Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: pagein runs at last, and a compressed file's pieces are no longer found twice

vcvkk4518206bdd14
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.c
index 6516841..f95ad03 100644
--- a/tools/apfs/apfs-tool.c
+++ b/tools/apfs/apfs-tool.c
@@ -280,6 +280,7 @@ cmd_cat(const char *path, const char *where)
struct apfs_nx nx;
struct apfs_vol vol;
struct apfs_inode ino;
+ struct apfs_cfile cf;
uint64_t id, off = 0;
uint8_t dt;
uint8_t *buf;
@@ -300,10 +301,12 @@ cmd_cat(const char *path, const char *where)
if (buf == NULL) {
return 1;
}
+ memset(&cf, 0, sizeof(cf));
+ cf.held = APFS_NO_PIECE;
while (off < ino.size) {
size_t got = 0;
- error = apfs_read_file(&vol, &ino, off, buf, 65536, &got);
+ error = apfs_read_file(&vol, &ino, &cf, off, buf, 65536, &got);
if (error) {
fprintf(stderr, "apfs-tool: read at %" PRIu64 ": %s\n", off,
strerror(error));
@@ -319,6 +322,7 @@ cmd_cat(const char *path, const char *where)
}
off += got;
}
+ apfs_cfile_fini(&cf);
free(buf);
apfs_nx_close(&nx);
return 0;
@@ -343,15 +347,19 @@ static uint64_t
file_hash(struct census *cs, const struct apfs_inode *ino, uint64_t size)
{
uint64_t h = FNV_INIT, off = 0;
+ struct apfs_cfile cf;
+ memset(&cf, 0, sizeof(cf));
+ cf.held = APFS_NO_PIECE;
for (;;) {
size_t got = 0;
- int error = apfs_read_file(cs->vol, ino, off, cs->buf, 65536, &got);
+ int error = apfs_read_file(cs->vol, ino, &cf, off, cs->buf, 65536, &got);
if (error) {
fprintf(stderr, "apfs-tool: %s: read at %" PRIu64 ": %s\n", cs->path,
off, strerror(error));
cs->error = error;
+ apfs_cfile_fini(&cf);
return 0;
}
if (got == 0) {
@@ -363,6 +371,7 @@ file_hash(struct census *cs, const struct apfs_inode *ino, uint64_t size)
break;
}
}
+ apfs_cfile_fini(&cf);
return h;
}
diff --git a/userland/build.sh b/userland/build.sh
index 8a43f24..7bac039 100755
--- a/userland/build.sh
+++ b/userland/build.sh
@@ -123,6 +123,7 @@ if [ -d $IOS/rootfs/usr/lib ]; then
$DYN -framework CoreFoundation -framework IOSurface -o obj/cow_test src/cow_test.c
$DYN -framework CoreFoundation -framework CoreGraphics -framework IOSurface -o obj/cg_test src/cg_test.c
$DYN -o obj/shm_test src/shm_test.c
+ $DYN -o obj/apfs_mmap_test src/apfs_mmap_test.c
$DYN -framework CoreFoundation -framework CoreGraphics -framework ImageIO -o obj/snap_test src/snap_test.c
$DYN -framework CoreFoundation -framework CoreGraphics -framework CoreText -o obj/font_test src/font_test.c
$DYN -fobjc-arc -framework Foundation -framework SystemConfiguration -o obj/net_test src/net_test.m
@@ -167,6 +168,7 @@ if [ -d $IOS/rootfs/usr/lib ]; then
[ -f obj/cow_test ] && cp obj/cow_test root/bin/
[ -f obj/cg_test ] && cp obj/cg_test root/bin/
[ -f obj/shm_test ] && cp obj/shm_test root/bin/
+ [ -f obj/apfs_mmap_test ] && cp obj/apfs_mmap_test root/bin/
[ -f obj/snap_test ] && cp obj/snap_test root/bin/
[ -f obj/font_test ] && cp obj/font_test root/bin/
[ -f obj/net_test ] && cp obj/net_test root/bin/
diff --git a/userland/rc.jobs.apfs b/userland/rc.jobs.apfs
new file mode 100644
index 0000000..a02d78b
--- /dev/null
+++ b/userland/rc.jobs.apfs
@@ -0,0 +1,10 @@
+# Commands run by the launchd job com.redwood.demo (see com.redwood.demo.plist).
+# Reading the APFS volume the kernel mounted on /private/var/apfs-check (boot-arg
+# apfs_check=3), from user space. The three roads, over one fixed set of files
+# (/usr/sbin: 32 files, 47.8 MB) so that two kernels can be compared on equal terms:
+# mapping the only road that makes the driver's vnop_pagein run
+# small read(2)s what the compressed path is worst at
+# large read(2)s the control: a piece per decompression even before any caching
+/redwood/bin/apfs_mmap_test mmap /private/var/apfs-check/usr/sbin
+/redwood/bin/apfs_mmap_test read /private/var/apfs-check/usr/sbin 4096
+/redwood/bin/apfs_mmap_test read /private/var/apfs-check/usr/sbin 65536
diff --git a/userland/src/apfs_mmap_test.c b/userland/src/apfs_mmap_test.c
new file mode 100644
index 0000000..ac051bb
--- /dev/null
+++ b/userland/src/apfs_mmap_test.c
@@ -0,0 +1,237 @@
+/*
+ * Read files off the APFS mount and hash them, two ways, so that what the driver hands
+ * back can be compared with what tools/apfs/apfs.py reads from the same image on the
+ * build machine.
+ *
+ * mmap map the file and read it through the mapping. This is the only way to make
+ * the driver's vnop_pagein run: the walk inside the kernel (boot-arg apfs_check)
+ * goes through vnop_read, and nothing on this volume is executed, so without
+ * this the paging path is never touched at all.
+ * read read(2) the file in pieces of a given size. With a small size this is the
+ * case the compressed-file path is worst at -- every read decompresses the whole
+ * 64 KB piece it falls in.
+ *
+ * The hash is FNV-1a over the whole file, which is what verify.py and the in-kernel walk
+ * use, so a line here can be compared with a line from either.
+ *
+ * usage: apfs_mmap_test mmap [PATH]
+ * apfs_mmap_test read [PATH] [CHUNK]
+ * PATH defaults to /private/var/apfs-check/usr, CHUNK to 4096.
+ */
+
+#include <dirent.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <time.h>
+#include <unistd.h>
+
+#define FNV_INIT 0xcbf29ce484222325ULL
+#define FNV_PRIME 0x100000001b3ULL
+
+#define MOUNT "/private/var/apfs-check"
+
+static const char *tag; /* what to print each line under */
+static int chunk_size = 4096;
+static int use_mmap;
+
+static unsigned long long total_sum;
+static unsigned long long total_bytes;
+static long total_files, total_errors;
+
+static unsigned long long
+fnv1a(const unsigned char *p, size_t n, unsigned long long h)
+{
+ size_t i;
+
+ for (i = 0; i < n; i++) {
+ h ^= p[i];
+ h *= FNV_PRIME;
+ }
+ return h;
+}
+
+static double
+now_ms(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec * 1000.0 + ts.tv_nsec / 1000000.0;
+}
+
+/* The bytes of one file, through a mapping. */
+static int
+hash_mapped(const char *path, off_t size, unsigned long long *out)
+{
+ int fd = open(path, O_RDONLY);
+ void *map;
+
+ if (fd < 0) {
+ return -1;
+ }
+ if (size == 0) {
+ close(fd);
+ *out = FNV_INIT; /* nothing to map, and nothing to hash */
+ return 0;
+ }
+ map = mmap(NULL, (size_t)size, PROT_READ, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (map == MAP_FAILED) {
+ return -1;
+ }
+ /* touching every page is what makes the kernel page the file in */
+ *out = fnv1a((const unsigned char *)map, (size_t)size, FNV_INIT);
+ munmap(map, (size_t)size);
+ return 0;
+}
+
+/* The bytes of one file, read in pieces. */
+static int
+hash_read(const char *path, off_t size, unsigned long long *out)
+{
+ int fd = open(path, O_RDONLY);
+ unsigned long long h = FNV_INIT;
+ unsigned char *buf;
+ ssize_t n;
+
+ (void)size;
+ if (fd < 0) {
+ return -1;
+ }
+ buf = malloc((size_t)chunk_size);
+ if (buf == NULL) {
+ close(fd);
+ return -1;
+ }
+ while ((n = read(fd, buf, (size_t)chunk_size)) > 0) {
+ h = fnv1a(buf, (size_t)n, h);
+ }
+ free(buf);
+ close(fd);
+ if (n < 0) {
+ return -1;
+ }
+ *out = h;
+ return 0;
+}
+
+static void
+do_file(const char *path, off_t size)
+{
+ unsigned long long h = 0;
+ char line[2048];
+ int rc = use_mmap ? hash_mapped(path, size, &h) : hash_read(path, size, &h);
+
+ if (rc != 0) {
+ printf("%s %s: cannot read it: %s\n", tag, path + sizeof(MOUNT) - 1,
+ strerror(errno));
+ total_errors++;
+ return;
+ }
+ snprintf(line, sizeof(line), "%s\t%lld\t%016llx\n", path + sizeof(MOUNT) - 1,
+ (long long)size, h);
+ printf("%s %s", tag, line);
+ total_sum += fnv1a((const unsigned char *)line, strlen(line), FNV_INIT);
+ total_files++;
+ total_bytes += (unsigned long long)size;
+}
+
+static void
+walk(char *path, size_t len)
+{
+ DIR *d = opendir(path);
+ struct dirent *de;
+
+ if (d == NULL) {
+ printf("%s %s: cannot open the directory: %s\n", tag,
+ path + sizeof(MOUNT) - 1, strerror(errno));
+ total_errors++;
+ return;
+ }
+ while ((de = readdir(d)) != NULL) {
+ struct stat st;
+ size_t n = strlen(de->d_name);
+
+ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {
+ continue;
+ }
+ if (len + n + 2 > PATH_MAX) {
+ total_errors++;
+ continue;
+ }
+ path[len] = '/';
+ memcpy(path + len + 1, de->d_name, n + 1);
+ /* lstat, not stat: a symbolic link is not followed and not read */
+ if (lstat(path, &st) != 0) {
+ printf("%s %s: lstat failed: %s\n", tag, path + sizeof(MOUNT) - 1,
+ strerror(errno));
+ total_errors++;
+ } else if (S_ISDIR(st.st_mode)) {
+ walk(path, len + 1 + n);
+ } else if (S_ISREG(st.st_mode)) {
+ do_file(path, st.st_size);
+ }
+ path[len] = '\0';
+ }
+ closedir(d);
+}
+
+int
+main(int argc, char **argv)
+{
+ static char path[PATH_MAX];
+ const char *where = MOUNT "/usr";
+ const char *mode = argc > 1 ? argv[1] : "mmap";
+ double t0, t1;
+ int tries;
+
+ use_mmap = (strcmp(mode, "read") != 0);
+ tag = use_mmap ? "apfs-mmap:" : "apfs-read:";
+ if (argc > 2) {
+ where = argv[2];
+ }
+ if (argc > 3) {
+ chunk_size = atoi(argv[3]);
+ if (chunk_size <= 0) {
+ chunk_size = 4096;
+ }
+ }
+ if (strncmp(where, MOUNT, sizeof(MOUNT) - 1) != 0) {
+ printf("%s a path under %s, please\n", tag, MOUNT);
+ return 2;
+ }
+
+ /* the kernel mounts the volume as soon as /private/var is there; wait for it */
+ for (tries = 0; tries < 120; tries++) {
+ struct stat st;
+
+ if (stat(where, &st) == 0) {
+ break;
+ }
+ sleep(1);
+ }
+
+ printf("%s %s, %s%s\n", tag, where, use_mmap ? "through a mapping" : "with read(2)",
+ use_mmap ? "" : " in pieces");
+ if (!use_mmap) {
+ printf("%s pieces of %d bytes\n", tag, chunk_size);
+ }
+ snprintf(path, sizeof(path), "%s", where);
+ t0 = now_ms();
+ walk(path, strlen(path));
+ t1 = now_ms();
+
+ printf("%s files %ld bytes %llu errors %ld\n", tag, total_files, total_bytes,
+ total_errors);
+ printf("%s line-hash sum %016llx\n", tag, total_sum);
+ printf("%s done in %.0f ms (%.1f MB/s)\n", tag, t1 - t0,
+ (t1 > t0) ? (double)total_bytes / 1048576.0 / ((t1 - t0) / 1000.0) : 0.0);
+ return total_errors ? 1 : 0;
+}
diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.c
index 707fce6..64a74d9 100644
--- a/xnu/files/bsd/dev/redwood_apfs_check.c
+++ b/xnu/files/bsd/dev/redwood_apfs_check.c
@@ -5,6 +5,9 @@
*
* apfs_check=1 walk the whole volume and print only the totals
* apfs_check=2 also print a line per entry
+ * apfs_check=3 mount it and stop, leaving it for user space to read
+ * (userland/src/apfs_mmap_test.c, which is the only way the
+ * driver's vnop_pagein ever runs)
* apfs_subtree=/usr walk only that part of the volume
*
* A line is the same one tools/apfs/apfs-tool's census and `verify.py clines` print:
@@ -409,6 +412,11 @@ apfs_check_thread(void *arg, __unused wait_result_t wr)
return;
}
printf("apfs-check: %s mounted on %s\n", CHECK_DEV, CHECK_MNT);
+ if (mode >= 3) {
+ /* someone in user space is going to read it; leave it alone */
+ printf("apfs-check: left mounted, not walking it here\n");
+ return;
+ }
cs.path = kalloc_data(PATH_CAP, Z_WAITOK | Z_ZERO);
cs.buf = kalloc_data(READ_CHUNK, Z_WAITOK);
diff --git a/xnu/files/bsd/miscfs/apfs/apfs.h b/xnu/files/bsd/miscfs/apfs/apfs.h
index 2f78602..ac76c3d 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs.h
@@ -50,6 +50,13 @@ struct apfs_node {
uint16_t an_mode;
uint64_t an_size; /* the logical size, as a reader of the file sees it */
bool an_attached; /* an_vp is set and the node is in the hash */
+ /*
+ * What has been worked out about how this file's contents are stored, kept for as
+ * long as the vnode lives: the table of compressed pieces and the piece last
+ * decompressed. Without it every read and every page fault would find them again.
+ * Guarded by am_lock, like everything else that reaches apfs_lib.
+ */
+ struct apfs_cfile an_cf;
};
#define APFS_MP(mp) ((struct apfs_mount *)vfs_fsprivate(mp))
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
index 7d55adf..63daed6 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
@@ -306,6 +306,9 @@ node_child(struct apfs_btree *tree, const struct node *n, uint32_t i, uint64_t *
return 0;
}
+static int cfile_header(struct apfs_vol *vol, const struct apfs_inode *ino,
+ bool *compressed, uint32_t *type, uint64_t *size, bool *in_rsrc);
+
/* Where a node identifier actually is: a block address, directly or through a map. */
static int
tree_node_addr(struct apfs_btree *tree, uint64_t oid, uint64_t *addr);
@@ -1011,7 +1014,9 @@ apfs_inode_get(struct apfs_vol *vol, uint64_t obj_id, struct apfs_inode *ino)
int
apfs_inode_size(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t *size)
{
- struct apfs_cfile cf;
+ bool compressed = false, in_rsrc = false;
+ uint32_t type = 0;
+ uint64_t csize = 0;
int error;
if ((ino->mode & APFS_S_IFMT) == APFS_S_IFLNK) {
@@ -1025,11 +1030,12 @@ apfs_inode_size(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t *si
*size = n;
return 0;
}
- error = apfs_cfile_init(vol, ino, &cf);
+ /* only the decmpfs header is needed here, not the table of pieces */
+ error = cfile_header(vol, ino, &compressed, &type, &csize, &in_rsrc);
if (error) {
return error;
}
- *size = cf.compressed ? cf.size : ino->size;
+ *size = compressed ? csize : ino->size;
return 0;
}
@@ -1503,15 +1509,19 @@ apfs_readlink(struct apfs_vol *vol, uint64_t obj_id, char *buf, size_t len, size
return 0;
}
-int
-apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,
- struct apfs_cfile *cf)
+/* Just the decmpfs header: what a caller needs to report a size, and nothing more. */
+static int
+cfile_header(struct apfs_vol *vol, const struct apfs_inode *ino, bool *compressed,
+ uint32_t *type, uint64_t *size, bool *in_rsrc)
{
uint8_t hdr[16];
size_t n = 0;
int error;
- memset(cf, 0, sizeof(*cf));
+ *compressed = false;
+ *type = 0;
+ *size = 0;
+ *in_rsrc = false;
error = apfs_xattr_read(vol, ino->obj_id, APFS_DECMPFS_XATTR_NAME, hdr, sizeof(hdr),
&n);
if (error == ENOENT) {
@@ -1523,25 +1533,189 @@ apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,
if (n < sizeof(hdr) || le32(hdr) != APFS_DECMPFS_MAGIC) {
return 0;
}
- cf->compressed = true;
- cf->type = le32(hdr + 4);
- cf->size = le64(hdr + 8);
- switch (cf->type) {
+ *compressed = true;
+ *type = le32(hdr + 4);
+ *size = le64(hdr + 8);
+ switch (*type) {
case APFS_DECMPFS_TYPE_ZLIB_ATTR:
case APFS_DECMPFS_TYPE_RAW_ATTR:
- cf->in_rsrc = false;
+ *in_rsrc = false;
return 0;
case APFS_DECMPFS_TYPE_ZLIB_RSRC:
case APFS_DECMPFS_TYPE_RAW_RSRC:
- cf->in_rsrc = true;
+ *in_rsrc = true;
return 0;
default:
apfs_log("inode %llu: decmpfs type %u is not supported\n",
- (unsigned long long)ino->obj_id, cf->type);
+ (unsigned long long)ino->obj_id, *type);
return ENOTSUP;
}
}
+void
+apfs_cfile_fini(struct apfs_cfile *cf)
+{
+ if (cf == NULL) {
+ return;
+ }
+ if (cf->piece_off != NULL) {
+ apfs_free(cf->piece_off, (size_t)cf->piece_count * sizeof(uint32_t));
+ }
+ if (cf->piece_len != NULL) {
+ apfs_free(cf->piece_len, (size_t)cf->piece_count * sizeof(uint32_t));
+ }
+ if (cf->held_data != NULL) {
+ apfs_free(cf->held_data, cf->held_cap);
+ }
+ memset(cf, 0, sizeof(*cf));
+ cf->held = APFS_NO_PIECE;
+}
+
+/*
+ * Read the resource fork's table of pieces, once. The fork is an HFS-Plus resource fork:
+ * sixteen big-endian bytes giving the offset and length of the resource data and of the
+ * resource map. At that data offset sits a four-byte big-endian length, and the table
+ * starts right after it -- a little-endian count, then an (offset, length) pair per piece,
+ * the offsets measured from the start of the table. The table is not at a fixed place:
+ * the data offset is 256 in every file here, so the table lands at 260, but it has to be
+ * read rather than assumed.
+ */
+static int
+cfile_read_table(struct apfs_vol *vol, const struct apfs_inode *ino,
+ struct apfs_cfile *cf)
+{
+ struct apfs_xattr xa;
+ uint8_t head[16];
+ uint8_t *tbl = NULL;
+ size_t got = 0, tbl_len;
+ uint32_t i;
+ int error;
+
+ error = apfs_xattr_lookup(vol, ino->obj_id, APFS_RSRC_FORK_XATTR_NAME, &xa);
+ if (error) {
+ return error;
+ }
+ if ((xa.flags & APFS_XATTR_DATA_STREAM) == 0) {
+ return ENOTSUP; /* a resource fork small enough to be embedded */
+ }
+ cf->rsrc_id = xa.stream_id;
+ cf->rsrc_size = xa.stream_size;
+
+ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size, 0, head, sizeof(head),
+ &got);
+ if (error) {
+ return error;
+ }
+ if (got < sizeof(head)) {
+ return EIO;
+ }
+ cf->table_base = (uint64_t)be32(head) + 4;
+ if (cf->table_base + 4 > cf->rsrc_size) {
+ return EIO;
+ }
+ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size, cf->table_base, head, 4,
+ &got);
+ if (error) {
+ return error;
+ }
+ if (got < 4) {
+ return EIO;
+ }
+ cf->piece_count = le32(head);
+ if (cf->piece_count == 0 ||
+ cf->piece_count != (uint32_t)((cf->size + APFS_DECMPFS_BLOCK_SIZE - 1) /
+ APFS_DECMPFS_BLOCK_SIZE)) {
+ apfs_log("inode %llu: resource fork says %u pieces, %llu bytes needs %llu\n",
+ (unsigned long long)ino->obj_id, cf->piece_count,
+ (unsigned long long)cf->size,
+ (unsigned long long)((cf->size + APFS_DECMPFS_BLOCK_SIZE - 1) /
+ APFS_DECMPFS_BLOCK_SIZE));
+ return EIO;
+ }
+ tbl_len = (size_t)cf->piece_count * 8;
+ if (cf->table_base + 4 + tbl_len > cf->rsrc_size) {
+ return EIO;
+ }
+ cf->piece_off = apfs_alloc((size_t)cf->piece_count * sizeof(uint32_t));
+ cf->piece_len = apfs_alloc((size_t)cf->piece_count * sizeof(uint32_t));
+ tbl = apfs_alloc(tbl_len);
+ if (cf->piece_off == NULL || cf->piece_len == NULL || tbl == NULL) {
+ error = ENOMEM;
+ goto out;
+ }
+ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size, cf->table_base + 4, tbl,
+ tbl_len, &got);
+ if (error) {
+ goto out;
+ }
+ if (got < tbl_len) {
+ error = EIO;
+ goto out;
+ }
+ for (i = 0; i < cf->piece_count; i++) {
+ uint64_t off = le32(tbl + 8 * i);
+ uint64_t len = le32(tbl + 8 * i + 4);
+
+ if (len == 0 || cf->table_base + off + len > cf->rsrc_size) {
+ error = EIO;
+ goto out;
+ }
+ cf->piece_off[i] = (uint32_t)off;
+ cf->piece_len[i] = (uint32_t)len;
+ }
+out:
+ if (tbl != NULL) {
+ apfs_free(tbl, tbl_len);
+ }
+ return error;
+}
+
+int
+apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,
+ struct apfs_cfile *cf)
+{
+ int error;
+
+ if (cf->ready && cf->obj_id == ino->obj_id) {
+ return 0; /* already worked out, with its table and piece */
+ }
+ apfs_cfile_fini(cf);
+ cf->obj_id = ino->obj_id;
+ cf->held = APFS_NO_PIECE;
+
+ error = cfile_header(vol, ino, &cf->compressed, &cf->type, &cf->size, &cf->in_rsrc);
+ if (error) {
+ return error;
+ }
+ if (!cf->compressed) {
+ cf->ready = true;
+ return 0;
+ }
+ if (cf->in_rsrc) {
+ error = cfile_read_table(vol, ino, cf);
+ if (error) {
+ apfs_cfile_fini(cf);
+ return error;
+ }
+ cf->held_cap = APFS_DECMPFS_BLOCK_SIZE;
+ } else if (cf->size <= APFS_DECMPFS_BLOCK_SIZE) {
+ /* the whole thing is in the attribute and fits: hold it as one piece */
+ cf->held_cap = (size_t)cf->size;
+ cf->piece_count = 1;
+ } else {
+ cf->held_cap = 0; /* too big to hold; it is read afresh each time */
+ }
+ if (cf->held_cap != 0) {
+ cf->held_data = apfs_alloc(cf->held_cap);
+ if (cf->held_data == NULL) {
+ apfs_cfile_fini(cf);
+ return ENOMEM;
+ }
+ }
+ cf->ready = true;
+ return 0;
+}
+
/*
* zlib asks for its own working memory through these. The kernel's copy of zlib has no
* allocator of its own -- a zeroed z_stream leaves zalloc NULL and inflateInit_ then calls
@@ -1627,26 +1801,74 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,
return 0;
}
+/*
+ * The contents of a file whose bytes are in the com.apple.decmpfs attribute, from the
+ * bytes after the header.
+ *
+ * Type 9 means they were stored as they are, with one marker byte in front. All 23947
+ * type-9 files in the iOS 16.5 root file system have 0xcc there and exactly `want` bytes
+ * after it, which is how the marker's length was settled; a payload that is already
+ * exactly the right length is taken whole.
+ */
+static int
+decmpfs_attr(uint32_t type, const uint8_t *src, size_t src_len, size_t want,
+ uint8_t *dst, size_t *out)
+{
+ switch (type) {
+ case APFS_DECMPFS_TYPE_RAW_ATTR: {
+ const uint8_t *p = src;
+ size_t n = src_len;
+
+ if (n != want) {
+ if (n == 0) {
+ return EIO;
+ }
+ p++;
+ n--;
+ }
+ if (n > want) {
+ n = want;
+ }
+ memcpy(dst, p, n);
+ *out = n;
+ return 0;
+ }
+ case APFS_DECMPFS_TYPE_ZLIB_ATTR:
+ return apfs_inflate(src, src_len, dst, want, out);
+ default:
+ return ENOTSUP;
+ }
+}
+
/*
* A compressed file whose bytes are in the attribute: the whole thing is there, after the
- * header, with one marker byte in front when it was stored rather than compressed.
+ * header, with one marker byte in front when it was stored rather than compressed. When
+ * the context can hold it, it is produced once and kept.
*/
static int
read_cfile_attr(struct apfs_vol *vol, const struct apfs_inode *ino,
- const struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out)
+ struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out)
{
struct apfs_xattr xa;
uint8_t *attr = NULL;
uint8_t *plain = NULL;
- size_t attr_len = 0, plain_len = 0;
- size_t alloc = APFS_XATTR_MAX_EMBEDDED_SIZE + 16;
- int error;
+ size_t attr_len = 0, plain_len = 0, alloc = 0;
+ bool held = false;
+ int error = 0;
*out = 0;
+ if (cf->held_cap != 0 && cf->held == 0) {
+ plain = cf->held_data; /* produced on an earlier call */
+ plain_len = cf->held_len;
+ held = true;
+ goto copy;
+ }
+
error = apfs_xattr_lookup(vol, ino->obj_id, APFS_DECMPFS_XATTR_NAME, &xa);
if (error) {
return error;
}
+ alloc = APFS_XATTR_MAX_EMBEDDED_SIZE + 16;
if (xa.flags & APFS_XATTR_DATA_STREAM) {
if (xa.stream_size > 64u * 1024u * 1024u) {
return EIO;
@@ -1670,40 +1892,26 @@ read_cfile_attr(struct apfs_vol *vol, const struct apfs_inode *ino,
error = EIO;
goto out;
}
- plain = apfs_alloc((size_t)cf->size ? (size_t)cf->size : 1);
- if (plain == NULL) {
- error = ENOMEM;
- goto out;
- }
- if (cf->type == APFS_DECMPFS_TYPE_RAW_ATTR) {
- const uint8_t *p = attr + 16;
- size_t n = attr_len - 16;
-
- /*
- * Stored as it is. Every type-9 file in the iOS 16.5 root file system has one
- * marker byte in front of the contents; a payload that is already exactly the
- * right length is taken whole.
- */
- if (n != cf->size) {
- if (n == 0) {
- error = EIO;
- goto out;
- }
- p++;
- n--;
- }
- if (n > cf->size) {
- n = (size_t)cf->size;
- }
- memcpy(plain, p, n);
- plain_len = n;
+ if (cf->held_cap != 0 && cf->held_cap >= (size_t)cf->size) {
+ plain = cf->held_data;
+ held = true;
} else {
- error = apfs_inflate(attr + 16, attr_len - 16, plain, (size_t)cf->size,
- &plain_len);
- if (error) {
+ plain = apfs_alloc((size_t)cf->size ? (size_t)cf->size : 1);
+ if (plain == NULL) {
+ error = ENOMEM;
goto out;
}
}
+ error = decmpfs_attr(cf->type, attr + 16, attr_len - 16, (size_t)cf->size,
+ plain, &plain_len);
+ if (error) {
+ goto out;
+ }
+ if (held) {
+ cf->held = 0;
+ cf->held_len = plain_len;
+ }
+copy:
if (off < plain_len) {
size_t n = plain_len - (size_t)off;
@@ -1717,7 +1925,7 @@ out:
if (attr != NULL) {
apfs_free(attr, alloc);
}
- if (plain != NULL) {
+ if (plain != NULL && !held) {
apfs_free(plain, (size_t)cf->size ? (size_t)cf->size : 1);
}
return error;
@@ -1725,50 +1933,23 @@ out:
/*
* A compressed file whose bytes are in com.apple.ResourceFork: the contents cut into
- * 64 KB pieces, each its own zlib stream. The fork is an HFS-Plus resource fork; the
- * piece table starts four bytes past the data offset in its big-endian header.
+ * 64 KB pieces, each its own zlib stream. The table saying where the pieces are was read
+ * once by apfs_cfile_init(); the piece last decompressed is kept, so a reader working
+ * through the file -- or a page fault asking for 16 KB of it -- does not decompress the
+ * same 64 KB again.
*/
static int
read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
- const struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out)
+ struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out)
{
- struct apfs_xattr xa;
- uint8_t head[16];
- uint8_t entry[8];
- uint8_t *piece = NULL, *plain = NULL;
- size_t piece_cap = 0, got = 0;
- uint64_t base, nblocks, first, last, i;
- int error;
+ uint8_t *piece = NULL;
+ size_t piece_cap = 0;
+ uint64_t first, last, i;
+ int error = 0;
*out = 0;
- error = apfs_xattr_lookup(vol, ino->obj_id, APFS_RSRC_FORK_XATTR_NAME, &xa);
- if (error) {
- return error;
- }
- if ((xa.flags & APFS_XATTR_DATA_STREAM) == 0) {
- return ENOTSUP; /* a resource fork small enough to be embedded */
- }
- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, 0, head, sizeof(head),
- &got);
- if (error) {
- return error;
- }
- if (got < sizeof(head)) {
- return EIO;
- }
- base = (uint64_t)be32(head) + 4; /* data offset, then the big-endian length */
- if (base + 4 > xa.stream_size) {
- return EIO;
- }
- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, base, entry, 4, &got);
- if (error) {
- return error;
- }
- if (got < 4) {
- return EIO;
- }
- nblocks = le32(entry);
- if (nblocks == 0 || nblocks > (cf->size / APFS_RSRC_PIECE) + 1) {
+ (void)ino;
+ if (cf->piece_count == 0 || cf->piece_off == NULL) {
return EIO;
}
if (off >= cf->size) {
@@ -1777,102 +1958,83 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,
if (len > cf->size - off) {
len = (size_t)(cf->size - off);
}
- first = off / APFS_RSRC_PIECE;
- last = (off + len - 1) / APFS_RSRC_PIECE;
- if (last >= nblocks) {
+ first = off / APFS_DECMPFS_BLOCK_SIZE;
+ last = (off + len - 1) / APFS_DECMPFS_BLOCK_SIZE;
+ if (last >= cf->piece_count) {
return EIO;
}
- plain = apfs_alloc(APFS_RSRC_PIECE);
- if (plain == NULL) {
- return ENOMEM;
- }
+
for (i = first; i <= last; i++) {
- uint64_t ent_off = base + 4 + 8 * i;
- uint64_t poff, plen;
- size_t want, copied, from_in_piece, n;
+ size_t want = APFS_DECMPFS_BLOCK_SIZE;
+ size_t from_in_piece, n, got = 0;
- if (ent_off + 8 > xa.stream_size) {
- error = EIO;
- goto out;
- }
- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, ent_off, entry, 8,
- &got);
- if (error) {
- goto out;
+ if ((i + 1) * APFS_DECMPFS_BLOCK_SIZE > cf->size) {
+ want = (size_t)(cf->size - i * APFS_DECMPFS_BLOCK_SIZE);
}
- if (got < 8) {
- error = EIO;
- goto out;
- }
- poff = le32(entry);
- plen = le32(entry + 4);
- if (plen == 0 || base + poff + plen > xa.stream_size) {
- error = EIO;
- goto out;
- }
- if (plen > piece_cap) {
- if (piece != NULL) {
- apfs_free(piece, piece_cap);
+ if (cf->held != i) {
+ uint32_t plen = cf->piece_len[i];
+
+ if (plen > piece_cap) {
+ if (piece != NULL) {
+ apfs_free(piece, piece_cap);
+ }
+ piece_cap = plen;
+ piece = apfs_alloc(piece_cap);
+ if (piece == NULL) {
+ error = ENOMEM;
+ goto out;
+ }
}
- piece_cap = (size_t)plen;
- piece = apfs_alloc(piece_cap);
- if (piece == NULL) {
- error = ENOMEM;
+ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size,
+ cf->table_base + cf->piece_off[i], piece, plen, &got);
+ if (error) {
goto out;
}
- }
- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, base + poff, piece,
- (size_t)plen, &got);
- if (error) {
- goto out;
- }
- if (got < plen) {
- error = EIO;
- goto out;
- }
- /* how much of the file this piece stands for */
- want = APFS_RSRC_PIECE;
- if ((i + 1) * APFS_RSRC_PIECE > cf->size) {
- want = (size_t)(cf->size - i * APFS_RSRC_PIECE);
- }
- if (cf->type == APFS_DECMPFS_TYPE_RAW_RSRC) {
- size_t m = plen < want ? (size_t)plen : want;
+ if (got < plen) {
+ error = EIO;
+ goto out;
+ }
+ cf->held = APFS_NO_PIECE; /* what is in there now is unknown */
+ if (cf->type == APFS_DECMPFS_TYPE_RAW_RSRC) {
+ size_t m = plen < want ? (size_t)plen : want;
- memcpy(plain, piece, m);
- copied = m;
- } else {
- error = apfs_inflate(piece, (size_t)plen, plain, want, &copied);
- if (error) {
+ memcpy(cf->held_data, piece, m);
+ cf->held_len = m;
+ } else {
+ error = apfs_inflate(piece, plen, cf->held_data, want,
+ &cf->held_len);
+ if (error) {
+ goto out;
+ }
+ }
+ if (cf->held_len != want) {
+ error = EIO;
goto out;
}
+ cf->held = i;
}
- if (copied != want) {
- error = EIO;
- goto out;
- }
- /* the overlap of this piece with what was asked for */
- from_in_piece = (off > i * APFS_RSRC_PIECE)
- ? (size_t)(off - i * APFS_RSRC_PIECE) : 0;
- n = copied - from_in_piece;
+ from_in_piece = (off > i * APFS_DECMPFS_BLOCK_SIZE)
+ ? (size_t)(off - i * APFS_DECMPFS_BLOCK_SIZE) : 0;
+ n = cf->held_len - from_in_piece;
if (n > len - *out) {
n = len - *out;
}
- memcpy(buf + *out, plain + from_in_piece, n);
+ memcpy(buf + *out, cf->held_data + from_in_piece, n);
*out += n;
}
out:
if (piece != NULL) {
apfs_free(piece, piece_cap);
}
- apfs_free(plain, APFS_RSRC_PIECE);
return error;
}
int
-apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t off,
- void *buf, size_t len, size_t *out)
+apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino,
+ struct apfs_cfile *cf, uint64_t off, void *buf, size_t len, size_t *out)
{
- struct apfs_cfile cf;
+ struct apfs_cfile own;
+ bool borrowed = false;
int error;
*out = 0;
@@ -1889,18 +2051,31 @@ apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t off,
}
return 0;
}
- error = apfs_cfile_init(vol, ino, &cf);
- if (error) {
- return error;
- }
- if (!cf.compressed) {
- if (!ino->has_dstream) {
- return 0;
+ /*
+ * Without a context from the caller, one is made for this call alone -- correct, and
+ * the slow way: the table and the piece are then found again on the next call.
+ */
+ if (cf == NULL) {
+ memset(&own, 0, sizeof(own));
+ own.held = APFS_NO_PIECE;
+ cf = &own;
+ borrowed = true;
+ }
+ error = apfs_cfile_init(vol, ino, cf);
+ if (error == 0) {
+ if (!cf->compressed) {
+ error = ino->has_dstream
+ ? apfs_read_stream(vol, ino->private_id, ino->size, off, buf, len,
+ out)
+ : 0;
+ } else if (cf->in_rsrc) {
+ error = read_cfile_rsrc(vol, ino, cf, off, buf, len, out);
+ } else {
+ error = read_cfile_attr(vol, ino, cf, off, buf, len, out);
}
- return apfs_read_stream(vol, ino->private_id, ino->size, off, buf, len, out);
}
- if (cf.in_rsrc) {
- return read_cfile_rsrc(vol, ino, &cf, off, buf, len, out);
+ if (borrowed) {
+ apfs_cfile_fini(cf);
}
- return read_cfile_attr(vol, ino, &cf, off, buf, len, out);
+ return error;
}
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
index c8b9038..5d29e00 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
@@ -197,12 +197,46 @@ struct apfs_xattr {
uint64_t stream_size;
};
-/* How a file's contents are stored, once the decmpfs attribute has been read. */
+/* No piece is being held. */
+#define APFS_NO_PIECE ((uint64_t)~0ULL)
+
+/*
+ * How a file's contents are stored, and what has been worked out about it so far.
+ *
+ * A compressed file's bytes are cut into 64 KB pieces, each its own zlib stream, with a
+ * table in the resource fork saying where each piece is. Reaching that table costs
+ * several walks of the file-system tree -- the attribute record, the fork's header, the
+ * table entry -- and a piece has to be decompressed whole however few of its bytes are
+ * wanted. A reader that comes back for the next page would pay all of that again, which
+ * is what made paging a mapped file slow.
+ *
+ * So this holds what has been found: the fork's stream, the whole table, and the last
+ * piece decompressed. One of these belongs to whoever is reading the file -- the driver
+ * keeps one per vnode -- and it is handed to apfs_read_file() on every call.
+ * apfs_cfile_fini() gives back what it holds. Passing NULL instead works and is correct,
+ * and is the slow way.
+ */
struct apfs_cfile {
- bool compressed;
- uint32_t type; /* decmpfs compression type */
- uint64_t size; /* uncompressed size */
- bool in_rsrc; /* the bytes are in com.apple.ResourceFork */
+ bool ready; /* set up, for the inode in obj_id */
+ uint64_t obj_id;
+ bool compressed;
+ uint32_t type; /* decmpfs compression type */
+ uint64_t size; /* uncompressed size */
+ bool in_rsrc; /* the bytes are in com.apple.ResourceFork */
+
+ /* the resource fork and its table of pieces, read once */
+ uint64_t rsrc_id;
+ uint64_t rsrc_size;
+ uint64_t table_base;
+ uint32_t piece_count;
+ uint32_t *piece_off; /* piece_count entries, from table_base */
+ uint32_t *piece_len;
+
+ /* the piece last decompressed */
+ uint64_t held; /* its index, or APFS_NO_PIECE */
+ uint8_t *held_data;
+ size_t held_len;
+ size_t held_cap; /* 0 when this file is not worth holding */
};
/* ---- container and volumes ---- */
@@ -287,16 +321,22 @@ int apfs_readlink(struct apfs_vol *vol, uint64_t obj_id, char *buf, size_t len,
int apfs_read_stream(struct apfs_vol *vol, uint64_t stream_id, uint64_t stream_size,
uint64_t off, void *buf, size_t len, size_t *out);
-/* Whether a file is compressed, and how. */
+/*
+ * Work out how a file's contents are stored and read in whatever is worth reading once.
+ * Calling it again for the same inode does nothing; for a different one it starts over.
+ */
int apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,
struct apfs_cfile *cf);
+/* Give back everything a context holds. Safe on a zeroed or already finished one. */
+void apfs_cfile_fini(struct apfs_cfile *cf);
+
/*
* The contents of a file as a reader of the file would see them: the data stream for a
* plain file, the decompressed bytes for a compressed one.
*/
-int apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t off,
- void *buf, size_t len, size_t *out);
+int apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino,
+ struct apfs_cfile *cf, uint64_t off, void *buf, size_t len, size_t *out);
/* ---- odds and ends worth testing on their own ---- */
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
index 1f74996..23e6fba 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c
@@ -200,8 +200,8 @@ apfs_vnop_read(struct vnop_read_args *ap)
size_t got = 0;
apfs_lock(am);
- error = apfs_read_file(&am->am_vol, &ino, (uint64_t)uio_offset(uio), buf, want,
- &got);
+ error = apfs_read_file(&am->am_vol, &ino, &APFS_VP(vp)->an_cf,
+ (uint64_t)uio_offset(uio), buf, want, &got);
apfs_unlock(am);
if (error) {
break;
@@ -273,7 +273,8 @@ apfs_vnop_pagein(struct vnop_pagein_args *ap)
}
want = (size_t)MIN(end - at, (off_t)(size - (uint64_t)at));
apfs_lock(am);
- error = apfs_read_file(&am->am_vol, &ino, (uint64_t)at, dst, want, &got);
+ error = apfs_read_file(&am->am_vol, &ino, &APFS_VP(vp)->an_cf, (uint64_t)at,
+ dst, want, &got);
apfs_unlock(am);
if (error) {
break;
@@ -593,6 +594,9 @@ apfs_vnop_reclaim(struct vnop_reclaim_args *ap)
struct apfs_node *an = APFS_VP(vp);
if (an != NULL) {
+ apfs_lock(an->an_mount);
+ apfs_cfile_fini(&an->an_cf);
+ apfs_unlock(an->an_mount);
apfs_node_detach(an);
vnode_clearfsnode(vp);
kfree_type(struct apfs_node, an);