Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
APFS: pagein runs at last, and a compressed file's pieces are no longer found twice
vcvkk4518206bdd14
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.cindex 6516841..f95ad03 100644--- a/tools/apfs/apfs-tool.c+++ b/tools/apfs/apfs-tool.c@@ -280,6 +280,7 @@ cmd_cat(const char *path, const char *where)struct apfs_nx nx;struct apfs_vol vol;struct apfs_inode ino;+ struct apfs_cfile cf;uint64_t id, off = 0;uint8_t dt;uint8_t *buf;@@ -300,10 +301,12 @@ cmd_cat(const char *path, const char *where)if (buf == NULL) {return 1;}+ memset(&cf, 0, sizeof(cf));+ cf.held = APFS_NO_PIECE;while (off < ino.size) {size_t got = 0;- error = apfs_read_file(&vol, &ino, off, buf, 65536, &got);+ error = apfs_read_file(&vol, &ino, &cf, off, buf, 65536, &got);if (error) {fprintf(stderr, "apfs-tool: read at %" PRIu64 ": %s\n", off,strerror(error));@@ -319,6 +322,7 @@ cmd_cat(const char *path, const char *where)}off += got;}+ apfs_cfile_fini(&cf);free(buf);apfs_nx_close(&nx);return 0;@@ -343,15 +347,19 @@ static uint64_tfile_hash(struct census *cs, const struct apfs_inode *ino, uint64_t size){uint64_t h = FNV_INIT, off = 0;+ struct apfs_cfile cf;+ memset(&cf, 0, sizeof(cf));+ cf.held = APFS_NO_PIECE;for (;;) {size_t got = 0;- int error = apfs_read_file(cs->vol, ino, off, cs->buf, 65536, &got);+ int error = apfs_read_file(cs->vol, ino, &cf, off, cs->buf, 65536, &got);if (error) {fprintf(stderr, "apfs-tool: %s: read at %" PRIu64 ": %s\n", cs->path,off, strerror(error));cs->error = error;+ apfs_cfile_fini(&cf);return 0;}if (got == 0) {@@ -363,6 +371,7 @@ file_hash(struct census *cs, const struct apfs_inode *ino, uint64_t size)break;}}+ apfs_cfile_fini(&cf);return h;}diff --git a/userland/build.sh b/userland/build.shindex 8a43f24..7bac039 100755--- a/userland/build.sh+++ b/userland/build.sh@@ -123,6 +123,7 @@ if [ -d $IOS/rootfs/usr/lib ]; then$DYN -framework CoreFoundation -framework IOSurface -o obj/cow_test src/cow_test.c$DYN -framework CoreFoundation -framework CoreGraphics -framework IOSurface -o obj/cg_test src/cg_test.c$DYN -o obj/shm_test src/shm_test.c+ $DYN -o obj/apfs_mmap_test src/apfs_mmap_test.c$DYN -framework CoreFoundation -framework CoreGraphics -framework ImageIO -o obj/snap_test src/snap_test.c$DYN -framework CoreFoundation -framework CoreGraphics -framework CoreText -o obj/font_test src/font_test.c$DYN -fobjc-arc -framework Foundation -framework SystemConfiguration -o obj/net_test src/net_test.m@@ -167,6 +168,7 @@ if [ -d $IOS/rootfs/usr/lib ]; then[ -f obj/cow_test ] && cp obj/cow_test root/bin/[ -f obj/cg_test ] && cp obj/cg_test root/bin/[ -f obj/shm_test ] && cp obj/shm_test root/bin/+ [ -f obj/apfs_mmap_test ] && cp obj/apfs_mmap_test root/bin/[ -f obj/snap_test ] && cp obj/snap_test root/bin/[ -f obj/font_test ] && cp obj/font_test root/bin/[ -f obj/net_test ] && cp obj/net_test root/bin/diff --git a/userland/rc.jobs.apfs b/userland/rc.jobs.apfsnew file mode 100644index 0000000..a02d78b--- /dev/null+++ b/userland/rc.jobs.apfs@@ -0,0 +1,10 @@+# Commands run by the launchd job com.redwood.demo (see com.redwood.demo.plist).+# Reading the APFS volume the kernel mounted on /private/var/apfs-check (boot-arg+# apfs_check=3), from user space. The three roads, over one fixed set of files+# (/usr/sbin: 32 files, 47.8 MB) so that two kernels can be compared on equal terms:+# mapping the only road that makes the driver's vnop_pagein run+# small read(2)s what the compressed path is worst at+# large read(2)s the control: a piece per decompression even before any caching+/redwood/bin/apfs_mmap_test mmap /private/var/apfs-check/usr/sbin+/redwood/bin/apfs_mmap_test read /private/var/apfs-check/usr/sbin 4096+/redwood/bin/apfs_mmap_test read /private/var/apfs-check/usr/sbin 65536diff --git a/userland/src/apfs_mmap_test.c b/userland/src/apfs_mmap_test.cnew file mode 100644index 0000000..ac051bb--- /dev/null+++ b/userland/src/apfs_mmap_test.c@@ -0,0 +1,237 @@+/*+ * Read files off the APFS mount and hash them, two ways, so that what the driver hands+ * back can be compared with what tools/apfs/apfs.py reads from the same image on the+ * build machine.+ *+ * mmap map the file and read it through the mapping. This is the only way to make+ * the driver's vnop_pagein run: the walk inside the kernel (boot-arg apfs_check)+ * goes through vnop_read, and nothing on this volume is executed, so without+ * this the paging path is never touched at all.+ * read read(2) the file in pieces of a given size. With a small size this is the+ * case the compressed-file path is worst at -- every read decompresses the whole+ * 64 KB piece it falls in.+ *+ * The hash is FNV-1a over the whole file, which is what verify.py and the in-kernel walk+ * use, so a line here can be compared with a line from either.+ *+ * usage: apfs_mmap_test mmap [PATH]+ * apfs_mmap_test read [PATH] [CHUNK]+ * PATH defaults to /private/var/apfs-check/usr, CHUNK to 4096.+ */++#include <dirent.h>+#include <errno.h>+#include <fcntl.h>+#include <limits.h>+#include <stdint.h>+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <sys/mman.h>+#include <sys/stat.h>+#include <time.h>+#include <unistd.h>++#define FNV_INIT 0xcbf29ce484222325ULL+#define FNV_PRIME 0x100000001b3ULL++#define MOUNT "/private/var/apfs-check"++static const char *tag; /* what to print each line under */+static int chunk_size = 4096;+static int use_mmap;++static unsigned long long total_sum;+static unsigned long long total_bytes;+static long total_files, total_errors;++static unsigned long long+fnv1a(const unsigned char *p, size_t n, unsigned long long h)+{+ size_t i;++ for (i = 0; i < n; i++) {+ h ^= p[i];+ h *= FNV_PRIME;+ }+ return h;+}++static double+now_ms(void)+{+ struct timespec ts;++ clock_gettime(CLOCK_MONOTONIC, &ts);+ return ts.tv_sec * 1000.0 + ts.tv_nsec / 1000000.0;+}++/* The bytes of one file, through a mapping. */+static int+hash_mapped(const char *path, off_t size, unsigned long long *out)+{+ int fd = open(path, O_RDONLY);+ void *map;++ if (fd < 0) {+ return -1;+ }+ if (size == 0) {+ close(fd);+ *out = FNV_INIT; /* nothing to map, and nothing to hash */+ return 0;+ }+ map = mmap(NULL, (size_t)size, PROT_READ, MAP_PRIVATE, fd, 0);+ close(fd);+ if (map == MAP_FAILED) {+ return -1;+ }+ /* touching every page is what makes the kernel page the file in */+ *out = fnv1a((const unsigned char *)map, (size_t)size, FNV_INIT);+ munmap(map, (size_t)size);+ return 0;+}++/* The bytes of one file, read in pieces. */+static int+hash_read(const char *path, off_t size, unsigned long long *out)+{+ int fd = open(path, O_RDONLY);+ unsigned long long h = FNV_INIT;+ unsigned char *buf;+ ssize_t n;++ (void)size;+ if (fd < 0) {+ return -1;+ }+ buf = malloc((size_t)chunk_size);+ if (buf == NULL) {+ close(fd);+ return -1;+ }+ while ((n = read(fd, buf, (size_t)chunk_size)) > 0) {+ h = fnv1a(buf, (size_t)n, h);+ }+ free(buf);+ close(fd);+ if (n < 0) {+ return -1;+ }+ *out = h;+ return 0;+}++static void+do_file(const char *path, off_t size)+{+ unsigned long long h = 0;+ char line[2048];+ int rc = use_mmap ? hash_mapped(path, size, &h) : hash_read(path, size, &h);++ if (rc != 0) {+ printf("%s %s: cannot read it: %s\n", tag, path + sizeof(MOUNT) - 1,+ strerror(errno));+ total_errors++;+ return;+ }+ snprintf(line, sizeof(line), "%s\t%lld\t%016llx\n", path + sizeof(MOUNT) - 1,+ (long long)size, h);+ printf("%s %s", tag, line);+ total_sum += fnv1a((const unsigned char *)line, strlen(line), FNV_INIT);+ total_files++;+ total_bytes += (unsigned long long)size;+}++static void+walk(char *path, size_t len)+{+ DIR *d = opendir(path);+ struct dirent *de;++ if (d == NULL) {+ printf("%s %s: cannot open the directory: %s\n", tag,+ path + sizeof(MOUNT) - 1, strerror(errno));+ total_errors++;+ return;+ }+ while ((de = readdir(d)) != NULL) {+ struct stat st;+ size_t n = strlen(de->d_name);++ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) {+ continue;+ }+ if (len + n + 2 > PATH_MAX) {+ total_errors++;+ continue;+ }+ path[len] = '/';+ memcpy(path + len + 1, de->d_name, n + 1);+ /* lstat, not stat: a symbolic link is not followed and not read */+ if (lstat(path, &st) != 0) {+ printf("%s %s: lstat failed: %s\n", tag, path + sizeof(MOUNT) - 1,+ strerror(errno));+ total_errors++;+ } else if (S_ISDIR(st.st_mode)) {+ walk(path, len + 1 + n);+ } else if (S_ISREG(st.st_mode)) {+ do_file(path, st.st_size);+ }+ path[len] = '\0';+ }+ closedir(d);+}++int+main(int argc, char **argv)+{+ static char path[PATH_MAX];+ const char *where = MOUNT "/usr";+ const char *mode = argc > 1 ? argv[1] : "mmap";+ double t0, t1;+ int tries;++ use_mmap = (strcmp(mode, "read") != 0);+ tag = use_mmap ? "apfs-mmap:" : "apfs-read:";+ if (argc > 2) {+ where = argv[2];+ }+ if (argc > 3) {+ chunk_size = atoi(argv[3]);+ if (chunk_size <= 0) {+ chunk_size = 4096;+ }+ }+ if (strncmp(where, MOUNT, sizeof(MOUNT) - 1) != 0) {+ printf("%s a path under %s, please\n", tag, MOUNT);+ return 2;+ }++ /* the kernel mounts the volume as soon as /private/var is there; wait for it */+ for (tries = 0; tries < 120; tries++) {+ struct stat st;++ if (stat(where, &st) == 0) {+ break;+ }+ sleep(1);+ }++ printf("%s %s, %s%s\n", tag, where, use_mmap ? "through a mapping" : "with read(2)",+ use_mmap ? "" : " in pieces");+ if (!use_mmap) {+ printf("%s pieces of %d bytes\n", tag, chunk_size);+ }+ snprintf(path, sizeof(path), "%s", where);+ t0 = now_ms();+ walk(path, strlen(path));+ t1 = now_ms();++ printf("%s files %ld bytes %llu errors %ld\n", tag, total_files, total_bytes,+ total_errors);+ printf("%s line-hash sum %016llx\n", tag, total_sum);+ printf("%s done in %.0f ms (%.1f MB/s)\n", tag, t1 - t0,+ (t1 > t0) ? (double)total_bytes / 1048576.0 / ((t1 - t0) / 1000.0) : 0.0);+ return total_errors ? 1 : 0;+}diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.cindex 707fce6..64a74d9 100644--- a/xnu/files/bsd/dev/redwood_apfs_check.c+++ b/xnu/files/bsd/dev/redwood_apfs_check.c@@ -5,6 +5,9 @@** apfs_check=1 walk the whole volume and print only the totals* apfs_check=2 also print a line per entry+ * apfs_check=3 mount it and stop, leaving it for user space to read+ * (userland/src/apfs_mmap_test.c, which is the only way the+ * driver's vnop_pagein ever runs)* apfs_subtree=/usr walk only that part of the volume** A line is the same one tools/apfs/apfs-tool's census and `verify.py clines` print:@@ -409,6 +412,11 @@ apfs_check_thread(void *arg, __unused wait_result_t wr)return;}printf("apfs-check: %s mounted on %s\n", CHECK_DEV, CHECK_MNT);+ if (mode >= 3) {+ /* someone in user space is going to read it; leave it alone */+ printf("apfs-check: left mounted, not walking it here\n");+ return;+ }cs.path = kalloc_data(PATH_CAP, Z_WAITOK | Z_ZERO);cs.buf = kalloc_data(READ_CHUNK, Z_WAITOK);diff --git a/xnu/files/bsd/miscfs/apfs/apfs.h b/xnu/files/bsd/miscfs/apfs/apfs.hindex 2f78602..ac76c3d 100644--- a/xnu/files/bsd/miscfs/apfs/apfs.h+++ b/xnu/files/bsd/miscfs/apfs/apfs.h@@ -50,6 +50,13 @@ struct apfs_node {uint16_t an_mode;uint64_t an_size; /* the logical size, as a reader of the file sees it */bool an_attached; /* an_vp is set and the node is in the hash */+ /*+ * What has been worked out about how this file's contents are stored, kept for as+ * long as the vnode lives: the table of compressed pieces and the piece last+ * decompressed. Without it every read and every page fault would find them again.+ * Guarded by am_lock, like everything else that reaches apfs_lib.+ */+ struct apfs_cfile an_cf;};#define APFS_MP(mp) ((struct apfs_mount *)vfs_fsprivate(mp))diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.cindex 7d55adf..63daed6 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c@@ -306,6 +306,9 @@ node_child(struct apfs_btree *tree, const struct node *n, uint32_t i, uint64_t *return 0;}+static int cfile_header(struct apfs_vol *vol, const struct apfs_inode *ino,+ bool *compressed, uint32_t *type, uint64_t *size, bool *in_rsrc);+/* Where a node identifier actually is: a block address, directly or through a map. */static inttree_node_addr(struct apfs_btree *tree, uint64_t oid, uint64_t *addr);@@ -1011,7 +1014,9 @@ apfs_inode_get(struct apfs_vol *vol, uint64_t obj_id, struct apfs_inode *ino)intapfs_inode_size(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t *size){- struct apfs_cfile cf;+ bool compressed = false, in_rsrc = false;+ uint32_t type = 0;+ uint64_t csize = 0;int error;if ((ino->mode & APFS_S_IFMT) == APFS_S_IFLNK) {@@ -1025,11 +1030,12 @@ apfs_inode_size(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t *si*size = n;return 0;}- error = apfs_cfile_init(vol, ino, &cf);+ /* only the decmpfs header is needed here, not the table of pieces */+ error = cfile_header(vol, ino, &compressed, &type, &csize, &in_rsrc);if (error) {return error;}- *size = cf.compressed ? cf.size : ino->size;+ *size = compressed ? csize : ino->size;return 0;}@@ -1503,15 +1509,19 @@ apfs_readlink(struct apfs_vol *vol, uint64_t obj_id, char *buf, size_t len, sizereturn 0;}-int-apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,- struct apfs_cfile *cf)+/* Just the decmpfs header: what a caller needs to report a size, and nothing more. */+static int+cfile_header(struct apfs_vol *vol, const struct apfs_inode *ino, bool *compressed,+ uint32_t *type, uint64_t *size, bool *in_rsrc){uint8_t hdr[16];size_t n = 0;int error;- memset(cf, 0, sizeof(*cf));+ *compressed = false;+ *type = 0;+ *size = 0;+ *in_rsrc = false;error = apfs_xattr_read(vol, ino->obj_id, APFS_DECMPFS_XATTR_NAME, hdr, sizeof(hdr),&n);if (error == ENOENT) {@@ -1523,25 +1533,189 @@ apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,if (n < sizeof(hdr) || le32(hdr) != APFS_DECMPFS_MAGIC) {return 0;}- cf->compressed = true;- cf->type = le32(hdr + 4);- cf->size = le64(hdr + 8);- switch (cf->type) {+ *compressed = true;+ *type = le32(hdr + 4);+ *size = le64(hdr + 8);+ switch (*type) {case APFS_DECMPFS_TYPE_ZLIB_ATTR:case APFS_DECMPFS_TYPE_RAW_ATTR:- cf->in_rsrc = false;+ *in_rsrc = false;return 0;case APFS_DECMPFS_TYPE_ZLIB_RSRC:case APFS_DECMPFS_TYPE_RAW_RSRC:- cf->in_rsrc = true;+ *in_rsrc = true;return 0;default:apfs_log("inode %llu: decmpfs type %u is not supported\n",- (unsigned long long)ino->obj_id, cf->type);+ (unsigned long long)ino->obj_id, *type);return ENOTSUP;}}+void+apfs_cfile_fini(struct apfs_cfile *cf)+{+ if (cf == NULL) {+ return;+ }+ if (cf->piece_off != NULL) {+ apfs_free(cf->piece_off, (size_t)cf->piece_count * sizeof(uint32_t));+ }+ if (cf->piece_len != NULL) {+ apfs_free(cf->piece_len, (size_t)cf->piece_count * sizeof(uint32_t));+ }+ if (cf->held_data != NULL) {+ apfs_free(cf->held_data, cf->held_cap);+ }+ memset(cf, 0, sizeof(*cf));+ cf->held = APFS_NO_PIECE;+}++/*+ * Read the resource fork's table of pieces, once. The fork is an HFS-Plus resource fork:+ * sixteen big-endian bytes giving the offset and length of the resource data and of the+ * resource map. At that data offset sits a four-byte big-endian length, and the table+ * starts right after it -- a little-endian count, then an (offset, length) pair per piece,+ * the offsets measured from the start of the table. The table is not at a fixed place:+ * the data offset is 256 in every file here, so the table lands at 260, but it has to be+ * read rather than assumed.+ */+static int+cfile_read_table(struct apfs_vol *vol, const struct apfs_inode *ino,+ struct apfs_cfile *cf)+{+ struct apfs_xattr xa;+ uint8_t head[16];+ uint8_t *tbl = NULL;+ size_t got = 0, tbl_len;+ uint32_t i;+ int error;++ error = apfs_xattr_lookup(vol, ino->obj_id, APFS_RSRC_FORK_XATTR_NAME, &xa);+ if (error) {+ return error;+ }+ if ((xa.flags & APFS_XATTR_DATA_STREAM) == 0) {+ return ENOTSUP; /* a resource fork small enough to be embedded */+ }+ cf->rsrc_id = xa.stream_id;+ cf->rsrc_size = xa.stream_size;++ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size, 0, head, sizeof(head),+ &got);+ if (error) {+ return error;+ }+ if (got < sizeof(head)) {+ return EIO;+ }+ cf->table_base = (uint64_t)be32(head) + 4;+ if (cf->table_base + 4 > cf->rsrc_size) {+ return EIO;+ }+ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size, cf->table_base, head, 4,+ &got);+ if (error) {+ return error;+ }+ if (got < 4) {+ return EIO;+ }+ cf->piece_count = le32(head);+ if (cf->piece_count == 0 ||+ cf->piece_count != (uint32_t)((cf->size + APFS_DECMPFS_BLOCK_SIZE - 1) /+ APFS_DECMPFS_BLOCK_SIZE)) {+ apfs_log("inode %llu: resource fork says %u pieces, %llu bytes needs %llu\n",+ (unsigned long long)ino->obj_id, cf->piece_count,+ (unsigned long long)cf->size,+ (unsigned long long)((cf->size + APFS_DECMPFS_BLOCK_SIZE - 1) /+ APFS_DECMPFS_BLOCK_SIZE));+ return EIO;+ }+ tbl_len = (size_t)cf->piece_count * 8;+ if (cf->table_base + 4 + tbl_len > cf->rsrc_size) {+ return EIO;+ }+ cf->piece_off = apfs_alloc((size_t)cf->piece_count * sizeof(uint32_t));+ cf->piece_len = apfs_alloc((size_t)cf->piece_count * sizeof(uint32_t));+ tbl = apfs_alloc(tbl_len);+ if (cf->piece_off == NULL || cf->piece_len == NULL || tbl == NULL) {+ error = ENOMEM;+ goto out;+ }+ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size, cf->table_base + 4, tbl,+ tbl_len, &got);+ if (error) {+ goto out;+ }+ if (got < tbl_len) {+ error = EIO;+ goto out;+ }+ for (i = 0; i < cf->piece_count; i++) {+ uint64_t off = le32(tbl + 8 * i);+ uint64_t len = le32(tbl + 8 * i + 4);++ if (len == 0 || cf->table_base + off + len > cf->rsrc_size) {+ error = EIO;+ goto out;+ }+ cf->piece_off[i] = (uint32_t)off;+ cf->piece_len[i] = (uint32_t)len;+ }+out:+ if (tbl != NULL) {+ apfs_free(tbl, tbl_len);+ }+ return error;+}++int+apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,+ struct apfs_cfile *cf)+{+ int error;++ if (cf->ready && cf->obj_id == ino->obj_id) {+ return 0; /* already worked out, with its table and piece */+ }+ apfs_cfile_fini(cf);+ cf->obj_id = ino->obj_id;+ cf->held = APFS_NO_PIECE;++ error = cfile_header(vol, ino, &cf->compressed, &cf->type, &cf->size, &cf->in_rsrc);+ if (error) {+ return error;+ }+ if (!cf->compressed) {+ cf->ready = true;+ return 0;+ }+ if (cf->in_rsrc) {+ error = cfile_read_table(vol, ino, cf);+ if (error) {+ apfs_cfile_fini(cf);+ return error;+ }+ cf->held_cap = APFS_DECMPFS_BLOCK_SIZE;+ } else if (cf->size <= APFS_DECMPFS_BLOCK_SIZE) {+ /* the whole thing is in the attribute and fits: hold it as one piece */+ cf->held_cap = (size_t)cf->size;+ cf->piece_count = 1;+ } else {+ cf->held_cap = 0; /* too big to hold; it is read afresh each time */+ }+ if (cf->held_cap != 0) {+ cf->held_data = apfs_alloc(cf->held_cap);+ if (cf->held_data == NULL) {+ apfs_cfile_fini(cf);+ return ENOMEM;+ }+ }+ cf->ready = true;+ return 0;+}+/** zlib asks for its own working memory through these. The kernel's copy of zlib has no* allocator of its own -- a zeroed z_stream leaves zalloc NULL and inflateInit_ then calls@@ -1627,26 +1801,74 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,return 0;}+/*+ * The contents of a file whose bytes are in the com.apple.decmpfs attribute, from the+ * bytes after the header.+ *+ * Type 9 means they were stored as they are, with one marker byte in front. All 23947+ * type-9 files in the iOS 16.5 root file system have 0xcc there and exactly `want` bytes+ * after it, which is how the marker's length was settled; a payload that is already+ * exactly the right length is taken whole.+ */+static int+decmpfs_attr(uint32_t type, const uint8_t *src, size_t src_len, size_t want,+ uint8_t *dst, size_t *out)+{+ switch (type) {+ case APFS_DECMPFS_TYPE_RAW_ATTR: {+ const uint8_t *p = src;+ size_t n = src_len;++ if (n != want) {+ if (n == 0) {+ return EIO;+ }+ p++;+ n--;+ }+ if (n > want) {+ n = want;+ }+ memcpy(dst, p, n);+ *out = n;+ return 0;+ }+ case APFS_DECMPFS_TYPE_ZLIB_ATTR:+ return apfs_inflate(src, src_len, dst, want, out);+ default:+ return ENOTSUP;+ }+}+/** A compressed file whose bytes are in the attribute: the whole thing is there, after the- * header, with one marker byte in front when it was stored rather than compressed.+ * header, with one marker byte in front when it was stored rather than compressed. When+ * the context can hold it, it is produced once and kept.*/static intread_cfile_attr(struct apfs_vol *vol, const struct apfs_inode *ino,- const struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out)+ struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out){struct apfs_xattr xa;uint8_t *attr = NULL;uint8_t *plain = NULL;- size_t attr_len = 0, plain_len = 0;- size_t alloc = APFS_XATTR_MAX_EMBEDDED_SIZE + 16;- int error;+ size_t attr_len = 0, plain_len = 0, alloc = 0;+ bool held = false;+ int error = 0;*out = 0;+ if (cf->held_cap != 0 && cf->held == 0) {+ plain = cf->held_data; /* produced on an earlier call */+ plain_len = cf->held_len;+ held = true;+ goto copy;+ }+error = apfs_xattr_lookup(vol, ino->obj_id, APFS_DECMPFS_XATTR_NAME, &xa);if (error) {return error;}+ alloc = APFS_XATTR_MAX_EMBEDDED_SIZE + 16;if (xa.flags & APFS_XATTR_DATA_STREAM) {if (xa.stream_size > 64u * 1024u * 1024u) {return EIO;@@ -1670,40 +1892,26 @@ read_cfile_attr(struct apfs_vol *vol, const struct apfs_inode *ino,error = EIO;goto out;}- plain = apfs_alloc((size_t)cf->size ? (size_t)cf->size : 1);- if (plain == NULL) {- error = ENOMEM;- goto out;- }- if (cf->type == APFS_DECMPFS_TYPE_RAW_ATTR) {- const uint8_t *p = attr + 16;- size_t n = attr_len - 16;-- /*- * Stored as it is. Every type-9 file in the iOS 16.5 root file system has one- * marker byte in front of the contents; a payload that is already exactly the- * right length is taken whole.- */- if (n != cf->size) {- if (n == 0) {- error = EIO;- goto out;- }- p++;- n--;- }- if (n > cf->size) {- n = (size_t)cf->size;- }- memcpy(plain, p, n);- plain_len = n;+ if (cf->held_cap != 0 && cf->held_cap >= (size_t)cf->size) {+ plain = cf->held_data;+ held = true;} else {- error = apfs_inflate(attr + 16, attr_len - 16, plain, (size_t)cf->size,- &plain_len);- if (error) {+ plain = apfs_alloc((size_t)cf->size ? (size_t)cf->size : 1);+ if (plain == NULL) {+ error = ENOMEM;goto out;}}+ error = decmpfs_attr(cf->type, attr + 16, attr_len - 16, (size_t)cf->size,+ plain, &plain_len);+ if (error) {+ goto out;+ }+ if (held) {+ cf->held = 0;+ cf->held_len = plain_len;+ }+copy:if (off < plain_len) {size_t n = plain_len - (size_t)off;@@ -1717,7 +1925,7 @@ out:if (attr != NULL) {apfs_free(attr, alloc);}- if (plain != NULL) {+ if (plain != NULL && !held) {apfs_free(plain, (size_t)cf->size ? (size_t)cf->size : 1);}return error;@@ -1725,50 +1933,23 @@ out:/** A compressed file whose bytes are in com.apple.ResourceFork: the contents cut into- * 64 KB pieces, each its own zlib stream. The fork is an HFS-Plus resource fork; the- * piece table starts four bytes past the data offset in its big-endian header.+ * 64 KB pieces, each its own zlib stream. The table saying where the pieces are was read+ * once by apfs_cfile_init(); the piece last decompressed is kept, so a reader working+ * through the file -- or a page fault asking for 16 KB of it -- does not decompress the+ * same 64 KB again.*/static intread_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,- const struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out)+ struct apfs_cfile *cf, uint64_t off, uint8_t *buf, size_t len, size_t *out){- struct apfs_xattr xa;- uint8_t head[16];- uint8_t entry[8];- uint8_t *piece = NULL, *plain = NULL;- size_t piece_cap = 0, got = 0;- uint64_t base, nblocks, first, last, i;- int error;+ uint8_t *piece = NULL;+ size_t piece_cap = 0;+ uint64_t first, last, i;+ int error = 0;*out = 0;- error = apfs_xattr_lookup(vol, ino->obj_id, APFS_RSRC_FORK_XATTR_NAME, &xa);- if (error) {- return error;- }- if ((xa.flags & APFS_XATTR_DATA_STREAM) == 0) {- return ENOTSUP; /* a resource fork small enough to be embedded */- }- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, 0, head, sizeof(head),- &got);- if (error) {- return error;- }- if (got < sizeof(head)) {- return EIO;- }- base = (uint64_t)be32(head) + 4; /* data offset, then the big-endian length */- if (base + 4 > xa.stream_size) {- return EIO;- }- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, base, entry, 4, &got);- if (error) {- return error;- }- if (got < 4) {- return EIO;- }- nblocks = le32(entry);- if (nblocks == 0 || nblocks > (cf->size / APFS_RSRC_PIECE) + 1) {+ (void)ino;+ if (cf->piece_count == 0 || cf->piece_off == NULL) {return EIO;}if (off >= cf->size) {@@ -1777,102 +1958,83 @@ read_cfile_rsrc(struct apfs_vol *vol, const struct apfs_inode *ino,if (len > cf->size - off) {len = (size_t)(cf->size - off);}- first = off / APFS_RSRC_PIECE;- last = (off + len - 1) / APFS_RSRC_PIECE;- if (last >= nblocks) {+ first = off / APFS_DECMPFS_BLOCK_SIZE;+ last = (off + len - 1) / APFS_DECMPFS_BLOCK_SIZE;+ if (last >= cf->piece_count) {return EIO;}- plain = apfs_alloc(APFS_RSRC_PIECE);- if (plain == NULL) {- return ENOMEM;- }+for (i = first; i <= last; i++) {- uint64_t ent_off = base + 4 + 8 * i;- uint64_t poff, plen;- size_t want, copied, from_in_piece, n;+ size_t want = APFS_DECMPFS_BLOCK_SIZE;+ size_t from_in_piece, n, got = 0;- if (ent_off + 8 > xa.stream_size) {- error = EIO;- goto out;- }- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, ent_off, entry, 8,- &got);- if (error) {- goto out;+ if ((i + 1) * APFS_DECMPFS_BLOCK_SIZE > cf->size) {+ want = (size_t)(cf->size - i * APFS_DECMPFS_BLOCK_SIZE);}- if (got < 8) {- error = EIO;- goto out;- }- poff = le32(entry);- plen = le32(entry + 4);- if (plen == 0 || base + poff + plen > xa.stream_size) {- error = EIO;- goto out;- }- if (plen > piece_cap) {- if (piece != NULL) {- apfs_free(piece, piece_cap);+ if (cf->held != i) {+ uint32_t plen = cf->piece_len[i];++ if (plen > piece_cap) {+ if (piece != NULL) {+ apfs_free(piece, piece_cap);+ }+ piece_cap = plen;+ piece = apfs_alloc(piece_cap);+ if (piece == NULL) {+ error = ENOMEM;+ goto out;+ }}- piece_cap = (size_t)plen;- piece = apfs_alloc(piece_cap);- if (piece == NULL) {- error = ENOMEM;+ error = apfs_read_stream(vol, cf->rsrc_id, cf->rsrc_size,+ cf->table_base + cf->piece_off[i], piece, plen, &got);+ if (error) {goto out;}- }- error = apfs_read_stream(vol, xa.stream_id, xa.stream_size, base + poff, piece,- (size_t)plen, &got);- if (error) {- goto out;- }- if (got < plen) {- error = EIO;- goto out;- }- /* how much of the file this piece stands for */- want = APFS_RSRC_PIECE;- if ((i + 1) * APFS_RSRC_PIECE > cf->size) {- want = (size_t)(cf->size - i * APFS_RSRC_PIECE);- }- if (cf->type == APFS_DECMPFS_TYPE_RAW_RSRC) {- size_t m = plen < want ? (size_t)plen : want;+ if (got < plen) {+ error = EIO;+ goto out;+ }+ cf->held = APFS_NO_PIECE; /* what is in there now is unknown */+ if (cf->type == APFS_DECMPFS_TYPE_RAW_RSRC) {+ size_t m = plen < want ? (size_t)plen : want;- memcpy(plain, piece, m);- copied = m;- } else {- error = apfs_inflate(piece, (size_t)plen, plain, want, &copied);- if (error) {+ memcpy(cf->held_data, piece, m);+ cf->held_len = m;+ } else {+ error = apfs_inflate(piece, plen, cf->held_data, want,+ &cf->held_len);+ if (error) {+ goto out;+ }+ }+ if (cf->held_len != want) {+ error = EIO;goto out;}+ cf->held = i;}- if (copied != want) {- error = EIO;- goto out;- }- /* the overlap of this piece with what was asked for */- from_in_piece = (off > i * APFS_RSRC_PIECE)- ? (size_t)(off - i * APFS_RSRC_PIECE) : 0;- n = copied - from_in_piece;+ from_in_piece = (off > i * APFS_DECMPFS_BLOCK_SIZE)+ ? (size_t)(off - i * APFS_DECMPFS_BLOCK_SIZE) : 0;+ n = cf->held_len - from_in_piece;if (n > len - *out) {n = len - *out;}- memcpy(buf + *out, plain + from_in_piece, n);+ memcpy(buf + *out, cf->held_data + from_in_piece, n);*out += n;}out:if (piece != NULL) {apfs_free(piece, piece_cap);}- apfs_free(plain, APFS_RSRC_PIECE);return error;}int-apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t off,- void *buf, size_t len, size_t *out)+apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino,+ struct apfs_cfile *cf, uint64_t off, void *buf, size_t len, size_t *out){- struct apfs_cfile cf;+ struct apfs_cfile own;+ bool borrowed = false;int error;*out = 0;@@ -1889,18 +2051,31 @@ apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t off,}return 0;}- error = apfs_cfile_init(vol, ino, &cf);- if (error) {- return error;- }- if (!cf.compressed) {- if (!ino->has_dstream) {- return 0;+ /*+ * Without a context from the caller, one is made for this call alone -- correct, and+ * the slow way: the table and the piece are then found again on the next call.+ */+ if (cf == NULL) {+ memset(&own, 0, sizeof(own));+ own.held = APFS_NO_PIECE;+ cf = &own;+ borrowed = true;+ }+ error = apfs_cfile_init(vol, ino, cf);+ if (error == 0) {+ if (!cf->compressed) {+ error = ino->has_dstream+ ? apfs_read_stream(vol, ino->private_id, ino->size, off, buf, len,+ out)+ : 0;+ } else if (cf->in_rsrc) {+ error = read_cfile_rsrc(vol, ino, cf, off, buf, len, out);+ } else {+ error = read_cfile_attr(vol, ino, cf, off, buf, len, out);}- return apfs_read_stream(vol, ino->private_id, ino->size, off, buf, len, out);}- if (cf.in_rsrc) {- return read_cfile_rsrc(vol, ino, &cf, off, buf, len, out);+ if (borrowed) {+ apfs_cfile_fini(cf);}- return read_cfile_attr(vol, ino, &cf, off, buf, len, out);+ return error;}diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.hindex c8b9038..5d29e00 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h@@ -197,12 +197,46 @@ struct apfs_xattr {uint64_t stream_size;};-/* How a file's contents are stored, once the decmpfs attribute has been read. */+/* No piece is being held. */+#define APFS_NO_PIECE ((uint64_t)~0ULL)++/*+ * How a file's contents are stored, and what has been worked out about it so far.+ *+ * A compressed file's bytes are cut into 64 KB pieces, each its own zlib stream, with a+ * table in the resource fork saying where each piece is. Reaching that table costs+ * several walks of the file-system tree -- the attribute record, the fork's header, the+ * table entry -- and a piece has to be decompressed whole however few of its bytes are+ * wanted. A reader that comes back for the next page would pay all of that again, which+ * is what made paging a mapped file slow.+ *+ * So this holds what has been found: the fork's stream, the whole table, and the last+ * piece decompressed. One of these belongs to whoever is reading the file -- the driver+ * keeps one per vnode -- and it is handed to apfs_read_file() on every call.+ * apfs_cfile_fini() gives back what it holds. Passing NULL instead works and is correct,+ * and is the slow way.+ */struct apfs_cfile {- bool compressed;- uint32_t type; /* decmpfs compression type */- uint64_t size; /* uncompressed size */- bool in_rsrc; /* the bytes are in com.apple.ResourceFork */+ bool ready; /* set up, for the inode in obj_id */+ uint64_t obj_id;+ bool compressed;+ uint32_t type; /* decmpfs compression type */+ uint64_t size; /* uncompressed size */+ bool in_rsrc; /* the bytes are in com.apple.ResourceFork */++ /* the resource fork and its table of pieces, read once */+ uint64_t rsrc_id;+ uint64_t rsrc_size;+ uint64_t table_base;+ uint32_t piece_count;+ uint32_t *piece_off; /* piece_count entries, from table_base */+ uint32_t *piece_len;++ /* the piece last decompressed */+ uint64_t held; /* its index, or APFS_NO_PIECE */+ uint8_t *held_data;+ size_t held_len;+ size_t held_cap; /* 0 when this file is not worth holding */};/* ---- container and volumes ---- */@@ -287,16 +321,22 @@ int apfs_readlink(struct apfs_vol *vol, uint64_t obj_id, char *buf, size_t len,int apfs_read_stream(struct apfs_vol *vol, uint64_t stream_id, uint64_t stream_size,uint64_t off, void *buf, size_t len, size_t *out);-/* Whether a file is compressed, and how. */+/*+ * Work out how a file's contents are stored and read in whatever is worth reading once.+ * Calling it again for the same inode does nothing; for a different one it starts over.+ */int apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,struct apfs_cfile *cf);+/* Give back everything a context holds. Safe on a zeroed or already finished one. */+void apfs_cfile_fini(struct apfs_cfile *cf);+/** The contents of a file as a reader of the file would see them: the data stream for a* plain file, the decompressed bytes for a compressed one.*/-int apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino, uint64_t off,- void *buf, size_t len, size_t *out);+int apfs_read_file(struct apfs_vol *vol, const struct apfs_inode *ino,+ struct apfs_cfile *cf, uint64_t off, void *buf, size_t len, size_t *out);/* ---- odds and ends worth testing on their own ---- */diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c b/xnu/files/bsd/miscfs/apfs/apfs_vnops.cindex 1f74996..23e6fba 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_vnops.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_vnops.c@@ -200,8 +200,8 @@ apfs_vnop_read(struct vnop_read_args *ap)size_t got = 0;apfs_lock(am);- error = apfs_read_file(&am->am_vol, &ino, (uint64_t)uio_offset(uio), buf, want,- &got);+ error = apfs_read_file(&am->am_vol, &ino, &APFS_VP(vp)->an_cf,+ (uint64_t)uio_offset(uio), buf, want, &got);apfs_unlock(am);if (error) {break;@@ -273,7 +273,8 @@ apfs_vnop_pagein(struct vnop_pagein_args *ap)}want = (size_t)MIN(end - at, (off_t)(size - (uint64_t)at));apfs_lock(am);- error = apfs_read_file(&am->am_vol, &ino, (uint64_t)at, dst, want, &got);+ error = apfs_read_file(&am->am_vol, &ino, &APFS_VP(vp)->an_cf, (uint64_t)at,+ dst, want, &got);apfs_unlock(am);if (error) {break;@@ -593,6 +594,9 @@ apfs_vnop_reclaim(struct vnop_reclaim_args *ap)struct apfs_node *an = APFS_VP(vp);if (an != NULL) {+ apfs_lock(an->an_mount);+ apfs_cfile_fini(&an->an_cf);+ apfs_unlock(an->an_mount);apfs_node_detach(an);vnode_clearfsnode(vp);kfree_type(struct apfs_node, an);