Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.
C
APFS: a name found by its hash, halving within B-tree nodes, object-map answers kept
vcvkk171b06af7fe9
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.cindex 3761936..e6b4bb9 100644--- a/tools/apfs/apfs-tool.c+++ b/tools/apfs/apfs-tool.c@@ -14,6 +14,7 @@* apfs-tool stat IMAGE PATH* apfs-tool cat IMAGE PATH* apfs-tool census IMAGE [PATH] [--meta] (--meta: do not read contents)+ * apfs-tool lookups IMAGE (every name looked up by name; see cmd_lookups)*//* pread(2) and clock_gettime(2) come with -D_DEFAULT_SOURCE, set by build-tool.sh */@@ -599,6 +600,162 @@ cmd_census(const char *path, const char *where, bool meta_only)return cs.error ? 1 : 0;}+/*+ * Every name in the volume looked up by name, the way the kernel's vnop_lookup does it,+ * against what reading its directory gave; and each name with "#" after it, which must+ * not be found. Also how long that took against finding each name by reading its+ * directory from the start, which is what apfs_lookup did before it went by the hash.+ */+struct lk_entry {+ char name[256];+ size_t len;+ uint64_t id;+ uint8_t type;+};++struct lk_dir {+ struct lk_entry *e;+ size_t n, cap;+};++static int+lk_collect(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)+{+ struct lk_dir *d = ctx;++ if (d->n == d->cap) {+ size_t cap = d->cap ? d->cap * 2 : 64;+ struct lk_entry *e = realloc(d->e, cap * sizeof(*e));++ if (e == NULL) {+ return ENOMEM;+ }+ d->e = e;+ d->cap = cap;+ }+ if (name_len >= sizeof(d->e[0].name)) {+ return EIO;+ }+ memcpy(d->e[d->n].name, name, name_len);+ d->e[d->n].name[name_len] = '\0';+ d->e[d->n].len = name_len;+ d->e[d->n].id = file_id;+ d->e[d->n].type = d_type;+ d->n++;+ return 0;+}++struct lk_scan {+ const char *name;+ size_t len;+ bool found;+};++static int+lk_scan_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)+{+ struct lk_scan *sc = ctx;++ (void)file_id;+ (void)d_type;+ if (name_len == sc->len && memcmp(name, sc->name, name_len) == 0) {+ sc->found = true;+ return EEXIST;+ }+ return 0;+}++static double+lk_now(void)+{+ struct timespec ts;++ clock_gettime(CLOCK_MONOTONIC, &ts);+ return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;+}++static int+cmd_lookups(const char *path)+{+ struct apfs_nx nx;+ struct apfs_vol vol;+ uint64_t *stack = NULL;+ size_t depth = 0, cap = 0;+ uint64_t dirs = 0, names = 0, good = 0, bad = 0, absent_good = 0, absent_bad = 0;+ double t_hash = 0, t_scan = 0;++ if (open_image(path, &nx, &vol) != 0) {+ return 1;+ }+ stack = malloc(sizeof(*stack) * 64);+ cap = 64;+ stack[depth++] = APFS_ROOT_DIR_INO_NUM;+ while (depth > 0) {+ uint64_t dir = stack[--depth];+ struct lk_dir d = { 0 };+ size_t i;+ int error = apfs_readdir(&vol, dir, lk_collect, &d);++ if (error) {+ fprintf(stderr, "apfs-tool: readdir %" PRIu64 ": %s\n", dir, strerror(error));+ bad++;+ continue;+ }+ dirs++;+ for (i = 0; i < d.n; i++) {+ struct lk_entry *e = &d.e[i];+ struct lk_scan sc = { e->name, e->len, false };+ char absent[260];+ uint64_t id = 0;+ uint8_t type = 0;+ double t0;++ names++;+ t0 = lk_now();+ error = apfs_lookup(&vol, dir, e->name, e->len, &id, &type);+ t_hash += lk_now() - t0;+ if (error == 0 && id == e->id && type == e->type) {+ good++;+ } else {+ bad++;+ if (bad <= 10) {+ fprintf(stderr, "apfs-tool: lookup %s in %" PRIu64 ": %d, %" PRIu64+ " (readdir: %" PRIu64 ")\n", e->name, dir, error, id, e->id);+ }+ }+ t0 = lk_now();+ error = apfs_readdir(&vol, dir, lk_scan_cb, &sc);+ t_scan += lk_now() - t0;+ if (!sc.found) {+ bad++;+ }+ snprintf(absent, sizeof(absent), "%s#", e->name);+ error = apfs_lookup(&vol, dir, absent, e->len + 1, NULL, NULL);+ if (error == ENOENT) {+ absent_good++;+ } else {+ absent_bad++;+ }+ if (e->type == APFS_DT_DIR) {+ if (depth == cap) {+ cap *= 2;+ stack = realloc(stack, sizeof(*stack) * cap);+ }+ stack[depth++] = e->id;+ }+ }+ free(d.e);+ }+ printf("directories %" PRIu64 ", names %" PRIu64 ": found as readdir has them %" PRIu64+ ", not %" PRIu64 "; with \"#\" after them not found %" PRIu64 ", found %" PRIu64 "\n",+ dirs, names, good, bad, absent_good, absent_bad);+ printf("time for all of them: apfs_lookup %.0f ms, reading the directory from its start "+ "%.0f ms\n", t_hash * 1000, t_scan * 1000);+ free(stack);+ apfs_nx_close(&nx);+ return (bad == 0 && absent_bad == 0) ? 0 : 1;+}+intmain(int argc, char **argv){@@ -608,7 +765,8 @@ main(int argc, char **argv)" apfs-tool ls IMAGE [PATH]\n"" apfs-tool stat IMAGE PATH\n"" apfs-tool cat IMAGE PATH\n"- " apfs-tool census IMAGE [PATH] [--meta]\n");+ " apfs-tool census IMAGE [PATH] [--meta]\n"+ " apfs-tool lookups IMAGE\n");return 2;}if (strcmp(argv[1], "info") == 0) {@@ -627,6 +785,9 @@ main(int argc, char **argv)return cmd_census(argv[2], (argc > 3 && argv[3][0] == '/') ? argv[3] : NULL,(argc > 3 && strcmp(argv[argc - 1], "--meta") == 0));}+ if (strcmp(argv[1], "lookups") == 0) {+ return cmd_lookups(argv[2]);+ }fprintf(stderr, "apfs-tool: unknown command %s\n", argv[1]);return 2;}diff --git a/tools/apfs/run-tests.sh b/tools/apfs/run-tests.shindex 035aecc..7f26c3d 100755--- a/tools/apfs/run-tests.sh+++ b/tools/apfs/run-tests.sh@@ -6,8 +6,10 @@# 3. the APFS reader against 7-Zip's, over every entry in the volume (needs ios165/)# 4. file contents against the files 7-Zip extracted, byte for byte (needs ios165/)# 5. the C the kernel is built from against apfs.py, over the whole volume+# 6. the hash in directory records, worked out as the C lookup does (needs the image)+# 7. every name looked up by name through the C, against its directory (needs the image)#-# Steps 3, 4 and 5 need the APFS partition unpacked out of ios165/rootfs.dmg; pass it as+# Steps 3 to 7 need the APFS partition unpacked out of ios165/rootfs.dmg; pass it as# $1, or set APFS_IMAGE. Without it only the first two steps run.## APFS_SUBTREE=/some/path keeps step 5 to one subtree, which is the quick way to run it;@@ -44,6 +46,17 @@ echo "== 1b. hole handling, on made-up extent lists (the image has no sparse filpython3 "$HERE/test_sparse.py" || fail=1echo+echo "== 1c. the C that reads file contents without the lock, on made-up runs (holes, ends inside blocks)"+if cc -std=c11 -O2 -w -I"$HERE/host-shim" -c -o "$TMP/deflate.o" "$HDR_DIR/apfs_deflate.c" &&+ cc -std=c11 -D_DEFAULT_SOURCE -O1 -Wall -Wextra -Wno-unused-parameter -Wno-unused-function \+ -I"$HDR_DIR" -o "$TMP/test_fetch" "$HERE/test_fetch.c" "$TMP/deflate.o" -lz -pthread; then+ "$TMP/test_fetch" || fail=1+else+ echo " would not build"+ fail=1+fi+echo+echo "== 2. the UDIF reader on ios165/rootfs.dmg"if [ -f "$ROOT/ios165/rootfs.dmg" ]; thenpython3 "$HERE/dmg.py" info "$ROOT/ios165/rootfs.dmg" | tail -n 14 || fail=1@@ -96,5 +109,18 @@ elseecho " skipped: the tool would not build"fail=1fi+echo++echo "== 6. the hash in directory records, as the lookup works it out"+python3 "$HERE/test_namehash.py" "$IMAGE" || fail=1+echo++echo "== 7. every name in the volume looked up by name (and with \"#\" after it, not found)"+if [ -x "$TMP/apfs-tool" ]; then+ "$TMP/apfs-tool" lookups "$IMAGE" || fail=1+else+ echo " skipped: the tool would not build"+ fail=1+fiexit $faildiff --git a/tools/apfs/test_fetch.c b/tools/apfs/test_fetch.cnew file mode 100644index 0000000..c18bb29--- /dev/null+++ b/tools/apfs/test_fetch.c@@ -0,0 +1,92 @@+/*+ * stream_fetch() -- the part of reading a file's contents that runs without the volume's+ * lock -- against made-up runs, since the iOS 16.5 root volume has no sparse file and so+ * never gives it a hole: holes at the front, between runs and at the end, runs that start+ * and end inside a block, a run inside one block, no runs at all. apfs_lib.c is included+ * whole, so that its static functions can be called. The device is 64 blocks of memory,+ * block n holding bytes that depend on n and on the place in the block, so a byte from+ * the wrong block or the wrong place shows. Built and run by run-tests.sh.+ */+#include "apfs_lib.c"++#define BS 4096+#define NBLK 64++static uint8_t disk[NBLK * BS];++static int+dev_read(void *ctx, uint64_t off, void *buf, size_t len)+{+ (void)ctx;+ if (off % BS != 0 || len % BS != 0 || off + len > sizeof(disk)) {+ return EINVAL; /* whole blocks only, as apfs_dev.read promises */+ }+ memcpy(buf, disk + off, len);+ return 0;+}++struct fcase {+ const char *what;+ uint32_t nruns;+ struct apfs_run runs[APFS_MAX_RUNS];+ uint64_t covered;+};++static const struct fcase cases[] = {+ { "whole blocks", 1, { { 0, 3, 0, 3 * BS } }, 3 * BS },+ { "starts and ends inside blocks", 1, { { 0, 5, 100, 2 * BS + 50 } }, 2 * BS + 50 },+ { "inside one block", 1, { { 0, 7, 10, 20 } }, 20 },+ { "ends where a block does", 1, { { 0, 10, 4000, 96 + BS } }, 96 + BS },+ { "a hole at the front", 1, { { 1000, 2, 0, 5000 } }, 6000 },+ { "a hole between two runs", 2, { { 0, 1, 0, BS }, { 2 * BS, 9, 0, BS } }, 3 * BS },+ { "a hole at the end", 1, { { 0, 4, 0, 100 } }, 2 * BS },+ { "no runs: all hole", 0, { { 0 } }, 5000 },+ { "eight runs, every one unaligned", 8, {+ { 0, 20, 1, 10 }, { 10, 21, 4095, 2 }, { 100, 22, 0, BS + 1 },+ { 2 * BS, 30, 2048, 2048 }, { 3 * BS, 31, 0, 1 }, { 3 * BS + 5, 40, 7, 3 * BS },+ { 7 * BS, 50, 4000, 200 }, { 8 * BS, 60, 0, 4 * BS } }, 12 * BS },+};++int+main(void)+{+ struct apfs_nx nx;+ static uint8_t dst[16 * BS], want[16 * BS], bounce[BS];+ unsigned c, failed = 0;+ uint64_t i;++ for (i = 0; i < sizeof(disk); i++) {+ disk[i] = (uint8_t)((i / BS) * 37 + (i % BS) * 7 + 1);+ }+ memset(&nx, 0, sizeof(nx));+ nx.block_size = BS;+ nx.block_count = NBLK;+ nx.dev.read = dev_read;++ for (c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {+ const struct fcase *fc = &cases[c];+ uint64_t calls = 0, bytes = 0;+ uint32_t r;+ int error;++ memset(want, 0, sizeof(want));+ for (r = 0; r < fc->nruns; r++) {+ const struct apfs_run *run = &fc->runs[r];++ memcpy(want + run->at, disk + run->addr * BS + run->skew, run->len);+ }+ memset(dst, 0xAA, sizeof(dst)); /* what is not written shows */+ error = stream_fetch(&nx, fc->runs, fc->nruns, fc->covered, dst, bounce, &calls,+ &bytes);+ if (error != 0 || memcmp(dst, want, fc->covered) != 0 || dst[fc->covered] != 0xAA) {+ printf(" FAIL %s (error %d)\n", fc->what, error);+ failed++;+ } else {+ printf(" ok %s (%llu requests, %llu bytes)\n", fc->what,+ (unsigned long long)calls, (unsigned long long)bytes);+ }+ }+ printf("stream_fetch: %u cases, %u failed\n", (unsigned)(sizeof(cases) / sizeof(cases[0])),+ failed);+ return failed ? 1 : 0;+}diff --git a/tools/apfs/test_namehash.py b/tools/apfs/test_namehash.pynew file mode 100644index 0000000..ef5ce1b--- /dev/null+++ b/tools/apfs/test_namehash.py@@ -0,0 +1,79 @@+#!/usr/bin/env python3+# The hash in a directory record's key, worked out the way apfs_lib.c's lookup works it+# out, against the hash APFS stored with every name of a volume that is plain ASCII:+# CRC-32C over the name as little-endian 32-bit code points, started at all ones and not+# inverted at the end, low 22 bits. (Other names are hashed after Unicode canonical+# decomposition, which the lookup does not do: it reads the directory through for them.)+# Also that the records of a directory sort by the word holding hash and length, then by+# name -- what lets the lookup stop at the first record past the hash.+#+# usage: test_namehash.py IMAGE++import os+import struct+import sys++sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+import apfs++POLY = 0x82F63B78+TABLE = []+for _i in range(256):+ _c = _i+ for _ in range(8):+ _c = (_c >> 1) ^ POLY if _c & 1 else _c >> 1+ TABLE.append(_c)+++def name_hash(name):+ crc = 0xFFFFFFFF+ for ch in name:+ for b in struct.pack("<I", ord(ch)):+ crc = TABLE[(crc ^ b) & 0xFF] ^ (crc >> 8)+ return crc & 0x3FFFFF+++def main():+ if len(sys.argv) != 2:+ print("usage: test_namehash.py IMAGE", file=sys.stderr)+ return 2+ c = apfs.Container(sys.argv[1])+ vol = next(iter(c.volumes()))+ if vol.case_insensitive:+ print("a case-insensitive volume: names are folded before hashing; not checked here")+ return 2+ total = ascii_n = good = 0+ bad = []+ disorder = 0+ prev = None+ for k, _v in vol.tree.iter_all():+ x = struct.unpack_from("<Q", k, 0)[0]+ oid, typ = x & apfs.OBJ_ID_MASK, x >> apfs.OBJ_TYPE_SHIFT+ if typ != apfs.TYPE_DIR_REC:+ prev = None+ continue+ nlh = struct.unpack_from("<I", k, 8)[0]+ raw = k[12:12 + (nlh & apfs.J_DREC_LEN_MASK)].split(b"\0", 1)[0]+ total += 1+ if prev is not None and prev[0] == oid and prev[1:] > (nlh, raw):+ disorder += 1+ prev = (oid, nlh, raw)+ try:+ name = raw.decode("ascii")+ except UnicodeDecodeError:+ continue+ ascii_n += 1+ if name_hash(name) == nlh >> apfs.J_DREC_HASH_SHIFT and \+ (nlh & apfs.J_DREC_LEN_MASK) == len(raw) + 1:+ good += 1+ elif len(bad) < 10:+ bad.append((name, nlh))+ print("directory records %d, plain ASCII %d, hash and length as worked out %d, not %d;"+ " out of order %d" % (total, ascii_n, good, ascii_n - good, disorder))+ for name, nlh in bad:+ print(" %r: stored %06x, worked out %06x" % (name, nlh >> 10, name_hash(name)))+ return 0 if good == ascii_n and disorder == 0 else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.cindex 7a18629..cc726fd 100644--- a/xnu/files/bsd/dev/redwood_apfs_check.c+++ b/xnu/files/bsd/dev/redwood_apfs_check.c@@ -179,8 +179,25 @@ struct census {*/bool no_hash;uint64_t hash_ticks;+ /*+ * Boot-arg apfs_contents=0: no file is read at all, only looked up and stat'ed --+ * how long the walk itself takes, the directories and the inodes.+ */+ bool no_contents;};+/* The boot-args both kinds of walk take the same way. */+static void+census_options(struct census *cs)+{+ uint32_t hash = 1, contents = 1;++ PE_parse_boot_argn("apfs_hash", &hash, sizeof(hash));+ PE_parse_boot_argn("apfs_contents", &contents, sizeof(contents));+ cs->no_hash = (hash == 0);+ cs->no_contents = (contents == 0);+}+/* The format bits of a mode, which vnode_getattr does not report in va_mode. */static uint16_tifmt_of(enum vtype t)@@ -301,11 +318,15 @@ census_entry(struct census *cs, uint8_t d_type)if (vnode_vtype(vp) == VREG) {cs->files++;PHASE("read");- error = hash_file(cs, vp, size, &h);+ if (!cs->no_contents) {+ error = hash_file(cs, vp, size, &h);+ }} else if (vnode_vtype(vp) == VLNK) {cs->links++;PHASE("readlink");- error = hash_link(cs, vp, &h);+ if (!cs->no_contents) {+ error = hash_link(cs, vp, &h);+ }} else if (vnode_vtype(vp) == VDIR) {cs->dirs++;} else {@@ -610,12 +631,10 @@ apfs_walk_one(void *arg, __unused wait_result_t wr)struct walk_job *job = arg;struct census cs;uint64_t start, end, ns = 0, hash_ns = 0;- uint32_t hash = 1;bzero(&cs, sizeof(cs));cs.ctx = vfs_context_kernel();- PE_parse_boot_argn("apfs_hash", &hash, sizeof(hash));- cs.no_hash = (hash == 0);+ census_options(&cs);PE_parse_boot_argn("apfs_depth", &cs.max_depth, sizeof(cs.max_depth));PE_parse_boot_argn("apfs_max", &cs.max_entries, sizeof(cs.max_entries));{@@ -725,6 +744,7 @@ apfs_check_thread(void *arg, __unused wait_result_t wr)bzero(&cs, sizeof(cs));cs.ctx = ctx;+ census_options(&cs);cs.print_lines = (mode == 2);/** NOCROSSMOUNT only where the volume is the root (modes 4 and 5). In mode 6 thediff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.cindex 245566c..1790b03 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c@@ -474,9 +474,23 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *aconst uint8_t *blk;int level, error;+ struct apfs_omap_hit *hit = NULL;+if (!omap->ready) {return EIO;}+ if (tree->nx->omap_cache != NULL) {+ uint64_t h = (oid ^ (tree->root * 0xff51afd7ed558ccdULL)) * 0x9E3779B97F4A7C15ULL;++ hit = &tree->nx->omap_cache[h >> 53 & (APFS_OMAP_CACHE - 1)];+ if (hit->oid == oid && oid != 0 && hit->xid == want_xid &&+ hit->map == tree->root) {+ tree->nx->omap_hits++;+ *addr = hit->addr;+ return 0;+ }+ tree->nx->omap_misses++;+ }error = nx_block(tree->nx, node_oid, &blk, true);if (error) {return error;@@ -486,19 +500,20 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *areturn error;}for (level = 0; level < APFS_MAX_LEVELS; level++) {- uint32_t i, best = 0;- bool found = false;+ uint32_t lo = 0, hi, best;if (n.nkeys == 0) {return ENOENT;}- /* the last entry whose key is at or before what we are looking for */- for (i = 0; i < n.nkeys; i++) {+ /* the last entry whose key is at or before what we are looking for: halving */+ hi = n.nkeys;+ while (lo < hi) {+ uint32_t mid = lo + (hi - lo) / 2;const uint8_t *k, *v;uint16_t kl, vl;uint64_t o, x;- error = node_entry(tree, &n, i, &k, &kl, &v, &vl);+ error = node_entry(tree, &n, mid, &k, &kl, &v, &vl);if (error) {return error;}@@ -508,15 +523,15 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *ao = le64(k);x = le64(k + 8);if (o < oid || (o == oid && x <= want_xid)) {- best = i;- found = true;+ lo = mid + 1;} else {- break;+ hi = mid;}}- if (!found) {+ if (lo == 0) {return ENOENT;}+ best = lo - 1;if (n.leaf) {const uint8_t *k, *v;uint16_t kl, vl;@@ -536,6 +551,12 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *aif (*addr == 0 || *addr >= tree->nx->block_count) {return EIO;}+ if (hit != NULL) {+ hit->map = tree->root;+ hit->oid = oid;+ hit->xid = want_xid;+ hit->addr = *addr;+ }return 0;}error = node_child(tree, &n, best, &node_oid);@@ -585,6 +606,8 @@ apfs_nx_open(struct apfs_nx *nx, const struct apfs_dev *dev)for (i = 0; i < APFS_CACHE_BLOCKS; i++) {nx->cache[i].addr = APFS_BUF_EMPTY;}+ /* zeroed, so every slot is empty; without it the object maps are simply walked */+ nx->omap_cache = apfs_alloc(APFS_OMAP_CACHE * sizeof(*nx->omap_cache));/** Block zero holds a container superblock, but it can be behind: the live one is@@ -698,6 +721,10 @@ apfs_nx_close(struct apfs_nx *nx)uint32_t i;scratch_drain(nx);+ if (nx->omap_cache != NULL) {+ apfs_free(nx->omap_cache, APFS_OMAP_CACHE * sizeof(*nx->omap_cache));+ nx->omap_cache = NULL;+ }for (i = 0; i < APFS_CACHE_BLOCKS; i++) {if (nx->cache[i].data != NULL) {@@ -770,9 +797,39 @@ apfs_vol_open(struct apfs_nx *nx, uint32_t index, struct apfs_vol *vol)/* ------------------------------------------------------------------ records */-int-apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,- struct apfs_cursor *cur)+/*+ * Where a seek is going: (obj_id, type), and for a directory record also the word that+ * holds its name's hash and length (with_nlh) -- the records of one directory sort by+ * that word, then by name.+ */+struct seek_target {+ uint64_t obj_id;+ uint8_t type;+ bool with_nlh;+ uint32_t nlh;+};++/* Below 0: the key sorts before the target; 0 or more: at it or after. */+static int+seek_cmp(const uint8_t *k, uint16_t kl, const struct seek_target *t)+{+ uint64_t id = jkey_id(k);+ uint8_t type = jkey_type(k);++ if (id != t->obj_id) {+ return id < t->obj_id ? -1 : 1;+ }+ if (type != t->type) {+ return type < t->type ? -1 : 1;+ }+ if (t->with_nlh && kl >= 12 && le32(k + 8) < t->nlh) {+ return -1;+ }+ return 0;+}++static int+seek_to(struct apfs_btree *tree, const struct seek_target *t, struct apfs_cursor *cur){const uint8_t *blk;struct node n;@@ -797,24 +854,31 @@ apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,cur->done = true;return 0;}- if (n.leaf) {- /* the first record at or after (obj_id, type) */- for (i = 0; i < n.nkeys; i++) {+ /* the first key at or after the target, by halving: keys sort the way seek_cmp does */+ {+ uint32_t lo = 0, hi = n.nkeys;++ while (lo < hi) {+ uint32_t mid = lo + (hi - lo) / 2;const uint8_t *k, *v;uint16_t kl, vl;- error = node_entry(tree, &n, i, &k, &kl, &v, &vl);+ error = node_entry(tree, &n, mid, &k, &kl, &v, &vl);if (error) {return error;}if (kl < 8) {return EIO;}- if (jkey_id(k) > obj_id ||- (jkey_id(k) == obj_id && jkey_type(k) >= type)) {- break;+ if (seek_cmp(k, kl, t) < 0) {+ lo = mid + 1;+ } else {+ hi = mid;}}+ i = lo;+ }+ if (n.leaf) {cur->index[cur->depth] = i;return 0;}@@ -823,24 +887,7 @@ apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,* smallest key in it, so going left on a tie cannot skip a record: the walk* forward will reach it.*/- for (i = 0; i < n.nkeys; i++) {- const uint8_t *k, *v;- uint16_t kl, vl;-- error = node_entry(tree, &n, i, &k, &kl, &v, &vl);- if (error) {- return error;- }- if (kl < 8) {- return EIO;- }- if (jkey_id(k) < obj_id ||- (jkey_id(k) == obj_id && jkey_type(k) < type)) {- pick = i;- } else {- break;- }- }+ pick = (i > 0) ? i - 1 : 0;cur->index[cur->depth] = pick;error = node_child_addr(tree, &n, pick, &addr);if (error) {@@ -854,6 +901,15 @@ apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,}}+int+apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,+ struct apfs_cursor *cur)+{+ struct seek_target t = { .obj_id = obj_id, .type = type };++ return seek_to(tree, &t, cur);+}+/** The record the cursor is on, then step past it. The pointers in `rec` point into a* cached block and stay good only until the next call that touches this container.@@ -1184,6 +1240,104 @@ lookup_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_return EEXIST; /* stop the walk; not an error */}+/*+ * The hash in a directory record's key, for a name of ASCII characters: CRC-32C over the+ * name as little-endian 32-bit code points, started at all ones and not inverted at the+ * end, low 22 bits. For other names APFS hashes the name after Unicode canonical+ * decomposition (and case folding, on a case-insensitive volume), which is not done+ * here. Checked against the hash stored with every ASCII name of the iOS 16.5 root+ * volume: 317749 of 317749 (tools/apfs/test_namehash.py).+ */+static const uint32_t crc32c_nibble[16] = {+ 0x00000000, 0x105ec76f, 0x20bd8ede, 0x30e349b1,+ 0x417b1dbc, 0x5125dad3, 0x61c69362, 0x7198540d,+ 0x82f63b78, 0x92a8fc17, 0xa24bb5a6, 0xb21572c9,+ 0xc38d26c4, 0xd3d3e1ab, 0xe330a81a, 0xf36e6f75,+};++static bool+ascii_name_hash(const char *name, size_t name_len, uint32_t *hash)+{+ uint32_t crc = 0xffffffffu;+ size_t i;++ for (i = 0; i < name_len; i++) {+ uint8_t c = (uint8_t)name[i];+ uint8_t b[4] = { c, 0, 0, 0 }; /* the code point, little-endian */+ int j;++ if (c == 0 || c >= 0x80) {+ return false;+ }+ for (j = 0; j < 4; j++) {+ crc ^= b[j];+ crc = (crc >> 4) ^ crc32c_nibble[crc & 15];+ crc = (crc >> 4) ^ crc32c_nibble[crc & 15];+ }+ }+ *hash = crc & 0x3fffff;+ return true;+}++/*+ * A name in a directory by its hash: straight to the records with that hash and compare+ * only those, instead of reading the directory from its start. Gives ENOENT for a name+ * that is not there, EAGAIN when this way does not apply (not hashed keys, a case-+ * insensitive volume, a name that is not plain ASCII) and the caller goes through the+ * directory instead. Either way the match is the same: the name byte for byte.+ */+static int+lookup_hashed(struct apfs_vol *vol, uint64_t dir_id, const char *name, size_t name_len,+ uint64_t *file_id, uint8_t *d_type)+{+ struct seek_target t = { .obj_id = dir_id, .type = APFS_TYPE_DIR_REC,+ .with_nlh = true };+ struct apfs_cursor cur;+ struct apfs_rec rec;+ uint32_t hash;+ int error;++ if (!vol->hashed_names || (vol->incompat & APFS_INCOMPAT_CASE_INSENSITIVE) != 0 ||+ name_len == 0 || name_len >= APFS_J_DREC_LEN_MASK ||+ !ascii_name_hash(name, name_len, &hash)) {+ return EAGAIN;+ }+ t.nlh = hash << APFS_J_DREC_HASH_SHIFT;+ error = seek_to(&vol->tree, &t, &cur);+ if (error) {+ return error;+ }+ for (;;) {+ uint32_t nlh, len;++ error = apfs_cursor_next(&cur, &rec);+ if (error) {+ return error; /* ENOENT at the end of the tree */+ }+ if (rec.obj_id != dir_id || rec.type != APFS_TYPE_DIR_REC || rec.key_len < 12) {+ return ENOENT;+ }+ nlh = le32(rec.key + 8);+ if ((nlh >> APFS_J_DREC_HASH_SHIFT) != hash) {+ return ENOENT; /* past the records with this hash */+ }+ len = nlh & APFS_J_DREC_LEN_MASK;+ if (len == name_len + 1 && 12 + len <= rec.key_len &&+ memcmp(rec.key + 12, name, name_len) == 0) {+ if (rec.val == NULL || rec.val_len < 18) {+ return EIO;+ }+ if (file_id != NULL) {+ *file_id = le64(rec.val);+ }+ if (d_type != NULL) {+ *d_type = (uint8_t)(le16(rec.val + 16) & APFS_DREC_TYPE_MASK);+ }+ return 0;+ }+ }+}+intapfs_lookup(struct apfs_vol *vol, uint64_t dir_id, const char *name, size_t name_len,uint64_t *file_id, uint8_t *d_type)@@ -1191,6 +1345,10 @@ apfs_lookup(struct apfs_vol *vol, uint64_t dir_id, const char *name, size_t namestruct lookup_ctx lc;int error;+ error = lookup_hashed(vol, dir_id, name, name_len, file_id, d_type);+ if (error != EAGAIN) {+ return error;+ }memset(&lc, 0, sizeof(lc));lc.name = name;lc.name_len = name_len;diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.hindex dce5f70..49783ee 100644--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h@@ -122,6 +122,21 @@ struct apfs_omap {bool ready;};+/*+ * What the object maps have answered: a B-tree node of the file-system tree is named by+ * a virtual id, and every step down that tree used to be a walk down an object map too.+ * The mount is read-only at one checkpoint, so an answer never goes stale. One cache+ * for the container's map and its volumes', direct-mapped, made at apfs_nx_open.+ */+#define APFS_OMAP_CACHE 2048 /* a power of two */++struct apfs_omap_hit {+ uint64_t map; /* which object map: the root of its tree */+ uint64_t oid; /* 0: empty */+ uint64_t xid;+ uint64_t addr;+};+struct apfs_scratch; /* apfs_lib.c's own *//* A mounted container. */@@ -193,6 +208,8 @@ struct apfs_nx {uint32_t requests;uint64_t batch_calls, batch_pieces; /* how many requests used it, for how much */uint64_t ra_batches, ra_pieces, ra_hits; /* read-ahead: filled, and used */+ struct apfs_omap_hit *omap_cache; /* APFS_OMAP_CACHE of them, or NULL */+ uint64_t omap_hits, omap_misses;};/* A volume inside the container. */