Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: a name found by its hash, halving within B-tree nodes, object-map answers kept

vcvkk171b06af7fe9
diff --git a/tools/apfs/apfs-tool.c b/tools/apfs/apfs-tool.c
index 3761936..e6b4bb9 100644
--- a/tools/apfs/apfs-tool.c
+++ b/tools/apfs/apfs-tool.c
@@ -14,6 +14,7 @@
* apfs-tool stat IMAGE PATH
* apfs-tool cat IMAGE PATH
* apfs-tool census IMAGE [PATH] [--meta] (--meta: do not read contents)
+ * apfs-tool lookups IMAGE (every name looked up by name; see cmd_lookups)
*/
/* pread(2) and clock_gettime(2) come with -D_DEFAULT_SOURCE, set by build-tool.sh */
@@ -599,6 +600,162 @@ cmd_census(const char *path, const char *where, bool meta_only)
return cs.error ? 1 : 0;
}
+/*
+ * Every name in the volume looked up by name, the way the kernel's vnop_lookup does it,
+ * against what reading its directory gave; and each name with "#" after it, which must
+ * not be found. Also how long that took against finding each name by reading its
+ * directory from the start, which is what apfs_lookup did before it went by the hash.
+ */
+struct lk_entry {
+ char name[256];
+ size_t len;
+ uint64_t id;
+ uint8_t type;
+};
+
+struct lk_dir {
+ struct lk_entry *e;
+ size_t n, cap;
+};
+
+static int
+lk_collect(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)
+{
+ struct lk_dir *d = ctx;
+
+ if (d->n == d->cap) {
+ size_t cap = d->cap ? d->cap * 2 : 64;
+ struct lk_entry *e = realloc(d->e, cap * sizeof(*e));
+
+ if (e == NULL) {
+ return ENOMEM;
+ }
+ d->e = e;
+ d->cap = cap;
+ }
+ if (name_len >= sizeof(d->e[0].name)) {
+ return EIO;
+ }
+ memcpy(d->e[d->n].name, name, name_len);
+ d->e[d->n].name[name_len] = '\0';
+ d->e[d->n].len = name_len;
+ d->e[d->n].id = file_id;
+ d->e[d->n].type = d_type;
+ d->n++;
+ return 0;
+}
+
+struct lk_scan {
+ const char *name;
+ size_t len;
+ bool found;
+};
+
+static int
+lk_scan_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_t d_type)
+{
+ struct lk_scan *sc = ctx;
+
+ (void)file_id;
+ (void)d_type;
+ if (name_len == sc->len && memcmp(name, sc->name, name_len) == 0) {
+ sc->found = true;
+ return EEXIST;
+ }
+ return 0;
+}
+
+static double
+lk_now(void)
+{
+ struct timespec ts;
+
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec + (double)ts.tv_nsec / 1e9;
+}
+
+static int
+cmd_lookups(const char *path)
+{
+ struct apfs_nx nx;
+ struct apfs_vol vol;
+ uint64_t *stack = NULL;
+ size_t depth = 0, cap = 0;
+ uint64_t dirs = 0, names = 0, good = 0, bad = 0, absent_good = 0, absent_bad = 0;
+ double t_hash = 0, t_scan = 0;
+
+ if (open_image(path, &nx, &vol) != 0) {
+ return 1;
+ }
+ stack = malloc(sizeof(*stack) * 64);
+ cap = 64;
+ stack[depth++] = APFS_ROOT_DIR_INO_NUM;
+ while (depth > 0) {
+ uint64_t dir = stack[--depth];
+ struct lk_dir d = { 0 };
+ size_t i;
+ int error = apfs_readdir(&vol, dir, lk_collect, &d);
+
+ if (error) {
+ fprintf(stderr, "apfs-tool: readdir %" PRIu64 ": %s\n", dir, strerror(error));
+ bad++;
+ continue;
+ }
+ dirs++;
+ for (i = 0; i < d.n; i++) {
+ struct lk_entry *e = &d.e[i];
+ struct lk_scan sc = { e->name, e->len, false };
+ char absent[260];
+ uint64_t id = 0;
+ uint8_t type = 0;
+ double t0;
+
+ names++;
+ t0 = lk_now();
+ error = apfs_lookup(&vol, dir, e->name, e->len, &id, &type);
+ t_hash += lk_now() - t0;
+ if (error == 0 && id == e->id && type == e->type) {
+ good++;
+ } else {
+ bad++;
+ if (bad <= 10) {
+ fprintf(stderr, "apfs-tool: lookup %s in %" PRIu64 ": %d, %" PRIu64
+ " (readdir: %" PRIu64 ")\n", e->name, dir, error, id, e->id);
+ }
+ }
+ t0 = lk_now();
+ error = apfs_readdir(&vol, dir, lk_scan_cb, &sc);
+ t_scan += lk_now() - t0;
+ if (!sc.found) {
+ bad++;
+ }
+ snprintf(absent, sizeof(absent), "%s#", e->name);
+ error = apfs_lookup(&vol, dir, absent, e->len + 1, NULL, NULL);
+ if (error == ENOENT) {
+ absent_good++;
+ } else {
+ absent_bad++;
+ }
+ if (e->type == APFS_DT_DIR) {
+ if (depth == cap) {
+ cap *= 2;
+ stack = realloc(stack, sizeof(*stack) * cap);
+ }
+ stack[depth++] = e->id;
+ }
+ }
+ free(d.e);
+ }
+ printf("directories %" PRIu64 ", names %" PRIu64 ": found as readdir has them %" PRIu64
+ ", not %" PRIu64 "; with \"#\" after them not found %" PRIu64 ", found %" PRIu64 "\n",
+ dirs, names, good, bad, absent_good, absent_bad);
+ printf("time for all of them: apfs_lookup %.0f ms, reading the directory from its start "
+ "%.0f ms\n", t_hash * 1000, t_scan * 1000);
+ free(stack);
+ apfs_nx_close(&nx);
+ return (bad == 0 && absent_bad == 0) ? 0 : 1;
+}
+
int
main(int argc, char **argv)
{
@@ -608,7 +765,8 @@ main(int argc, char **argv)
" apfs-tool ls IMAGE [PATH]\n"
" apfs-tool stat IMAGE PATH\n"
" apfs-tool cat IMAGE PATH\n"
- " apfs-tool census IMAGE [PATH] [--meta]\n");
+ " apfs-tool census IMAGE [PATH] [--meta]\n"
+ " apfs-tool lookups IMAGE\n");
return 2;
}
if (strcmp(argv[1], "info") == 0) {
@@ -627,6 +785,9 @@ main(int argc, char **argv)
return cmd_census(argv[2], (argc > 3 && argv[3][0] == '/') ? argv[3] : NULL,
(argc > 3 && strcmp(argv[argc - 1], "--meta") == 0));
}
+ if (strcmp(argv[1], "lookups") == 0) {
+ return cmd_lookups(argv[2]);
+ }
fprintf(stderr, "apfs-tool: unknown command %s\n", argv[1]);
return 2;
}
diff --git a/tools/apfs/run-tests.sh b/tools/apfs/run-tests.sh
index 035aecc..7f26c3d 100755
--- a/tools/apfs/run-tests.sh
+++ b/tools/apfs/run-tests.sh
@@ -6,8 +6,10 @@
# 3. the APFS reader against 7-Zip's, over every entry in the volume (needs ios165/)
# 4. file contents against the files 7-Zip extracted, byte for byte (needs ios165/)
# 5. the C the kernel is built from against apfs.py, over the whole volume
+# 6. the hash in directory records, worked out as the C lookup does (needs the image)
+# 7. every name looked up by name through the C, against its directory (needs the image)
#
-# Steps 3, 4 and 5 need the APFS partition unpacked out of ios165/rootfs.dmg; pass it as
+# Steps 3 to 7 need the APFS partition unpacked out of ios165/rootfs.dmg; pass it as
# $1, or set APFS_IMAGE. Without it only the first two steps run.
#
# APFS_SUBTREE=/some/path keeps step 5 to one subtree, which is the quick way to run it;
@@ -44,6 +46,17 @@ echo "== 1b. hole handling, on made-up extent lists (the image has no sparse fil
python3 "$HERE/test_sparse.py" || fail=1
echo
+echo "== 1c. the C that reads file contents without the lock, on made-up runs (holes, ends inside blocks)"
+if cc -std=c11 -O2 -w -I"$HERE/host-shim" -c -o "$TMP/deflate.o" "$HDR_DIR/apfs_deflate.c" &&
+ cc -std=c11 -D_DEFAULT_SOURCE -O1 -Wall -Wextra -Wno-unused-parameter -Wno-unused-function \
+ -I"$HDR_DIR" -o "$TMP/test_fetch" "$HERE/test_fetch.c" "$TMP/deflate.o" -lz -pthread; then
+ "$TMP/test_fetch" || fail=1
+else
+ echo " would not build"
+ fail=1
+fi
+echo
+
echo "== 2. the UDIF reader on ios165/rootfs.dmg"
if [ -f "$ROOT/ios165/rootfs.dmg" ]; then
python3 "$HERE/dmg.py" info "$ROOT/ios165/rootfs.dmg" | tail -n 14 || fail=1
@@ -96,5 +109,18 @@ else
echo " skipped: the tool would not build"
fail=1
fi
+echo
+
+echo "== 6. the hash in directory records, as the lookup works it out"
+python3 "$HERE/test_namehash.py" "$IMAGE" || fail=1
+echo
+
+echo "== 7. every name in the volume looked up by name (and with \"#\" after it, not found)"
+if [ -x "$TMP/apfs-tool" ]; then
+ "$TMP/apfs-tool" lookups "$IMAGE" || fail=1
+else
+ echo " skipped: the tool would not build"
+ fail=1
+fi
exit $fail
diff --git a/tools/apfs/test_fetch.c b/tools/apfs/test_fetch.c
new file mode 100644
index 0000000..c18bb29
--- /dev/null
+++ b/tools/apfs/test_fetch.c
@@ -0,0 +1,92 @@
+/*
+ * stream_fetch() -- the part of reading a file's contents that runs without the volume's
+ * lock -- against made-up runs, since the iOS 16.5 root volume has no sparse file and so
+ * never gives it a hole: holes at the front, between runs and at the end, runs that start
+ * and end inside a block, a run inside one block, no runs at all. apfs_lib.c is included
+ * whole, so that its static functions can be called. The device is 64 blocks of memory,
+ * block n holding bytes that depend on n and on the place in the block, so a byte from
+ * the wrong block or the wrong place shows. Built and run by run-tests.sh.
+ */
+#include "apfs_lib.c"
+
+#define BS 4096
+#define NBLK 64
+
+static uint8_t disk[NBLK * BS];
+
+static int
+dev_read(void *ctx, uint64_t off, void *buf, size_t len)
+{
+ (void)ctx;
+ if (off % BS != 0 || len % BS != 0 || off + len > sizeof(disk)) {
+ return EINVAL; /* whole blocks only, as apfs_dev.read promises */
+ }
+ memcpy(buf, disk + off, len);
+ return 0;
+}
+
+struct fcase {
+ const char *what;
+ uint32_t nruns;
+ struct apfs_run runs[APFS_MAX_RUNS];
+ uint64_t covered;
+};
+
+static const struct fcase cases[] = {
+ { "whole blocks", 1, { { 0, 3, 0, 3 * BS } }, 3 * BS },
+ { "starts and ends inside blocks", 1, { { 0, 5, 100, 2 * BS + 50 } }, 2 * BS + 50 },
+ { "inside one block", 1, { { 0, 7, 10, 20 } }, 20 },
+ { "ends where a block does", 1, { { 0, 10, 4000, 96 + BS } }, 96 + BS },
+ { "a hole at the front", 1, { { 1000, 2, 0, 5000 } }, 6000 },
+ { "a hole between two runs", 2, { { 0, 1, 0, BS }, { 2 * BS, 9, 0, BS } }, 3 * BS },
+ { "a hole at the end", 1, { { 0, 4, 0, 100 } }, 2 * BS },
+ { "no runs: all hole", 0, { { 0 } }, 5000 },
+ { "eight runs, every one unaligned", 8, {
+ { 0, 20, 1, 10 }, { 10, 21, 4095, 2 }, { 100, 22, 0, BS + 1 },
+ { 2 * BS, 30, 2048, 2048 }, { 3 * BS, 31, 0, 1 }, { 3 * BS + 5, 40, 7, 3 * BS },
+ { 7 * BS, 50, 4000, 200 }, { 8 * BS, 60, 0, 4 * BS } }, 12 * BS },
+};
+
+int
+main(void)
+{
+ struct apfs_nx nx;
+ static uint8_t dst[16 * BS], want[16 * BS], bounce[BS];
+ unsigned c, failed = 0;
+ uint64_t i;
+
+ for (i = 0; i < sizeof(disk); i++) {
+ disk[i] = (uint8_t)((i / BS) * 37 + (i % BS) * 7 + 1);
+ }
+ memset(&nx, 0, sizeof(nx));
+ nx.block_size = BS;
+ nx.block_count = NBLK;
+ nx.dev.read = dev_read;
+
+ for (c = 0; c < sizeof(cases) / sizeof(cases[0]); c++) {
+ const struct fcase *fc = &cases[c];
+ uint64_t calls = 0, bytes = 0;
+ uint32_t r;
+ int error;
+
+ memset(want, 0, sizeof(want));
+ for (r = 0; r < fc->nruns; r++) {
+ const struct apfs_run *run = &fc->runs[r];
+
+ memcpy(want + run->at, disk + run->addr * BS + run->skew, run->len);
+ }
+ memset(dst, 0xAA, sizeof(dst)); /* what is not written shows */
+ error = stream_fetch(&nx, fc->runs, fc->nruns, fc->covered, dst, bounce, &calls,
+ &bytes);
+ if (error != 0 || memcmp(dst, want, fc->covered) != 0 || dst[fc->covered] != 0xAA) {
+ printf(" FAIL %s (error %d)\n", fc->what, error);
+ failed++;
+ } else {
+ printf(" ok %s (%llu requests, %llu bytes)\n", fc->what,
+ (unsigned long long)calls, (unsigned long long)bytes);
+ }
+ }
+ printf("stream_fetch: %u cases, %u failed\n", (unsigned)(sizeof(cases) / sizeof(cases[0])),
+ failed);
+ return failed ? 1 : 0;
+}
diff --git a/tools/apfs/test_namehash.py b/tools/apfs/test_namehash.py
new file mode 100644
index 0000000..ef5ce1b
--- /dev/null
+++ b/tools/apfs/test_namehash.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+# The hash in a directory record's key, worked out the way apfs_lib.c's lookup works it
+# out, against the hash APFS stored with every name of a volume that is plain ASCII:
+# CRC-32C over the name as little-endian 32-bit code points, started at all ones and not
+# inverted at the end, low 22 bits. (Other names are hashed after Unicode canonical
+# decomposition, which the lookup does not do: it reads the directory through for them.)
+# Also that the records of a directory sort by the word holding hash and length, then by
+# name -- what lets the lookup stop at the first record past the hash.
+#
+# usage: test_namehash.py IMAGE
+
+import os
+import struct
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import apfs
+
+POLY = 0x82F63B78
+TABLE = []
+for _i in range(256):
+ _c = _i
+ for _ in range(8):
+ _c = (_c >> 1) ^ POLY if _c & 1 else _c >> 1
+ TABLE.append(_c)
+
+
+def name_hash(name):
+ crc = 0xFFFFFFFF
+ for ch in name:
+ for b in struct.pack("<I", ord(ch)):
+ crc = TABLE[(crc ^ b) & 0xFF] ^ (crc >> 8)
+ return crc & 0x3FFFFF
+
+
+def main():
+ if len(sys.argv) != 2:
+ print("usage: test_namehash.py IMAGE", file=sys.stderr)
+ return 2
+ c = apfs.Container(sys.argv[1])
+ vol = next(iter(c.volumes()))
+ if vol.case_insensitive:
+ print("a case-insensitive volume: names are folded before hashing; not checked here")
+ return 2
+ total = ascii_n = good = 0
+ bad = []
+ disorder = 0
+ prev = None
+ for k, _v in vol.tree.iter_all():
+ x = struct.unpack_from("<Q", k, 0)[0]
+ oid, typ = x & apfs.OBJ_ID_MASK, x >> apfs.OBJ_TYPE_SHIFT
+ if typ != apfs.TYPE_DIR_REC:
+ prev = None
+ continue
+ nlh = struct.unpack_from("<I", k, 8)[0]
+ raw = k[12:12 + (nlh & apfs.J_DREC_LEN_MASK)].split(b"\0", 1)[0]
+ total += 1
+ if prev is not None and prev[0] == oid and prev[1:] > (nlh, raw):
+ disorder += 1
+ prev = (oid, nlh, raw)
+ try:
+ name = raw.decode("ascii")
+ except UnicodeDecodeError:
+ continue
+ ascii_n += 1
+ if name_hash(name) == nlh >> apfs.J_DREC_HASH_SHIFT and \
+ (nlh & apfs.J_DREC_LEN_MASK) == len(raw) + 1:
+ good += 1
+ elif len(bad) < 10:
+ bad.append((name, nlh))
+ print("directory records %d, plain ASCII %d, hash and length as worked out %d, not %d;"
+ " out of order %d" % (total, ascii_n, good, ascii_n - good, disorder))
+ for name, nlh in bad:
+ print(" %r: stored %06x, worked out %06x" % (name, nlh >> 10, name_hash(name)))
+ return 0 if good == ascii_n and disorder == 0 else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.c
index 7a18629..cc726fd 100644
--- a/xnu/files/bsd/dev/redwood_apfs_check.c
+++ b/xnu/files/bsd/dev/redwood_apfs_check.c
@@ -179,8 +179,25 @@ struct census {
*/
bool no_hash;
uint64_t hash_ticks;
+ /*
+ * Boot-arg apfs_contents=0: no file is read at all, only looked up and stat'ed --
+ * how long the walk itself takes, the directories and the inodes.
+ */
+ bool no_contents;
};
+/* The boot-args both kinds of walk take the same way. */
+static void
+census_options(struct census *cs)
+{
+ uint32_t hash = 1, contents = 1;
+
+ PE_parse_boot_argn("apfs_hash", &hash, sizeof(hash));
+ PE_parse_boot_argn("apfs_contents", &contents, sizeof(contents));
+ cs->no_hash = (hash == 0);
+ cs->no_contents = (contents == 0);
+}
+
/* The format bits of a mode, which vnode_getattr does not report in va_mode. */
static uint16_t
ifmt_of(enum vtype t)
@@ -301,11 +318,15 @@ census_entry(struct census *cs, uint8_t d_type)
if (vnode_vtype(vp) == VREG) {
cs->files++;
PHASE("read");
- error = hash_file(cs, vp, size, &h);
+ if (!cs->no_contents) {
+ error = hash_file(cs, vp, size, &h);
+ }
} else if (vnode_vtype(vp) == VLNK) {
cs->links++;
PHASE("readlink");
- error = hash_link(cs, vp, &h);
+ if (!cs->no_contents) {
+ error = hash_link(cs, vp, &h);
+ }
} else if (vnode_vtype(vp) == VDIR) {
cs->dirs++;
} else {
@@ -610,12 +631,10 @@ apfs_walk_one(void *arg, __unused wait_result_t wr)
struct walk_job *job = arg;
struct census cs;
uint64_t start, end, ns = 0, hash_ns = 0;
- uint32_t hash = 1;
bzero(&cs, sizeof(cs));
cs.ctx = vfs_context_kernel();
- PE_parse_boot_argn("apfs_hash", &hash, sizeof(hash));
- cs.no_hash = (hash == 0);
+ census_options(&cs);
PE_parse_boot_argn("apfs_depth", &cs.max_depth, sizeof(cs.max_depth));
PE_parse_boot_argn("apfs_max", &cs.max_entries, sizeof(cs.max_entries));
{
@@ -725,6 +744,7 @@ apfs_check_thread(void *arg, __unused wait_result_t wr)
bzero(&cs, sizeof(cs));
cs.ctx = ctx;
+ census_options(&cs);
cs.print_lines = (mode == 2);
/*
* NOCROSSMOUNT only where the volume is the root (modes 4 and 5). In mode 6 the
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
index 245566c..1790b03 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
@@ -474,9 +474,23 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *a
const uint8_t *blk;
int level, error;
+ struct apfs_omap_hit *hit = NULL;
+
if (!omap->ready) {
return EIO;
}
+ if (tree->nx->omap_cache != NULL) {
+ uint64_t h = (oid ^ (tree->root * 0xff51afd7ed558ccdULL)) * 0x9E3779B97F4A7C15ULL;
+
+ hit = &tree->nx->omap_cache[h >> 53 & (APFS_OMAP_CACHE - 1)];
+ if (hit->oid == oid && oid != 0 && hit->xid == want_xid &&
+ hit->map == tree->root) {
+ tree->nx->omap_hits++;
+ *addr = hit->addr;
+ return 0;
+ }
+ tree->nx->omap_misses++;
+ }
error = nx_block(tree->nx, node_oid, &blk, true);
if (error) {
return error;
@@ -486,19 +500,20 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *a
return error;
}
for (level = 0; level < APFS_MAX_LEVELS; level++) {
- uint32_t i, best = 0;
- bool found = false;
+ uint32_t lo = 0, hi, best;
if (n.nkeys == 0) {
return ENOENT;
}
- /* the last entry whose key is at or before what we are looking for */
- for (i = 0; i < n.nkeys; i++) {
+ /* the last entry whose key is at or before what we are looking for: halving */
+ hi = n.nkeys;
+ while (lo < hi) {
+ uint32_t mid = lo + (hi - lo) / 2;
const uint8_t *k, *v;
uint16_t kl, vl;
uint64_t o, x;
- error = node_entry(tree, &n, i, &k, &kl, &v, &vl);
+ error = node_entry(tree, &n, mid, &k, &kl, &v, &vl);
if (error) {
return error;
}
@@ -508,15 +523,15 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *a
o = le64(k);
x = le64(k + 8);
if (o < oid || (o == oid && x <= want_xid)) {
- best = i;
- found = true;
+ lo = mid + 1;
} else {
- break;
+ hi = mid;
}
}
- if (!found) {
+ if (lo == 0) {
return ENOENT;
}
+ best = lo - 1;
if (n.leaf) {
const uint8_t *k, *v;
uint16_t kl, vl;
@@ -536,6 +551,12 @@ omap_find(struct apfs_omap *omap, uint64_t oid, apfs_xid_t want_xid, uint64_t *a
if (*addr == 0 || *addr >= tree->nx->block_count) {
return EIO;
}
+ if (hit != NULL) {
+ hit->map = tree->root;
+ hit->oid = oid;
+ hit->xid = want_xid;
+ hit->addr = *addr;
+ }
return 0;
}
error = node_child(tree, &n, best, &node_oid);
@@ -585,6 +606,8 @@ apfs_nx_open(struct apfs_nx *nx, const struct apfs_dev *dev)
for (i = 0; i < APFS_CACHE_BLOCKS; i++) {
nx->cache[i].addr = APFS_BUF_EMPTY;
}
+ /* zeroed, so every slot is empty; without it the object maps are simply walked */
+ nx->omap_cache = apfs_alloc(APFS_OMAP_CACHE * sizeof(*nx->omap_cache));
/*
* Block zero holds a container superblock, but it can be behind: the live one is
@@ -698,6 +721,10 @@ apfs_nx_close(struct apfs_nx *nx)
uint32_t i;
scratch_drain(nx);
+ if (nx->omap_cache != NULL) {
+ apfs_free(nx->omap_cache, APFS_OMAP_CACHE * sizeof(*nx->omap_cache));
+ nx->omap_cache = NULL;
+ }
for (i = 0; i < APFS_CACHE_BLOCKS; i++) {
if (nx->cache[i].data != NULL) {
@@ -770,9 +797,39 @@ apfs_vol_open(struct apfs_nx *nx, uint32_t index, struct apfs_vol *vol)
/* ------------------------------------------------------------------ records */
-int
-apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,
- struct apfs_cursor *cur)
+/*
+ * Where a seek is going: (obj_id, type), and for a directory record also the word that
+ * holds its name's hash and length (with_nlh) -- the records of one directory sort by
+ * that word, then by name.
+ */
+struct seek_target {
+ uint64_t obj_id;
+ uint8_t type;
+ bool with_nlh;
+ uint32_t nlh;
+};
+
+/* Below 0: the key sorts before the target; 0 or more: at it or after. */
+static int
+seek_cmp(const uint8_t *k, uint16_t kl, const struct seek_target *t)
+{
+ uint64_t id = jkey_id(k);
+ uint8_t type = jkey_type(k);
+
+ if (id != t->obj_id) {
+ return id < t->obj_id ? -1 : 1;
+ }
+ if (type != t->type) {
+ return type < t->type ? -1 : 1;
+ }
+ if (t->with_nlh && kl >= 12 && le32(k + 8) < t->nlh) {
+ return -1;
+ }
+ return 0;
+}
+
+static int
+seek_to(struct apfs_btree *tree, const struct seek_target *t, struct apfs_cursor *cur)
{
const uint8_t *blk;
struct node n;
@@ -797,24 +854,31 @@ apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,
cur->done = true;
return 0;
}
- if (n.leaf) {
- /* the first record at or after (obj_id, type) */
- for (i = 0; i < n.nkeys; i++) {
+ /* the first key at or after the target, by halving: keys sort the way seek_cmp does */
+ {
+ uint32_t lo = 0, hi = n.nkeys;
+
+ while (lo < hi) {
+ uint32_t mid = lo + (hi - lo) / 2;
const uint8_t *k, *v;
uint16_t kl, vl;
- error = node_entry(tree, &n, i, &k, &kl, &v, &vl);
+ error = node_entry(tree, &n, mid, &k, &kl, &v, &vl);
if (error) {
return error;
}
if (kl < 8) {
return EIO;
}
- if (jkey_id(k) > obj_id ||
- (jkey_id(k) == obj_id && jkey_type(k) >= type)) {
- break;
+ if (seek_cmp(k, kl, t) < 0) {
+ lo = mid + 1;
+ } else {
+ hi = mid;
}
}
+ i = lo;
+ }
+ if (n.leaf) {
cur->index[cur->depth] = i;
return 0;
}
@@ -823,24 +887,7 @@ apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,
* smallest key in it, so going left on a tie cannot skip a record: the walk
* forward will reach it.
*/
- for (i = 0; i < n.nkeys; i++) {
- const uint8_t *k, *v;
- uint16_t kl, vl;
-
- error = node_entry(tree, &n, i, &k, &kl, &v, &vl);
- if (error) {
- return error;
- }
- if (kl < 8) {
- return EIO;
- }
- if (jkey_id(k) < obj_id ||
- (jkey_id(k) == obj_id && jkey_type(k) < type)) {
- pick = i;
- } else {
- break;
- }
- }
+ pick = (i > 0) ? i - 1 : 0;
cur->index[cur->depth] = pick;
error = node_child_addr(tree, &n, pick, &addr);
if (error) {
@@ -854,6 +901,15 @@ apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,
}
}
+int
+apfs_seek(struct apfs_btree *tree, uint64_t obj_id, uint8_t type,
+ struct apfs_cursor *cur)
+{
+ struct seek_target t = { .obj_id = obj_id, .type = type };
+
+ return seek_to(tree, &t, cur);
+}
+
/*
* The record the cursor is on, then step past it. The pointers in `rec` point into a
* cached block and stay good only until the next call that touches this container.
@@ -1184,6 +1240,104 @@ lookup_cb(void *ctx, const char *name, size_t name_len, uint64_t file_id, uint8_
return EEXIST; /* stop the walk; not an error */
}
+/*
+ * The hash in a directory record's key, for a name of ASCII characters: CRC-32C over the
+ * name as little-endian 32-bit code points, started at all ones and not inverted at the
+ * end, low 22 bits. For other names APFS hashes the name after Unicode canonical
+ * decomposition (and case folding, on a case-insensitive volume), which is not done
+ * here. Checked against the hash stored with every ASCII name of the iOS 16.5 root
+ * volume: 317749 of 317749 (tools/apfs/test_namehash.py).
+ */
+static const uint32_t crc32c_nibble[16] = {
+ 0x00000000, 0x105ec76f, 0x20bd8ede, 0x30e349b1,
+ 0x417b1dbc, 0x5125dad3, 0x61c69362, 0x7198540d,
+ 0x82f63b78, 0x92a8fc17, 0xa24bb5a6, 0xb21572c9,
+ 0xc38d26c4, 0xd3d3e1ab, 0xe330a81a, 0xf36e6f75,
+};
+
+static bool
+ascii_name_hash(const char *name, size_t name_len, uint32_t *hash)
+{
+ uint32_t crc = 0xffffffffu;
+ size_t i;
+
+ for (i = 0; i < name_len; i++) {
+ uint8_t c = (uint8_t)name[i];
+ uint8_t b[4] = { c, 0, 0, 0 }; /* the code point, little-endian */
+ int j;
+
+ if (c == 0 || c >= 0x80) {
+ return false;
+ }
+ for (j = 0; j < 4; j++) {
+ crc ^= b[j];
+ crc = (crc >> 4) ^ crc32c_nibble[crc & 15];
+ crc = (crc >> 4) ^ crc32c_nibble[crc & 15];
+ }
+ }
+ *hash = crc & 0x3fffff;
+ return true;
+}
+
+/*
+ * A name in a directory by its hash: straight to the records with that hash and compare
+ * only those, instead of reading the directory from its start. Gives ENOENT for a name
+ * that is not there, EAGAIN when this way does not apply (not hashed keys, a case-
+ * insensitive volume, a name that is not plain ASCII) and the caller goes through the
+ * directory instead. Either way the match is the same: the name byte for byte.
+ */
+static int
+lookup_hashed(struct apfs_vol *vol, uint64_t dir_id, const char *name, size_t name_len,
+ uint64_t *file_id, uint8_t *d_type)
+{
+ struct seek_target t = { .obj_id = dir_id, .type = APFS_TYPE_DIR_REC,
+ .with_nlh = true };
+ struct apfs_cursor cur;
+ struct apfs_rec rec;
+ uint32_t hash;
+ int error;
+
+ if (!vol->hashed_names || (vol->incompat & APFS_INCOMPAT_CASE_INSENSITIVE) != 0 ||
+ name_len == 0 || name_len >= APFS_J_DREC_LEN_MASK ||
+ !ascii_name_hash(name, name_len, &hash)) {
+ return EAGAIN;
+ }
+ t.nlh = hash << APFS_J_DREC_HASH_SHIFT;
+ error = seek_to(&vol->tree, &t, &cur);
+ if (error) {
+ return error;
+ }
+ for (;;) {
+ uint32_t nlh, len;
+
+ error = apfs_cursor_next(&cur, &rec);
+ if (error) {
+ return error; /* ENOENT at the end of the tree */
+ }
+ if (rec.obj_id != dir_id || rec.type != APFS_TYPE_DIR_REC || rec.key_len < 12) {
+ return ENOENT;
+ }
+ nlh = le32(rec.key + 8);
+ if ((nlh >> APFS_J_DREC_HASH_SHIFT) != hash) {
+ return ENOENT; /* past the records with this hash */
+ }
+ len = nlh & APFS_J_DREC_LEN_MASK;
+ if (len == name_len + 1 && 12 + len <= rec.key_len &&
+ memcmp(rec.key + 12, name, name_len) == 0) {
+ if (rec.val == NULL || rec.val_len < 18) {
+ return EIO;
+ }
+ if (file_id != NULL) {
+ *file_id = le64(rec.val);
+ }
+ if (d_type != NULL) {
+ *d_type = (uint8_t)(le16(rec.val + 16) & APFS_DREC_TYPE_MASK);
+ }
+ return 0;
+ }
+ }
+}
+
int
apfs_lookup(struct apfs_vol *vol, uint64_t dir_id, const char *name, size_t name_len,
uint64_t *file_id, uint8_t *d_type)
@@ -1191,6 +1345,10 @@ apfs_lookup(struct apfs_vol *vol, uint64_t dir_id, const char *name, size_t name
struct lookup_ctx lc;
int error;
+ error = lookup_hashed(vol, dir_id, name, name_len, file_id, d_type);
+ if (error != EAGAIN) {
+ return error;
+ }
memset(&lc, 0, sizeof(lc));
lc.name = name;
lc.name_len = name_len;
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.h b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
index dce5f70..49783ee 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.h
@@ -122,6 +122,21 @@ struct apfs_omap {
bool ready;
};
+/*
+ * What the object maps have answered: a B-tree node of the file-system tree is named by
+ * a virtual id, and every step down that tree used to be a walk down an object map too.
+ * The mount is read-only at one checkpoint, so an answer never goes stale. One cache
+ * for the container's map and its volumes', direct-mapped, made at apfs_nx_open.
+ */
+#define APFS_OMAP_CACHE 2048 /* a power of two */
+
+struct apfs_omap_hit {
+ uint64_t map; /* which object map: the root of its tree */
+ uint64_t oid; /* 0: empty */
+ uint64_t xid;
+ uint64_t addr;
+};
+
struct apfs_scratch; /* apfs_lib.c's own */
/* A mounted container. */
@@ -193,6 +208,8 @@ struct apfs_nx {
uint32_t requests;
uint64_t batch_calls, batch_pieces; /* how many requests used it, for how much */
uint64_t ra_batches, ra_pieces, ra_hits; /* read-ahead: filled, and used */
+ struct apfs_omap_hit *omap_cache; /* APFS_OMAP_CACHE of them, or NULL */
+ uint64_t omap_hits, omap_misses;
};
/* A volume inside the container. */