Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS: cut a volume too big to hold into parts that cover it exactly once

vcvkk406d3593474d
diff --git a/tools/apfs/mksparse.py b/tools/apfs/mksparse.py
index d465a32..a108515 100755
--- a/tools/apfs/mksparse.py
+++ b/tools/apfs/mksparse.py
@@ -21,8 +21,11 @@
# into the data area plus one
# data: the blocks that are present, in the order the table numbers them
#
-# usage: mksparse.py trace IMAGE [PATH ...] just say what it would take
-# mksparse.py build IMAGE OUT [PATH ...] write the sparse image
+# usage: mksparse.py trace IMAGE [PATH ...] [--depth N] just say what it would take
+# mksparse.py build IMAGE OUT [PATH ...] [--depth N] [--skip /a,/b]
+# write the sparse image
+# --skip leaves out the contents of those directories, the way boot-arg apfs_skip does in
+# the kernel, so that several images between them cover a volume exactly once.
# With no PATH the whole volume is walked.
import os
@@ -53,7 +56,7 @@ class Tracing(apfs.Blocks):
return super().raw(addr, count)
-def walk_and_trace(image, paths):
+def walk_and_trace(image, paths, max_depth=0, skip=()):
"""
Walk and record. The recording reader has to be in place before the container is
opened, not swapped in after: opening it reads block zero, the checkpoint area, the
@@ -83,9 +86,9 @@ def walk_and_trace(image, paths):
n = 0
for oid, prefix in starts:
- stack = [(oid, prefix)]
+ stack = [(oid, prefix, 1)]
while stack:
- d, px = stack.pop()
+ d, px, depth = stack.pop()
for name, fid, dt in v.dir_entries(d):
p = px + "/" + name
ino = v.inode(fid)
@@ -94,8 +97,8 @@ def walk_and_trace(image, paths):
continue
if ino.is_reg or ino.is_link:
ino.contents() # what makes the data blocks be read
- if dt == 4:
- stack.append((fid, p))
+ if dt == 4 and (max_depth == 0 or depth < max_depth) and p not in skip:
+ stack.append((fid, p, depth + 1))
if n % 20000 == 0:
sys.stderr.write("\r %d entries, %d blocks so far..."
% (n, len(tr.touched)))
@@ -108,6 +111,18 @@ def main(argv):
if len(argv) < 3:
print(__doc__ or "", file=sys.stderr)
return 2
+ args = list(argv[1:])
+ depth = 0
+ skip = ()
+ if "--depth" in args:
+ i = args.index("--depth")
+ depth = int(args[i + 1])
+ del args[i:i + 2]
+ if "--skip" in args:
+ i = args.index("--skip")
+ skip = tuple(args[i + 1].split(","))
+ del args[i:i + 2]
+ argv = [argv[0]] + args
cmd = argv[1]
if cmd == "trace":
image, out, paths = argv[2], None, argv[3:]
@@ -120,7 +135,7 @@ def main(argv):
print("mksparse.py: unknown command %r" % cmd, file=sys.stderr)
return 2
- c, tr, entries = walk_and_trace(image, paths)
+ c, tr, entries = walk_and_trace(image, paths, depth, skip)
if c is None:
return 1
present = sorted(tr.touched)
diff --git a/tools/apfs/verify.py b/tools/apfs/verify.py
index ec9b5fb..9460277 100755
--- a/tools/apfs/verify.py
+++ b/tools/apfs/verify.py
@@ -24,7 +24,7 @@
# verify.py content IMAGE REFDIR [-n COUNT] [--all]
# verify.py sample IMAGE REFDIR (a few dozen files of every kind)
# verify.py clines IMAGE [PATH]
-# verify.py sum IMAGE [PATH]
+# verify.py sum IMAGE [PATH] [--depth N]
import os
import random
@@ -334,11 +334,11 @@ def fnv1a(buf, h=FNV_INIT):
return h
-def cmd_sum(image, where):
+def cmd_sum(image, where, max_depth=0):
"""The totals the kernel's apfs_check prints, computed the same way."""
import io
out = io.StringIO()
- rc = _clines(image, where, out)
+ rc = _clines(image, where, out, max_depth)
if rc:
return rc
total = 0
@@ -366,7 +366,7 @@ def cmd_sum(image, where):
return 0
-def cmd_clines(image, where):
+def cmd_clines(image, where, max_depth=0):
"""
One line per directory entry, the same shape apfs-tool's census prints:
path, d_type, mode, uid, gid, logical size, FNV-1a of the contents
@@ -382,10 +382,10 @@ def cmd_clines(image, where):
return 1
start = hit[0]
prefix = "/" + where.strip("/")
- return _walk_lines(v, c, start, prefix, sys.stdout)
+ return _walk_lines(v, c, start, prefix, sys.stdout, max_depth)
-def _clines(image, where, out):
+def _clines(image, where, out, max_depth=0):
c, v = open_volume(image)
start = apfs.ROOT_DIR_INO_NUM
prefix = ""
@@ -396,13 +396,13 @@ def _clines(image, where, out):
return 1
start = hit[0]
prefix = "/" + where.strip("/")
- return _walk_lines(v, c, start, prefix, out)
+ return _walk_lines(v, c, start, prefix, out, max_depth)
-def _walk_lines(v, c, start, prefix, out):
- stack = [(start, prefix)]
+def _walk_lines(v, c, start, prefix, out, max_depth=0):
+ stack = [(start, prefix, 1)]
while stack:
- oid, px = stack.pop()
+ oid, px, depth = stack.pop()
for name, fid, dt in v.dir_entries(oid):
p = px + "/" + name
ino = v.inode(fid)
@@ -414,14 +414,15 @@ def _walk_lines(v, c, start, prefix, out):
h = fnv1a(ino.contents())
out.write("%s\t%d\t%06o\t%d\t%d\t%d\t%016x\n"
% (p, dt, ino.mode, ino.owner, ino.group, ino.size, h))
- if dt == 4:
- stack.append((fid, p))
+ if dt == 4 and (max_depth == 0 or depth < max_depth):
+ stack.append((fid, p, depth + 1))
c.close()
return 0
def main(argv):
limit = 200
+ depth = 0
do_all = False
args = []
i = 1
@@ -435,6 +436,10 @@ def main(argv):
do_all = True
i += 1
continue
+ if a == "--depth" and i + 1 < len(argv):
+ depth = int(argv[i + 1])
+ i += 2
+ continue
args.append(a)
i += 1
if len(args) < 2 or (args[0] not in ("clines", "sum") and len(args) < 3):
@@ -452,9 +457,9 @@ def main(argv):
if cmd == "sample":
return cmd_content(args[1], args[2], limit, False, True)
if cmd == "clines":
- return cmd_clines(args[1], args[2] if len(args) > 2 else None)
+ return cmd_clines(args[1], args[2] if len(args) > 2 else None, depth)
if cmd == "sum":
- return cmd_sum(args[1], args[2] if len(args) > 2 else None)
+ return cmd_sum(args[1], args[2] if len(args) > 2 else None, depth)
print("verify.py: unknown command %r" % cmd, file=sys.stderr)
return 2
diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.c
index 5965a4b..0331ac4 100644
--- a/xnu/files/bsd/dev/redwood_apfs_check.c
+++ b/xnu/files/bsd/dev/redwood_apfs_check.c
@@ -24,6 +24,16 @@
* point is a directory of the root image itself.
* apfs_dev=md1 the device to mount for modes 1 to 3 and 6 (default rvblk0)
* apfs_mnt=/lib where to mount it for mode 6 (default /private/var/apfs-check)
+ * apfs_depth=N go no deeper than N levels below where the walk starts
+ * (1 = only the entries of that directory; 0 = no limit).
+ * For splitting a volume too big to hold into parts: one part
+ * takes the root's own entries, the others take a subtree
+ * each, and the per-line hashes add up to the whole.
+ * apfs_skip=/a,/b directories not to go into, as paths inside the volume,
+ * comma separated. Their own line is still counted, only
+ * their contents are left out -- which is how a volume too
+ * big to hold is cut into parts that between them cover it
+ * exactly once.
* apfs_subtree=/usr walk only that part of the volume
*
* A line is the same one tools/apfs/apfs-tool's census and `verify.py clines` print:
@@ -121,6 +131,8 @@ struct census {
* through a path that itself crosses into it.
*/
int lookup_flags;
+ uint32_t max_depth; /* 0: no limit */
+ char *skip; /* comma separated paths not to descend into */
};
/* The format bits of a mode, which vnode_getattr does not report in va_mode. */
@@ -270,6 +282,32 @@ out:
static void census_dir(struct census *cs);
+/* Is this directory one of the ones to stay out of? Compares the path inside the volume. */
+static bool
+census_skipped(struct census *cs)
+{
+ const char *want = cs->path + cs->rel;
+ size_t want_len = strlen(want);
+ const char *at = cs->skip;
+
+ if (at == NULL) {
+ return false;
+ }
+ while (*at != '\0') {
+ const char *end = strchr(at, ',');
+ size_t n = (end != NULL) ? (size_t)(end - at) : strlen(at);
+
+ if (n == want_len && memcmp(at, want, n) == 0) {
+ return true;
+ }
+ if (end == NULL) {
+ break;
+ }
+ at = end + 1;
+ }
+ return false;
+}
+
/* Add "/name" to the path, do the entry, and put the path back as it was. */
static void
census_child(struct census *cs, const char *name, size_t name_len, uint8_t d_type)
@@ -286,7 +324,9 @@ census_child(struct census *cs, const char *name, size_t name_len, uint8_t d_typ
cs->len += name_len;
cs->path[cs->len] = '\0';
- if (census_entry(cs, d_type) && d_type == DT_DIR && cs->depth < MAX_DEPTH) {
+ if (census_entry(cs, d_type) && d_type == DT_DIR && cs->depth < MAX_DEPTH &&
+ (cs->max_depth == 0 || (uint32_t)cs->depth + 1 < cs->max_depth) &&
+ !census_skipped(cs)) {
cs->depth++;
census_dir(cs);
cs->depth--;
@@ -436,6 +476,15 @@ apfs_walk_one(void *arg, __unused wait_result_t wr)
bzero(&cs, sizeof(cs));
cs.ctx = vfs_context_kernel();
+ PE_parse_boot_argn("apfs_depth", &cs.max_depth, sizeof(cs.max_depth));
+ {
+ static char skip[256];
+
+ if (PE_parse_boot_arg_str("apfs_skip", skip, sizeof(skip)) && skip[0] != '\0') {
+ cs.skip = skip;
+ printf("apfs-check: staying out of %s\n", skip);
+ }
+ }
cs.lookup_flags = VNODE_LOOKUP_NOFOLLOW | VNODE_LOOKUP_NOCROSSMOUNT;
cs.path = kalloc_data(PATH_CAP, Z_WAITOK | Z_ZERO);
cs.buf = kalloc_data(READ_CHUNK, Z_WAITOK);
@@ -539,6 +588,15 @@ apfs_check_thread(void *arg, __unused wait_result_t wr)
* path leads from the root image into the mount, so crossing has to be allowed; the
* walk is kept on the volume by the check against the mount it started on.
*/
+ PE_parse_boot_argn("apfs_depth", &cs.max_depth, sizeof(cs.max_depth));
+ {
+ static char skip[256];
+
+ if (PE_parse_boot_arg_str("apfs_skip", skip, sizeof(skip)) && skip[0] != '\0') {
+ cs.skip = skip;
+ printf("apfs-check: staying out of %s\n", skip);
+ }
+ }
cs.lookup_flags = VNODE_LOOKUP_NOFOLLOW |
((mode == 4 || mode == 5) ? VNODE_LOOKUP_NOCROSSMOUNT : 0);
if (PE_parse_boot_arg_str("apfs_dev", subtree, sizeof(subtree)) && subtree[0] != '\0') {