Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C

APFS mounts in QEMU: a virtio block device, and a walk of the mounted volume

vcvkkfd69c883853e
diff --git a/tools/apfs/verify.py b/tools/apfs/verify.py
index 883141c..ec9b5fb 100755
--- a/tools/apfs/verify.py
+++ b/tools/apfs/verify.py
@@ -13,10 +13,18 @@
# clines the same lines tools/apfs/apfs-tool prints, so the C code that goes into the
# kernel can be diffed against this reader over a whole tree.
#
+# sum those same lines folded the way the kernel folds them (boot-arg apfs_check):
+# each line hashed on its own and the hashes added together. Printing 318098
+# lines out of a guest over a serial line is 47 MB, so the driver running in
+# QEMU reports this sum instead and it is compared with this one. Addition
+# does not care about the order the entries came in, which the two walks need
+# not share.
+#
# usage: verify.py listing IMAGE SLT
# verify.py content IMAGE REFDIR [-n COUNT] [--all]
# verify.py sample IMAGE REFDIR (a few dozen files of every kind)
# verify.py clines IMAGE [PATH]
+# verify.py sum IMAGE [PATH]
import os
import random
@@ -326,6 +334,38 @@ def fnv1a(buf, h=FNV_INIT):
return h
+def cmd_sum(image, where):
+ """The totals the kernel's apfs_check prints, computed the same way."""
+ import io
+ out = io.StringIO()
+ rc = _clines(image, where, out)
+ if rc:
+ return rc
+ total = 0
+ entries = dirs = files = links = others = 0
+ content = 0
+ for line in out.getvalue().splitlines():
+ total = (total + fnv1a((line + "\n").encode())) & MASK64
+ entries += 1
+ f = line.split("\t")
+ mode = int(f[2], 8)
+ if (mode & 0o170000) == 0o040000:
+ dirs += 1
+ elif (mode & 0o170000) == 0o120000:
+ links += 1
+ content += int(f[5])
+ elif (mode & 0o170000) == 0o100000:
+ files += 1
+ content += int(f[5])
+ else:
+ others += 1
+ print("entries %d dirs %d files %d symlinks %d other %d"
+ % (entries, dirs, files, links, others))
+ print("content bytes %d" % content)
+ print("line-hash sum %016x" % total)
+ return 0
+
+
def cmd_clines(image, where):
"""
One line per directory entry, the same shape apfs-tool's census prints:
@@ -342,7 +382,24 @@ def cmd_clines(image, where):
return 1
start = hit[0]
prefix = "/" + where.strip("/")
- out = sys.stdout
+ return _walk_lines(v, c, start, prefix, sys.stdout)
+
+
+def _clines(image, where, out):
+ c, v = open_volume(image)
+ start = apfs.ROOT_DIR_INO_NUM
+ prefix = ""
+ if where:
+ hit = v.lookup(where)
+ if hit is None:
+ print("verify.py: %s: not found" % where, file=sys.stderr)
+ return 1
+ start = hit[0]
+ prefix = "/" + where.strip("/")
+ return _walk_lines(v, c, start, prefix, out)
+
+
+def _walk_lines(v, c, start, prefix, out):
stack = [(start, prefix)]
while stack:
oid, px = stack.pop()
@@ -380,11 +437,12 @@ def main(argv):
continue
args.append(a)
i += 1
- if len(args) < 2 or (args[0] != "clines" and len(args) < 3):
+ if len(args) < 2 or (args[0] not in ("clines", "sum") and len(args) < 3):
print("usage: verify.py listing IMAGE SLT\n"
" verify.py content IMAGE REFDIR [-n COUNT] [--all]\n"
" verify.py sample IMAGE REFDIR\n"
- " verify.py clines IMAGE [PATH]", file=sys.stderr)
+ " verify.py clines IMAGE [PATH]\n"
+ " verify.py sum IMAGE [PATH]", file=sys.stderr)
return 2
cmd = args[0]
if cmd == "listing":
@@ -395,6 +453,8 @@ def main(argv):
return cmd_content(args[1], args[2], limit, False, True)
if cmd == "clines":
return cmd_clines(args[1], args[2] if len(args) > 2 else None)
+ if cmd == "sum":
+ return cmd_sum(args[1], args[2] if len(args) > 2 else None)
print("verify.py: unknown command %r" % cmd, file=sys.stderr)
return 2
diff --git a/xnu/files/bsd/dev/redwood_apfs_check.c b/xnu/files/bsd/dev/redwood_apfs_check.c
new file mode 100644
index 0000000..707fce6
--- /dev/null
+++ b/xnu/files/bsd/dev/redwood_apfs_check.c
@@ -0,0 +1,470 @@
+/*
+ * Mount the APFS volume on /dev/rvblk0 and walk it through VFS, so that what the driver
+ * does can be compared with what tools/apfs/apfs.py reads from the same image on the
+ * build machine. Only with boot-arg apfs_check.
+ *
+ * apfs_check=1 walk the whole volume and print only the totals
+ * apfs_check=2 also print a line per entry
+ * apfs_subtree=/usr walk only that part of the volume
+ *
+ * A line is the same one tools/apfs/apfs-tool's census and `verify.py clines` print:
+ *
+ * path <TAB> d_type <TAB> mode <TAB> uid <TAB> gid <TAB> size <TAB> hash-of-contents
+ *
+ * The whole volume is 318098 entries, and printing them all through the kernel log would
+ * be 47 MB down a serial line, so by default each line is folded into a sum instead: the
+ * lines are hashed one at a time and the hashes added together. Addition does not care
+ * what order the entries came in, which matters because this walk and the one on the host
+ * need not visit a directory's entries in the same order. The host adds up the same
+ * hashes over its own lines; if the two sums and counts agree, every path, kind, mode,
+ * owner, group, size and byte of content agreed. When they do not, apfs_check=2 over a
+ * subtree gives lines to diff.
+ *
+ * Everything here goes through VFS -- vnode_lookup, vnode_getattr, VNOP_READDIR,
+ * VNOP_READLINK, vn_rdwr -- so it exercises the vnode operations and not just the format
+ * reading underneath them.
+ */
+
+#include <kern/thread.h>
+#include <kern/clock.h>
+#include <kern/kalloc.h>
+#include <libkern/libkern.h>
+#include <mach/mach_types.h>
+#include <sys/dirent.h>
+#include <sys/errno.h>
+#include <sys/kauth.h>
+#include <sys/mount.h>
+#include <sys/mount_internal.h>
+#include <sys/namei.h>
+#include <sys/proc.h>
+#include <sys/stat.h>
+#include <sys/systm.h>
+#include <sys/uio.h>
+#include <sys/uio_internal.h>
+#include <sys/vnode.h>
+#include <sys/vnode_if.h>
+#include <sys/vnode_internal.h>
+#include <pexpert/pexpert.h>
+#include <sys/redwood_apfs_check.h>
+
+#define CHECK_MNT "/private/var/apfs-check"
+#define CHECK_PARENT "/private/var"
+#define CHECK_LEAF "apfs-check"
+#define CHECK_DEV "/dev/rvblk0"
+
+#define FNV_INIT 0xcbf29ce484222325ULL
+#define FNV_PRIME 0x100000001b3ULL
+
+#define READ_CHUNK (64 * 1024)
+#define DIR_CHUNK 4096
+#define MAX_DEPTH 48
+#define PATH_CAP 2048
+
+static uint64_t
+fnv1a(const uint8_t *p, size_t n, uint64_t h)
+{
+ size_t i;
+
+ for (i = 0; i < n; i++) {
+ h ^= p[i];
+ h *= FNV_PRIME;
+ }
+ return h;
+}
+
+struct census {
+ vfs_context_t ctx;
+ char *path; /* the whole path, starting with CHECK_MNT */
+ size_t rel; /* where the part inside the volume starts */
+ size_t len;
+ uint8_t *buf; /* for reading file contents */
+ uint64_t sum; /* the per-line hashes, added up */
+ uint64_t entries, dirs, files, links, others, bytes;
+ uint32_t errors;
+ bool print_lines;
+ int depth;
+};
+
+/* The format bits of a mode, which vnode_getattr does not report in va_mode. */
+static uint16_t
+ifmt_of(enum vtype t)
+{
+ switch (t) {
+ case VREG: return S_IFREG;
+ case VDIR: return S_IFDIR;
+ case VLNK: return S_IFLNK;
+ case VCHR: return S_IFCHR;
+ case VBLK: return S_IFBLK;
+ case VFIFO: return S_IFIFO;
+ case VSOCK: return S_IFSOCK;
+ default: return 0;
+ }
+}
+
+/* FNV-1a over a regular file's contents, read through VNOP_READ. */
+static int
+hash_file(struct census *cs, vnode_t vp, uint64_t size, uint64_t *out)
+{
+ uint64_t h = FNV_INIT, off = 0;
+
+ while (off < size) {
+ int want = (int)((size - off > READ_CHUNK) ? READ_CHUNK : (size - off));
+ int resid = 0;
+ int error = vn_rdwr(UIO_READ, vp, (caddr_t)cs->buf, want, (off_t)off,
+ UIO_SYSSPACE, IO_NOCACHE, vfs_context_ucred(cs->ctx), &resid,
+ vfs_context_proc(cs->ctx));
+
+ if (error) {
+ return error;
+ }
+ if (resid == want) {
+ break; /* nothing more to be had */
+ }
+ h = fnv1a(cs->buf, (size_t)(want - resid), h);
+ cs->bytes += (uint64_t)(want - resid);
+ off += (uint64_t)(want - resid);
+ }
+ *out = h;
+ return 0;
+}
+
+/* FNV-1a over a symbolic link's target, read through VNOP_READLINK. */
+static int
+hash_link(struct census *cs, vnode_t vp, uint64_t *out)
+{
+ uio_t uio;
+ char uio_buf[UIO_SIZEOF(1)];
+ int error;
+ size_t n;
+
+ uio = uio_createwithbuffer(1, 0, UIO_SYSSPACE, UIO_READ, uio_buf, sizeof(uio_buf));
+ if (uio == NULL) {
+ return ENOMEM;
+ }
+ uio_addiov(uio, CAST_USER_ADDR_T(cs->buf), MAXPATHLEN);
+ error = VNOP_READLINK(vp, uio, cs->ctx);
+ if (error) {
+ return error;
+ }
+ n = MAXPATHLEN - (size_t)uio_resid(uio);
+ cs->bytes += n;
+ *out = fnv1a(cs->buf, n, FNV_INIT);
+ return 0;
+}
+
+static void
+census_entry(struct census *cs, uint8_t d_type)
+{
+ struct vnode_attr va;
+ vnode_t vp = NULLVP;
+ uint64_t h = 0, size = 0;
+ uint16_t mode = 0;
+ char line[PATH_CAP + 128];
+ int error;
+ int n;
+
+ /*
+ * Do not follow a symbolic link: the point is to look at the link itself, as the host
+ * reader does. Following them would report the target's kind, size and contents
+ * instead, and a link whose target is missing -- of which this volume has several --
+ * would look like a lookup failure. Crossing mount points has to stay allowed: the
+ * path leads from the ramfs that holds the mount point into the volume itself.
+ */
+ error = vnode_lookup(cs->path, VNODE_LOOKUP_NOFOLLOW, &vp, cs->ctx);
+ if (error) {
+ printf("apfs-check: %s: lookup failed: %d\n", cs->path + cs->rel, error);
+ cs->errors++;
+ return;
+ }
+ VATTR_INIT(&va);
+ VATTR_WANTED(&va, va_mode);
+ VATTR_WANTED(&va, va_uid);
+ VATTR_WANTED(&va, va_gid);
+ VATTR_WANTED(&va, va_data_size);
+ error = vnode_getattr(vp, &va, cs->ctx);
+ if (error) {
+ printf("apfs-check: %s: getattr failed: %d\n", cs->path + cs->rel, error);
+ cs->errors++;
+ goto out;
+ }
+ mode = (uint16_t)(ifmt_of(vnode_vtype(vp)) | (VATTR_IS_SUPPORTED(&va, va_mode)
+ ? (va.va_mode & 07777) : 0));
+ size = VATTR_IS_SUPPORTED(&va, va_data_size) ? va.va_data_size : 0;
+
+ if (vnode_vtype(vp) == VREG) {
+ cs->files++;
+ error = hash_file(cs, vp, size, &h);
+ } else if (vnode_vtype(vp) == VLNK) {
+ cs->links++;
+ error = hash_link(cs, vp, &h);
+ } else if (vnode_vtype(vp) == VDIR) {
+ cs->dirs++;
+ } else {
+ cs->others++;
+ }
+ if (error) {
+ printf("apfs-check: %s: reading it failed: %d\n", cs->path + cs->rel, error);
+ cs->errors++;
+ goto out;
+ }
+
+ n = scnprintf(line, sizeof(line), "%s\t%u\t%06o\t%u\t%u\t%llu\t%016llx\n",
+ cs->path + cs->rel, d_type, mode,
+ VATTR_IS_SUPPORTED(&va, va_uid) ? va.va_uid : 0,
+ VATTR_IS_SUPPORTED(&va, va_gid) ? va.va_gid : 0,
+ (unsigned long long)size, (unsigned long long)h);
+ cs->sum += fnv1a((const uint8_t *)line, (size_t)n, FNV_INIT);
+ cs->entries++;
+ if (cs->print_lines) {
+ printf("apfs-line: %s", line);
+ }
+ if ((cs->entries % 20000) == 0) {
+ printf("apfs-check: %llu entries so far\n", cs->entries);
+ }
+out:
+ vnode_put(vp);
+}
+
+static void census_dir(struct census *cs);
+
+/* Add "/name" to the path, do the entry, and put the path back as it was. */
+static void
+census_child(struct census *cs, const char *name, size_t name_len, uint8_t d_type)
+{
+ size_t saved = cs->len;
+
+ if (cs->len + name_len + 2 > PATH_CAP) {
+ printf("apfs-check: path below %s is too long\n", cs->path + cs->rel);
+ cs->errors++;
+ return;
+ }
+ cs->path[cs->len++] = '/';
+ memcpy(cs->path + cs->len, name, name_len);
+ cs->len += name_len;
+ cs->path[cs->len] = '\0';
+
+ census_entry(cs, d_type);
+ if (d_type == DT_DIR && cs->depth < MAX_DEPTH) {
+ cs->depth++;
+ census_dir(cs);
+ cs->depth--;
+ }
+ cs->len = saved;
+ cs->path[cs->len] = '\0';
+}
+
+/* Every entry of the directory the path currently names. */
+static void
+census_dir(struct census *cs)
+{
+ vnode_t dvp = NULLVP;
+ char *dbuf;
+ off_t offset = 0;
+ int error;
+
+ error = vnode_lookup(cs->path, VNODE_LOOKUP_NOFOLLOW, &dvp, cs->ctx);
+ if (error) {
+ printf("apfs-check: %s: lookup failed: %d\n", cs->path + cs->rel, error);
+ cs->errors++;
+ return;
+ }
+ dbuf = kalloc_data(DIR_CHUNK, Z_WAITOK);
+ if (dbuf == NULL) {
+ vnode_put(dvp);
+ cs->errors++;
+ return;
+ }
+ for (;;) {
+ uio_t uio;
+ char uio_buf[UIO_SIZEOF(1)];
+ int eof = 0, items = 0;
+ size_t got;
+ char *p;
+
+ uio = uio_createwithbuffer(1, offset, UIO_SYSSPACE, UIO_READ, uio_buf,
+ sizeof(uio_buf));
+ if (uio == NULL) {
+ cs->errors++;
+ break;
+ }
+ uio_addiov(uio, CAST_USER_ADDR_T(dbuf), DIR_CHUNK);
+ error = VNOP_READDIR(dvp, uio, 0, &eof, &items, cs->ctx);
+ if (error) {
+ printf("apfs-check: %s: readdir failed: %d\n", cs->path + cs->rel, error);
+ cs->errors++;
+ break;
+ }
+ got = DIR_CHUNK - (size_t)uio_resid(uio);
+ if (got == 0) {
+ break;
+ }
+ offset = uio_offset(uio);
+
+ for (p = dbuf; p < dbuf + got;) {
+ struct dirent *de = (struct dirent *)p;
+
+ if (de->d_reclen == 0 || p + de->d_reclen > dbuf + got) {
+ printf("apfs-check: %s: a directory entry of %u bytes\n",
+ cs->path + cs->rel, de->d_reclen);
+ cs->errors++;
+ break;
+ }
+ p += de->d_reclen;
+ if (de->d_namlen == 0 ||
+ (de->d_namlen == 1 && de->d_name[0] == '.') ||
+ (de->d_namlen == 2 && de->d_name[0] == '.' && de->d_name[1] == '.')) {
+ continue;
+ }
+ census_child(cs, de->d_name, de->d_namlen, de->d_type);
+ }
+ if (eof) {
+ break;
+ }
+ }
+ kfree_data(dbuf, DIR_CHUNK);
+ vnode_put(dvp);
+}
+
+/* ---- getting there ---- */
+
+static int
+make_mount_point(vfs_context_t ctx)
+{
+ struct componentname cn;
+ struct vnode_attr va;
+ vnode_t dvp = NULLVP, vp = NULLVP;
+ char leaf[] = CHECK_LEAF;
+ int error;
+
+ error = vnode_lookup(CHECK_PARENT, 0, &dvp, ctx);
+ if (error) {
+ return error;
+ }
+ bzero(&cn, sizeof(cn));
+ cn.cn_nameiop = CREATE;
+ cn.cn_flags = ISLASTCN;
+ cn.cn_context = ctx;
+ cn.cn_pnbuf = leaf;
+ cn.cn_pnlen = (int)sizeof(leaf);
+ cn.cn_nameptr = leaf;
+ cn.cn_namelen = (int)(sizeof(leaf) - 1);
+
+ VATTR_INIT(&va);
+ VATTR_SET(&va, va_type, VDIR);
+ VATTR_SET(&va, va_mode, 0755);
+
+ error = VNOP_MKDIR(dvp, &vp, &cn, &va, ctx);
+ if (error == 0 && vp != NULLVP) {
+ vnode_put(vp);
+ }
+ vnode_put(dvp);
+ if (error == EEXIST) {
+ error = 0;
+ }
+ return error;
+}
+
+static void
+apfs_check_thread(void *arg, __unused wait_result_t wr)
+{
+ uint32_t mode = (uint32_t)(uintptr_t)arg;
+ vfs_context_t ctx = vfs_context_kernel();
+ struct census cs;
+ char devname[] = CHECK_DEV;
+ char subtree[256] = "";
+ uint64_t start, end;
+ int error, tries;
+
+ bzero(&cs, sizeof(cs));
+ cs.ctx = ctx;
+ cs.print_lines = (mode >= 2);
+
+ /*
+ * /private/var is a ramfs that userland's init mounts, so the mount point cannot be
+ * made until that has happened. Wait for it rather than guess how long it takes.
+ */
+ for (tries = 0; tries < 180; tries++) {
+ error = make_mount_point(ctx);
+ if (error == 0) {
+ break;
+ }
+ delay_for_interval(500, NSEC_PER_MSEC);
+ }
+ if (error) {
+ printf("apfs-check: cannot make %s: %d\n", CHECK_MNT, error);
+ return;
+ }
+
+ /*
+ * The mount data is the device's path itself, not a pointer to it: the extra
+ * indirection imageboot_pivot_image() uses is unpacked only on the BaseSystem path,
+ * which this build does not have, so apfs_mount() reads the string straight off
+ * this pointer.
+ */
+ error = kernel_mount("apfs", NULLVP, NULLVP, CHECK_MNT, devname, 0,
+ MNT_RDONLY | MNT_DONTBROWSE, KERNEL_MOUNT_NOAUTH, ctx);
+ if (error) {
+ printf("apfs-check: mounting %s on %s failed: %d\n", CHECK_DEV, CHECK_MNT,
+ error);
+ return;
+ }
+ printf("apfs-check: %s mounted on %s\n", CHECK_DEV, CHECK_MNT);
+
+ cs.path = kalloc_data(PATH_CAP, Z_WAITOK | Z_ZERO);
+ cs.buf = kalloc_data(READ_CHUNK, Z_WAITOK);
+ if (cs.path == NULL || cs.buf == NULL) {
+ printf("apfs-check: out of memory\n");
+ return;
+ }
+ cs.rel = sizeof(CHECK_MNT) - 1;
+ strlcpy(cs.path, CHECK_MNT, PATH_CAP);
+ cs.len = cs.rel;
+ if (PE_parse_boot_arg_str("apfs_subtree", subtree, sizeof(subtree)) &&
+ subtree[0] == '/') {
+ strlcat(cs.path, subtree, PATH_CAP);
+ cs.len = strlen(cs.path);
+ while (cs.len > cs.rel && cs.path[cs.len - 1] == '/') {
+ cs.path[--cs.len] = '\0';
+ }
+ printf("apfs-check: walking %s\n", cs.path + cs.rel);
+ } else {
+ printf("apfs-check: walking the whole volume\n");
+ }
+
+ start = mach_absolute_time();
+ census_dir(&cs);
+ end = mach_absolute_time();
+
+ {
+ uint64_t ns = 0;
+
+ absolutetime_to_nanoseconds(end - start, &ns);
+ printf("apfs-check: done in %llu ms\n", ns / 1000000);
+ }
+ printf("apfs-check: entries %llu dirs %llu files %llu symlinks %llu other %llu\n",
+ cs.entries, cs.dirs, cs.files, cs.links, cs.others);
+ printf("apfs-check: content bytes %llu\n", cs.bytes);
+ printf("apfs-check: line-hash sum %016llx\n", (unsigned long long)cs.sum);
+ printf("apfs-check: errors %u\n", cs.errors);
+ printf("apfs-check: compare with tools/apfs/verify.py sum\n");
+
+ kfree_data(cs.path, PATH_CAP);
+ kfree_data(cs.buf, READ_CHUNK);
+}
+
+void
+redwood_apfs_check_init(void)
+{
+ uint32_t mode = 0;
+ thread_t th;
+
+ if (!PE_parse_boot_argn("apfs_check", &mode, sizeof(mode)) || mode == 0) {
+ return;
+ }
+ if (kernel_thread_start(apfs_check_thread, (void *)(uintptr_t)mode, &th) !=
+ KERN_SUCCESS) {
+ printf("apfs-check: cannot start the thread\n");
+ return;
+ }
+ thread_deallocate(th);
+}
diff --git a/xnu/files/bsd/dev/redwood_virtio_blk.c b/xnu/files/bsd/dev/redwood_virtio_blk.c
new file mode 100644
index 0000000..630664d
--- /dev/null
+++ b/xnu/files/bsd/dev/redwood_virtio_blk.c
@@ -0,0 +1,522 @@
+/*
+ * A virtio block device, so that a disk image on the host can be a block device in the
+ * guest (/dev/rvblk0). Only with boot-arg virtio_blk=1: the bus address is QEMU's and
+ * means nothing on the phone.
+ *
+ * This exists for the APFS driver. The volume in ios165/rootfs.dmg is 5.6 GB, which is
+ * too much to hold in guest memory as a RAM disk the way packfs's image is held, and the
+ * point of reading APFS is to read a real image rather than one cut down to fit. QEMU
+ * serves it straight off the host's disk:
+ *
+ * -drive file=rootfs.apfs,format=raw,if=none,id=d,readonly=on
+ * -device virtio-blk-device,drive=d
+ *
+ * Under QEMU's "virt" machine the transport is virtio-mmio and must be the current one,
+ * not "legacy" (-global virtio-mmio.force-legacy=false).
+ *
+ * No interrupt is used and only one request is outstanding at a time: a request is put on
+ * the queue, the device is notified, and the used ring is watched until it comes back.
+ * That is slow in principle and entirely adequate here -- reads come from the APFS
+ * driver's own block cache misses, and the host is serving them out of its page cache.
+ *
+ * Read only. A write request is refused: nothing in this port has any business writing
+ * to the image, and refusing is one less way to damage it by accident.
+ */
+
+#include <kern/clock.h>
+#include <kern/locks.h>
+#include <libkern/libkern.h>
+#include <mach/mach_types.h>
+#include <mach/vm_param.h>
+#include <miscfs/devfs/devfs.h>
+#include <pexpert/pexpert.h>
+#include <sys/buf.h>
+#include <sys/conf.h>
+#include <sys/disk.h>
+#include <sys/errno.h>
+#include <sys/systm.h>
+#include <sys/redwood_virtio_blk.h>
+#include <vm/vm_kern.h>
+
+extern vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);
+extern uint64_t kvtophys(vm_offset_t va);
+
+#define VIRTIO_BUS 0x0a000000UL /* QEMU virt: 32 transports of 0x200 bytes */
+#define VIRTIO_SLOTS 32
+#define VIRTIO_SLOT_SIZE 0x200
+
+#define VIRTIO_MAGIC 0x000 /* "virt" */
+#define VIRTIO_VERSION 0x004
+#define VIRTIO_DEVICE_ID 0x008
+#define VIRTIO_DRIVER_FEATURES 0x020
+#define VIRTIO_DRIVER_FEATURES_SEL 0x024
+#define VIRTIO_QUEUE_SEL 0x030
+#define VIRTIO_QUEUE_NUM_MAX 0x034
+#define VIRTIO_QUEUE_NUM 0x038
+#define VIRTIO_QUEUE_READY 0x044
+#define VIRTIO_QUEUE_NOTIFY 0x050
+#define VIRTIO_INTERRUPT_ACK 0x064
+#define VIRTIO_STATUS 0x070
+#define VIRTIO_QUEUE_DESC 0x080
+#define VIRTIO_QUEUE_DRIVER 0x090
+#define VIRTIO_QUEUE_DEVICE 0x0a0
+#define VIRTIO_CONFIG 0x100 /* the device's own configuration */
+
+#define VIRTIO_ID_BLOCK 2
+#define STATUS_ACKNOWLEDGE 1
+#define STATUS_DRIVER 2
+#define STATUS_DRIVER_OK 4
+#define STATUS_FEATURES_OK 8
+
+#define QUEUE_SIZE 64
+#define DESC_F_NEXT 1
+#define DESC_F_WRITE 2
+#define AVAIL_F_NO_INTERRUPT 1
+
+/* the request types we use */
+#define VIRTIO_BLK_T_IN 0
+#define VIRTIO_BLK_S_OK 0
+
+#define VIRTIO_BLK_SECTOR 512 /* virtio-blk counts in these, always */
+
+/*
+ * The most one request may carry. A request needs one descriptor for its header, one for
+ * its status byte and one per page of the data, so this bounds the chain well inside the
+ * queue.
+ */
+#define BLK_MAX_XFER (64 * 1024)
+#define BLK_MAX_DATA_DESC (BLK_MAX_XFER / PAGE_SIZE + 1)
+
+struct vq_desc {
+ uint64_t addr;
+ uint32_t len;
+ uint16_t flags;
+ uint16_t next;
+};
+struct vq_avail {
+ uint16_t flags;
+ uint16_t idx;
+ uint16_t ring[QUEUE_SIZE];
+};
+struct vq_used {
+ uint16_t flags;
+ uint16_t idx;
+ struct {
+ uint32_t id;
+ uint32_t len;
+ } ring[QUEUE_SIZE];
+};
+struct blk_req_hdr {
+ uint32_t type;
+ uint32_t reserved;
+ uint64_t sector;
+};
+
+/* the ring and the small request pieces share one page */
+#define OFFSET_DESC 0x0000
+#define OFFSET_AVAIL 0x0400
+#define OFFSET_USED 0x0800
+#define OFFSET_HDR 0x0c00
+#define OFFSET_STATUS 0x0c40
+
+struct blk_device {
+ vm_offset_t regs;
+ vm_offset_t page;
+ uint64_t page_phys;
+ struct vq_desc *desc;
+ struct vq_avail *avail;
+ struct vq_used *used;
+ struct blk_req_hdr *hdr;
+ volatile uint8_t *status;
+ uint16_t seen; /* how far the used ring has been read */
+ uint64_t capacity; /* in 512-byte sectors */
+ lck_mtx_t lock;
+};
+
+static LCK_GRP_DECLARE(blk_lck_grp, "redwood-virtio-blk");
+static struct blk_device blk_dev;
+static bool blk_ready;
+static int blk_bmajor = -1, blk_cmajor = -1;
+
+static inline uint32_t
+reg_read(struct blk_device *d, uint32_t reg)
+{
+ return *(volatile uint32_t *)(d->regs + reg);
+}
+
+static inline void
+reg_write(struct blk_device *d, uint32_t reg, uint32_t value)
+{
+ *(volatile uint32_t *)(d->regs + reg) = value;
+}
+
+static inline void
+reg_write64(struct blk_device *d, uint32_t reg, uint64_t value)
+{
+ reg_write(d, reg, (uint32_t)value);
+ reg_write(d, reg + 4, (uint32_t)(value >> 32));
+}
+
+/*
+ * One read. `data` is kernel memory that need not be page aligned and whose pages need
+ * not be next to each other in physical memory, so it goes on the queue a page at a time.
+ */
+static int
+blk_read(struct blk_device *d, uint64_t offset, void *data, size_t len)
+{
+ vm_offset_t va = (vm_offset_t)data;
+ uint16_t idx = 0;
+ size_t done = 0;
+ uint64_t deadline;
+ int error = 0;
+
+ if (len == 0 || len > BLK_MAX_XFER) {
+ return EINVAL;
+ }
+ if ((offset % VIRTIO_BLK_SECTOR) != 0) {
+ return EINVAL;
+ }
+ if (offset / VIRTIO_BLK_SECTOR >= d->capacity ||
+ (offset + len) / VIRTIO_BLK_SECTOR > d->capacity) {
+ return EINVAL;
+ }
+
+ lck_mtx_lock(&d->lock);
+
+ d->hdr->type = VIRTIO_BLK_T_IN;
+ d->hdr->reserved = 0;
+ d->hdr->sector = offset / VIRTIO_BLK_SECTOR;
+ *d->status = 0xff;
+
+ /* descriptor 0: the header, which the device reads */
+ d->desc[0].addr = d->page_phys + OFFSET_HDR;
+ d->desc[0].len = sizeof(struct blk_req_hdr);
+ d->desc[0].flags = DESC_F_NEXT;
+ d->desc[0].next = 1;
+ idx = 1;
+
+ /* then the data, one descriptor per run of bytes inside a page */
+ while (done < len) {
+ vm_offset_t at = va + done;
+ size_t chunk = PAGE_SIZE - (at & PAGE_MASK);
+
+ if (chunk > len - done) {
+ chunk = len - done;
+ }
+ if (idx >= 1 + BLK_MAX_DATA_DESC) {
+ lck_mtx_unlock(&d->lock);
+ return EINVAL;
+ }
+ d->desc[idx].addr = kvtophys(at);
+ d->desc[idx].len = (uint32_t)chunk;
+ d->desc[idx].flags = DESC_F_WRITE | DESC_F_NEXT;
+ d->desc[idx].next = (uint16_t)(idx + 1);
+ idx++;
+ done += chunk;
+ }
+
+ /* last: the status byte, which the device writes */
+ d->desc[idx].addr = d->page_phys + OFFSET_STATUS;
+ d->desc[idx].len = 1;
+ d->desc[idx].flags = DESC_F_WRITE;
+ d->desc[idx].next = 0;
+
+ d->avail->flags = AVAIL_F_NO_INTERRUPT;
+ d->avail->ring[d->avail->idx % QUEUE_SIZE] = 0; /* the chain starts at 0 */
+ __builtin_arm_dmb(0xb);
+ d->avail->idx++;
+ __builtin_arm_dmb(0xb);
+ reg_write(d, VIRTIO_QUEUE_NOTIFY, 0);
+
+ /* wait for it to come back; a read off the host's disk should take microseconds */
+ clock_interval_to_deadline(10, NSEC_PER_SEC, &deadline);
+ for (;;) {
+ __builtin_arm_dmb(0xb);
+ if (d->used->idx != d->seen) {
+ d->seen = d->used->idx;
+ break;
+ }
+ if (mach_absolute_time() > deadline) {
+ printf("virtio-blk: a read of %lu bytes at %llu went unanswered\n",
+ (unsigned long)len, offset);
+ lck_mtx_unlock(&d->lock);
+ return EIO;
+ }
+ delay(10);
+ }
+ reg_write(d, VIRTIO_INTERRUPT_ACK, reg_read(d, 0x060)); /* keep the device quiet */
+
+ if (*d->status != VIRTIO_BLK_S_OK) {
+ printf("virtio-blk: a read of %lu bytes at %llu failed with status %u\n",
+ (unsigned long)len, offset, *d->status);
+ error = EIO;
+ }
+ lck_mtx_unlock(&d->lock);
+ return error;
+}
+
+/* ---- the block device ---- */
+
+static int
+blk_dev_open(__unused dev_t dev, int flags, __unused int devtype, __unused proc_t p)
+{
+ if (!blk_ready) {
+ return ENXIO;
+ }
+ if (flags & FWRITE) {
+ return EROFS;
+ }
+ return 0;
+}
+
+static int
+blk_dev_close(__unused dev_t dev, __unused int flags, __unused int devtype,
+ __unused proc_t p)
+{
+ return 0;
+}
+
+static void
+blk_dev_strategy(buf_t bp)
+{
+ struct blk_device *d = &blk_dev;
+ uint64_t offset;
+ size_t len;
+ int error;
+
+ if (!blk_ready) {
+ buf_seterror(bp, ENXIO);
+ buf_biodone(bp);
+ return;
+ }
+ if (!(buf_flags(bp) & B_READ)) {
+ buf_seterror(bp, EROFS); /* this device never writes */
+ buf_biodone(bp);
+ return;
+ }
+ offset = (uint64_t)buf_blkno(bp) * VIRTIO_BLK_SECTOR;
+ len = buf_count(bp);
+
+ /* trim a request that runs off the end, and refuse one that starts past it */
+ if (offset >= d->capacity * VIRTIO_BLK_SECTOR) {
+ buf_seterror(bp, offset == d->capacity * VIRTIO_BLK_SECTOR ? 0 : EINVAL);
+ buf_setresid(bp, buf_count(bp));
+ buf_biodone(bp);
+ return;
+ }
+ if (offset + len > d->capacity * VIRTIO_BLK_SECTOR) {
+ len = (size_t)(d->capacity * VIRTIO_BLK_SECTOR - offset);
+ }
+
+ error = 0;
+ {
+ uint8_t *dst = (uint8_t *)buf_dataptr(bp);
+ size_t done = 0;
+
+ while (done < len && error == 0) {
+ size_t chunk = len - done;
+
+ if (chunk > BLK_MAX_XFER) {
+ chunk = BLK_MAX_XFER;
+ }
+ error = blk_read(d, offset + done, dst + done, chunk);
+ done += chunk;
+ }
+ buf_setresid(bp, (uint32_t)(buf_count(bp) - (error ? 0 : done)));
+ }
+ if (error) {
+ buf_seterror(bp, error);
+ }
+ buf_biodone(bp);
+}
+
+static int
+blk_dev_ioctl(__unused dev_t dev, u_long cmd, caddr_t data, __unused int fflag,
+ __unused proc_t p)
+{
+ struct blk_device *d = &blk_dev;
+
+ if (!blk_ready) {
+ return ENXIO;
+ }
+ switch (cmd) {
+ case DKIOCGETBLOCKSIZE:
+ *(uint32_t *)data = VIRTIO_BLK_SECTOR;
+ return 0;
+ case DKIOCSETBLOCKSIZE:
+ /*
+ * spec_open() sets the block size to 512 to count the device in 512-byte
+ * blocks and then sets it back, and treats a refusal of either as the device
+ * being broken (ENXIO). virtio-blk counts in 512-byte sectors and nothing
+ * else, so that is the only size to agree to.
+ */
+ if (*(uint32_t *)data != VIRTIO_BLK_SECTOR) {
+ return EINVAL;
+ }
+ return 0;
+ case DKIOCGETBLOCKCOUNT:
+ *(uint64_t *)data = d->capacity;
+ return 0;
+ case DKIOCISWRITABLE:
+ *(uint32_t *)data = 0;
+ return 0;
+ case DKIOCGETMAXBLOCKCOUNTREAD:
+ *(uint64_t *)data = BLK_MAX_XFER / VIRTIO_BLK_SECTOR;
+ return 0;
+ default:
+ return ENOTTY;
+ }
+}
+
+static int
+blk_dev_size(dev_t dev)
+{
+ (void)dev;
+ return blk_ready ? VIRTIO_BLK_SECTOR : -1;
+}
+
+static int
+blk_dev_read(__unused dev_t dev, struct uio *uio, __unused int ioflag)
+{
+ /* the character device is here so that tools can see the disk; reads go by buffer */
+ return physio(blk_dev_strategy, NULL, dev, B_READ, (u_int (*)(buf_t))blk_dev_size,
+ uio, VIRTIO_BLK_SECTOR);
+}
+
+static const struct bdevsw blk_bdevsw = {
+ .d_open = blk_dev_open,
+ .d_close = blk_dev_close,
+ .d_strategy = blk_dev_strategy,
+ .d_ioctl = blk_dev_ioctl,
+ .d_dump = eno_dump,
+ .d_psize = blk_dev_size,
+ .d_type = D_DISK,
+};
+
+static const struct cdevsw blk_cdevsw = {
+ .d_open = blk_dev_open,
+ .d_close = blk_dev_close,
+ .d_read = blk_dev_read,
+ .d_write = eno_rdwrt,
+ .d_ioctl = blk_dev_ioctl,
+ .d_stop = eno_stop,
+ .d_reset = eno_reset,
+ .d_ttys = NULL,
+ .d_select = eno_select,
+ .d_mmap = eno_mmap,
+ .d_strategy = eno_strat,
+ .d_reserved_1 = eno_getc,
+ .d_reserved_2 = eno_putc,
+ .d_type = D_DISK,
+};
+
+/* ---- bringing it up ---- */
+
+static bool
+blk_attach(struct blk_device *d, int slot)
+{
+ uint32_t cap_lo, cap_hi;
+
+ if (kmem_alloc(kernel_map, &d->page, PAGE_SIZE,
+ KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
+ return false;
+ }
+ d->page_phys = kvtophys(d->page);
+ d->desc = (struct vq_desc *)(d->page + OFFSET_DESC);
+ d->avail = (struct vq_avail *)(d->page + OFFSET_AVAIL);
+ d->used = (struct vq_used *)(d->page + OFFSET_USED);
+ d->hdr = (struct blk_req_hdr *)(d->page + OFFSET_HDR);
+ d->status = (volatile uint8_t *)(d->page + OFFSET_STATUS);
+
+ reg_write(d, VIRTIO_STATUS, 0); /* reset */
+ reg_write(d, VIRTIO_STATUS, STATUS_ACKNOWLEDGE);
+ reg_write(d, VIRTIO_STATUS, STATUS_ACKNOWLEDGE | STATUS_DRIVER);
+ /* nothing in the low half is wanted; bit 32 is VIRTIO_F_VERSION_1 */
+ reg_write(d, VIRTIO_DRIVER_FEATURES_SEL, 0);
+ reg_write(d, VIRTIO_DRIVER_FEATURES, 0);
+ reg_write(d, VIRTIO_DRIVER_FEATURES_SEL, 1);
+ reg_write(d, VIRTIO_DRIVER_FEATURES, 1);
+ reg_write(d, VIRTIO_STATUS, STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK);
+ if (!(reg_read(d, VIRTIO_STATUS) & STATUS_FEATURES_OK)) {
+ printf("virtio-blk: slot %d: features refused\n", slot);
+ return false;
+ }
+
+ cap_lo = reg_read(d, VIRTIO_CONFIG);
+ cap_hi = reg_read(d, VIRTIO_CONFIG + 4);
+ d->capacity = ((uint64_t)cap_hi << 32) | cap_lo;
+ if (d->capacity == 0) {
+ printf("virtio-blk: slot %d: the disk is empty\n", slot);
+ return false;
+ }
+
+ reg_write(d, VIRTIO_QUEUE_SEL, 0);
+ if (reg_read(d, VIRTIO_QUEUE_NUM_MAX) < QUEUE_SIZE) {
+ printf("virtio-blk: slot %d: request queue of %u\n", slot,
+ reg_read(d, VIRTIO_QUEUE_NUM_MAX));
+ return false;
+ }
+ reg_write(d, VIRTIO_QUEUE_NUM, QUEUE_SIZE);
+ reg_write64(d, VIRTIO_QUEUE_DESC, d->page_phys + OFFSET_DESC);
+ reg_write64(d, VIRTIO_QUEUE_DRIVER, d->page_phys + OFFSET_AVAIL);
+ reg_write64(d, VIRTIO_QUEUE_DEVICE, d->page_phys + OFFSET_USED);
+ __builtin_arm_dmb(0xb);
+ reg_write(d, VIRTIO_QUEUE_READY, 1);
+ reg_write(d, VIRTIO_STATUS,
+ STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK);
+ return true;
+}
+
+void
+redwood_virtio_blk_init(void)
+{
+ uint32_t enabled = 0;
+ vm_offset_t bus;
+ struct blk_device *d = &blk_dev;
+ int slot;
+
+ if (!PE_parse_boot_argn("virtio_blk", &enabled, sizeof(enabled)) || !enabled) {
+ return;
+ }
+ bus = ml_io_map(VIRTIO_BUS, VIRTIO_SLOTS * VIRTIO_SLOT_SIZE);
+ for (slot = 0; slot < VIRTIO_SLOTS; slot++) {
+ d->regs = bus + (vm_offset_t)slot * VIRTIO_SLOT_SIZE;
+ if (reg_read(d, VIRTIO_MAGIC) != 0x74726976 ||
+ reg_read(d, VIRTIO_DEVICE_ID) != VIRTIO_ID_BLOCK) {
+ continue;
+ }
+ if (reg_read(d, VIRTIO_VERSION) != 2) {
+ printf("virtio-blk: slot %d is a legacy transport; start QEMU with "
+ "-global virtio-mmio.force-legacy=false\n", slot);
+ continue;
+ }
+ lck_mtx_init(&d->lock, &blk_lck_grp, LCK_ATTR_NULL);
+ if (!blk_attach(d, slot)) {
+ lck_mtx_destroy(&d->lock, &blk_lck_grp);
+ continue;
+ }
+ blk_bmajor = bdevsw_add(-1, &blk_bdevsw);
+ if (blk_bmajor < 0) {
+ printf("virtio-blk: bdevsw_add failed\n");
+ return;
+ }
+ blk_cmajor = cdevsw_add_with_bdev(-1, &blk_cdevsw, blk_bmajor);
+ if (blk_cmajor < 0) {
+ printf("virtio-blk: cdevsw_add failed\n");
+ return;
+ }
+ if (devfs_make_node(makedev(blk_bmajor, 0), DEVFS_BLOCK, UID_ROOT,
+ GID_OPERATOR, 0600, "rvblk0") == NULL ||
+ devfs_make_node(makedev(blk_cmajor, 0), DEVFS_CHAR, UID_ROOT,
+ GID_OPERATOR, 0600, "rrvblk0") == NULL) {
+ printf("virtio-blk: devfs_make_node failed\n");
+ return;
+ }
+ blk_ready = true;
+ printf("virtio-blk: /dev/rvblk0 in slot %d, %llu sectors of %u (%llu MB)\n",
+ slot, d->capacity, VIRTIO_BLK_SECTOR,
+ d->capacity * VIRTIO_BLK_SECTOR / (1024 * 1024));
+ return;
+ }
+ printf("virtio-blk: no block device on the bus\n");
+}
diff --git a/xnu/files/bsd/miscfs/apfs/apfs.h b/xnu/files/bsd/miscfs/apfs/apfs.h
index ffa9480..2f78602 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs.h
+++ b/xnu/files/bsd/miscfs/apfs/apfs.h
@@ -35,6 +35,7 @@ struct apfs_mount {
struct apfs_nx am_nx;
struct apfs_vol am_vol;
uint32_t am_devblock;
+ bool am_owns_devvp; /* we opened it, so we close it */
struct apfs_node *am_hash[APFS_HASH_BUCKETS];
uint64_t am_root_id;
};
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_lib.c b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
index c4775bb..7d55adf 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_lib.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_lib.c
@@ -1542,6 +1542,50 @@ apfs_cfile_init(struct apfs_vol *vol, const struct apfs_inode *ino,
}
}
+/*
+ * zlib asks for its own working memory through these. The kernel's copy of zlib has no
+ * allocator of its own -- a zeroed z_stream leaves zalloc NULL and inflateInit_ then calls
+ * through it -- so they have to be set, and they are set on both sides so that the kernel
+ * and the program on the build machine run the same code. zfree is told only an address,
+ * so each block carries its own length in front of it.
+ */
+static voidpf
+apfs_zalloc(voidpf opaque, unsigned items, unsigned size)
+{
+ size_t n;
+ uint8_t *p;
+
+ (void)opaque;
+ if (items == 0 || size == 0) {
+ return Z_NULL;
+ }
+ if ((size_t)items > (~(size_t)0 - sizeof(size_t)) / size) {
+ return Z_NULL;
+ }
+ n = (size_t)items * size + sizeof(size_t);
+ p = apfs_alloc(n);
+ if (p == NULL) {
+ return Z_NULL;
+ }
+ memcpy(p, &n, sizeof(n));
+ return p + sizeof(size_t);
+}
+
+static void
+apfs_zfree(voidpf opaque, voidpf address)
+{
+ uint8_t *p;
+ size_t n;
+
+ (void)opaque;
+ if (address == NULL) {
+ return;
+ }
+ p = (uint8_t *)address - sizeof(size_t);
+ memcpy(&n, p, sizeof(n));
+ apfs_free(p, n);
+}
+
/* One zlib stream. A leading 0xff means the piece was stored rather than compressed. */
static int
apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,
@@ -1564,6 +1608,9 @@ apfs_inflate(const uint8_t *src, size_t src_len, uint8_t *dst, size_t dst_len,
return 0;
}
memset(&zs, 0, sizeof(zs));
+ zs.zalloc = apfs_zalloc;
+ zs.zfree = apfs_zfree;
+ zs.opaque = Z_NULL;
zs.next_in = (Bytef *)(uintptr_t)src;
zs.avail_in = (uInt)src_len;
zs.next_out = (Bytef *)dst;
diff --git a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
index d7c1a0c..50879aa 100644
--- a/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
+++ b/xnu/files/bsd/miscfs/apfs/apfs_vfsops.c
@@ -17,6 +17,7 @@
#include <sys/time.h>
#include <uuid/uuid.h>
#include <sys/vnode_internal.h>
+#include <sys/fcntl.h>
static LCK_GRP_DECLARE(apfs_lck_grp, "apfs");
@@ -317,21 +318,74 @@ apfs_mountroot(mount_t mp, vnode_t devvp, vfs_context_t ctx)
return apfs_mount_common(mp, devvp, ctx);
}
+/*
+ * mount(2), and kernel_mount(). The mount data is the path of the block device holding
+ * the container, as a string. VFS does not resolve it for us the way it does for a
+ * filesystem marked VFC_VFSLOCALARGS -- see the note in vfs_conf.c for why this one is
+ * not -- so it is looked up and opened here, and closed again in apfs_unmount().
+ */
static int
-apfs_mount(mount_t mp, vnode_t devvp, __unused user_addr_t data, vfs_context_t ctx)
+apfs_mount(mount_t mp, vnode_t devvp, user_addr_t data, vfs_context_t ctx)
{
+ char *path;
+ vnode_t vp = NULLVP;
+ size_t len = 0;
+ int error;
+
if (vfs_isupdate(mp)) {
/* nothing about a mounted volume can change, least of all to read-write */
return vfs_iswriteupgrade(mp) ? EROFS : 0;
}
- if (devvp == NULLVP) {
+ if (devvp != NULLVP) {
+ /* mounted as the root: VFS found the device and owns the vnode */
+ return apfs_mount_common(mp, devvp, ctx);
+ }
+ if (data == USER_ADDR_NULL) {
return EINVAL;
}
- return apfs_mount_common(mp, devvp, ctx);
+ path = kalloc_data(MAXPATHLEN, Z_WAITOK);
+ if (path == NULL) {
+ return ENOMEM;
+ }
+ if (vfs_iskernelmount(mp)) {
+ /* kernel_mount() hands the pointer through untouched */
+ error = (strlcpy(path, CAST_DOWN(char *, data), MAXPATHLEN) < MAXPATHLEN)
+ ? 0 : ENAMETOOLONG;
+ } else {
+ error = copyinstr(data, path, MAXPATHLEN, &len);
+ }
+ if (error == 0) {
+ error = vnode_lookup(path, 0, &vp, ctx);
+ }
+ if (error == 0 && !vnode_isblk(vp)) {
+ error = ENOTBLK;
+ }
+ if (error == 0) {
+ error = VNOP_OPEN(vp, FREAD, ctx);
+ if (error == 0) {
+ /*
+ * The iocount from the lookup is kept for as long as the volume is
+ * mounted; apfs_unmount() closes and drops it.
+ */
+ error = apfs_mount_common(mp, vp, ctx);
+ if (error == 0) {
+ APFS_MP(mp)->am_owns_devvp = true;
+ } else {
+ (void)VNOP_CLOSE(vp, FREAD, ctx);
+ }
+ }
+ if (error != 0) {
+ vnode_put(vp);
+ }
+ } else if (vp != NULLVP) {
+ vnode_put(vp);
+ }
+ kfree_data(path, MAXPATHLEN);
+ return error;
}
static int
-apfs_unmount(mount_t mp, int mntflags, __unused vfs_context_t ctx)
+apfs_unmount(mount_t mp, int mntflags, vfs_context_t ctx)
{
struct apfs_mount *am = APFS_MP(mp);
int error = vflush(mp, NULLVP, (mntflags & MNT_FORCE) ? FORCECLOSE : 0);
@@ -340,6 +394,10 @@ apfs_unmount(mount_t mp, int mntflags, __unused vfs_context_t ctx)
return error;
}
vfs_setfsprivate(mp, NULL);
+ if (am->am_owns_devvp && am->am_devvp != NULLVP) {
+ (void)VNOP_CLOSE(am->am_devvp, FREAD, ctx);
+ vnode_put(am->am_devvp);
+ }
apfs_free_mount(am);
return 0;
}
diff --git a/xnu/files/bsd/sys/redwood_apfs_check.h b/xnu/files/bsd/sys/redwood_apfs_check.h
new file mode 100644
index 0000000..2babd7a
--- /dev/null
+++ b/xnu/files/bsd/sys/redwood_apfs_check.h
@@ -0,0 +1,11 @@
+/*
+ * Mount the APFS volume on /dev/rvblk0 and walk it, to compare with what the host reader
+ * sees in the same image. Only with boot-arg apfs_check.
+ * See bsd/dev/redwood_apfs_check.c.
+ */
+#ifndef _SYS_REDWOOD_APFS_CHECK_H_
+#define _SYS_REDWOOD_APFS_CHECK_H_
+
+void redwood_apfs_check_init(void);
+
+#endif /* _SYS_REDWOOD_APFS_CHECK_H_ */
diff --git a/xnu/files/bsd/sys/redwood_virtio_blk.h b/xnu/files/bsd/sys/redwood_virtio_blk.h
new file mode 100644
index 0000000..b97e0b0
--- /dev/null
+++ b/xnu/files/bsd/sys/redwood_virtio_blk.h
@@ -0,0 +1,10 @@
+/*
+ * A virtio block device for QEMU: a disk image on the host becomes /dev/rvblk0 in the
+ * guest. Only with boot-arg virtio_blk=1. See bsd/dev/redwood_virtio_blk.c.
+ */
+#ifndef _SYS_REDWOOD_VIRTIO_BLK_H_
+#define _SYS_REDWOOD_VIRTIO_BLK_H_
+
+void redwood_virtio_blk_init(void);
+
+#endif /* _SYS_REDWOOD_VIRTIO_BLK_H_ */
diff --git a/xnu/redwood.patch b/xnu/redwood.patch
index 442b842..fbcca90 100644
--- a/xnu/redwood.patch
+++ b/xnu/redwood.patch
@@ -46,10 +46,10 @@ index 222158a93918..68bb0f3e56bf 100755
######################################################################
# HISTORY
diff --git a/bsd/conf/files b/bsd/conf/files
-index 44274016a4cf..cbbb936699fc 100644
+index 44274016a4cf..9902234f568c 100644
--- a/bsd/conf/files
+++ b/bsd/conf/files
-@@ -109,6 +109,31 @@ bsd/dev/munge.c optional config_requires_u32_munging
+@@ -109,6 +109,33 @@ bsd/dev/munge.c optional config_requires_u32_munging
bsd/dev/unix_startup.c standard
bsd/dev/monotonic.c optional monotonic
@@ -73,6 +73,8 @@ index 44274016a4cf..cbbb936699fc 100644
+bsd/dev/redwood_mic.c standard
+bsd/dev/redwood_wifi.c standard
+bsd/dev/redwood_adsp.c standard
++bsd/dev/redwood_virtio_blk.c standard
++bsd/dev/redwood_apfs_check.c optional apfs
+bsd/dev/redwood_sound.c standard
+bsd/dev/redwood_qmi.c standard
+bsd/dev/redwood_wlfw.c standard
@@ -81,7 +83,7 @@ index 44274016a4cf..cbbb936699fc 100644
bsd/dev/dev_perfmon.c optional config_perfmon
bsd/libkern/crc32.c standard
-@@ -155,6 +180,13 @@ bsd/miscfs/devfs/devfs_vfsops.c optional devfs
+@@ -155,6 +182,13 @@ bsd/miscfs/devfs/devfs_vfsops.c optional devfs
bsd/miscfs/mockfs/mockfs_fsnode.c optional mockfs
bsd/miscfs/mockfs/mockfs_vfsops.c optional mockfs
bsd/miscfs/mockfs/mockfs_vnops.c optional mockfs
@@ -95,7 +97,7 @@ index 44274016a4cf..cbbb936699fc 100644
bsd/miscfs/routefs/routefs_ops.c optional routefs
-@@ -361,6 +393,7 @@ bsd/security/audit/audit_syscalls.c standard
+@@ -361,6 +395,7 @@ bsd/security/audit/audit_syscalls.c standard
bsd/security/audit/audit_worker.c optional config_audit
bsd/pthread/pthread_shims.c standard
@@ -103,7 +105,7 @@ index 44274016a4cf..cbbb936699fc 100644
bsd/pthread/pthread_priority.c standard
bsd/pthread/pthread_workqueue.c standard
-@@ -383,6 +416,7 @@ bsd/kern/kern_debug.c standard
+@@ -383,6 +418,7 @@ bsd/kern/kern_debug.c standard
bsd/kern/kern_symfile.c standard
bsd/kern/kern_descrip.c standard
bsd/kern/kern_guarded.c standard
@@ -296,19 +298,23 @@ index 3e6bffa8c48e..43cffb98bdcc 100644
mdevremoveall(void)
{
diff --git a/bsd/kern/bsd_init.c b/bsd/kern/bsd_init.c
-index 4df41ba1f61b..d532c567ba6b 100644
+index 4df41ba1f61b..0c71e8a534bb 100644
--- a/bsd/kern/bsd_init.c
+++ b/bsd/kern/bsd_init.c
-@@ -147,6 +147,8 @@
+@@ -147,6 +147,12 @@
#include <net/if_ether.h> /* for ether_family_init() */
#include <net/if_gif.h> /* for gif_init() */
#include <miscfs/devfs/devfsdefs.h> /* for devfs_kernel_mount() */
+#include <sys/redwood_input.h>
+#include <sys/redwood_audio.h>
++#include <sys/redwood_virtio_blk.h>
++#if APFS
++#include <sys/redwood_apfs_check.h>
++#endif
#include <vm/vm_kern.h> /* for kmem_suballoc() */
#include <sys/proc_uuid_policy.h> /* proc_uuid_policy_init() */
#include <netinet/flow_divert.h> /* flow_divert_init() */
-@@ -490,7 +492,7 @@ bsd_init(void)
+@@ -490,7 +496,7 @@ bsd_init(void)
platform_stall_panic_or_spin(PLATFORM_STALL_XNU_LOCATION_BSD_INIT);
#endif
@@ -317,12 +323,17 @@ index 4df41ba1f61b..d532c567ba6b 100644
#if DEBUG_BSDINIT
#define bsd_init_kprintf(x, ...) kprintf("bsd_init: " x, ## __VA_ARGS__)
-@@ -976,6 +978,8 @@ bsd_init(void)
+@@ -976,6 +982,13 @@ bsd_init(void)
bsd_init_kprintf("calling devfs_kernel_mount\n");
devfs_kernel_mount(mounthere);
+ redwood_input_init(); /* /dev/redwood-touch */
+ redwood_audio_init(); /* /dev/redwood-audio */
++ redwood_virtio_blk_init(); /* /dev/rvblk0, a disk image of the host (QEMU) */
++#if APFS
++ /* mounts and walks /dev/rvblk0 once userland has /private/var up */
++ redwood_apfs_check_init();
++#endif
}
#endif /* DEVFS */
@@ -924,7 +935,7 @@ index cbaa0b221894..78315a2d54cf 100644
int eofflag;
uint64_t options;
diff --git a/bsd/vfs/vfs_conf.c b/bsd/vfs/vfs_conf.c
-index bae70cda3a96..444f414532f5 100644
+index bae70cda3a96..d7bf8c05b527 100644
--- a/bsd/vfs/vfs_conf.c
+++ b/bsd/vfs/vfs_conf.c
@@ -103,6 +103,20 @@ extern struct vfsops mockfs_vfsops;
@@ -960,7 +971,7 @@ index bae70cda3a96..444f414532f5 100644
};
int fstypenumstart = FT_BINDFS + 1;
-@@ -194,6 +211,67 @@ static struct vfstable vfstbllist[] = {
+@@ -194,6 +211,73 @@ static struct vfstable vfstbllist[] = {
},
#endif /* BINDFS */
@@ -999,7 +1010,13 @@ index bae70cda3a96..444f414532f5 100644
+ .vfc_next = NULL,
+ .vfc_reserved1 = 0,
+ .vfc_reserved2 = 0,
-+ .vfc_vfsflags = VFC_VFSLOCALARGS | VFC_VFS64BITREADY | VFC_VFSNATIVEXATTR,
++ /*
++ * Generic args, not local: a kernel mount of a filesystem marked
++ * VFC_VFSLOCALARGS is refused outright (mount_common), and this one has to be
++ * mountable from the kernel. So apfs_mount() takes the device's path as its
++ * mount data and opens it itself.
++ */
++ .vfc_vfsflags = VFC_VFSGENERICARGS | VFC_VFS64BITREADY | VFC_VFSNATIVEXATTR,
+ .vfc_descptr = NULL,
+ .vfc_descsize = 0,
+ .vfc_sysctl = NULL
@@ -1028,7 +1045,7 @@ index bae70cda3a96..444f414532f5 100644
#if MOCKFS
/* If we are configured for it, mockfs should always be the last standard entry (and thus the last FS we attempt mountroot with) */
{
-@@ -298,6 +376,18 @@ extern const struct vnodeopv_desc devfs_fdesc_vnodeop_opv_desc;
+@@ -298,6 +382,18 @@ extern const struct vnodeopv_desc devfs_fdesc_vnodeop_opv_desc;
extern const struct vnodeopv_desc mockfs_vnodeop_opv_desc;
#endif /* MOCKFS */
@@ -1047,7 +1064,7 @@ index bae70cda3a96..444f414532f5 100644
extern const struct vnodeopv_desc nullfs_vnodeop_opv_desc;
extern const struct vnodeopv_desc bindfs_vnodeop_opv_desc;
-@@ -327,5 +417,14 @@ const struct vnodeopv_desc *vfs_opv_descs[] = {
+@@ -327,5 +423,14 @@ const struct vnodeopv_desc *vfs_opv_descs[] = {
#if MOCKFS
&mockfs_vnodeop_opv_desc,
#endif /* MOCKFS */