Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C
17.2 KB
Raw Blame
/*
 * A virtio block device, so that a disk image on the host can be a block device in the
 * guest (/dev/rvblk0).  Only with boot-arg virtio_blk=1: the bus address is QEMU's and
 * means nothing on the phone.
 *
 * This exists for the APFS driver.  The volume in darwin165/rootfs.dmg is 5.6 GB, which is
 * too much to hold in guest memory as a RAM disk the way this port's own images are held, and the
 * point of reading APFS is to read a real image rather than one cut down to fit.  QEMU
 * serves it straight off the host's disk:
 *
 *   -drive file=rootfs.apfs,format=raw,if=none,id=d,readonly=on
 *   -device virtio-blk-device,drive=d
 *
 * Under QEMU's "virt" machine the transport is virtio-mmio and must be the current one,
 * not "legacy" (-global virtio-mmio.force-legacy=false).
 *
 * No interrupt is used and only one request is outstanding at a time: a request is put on
 * the queue, the device is notified, and the used ring is watched until it comes back.
 * That is slow in principle and entirely adequate here -- reads come from the APFS
 * driver's own block cache misses, and the host is serving them out of its page cache.
 *
 * Read only, unless the boot-arg is virtio_blk=2: then it writes too, for trying a file system that writes (HFS Plus,
 * the data volume) on a disk image before it is let near the phone's flash.  With 1 a write request is refused, which
 * is one less way to damage an image that is only to be read.
 */

#include <kern/clock.h>
#include <kern/locks.h>
#include <libkern/libkern.h>
#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <miscfs/devfs/devfs.h>
#include <pexpert/pexpert.h>
#include <sys/buf.h>
#include <sys/conf.h>
#include <sys/disk.h>
#include <sys/errno.h>
#include <sys/systm.h>
#include <sys/hakios.virtio_blk.h>
#include <vm/vm_kern.h>

extern vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);
extern uint64_t kvtophys(vm_offset_t va);

#define VIRTIO_BUS              0x0a000000UL    /* QEMU virt: 32 transports of 0x200 bytes */
#define VIRTIO_SLOTS            32
#define VIRTIO_SLOT_SIZE        0x200

#define VIRTIO_MAGIC            0x000           /* "virt" */
#define VIRTIO_VERSION          0x004
#define VIRTIO_DEVICE_ID        0x008
#define VIRTIO_DRIVER_FEATURES  0x020
#define VIRTIO_DRIVER_FEATURES_SEL 0x024
#define VIRTIO_QUEUE_SEL        0x030
#define VIRTIO_QUEUE_NUM_MAX    0x034
#define VIRTIO_QUEUE_NUM        0x038
#define VIRTIO_QUEUE_READY      0x044
#define VIRTIO_QUEUE_NOTIFY     0x050
#define VIRTIO_INTERRUPT_ACK    0x064
#define VIRTIO_STATUS           0x070
#define VIRTIO_QUEUE_DESC       0x080
#define VIRTIO_QUEUE_DRIVER     0x090
#define VIRTIO_QUEUE_DEVICE     0x0a0
#define VIRTIO_CONFIG           0x100           /* the device's own configuration */

#define VIRTIO_ID_BLOCK         2
#define STATUS_ACKNOWLEDGE      1
#define STATUS_DRIVER           2
#define STATUS_DRIVER_OK        4
#define STATUS_FEATURES_OK      8

#define QUEUE_SIZE              64
#define DESC_F_NEXT             1
#define DESC_F_WRITE            2
#define AVAIL_F_NO_INTERRUPT    1

/* the request types we use */
#define VIRTIO_BLK_T_IN         0
#define VIRTIO_BLK_T_OUT        1
#define VIRTIO_BLK_S_OK         0

#define VIRTIO_BLK_SECTOR       512             /* virtio-blk counts in these, always */

/*
 * The most one request may carry.  A request needs one descriptor for its header, one for
 * its status byte and one per page of the data, so this bounds the chain well inside the
 * queue.
 */
#define BLK_MAX_XFER            (64 * 1024)
#define BLK_MAX_DATA_DESC       (BLK_MAX_XFER / PAGE_SIZE + 1)

struct vq_desc {
	uint64_t addr;
	uint32_t len;
	uint16_t flags;
	uint16_t next;
};
struct vq_avail {
	uint16_t flags;
	uint16_t idx;
	uint16_t ring[QUEUE_SIZE];
};
struct vq_used {
	uint16_t flags;
	uint16_t idx;
	struct {
		uint32_t id;
		uint32_t len;
	} ring[QUEUE_SIZE];
};
struct blk_req_hdr {
	uint32_t type;
	uint32_t reserved;
	uint64_t sector;
};

/* the ring and the small request pieces share one page */
#define OFFSET_DESC             0x0000
#define OFFSET_AVAIL            0x0400
#define OFFSET_USED             0x0800
#define OFFSET_HDR              0x0c00
#define OFFSET_STATUS           0x0c40

struct blk_device {
	vm_offset_t          regs;
	vm_offset_t          page;
	uint64_t             page_phys;
	struct vq_desc      *desc;
	struct vq_avail     *avail;
	struct vq_used      *used;
	struct blk_req_hdr  *hdr;
	volatile uint8_t    *status;
	uint16_t             seen;      /* how far the used ring has been read */
	vm_offset_t          bounce;    /* BLK_MAX_XFER of wired memory the device reads and writes */
	uint64_t             bounce_phys[BLK_MAX_XFER / PAGE_MIN_SIZE];
	uint64_t             capacity;  /* in 512-byte sectors */
	lck_mtx_t            lock;
};

static LCK_GRP_DECLARE(blk_lck_grp, "hakios-virtio-blk");
static struct blk_device blk_dev;
static bool blk_ready;
static bool blk_writable;               /* virtio_blk=2 */
static int blk_bmajor = -1, blk_cmajor = -1;

/* One of this driver's devices: every request is a trip through the hypervisor */
bool
hakios_virtio_blk_dev(dev_t dev)
{
	return (blk_bmajor >= 0 && major(dev) == blk_bmajor) ||
	       (blk_cmajor >= 0 && major(dev) == blk_cmajor);
}

static inline uint32_t
reg_read(struct blk_device *d, uint32_t reg)
{
	return *(volatile uint32_t *)(d->regs + reg);
}

static inline void
reg_write(struct blk_device *d, uint32_t reg, uint32_t value)
{
	*(volatile uint32_t *)(d->regs + reg) = value;
}

static inline void
reg_write64(struct blk_device *d, uint32_t reg, uint64_t value)
{
	reg_write(d, reg, (uint32_t)value);
	reg_write(d, reg + 4, (uint32_t)(value >> 32));
}

/*
 * One read, or one write.  The device works on the driver's own wired pages (d->bounce), and `data` is copied in or
 * out: a file system's buffer is not always memory whose physical address kvtophys() can give (a write after a
 * remount had QEMU refuse the descriptor), and a copy of 64 KB costs nothing next to the trip to the host.
 */
static int
blk_io(struct blk_device *d, bool write, uint64_t offset, void *data, size_t len)
{
	uint16_t idx = 0;
	size_t done = 0;
	uint64_t deadline;
	int error = 0;

	if (len == 0 || len > BLK_MAX_XFER) {
		return EINVAL;
	}
	if ((offset % VIRTIO_BLK_SECTOR) != 0) {
		return EINVAL;
	}
	if (offset / VIRTIO_BLK_SECTOR >= d->capacity ||
	    (offset + len) / VIRTIO_BLK_SECTOR > d->capacity) {
		return EINVAL;
	}

	lck_mtx_lock(&d->lock);
	/* a request that waited for the lock while an earlier one timed out must not start on a queue left in that state */
	if (!blk_ready) {
		lck_mtx_unlock(&d->lock);
		return ENXIO;
	}

	d->hdr->type = write ? VIRTIO_BLK_T_OUT : VIRTIO_BLK_T_IN;
	d->hdr->reserved = 0;
	d->hdr->sector = offset / VIRTIO_BLK_SECTOR;
	*d->status = 0xff;

	/* descriptor 0: the header, which the device reads */
	d->desc[0].addr = d->page_phys + OFFSET_HDR;
	d->desc[0].len = sizeof(struct blk_req_hdr);
	d->desc[0].flags = DESC_F_NEXT;
	d->desc[0].next = 1;
	idx = 1;

	/* then the data, one descriptor per page of the bounce buffer */
	if (write) {
		bcopy(data, (void *)d->bounce, len);
	}
	while (done < len) {
		size_t chunk = MIN(PAGE_SIZE, len - done);

		d->desc[idx].addr = d->bounce_phys[done / PAGE_SIZE];
		d->desc[idx].len = (uint32_t)chunk;
		d->desc[idx].flags = (write ? 0 : DESC_F_WRITE) | DESC_F_NEXT;
		d->desc[idx].next = (uint16_t)(idx + 1);
		idx++;
		done += chunk;
	}

	/* last: the status byte, which the device writes */
	d->desc[idx].addr = d->page_phys + OFFSET_STATUS;
	d->desc[idx].len = 1;
	d->desc[idx].flags = DESC_F_WRITE;
	d->desc[idx].next = 0;

	d->avail->flags = AVAIL_F_NO_INTERRUPT;
	d->avail->ring[d->avail->idx % QUEUE_SIZE] = 0;         /* the chain starts at 0 */
	__builtin_arm_dmb(0xb);
	d->avail->idx++;
	__builtin_arm_dmb(0xb);
	reg_write(d, VIRTIO_QUEUE_NOTIFY, 0);

	/* wait for it to come back; a read off the host's disk should take microseconds */
	clock_interval_to_deadline(10, NSEC_PER_SEC, &deadline);
	for (;;) {
		__builtin_arm_dmb(0xb);
		if (d->used->idx != d->seen) {
			d->seen = d->used->idx;
			break;
		}
		if (mach_absolute_time() > deadline) {
			printf("virtio-blk: a %s of %lu bytes at %llu went unanswered\n",
			    write ? "write" : "read", (unsigned long)len, offset);
			/*
			 * The request stays published on the queue and the caller may reuse its buffer: a late
			 * answer would land in it, and the next request would take this stale completion as
			 * its own. So the device is off from now on.
			 */
			blk_ready = false;
			lck_mtx_unlock(&d->lock);
			return EIO;
		}
		delay(10);
	}
	reg_write(d, VIRTIO_INTERRUPT_ACK, reg_read(d, 0x060));  /* keep the device quiet */

	if (*d->status != VIRTIO_BLK_S_OK) {
		printf("virtio-blk: a %s of %lu bytes at %llu failed with status %u\n",
		    write ? "write" : "read", (unsigned long)len, offset, *d->status);
		error = EIO;
	} else if (!write) {
		bcopy((void *)d->bounce, data, len);
	}
	lck_mtx_unlock(&d->lock);
	return error;
}

/* ---- the block device ---- */

static int
blk_dev_open(__unused dev_t dev, int flags, __unused int devtype, __unused proc_t p)
{
	if (!blk_ready) {
		return ENXIO;
	}
	if ((flags & FWRITE) && !blk_writable) {
		return EROFS;
	}
	return 0;
}

static int
blk_dev_close(__unused dev_t dev, __unused int flags, __unused int devtype,
    __unused proc_t p)
{
	return 0;
}

static void
blk_dev_strategy(buf_t bp)
{
	struct blk_device *d = &blk_dev;
	uint64_t offset;
	size_t len;
	int error;

	if (!blk_ready) {
		buf_seterror(bp, ENXIO);
		buf_biodone(bp);
		return;
	}
	if (!(buf_flags(bp) & B_READ) && !blk_writable) {
		buf_seterror(bp, EROFS);         /* this device writes only with virtio_blk=2 */
		buf_biodone(bp);
		return;
	}
	offset = (uint64_t)buf_blkno(bp) * VIRTIO_BLK_SECTOR;
	len = buf_count(bp);

	/* trim a request that runs off the end, and refuse one that starts past it */
	if (offset >= d->capacity * VIRTIO_BLK_SECTOR) {
		buf_seterror(bp, offset == d->capacity * VIRTIO_BLK_SECTOR ? 0 : EINVAL);
		buf_setresid(bp, buf_count(bp));
		buf_biodone(bp);
		return;
	}
	if (offset + len > d->capacity * VIRTIO_BLK_SECTOR) {
		len = (size_t)(d->capacity * VIRTIO_BLK_SECTOR - offset);
	}

	/*
	 * A file system's cluster I/O (HFS Plus's) hands over a buffer of pages that has no kernel address yet:
	 * buf_dataptr() of it is an offset, and buf_map() is what gives it one.
	 */
	caddr_t mapped;
	if ((error = buf_map(bp, &mapped)) != 0) {
		buf_seterror(bp, error);
		buf_biodone(bp);
		return;
	}
	{
		uint8_t *dst = (uint8_t *)mapped;
		size_t done = 0;

		while (done < len && error == 0) {
			size_t chunk = len - done;

			if (chunk > BLK_MAX_XFER) {
				chunk = BLK_MAX_XFER;
			}
			error = blk_io(d, !(buf_flags(bp) & B_READ), offset + done, dst + done, chunk);
			done += chunk;
		}
		buf_setresid(bp, (uint32_t)(buf_count(bp) - (error ? 0 : done)));
	}
	buf_unmap(bp);
	if (error) {
		buf_seterror(bp, error);
	}
	buf_biodone(bp);
}

static int
blk_dev_ioctl(__unused dev_t dev, u_long cmd, caddr_t data, __unused int fflag,
    __unused proc_t p)
{
	struct blk_device *d = &blk_dev;

	if (!blk_ready) {
		return ENXIO;
	}
	switch (cmd) {
	case DKIOCGETBLOCKSIZE:
		*(uint32_t *)data = VIRTIO_BLK_SECTOR;
		return 0;
	case DKIOCSETBLOCKSIZE:
		/*
		 * spec_open() sets the block size to 512 to count the device in 512-byte
		 * blocks and then sets it back, and treats a refusal of either as the device
		 * being broken (ENXIO).  virtio-blk counts in 512-byte sectors and nothing
		 * else, so that is the only size to agree to.
		 */
		if (*(uint32_t *)data != VIRTIO_BLK_SECTOR) {
			return EINVAL;
		}
		return 0;
	case DKIOCGETBLOCKCOUNT:
		*(uint64_t *)data = d->capacity;
		return 0;
	case DKIOCISWRITABLE:
		*(uint32_t *)data = blk_writable;
		return 0;
	case DKIOCGETMAXBLOCKCOUNTREAD:
	case DKIOCGETMAXBLOCKCOUNTWRITE:
		*(uint64_t *)data = BLK_MAX_XFER / VIRTIO_BLK_SECTOR;
		return 0;
	case DKIOCSYNCHRONIZE:
	case DKIOCSYNCHRONIZECACHE:
		/* no VIRTIO_BLK_F_FLUSH is taken, so the device writes through: a write that came back is on the disk */
		return 0;
	default:
		return ENOTTY;
	}
}

static int
blk_dev_size(dev_t dev)
{
	(void)dev;
	return blk_ready ? VIRTIO_BLK_SECTOR : -1;
}

static int
blk_dev_read(__unused dev_t dev, struct uio *uio, __unused int ioflag)
{
	/* the character device is here so that tools can see the disk; reads go by buffer */
	return physio(blk_dev_strategy, NULL, dev, B_READ, (u_int (*)(buf_t))blk_dev_size,
	    uio, VIRTIO_BLK_SECTOR);
}

static int
blk_dev_write(__unused dev_t dev, struct uio *uio, __unused int ioflag)
{
	return physio(blk_dev_strategy, NULL, dev, B_WRITE, (u_int (*)(buf_t))blk_dev_size,
	    uio, VIRTIO_BLK_SECTOR);
}

static const struct bdevsw blk_bdevsw = {
	.d_open     = blk_dev_open,
	.d_close    = blk_dev_close,
	.d_strategy = blk_dev_strategy,
	.d_ioctl    = blk_dev_ioctl,
	.d_dump     = eno_dump,
	.d_psize    = blk_dev_size,
	.d_type     = D_DISK,
};

static const struct cdevsw blk_cdevsw = {
	.d_open       = blk_dev_open,
	.d_close      = blk_dev_close,
	.d_read       = blk_dev_read,
	.d_write      = blk_dev_write,
	.d_ioctl      = blk_dev_ioctl,
	.d_stop       = eno_stop,
	.d_reset      = eno_reset,
	.d_ttys       = NULL,
	.d_select     = eno_select,
	.d_mmap       = eno_mmap,
	.d_strategy   = eno_strat,
	.d_reserved_1 = eno_getc,
	.d_reserved_2 = eno_putc,
	.d_type       = D_DISK,
};

/* ---- bringing it up ---- */

static bool
blk_attach(struct blk_device *d, int slot)
{
	uint32_t cap_lo, cap_hi;

	if (kmem_alloc(kernel_map, &d->page, PAGE_SIZE,
	    KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
		return false;
	}
	d->page_phys = kvtophys(d->page);
	d->desc = (struct vq_desc *)(d->page + OFFSET_DESC);
	d->avail = (struct vq_avail *)(d->page + OFFSET_AVAIL);
	d->used = (struct vq_used *)(d->page + OFFSET_USED);
	d->hdr = (struct blk_req_hdr *)(d->page + OFFSET_HDR);
	d->status = (volatile uint8_t *)(d->page + OFFSET_STATUS);
	if (kmem_alloc(kernel_map, &d->bounce, BLK_MAX_XFER,
	    KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
		return false;
	}
	for (int i = 0; i < BLK_MAX_XFER / PAGE_SIZE; i++) {
		d->bounce_phys[i] = kvtophys(d->bounce + (vm_offset_t)i * PAGE_SIZE);
	}

	reg_write(d, VIRTIO_STATUS, 0);         /* reset */
	reg_write(d, VIRTIO_STATUS, STATUS_ACKNOWLEDGE);
	reg_write(d, VIRTIO_STATUS, STATUS_ACKNOWLEDGE | STATUS_DRIVER);
	/* nothing in the low half is wanted; bit 32 is VIRTIO_F_VERSION_1 */
	reg_write(d, VIRTIO_DRIVER_FEATURES_SEL, 0);
	reg_write(d, VIRTIO_DRIVER_FEATURES, 0);
	reg_write(d, VIRTIO_DRIVER_FEATURES_SEL, 1);
	reg_write(d, VIRTIO_DRIVER_FEATURES, 1);
	reg_write(d, VIRTIO_STATUS, STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK);
	if (!(reg_read(d, VIRTIO_STATUS) & STATUS_FEATURES_OK)) {
		printf("virtio-blk: slot %d: features refused\n", slot);
		return false;
	}

	cap_lo = reg_read(d, VIRTIO_CONFIG);
	cap_hi = reg_read(d, VIRTIO_CONFIG + 4);
	d->capacity = ((uint64_t)cap_hi << 32) | cap_lo;
	if (d->capacity == 0) {
		printf("virtio-blk: slot %d: the disk is empty\n", slot);
		return false;
	}

	reg_write(d, VIRTIO_QUEUE_SEL, 0);
	if (reg_read(d, VIRTIO_QUEUE_NUM_MAX) < QUEUE_SIZE) {
		printf("virtio-blk: slot %d: request queue of %u\n", slot,
		    reg_read(d, VIRTIO_QUEUE_NUM_MAX));
		return false;
	}
	reg_write(d, VIRTIO_QUEUE_NUM, QUEUE_SIZE);
	reg_write64(d, VIRTIO_QUEUE_DESC, d->page_phys + OFFSET_DESC);
	reg_write64(d, VIRTIO_QUEUE_DRIVER, d->page_phys + OFFSET_AVAIL);
	reg_write64(d, VIRTIO_QUEUE_DEVICE, d->page_phys + OFFSET_USED);
	__builtin_arm_dmb(0xb);
	reg_write(d, VIRTIO_QUEUE_READY, 1);
	reg_write(d, VIRTIO_STATUS,
	    STATUS_ACKNOWLEDGE | STATUS_DRIVER | STATUS_FEATURES_OK | STATUS_DRIVER_OK);
	return true;
}

void
hakios_virtio_blk_init(void)
{
	uint32_t enabled = 0;
	vm_offset_t bus;
	struct blk_device *d = &blk_dev;
	int slot;

	if (!PE_parse_boot_argn("virtio_blk", &enabled, sizeof(enabled)) || !enabled) {
		return;
	}
	blk_writable = enabled == 2;
	bus = ml_io_map(VIRTIO_BUS, VIRTIO_SLOTS * VIRTIO_SLOT_SIZE);
	for (slot = 0; slot < VIRTIO_SLOTS; slot++) {
		d->regs = bus + (vm_offset_t)slot * VIRTIO_SLOT_SIZE;
		if (reg_read(d, VIRTIO_MAGIC) != 0x74726976 ||
		    reg_read(d, VIRTIO_DEVICE_ID) != VIRTIO_ID_BLOCK) {
			continue;
		}
		if (reg_read(d, VIRTIO_VERSION) != 2) {
			printf("virtio-blk: slot %d is a legacy transport; start QEMU with "
			    "-global virtio-mmio.force-legacy=false\n", slot);
			continue;
		}
		lck_mtx_init(&d->lock, &blk_lck_grp, LCK_ATTR_NULL);
		if (!blk_attach(d, slot)) {
			lck_mtx_destroy(&d->lock, &blk_lck_grp);
			continue;
		}
		blk_bmajor = bdevsw_add(-1, &blk_bdevsw);
		if (blk_bmajor < 0) {
			printf("virtio-blk: bdevsw_add failed\n");
			return;
		}
		blk_cmajor = cdevsw_add_with_bdev(-1, &blk_cdevsw, blk_bmajor);
		if (blk_cmajor < 0) {
			printf("virtio-blk: cdevsw_add failed\n");
			return;
		}
		if (devfs_make_node(makedev(blk_bmajor, 0), DEVFS_BLOCK, UID_ROOT,
		    GID_OPERATOR, 0600, "rvblk0") == NULL ||
		    devfs_make_node(makedev(blk_cmajor, 0), DEVFS_CHAR, UID_ROOT,
		    GID_OPERATOR, 0600, "rrvblk0") == NULL) {
			printf("virtio-blk: devfs_make_node failed\n");
			return;
		}
		blk_ready = true;
		printf("virtio-blk: /dev/rvblk0 in slot %d, %llu sectors of %u (%llu MB)%s\n",
		    slot, d->capacity, VIRTIO_BLK_SECTOR,
		    d->capacity * VIRTIO_BLK_SECTOR / (1024 * 1024), blk_writable ? ", writable" : "");
		return;
	}
	printf("virtio-blk: no block device on the bus\n");
}