Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C
245 KB
Raw Blame
/*
 * The modem of the POCO X5 Pro (redwood), started, and asked about the network: step 1 of
 * cellular. Every address below belongs to this phone, and nothing here is touched unless a
 * boot-arg asks for it.
 *
 * The modem ("MPSS", Android's "modem") is a Hexagon of its own inside the SoC, exactly as the
 * Wi-Fi's and the audio's are (hakios.wifi.c, hakios.adsp.c): it stays in reset until somebody
 * loads its signed image -- modem.mdt and modem.b00 to modem.b29 from /vendor/firmware_mnt/image
 * of its Android, 85 MB, in the system image under /fw (MODEM=1 SYSTEM=1 userland/build.sh, from
 * fw-modem/) -- into the 246 MB reserved for it and asks TrustZone to authenticate and start it.
 * From there everything is conversation through the shared memory: SMP2P for "up", "given up",
 * "stopped"; GLINK for channels; QRTR on the channel "IPCRTR" for the modem's services, which
 * speak QMI (drivers/modem/hakios.qmi.c knows the protocol; here only two of its messages are made by
 * hand): the device management service is asked to put the modem online, and the network access
 * service what network it is on and how strong the signal is (drivers/modem/hakios.nas.c). What this
 * learns goes to sysctl kern.hakios.cellular_* for the rest of the system; calls, SMS and data
 * are for the steps after this one.
 *
 * The way is the modem's, which is the Wi-Fi DSP's: hakios.adsp.c's comments say why each
 * thing is done so; this file is that one with the modem's numbers (dt: qcom,mss@4080000,
 * qcom,smp2p-modem, qcom,glink/modem, reserved-memory mpss@8b800000):
 *   - processor 1 in the shared memory; its SMP2P items 435 (its) and 428 (ours); its last words
 *     in item 421; GLINK's three items in our partition with it
 *   - its doorbells are IPCC client 2 (MPSS): signal 0 GLINK, signal 2 SMP2P
 *   - TrustZone knows it as image 4; its memory is 0x8b800000, 246 MB, of which the image takes
 *     the first 85 (TrustZone is told of all of it: the modem uses the rest)
 *   - its rails are CX and MSS (cx.lvl and mss.lvl), asked for at level 384 while it boots
 *   - the always-on processor is told of it as "modem"
 *   - the channel is IPCRTR only; the modem's others (DS, DATA*, glink_ssr) are left unanswered
 *   - the ports of QRTR are the Wi-Fi driver's (sys/hakios.qrtr.h): what the NAS client needs
 * The modem and the Wi-Fi DSP may run at the same time: the mailbox to the always-on
 * processor and the lock for allocating in the shared memory are shared, each behind a lock.
 *
 * Boot-arg modem_step=N performs steps 1 to N and stops; modem=1 is all of them, on a thread
 * beside the system's start, and the modem stays on, its network state asked every two seconds.
 *
 *   1  reads   the shared memory: our partition with the modem, its items, whether it has run
 *   2  reads   the firmware on the root (or in the small image): is it whole, does it fit
 *   3  writes  the always-on processor's mailbox open, unless it is
 *   4  asks    TrustZone to take the image's header (image 4) and where the image will be; a
 *              modem that a kernel before this one left running is asked to stop and shut down first
 *   5  writes  the image into the modem's memory, and reads it back
 *   6  writes  our SMP2P item (428) into the shared memory, and our two marks of GLINK to zero
 *   7  asks    RPMh for the modem's rails, writes "modem: on" to the always-on processor, asks
 *              TrustZone to authenticate and start the modem, and watches for its first words
 *   8  reads   GLINK's queues
 *   9  writes  our side of GLINK: the versions, the channel "IPCRTR" opened both ways, our intents;
 *              from here a thread of ours listens to the modem
 *  10  talks   QRTR on that channel: hello both ways, and the services the modem tells of
 *  11  talks   QMI: DMS "operating mode online" (modem_online=0: not), then NAS "serving system"
 *              and "signal info" every second for modem_nas_wait seconds (default 30), or until
 *              the modem says it is registered and how strong the signal is
 *
 * What is started is stopped: when the steps are over (unless modem_keep=1, and then when the run
 * ends), and on a panic's way back to the loader. More boot-args: modem_new=K, modem_pause=MS,
 * modem_wait=S (step 7's watch, default 10), modem_listen=S (step 10, default 3), modem_again=1,
 * modem_return=1, as the modem's.
 *
 * Nothing here writes a clock or changes a pin. One request goes to RPMh (the two rails), as
 * Linux's remoteproc makes it; the SIM is not touched, no call is made, no data session opened.
 */
#include <kern/clock.h>
#include <kern/kalloc.h>
#include <kern/locks.h>
#include <kern/thread.h>
#include <libkern/libkern.h>
#include <os/atomic_private.h>
#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <vm/memory_types.h>
#include <pexpert/pexpert.h>
#include <sys/errno.h>
#include <sys/kauth.h>
#include <sys/stat.h>
#include <sys/systm.h>
#include <sys/vnode.h>
#include <sys/vnode_internal.h>
#include <vm/vm_kern.h>
#include <sys/hakios.input.h>
#include <sys/hakios.ipa.h>
#include <sys/hakios.nas.h>
#include <sys/hakios.qrtr.h>
#include <sys/sysctl.h>

extern vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);
extern vm_offset_t ml_io_map_unmappable(vm_offset_t phys_addr, vm_size_t size, unsigned int flags);
extern void ml_io_unmap(vm_offset_t addr, vm_size_t sz);
extern uint64_t kvtophys(vm_offset_t va);
extern void loader_return(void);
extern kern_return_t thread_terminate(thread_t thread);
extern vm_offset_t pe_arm_get_soc_base_phys(void);
extern dev_t mdevlookup(int devid);
extern int mdevgetkernelrange(dev_t dev, vm_offset_t *base, uint64_t *size);
extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length);   /* osfmk/arm64/caches_asm.s */

/* What the Wi-Fi's DSP and this one both use (hakios.wifi.c). */
extern lck_mtx_t hakios_aop_mailbox_lock;
extern lck_mtx_t hakios_smem_alloc_lock;
extern lck_mtx_t hakios_rsc_lock;            /* RPMh's sequencer, shared with the other DSP's driver */

static LCK_GRP_DECLARE(mpss_lock_group, "hakios-mpss");

/* the modem's file systems, served from here (rmtfs, below) */
static bool rmtfs_buffer_make(void);
static bool ipa_firmware_load(void);
static void ipa_firmware_stop(void);
static bool ipa_loaded;                 /* the IPA's firmware runs (ipa_firmware_load()) */
static uint32_t ipa_mem_offset;          /* where the IPA's local memory begins, as its register says */
static bool rmtfs_share(bool in);
static bool rmtfs_start(void);
static void rmtfs_stop(void);

#define STEPS                   11
#define HAKIOS_SOC_BASE        0x00400000ul    /* where the loader says the phone's peripherals begin */

static uint32_t last_step, pause_ms = 2200, listen_seconds = 3;
static uint32_t step;                   /* the one being performed */
static uint32_t new_from;               /* the first of the steps that have not been seen to work: they wait */
static char last_write[96] = "nothing written";
static bool leaving;                    /* on the way back to the loader after a panic: no lock is taken, nothing sleeps */

/*
 * Every line goes to the log the loader reads from another core as well (as in hakios.wifi.c);
 * with modem=1 only the few that say how it went (tell()), the rest to the kernel's own log.
 */
extern void loader_log_putc(char c);

static LCK_MTX_DECLARE(say_lock, &mpss_lock_group);
static bool quiet;

static void
say_line(const char *line, bool aloud)
{
	if (leaving) {
		for (const char *c = line; *c != 0; c++) {
			loader_log_putc(*c);
		}
		return;
	}
	lck_mtx_lock(&say_lock);
	if (!quiet || aloud) {
		for (const char *c = line; *c != 0; c++) {
			loader_log_putc(*c);
		}
	}
	printf("%s", line);
	lck_mtx_unlock(&say_lock);
}

static int say(const char *format, ...) __printflike(1, 2);
static int tell(const char *format, ...) __printflike(1, 2);

static int
say(const char *format, ...)
{
	char line[400];
	va_list list;
	int length;

	va_start(list, format);
	length = vsnprintf(line, sizeof(line), format, list);
	va_end(list);
	say_line(line, false);
	return length;
}

static int
tell(const char *format, ...)
{
	char line[400];
	va_list list;
	int length;

	va_start(list, format);
	length = vsnprintf(line, sizeof(line), format, list);
	va_end(list);
	say_line(line, true);
	return length;
}
#define printf say

static void
sleep_us(uint32_t us)
{
	delay_for_interval(us, NSEC_PER_USEC);
}

static void
sleep_ms(uint32_t ms)
{
	delay_for_interval(ms, NSEC_PER_MSEC);
}

static uint64_t
now_us(void)
{
	uint64_t nanoseconds;

	absolutetime_to_nanoseconds(mach_absolute_time(), &nanoseconds);
	return nanoseconds / 1000;
}

/* Until every write to memory so far has been carried out. */
static void
settle(void)
{
	__asm__ volatile ("dsb sy");
}

static bool
option(const char *name)
{
	uint32_t value = 0;

	return PE_parse_boot_argn(name, &value, sizeof(value)) && value != 0;
}

/* ---- register access (as in hakios.wifi.c) ------------------------------------------------------- */

static uint32_t __attribute__((noinline))
rd(vm_offset_t address)
{
	uint32_t value;

	__asm__ volatile ("ldr %w0, [%1]" : "=r" (value) : "r" (address) : "memory");
	return value;
}

static void __attribute__((noinline))
wr(vm_offset_t address, uint32_t value)
{
	__asm__ volatile ("str %w0, [%1]" : : "r" (value), "r" (address) : "memory");
}

struct block {
	const char *name;
	uint32_t phys, size;
	vm_offset_t base;
};

static void
need(struct block *b)
{
	if (b->base == 0) {
		b->base = ml_io_map(b->phys, b->size);
	}
}

/* Time for the loader to note what has just been said, where it is not yet known to be harmless. */
static void
hold(void)
{
	if (step >= new_from && pause_ms && !leaving) {
		sleep_ms(pause_ms);
	}
}

static void
announce(const char *what)
{
	printf("mpss: step %u: %s\n", step, what);
	hold();
}

static uint32_t
get(struct block *b, const char *name, uint32_t offset)
{
	uint32_t value;

	need(b);
	value = rd(b->base + offset);
	printf("mpss: %u: %s %s (%#x) = %#x\n", step, b->name, name, b->phys + offset, value);
	return value;
}

static void
put(struct block *b, const char *name, uint32_t offset, uint32_t value)
{
	need(b);
	printf("mpss: %u: about to write %s %s at %#x: %#x\n", step, b->name, name, b->phys + offset, value);
	snprintf(last_write, sizeof(last_write), "step %u wrote %#x to %s %s at %#x", step, value, b->name, name, b->phys + offset);
	wr(b->base + offset, value);
}

/* ---- the shared memory (hakios.wifi.c, step 1) --------------------------------------------------- */

#define SMEM_PHYS_DEFAULT       0x80900000u   /* dt: reg of /soc/qcom,smem */
static uint32_t smem_phys = SMEM_PHYS_DEFAULT;
#define SMEM_SIZE               0x00200000u
#define SMEM_US                 0
#define SMEM_MPSS               1               /* dt: qcom,remote-pid of qcom,smp2p-modem and qcom,glink/modem */
#define SMEM_GLOBAL             0xfffe
#define SMEM_CANARY             0xa5a5

struct smem_table_entry {
	uint32_t offset, size, flags;
	uint16_t host0, host1;
	uint32_t cacheline, reserved[7];
};
struct smem_table {
	uint8_t magic[4];
	uint32_t version, count, reserved[5];
};
struct smem_partition {
	uint8_t magic[4];
	uint16_t host0, host1;
	uint32_t size, free_uncached, free_cached, reserved[3];
};
struct smem_entry {
	uint16_t canary, item;
	uint32_t size;
	uint16_t padding_data, padding_header;
	uint32_t reserved;
};
struct smem_item {
	uint32_t item, offset, size;
};

/*
 * The DSP's items (dt: qcom,smem of qcom,smp2p-modem, <435 428>: the item the DSP writes first, then
 * ours -- the Wi-Fi's <617 616> are in the same order; taken the other way round, the DSP made
 * our item its own and never said "up" --, and qcom,smem-id of qcom,mss@4080000).
 */
#define ITEM_SMP2P_OURS         428
#define ITEM_SMP2P_THEIRS       435
#define ITEM_CRASH_REASON       421

static const volatile uint8_t *smem;

static bool
smem_peek(uint64_t offset, void *to, uint32_t size)
{
	if (smem == NULL) {
		smem = (const volatile uint8_t *)ml_io_map_unmappable(smem_phys, SMEM_SIZE, VM_WIMG_WCOMB);
	}
	if (offset + size > SMEM_SIZE) {
		return false;
	}
	for (uint32_t i = 0; i < size; i++) {
		((uint8_t *)to)[i] = smem[offset + i];
	}
	return true;
}

static void
smem_poke(uint32_t offset, uint32_t value)
{
	wr((vm_offset_t)smem + offset, value);
}

static uint32_t
smem_word(uint32_t offset)
{
	return rd((vm_offset_t)smem + offset);
}

static uint32_t
smem_partition(uint16_t one, uint16_t other, uint32_t *cacheline, uint32_t *size)
{
	struct smem_table table;
	struct smem_table_entry entry;
	struct smem_partition header;

	if (!smem_peek(SMEM_SIZE - 0x1000, &table, sizeof(table)) || memcmp(table.magic, "$TOC", 4) != 0 || table.version != 1 ||
	    table.count > 64) {
		return 0;
	}
	for (uint32_t i = 0; i < table.count; i++) {
		if (!smem_peek(SMEM_SIZE - 0x1000 + sizeof(table) + i * sizeof(entry), &entry, sizeof(entry)) || entry.offset == 0 ||
		    entry.size == 0) {
			continue;
		}
		if (!((entry.host0 == one && entry.host1 == other) || (entry.host0 == other && entry.host1 == one))) {
			continue;
		}
		if ((uint64_t)entry.offset + entry.size > SMEM_SIZE - 0x1000 || !smem_peek(entry.offset, &header, sizeof(header)) ||
		    memcmp(header.magic, "$PRT", 4) != 0 || header.size != entry.size) {
			return 0;
		}
		*cacheline = entry.cacheline;
		*size = entry.size;
		return entry.offset;
	}
	return 0;
}

static unsigned
smem_items(uint32_t partition, uint32_t cacheline, struct smem_item *found, unsigned max)
{
	struct smem_partition header;
	struct smem_entry entry;
	uint32_t at, step_back;
	unsigned count = 0;

	if (partition == 0 || !smem_peek(partition, &header, sizeof(header)) || header.free_uncached > header.size ||
	    header.free_cached > header.size) {
		return 0;
	}
	for (at = sizeof(header); at + sizeof(entry) <= header.free_uncached;) {
		if (!smem_peek(partition + at, &entry, sizeof(entry)) || entry.canary != SMEM_CANARY || entry.size > header.size) {
			break;
		}
		if (count < max) {
			found[count].item = entry.item;
			found[count].offset = partition + at + sizeof(entry) + entry.padding_header;
			found[count].size = entry.size - entry.padding_data;
		}
		count++;
		at += sizeof(entry) + entry.padding_header + entry.size;
	}
	if (cacheline == 0 || (cacheline & (cacheline - 1))) {
		return count;
	}
	step_back = ((uint32_t)sizeof(entry) + cacheline - 1) & ~(cacheline - 1);
	for (at = header.size - step_back; at > header.free_cached && at < header.size;) {
		if (!smem_peek(partition + at, &entry, sizeof(entry)) || entry.canary != SMEM_CANARY || entry.size > at) {
			break;
		}
		if (count < max) {
			found[count].item = entry.item;
			found[count].offset = partition + at - entry.size;
			found[count].size = entry.size - entry.padding_data;
		}
		count++;
		if (entry.size + step_back > at) {
			break;
		}
		at -= entry.size + step_back;
	}
	return count;
}

/* An item the DSP and we share: in the partition of the two, or else in the global one. Where it is; 0: nowhere. */
static uint32_t
smem_item(uint32_t item, uint32_t *size)
{
	static struct smem_item found[256];
	static const uint16_t hosts[2][2] = { { SMEM_US, SMEM_MPSS }, { SMEM_GLOBAL, SMEM_GLOBAL } };

	for (unsigned h = 0; h < 2; h++) {
		uint32_t cacheline = 0, partition_size = 0;
		uint32_t partition = smem_partition(hosts[h][0], hosts[h][1], &cacheline, &partition_size);
		unsigned count = smem_items(partition, cacheline, found, 256);

		for (unsigned i = 0; i < count && i < 256; i++) {
			if (found[i].item == item) {
				*size = found[i].size;
				return found[i].offset;
			}
		}
	}
	return 0;
}

static void
show_partition(void)
{
	static struct smem_item found[256];
	struct smem_partition header;
	uint32_t cacheline = 0, size = 0, partition = smem_partition(SMEM_US, SMEM_MPSS, &cacheline, &size);
	unsigned count;
	char line[320];

	if (partition == 0 || !smem_peek(partition, &header, sizeof(header))) {
		printf("mpss: %u: there is no partition for us and the modem (0 and 1)\n", step);
		return;
	}
	count = smem_items(partition, cacheline, found, 256);
	printf("mpss: %u: the partition for us and the modem (0 and 1): at %#lx, %#x bytes, of which %#x free; %u items\n", step,
	    smem_phys + partition, size, header.free_cached - header.free_uncached, count);
	for (unsigned i = 0; i < count && i < 256;) {
		int at = 0;

		for (unsigned n = 0; n < 12 && i < count && i < 256; n++, i++) {
			at += snprintf(line + at, sizeof(line) - (size_t)at, " %u (%u)", found[i].item, found[i].size);
		}
		printf("mpss: %u:   items (bytes):%s\n", step, line);
	}
}

/* ---- SMP2P ---------------------------------------------------------------------------------------- */

struct smp2p {
	uint32_t magic;
	uint32_t version;
	uint16_t local, remote, total, valid;
	uint32_t flags;
	struct {
		char name[16];
		uint32_t value;
	} entry[16];
};
#define SMP2P_MAGIC             0x504d5324
#define SMP2P_VERSION           1
#define SMP2P_RESTART_ACKS      1u
#define SMP2P_RESTART_DONE      (1u << 0)
#define SMP2P_RESTART_ACK       (1u << 1)
#define SMP2P_AT_VERSION        4
#define SMP2P_AT_PROCESSORS     8
#define SMP2P_AT_COUNTS         12
#define SMP2P_AT_FLAGS          16
#define SMP2P_AT_NAME(i)        (20 + 20 * (i))
#define SMP2P_AT_VALUE(i)       (36 + 20 * (i))

/* The bits of the DSP's entry "slave-kernel" (dt: the interrupts of qcom,lpass@3700000). */
#define DSP_FATAL               (1u << 0)
#define DSP_READY               (1u << 1)
#define DSP_HANDOVER            (1u << 2)
#define DSP_STOP_ACK            (1u << 3)
#define DSP_SHUTDOWN_ACK        (1u << 7)

/* Our entry (dt: the child "master-kernel" of qcom,smp2p-adsp; "qcom,force-stop" of qcom,lpass@3700000). */
#define OUR_ENTRY_STOP          "master-kernel"
#define DSP_STOP                (1u << 0)

static bool
dsp_words(struct smp2p *words)
{
	static uint32_t at;
	uint32_t size = 0;

	if (at == 0) {
		at = smem_item(ITEM_SMP2P_THEIRS, &size);
		if (at == 0 || size < sizeof(*words)) {
			at = 0;
			return false;
		}
	}
	return smem_peek(at, words, sizeof(*words)) && words->magic == SMP2P_MAGIC;
}

static int
smp2p_entry(const struct smp2p *words, const char *name)
{
	for (unsigned i = 0; i < words->valid && i < 16; i++) {
		if (strncmp(words->entry[i].name, name, sizeof(words->entry[i].name)) == 0) {
			return (int)i;
		}
	}
	return -1;
}

static uint32_t
dsp_state(const struct smp2p *words)
{
	int entry = smp2p_entry(words, "slave-kernel");

	return entry < 0 ? 0 : words->entry[entry].value;
}

static void
show_words(const char *prefix, const struct smp2p *words)
{
	char line[320];
	int at = 0;

	line[0] = 0;
	for (unsigned i = 0; i < words->valid && i < 16 && at < (int)sizeof(line) - 40; i++) {
		at += snprintf(line + at, sizeof(line) - (size_t)at, " %.16s = %#x;", words->entry[i].name, words->entry[i].value);
	}
	printf("mpss: %u: %s: SMP2P version %u, features %#x, from processor %u to %u, flags %#x, %u of %u entries:%s\n", step, prefix,
	    words->version & 0xff, words->version >> 8, words->local, words->remote, words->flags, words->valid, words->total, line);
}

static void
show_crash_reason(void)
{
	char text[257];
	uint32_t size = 0, at = smem_item(ITEM_CRASH_REASON, &size);

	if (at == 0 || size == 0) {
		printf("mpss: %u: item %u, the DSP's last words: not there\n", step, ITEM_CRASH_REASON);
		return;
	}
	bzero(text, sizeof(text));
	smem_peek(at, text, size < sizeof(text) - 1 ? size : sizeof(text) - 1);
	if (text[0] == 0) {
		printf("mpss: %u: item %u, the DSP's last words (%u bytes): none\n", step, ITEM_CRASH_REASON, size);
	} else {
		printf("mpss: %u: THE MODEM'S LAST WORDS (item %u, %u bytes): \"%s\"\n", step, ITEM_CRASH_REASON, size, text);
	}
}

/* ---- the doorbells, and the mailbox to the always-on processor (hakios.wifi.c, steps 2 and 7) ------ */

#define QMP_MAGIC               0x00
#define QMP_MAGIC_VALUE         0x4d41494c
#define QMP_VERSION             0x04
#define QMP_THEIR_LINK          0x0c
#define QMP_THEIR_LINK_ACK      0x10
#define QMP_THEIR_CHANNEL       0x14
#define QMP_THEIR_CHANNEL_ACK   0x18
#define QMP_OUR_LINK            0x24
#define QMP_OUR_LINK_ACK        0x28
#define QMP_OUR_CHANNEL         0x2c
#define QMP_OUR_CHANNEL_ACK     0x30
#define QMP_OUR_BOX_SIZE        0x34
#define QMP_OUR_BOX             0x38
#define QMP_UP                  0x0000ffffu
#define QMP_DOWN                0xffff0000u
#define QMP_MESSAGE             64

static struct block aoss = { "mailbox", 0x0c300000, 0x400 };

#define IPCC_SEND               0x0c
#define IPCC_RECEIVED           0x10
#define IPCC_ENABLE             0x14
#define IPCC_DISABLE            0x18
#define IPCC_CLEAR              0x1c
#define IPCC_NOBODY             0xffffffffu
#define IPCC_AOP                0u
#define IPCC_MPSS              2u              /* dt: mboxes of qcom,smp2p-modem and qcom,glink/modem */
#define IPCC_GLINK              0u
#define IPCC_SMP2P              2u

static struct block ipcc = { "ipcc", 0x00408000, 0x1000 };

static struct {
	uint32_t glink, smp2p, said;
	uint64_t since;
} doorbell;

/*
 * The modem's rings, counted and cleared. Another's ring (the Wi-Fi DSP's, the always-on
 * processor's) is left for whoever listens for it: IPCC names the same one until it is cleared,
 * so the look ends there.
 */
static void
doorbells(void)
{
	for (unsigned i = 0; i < 8; i++) {
		uint32_t who = rd(ipcc.base + IPCC_RECEIVED);

		if (who == IPCC_NOBODY || who >> 16 != IPCC_MPSS) {
			return;
		}
		wr(ipcc.base + IPCC_CLEAR, who);
		doorbell.glink += (who & 0xffff) == IPCC_GLINK;
		doorbell.smp2p += (who & 0xffff) == IPCC_SMP2P;
		if (doorbell.said < 24) {
			uint64_t now = now_us() - doorbell.since;

			doorbell.said++;
			printf("mpss: doorbell: +%llu.%03llu ms: signal %u of the modem; heard and cleared\n", now / 1000, now % 1000,
			    who & 0xffff);
		}
	}
}

static uint32_t our_box;                /* where in the message memory our mailbox is; 0: not open */

static bool
mailbox_wait(uint32_t offset, uint32_t value, uint32_t ms)
{
	for (uint32_t i = 0; i < ms * 10; i++) {
		if (rd(aoss.base + offset) == value) {
			return true;
		}
		sleep_us(100);
	}
	return rd(aoss.base + offset) == value;
}

static void
ring_aop(void)
{
	wr(ipcc.base + IPCC_SEND, IPCC_AOP << 16 | 0);
}

/* Linux's qmp_open(), as hakios.wifi.c does it; what is done already is found done. Under the mailbox's lock. */
static bool
mailbox_open(void)
{
	uint32_t box, size;

	need(&aoss);
	need(&ipcc);
	if (get(&aoss, "magic", QMP_MAGIC) != QMP_MAGIC_VALUE || get(&aoss, "version", QMP_VERSION) != 1) {
		printf("mpss: %u: not a mailbox this knows\n", step);
		return false;
	}
	box = get(&aoss, "our mailbox", QMP_OUR_BOX);
	size = get(&aoss, "our mailbox's size", QMP_OUR_BOX_SIZE);
	if (size < QMP_MESSAGE + 4 || box < 0x3c || box + size > aoss.size || (box & 3)) {
		printf("mpss: %u: our mailbox is not where a message fits\n", step);
		return false;
	}
	if (rd(aoss.base + QMP_OUR_LINK) == QMP_UP && rd(aoss.base + QMP_OUR_LINK_ACK) == QMP_UP &&
	    rd(aoss.base + QMP_OUR_CHANNEL) == QMP_UP && rd(aoss.base + QMP_THEIR_CHANNEL) == QMP_UP &&
	    rd(aoss.base + QMP_OUR_CHANNEL_ACK) == QMP_UP) {
		printf("mpss: %u: the mailbox is open already (the Wi-Fi's start, or a kernel before this one, opened it)\n", step);
		our_box = box;
		return true;
	}
	put(&aoss, "their link, seen by us", QMP_THEIR_LINK_ACK, rd(aoss.base + QMP_THEIR_LINK));
	put(&aoss, "our link", QMP_OUR_LINK, QMP_UP);
	ring_aop();
	if (!mailbox_wait(QMP_OUR_LINK_ACK, QMP_UP, 1000)) {
		printf("mpss: %u: THE ALWAYS-ON PROCESSOR DOES NOT ACKNOWLEDGE OUR LINK within a second; taken down again\n", step);
		put(&aoss, "our link", QMP_OUR_LINK, QMP_DOWN);
		ring_aop();
		return false;
	}
	put(&aoss, "our channel", QMP_OUR_CHANNEL, QMP_UP);
	ring_aop();
	if (!mailbox_wait(QMP_THEIR_CHANNEL, QMP_UP, 1000)) {
		printf("mpss: %u: THE ALWAYS-ON PROCESSOR DOES NOT OPEN ITS CHANNEL within a second; ours is taken down again\n", step);
		put(&aoss, "our channel", QMP_OUR_CHANNEL, QMP_DOWN);
		put(&aoss, "our link", QMP_OUR_LINK, QMP_DOWN);
		ring_aop();
		return false;
	}
	put(&aoss, "their channel, seen by us", QMP_THEIR_CHANNEL_ACK, QMP_UP);
	ring_aop();
	if (!mailbox_wait(QMP_OUR_CHANNEL_ACK, QMP_UP, 1000)) {
		printf("mpss: %u: THE ALWAYS-ON PROCESSOR DOES NOT ACKNOWLEDGE OUR CHANNEL within a second; taken down again\n", step);
		put(&aoss, "our channel", QMP_OUR_CHANNEL, QMP_DOWN);
		put(&aoss, "our link", QMP_OUR_LINK, QMP_DOWN);
		ring_aop();
		return false;
	}
	printf("mpss: %u: THE MAILBOX TO THE ALWAYS-ON PROCESSOR IS OPEN\n", step);
	our_box = box;
	return true;
}

/* A message to the always-on processor (Linux's qmp_send()), under the lock the Wi-Fi's start takes too. */
static bool
mailbox_send(const char *text)
{
	uint32_t words[QMP_MESSAGE / 4] = { 0 };
	bool sent = false;

	if (!leaving) {
		lck_mtx_lock(&hakios_aop_mailbox_lock);
	}
	if (our_box == 0 && (leaving || !mailbox_open())) {
		goto out;
	}
	strlcpy((char *)words, text, sizeof(words));
	printf("mpss: %u: about to tell the always-on processor \"%s\"\n", step, text);
	snprintf(last_write, sizeof(last_write), "step %u sent the always-on processor a message", step);
	if (rd(aoss.base + our_box) != 0) {
		printf("mpss: %u: our mailbox still holds a message of %u bytes: not sent\n", step, rd(aoss.base + our_box));
		goto out;
	}
	for (unsigned i = 0; i < QMP_MESSAGE / 4; i++) {
		wr(aoss.base + our_box + 4 + 4 * i, words[i]);
	}
	wr(aoss.base + our_box, QMP_MESSAGE);
	(void)rd(aoss.base + our_box);
	ring_aop();
	if (!mailbox_wait(our_box, 0, 1000)) {
		printf("mpss: %u: THE ALWAYS-ON PROCESSOR DOES NOT TAKE THE MESSAGE within a second; it is taken back\n", step);
		wr(aoss.base + our_box, 0);
		goto out;
	}
	printf("mpss: %u: the always-on processor has taken it\n", step);
	sent = true;
out:
	if (!leaving) {
		lck_mtx_unlock(&hakios_aop_mailbox_lock);
	}
	return sent;
}

/* ---- RPMh: the DSP's two rails, asked for while it boots ------------------------------------------------ */

/*
 * The modem's logic and memories run from LPASS's own rails, LCX and LMX (dt: vdd_cx and
 * vdd_mx of qcom,lpass@3700000 are PM7325 L5 and L4 "level" regulators, RPMh resources cx.lvl and
 * mss.lvl), which the boot loader leaves at retention (16). Linux and Android ask for level 384
 * while the DSP boots, until it says it has asked for its own (bit 2, "proxy-unvote"); without it
 * the DSP rang once and left "SFR Init: wdog or kernel error suspected." in its last words.
 *
 * A request goes to RPMh through the application processor's sequencer (dt: rsc@18200000, DRV 2,
 * qcom,tcs-offset 0xd00, qcom,tcs-config: TCSs 0 and 1 are for requests made at once), as Linux's
 * rpmh-rsc.c writes one: the command's message id, address and value into a TCS that is idle, the
 * command enabled and marked to be waited for, the TCS put in "active" mode and triggered, and its
 * bit in the status of completions looked for. A rail asked for by level is asked by position in
 * the list of its levels that the boot loader's command database keeps (cmd-db.c), the first
 * position at or above the level wanted.
 */
#define CMD_DB_PHYS_DEFAULT     0x80860000u   /* dt: reg of /reserved-memory/cmd_db */
static uint32_t cmd_db_phys = CMD_DB_PHYS_DEFAULT;
#define CMD_DB_SIZE             0x20000UL

struct cmd_db_slave {
	uint16_t id, header_offset, data_offset, count, version, reserved[3];
};
struct cmd_db_header {
	uint32_t version;
	uint8_t magic[4];
	struct cmd_db_slave slave[8];
	uint32_t checksum, reserved;
	uint8_t data[];
};
struct cmd_db_entry {
	char id[8];
	uint32_t priority[2];
	uint32_t address;
	uint16_t length, offset;
};

static const struct cmd_db_header *cmd_db;

static struct block rsc = { "rsc", 0x18220000, 0x10000 };
#define RSC_TCS_BASE            0xd00
#define RSC_IRQ_STATUS          (RSC_TCS_BASE + 0x04)   /* a bit per TCS: done */
#define RSC_IRQ_CLEAR           (RSC_TCS_BASE + 0x08)
#define RSC_TCS(n)              (RSC_TCS_BASE + 672 * (n))
#define TCS_WAIT_FOR_DONE       0x10
#define TCS_CONTROL             0x14
#define TCS_STATUS              0x18                    /* not zero: idle */
#define TCS_ENABLED             0x1c
#define TCS_COMMAND(c)          (0x30 + 20 * (c))       /* message id, address, value, status */
#define TCS_ACTIVE_MODE         (1u << 16)
#define TCS_TRIGGER             (1u << 24)
#define MSGID_WRITE_WAITED      (8u | 1u << 8 | 1u << 16)       /* four bytes, an answer wanted, a write */
#define RPMH_TCS                0                       /* the first of the two for requests made at once */

/* The level position RPMh takes for that rail and level; -1: not found. */
static int
rail_position(const char *name, uint32_t level, uint32_t *address)
{
	char id[8] = { 0 };

	if (cmd_db == NULL) {
		cmd_db = (const struct cmd_db_header *)ml_io_map_unmappable(cmd_db_phys, CMD_DB_SIZE, VM_WIMG_WCOMB);
	}
	if (!(cmd_db->magic[0] == 0xdb && cmd_db->magic[1] == 0x30 && cmd_db->magic[2] == 0x03 && cmd_db->magic[3] == 0x0c)) {
		printf("mpss: %u: the boot loader's command database is not there\n", step);
		return -1;
	}
	strlcpy(id, name, sizeof(id));
	for (unsigned s = 0; s < 8; s++) {
		const struct cmd_db_slave *slave = &cmd_db->slave[s];
		const struct cmd_db_entry *entry = (const struct cmd_db_entry *)(cmd_db->data + slave->header_offset);

		if (slave->id == 0 || slave->count > 512) {
			continue;
		}
		for (unsigned i = 0; i < slave->count; i++) {
			const uint8_t *data;
			char levels[160];
			int at;

			if (memcmp(entry[i].id, id, sizeof(id)) != 0) {
				continue;
			}
			if ((unsigned)slave->data_offset + entry[i].offset + entry[i].length > CMD_DB_SIZE - 0x1000) {
				return -1;
			}
			data = cmd_db->data + slave->data_offset + entry[i].offset;
			at = snprintf(levels, sizeof(levels), "%s at %#x, levels:", name, entry[i].address);
			for (unsigned l = 0; l < entry[i].length / 2u && l < 16; l++) {
				unsigned value = data[2 * l] | data[2 * l + 1] << 8;

				if (l > 0 && value == 0) {
					break;
				}
				at += snprintf(levels + at, sizeof(levels) - (size_t)at, " %u", value);
			}
			printf("mpss: %u: RPMh has %s\n", step, levels);
			for (unsigned l = 0; l < entry[i].length / 2u && l < 16; l++) {
				if ((uint32_t)(data[2 * l] | data[2 * l + 1] << 8) >= level) {
					*address = entry[i].address;
					return (int)l;
				}
			}
			return -1;
		}
	}
	printf("mpss: %u: RPMh has no \"%s\"\n", step, name);
	return -1;
}

/* The two rails at that level (Linux's proxy request), or given back (level 0). True: RPMh has said done. */
static bool
rails(uint32_t level)
{
	static const char *const names[2] = { "cx.lvl", "mss.lvl" };
	uint32_t addresses[2] = { 0, 0 }, values[2] = { 0, 0 }, done = 0;
	uint64_t began;

	for (unsigned r = 0; r < 2; r++) {
		int position = level == 0 ? 0 : rail_position(names[r], level, &addresses[r]);

		if (level == 0) {
			rail_position(names[r], 0, &addresses[r]);
		}
		if (position < 0 || addresses[r] == 0) {
			printf("mpss: %u: no position for %s at level %u: the rails are not asked\n", step, names[r], level);
			return false;
		}
		values[r] = (uint32_t)position;
	}
	need(&rsc);
	lck_mtx_lock(&hakios_rsc_lock);
	/* (the boot loader leaves the trigger set after its last request, done: idle is what the status says) */
	if (rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_STATUS) == 0) {
		printf("mpss: %u: TCS %u of the sequencer is busy (status %#x, control %#x): the rails are not asked\n", step, RPMH_TCS,
		    rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_STATUS), rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL));
		lck_mtx_unlock(&hakios_rsc_lock);
		return false;
	}
	printf("mpss: %u: about to ask RPMh, through TCS %u, for cx.lvl (%#x) = %u and mss.lvl (%#x) = %u (level %u%s)\n", step,
	    RPMH_TCS, addresses[0], values[0], addresses[1], values[1], level, level == 0 ? ": given back" : "");
	snprintf(last_write, sizeof(last_write), "step %u asked RPMh for the modem's rails at level %u", step, level);
	hold();
	wr(rsc.base + RSC_IRQ_CLEAR, 1u << RPMH_TCS);
	for (unsigned r = 0; r < 2; r++) {
		wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(r), MSGID_WRITE_WAITED);
		wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(r) + 4, addresses[r]);
		wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(r) + 8, values[r]);
	}
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_WAIT_FOR_DONE, 3);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_ENABLED, 3);
	settle();
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) & ~TCS_TRIGGER);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) | TCS_ACTIVE_MODE);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) | TCS_TRIGGER);
	began = now_us();
	while (now_us() - began < 100000) {
		done = rd(rsc.base + RSC_IRQ_STATUS) & (1u << RPMH_TCS);
		if (done) {
			break;
		}
		sleep_us(50);
	}
	printf("mpss: %u: RPMh %s after %llu us; the commands' status %#x %#x, the TCS's %#x\n", step, done ? "HAS DONE IT" :
	    "HAS NOT SAID DONE", now_us() - began, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(0) + 12),
	    rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(1) + 12), rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_STATUS));
	wr(rsc.base + RSC_IRQ_CLEAR, 1u << RPMH_TCS);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) & ~(TCS_TRIGGER | TCS_ACTIVE_MODE));
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_ENABLED, 0);
	lck_mtx_unlock(&hakios_rsc_lock);
	return done != 0;
}


/*
 * One command to RPMh through the sequencer, as rails() sends its two: a resource's address from
 * the command database and the word for it. True: RPMh has said done. The IPA's clock is such a
 * resource, a "bus clock manager" named IP0 (Linux's clk-rpmh.c: BCM votes are a word of commit
 * (bit 30), valid (bit 29) and a vote of 14 bits).
 */
static bool
rpmh_command(const char *name, uint32_t value)
{
	uint32_t address = 0, done = 0;
	uint64_t began;

	if (rail_position(name, 0, &address) < 0 || address == 0) {
		printf("mpss: %u: RPMh has no \"%s\": nothing is asked\n", step, name);
		return false;
	}
	need(&rsc);
	lck_mtx_lock(&hakios_rsc_lock);
	if (rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_STATUS) == 0) {
		printf("mpss: %u: TCS %u of the sequencer is busy: \"%s\" is not asked\n", step, RPMH_TCS, name);
		lck_mtx_unlock(&hakios_rsc_lock);
		return false;
	}
	printf("mpss: %u: about to ask RPMh, through TCS %u, for %s (%#x) = %#x\n", step, RPMH_TCS, name, address, value);
	snprintf(last_write, sizeof(last_write), "step %u asked RPMh for %s = %#x", step, name, value);
	hold();
	wr(rsc.base + RSC_IRQ_CLEAR, 1u << RPMH_TCS);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(0), MSGID_WRITE_WAITED);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(0) + 4, address);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(0) + 8, value);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_WAIT_FOR_DONE, 1);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_ENABLED, 1);
	settle();
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) & ~TCS_TRIGGER);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) | TCS_ACTIVE_MODE);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) | TCS_TRIGGER);
	began = now_us();
	while (now_us() - began < 100000) {
		done = rd(rsc.base + RSC_IRQ_STATUS) & (1u << RPMH_TCS);
		if (done) {
			break;
		}
		sleep_us(50);
	}
	printf("mpss: %u: RPMh %s after %llu us for %s; the command's status %#x\n", step, done ? "HAS DONE IT" : "HAS NOT SAID DONE",
	    now_us() - began, name, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_COMMAND(0) + 12));
	wr(rsc.base + RSC_IRQ_CLEAR, 1u << RPMH_TCS);
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL, rd(rsc.base + RSC_TCS(RPMH_TCS) + TCS_CONTROL) & ~(TCS_TRIGGER | TCS_ACTIVE_MODE));
	wr(rsc.base + RSC_TCS(RPMH_TCS) + TCS_ENABLED, 0);
	lck_mtx_unlock(&hakios_rsc_lock);
	return done != 0;
}

#define BCM_VOTE(on, vote)      (1u << 30 | (uint32_t)(on) << 29 | ((vote) & 0x3fffu))

/* The IPA's clock, on or off: the BCM "IP0" (dt of Linux: clocks = <&rpmhcc RPMH_IPA_CLK>). */
static bool
ipa_clock(bool on)
{
	bool ok = rpmh_command("IP0", BCM_VOTE(on, on ? 0x3fffu : 0));

	/*
	 * The buses on the IPA's way to memory, voted for as Android's driver does while the IPA is on (its
	 * ipa3_enable_clks(): "ICC Voting for BW", the paths ipa_to_llcc, llcc_to_ebi1 and appss_to_ipa of the
	 * device tree): the BCMs of the sc7280 interconnect driver on that way -- SN8 (aggre2 noc, the IPA's
	 * master), SH0 (gem noc to the last-level cache), MC0 and ACV (the memory controller) -- all in the
	 * command database. Whether a NoC that nobody votes for stays clocked for the IPA's own traffic is
	 * what this tells (modem_ipa_bus=0: not voted).
	 */
	if (!option("modem_ipa_bus_off")) {
		static const char *const buses[] = { "SN8", "SH0", "MC0", "ACV" };

		for (unsigned b = 0; b < sizeof(buses) / sizeof(buses[0]); b++) {
			rpmh_command(buses[b], BCM_VOTE(on, on ? 0x3fffu : 0));
		}
	}
	return ok;
}

/* ---- step 1: the shared memory --------------------------------------------------------------------- */

static bool
step_survey(void)
{
	struct smp2p words;
	uint32_t size = 0;

	announce("read our partition with the modem in the shared memory (0x80900000), and its items");
	show_partition();
	if (dsp_words(&words)) {
		show_words("the DSP's item (435) is there: it has run since the phone was started", &words);
	} else {
		printf("mpss: %u: the DSP's SMP2P item (435) is not there: it has not run since the phone was started\n", step);
	}
	printf("mpss: %u: our SMP2P item (428): %s; GLINK's descriptor (478): %s, our queue (479): %s, the DSP's (480): %s\n", step,
	    smem_item(ITEM_SMP2P_OURS, &size) ? "there" : "not there", smem_item(478, &size) ? "there" : "NOT THERE",
	    smem_item(479, &size) ? "there" : "NOT THERE", smem_item(480, &size) ? "there" : "not there (the DSP makes it)");
	show_crash_reason();
	need(&ipcc);
	printf("mpss: %u: the doorbells say %#x\n", step, rd(ipcc.base + IPCC_RECEIVED));
	return smem_partition(SMEM_US, SMEM_MPSS, &size, &size) != 0;
}

/* ---- step 2: the firmware -------------------------------------------------------------------------- */

#define MPSS_MEMORY_DEFAULT     0x8b800000u   /* dt: reserved-memory mpss@..., memory-region of qcom,mss */
static uint32_t mpss_memory = MPSS_MEMORY_DEFAULT;
#define MPSS_MEMORY_SIZE        0x0f600000u
#define PAS_MPSS_DEFAULT        4               /* dt: qcom,pas-id of qcom,mss */
static uint32_t pas_mpss = PAS_MPSS_DEFAULT;


/*
 * A file of /fw on the root file system, read whole into memory of ours: the system image, where
 * userland/build.sh puts fw-modem/ when it makes that image. It is compressed there, which the file
 * system undoes and the reading of the small image's memory below does not; and it is there only
 * once the root is mounted, so only modem=1, which waits for that on its own thread, finds it so.
 */
static const uint8_t *
root_file(const char *name, uint64_t *size)
{
	char path[256];
	vfs_context_t context = vfs_context_kernel();
	vnode_t vnode = NULLVP;
	struct vnode_attr attributes;
	uint8_t *data;
	int left = 0, error;

	if (rootvnode == NULLVP) {
		return NULL;
	}
	snprintf(path, sizeof(path), "/fw/%s", name);
	if (vnode_lookup(path, 0, &vnode, context) != 0) {
		return NULL;
	}
	VATTR_INIT(&attributes);
	VATTR_WANTED(&attributes, va_data_size);
	if (vnode_getattr(vnode, &attributes, context) != 0 || attributes.va_data_size == 0 ||
	    attributes.va_data_size > MPSS_MEMORY_SIZE) {
		vnode_put(vnode);
		return NULL;
	}
	data = kalloc_data(attributes.va_data_size, Z_WAITOK);
	if (data == NULL) {
		vnode_put(vnode);
		return NULL;
	}
	error = vn_rdwr(UIO_READ, vnode, (caddr_t)data, (int)attributes.va_data_size, 0, UIO_SYSSPACE, 0,
	    vfs_context_ucred(context), &left, vfs_context_proc(context));
	vnode_put(vnode);
	if (error != 0 || left != 0) {
		printf("mpss: %u: %s: read error %d, %d bytes left\n", step, path, error, left);
		kfree_data(data, attributes.va_data_size);
		return NULL;
	}
	*size = attributes.va_data_size;
	return data;
}

static bool from_root;                  /* the image is in memory of ours (root_file()), to be given back once written */

/* A file of fw/ in the small image (MODEM=1 userland/build.sh puts fw-modem/ there), else of /fw on the root (above). */
extern const uint8_t *hakios_image_file(const char *dir, const char *name, uint64_t *size, int *disk_out);
static const uint8_t *
image_file(const char *name, uint64_t *size)
{
	{
		const uint8_t *data = hakios_image_file("fw", name, size, NULL);  /* (hakios.image.c) */

		if (data != NULL) {
			return data;
		}
	}
	{
		const uint8_t *data = root_file(name, size);

		if (data != NULL) {
			from_root = true;
			return data;
		}
	}
	printf("mpss: %u: no fw/%s in the small image nor on the root: put the phone's /vendor/firmware_mnt/image/%s into "
	    "fw-modem/ and run SYSTEM=1 userland/build.sh (or MODEM=1, for the small image)\n", step, name, name);
	return NULL;
}


#define MDT_HASH(flags)         (((flags) >> 24 & 7) == 2)
#define MAX_SEGMENTS            48

struct elf_segment {
	uint32_t type, offset, vaddr, paddr, file_size, memory_size, flags, align;
};

static struct {
	const uint8_t *header;
	uint64_t header_size;
	struct {
		uint32_t at;
		uint32_t file_size, memory_size;
		const uint8_t *file;
	} segment[MAX_SEGMENTS];
	unsigned segments;
	uint32_t size;
} image;

/* The image's files given back, if they were read into memory of ours. */
static void
image_forget(void)
{
	if (!from_root) {
		return;
	}
	if (image.header != NULL) {
		void *data = (void *)(uintptr_t)image.header;

		kfree_data(data, image.header_size);
	}
	for (unsigned s = 0; s < image.segments; s++) {
		if (image.segment[s].file != NULL) {
			void *data = (void *)(uintptr_t)image.segment[s].file;

			kfree_data(data, image.segment[s].file_size);
		}
	}
	bzero(&image, sizeof(image));
	from_root = false;
}

static unsigned rmtfs_load(void);
static void servreg_load(void);

static bool
step_firmware(void)
{
	struct elf_segment segment;
	const uint8_t *header;
	uint64_t size = 0, files = 0;
	uint32_t count, hashes = 0, lowest = ~0u, highest = 0;

	announce("find the modem's image in the small image or on the root, and check it against its own header");
	image_forget();
	image.segments = 0;
	header = image_file("modem.mdt", &size);
	if (header == NULL) {
		return false;
	}
	image.header = header;
	image.header_size = size;
	count = size >= 52 + 2 * 32 ? header[44] | header[45] << 8 : 0;
	if (memcmp(header, "\177ELF\1", 5) != 0 || count < 3 || count > MAX_SEGMENTS || 52 + 32 * count > size || size > PAGE_SIZE) {
		printf("mpss: %u: fw/modem.mdt is not such a header (%02x %02x %02x %02x, %u segments, %llu bytes)\n", step, header[0],
		    header[1], header[2], header[3], count, size);
		goto fail;
	}
	for (uint32_t i = 0; i < count; i++) {
		char name[16];

		memcpy(&segment, header + 52 + 32 * i, sizeof(segment));
		if (MDT_HASH(segment.flags)) {
			hashes = segment.file_size;
		}
		if (segment.type != 1 || MDT_HASH(segment.flags) || segment.memory_size == 0) {
			continue;
		}
		if (segment.paddr < mpss_memory || segment.file_size > segment.memory_size ||
		    (uint64_t)segment.paddr + segment.memory_size > (uint64_t)mpss_memory + MPSS_MEMORY_SIZE) {
			printf("mpss: %u: segment %u (%#x, %#x bytes) is not within the DSP's memory\n", step, i, segment.paddr,
			    segment.memory_size);
			goto fail;
		}
		image.segment[image.segments].at = segment.paddr - mpss_memory;
		image.segment[image.segments].file_size = segment.file_size;
		image.segment[image.segments].memory_size = segment.memory_size;
		image.segment[image.segments].file = NULL;
		if (segment.file_size != 0) {
			uint64_t file_size = 0;

			snprintf(name, sizeof(name), "modem.b%02u", i);
			image.segment[image.segments].file = image_file(name, &file_size);
			if (image.segment[image.segments].file == NULL) {
				goto fail;
			}
			if (file_size != segment.file_size) {
				printf("mpss: %u: fw/%s has %llu bytes, the header says %u\n", step, name, file_size, segment.file_size);
				image.segments++;               /* (for its file to be given back) */
				goto fail;
			}
			files += file_size;
		}
		lowest = segment.paddr < lowest ? segment.paddr : lowest;
		highest = segment.paddr + segment.memory_size > highest ? segment.paddr + segment.memory_size : highest;
		image.segments++;
	}
	if (image.segments == 0 || lowest != mpss_memory || 52 + 32 * count + (uint64_t)hashes != size) {
		printf("mpss: %u: not what TrustZone takes: %u segments to load, the lowest at %#x; the header file has %llu bytes, "
		    "the header %u and the hashes %u\n", step, image.segments, lowest, size, 52 + 32 * count, hashes);
		goto fail;
	}
	image.header = header;
	image.header_size = size;
	image.size = ((highest + 0xfff) & ~0xfffu) - mpss_memory;
	printf("mpss: %u: the image is whole (%s): a header of %llu bytes (%u of hashes), %u segments, %llu bytes of files, %#x "
	    "bytes of the DSP's %#x from %#x\n", step, from_root ? "from /fw on the root" : "from the small image", size, hashes,
	    image.segments, files, image.size, MPSS_MEMORY_SIZE, mpss_memory);
	announce("read the modem's four file systems from the phone's storage into memory of ours (rmtfs below)");
	{
		unsigned loaded = rmtfs_load();

		printf("mpss: %u: %u of the modem's 4 file systems are here%s\n", step, loaded, loaded == 4 ? "" :
		    ": WITHOUT THEM ALL THE MODEM OFFERS NO SERVICES BUT ITS OWN (DMS, NAS... stay away)");
		servreg_load();
	}
	return true;
fail:
	image_forget();
	image.segments = 0;
	return false;
}

/* ---- step 3: the mailbox --------------------------------------------------------------------------- */

static bool
step_mailbox(void)
{
	bool open;

	announce("open the mailbox to the always-on processor, unless it is open: words of its message memory, and its doorbell");
	lck_mtx_lock(&hakios_aop_mailbox_lock);
	open = mailbox_open();
	lck_mtx_unlock(&hakios_aop_mailbox_lock);
	return open;
}

/* ---- TrustZone, and stopping the DSP --------------------------------------------------------------- */

#define SCM_64                  (1ull << 30)
#define SCM_SIP                 (2ull << 24)
#define SCM_CALL(service, command) ((uint64_t)(service) << 8 | (command))
#define SCM_PAS_INIT_IMAGE      SCM_CALL(2, 1)
#define SCM_PAS_MEM_SETUP       SCM_CALL(2, 2)
#define SCM_PAS_AUTH_AND_RESET  SCM_CALL(2, 5)
#define SCM_PAS_SHUTDOWN        SCM_CALL(2, 6)
#define SCM_ONE_VALUE           1
#define SCM_THREE_VALUES        3
#define SCM_VALUE_AND_BUFFER    (2 | 2 << 6)
#define SCM_INTERRUPTED         1

static LCK_MTX_DECLARE(dsp_lock, &mpss_lock_group);     /* the DSP being started or stopped */
static bool rails_asked;                /* our request for its rails stands (until it asks for its own, or is shut down) */
static bool dsp_started;
static bool stopped_by_its_word;
static uint32_t our_item;
static bool smp2p_agreed, smp2p_restart_acks, smp2p_restart_acked;

static int32_t
scm(const char *what, uint64_t function, uint64_t arguments, uint64_t first, uint64_t second, uint64_t third, uint64_t fourth,
    uint64_t *answer)
{
	uint64_t session = 0;

	function |= SCM_64 | SCM_SIP;
	printf("mpss: %u: about to ask TrustZone %s: smc %#llx, arguments %#llx, %#llx, %#llx\n", step, what, function, first,
	    second, third);
	snprintf(last_write, sizeof(last_write), "step %u asked TrustZone %#llx(%#llx, %#llx)", step, function, first, second);
	for (int i = 0; i < 100; i++) {
		register uint64_t x0 __asm__("x0") = i == 0 ? function : SCM_INTERRUPTED;
		register uint64_t x1 __asm__("x1") = arguments;
		register uint64_t x2 __asm__("x2") = first;
		register uint64_t x3 __asm__("x3") = second;
		register uint64_t x4 __asm__("x4") = third;
		register uint64_t x5 __asm__("x5") = fourth;
		register uint64_t x6 __asm__("x6") = session;
		register uint64_t x7 __asm__("x7") = 0;

		__asm__ volatile ("smc #0" : "+r" (x0), "+r" (x1), "+r" (x2), "+r" (x3), "+r" (x4), "+r" (x5), "+r" (x6),
		    "+r" (x7) : : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "memory");
		session = x6;
		if (x0 != SCM_INTERRUPTED) {
			printf("mpss: %u: TrustZone returns %d, answer %#llx\n", step, (int32_t)x0, x1);
			*answer = x1;
			return (int32_t)x0;
		}
	}
	printf("mpss: %u: TrustZone keeps saying the call was interrupted\n", step);
	return -1;
}

static uint32_t
our_item_find(void)
{
	uint32_t size = 0, at = smem_item(ITEM_SMP2P_OURS, &size);

	return at != 0 && size >= sizeof(struct smp2p) ? at : 0;
}

/* What Linux does when the DSP rings for SMP2P: agree on the features once, and answer a restart. */
static void
dsp_listen(const struct smp2p *theirs)
{
	struct smp2p ours;

	if (our_item == 0 || !smem_peek(our_item, &ours, sizeof(ours)) || ours.magic != SMP2P_MAGIC) {
		return;
	}
	if (!smp2p_agreed) {
		uint32_t features = (ours.version >> 8) & (theirs->version >> 8);

		if ((theirs->version & 0xff) != (ours.version & 0xff)) {
			return;
		}
		if (features != ours.version >> 8) {
			smem_poke(our_item + SMP2P_AT_VERSION, features << 8 | (ours.version & 0xff));
		}
		smp2p_restart_acks = features & SMP2P_RESTART_ACKS;
		smp2p_agreed = true;
		printf("mpss: %u: the DSP and we agree on SMP2P: version %u, features %#x\n", step, ours.version & 0xff, features);
	}
	if (smp2p_restart_acks && ((theirs->flags & SMP2P_RESTART_DONE) != 0) != smp2p_restart_acked) {
		smp2p_restart_acked = !smp2p_restart_acked;
		printf("mpss: %u: the DSP's flags say it has restarted (%#x): answered in our flags (bit 1 = %u), and its SMP2P doorbell\n",
		    step, theirs->flags, smp2p_restart_acked);
		smem_poke(our_item + SMP2P_AT_FLAGS, (ours.flags & ~SMP2P_RESTART_ACK) | (smp2p_restart_acked ? SMP2P_RESTART_ACK : 0));
		settle();
		wr(ipcc.base + IPCC_SEND, IPCC_MPSS << 16 | IPCC_SMP2P);
	}
}

/* Linux's qcom_q6v5_request_stop(): our bit "stop", the DSP's "stop ack" waited for, our bit taken back. */
static bool
dsp_ask_to_stop(uint32_t bound_ms)
{
	struct smp2p ours, theirs;
	uint32_t at = our_item_find(), state, value_at;
	uint64_t asked, now;
	int entry = -1;

	if (at == 0 || !smem_peek(at, &ours, sizeof(ours)) || ours.magic != SMP2P_MAGIC || (ours.version & 0xff) != SMP2P_VERSION ||
	    (entry = smp2p_entry(&ours, OUR_ENTRY_STOP)) < 0) {
		printf("mpss: %u: the DSP is not asked to stop first: there is no SMP2P item of ours to ask it with\n", step);
		return false;
	}
	if (!dsp_words(&theirs) || smp2p_entry(&theirs, "slave-kernel") < 0) {
		printf("mpss: %u: the DSP is not asked to stop first: its item has no entry to answer in\n", step);
		return false;
	}
	state = dsp_state(&theirs);
	if (state & DSP_STOP_ACK) {
		printf("mpss: %u: the DSP is not asked to stop: its entry says %#x, that it has stopped when asked before\n", step, state);
		return true;
	}
	if (!(state & DSP_READY) || (state & DSP_FATAL)) {
		printf("mpss: %u: the DSP is not asked to stop first: its entry says %#x, not \"up\"\n", step, state);
		return false;
	}
	need(&ipcc);
	value_at = at + SMP2P_AT_VALUE((uint32_t)entry);
	printf("mpss: %u: about to ask the DSP to stop: bit 0 of our entry \"%s\" at %#lx, and its SMP2P doorbell\n", step,
	    OUR_ENTRY_STOP, smem_phys + value_at);
	snprintf(last_write, sizeof(last_write), "step %u asked the modem to stop", step);
	smem_poke(value_at, ours.entry[entry].value | DSP_STOP);
	settle();
	wr(ipcc.base + IPCC_SEND, IPCC_MPSS << 16 | IPCC_SMP2P);
	asked = now = now_us();
	while (now - asked < bound_ms * 1000ull) {
		if (dsp_words(&theirs) && (dsp_state(&theirs) & (DSP_STOP_ACK | DSP_FATAL))) {
			break;
		}
		sleep_us(200);
		now = now_us();
	}
	state = dsp_words(&theirs) ? dsp_state(&theirs) : 0;
	printf("mpss: %u: %s after %llu.%03llu ms; its entry says %#x\n", step, state & DSP_STOP_ACK ?
	    "THE MODEM ACKNOWLEDGES THE STOP" : "THE MODEM DOES NOT ACKNOWLEDGE THE STOP", (now - asked) / 1000,
	    (now - asked) % 1000, state);
	smem_poke(value_at, ours.entry[entry].value & ~DSP_STOP);
	settle();
	wr(ipcc.base + IPCC_SEND, IPCC_MPSS << 16 | IPCC_SMP2P);
	return (state & DSP_STOP_ACK) != 0;
}

static void link_stop(void);

/* Asked to stop, then shut down by TrustZone, then the always-on processor told that the image is gone. */
static bool
dsp_shutdown(const char *why)
{
	uint64_t answer = ~0ull;
	int32_t result;

	printf("mpss: %u: %s: the modem is asked to stop, then TrustZone to shut it down\n", step, why);
	if (!leaving) {
		link_stop();
	}
	stopped_by_its_word = dsp_ask_to_stop(leaving ? 2000 : 5000);
	result = scm("to shut the modem down", SCM_PAS_SHUTDOWN, SCM_ONE_VALUE, pas_mpss, 0, 0, 0, &answer);
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT SHUT THE MODEM DOWN: the call returns %d, its answer is %#llx\n", step, result,
		    answer);
		return false;
	}
	printf("mpss: %u: THE MODEM IS SHUT DOWN\n", step);
	if (ipa_loaded) {
		ipa_firmware_stop();
		ipa_loaded = false;
	}
	if (!leaving) {
		rmtfs_share(false);
	}
	need(&ipcc);
	wr(ipcc.base + IPCC_DISABLE, IPCC_MPSS << 16 | IPCC_SMP2P);
	doorbells();
	dsp_started = false;
	mailbox_send("{class: image, res: load_state, name: modem, val: off}");
	if (rails_asked) {
		rails(0);
		rails_asked = false;
	}
	return true;
}

/* ---- step 4: TrustZone takes the header ------------------------------------------------------------ */

static bool
step_header(void)
{
	struct smp2p words;
	vm_offset_t page = 0;
	uint64_t answer = ~0ull;
	int32_t result;

	if (image.segments == 0) {
		return false;
	}
	if (dsp_words(&words)) {
		show_words("the DSP has left words in the shared memory", &words);
		announce("ask TrustZone to shut down an modem that a kernel before this one may have left running");
		if (!dsp_shutdown("it may be running")) {
			return false;
		}
		if (!stopped_by_its_word && !option("modem_again")) {
			printf("mpss: %u: THE MODEM IS NOT STARTED AGAIN: it was shut down without having acknowledged a stop, and "
			    "with the Wi-Fi's DSP a start after that reset the phone (modem_again=1 to try all the same)\n", step);
			return false;
		}
	} else {
		printf("mpss: %u: the DSP has left nothing in the shared memory: it has not run since the phone was started\n", step);
	}

	announce("hand the image's header to TrustZone to check its signature (image 4), in a page of ours");
	if (kmem_alloc(kernel_map, &page, PAGE_SIZE, KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
		printf("mpss: %u: no memory\n", step);
		return false;
	}
	memcpy((void *)page, image.header, image.header_size);
	for (vm_offset_t line = page; line < page + PAGE_SIZE; line += 64) {
		__asm__ volatile ("dc civac, %0" : : "r" (line));
	}
	settle();
	result = scm("to take the modem's header", SCM_PAS_INIT_IMAGE, SCM_VALUE_AND_BUFFER, pas_mpss, kvtophys(page), 0, 0, &answer);
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT TAKE THE HEADER: the call returns %d, its answer is %#llx\n", step, result, answer);
		return false;
	}
	printf("mpss: %u: TRUSTZONE ACCEPTS THE MODEM'S SIGNATURE FROM THIS KERNEL\n", step);

	announce("tell TrustZone where the image will be: the whole of the modem's memory, as Linux does");
	result = scm("where the image will be", SCM_PAS_MEM_SETUP, SCM_THREE_VALUES, pas_mpss, mpss_memory, MPSS_MEMORY_SIZE, 0, &answer);
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT TAKE THE PLACE: the call returns %d, its answer is %#llx\n", step, result, answer);
		return false;
	}
	printf("mpss: %u: TrustZone takes the place: %#x bytes at %#x (the image takes %#x of them)\n", step, MPSS_MEMORY_SIZE, mpss_memory,
	    image.size);
	return true;
}

/* ---- step 5: the image, in the DSP's memory --------------------------------------------------------- */

static bool image_written;

static uint64_t
segment_word(unsigned s, uint32_t at)
{
	uint64_t word = 0;

	for (unsigned b = 0; b < 8 && at + b < image.segment[s].file_size; b++) {
		word |= (uint64_t)image.segment[s].file[at + b] << (8 * b);
	}
	return word;
}

/* A segment written at its place: words of eight where it begins on one, bytes where it does not or at its end. */
static void
segment_put(volatile uint8_t *dsp_memory, unsigned s)
{
	uint32_t at = image.segment[s].at, size = image.segment[s].memory_size, i = 0;

	if ((at & 7) == 0) {
		volatile uint64_t *to = (volatile uint64_t *)(dsp_memory + at);

		for (; i + 8 <= size; i += 8) {
			to[i / 8] = segment_word(s, i);
		}
	}
	for (; i < size; i++) {
		dsp_memory[at + i] = i < image.segment[s].file_size ? image.segment[s].file[i] : 0;
	}
}

/* ... and read back: how many words of eight (or bytes, where it is not written by words) differ. */
static uint64_t
segment_check(volatile uint8_t *dsp_memory, unsigned s)
{
	uint32_t at = image.segment[s].at, size = image.segment[s].memory_size, i = 0;
	uint64_t wrong = 0;

	if ((at & 7) == 0) {
		volatile uint64_t *from = (volatile uint64_t *)(dsp_memory + at);

		for (; i + 8 <= size; i += 8) {
			wrong += from[i / 8] != segment_word(s, i);
		}
	}
	for (; i < size; i++) {
		wrong += dsp_memory[at + i] != (i < image.segment[s].file_size ? image.segment[s].file[i] : 0);
	}
	return wrong;
}

static bool
step_image(void)
{
	volatile uint8_t *dsp_memory;
	uint64_t written = 0, cleared = 0, wrong = 0, began;

	if (image.segments == 0) {
		return false;
	}
	announce("put the image's segments into the modem's memory (0x8b800000, 246 MB: memory of nobody's until TrustZone starts "
	    "the modem), with zeroes where a segment is longer than its file, and read them back");
	dsp_memory = (volatile uint8_t *)ml_io_map_unmappable(mpss_memory, image.size, VM_WIMG_WCOMB);
	if (dsp_memory == NULL) {
		printf("mpss: %u: the modem's memory cannot be mapped (%#x bytes)\n", step, image.size);
		return false;
	}
	snprintf(last_write, sizeof(last_write), "step %u wrote the image at %#x", step, mpss_memory);
	began = now_us();
	for (unsigned s = 0; s < image.segments; s++) {
		segment_put(dsp_memory, s);
		written += image.segment[s].file_size;
		cleared += image.segment[s].memory_size - image.segment[s].file_size;
	}
	settle();
	for (unsigned s = 0; s < image.segments; s++) {
		wrong += segment_check(dsp_memory, s);
	}
	printf("mpss: %u: %u segments, %llu bytes written and %llu cleared in %llu ms; read back, %llu words or bytes differ\n", step,
	    image.segments, written, cleared, (now_us() - began) / 1000, wrong);
	settle();
	ml_io_unmap((vm_offset_t)dsp_memory, image.size);
	image_written = wrong == 0;
	image_forget();                         /* (it is in the modem's memory now: what was read of the root, given back) */
	return image_written;
}

/* ---- step 6: our SMP2P item ------------------------------------------------------------------------ */

static struct block smem_lock = { "shared memory's lock", 0x01f43000, 0x1000 };
#define LOCK_OWNER              0x000
#define LOCK_US                 1u

/* Our item made in the partition, as Linux's qcom_smem_alloc_private() makes one (hakios.wifi.c: our_item_make()). */
static bool
our_item_make(uint32_t partition, uint32_t item, uint32_t bytes)
{
	struct smem_partition header;
	struct smem_entry entry;
	const uint32_t item_size = (bytes + 7) & ~7u, need_bytes = sizeof(entry) + item_size;
	uint32_t owner = ~0u, at;
	bool made = false;

	lck_mtx_lock(&hakios_smem_alloc_lock);         /* (the Wi-Fi's start allocates under the same hardware lock) */
	need(&smem_lock);
	printf("mpss: %u: about to take the lock (%#x), write an entry for item %u with %u bytes behind it at the partition's mark, "
	    "move the mark by %u, and release the lock\n", step, smem_lock.phys + LOCK_OWNER, item, item_size, need_bytes);
	snprintf(last_write, sizeof(last_write), "step %u took the shared memory's lock", step);
	for (uint32_t ms = 0; ms <= 1000; ms++) {
		wr(smem_lock.base + LOCK_OWNER, LOCK_US);
		owner = rd(smem_lock.base + LOCK_OWNER);
		if (owner == LOCK_US) {
			break;
		}
		sleep_ms(1);
	}
	if (owner != LOCK_US) {
		printf("mpss: %u: THE LOCK IS HELD BY PROCESSOR %u AND STAYS SO FOR A SECOND: nothing is written\n", step, owner);
		lck_mtx_unlock(&hakios_smem_alloc_lock);
		return false;
	}
	if (!smem_peek(partition, &header, sizeof(header)) || header.free_uncached > header.free_cached ||
	    header.free_cached > header.size) {
		printf("mpss: %u: the partition's marks make no sense: nothing is written\n", step);
		goto out;
	}
	for (at = sizeof(header); at < header.free_uncached;) {
		if (!smem_peek(partition + at, &entry, sizeof(entry)) || entry.canary != SMEM_CANARY || entry.item == item ||
		    entry.size > header.size) {
			break;
		}
		at += sizeof(entry) + entry.padding_header + entry.size;
	}
	if (at != header.free_uncached) {
		printf("mpss: %u: the partition's entries do not end at its mark (+%#x, the mark is +%#x), or the item has appeared: "
		    "nothing is written\n", step, at, header.free_uncached);
		goto out;
	}
	if (at + need_bytes > header.free_cached) {
		printf("mpss: %u: no room: %u bytes wanted, %u free: nothing is written\n", step, need_bytes, header.free_cached - at);
		goto out;
	}
	smem_poke(partition + at, SMEM_CANARY | item << 16);
	smem_poke(partition + at + 4, item_size);
	smem_poke(partition + at + 8, item_size - bytes);
	for (uint32_t z = 0; z < item_size; z += 4) {
		smem_poke(partition + at + (uint32_t)sizeof(entry) + z, 0);
	}
	smem_poke(partition + at + 12, 0);
	settle();
	smem_poke(partition + __offsetof(struct smem_partition, free_uncached), at + need_bytes);
	settle();
	made = true;
out:
	wr(smem_lock.base + LOCK_OWNER, 0);
	printf("mpss: %u: the lock is released (it reads %u)\n", step, rd(smem_lock.base + LOCK_OWNER));
	lck_mtx_unlock(&hakios_smem_alloc_lock);
	return made;
}

static void glink_reset(void);

static bool
step_our_item(void)
{
	struct smem_partition header;
	struct smp2p ours;
	uint32_t cacheline = 0, size = 0, partition, name[4] = { 0 };

	announce("look for an SMP2P item of ours (428) in our partition with the modem, make it if there is none, and fill it in");
	partition = smem_partition(SMEM_US, SMEM_MPSS, &cacheline, &size);
	if (partition == 0 || !smem_peek(partition, &header, sizeof(header))) {
		printf("mpss: %u: there is no such partition\n", step);
		return false;
	}
	our_item = our_item_find();
	if (our_item != 0) {
		printf("mpss: %u: item %u is there, at %#lx: a kernel before this one made it, and it is used again\n", step,
		    ITEM_SMP2P_OURS, smem_phys + our_item);
	} else {
		if (!our_item_make(partition, ITEM_SMP2P_OURS, sizeof(struct smp2p))) {
			return false;
		}
		our_item = our_item_find();
		if (our_item == 0) {
			printf("mpss: %u: THE ITEM IS NOT FOUND where it was just made\n", step);
			return false;
		}
		printf("mpss: %u: item %u made, at %#lx\n", step, ITEM_SMP2P_OURS, smem_phys + our_item);
	}
	/* zeroes; from processor 0 to 1, room for 16 entries, "master-kernel" at zero, feature bit 0; last the version */
	snprintf(last_write, sizeof(last_write), "step %u filled in our SMP2P item", step);
	for (uint32_t at = 0; at < sizeof(ours); at += 4) {
		smem_poke(our_item + at, 0);
	}
	settle();
	smem_poke(our_item, SMP2P_MAGIC);
	smem_poke(our_item + SMP2P_AT_PROCESSORS, SMEM_US | SMEM_MPSS << 16);
	strlcpy((char *)name, OUR_ENTRY_STOP, sizeof(name));
	for (uint32_t word = 0; word < 4; word++) {
		smem_poke(our_item + SMP2P_AT_NAME(0) + 4 * word, name[word]);
	}
	/*
	 * A second entry, "ipa" (dt: qcom,smp2p-ipa-1-out of qcom,smp2p-modem): the IPA driver's answer to the
	 * modem's question whether the IPA's clock is on, as Linux's ipa_smp2p.c gives it: bit 0 "the answer
	 * is valid", bit 1 "the clock is on". The modem, told so, keeps away from the IPA (there is no IPA
	 * driver here). Off unless modem_ipa=1 (valid, off) or 3 (on): with the entry the run of 23 October hung.
	 */
	{
		uint32_t ipa = 3;                       /* (the modem's IPA driver waits for this entry to exist: see the IPA's handshake) */

		PE_parse_boot_argn("modem_ipa", &ipa, sizeof(ipa));
		if (ipa != 0) {
			bzero(name, sizeof(name));
			strlcpy((char *)name, "ipa", sizeof(name));
			for (uint32_t word = 0; word < 4; word++) {
				smem_poke(our_item + SMP2P_AT_NAME(1) + 4 * word, name[word]);
			}
			smem_poke(our_item + SMP2P_AT_VALUE(1), ipa);
			printf("mpss: %u: our entry \"ipa\" = %#x\n", step, ipa);
		}
		smem_poke(our_item + SMP2P_AT_COUNTS, 16 | (ipa != 0 ? 2u : 1u) << 16);
	}
	smem_poke(our_item + SMP2P_AT_VERSION, SMP2P_RESTART_ACKS << 8);
	settle();
	smem_poke(our_item + SMP2P_AT_VERSION, SMP2P_RESTART_ACKS << 8 | SMP2P_VERSION);
	settle();
	smp2p_agreed = smp2p_restart_acks = smp2p_restart_acked = false;
	need(&ipcc);
	wr(ipcc.base + IPCC_SEND, IPCC_MPSS << 16 | IPCC_SMP2P);
	if (!smem_peek(our_item, &ours, sizeof(ours)) || ours.magic != SMP2P_MAGIC || smp2p_entry(&ours, OUR_ENTRY_STOP) < 0) {
		printf("mpss: %u: OUR ITEM DOES NOT READ BACK AS WRITTEN\n", step);
		return false;
	}
	show_words("our item reads back", &ours);
	/*
	 * The IPA's item: SMEM_IPA_FILTER_TABLE (497), 0x9000 bytes (dt: qcom,ipa-q6-smem-size of ipa_smmu_ap), which
	 * Android's IPA driver allocates in this partition at its probe (techpack/dataipa, ipa_smmu_ap_cb_probe():
	 * qcom_smem_alloc(SMEM_MODEM, SMEM_IPA_FILTER_TABLE, size)) and maps into the IPA's SMMU context as it is:
	 * the modem puts its filter tables there for the IPA to read. The modem looks for it when its IPA driver
	 * initializes; before the IPA ran here (22 October) an item of 0x2000 made the modem give up before any
	 * service, since it then went on to the unclocked IPA. Now that the IPA runs, made by default, zeroed
	 * (modem_ipa_item=0: not; our SMMU context maps all of DDR, so the item needs no mapping of its own).
	 */
	{
		uint32_t ipa_item = 497, ipa_size = 0;

		PE_parse_boot_argn("modem_ipa_item", &ipa_item, sizeof(ipa_item));
		if (ipa_item != 0) {
			if (smem_item(ipa_item, &ipa_size) != 0) {
				printf("mpss: %u: the IPA's item %u is there already (%u bytes)\n", step, ipa_item, ipa_size);
			} else if (our_item_make(partition, ipa_item, 0x9000)) {
				tell("mpss: %u: THE IPA'S ITEM %u MADE, 0x9000 bytes, zeroed (the modem's filter tables go there)\n", step, ipa_item);
			} else {
				printf("mpss: %u: the IPA's item %u could not be made\n", step, ipa_item);
			}
		}
	}
	glink_reset();
	return true;
}


/* ---- the IPA's firmware, loaded before the modem (Android's pil "ipa_fws") -------------------------- */

/*
 * The IPA (the SoC's packet engine, the modem's path for data) has a small firmware of its own,
 * which Android's IPA driver has TrustZone load before the modem starts (dt: qcom,ipa_fws, image
 * 15, yupik_ipa_fws.mdt + .b*, into reserved-memory ipa_fw@8b700000 and ipa_gsi@8b710000). A modem
 * that goes online without it dies in its LTE downlink task (23 October: "EX:kernel:0x0:enl2_dl",
 * a null pointer, twice of two). Nothing else of the IPA is done here yet: no clock, no tables.
 * modem_ipa_fw=0 leaves it out.
 */
#define PAS_IPA_DEFAULT         15              /* dt: qcom,pas-id of qcom,ipa_fws */
static uint32_t pas_ipa = PAS_IPA_DEFAULT;
/*
 * Where the image goes. Android's pil (its dmesg: "yupik_ipa_fws: loading from 0x8b710000 to 0x8b717000")
 * puts it at the ipa_gsi reservation (0x8b710000, 0xa000; gpu_micro_code follows at 0x8b71a000), not at
 * ipa_fw@8b700000 as the reservations' names suggest; loaded at 0x8b700000 (runs up to 23 of 24 October)
 * TrustZone started it and GSI came up, but the microcontroller never wrote "booted" and the modem,
 * which starts it, never answered INIT_DRIVER. Should the microcontroller fetch its code from DDR at
 * the address Android uses, this is why. modem_ipa_fw_at=ADDRESS chooses (0x8b700000 allows 0x1a000).
 */
static uint32_t ipa_fw_memory = 0x8b710000u, ipa_fw_size = 0xa000u;


/*
 * What the IPA says of itself, read with its clock on: its shared memory register (where the local
 * memory is), the microcontroller's area at the start of that memory (Linux's ipa_uc.c: command,
 * response -- 1 once it has booted --, event, error, hardware state, interface version) and GSI's
 * status (bit 0: the GSI firmware is enabled) and error log, for our execution environment.
 */
static struct block ipa_regs = { "ipa", 0x01e40000, 0x1000 };
static struct block ipa_sram = { "ipa-shared", 0x01e50000, 0x4000 };  /* the local memory: 0x3000 bytes (SHARED_MEM_SIZE) */
static struct block gsi_ee0 = { "gsi-ee0", 0x01e16000, 0x1000 };

static void
ipa_status(const char *when)
{
	uint32_t shared, uc0, uc3, uc5, uc7, uc8, gsi, error;

	need(&ipa_regs);
	need(&ipa_sram);
	need(&gsi_ee0);
	shared = rd(ipa_regs.base + 0x54);
	uc0 = rd(ipa_sram.base + 0x00);         /* command (byte 0) */
	uc3 = rd(ipa_sram.base + 0x0c);         /* response (byte 0) */
	uc5 = rd(ipa_sram.base + 0x14);         /* event (byte 0) */
	uc7 = rd(ipa_sram.base + 0x1c);         /* first error address */
	uc8 = rd(ipa_sram.base + 0x20);         /* hw_state (byte 0), warning counter (byte 1) */
	gsi = rd(gsi_ee0.base + 0x000);         /* GSI_STATUS */
	error = rd(gsi_ee0.base + 0x200);       /* ERROR_LOG */
	tell("mpss: ipa: %s: shared memory %#x (offset %u, %u bytes); uC command %#x response %#x (1: booted) event %#x error address %#x "
	    "hw_state %u warnings %u, interface %#x; GSI status %#x (%s), error log %#x\n", when, shared, 8 * (shared >> 16),
	    8 * (shared & 0xffff), uc0 & 0xff, uc3 & 0xff, uc5 & 0xff, uc7, uc8 & 0xff, (uc8 >> 8) & 0xff, rd(ipa_sram.base + 0x24) & 0xffff,
	    gsi, gsi & 1 ? "ENABLED" : "NOT ENABLED", error);
	ipa_mem_offset = 8 * (shared >> 16);
}


/*
 * The IPA's local memory as Linux's ipa_mem_config() and ipa_mem_setup() leave it before the modem
 * is told of it: zeroed, with "canaries" (0xdeadbeef) in the words before the regions that have
 * them (IPA v4.11: ipa_data-v4.11.c), and the register that says where the modem's processing
 * contexts begin. The modem reads this memory for the microcontroller's state and its own tables;
 * what it finds there after power-on is whatever was left.
 */
static void
ipa_memory_init(void)
{
	static const struct {
		uint16_t offset, canaries;
	} regions[] = {
		/* Android's _ipa_init_sram_v3() for IPA 4.11 (its ipa_4_11_mem_part): two before each table and the modem's
		 * headers and processing contexts, one 16 bytes before the NAT table (0x17a0) and the modem's quota statistics
		 * (0x1d00), two before the modem's region (0x27d8), one at the end */
		{ 0x0288, 2 }, { 0x0308, 2 }, { 0x0388, 2 }, { 0x0408, 2 }, { 0x0488, 2 }, { 0x0508, 2 }, { 0x0588, 2 }, { 0x0608, 2 },
		{ 0x0688, 2 }, { 0x0ad0, 2 }, { 0x1794, 1 }, { 0x1cf4, 1 }, { 0x27d8, 2 }, { 0x3000, 1 },
	};
	const uint32_t size = 0x3000;

	need(&ipa_sram);
	need(&ipa_regs);
	/* from 0x288: the microcontroller's own area (its shared words at 0, its info at 0x80) is left alone, as Linux leaves it */
	for (uint32_t at = 0x288; at < size; at += 4) {
		wr(ipa_sram.base + ipa_mem_offset + at, 0);
	}
	for (unsigned r = 0; r < sizeof(regions) / sizeof(regions[0]); r++) {
		for (unsigned c = 1; c <= regions[r].canaries; c++) {
			wr(ipa_sram.base + ipa_mem_offset + regions[r].offset - 4 * c, 0xdeadbeef);
		}
	}
	wr(ipa_regs.base + 0x1e8, ipa_mem_offset + 0x0ad0);    /* LOCAL_PKT_PROC_CNTXT: the modem's processing contexts */
	settle();
	printf("mpss: ipa: its local memory zeroed (0x288..%#x at offset %#x), the canaries written, the processing contexts' base set\n",
	    size, ipa_mem_offset);
}


/*
 * What Android's IPA driver programs into the IPA after its firmware starts and before the modem
 * boots (techpack/dataipa, ipa3_post_init(): ipa3_init_hw(), ipa3_set_resorce_groups_min_max_limits(),
 * ipa3_cfg_route()), as read from its source for IPA 4.11 and from the live registers of the phone's
 * Android (debugfs gen_reg): the QSB masters' outstanding reads and writes, the Qtime timers, the
 * GSI's in-order bits of COMP_CFG, the idle indication for clock division, the resource groups'
 * limits (without them nothing can be processed: the modem's own commands to the IPA would never
 * complete, and its answer to INIT_DRIVER never come), and the default route. Nothing of the
 * application processor's own pipes: there is no data path on this side yet.
 */
#define IPA_COMP_CFG            0x3c
#define IPA_ROUTE               0x48
#define IPA_QSB_MAX_WRITES      0x74
#define IPA_QSB_MAX_READS       0x78
#define IPA_IDLE_INDICATION_CFG 0x240
#define IPA_QTIME_TIMESTAMP_CFG 0x24c
#define IPA_TIMERS_XO_CLK_DIV_CFG 0x250
#define IPA_TIMERS_PULSE_GRAN_CFG 0x254
#define IPA_SRC_RSRC_GRP_01(n)  (0x400 + 0x20 * (n))    /* groups 0 and 1: x_min [5:0] x_max [13:8] y_min [21:16] y_max [29:24] */
#define IPA_SRC_RSRC_GRP_23(n)  (0x404 + 0x20 * (n))
#define IPA_DST_RSRC_GRP_01(n)  (0x500 + 0x20 * (n))
#define IPA_DST_RSRC_GRP_23(n)  (0x504 + 0x20 * (n))
#define RSRC(xmin, xmax, ymin, ymax) ((uint32_t)(xmin) | (uint32_t)(xmax) << 8 | (uint32_t)(ymin) << 16 | (uint32_t)(ymax) << 24)

static void
ipa_hardware_init(void)
{
	static const uint8_t source[5] = { 6, 8, 18, 2, 15 };  /* packet contexts, descriptor lists, descriptor buffers, HPS DMARs, ACK entries: group 0 (UL/DL) */
	uint32_t value;

	need(&ipa_regs);
	/* QSB: 12 outstanding writes, 13 reads of 120 beats on the DDR master (ipa3_qmb_outstanding[IPA_4_11]) */
	value = rd(ipa_regs.base + IPA_QSB_MAX_WRITES);
	wr(ipa_regs.base + IPA_QSB_MAX_WRITES, (value & ~0xfu) | 12);
	value = rd(ipa_regs.base + IPA_QSB_MAX_READS);
	wr(ipa_regs.base + IPA_QSB_MAX_READS, (value & ~0x00ff000fu) | 13 | 120u << 16);
	/* Qtime: the divider off, the timestamps and pulse generators as Android leaves them, the divider 192 on */
	wr(ipa_regs.base + IPA_TIMERS_XO_CLK_DIV_CFG, 0);
	wr(ipa_regs.base + IPA_QTIME_TIMESTAMP_CFG, 0x180e8e);
	wr(ipa_regs.base + IPA_TIMERS_PULSE_GRAN_CFG, 0x123);
	wr(ipa_regs.base + IPA_TIMERS_XO_CLK_DIV_CFG, 0xbf);
	wr(ipa_regs.base + IPA_TIMERS_XO_CLK_DIV_CFG, 0x800000bf);
	/* COMP_CFG: GSI_MULTI_INORDER_RD_DIS and _WR_DIS */
	value = rd(ipa_regs.base + IPA_COMP_CFG);
	wr(ipa_regs.base + IPA_COMP_CFG, value | 1u << 7 | 1u << 8);
	/* dynamic clock division: enter idle after 256, no constant non-idle */
	wr(ipa_regs.base + IPA_IDLE_INDICATION_CFG, 256);
	/* the resource groups: the sources' five types for group 0 alone; the destinations' data sectors 3 for group 0
	 * and 25 for group 2 (DRB_IP), DPS DMARs 2 for group 0 */
	for (unsigned n = 0; n < 5; n++) {
		wr(ipa_regs.base + IPA_SRC_RSRC_GRP_01(n), RSRC(source[n], source[n], 0, 0));
	}
	wr(ipa_regs.base + IPA_DST_RSRC_GRP_01(0), RSRC(3, 3, 0, 0));
	wr(ipa_regs.base + IPA_DST_RSRC_GRP_23(0), RSRC(25, 25, 0, 0));
	wr(ipa_regs.base + IPA_DST_RSRC_GRP_01(1), RSRC(2, 2, 0, 0));
	wr(ipa_regs.base + IPA_DST_RSRC_GRP_23(1), 0);
	/* the default route as Android's: not disabled, default pipe 9, header table 1, fragments to pipe 9, header retained */
	wr(ipa_regs.base + IPA_ROUTE, 0x1120052);
	settle();
	printf("mpss: ipa: the hardware set up as Android's driver does before the modem: QSB %#x %#x, COMP_CFG %#x, idle %#x, route %#x, "
	    "source group 0 %u/%u/%u/%u/%u, destination sectors %#x %#x\n", rd(ipa_regs.base + IPA_QSB_MAX_WRITES),
	    rd(ipa_regs.base + IPA_QSB_MAX_READS), rd(ipa_regs.base + IPA_COMP_CFG), rd(ipa_regs.base + IPA_IDLE_INDICATION_CFG),
	    rd(ipa_regs.base + IPA_ROUTE), rd(ipa_regs.base + IPA_SRC_RSRC_GRP_01(0)) & 0x3f, rd(ipa_regs.base + IPA_SRC_RSRC_GRP_01(1)) & 0x3f,
	    rd(ipa_regs.base + IPA_SRC_RSRC_GRP_01(2)) & 0x3f, rd(ipa_regs.base + IPA_SRC_RSRC_GRP_01(3)) & 0x3f,
	    rd(ipa_regs.base + IPA_SRC_RSRC_GRP_01(4)) & 0x3f, rd(ipa_regs.base + IPA_DST_RSRC_GRP_01(0)), rd(ipa_regs.base + IPA_DST_RSRC_GRP_23(0)));
}


/*
 * The IPA reaches memory through the application processors' SMMU (dt of Linux: iommus 0x480 and
 * 0x482 of apps_smmu), where a stream nobody matches faults, and the hypervisor grants no bypass
 * (tried on 24 October: the S2CR stayed "fault"). So, as the Wi-Fi's radio has (hakios.wifi.c,
 * radio_space_on(), where the formats and the order of programming are explained): a translation
 * context of our own, with the first free match register and the lowest free context, and page
 * tables of ours. The IPA, though, is shown memory as it is: one table of 2 MB blocks maps the
 * gigabyte of DDR from 0x80000000 (the shared memory, the modem's memory, our buffers) onto
 * itself, uncached, and one block the SoC's internal memory at 0x146a8000 (Linux's ipa_mem_data:
 * imem), as a device. Put back when the IPA is stopped (modem_ipa_smmu_off=1: not done at all).
 *
 * The hypervisor checks what is written to a match register against the masters it knows (the
 * device tree's "iommus" entries, stream and mask): on 24 October one register with stream 0x480
 * and the mask 0x2 (both streams at once) was refused -- the S2CR took, the SMR read back 0 --
 * while the Wi-Fi's register, whose mask 1 is the device tree's, was granted. The device tree
 * gives the IPA's streams the mask 0, so each takes a match register of its own, with the mask 0,
 * both sending to the one context.
 */
#define IPA_STREAM_DEFAULT      0x480u
#define IPA_STREAM_UC_DEFAULT   0x482u          /* the IPA's microcontroller: its own match register */
static uint32_t ipa_stream = IPA_STREAM_DEFAULT, ipa_stream_uc = IPA_STREAM_UC_DEFAULT;  /* dt: iommus of ipa_smmu_ap and ipa_smmu_uc */
#define IPA_SPACE_BASE          0x80000000ull   /* a gigabyte from here, as it is */
#define IPA_IMEM_DEFAULT        0x14600000u   /* the 2 MB block that holds the SoC internal memory (dt: qcom,msm-imem) */
static uint32_t ipa_imem = IPA_IMEM_DEFAULT;
static struct block apps_smmu = { "apps smmu", 0x15000000, 0x2000 };
static struct block apps_smmu_second = { "apps smmu", 0, 0x1000 };
static struct block apps_smmu_context = { "apps smmu, the IPA's context", 0, 0x1000 };
#define SMMU_SCR0               0x000
#define SMMU_IDR0               0x020
#define SMMU_IDR1               0x024
#define SMMU_SMR(n)             (0x800 + 4 * (n))
#define SMMU_S2CR(n)            (0xc00 + 4 * (n))
#define SMMU_SGFSR              0x048
#define SMMU_SGFSYNR0           0x050
#define SMMU_SGFSYNR1           0x054
#define SMMU_CBAR(n)            (4 * (n))
#define SMMU_CBAR_STAGE_1       (1u << 16 | 3u << 8 | 0xfu << 12)
#define SMMU_CBA2R(n)           (0x800 + 4 * (n))
#define SMMU_CBA2R_64_BIT       1
#define SMMU_CB_SCTLR           0x00
#define SMMU_CB_TCR2            0x10
#define SMMU_CB_TTBR0           0x20
#define SMMU_CB_TTBR1           0x28
#define SMMU_CB_TCR             0x30
#define SMMU_CB_MAIR0           0x38
#define SMMU_CB_MAIR1           0x3c
#define SMMU_CB_FSR             0x58
#define SMMU_CB_FAR             0x60
#define SMMU_CB_FSYNR0          0x68
#define SMMU_CB_TLBIASID        0x610
#define SMMU_CB_TLBSYNC         0x7f0
#define SMMU_CB_TLBSTATUS       0x7f4
#define SMMU_SCTLR_ON           (1u << 0 | 1u << 1 | 1u << 2 | 1u << 5 | 1u << 8 | 1u << 12)
#define SMMU_TCR_39_BITS        (25u | 1u << 8 | 1u << 10 | 3u << 12 | 1u << 23)
#define SMMU_TCR2               (1u | 1u << 4 | 7u << 15)
#define SMMU_MAIR0              0xf404ff44      /* 0 uncached, 1 cached, 2 device */
#define SMMU_FSR_ALL            0xc00001fe
#define IPA_BLOCK               (1ull | 0ull << 2 | 1ull << 6 | 2ull << 8 | 1ull << 10 | 1ull << 11)  /* a 2 MB block, uncached, RW, shared */
#define IPA_BLOCK_DEVICE        (1ull | 2ull << 2 | 1ull << 6 | 2ull << 8 | 1ull << 10 | 1ull << 11)
#define IPA_TABLE               3ull            /* a table entry of the root */
extern lck_mtx_t hakios_smmu_lock;
static struct {
	bool on;
	uint32_t page, pages, match, match_uc, context, old_smr, old_s2cr, old_smr_uc, old_s2cr_uc, old_sctlr;
	vm_offset_t tables;                     /* three pages: the root, the DDR gigabyte's blocks, the SoC's */
} ipa_space;

static void
ipa_space_faults(const char *when)
{
	uint32_t status, global;

	if (!ipa_space.on) {
		return;
	}
	status = rd(apps_smmu_context.base + SMMU_CB_FSR);
	global = rd(apps_smmu.base + SMMU_SGFSR);
	if ((status & SMMU_FSR_ALL) == 0 && global == 0) {
		printf("mpss: ipa: %s: the SMMU has refused the IPA nothing (context %u)\n", when, ipa_space.context);
		return;
	}
	tell("mpss: ipa: %s: THE SMMU HAS REFUSED A REQUEST OF THE IPA'S: context %u: status %#x, address %#llx, syndrome %#x; the SMMU "
	    "as a whole: status %#x, syndromes %#x %#x\n", when, ipa_space.context, status,
	    (uint64_t)rd(apps_smmu_context.base + SMMU_CB_FAR + 4) << 32 | rd(apps_smmu_context.base + SMMU_CB_FAR),
	    rd(apps_smmu_context.base + SMMU_CB_FSYNR0), global, rd(apps_smmu.base + SMMU_SGFSYNR0), rd(apps_smmu.base + SMMU_SGFSYNR1));
	wr(apps_smmu_context.base + SMMU_CB_FSR, SMMU_FSR_ALL);
}

static bool
ipa_space_on(void)
{
	bool used[256] = { false };
	uint32_t scr0, idr0, idr1, matches, contexts, sees = 1u << 31 | ipa_stream, sees_uc = 1u << 31 | ipa_stream_uc;
	uint64_t *root, *ddr, *soc, ttbr;

	if (ipa_space.on) {
		return true;
	}
	lck_mtx_lock(&hakios_smmu_lock);
	need(&apps_smmu);
	scr0 = rd(apps_smmu.base + SMMU_SCR0);
	idr0 = rd(apps_smmu.base + SMMU_IDR0);
	idr1 = rd(apps_smmu.base + SMMU_IDR1);
	matches = idr0 & 0xff;
	contexts = idr1 & 0xff;
	ipa_space.page = idr1 >> 31 ? 0x10000 : 0x1000;
	ipa_space.pages = 2u << (idr1 >> 28 & 7);
	if ((scr0 & 1u << 3) || 2 * ipa_space.pages * ipa_space.page != 0x100000) {
		lck_mtx_unlock(&hakios_smmu_lock);
		printf("mpss: ipa: the SMMU is not of the shape this knows (scr0 %#x, idr1 %#x)\n", scr0, idr1);
		return false;
	}
	ipa_space.match = ipa_space.match_uc = ~0u;
	for (uint32_t n = 0; n < matches && SMMU_SMR(n) < 0xc00; n++) {
		uint32_t smr = rd(apps_smmu.base + SMMU_SMR(n)), s2cr = rd(apps_smmu.base + SMMU_S2CR(n));

		if (smr >> 31) {
			used[s2cr & 0xff] = true;
		} else if (ipa_space.match == ~0u) {
			ipa_space.match = n;
			ipa_space.old_smr = smr;
			ipa_space.old_s2cr = s2cr;
		} else if (ipa_space.match_uc == ~0u) {
			ipa_space.match_uc = n;
			ipa_space.old_smr_uc = smr;
			ipa_space.old_s2cr_uc = s2cr;
		}
	}
	for (ipa_space.context = 0; ipa_space.context < contexts && used[ipa_space.context]; ipa_space.context++) {
	}
	if (ipa_space.match == ~0u || ipa_space.match_uc == ~0u || ipa_space.context >= contexts) {
		lck_mtx_unlock(&hakios_smmu_lock);
		printf("mpss: ipa: the SMMU has not two match registers or no context left\n");
		return false;
	}
	apps_smmu_second.phys = apps_smmu.phys + ipa_space.page;
	apps_smmu_context.phys = apps_smmu.phys + (ipa_space.pages + ipa_space.context) * ipa_space.page;
	need(&apps_smmu_second);
	need(&apps_smmu_context);
	ipa_space.old_sctlr = rd(apps_smmu_context.base + SMMU_CB_SCTLR);
	if (ipa_space.old_sctlr & 1) {
		lck_mtx_unlock(&hakios_smmu_lock);
		printf("mpss: ipa: context %u of the SMMU is switched on although no stream goes through it: left alone\n", ipa_space.context);
		return false;
	}
	if (ipa_space.tables == 0 &&
	    kmem_alloc(kernel_map, &ipa_space.tables, 3 * 4096, KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
		lck_mtx_unlock(&hakios_smmu_lock);
		return false;
	}
	root = (uint64_t *)ipa_space.tables;
	ddr = (uint64_t *)(ipa_space.tables + 4096);
	soc = (uint64_t *)(ipa_space.tables + 8192);
	bzero(root, 3 * 4096);
	for (unsigned i = 0; i < 512; i++) {
		ddr[i] = (IPA_SPACE_BASE + (uint64_t)i * 0x200000) | IPA_BLOCK;
	}
	soc[(ipa_imem >> 21) & 0x1ff] = ipa_imem | IPA_BLOCK_DEVICE;
	root[IPA_SPACE_BASE >> 30] = kvtophys((vm_offset_t)ddr) | IPA_TABLE;
	/* the rest of DDR (the phone has 6 or 8 GB from 0x80000000), as 1 GB blocks of the root: the data path's rings and
	 * buffers (hakios.ipa.c) are wherever the kernel's memory is */
	for (unsigned g = (IPA_SPACE_BASE >> 30) + 1; g < (IPA_SPACE_BASE >> 30) + 8; g++) {
		root[g] = ((uint64_t)g << 30) | IPA_BLOCK;
	}
	root[ipa_imem >> 30] = kvtophys((vm_offset_t)soc) | IPA_TABLE;
	FlushPoC_DcacheRegion(ipa_space.tables, 3 * 4096);
	settle();
	ttbr = kvtophys(ipa_space.tables) | (uint64_t)ipa_space.context << 48;
	printf("mpss: ipa: THE SMMU IS ABOUT TO BE WRITTEN: context %u gets our tables (root %#llx: DDR from %#llx as it is, 1 GB, and "
	    "the SoC's memory at %#llx) and match registers %u and %u send streams %#x and %#x through it (before: smr %#x s2cr %#x, "
	    "smr %#x s2cr %#x, sctlr %#x)\n", ipa_space.context, ttbr & 0x0000fffffffff000ull, IPA_SPACE_BASE, ipa_imem, ipa_space.match,
	    ipa_space.match_uc, ipa_stream, ipa_stream_uc, ipa_space.old_smr, ipa_space.old_s2cr, ipa_space.old_smr_uc, ipa_space.old_s2cr_uc,
	    ipa_space.old_sctlr);
	wr(apps_smmu_context.base + SMMU_CB_SCTLR, 0);
	wr(apps_smmu_second.base + SMMU_CBA2R(ipa_space.context), SMMU_CBA2R_64_BIT);
	wr(apps_smmu_second.base + SMMU_CBAR(ipa_space.context), SMMU_CBAR_STAGE_1);
	wr(apps_smmu_context.base + SMMU_CB_TCR2, SMMU_TCR2);
	wr(apps_smmu_context.base + SMMU_CB_TCR, SMMU_TCR_39_BITS);
	wr(apps_smmu_context.base + SMMU_CB_TTBR0, (uint32_t)ttbr);
	wr(apps_smmu_context.base + SMMU_CB_TTBR0 + 4, (uint32_t)(ttbr >> 32));
	wr(apps_smmu_context.base + SMMU_CB_TTBR1, 0);
	wr(apps_smmu_context.base + SMMU_CB_TTBR1 + 4, 0);
	wr(apps_smmu_context.base + SMMU_CB_MAIR0, SMMU_MAIR0);
	wr(apps_smmu_context.base + SMMU_CB_MAIR1, 0);
	wr(apps_smmu_context.base + SMMU_CB_FSR, SMMU_FSR_ALL);
	wr(apps_smmu_context.base + SMMU_CB_TLBIASID, ipa_space.context);
	wr(apps_smmu_context.base + SMMU_CB_TLBSYNC, 1);
	for (unsigned ms = 0; rd(apps_smmu_context.base + SMMU_CB_TLBSTATUS) & 1; ms++) {
		if (ms == 100) {
			lck_mtx_unlock(&hakios_smmu_lock);
			printf("mpss: ipa: the SMMU does not finish forgetting old translations\n");
			return false;
		}
		sleep_ms(1);
	}
	wr(apps_smmu_context.base + SMMU_CB_SCTLR, SMMU_SCTLR_ON);
	ipa_space.on = true;
	wr(apps_smmu.base + SMMU_S2CR(ipa_space.match), ipa_space.context);
	wr(apps_smmu.base + SMMU_SMR(ipa_space.match), sees);
	wr(apps_smmu.base + SMMU_S2CR(ipa_space.match_uc), ipa_space.context);
	wr(apps_smmu.base + SMMU_SMR(ipa_space.match_uc), sees_uc);
	if (rd(apps_smmu.base + SMMU_SMR(ipa_space.match)) != sees ||
	    (rd(apps_smmu.base + SMMU_S2CR(ipa_space.match)) & 0x300ff) != ipa_space.context ||
	    rd(apps_smmu.base + SMMU_SMR(ipa_space.match_uc)) != sees_uc ||
	    (rd(apps_smmu.base + SMMU_S2CR(ipa_space.match_uc)) & 0x300ff) != ipa_space.context ||
	    !(rd(apps_smmu_context.base + SMMU_CB_SCTLR) & 1) || rd(apps_smmu_context.base + SMMU_CB_TTBR0) != (uint32_t)ttbr) {
		lck_mtx_unlock(&hakios_smmu_lock);
		printf("mpss: ipa: THE SMMU DID NOT TAKE WHAT WAS WRITTEN: streams %#x and %#x are not both going through context %u (smr %#x "
		    "s2cr %#x, smr %#x s2cr %#x, sctlr %#x)\n", ipa_stream, ipa_stream_uc, ipa_space.context,
		    rd(apps_smmu.base + SMMU_SMR(ipa_space.match)), rd(apps_smmu.base + SMMU_S2CR(ipa_space.match)),
		    rd(apps_smmu.base + SMMU_SMR(ipa_space.match_uc)), rd(apps_smmu.base + SMMU_S2CR(ipa_space.match_uc)),
		    rd(apps_smmu_context.base + SMMU_CB_SCTLR));
		return false;
	}
	lck_mtx_unlock(&hakios_smmu_lock);
	tell("mpss: ipa: THE IPA'S STREAMS GO THROUGH CONTEXT %u OF THE SMMU, with our tables (match registers %u and %u)\n", ipa_space.context,
	    ipa_space.match, ipa_space.match_uc);
	return true;
}

static void
ipa_space_off(void)
{
	if (!ipa_space.on) {
		return;
	}
	ipa_space_faults("at the end");
	lck_mtx_lock(&hakios_smmu_lock);
	wr(apps_smmu.base + SMMU_SMR(ipa_space.match), ipa_space.old_smr);
	wr(apps_smmu.base + SMMU_S2CR(ipa_space.match), ipa_space.old_s2cr);
	wr(apps_smmu.base + SMMU_SMR(ipa_space.match_uc), ipa_space.old_smr_uc);
	wr(apps_smmu.base + SMMU_S2CR(ipa_space.match_uc), ipa_space.old_s2cr_uc);
	wr(apps_smmu_context.base + SMMU_CB_SCTLR, ipa_space.old_sctlr);
	lck_mtx_unlock(&hakios_smmu_lock);
	ipa_space.on = false;
	printf("mpss: ipa: the IPA's streams and context %u of the SMMU are put back as they were found\n", ipa_space.context);
}

static bool
ipa_firmware_load(void)
{
	const uint8_t *header, *segment_file[8] = { NULL };
	uint64_t header_size = 0, segment_size[8] = { 0 }, answer = ~0ull;
	struct elf_segment segments[8];
	volatile uint8_t *memory;
	vm_offset_t page = 0;
	uint32_t count, loaded = 0, highest = 0, size;
	int32_t result;
	bool ok = false;

	PE_parse_boot_argn("modem_ipa_fw_at", &ipa_fw_memory, sizeof(ipa_fw_memory));
	ipa_fw_size = ipa_fw_memory == 0x8b700000u ? 0x1a000u : 0xa000u;
	announce("turn the IPA's clock on (RPMh), load the IPA's firmware (image 15) into its memory and have TrustZone start it, as "
	    "Android does before the modem");
	printf("mpss: %u: the IPA's image goes at %#x (%#x bytes allowed)\n", step, ipa_fw_memory, ipa_fw_size);
	if (!ipa_clock(true)) {
		printf("mpss: %u: the IPA's clock is not on: its firmware is not loaded (a start without the clock ends the kernel)\n", step);
		return false;
	}
	if (!option("modem_ipa_smmu_off")) {
		ipa_space_on();
	}
	{
		/* the IPA's firmware file is SoC-specific (dt: qcom,firmware-name of qcom,ipa_fws; "yupik_ipa_fws"
		 * on the first phone, something else on another SoC); soc_ipa_fw= overrides the name */
		char fw[80], path[96];

		strlcpy(fw, "yupik_ipa_fws", sizeof(fw));
		PE_parse_boot_argn("soc_ipa_fw", fw, sizeof(fw));
		snprintf(path, sizeof(path), "image/%s.mdt", fw);
		header = root_file(path, &header_size);
		if (header == NULL) {
			printf("mpss: %u: no fw/%s: the IPA's firmware is not loaded\n", step, path);
			return false;
		}
	}
	count = header_size >= 52 ? header[44] | header[45] << 8 : 0;
	if (memcmp(header, "\177ELF\1", 5) != 0 || count == 0 || count > 8 || 52 + 32 * count > header_size) {
		printf("mpss: %u: yupik_ipa_fws.mdt is not such a header (%u segments)\n", step, count);
		goto out;
	}
	/* (the image is relocatable, bit 27 of the flags: its addresses count from the lowest, which goes at the region's start) */
	for (uint32_t i = 0; i < count; i++) {
		memcpy(&segments[i], header + 52 + 32 * i, sizeof(segments[i]));
	}
	{
		uint32_t lowest = ~0u;
		bool relocatable = false;

		for (uint32_t i = 0; i < count; i++) {
			if (segments[i].type == 1 && !MDT_HASH(segments[i].flags) && segments[i].memory_size != 0) {
				relocatable |= (segments[i].flags & (1u << 27)) != 0;
				lowest = segments[i].paddr < lowest ? segments[i].paddr : lowest;
			}
		}
		if (relocatable && lowest != ~0u) {
			for (uint32_t i = 0; i < count; i++) {
				segments[i].paddr = segments[i].paddr - lowest + ipa_fw_memory;
			}
		}
	}
	for (uint32_t i = 0; i < count; i++) {
		char name[48];

		if (segments[i].type != 1 || MDT_HASH(segments[i].flags) || segments[i].memory_size == 0) {
			continue;
		}
		if (segments[i].paddr < ipa_fw_memory || (uint64_t)segments[i].paddr + segments[i].memory_size > ipa_fw_memory + ipa_fw_size) {
			printf("mpss: %u: the IPA's segment %u (%#x, %#x bytes) is not within its memory\n", step, i, segments[i].paddr,
			    segments[i].memory_size);
			goto out;
		}
		if (segments[i].file_size != 0) {
			snprintf(name, sizeof(name), "image/yupik_ipa_fws.b%02u", i);
			segment_file[i] = root_file(name, &segment_size[i]);
			if (segment_file[i] == NULL || segment_size[i] != segments[i].file_size) {
				printf("mpss: %u: fw/%s: %s\n", step, name, segment_file[i] == NULL ? "not there" : "not the size the header says");
				goto out;
			}
		}
		if (segments[i].paddr + segments[i].memory_size > highest) {
			highest = segments[i].paddr + segments[i].memory_size;
		}
		loaded++;
	}
	size = ((highest + 0xfff) & ~0xfffu) - ipa_fw_memory;
	printf("mpss: %u: the IPA's image: %u segments to load, %#x bytes from %#x\n", step, loaded, size, ipa_fw_memory);
	if (kmem_alloc(kernel_map, &page, PAGE_SIZE, KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
		goto out;
	}
	memcpy((void *)page, header, header_size < PAGE_SIZE ? header_size : PAGE_SIZE);
	for (vm_offset_t line = page; line < page + PAGE_SIZE; line += 64) {
		__asm__ volatile ("dc civac, %0" : : "r" (line));
	}
	settle();
	result = scm("to take the IPA's header", SCM_PAS_INIT_IMAGE, SCM_VALUE_AND_BUFFER, pas_ipa, kvtophys(page), 0, 0, &answer);
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT TAKE THE IPA'S HEADER (%d, %#llx)\n", step, result, answer);
		goto out;
	}
	result = scm("where the IPA's image will be", SCM_PAS_MEM_SETUP, SCM_THREE_VALUES, pas_ipa, ipa_fw_memory, size, 0, &answer);
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT TAKE THE IPA'S PLACE (%d, %#llx)\n", step, result, answer);
		goto out;
	}
	memory = (volatile uint8_t *)ml_io_map_unmappable(ipa_fw_memory, size, VM_WIMG_WCOMB);
	if (memory == NULL) {
		goto out;
	}
	for (uint32_t i = 0; i < count; i++) {
		uint32_t at = segments[i].paddr - ipa_fw_memory;

		if (segments[i].type != 1 || MDT_HASH(segments[i].flags) || segments[i].memory_size == 0) {
			continue;
		}
		for (uint32_t b = 0; b < segments[i].memory_size; b++) {
			memory[at + b] = b < segments[i].file_size ? segment_file[i][b] : 0;
		}
	}
	settle();
	ml_io_unmap((vm_offset_t)memory, size);
	result = scm("to authenticate the IPA's image and start it", SCM_PAS_AUTH_AND_RESET, SCM_ONE_VALUE, pas_ipa, 0, 0, 0, &answer);
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT START THE IPA'S FIRMWARE (%d, %#llx)\n", step, result, answer);
		goto out;
	}
	tell("mpss: %u: THE IPA'S FIRMWARE IS LOADED AND STARTED (image 15, %u segments)\n", step, loaded);
	/*
	 * Whether TrustZone has copied the segments where the image says they run (their virtual addresses:
	 * GSI's instruction memory at 0x1e1f000 and the microcontroller's at 0x1e81000, both in the IPA's
	 * register space) is checked by reading them back: the GSI's status bit alone does not prove its
	 * firmware runs (24 October: the modem's IPA_CTL task never answers, as if GSI answered nothing).
	 */
	for (uint32_t i = 0; i < count; i++) {
		if (segments[i].type == 1 && !MDT_HASH(segments[i].flags) && segment_file[i] != NULL && segments[i].file_size >= 16 &&
		    segments[i].vaddr >= 0x01e00000u && segments[i].vaddr < 0x01f00000u) {
			struct block iram = { "ipa iram", segments[i].vaddr & ~0xfffu, 0x8000 };
			uint32_t n = segments[i].file_size < 256 ? segments[i].file_size : 256, same = 0, first[4];

			need(&iram);
			for (uint32_t at = 0; at < n; at += 4) {
				uint32_t ours = rd(iram.base + (segments[i].vaddr & 0xfff) + at), file;

				memcpy(&file, segment_file[i] + at, 4);
				same += ours == file ? 4 : 0;
				if (at < 16) {
					first[at / 4] = ours;
				}
			}
			tell("mpss: %u: the IPA's segment %u at %#x: %u of the first %u bytes read back as the file has them (%s); it begins %#x %#x %#x %#x\n",
			    step, i, segments[i].vaddr, same, n, same == n ? "TRUSTZONE HAS PUT IT THERE" : "NOT WHAT TRUSTZONE WAS GIVEN",
			    first[0], first[1], first[2], first[3]);
		}
	}
	{
		struct block gsi = { "gsi", 0x01e04000, 0xc000 };       /* GSI_CFG at 0, MCS_CFG at 0xb000 (the panic of cycle 26: 0x1000 mapped) */

		need(&gsi);
		tell("mpss: %u: GSI_CFG %#x (bit 0 GSI enable, 1 MCS enable, 2 double MCS clock, 3 uC is MCS, 4 power collapse, 5 bypass matrix off, "
		    "[11:8] sleep clock divider), MCS_CFG %#x (bit 0: the sequencer runs)\n", step, rd(gsi.base + 0), rd(gsi.base + 0xb000));
		if (option("modem_ipa_gsi_enable")) {
			/*
			 * What Android's gsi_enable_fw() writes when the firmware was put into the instruction memory by hand
			 * rather than by TrustZone (GSI 2.5 and later): the sequencer on, then GSI on with the doubled
			 * sequencer clock and the sleep clock divided by 2. TrustZone's start is supposed to do the same.
			 */
			wr(gsi.base + 0xb000, 1);
			wr(gsi.base + 0, 1u | 1u << 2 | 2u << 8);
			settle();
			need(&gsi_ee0);
			tell("mpss: %u: GSI enabled by hand: GSI_CFG %#x, MCS_CFG %#x, status %#x\n", step, rd(gsi.base + 0), rd(gsi.base + 0xb000),
			    rd(gsi_ee0.base + 0));
		}
	}
	sleep_ms(50);
	if (!option("modem_ipa_quiet")) {
		ipa_status("after its firmware's start");
	}
	if (!option("modem_ipa_raw")) {
		ipa_memory_init();
		ipa_hardware_init();
		/* this processor's own channels and endpoints, and the IPA's tables, before the modem (hakios.ipa.c); then en1 on them */
		if (hakios_ipa_data_setup()) {
			hakios_cell_attach();
		}
	}
	ok = true;
out:
	if (page != 0) {
		kmem_free(kernel_map, page, PAGE_SIZE);
	}
	{
		uint8_t *gone = __DECONST(uint8_t *, header);

		kfree_data(gone, header_size);
	}
	for (uint32_t i = 0; i < 8; i++) {
		if (segment_file[i] != NULL) {
			uint8_t *gone = __DECONST(uint8_t *, segment_file[i]);

			kfree_data(gone, segment_size[i]);
		}
	}
	return ok;
}

static void
ipa_firmware_stop(void)
{
	uint64_t answer = ~0ull;

	(void)scm("to shut the IPA's firmware down", SCM_PAS_SHUTDOWN, SCM_ONE_VALUE, pas_ipa, 0, 0, 0, &answer);
	if (!leaving) {
		ipa_space_off();
		ipa_clock(false);
	}
}

/* ---- step 7: the DSP, started ---------------------------------------------------------------------- */

/*
 * Android's kernel (this phone, dmesg): the IPA's firmware first, then modem.mdt and its segments
 * into 0x8b800000..0x9ae00000, "Power/Clock ready", and QRTR's "ready" after that. Here the modem
 * is watched for as the audio DSP is (hakios.adsp.c, step 7): its SMP2P item, its doorbells, its
 * "slave-kernel" word, and its last words in item 421 if it gives up.
 */
static bool
step_start(void)
{
	struct smp2p words, last;
	uint64_t answer = ~0ull, started, now, ready_at = 0, limit;
	uint32_t state = 0, rang, changes = 0, wait_seconds = 10;
	int32_t result;
	bool have, spoke = false, handed_over = false;

	if (!image_written) {
		printf("mpss: %u: the image is not in place\n", step);
		return false;
	}
	PE_parse_boot_argn("modem_wait", &wait_seconds, sizeof(wait_seconds));
	bzero(&last, sizeof(last));
	have = dsp_words(&last);
	announce("listen for the modem's doorbells, ask RPMh for its rails, tell the always-on processor that its image is "
	    "loaded, and ask TrustZone to authenticate the image and let the DSP out of reset");
	need(&ipcc);
	for (unsigned i = 0; i < 8 && (rang = rd(ipcc.base + IPCC_RECEIVED)) != IPCC_NOBODY && rang >> 16 == IPCC_MPSS; i++) {
		put(&ipcc, "heard", IPCC_CLEAR, rang);
	}
	put(&ipcc, "listened for", IPCC_ENABLE, IPCC_MPSS << 16 | IPCC_SMP2P);
	put(&ipcc, "listened for", IPCC_ENABLE, IPCC_MPSS << 16 | IPCC_GLINK);
	bzero(&doorbell, sizeof(doorbell));
	lck_mtx_lock(&dsp_lock);
	{
		uint32_t ipa_fw = 1;

		PE_parse_boot_argn("modem_ipa_fw", &ipa_fw, sizeof(ipa_fw));
		ipa_loaded = ipa_fw != 0 && ipa_firmware_load();
	}
	if (option("modem_share_off")) {
		printf("mpss: %u: the buffer is not shared with the modem (modem_share_off)\n", step);
	} else if (!rmtfs_buffer_make() || !rmtfs_share(true)) {
		printf("mpss: %u: THE MODEM GETS NO BUFFER FOR ITS FILES: it is started all the same, and will offer little\n", step);
	}
	if (!rails(384)) {
		lck_mtx_unlock(&dsp_lock);
		return false;
	}
	rails_asked = true;
	if (!mailbox_send("{class: image, res: load_state, name: modem, val: on}")) {
		rails(0);
		rails_asked = false;
		lck_mtx_unlock(&dsp_lock);
		return false;
	}
	result = scm("to authenticate the modem's image and start it", SCM_PAS_AUTH_AND_RESET, SCM_ONE_VALUE, pas_mpss, 0, 0, 0,
	    &answer);
	doorbell.since = started = now_us();
	image_written = false;
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT START THE MODEM: the call returns %d, its answer is %#llx\n", step, result,
		    answer);
		mailbox_send("{class: image, res: load_state, name: modem, val: off}");
		rails(0);
		rails_asked = false;
		lck_mtx_unlock(&dsp_lock);
		return false;
	}
	dsp_started = true;
	lck_mtx_unlock(&dsp_lock);
	printf("mpss: %u: TRUSTZONE HAS AUTHENTICATED THE IMAGE AND LET THE MODEM OUT OF RESET; watching for %u s\n", step,
	    wait_seconds);
	limit = started + wait_seconds * 1000000ull;
	for (;;) {
		now = now_us();
		if (now >= limit || !dsp_started) {
			break;
		}
		doorbells();
		spoke = spoke || doorbell.glink + doorbell.smp2p != 0;
		if (dsp_words(&words) && (!have || memcmp(&words, &last, sizeof(words)) != 0)) {
			char when[64];

			snprintf(when, sizeof(when), "+%llu.%03llu ms: the DSP's item %s", (now - started) / 1000, (now - started) % 1000,
			    have ? "changes" : "APPEARS");
			if (changes++ < 40) {
				show_words(when, &words);
			}
			last = words;
			have = spoke = true;
			dsp_listen(&words);
			state = dsp_state(&words);
			if ((state & DSP_READY) && ready_at == 0) {
				ready_at = now;
				limit = now + 3000000ull;
				printf("mpss: %u: THE MODEM SAYS IT IS UP: \"slave-kernel\" says %#x, %llu.%03llu ms after TrustZone started "
				    "it\n", step, state, (now - started) / 1000, (now - started) % 1000);
			}
			if ((state & DSP_HANDOVER) && !handed_over) {
				handed_over = true;
				printf("mpss: %u: the DSP says it has asked for its own power (bit 2), %llu.%03llu ms after the start: ours is "
				    "given back\n", step, (now - started) / 1000, (now - started) % 1000);
				lck_mtx_lock(&dsp_lock);
				if (rails_asked) {
					rails(0);
					rails_asked = false;
				}
				lck_mtx_unlock(&dsp_lock);
			}
			if (state & DSP_FATAL) {
				printf("mpss: %u: THE MODEM SAYS IT HAS GIVEN UP\n", step);
				show_crash_reason();
				break;
			}
		}
		sleep_us(200);
	}
	printf("mpss: %u: the watch is over after %llu ms; its doorbells heard: GLINK %u, SMP2P %u\n", step,
	    (now_us() - started) / 1000, doorbell.glink, doorbell.smp2p);
	show_partition();
	if (our_item != 0 && smem_peek(our_item, &words, sizeof(words))) {
		show_words("our item (428) now", &words);
	}
	{
		/* (what is at the bottom of our partition, entry by entry, as it is now) */
		uint32_t cacheline = 0, size = 0, partition = smem_partition(SMEM_US, SMEM_MPSS, &cacheline, &size);
		struct smem_partition header;
		struct smem_entry entry;

		if (partition != 0 && smem_peek(partition, &header, sizeof(header))) {
			printf("mpss: %u: the partition's header: hosts %u and %u, size %#x, free from the bottom at +%#x, from the top at "
			    "+%#x; the table's line size %u\n", step, header.host0, header.host1, header.size, header.free_uncached,
			    header.free_cached, cacheline);
			for (uint32_t at = sizeof(header), n = 0; at + sizeof(entry) <= header.size && n < 8; n++) {
				smem_peek(partition + at, &entry, sizeof(entry));
				printf("mpss: %u:   +%#x: canary %#x item %u size %u padding %u/%u\n", step, at, entry.canary, entry.item,
				    entry.size, entry.padding_data, entry.padding_header);
				if (entry.canary != SMEM_CANARY) {
					break;
				}
				at += sizeof(entry) + entry.padding_header + entry.size;
			}
		}
	}
	if (state & DSP_FATAL) {
		return false;
	}
	if (ready_at == 0) {
		printf("mpss: %u: %s\n", step, spoke ? "THE MODEM RUNS BUT HAS NOT SAID \"UP\"" : "NOT A WORD FROM THE MODEM");
		show_crash_reason();
	}
	return ready_at != 0;
}

/* ---- GLINK (hakios.wifi.c; items 478 to 480 of our partition with the modem) ------------------- */

#define ITEM_GLINK_DESCRIPTOR   478
#define ITEM_GLINK_OUR_QUEUE    479
#define ITEM_GLINK_THEIR_QUEUE  480
#define GLINK_OUR_TAIL          0
#define GLINK_OUR_HEAD          4
#define GLINK_THEIR_TAIL        8
#define GLINK_THEIR_HEAD        12
#define GLINK_RESERVE           16

enum {
	GLINK_VERSION = 0, GLINK_VERSION_ACK = 1, GLINK_OPEN = 2, GLINK_CLOSE = 3, GLINK_OPEN_ACK = 4, GLINK_INTENT = 5,
	GLINK_DONE = 6, GLINK_INTENT_REQUEST = 7, GLINK_INTENT_REQUEST_ACK = 8, GLINK_DATA = 9, GLINK_CLOSE_ACK = 11,
	GLINK_DATA_MORE = 12, GLINK_READ_NOTIFY = 13, GLINK_DONE_REUSE = 14, GLINK_SIGNALS = 15,
};
#define GLINK_VERSION_1         1
#define GLINK_INTENT_REUSE      1u

struct glink_message {
	uint16_t command, param1;
	uint32_t param2;
};
struct glink_packet {
	struct glink_packet *next;
	uint32_t length;
	uint8_t data[];
};

/* The one channel we answer when the modem opens it: IPCRTR, which carries QRTR (dt: qcom,intents of qcom,modem_qrtr). */
enum { CHANNEL_QRTR, CHANNELS };

struct glink_channel {
	const char *name;
	uint32_t ours;                          /* our number for it */
	struct {
		uint32_t size, count;
	} intents[3];
	uint32_t theirs;                        /* the DSP's number for it; 0: it has not opened it */
	bool asked, acknowledged, open;         /* our open sent; acknowledged; both ways done and our intents told */
	bool request_answered, request_granted; /* our last request for an intent */
	struct {
		uint32_t id, size;
		bool there, in_use;
	} their_intents[64];                    /* what we may send into */
	struct {
		uint32_t size, fill;
		uint8_t *data;
	} our_intents[24];                      /* what the DSP may send into; an intent's number is its place here + 1 */
	uint32_t our_intent_count;
	struct glink_packet *first, *last;      /* received, in order */
};

static LCK_MTX_DECLARE(glink_lock, &mpss_lock_group);
static struct {
	uint32_t descriptor, ours, ours_size, theirs, theirs_size;
	bool up, broken;
	uint32_t features;
	bool agreed;
	struct glink_channel channel[CHANNELS];
	uint32_t said, heard, trace;
	char others[4][33];                     /* the channels the DSP opens that nobody here speaks (for the log) */
} glink = {
	.channel = {
		[CHANNEL_QRTR] = { .name = "IPCRTR", .ours = 1, .intents = { { 0x800, 5 }, { 0x2000, 3 }, { 0x4400, 2 } } },
	},
};

static struct glink_channel *
glink_theirs(uint32_t theirs)
{
	for (unsigned c = 0; c < CHANNELS; c++) {
		if (glink.channel[c].theirs != 0 && glink.channel[c].theirs == theirs) {
			return &glink.channel[c];
		}
	}
	return NULL;
}

static bool
glink_find(void)
{
	uint32_t size = 0;

	glink.descriptor = smem_item(ITEM_GLINK_DESCRIPTOR, &size);
	if (glink.descriptor == 0 || size != 32) {
		return false;
	}
	glink.ours = smem_item(ITEM_GLINK_OUR_QUEUE, &glink.ours_size);
	glink.theirs = smem_item(ITEM_GLINK_THEIR_QUEUE, &glink.theirs_size);
	return glink.ours != 0 && glink.theirs != 0 && glink.ours_size >= 1024 && glink.theirs_size >= 1024 &&
	       !(glink.ours_size & 7) && !(glink.theirs_size & 7);
}

static void
glink_reset(void)
{
	uint32_t size = 0, descriptor = smem_item(ITEM_GLINK_DESCRIPTOR, &size);

	if (descriptor == 0 || size != 32) {
		printf("mpss: %u: GLINK's descriptor (item %u) is not there: nothing to reset\n", step, ITEM_GLINK_DESCRIPTOR);
		return;
	}
	printf("mpss: %u: GLINK's descriptor at %#lx: our queue read to %#x and written to %#x, the DSP's read to %#x and written "
	    "to %#x\n", step, smem_phys + descriptor, smem_word(descriptor + GLINK_OUR_TAIL), smem_word(descriptor + GLINK_OUR_HEAD),
	    smem_word(descriptor + GLINK_THEIR_TAIL), smem_word(descriptor + GLINK_THEIR_HEAD));
	if (smem_word(descriptor + GLINK_OUR_HEAD) != 0 || smem_word(descriptor + GLINK_THEIR_TAIL) != 0) {
		printf("mpss: %u: about to write our two marks there (+4, +8): 0\n", step);
		smem_poke(descriptor + GLINK_OUR_HEAD, 0);
		smem_poke(descriptor + GLINK_THEIR_TAIL, 0);
		settle();
	}
}

static uint32_t
glink_waiting(void)
{
	uint32_t head = smem_word(glink.descriptor + GLINK_THEIR_HEAD), tail = smem_word(glink.descriptor + GLINK_THEIR_TAIL);

	if (head >= glink.theirs_size || tail >= glink.theirs_size) {
		return 0;
	}
	return head >= tail ? head - tail : glink.theirs_size - tail + head;
}

static void
glink_peek(void *to, uint32_t offset, uint32_t count)
{
	uint32_t at = (smem_word(glink.descriptor + GLINK_THEIR_TAIL) + offset) % glink.theirs_size;

	for (uint32_t i = 0; i < count; i++) {
		((uint8_t *)to)[i] = smem[glink.theirs + (at + i) % glink.theirs_size];
	}
}

static void
glink_advance(uint32_t count)
{
	smem_poke(glink.descriptor + GLINK_THEIR_TAIL, (smem_word(glink.descriptor + GLINK_THEIR_TAIL) + count) % glink.theirs_size);
}

static uint32_t
glink_room(void)
{
	uint32_t head = smem_word(glink.descriptor + GLINK_OUR_HEAD), tail = smem_word(glink.descriptor + GLINK_OUR_TAIL), room;

	if (head >= glink.ours_size || tail >= glink.ours_size) {
		return 0;
	}
	room = tail <= head ? glink.ours_size - head + tail : tail - head;
	return room < GLINK_RESERVE ? 0 : room - GLINK_RESERVE;
}

static bool
glink_put(const void *first, uint32_t first_length, const void *second, uint32_t second_length)
{
	volatile uint8_t *queue = (volatile uint8_t *)(uintptr_t)smem + glink.ours;
	uint32_t head = smem_word(glink.descriptor + GLINK_OUR_HEAD), length = first_length + second_length;
	uint32_t padded = (length + 7) & ~7u;

	if (!glink.up || glink_room() < padded) {
		return false;
	}
	for (uint32_t i = 0; i < padded; i++) {
		queue[(head + i) % glink.ours_size] = i < first_length ? ((const uint8_t *)first)[i] :
		    i < length ? ((const uint8_t *)second)[i - first_length] : 0;
	}
	settle();
	smem_poke(glink.descriptor + GLINK_OUR_HEAD, (head + padded) % glink.ours_size);
	settle();
	wr(ipcc.base + IPCC_SEND, IPCC_MPSS << 16 | IPCC_GLINK);
	glink.said++;
	return true;
}

static const char *
glink_name(unsigned command)
{
	static const char *const names[] = {
		"version", "version-ack", "open", "close", "open-ack", "intent", "done", "intent-request", "intent-request-ack",
		"data", "10", "close-ack", "data-more", "read-notify", "done-reuse", "signals"
	};

	return command < sizeof(names) / sizeof(names[0]) ? names[command] : "?";
}

static void
glink_say(unsigned command, uint32_t param1, uint32_t param2, const void *more, uint32_t more_length)
{
	struct glink_message message = { .command = (uint16_t)command, .param1 = (uint16_t)param1, .param2 = param2 };
	bool sent = glink_put(&message, sizeof(message), more, more_length);

	if (glink.trace > 0 || !sent) {
		glink.trace -= glink.trace > 0;
		printf("mpss: glink: > %s %u %#x%s\n", glink_name(command), param1, param2, sent ? "" : ": NO ROOM IN OUR QUEUE, not sent");
	}
}

static bool
glink_offer(struct glink_channel *channel, uint32_t size)
{
	uint32_t words[2] = { size, channel->our_intent_count + 1 }, i = channel->our_intent_count;

	if (i >= sizeof(channel->our_intents) / sizeof(channel->our_intents[0]) || size == 0 || size > 0x10000) {
		return false;
	}
	if (channel->our_intents[i].data == NULL || channel->our_intents[i].size != size) {
		channel->our_intents[i].data = kalloc_data(size, Z_WAITOK | Z_ZERO);
	}
	if (channel->our_intents[i].data == NULL) {
		return false;
	}
	channel->our_intents[i].size = size;
	channel->our_intents[i].fill = 0;
	channel->our_intent_count++;
	glink_say(GLINK_INTENT, channel->ours, 1, words, sizeof(words));
	return true;
}

/* Everything the DSP has put into its queue, taken out and acted on (Linux's qcom_glink_native_rx()). With glink_lock held. */
static void
glink_pump(void)
{
	struct glink_message message;
	struct glink_channel *channel;
	uint32_t words[4];
	char name[33];

	while (glink.up && !glink.broken) {
		uint32_t waiting = glink_waiting(), length = sizeof(message);

		if (waiting < sizeof(message)) {
			return;
		}
		glink_peek(&message, 0, sizeof(message));
		if (message.command == GLINK_OPEN) {
			length += message.param2;
		} else if (message.command == GLINK_INTENT) {
			length += 8 * message.param2;
		} else if (message.command == GLINK_DATA || message.command == GLINK_DATA_MORE) {
			if (waiting < 16) {
				return;
			}
			glink_peek(words, 8, 8);
			length = 16 + words[0];
		}
		length = (length + 7) & ~7u;
		if (length > glink.theirs_size - 8) {
			printf("mpss: glink: A MESSAGE OF %u BYTES (%s %u %#x): not believed, and nothing more is read\n", length,
			    glink_name(message.command), message.param1, message.param2);
			glink.broken = true;
			return;
		}
		if (waiting < length) {
			return;
		}
		glink.heard++;
		if (glink.trace > 0 && message.command != GLINK_DATA && message.command != GLINK_DATA_MORE &&
		    message.command != GLINK_INTENT && message.command != GLINK_DONE_REUSE) {
			glink.trace--;
			printf("mpss: glink: < %s %u %#x\n", glink_name(message.command), message.param1, message.param2);
		}
		switch (message.command) {
		case GLINK_VERSION:
			if (message.param1 == GLINK_VERSION_1) {
				glink.features &= message.param2;
			}
			glink_advance(length);
			glink_say(GLINK_VERSION_ACK, GLINK_VERSION_1, glink.features, NULL, 0);
			break;
		case GLINK_VERSION_ACK:
			glink_advance(length);
			if (message.param1 == GLINK_VERSION_1 && message.param2 == glink.features) {
				glink.agreed = true;
			} else if (message.param1 != 0) {
				glink.features &= message.param2;
				glink_say(GLINK_VERSION, GLINK_VERSION_1, glink.features, NULL, 0);
			}
			break;
		case GLINK_OPEN:
			bzero(name, sizeof(name));
			glink_peek(name, sizeof(message), message.param2 < sizeof(name) - 1 ? message.param2 : sizeof(name) - 1);
			glink_advance(length);
			channel = NULL;
			for (unsigned c = 0; c < CHANNELS; c++) {
				if (strcmp(name, glink.channel[c].name) == 0) {
					channel = &glink.channel[c];
				}
			}
			if (channel == NULL) {
				printf("mpss: glink: the DSP opens \"%s\" as its channel %u: left unanswered, nobody here speaks it\n", name,
				    message.param1);
				for (unsigned i = 0; i < 4; i++) {
					if (glink.others[i][0] == 0) {
						strlcpy(glink.others[i], name, sizeof(glink.others[i]));
						break;
					}
				}
				break;
			}
			printf("mpss: glink: the DSP opens \"%s\" as its channel %u: acknowledged, and opened from our side as channel %u\n",
			    name, message.param1, channel->ours);
			channel->theirs = message.param1;
			glink_say(GLINK_OPEN_ACK, channel->theirs, 0, NULL, 0);
			if (!channel->asked) {
				uint32_t size = (uint32_t)strlen(channel->name) + 1;

				bzero(name, sizeof(name));
				strlcpy(name, channel->name, sizeof(name));
				glink_say(GLINK_OPEN, channel->ours, size, name, (size + 7) & ~7u);
				channel->asked = true;
			}
			break;
		case GLINK_OPEN_ACK:
			glink_advance(length);
			for (unsigned c = 0; c < CHANNELS; c++) {
				if (message.param1 == glink.channel[c].ours) {
					glink.channel[c].acknowledged = true;
				}
			}
			break;
		case GLINK_INTENT:
			channel = glink_theirs(message.param1);
			for (uint32_t i = 0; channel != NULL && i < message.param2; i++) {
				glink_peek(words, 8 + 8 * i, 8);
				for (unsigned n = 0; n < sizeof(channel->their_intents) / sizeof(channel->their_intents[0]); n++) {
					if (!channel->their_intents[n].there || channel->their_intents[n].id == words[1]) {
						channel->their_intents[n].there = true;
						channel->their_intents[n].in_use = false;
						channel->their_intents[n].size = words[0];
						channel->their_intents[n].id = words[1];
						break;
					}
				}
			}
			glink_advance(length);
			break;
		case GLINK_DONE:
		case GLINK_DONE_REUSE:
			channel = glink_theirs(message.param1);
			for (unsigned n = 0; channel != NULL && n < sizeof(channel->their_intents) / sizeof(channel->their_intents[0]); n++) {
				if (channel->their_intents[n].there && channel->their_intents[n].id == message.param2) {
					channel->their_intents[n].in_use = false;
					channel->their_intents[n].there = message.command == GLINK_DONE_REUSE;
				}
			}
			glink_advance(length);
			break;
		case GLINK_INTENT_REQUEST:
			glink_advance(length);
			if ((channel = glink_theirs(message.param1)) != NULL) {
				bool granted = glink_offer(channel, message.param2);

				glink_say(GLINK_INTENT_REQUEST_ACK, channel->ours, granted, NULL, 0);
			}
			break;
		case GLINK_INTENT_REQUEST_ACK:
			glink_advance(length);
			for (unsigned c = 0; c < CHANNELS; c++) {
				if (message.param1 == glink.channel[c].theirs || message.param1 == glink.channel[c].ours) {
					glink.channel[c].request_answered = true;
					glink.channel[c].request_granted = message.param2 != 0;
				}
			}
			break;
		case GLINK_DATA:
		case GLINK_DATA_MORE: {
			uint32_t intent = message.param2 - 1;

			channel = glink_theirs(message.param1);
			if (channel == NULL || intent >= channel->our_intent_count ||
			    words[0] > channel->our_intents[intent].size - channel->our_intents[intent].fill) {
				printf("mpss: glink: < data of %u bytes on channel %u for intent %u: NOT OURS OR TOO MUCH, dropped\n", words[0],
				    message.param1, message.param2);
				glink_advance(length);
				break;
			}
			glink_peek(channel->our_intents[intent].data + channel->our_intents[intent].fill, 16, words[0]);
			channel->our_intents[intent].fill += words[0];
			glink_advance(length);
			if (words[1] == 0) {
				struct glink_packet *packet = kalloc_data(sizeof(*packet) + channel->our_intents[intent].fill, Z_WAITOK);

				if (packet != NULL) {
					packet->next = NULL;
					packet->length = channel->our_intents[intent].fill;
					memcpy(packet->data, channel->our_intents[intent].data, packet->length);
					if (channel->last != NULL) {
						channel->last->next = packet;
					} else {
						channel->first = packet;
					}
					channel->last = packet;
				}
				channel->our_intents[intent].fill = 0;
				glink_say(GLINK_DONE_REUSE, channel->ours, message.param2, NULL, 0);
			}
			break;
		}
		case GLINK_READ_NOTIFY:
			glink_advance(length);
			wr(ipcc.base + IPCC_SEND, IPCC_MPSS << 16 | IPCC_GLINK);
			break;
		case GLINK_CLOSE:
			glink_advance(length);
			glink_say(GLINK_CLOSE_ACK, message.param1, 0, NULL, 0);
			if ((channel = glink_theirs(message.param1)) != NULL) {
				printf("mpss: glink: THE DSP CLOSES \"%s\"\n", channel->name);
				channel->open = false;
				channel->theirs = 0;
			}
			break;
		case GLINK_CLOSE_ACK:
		case GLINK_SIGNALS:
			glink_advance(length);
			break;
		default:
			printf("mpss: glink: A MESSAGE THIS DOES NOT KNOW (%u, %u, %#x): nothing more is read\n", message.command,
			    message.param1, message.param2);
			glink.broken = true;
			return;
		}
		for (unsigned c = 0; c < CHANNELS; c++) {
			channel = &glink.channel[c];
			if (channel->theirs != 0 && channel->acknowledged && !channel->open) {
				channel->open = true;
				printf("mpss: glink: THE CHANNEL \"%s\" IS OPEN both ways (ours %u, the DSP's %u); telling the DSP our intents\n",
				    channel->name, channel->ours, channel->theirs);
				for (unsigned i = 0; i < 3; i++) {
					for (unsigned n = 0; n < channel->intents[i].count; n++) {
						glink_offer(channel, channel->intents[i].size);
					}
				}
			}
		}
	}
}

/* One whole message to the DSP on a channel; waits up to five seconds for an intent of the DSP's that it fits. */
static int
glink_send(struct glink_channel *channel, const void *data, uint32_t length)
{
	struct {
		struct glink_message message;
		uint32_t size, left;
	} header = { { GLINK_DATA, (uint16_t)channel->ours, 0 }, length, 0 };
	bool asked = false, fits;
	int found = -1;

	lck_mtx_lock(&glink_lock);
	if (glink.up && length + sizeof(header) + 8 > glink.ours_size - GLINK_RESERVE) {
		lck_mtx_unlock(&glink_lock);
		return EMSGSIZE;
	}
	for (unsigned waited = 0;; waited++) {
		glink_pump();
		if (!glink.up || !channel->open || glink.broken) {
			lck_mtx_unlock(&glink_lock);
			return ENOTCONN;
		}
		found = -1;
		fits = false;
		for (unsigned n = 0; n < sizeof(channel->their_intents) / sizeof(channel->their_intents[0]); n++) {
			if (!channel->their_intents[n].there || channel->their_intents[n].size < length) {
				continue;
			}
			fits = true;
			if (!channel->their_intents[n].in_use &&
			    (found < 0 || channel->their_intents[n].size < channel->their_intents[found].size)) {
				found = (int)n;
			}
		}
		if (found >= 0 && glink_room() >= ((length + sizeof(header) + 7) & ~7u)) {
			break;
		}
		if (!fits && !asked) {
			channel->request_answered = false;
			glink_say(GLINK_INTENT_REQUEST, channel->ours, length, NULL, 0);
			asked = true;
		}
		if (waited >= 5000 || (!fits && asked && channel->request_answered && !channel->request_granted)) {
			lck_mtx_unlock(&glink_lock);
			return found < 0 ? ENOBUFS : EAGAIN;
		}
		lck_mtx_unlock(&glink_lock);
		sleep_ms(1);
		lck_mtx_lock(&glink_lock);
	}
	channel->their_intents[found].in_use = true;
	header.message.param2 = channel->their_intents[found].id;
	glink_put(&header, sizeof(header), data, length);
	lck_mtx_unlock(&glink_lock);
	return 0;
}

/* ---- QRTR: the router's datagrams, and ports of ours (hakios.wifi.c) ------------------------------ */

/*
 * What goes over the channel: datagrams of Qualcomm's IPC router, each with a header that says
 * who sends it (a node, a port) and to whom, and of which kind. The control port hears of the
 * services the modem offers ("new server": a service, an instance, a node, a port), and later
 * their going. We are node 1 (Linux's default, and what this phone's Android is); the modem says
 * who it is in what it sends. The interface for the rest of the kernel is sys/hakios.qrtr.h:
 * the four calls at the end of this section, which the NAS client (hakios.nas.c) and the QMI
 * helpers (hakios.qmi.c) take.
 */
#define HAKIOS_QRTR_MAX        8192            /* the most a datagram holds, either way */
#define QRTR_US                 1u
#define QRTR_CONTROL            0xfffffffeu     /* the control port */
#define QRTR_FIRST_PORT         0x4000u         /* where Linux begins numbering ports that ask for no number */
enum { QRTR_DATA = 1, QRTR_HELLO = 2, QRTR_BYE = 3, QRTR_NEW_SERVER = 4, QRTR_DEL_SERVER = 5, QRTR_DEL_CLIENT = 6, QRTR_RESUME_TX = 7,
       QRTR_NEW_LOOKUP = 10, QRTR_DEL_LOOKUP = 11 };

struct qrtr_header {                            /* version 1, which Linux sends; version 2 is read as well */
	uint32_t version, type, from_node, from_port, confirm, size, to_node, to_port;
};
struct qrtr_control {
	uint32_t command;
	uint32_t words[4];                      /* a server: service, instance, node, port; a client: node, port */
};

struct hakios_qrtr_port {
	uint32_t number;                        /* 0: free */
	hakios_qrtr_receive_t receive;
	hakios_qrtr_server_t server;
	void *context;
	struct {
		uint32_t service, instance;     /* the instance as on the wire: version | instance << 8; 0: any */
		bool asked;
		bool fresh;                     /* not yet told of the services that were there before it */
	} lookups[4];
};

static LCK_MTX_DECLARE(qrtr_lock, &mpss_lock_group);    /* the tables below */
static LCK_MTX_DECLARE(qrtr_calling, &mpss_lock_group); /* held by the link's thread around a callback */
static struct hakios_qrtr_port qrtr_ports[16];
static struct {
	uint32_t service, instance, node, port;
	bool there;
} qrtr_servers[64];
static uint32_t qrtr_peer;                      /* the modem's node; 0: it has not said */
static uint32_t qrtr_next_port = QRTR_FIRST_PORT, qrtr_in, qrtr_out, qrtr_hellos, qrtr_lookups;
static thread_t link_thread_is;

/* The name service's side: our servers, told of on hello and on lookup (the one there is: rmtfs, below). */
static struct {
	uint32_t service, instance, port;
} our_servers[4];
static uint32_t rmtfs_requests, rmtfs_reads, rmtfs_writes, rmtfs_sectors_read, rmtfs_sectors_written, rmtfs_refused;
static uint32_t tftp_requests, tftp_served, tftp_refused, servreg_requests;

/* One datagram out, of any type. */
static int
qrtr_out_packet(uint32_t type, uint32_t from_port, uint32_t to_node, uint32_t to_port, const void *data, uint32_t length)
{
	uint32_t padded = (length + 3) & ~3u;
	struct qrtr_header *packet;
	int error;

	if (length == 0 || length > HAKIOS_QRTR_MAX) {
		return EMSGSIZE;
	}
	packet = kalloc_data(sizeof(*packet) + padded, Z_WAITOK | Z_ZERO);
	if (packet == NULL) {
		return ENOMEM;
	}
	packet->version = 1;
	packet->type = type;
	packet->from_node = QRTR_US;
	packet->from_port = from_port;
	packet->size = length;
	packet->to_node = to_node;
	packet->to_port = to_port;
	memcpy(packet + 1, data, length);
	error = glink_send(&glink.channel[CHANNEL_QRTR], packet, (uint32_t)sizeof(*packet) + padded);
	kfree_data(packet, sizeof(*packet) + padded);
	qrtr_out += error == 0;
	return error;
}

static bool
qrtr_matches(uint32_t service, uint32_t instance, unsigned server)
{
	return qrtr_servers[server].there && qrtr_servers[server].service == service &&
	       (instance == 0 || qrtr_servers[server].instance == instance);
}

/* Everybody who looked a service up is told that it has come, or gone. From the link's thread. */
static void
qrtr_tell(uint32_t service, uint32_t instance, uint32_t node, uint32_t port, bool arrived)
{
	for (unsigned p = 0; p < sizeof(qrtr_ports) / sizeof(qrtr_ports[0]); p++) {
		for (unsigned l = 0; l < 4; l++) {
			hakios_qrtr_server_t server = NULL;
			void *context;

			lck_mtx_lock(&qrtr_lock);
			if (qrtr_ports[p].number != 0 && qrtr_ports[p].lookups[l].asked && !qrtr_ports[p].lookups[l].fresh &&
			    qrtr_ports[p].lookups[l].service == service &&
			    (qrtr_ports[p].lookups[l].instance == 0 || qrtr_ports[p].lookups[l].instance == instance)) {
				server = qrtr_ports[p].server;
			}
			context = qrtr_ports[p].context;
			lck_mtx_unlock(&qrtr_lock);
			if (server != NULL) {
				server(context, arrived, service, instance & 0xff, instance >> 8, node, port);
			}
		}
	}
}

static void qrtr_announce(uint32_t node, unsigned server, bool end);
static void qrtr_announce_all(uint32_t node);

/* The control port's part: Linux's name service, with our one service to tell of. */
static void
qrtr_control(const struct qrtr_header *header, const struct qrtr_control *control)
{
	struct qrtr_control hello = { .command = QRTR_HELLO };
	int found = -1, error;

	switch (control->command) {
	case QRTR_HELLO:
		if (qrtr_hellos++ != 0) {
			break;                                  /* (answered once: two that answer every hello never end) */
		}
		error = qrtr_out_packet(QRTR_HELLO, QRTR_CONTROL, header->from_node, QRTR_CONTROL, &hello, sizeof(hello));
		printf("mpss: qrtr: HELLO FROM NODE %u; ours to it: %s\n", header->from_node, error == 0 ? "SENT" : "NOT SENT");
		qrtr_announce_all(header->from_node);    /* (the name service's part: what we offer) */
		break;
	case QRTR_NEW_SERVER:
		lck_mtx_lock(&qrtr_lock);
		for (unsigned i = 0; i < sizeof(qrtr_servers) / sizeof(qrtr_servers[0]); i++) {
			if (qrtr_servers[i].there && qrtr_servers[i].node == control->words[2] && qrtr_servers[i].port == control->words[3]) {
				found = (int)i;
				break;
			}
			if (!qrtr_servers[i].there && found < 0) {
				found = (int)i;
			}
		}
		if (found >= 0) {
			qrtr_servers[found].there = true;
			qrtr_servers[found].service = control->words[0];
			qrtr_servers[found].instance = control->words[1];
			qrtr_servers[found].node = control->words[2];
			qrtr_servers[found].port = control->words[3];
		}
		lck_mtx_unlock(&qrtr_lock);
		printf("mpss: qrtr: a service: %u (%#x), version %u, instance %u, at node %u port %u\n", control->words[0],
		    control->words[0], control->words[1] & 0xff, control->words[1] >> 8, control->words[2], control->words[3]);
		qrtr_tell(control->words[0], control->words[1], control->words[2], control->words[3], true);
		break;
	case QRTR_DEL_SERVER:
	case QRTR_BYE:
		for (unsigned i = 0; i < sizeof(qrtr_servers) / sizeof(qrtr_servers[0]); i++) {
			bool gone;

			lck_mtx_lock(&qrtr_lock);
			gone = qrtr_servers[i].there && (control->command == QRTR_BYE ? qrtr_servers[i].node == header->from_node :
			    qrtr_servers[i].node == control->words[2] && qrtr_servers[i].port == control->words[3]);
			qrtr_servers[i].there = qrtr_servers[i].there && !gone;
			lck_mtx_unlock(&qrtr_lock);
			if (gone) {
				printf("mpss: qrtr: service %u at node %u port %u has gone\n", qrtr_servers[i].service, qrtr_servers[i].node,
				    qrtr_servers[i].port);
				qrtr_tell(qrtr_servers[i].service, qrtr_servers[i].instance, qrtr_servers[i].node, qrtr_servers[i].port, false);
			}
		}
		break;
	case QRTR_NEW_LOOKUP: {
		/* the modem looks for a service: ours that match, then the empty "new server" that ends the answer */
		unsigned matches = 0;

		for (unsigned i = 0; i < sizeof(our_servers) / sizeof(our_servers[0]); i++) {
			if (our_servers[i].port != 0 && (control->words[0] == 0 || our_servers[i].service == control->words[0]) &&
			    (control->words[1] == 0 || our_servers[i].instance == control->words[1])) {
				qrtr_announce(header->from_node, i, false);
				matches++;
			}
		}
		qrtr_announce(header->from_node, 0, true);
		if (qrtr_lookups++ < 16) {
			printf("mpss: qrtr: the modem looks for service %u (%#x), instance %u: %u of ours\n", control->words[0],
			    control->words[0], control->words[1], matches);
		}
		break;
	}
	default:
		break;                                  /* (a client's going, and what else there is: nothing here wants it) */
	}
}

/* A message from the channel: one datagram. From the link's thread. */
static void
qrtr_input(const uint8_t *data, uint32_t length)
{
	struct qrtr_header header;
	hakios_qrtr_receive_t receive = NULL;
	void *context = NULL;
	uint32_t at;

	if (length >= sizeof(header) && data[0] == 1) {
		memcpy(&header, data, sizeof(header));
		at = sizeof(header);
	} else if (length >= 16 && data[0] == 3) {
		/* version 2: version, type, flags, a length of options; the size; then four numbers of 16 bits */
		bzero(&header, sizeof(header));
		header.type = data[1];
		header.confirm = data[2] & 1;
		memcpy(&header.size, data + 4, 4);
		header.from_node = (uint32_t)(data[8] | data[9] << 8);
		header.from_port = (uint32_t)(data[10] | data[11] << 8);
		header.to_node = (uint32_t)(data[12] | data[13] << 8);
		header.to_port = (uint32_t)(data[14] | data[15] << 8);
		header.from_port = header.from_port == 0xfffe ? QRTR_CONTROL : header.from_port;
		header.to_port = header.to_port == 0xfffe ? QRTR_CONTROL : header.to_port;
		at = 16 + data[3];
	} else {
		printf("mpss: qrtr: %u bytes that are no datagram (they begin %#x)\n", length, length ? data[0] : 0);
		return;
	}
	if (header.size == 0 || at + header.size > length) {
		printf("mpss: qrtr: a datagram of %u bytes in a message of %u: dropped\n", header.size, length);
		return;
	}
	qrtr_in++;
	if (qrtr_peer == 0) {
		qrtr_peer = header.from_node;
	}
	if (header.type == QRTR_RESUME_TX) {
		return;                                 /* (we never ask for one: GLINK's intents hold us back already) */
	}
	if (header.confirm) {
		/* the sender counts what it has sent to this port, and waits for this before it sends many more */
		struct qrtr_control resume = { .command = QRTR_RESUME_TX, .words = { header.to_node, header.to_port } };

		qrtr_out_packet(QRTR_RESUME_TX, header.to_port, header.from_node, header.from_port, &resume, sizeof(resume));
	}
	if (header.to_port == QRTR_CONTROL || header.type == QRTR_HELLO || header.type == QRTR_BYE ||
	    header.type == QRTR_NEW_SERVER || header.type == QRTR_DEL_SERVER || header.type == QRTR_NEW_LOOKUP) {
		struct qrtr_control control = { 0 };

		memcpy(&control, data + at, header.size < sizeof(control) ? header.size : sizeof(control));
		qrtr_control(&header, &control);
		return;
	}
	lck_mtx_lock(&qrtr_lock);
	for (unsigned p = 0; p < sizeof(qrtr_ports) / sizeof(qrtr_ports[0]); p++) {
		if (qrtr_ports[p].number == header.to_port && header.type == QRTR_DATA) {
			receive = qrtr_ports[p].receive;
			context = qrtr_ports[p].context;
		}
	}
	lck_mtx_unlock(&qrtr_lock);
	if (receive != NULL) {
		receive(context, header.from_node, header.from_port, data + at, header.size);
	} else if (qrtr_in <= 64) {
		printf("mpss: qrtr: a datagram of %u bytes from node %u port %u to our port %u: nobody here listens there\n", header.size,
		    header.from_node, header.from_port, header.to_port);
	}
}

/* Lookups made since the last look are told of the services that were there before them. From the link's thread. */
static void
qrtr_fresh(void)
{
	for (unsigned p = 0; p < sizeof(qrtr_ports) / sizeof(qrtr_ports[0]); p++) {
		for (unsigned l = 0; l < 4; l++) {
			hakios_qrtr_server_t server;
			uint32_t service, instance;
			void *context;

			lck_mtx_lock(&qrtr_lock);
			if (qrtr_ports[p].number == 0 || !qrtr_ports[p].lookups[l].fresh) {
				lck_mtx_unlock(&qrtr_lock);
				continue;
			}
			qrtr_ports[p].lookups[l].fresh = false;
			server = qrtr_ports[p].server;
			service = qrtr_ports[p].lookups[l].service;
			instance = qrtr_ports[p].lookups[l].instance;
			context = qrtr_ports[p].context;
			lck_mtx_unlock(&qrtr_lock);
			for (unsigned i = 0; i < sizeof(qrtr_servers) / sizeof(qrtr_servers[0]) && server != NULL; i++) {
				if (qrtr_matches(service, instance, i)) {
					server(context, true, qrtr_servers[i].service, qrtr_servers[i].instance & 0xff,
					    qrtr_servers[i].instance >> 8, qrtr_servers[i].node, qrtr_servers[i].port);
				}
			}
		}
	}
}

static struct hakios_qrtr_port *
qrtr_open(hakios_qrtr_receive_t receive, hakios_qrtr_server_t server, void *context)
{
	struct hakios_qrtr_port *port = NULL;

	lck_mtx_lock(&qrtr_lock);
	for (unsigned p = 0; p < sizeof(qrtr_ports) / sizeof(qrtr_ports[0]) && port == NULL; p++) {
		if (qrtr_ports[p].number == 0) {
			port = &qrtr_ports[p];
			bzero(port, sizeof(*port));
			port->number = qrtr_next_port++;
			port->receive = receive;
			port->server = server;
			port->context = context;
		}
	}
	lck_mtx_unlock(&qrtr_lock);
	return port;
}

static int
qrtr_send(struct hakios_qrtr_port *port, uint32_t node, uint32_t to, const void *data, size_t length)
{
	if (port == NULL || port->number == 0 || length > HAKIOS_QRTR_MAX) {
		return EINVAL;
	}
	return qrtr_out_packet(QRTR_DATA, port->number, node, to, data, (uint32_t)length);
}

static int
qrtr_lookup(struct hakios_qrtr_port *port, uint32_t service, uint32_t version, uint32_t instance)
{
	int error = ENOSPC;

	lck_mtx_lock(&qrtr_lock);
	for (unsigned l = 0; l < 4 && error != 0 && port != NULL && port->number != 0; l++) {
		if (!port->lookups[l].asked) {
			port->lookups[l].service = service;
			port->lookups[l].instance = instance != 0 ? (version & 0xff) | instance << 8 : 0;
			port->lookups[l].asked = true;
			port->lookups[l].fresh = true;
			error = 0;
		}
	}
	lck_mtx_unlock(&qrtr_lock);
	return error;
}

static void
qrtr_close(struct hakios_qrtr_port *port)
{
	struct qrtr_control gone = { .command = QRTR_DEL_CLIENT, .words = { QRTR_US, port ? port->number : 0 } };
	bool outside = current_thread() != link_thread_is;

	if (port == NULL || port->number == 0) {
		return;
	}
	if (outside) {
		lck_mtx_lock(&qrtr_calling);            /* (a callback of the port's that is running ends first) */
	}
	lck_mtx_lock(&qrtr_lock);
	port->number = 0;
	lck_mtx_unlock(&qrtr_lock);
	if (outside) {
		lck_mtx_unlock(&qrtr_calling);
	}
	if (qrtr_peer != 0 && glink.channel[CHANNEL_QRTR].open) {
		qrtr_out_packet(QRTR_DEL_CLIENT, gone.words[1], qrtr_peer, QRTR_CONTROL, &gone, sizeof(gone));
	}
}

static const struct hakios_qrtr_ops qrtr_ops = { .open = qrtr_open, .send = qrtr_send, .lookup = qrtr_lookup, .close = qrtr_close };

/* The modem's QRTR for the rest of the kernel: NULL until the link is up (sys/hakios.qrtr.h). */
const struct hakios_qrtr_ops *
hakios_mpss_qrtr(void)
{
	return glink.channel[CHANNEL_QRTR].open ? &qrtr_ops : NULL;
}

/* ---- the link's thread ----------------------------------------------------------------------------- */

static volatile bool link_wanted, link_runs;

/* Everything the modem says once it runs: its SMP2P item, its queue, and from that the datagrams, handed on from here. */
static void
link_thread(__unused void *parameter, __unused wait_result_t result)
{
	struct smp2p words, last;
	struct glink_packet *packet;
	bool gave_up = false;

	bzero(&last, sizeof(last));
	dsp_words(&last);
	while (link_wanted) {
		sleep_us(1000);
		doorbells();
		if (dsp_words(&words) && memcmp(&words, &last, sizeof(words)) != 0) {
			last = words;
			dsp_listen(&words);
			if ((dsp_state(&words) & DSP_FATAL) && !gave_up) {
				gave_up = true;
				printf("mpss: link: THE MODEM SAYS IT HAS GIVEN UP (\"slave-kernel\" = %#x)\n", dsp_state(&words));
				show_crash_reason();
			}
		}
		for (;;) {
			struct glink_channel *channel = &glink.channel[CHANNEL_QRTR];

			lck_mtx_lock(&glink_lock);
			glink_pump();
			packet = channel->first;
			if (packet != NULL) {
				channel->first = packet->next;
				channel->last = channel->first == NULL ? NULL : channel->last;
			}
			lck_mtx_unlock(&glink_lock);
			if (packet == NULL) {
				break;
			}
			lck_mtx_lock(&qrtr_calling);
			qrtr_input(packet->data, packet->length);
			lck_mtx_unlock(&qrtr_calling);
			kfree_data(packet, sizeof(*packet) + packet->length);
		}
		lck_mtx_lock(&qrtr_calling);
		qrtr_fresh();
		lck_mtx_unlock(&qrtr_calling);
	}
	link_runs = false;
	thread_terminate(current_thread());
}

/* The link given up, before the modem is stopped: Linux too takes GLINK down first and tells the modem nothing of it. */
static void
link_stop(void)
{
	struct glink_packet *packet;

	if (link_runs) {
		link_wanted = false;
		for (unsigned ms = 0; ms < 2000 && link_runs; ms++) {
			sleep_ms(1);
		}
		printf("mpss: link: %s; GLINK said %u and heard %u messages, QRTR sent %u and received %u datagrams; its doorbells heard: "
		    "GLINK %u, SMP2P %u; rmtfs: %u requests, %u reads (%u sectors), %u writes (%u sectors), %u refused\n", link_runs ?
		    "THE THREAD DOES NOT END" : "down", glink.said, glink.heard, qrtr_out, qrtr_in, doorbell.glink, doorbell.smp2p,
		    rmtfs_requests, rmtfs_reads, rmtfs_sectors_read, rmtfs_writes, rmtfs_sectors_written, rmtfs_refused);
		printf("mpss: link: tftp: %u requests, %u served whole, %u refused; the registry: %u questions\n", tftp_requests, tftp_served,
		    tftp_refused, servreg_requests);
	}
	lck_mtx_lock(&glink_lock);
	glink.up = false;
	for (unsigned c = 0; c < CHANNELS; c++) {
		struct glink_channel *channel = &glink.channel[c];

		channel->open = false;
		while ((packet = channel->first) != NULL) {
			channel->first = packet->next;
			kfree_data(packet, sizeof(*packet) + packet->length);
		}
		channel->last = NULL;
	}
	lck_mtx_unlock(&glink_lock);
	lck_mtx_lock(&qrtr_lock);
	bzero(qrtr_servers, sizeof(qrtr_servers));
	qrtr_peer = 0;
	lck_mtx_unlock(&qrtr_lock);
}

/* ---- the modem's file systems: RMTFS, a service of ours on QRTR ----------------------------------- */

/*
 * The modem keeps its own files -- its settings, its calibration, what it has learnt of the
 * networks -- on four partitions of the phone's storage, which it cannot reach itself: the
 * application processor reads and writes them for it, through QMI service 14, "RMTFS" (Linux:
 * github.com/linux-msm/rmtfs, which this follows). Until that service is there the modem offers
 * nothing beyond SSCTL and its service registry (service 43 and 66: what the first run saw), and
 * waits. The modem opens a file by path (/boot/modem_fs1 and so on), asks for a buffer (2.5 MB
 * of memory both can reach: dt qcom,rmtfs_sharedmem@0; TrustZone is asked to let the modem into
 * it, as Linux's rmtfs_mem and Android's sharedmem do, and out again when it is shut down), and
 * then reads and writes sectors of 512 bytes through that buffer.
 *
 * Here the four partitions are read whole into memory of ours when the steps begin (the phone's
 * storage through the loader, hakios.ufs.c: ufs=2, which the ordinary run has), the modem is
 * served out of that, and what it writes goes into that memory and no further: the storage is
 * not written. The modem is content with that for a run; what it would have saved is lost when
 * the run ends, as the next run reads the partitions afresh.
 *
 * QRTR's name service is the application processor's job as well (Linux: net/qrtr/ns.c): the
 * modem is told of our service when it says hello and when it looks for one, with an empty
 * "new server" to end an answer to a lookup, as Linux ends it.
 */
#define RMTFS_SERVICE           14
#define RMTFS_VERSION           1
#define RMTFS_INSTANCE          0
#define RMTFS_SECTOR            512
#define RMTFS_BUFFER_SIZE       0x280000u       /* dt: reg of qcom,rmtfs_sharedmem@0 */
#define RMTFS_OPEN              1
#define RMTFS_CLOSE             2
#define RMTFS_RW_IOVEC          3
#define RMTFS_ALLOC_BUFF        4
#define RMTFS_GET_DEV_ERROR     5
#define QMI_REQUEST             0
#define QMI_RESPONSE            2
#define QMI_RESULT_FAILURE      1
#define QMI_ERR_INTERNAL        1
#define QMI_ERR_MALFORMED       2
#define SCM_MP_ASSIGN           SCM_CALL(0xc, 0x16)
#define SCM_ASSIGN_ARGUMENTS    0x1117          /* seven: a buffer, a value, a buffer, a value, a buffer, two values */
#define VMID_HLOS               3
#define VMID_MSS_MSA            15
#define VMID_NAV                0x2b            /* the modem's navigation side; dt: qcom,vm-nav-path of qcom,rmtfs_sharedmem@0 */
#define VM_PERM_RW              6

static struct {
	const char *path, *partition;
	uint8_t *data;
	uint64_t size;
	bool open;
} rmtfs_files[4] = {
	{ "/boot/modem_fs1", "modemst1" }, { "/boot/modem_fs2", "modemst2" }, { "/boot/modem_fsc", "fsc" },
	{ "/boot/modem_fsg", "fsg" },
};
static struct hakios_qrtr_port *rmtfs_port;
static vm_offset_t rmtfs_buffer;                /* the buffer, as this kernel sees it; 0: none yet */
static volatile uint8_t *rmtfs_window;          /* ... and the uncached one it is used through */
static uint64_t rmtfs_buffer_pa;
static bool rmtfs_shared;                       /* the modem has been let into it */

static unsigned
rmtfs_load(void)
{
	unsigned loaded = 0;

	for (unsigned f = 0; f < 4; f++) {
		uint64_t size = 0;
		int error = hakios_ufs_read_partition(rmtfs_files[f].partition, 0, NULL, 0, &size);

		if (error != 0 || size == 0 || size > (64u << 20)) {
			printf("mpss: %u: the partition %s: %s (%d)\n", step, rmtfs_files[f].partition, error == ENXIO ?
			    "no storage to read it from (the loader was not asked for ufs=2)" : "not read", error);
			continue;
		}
		if (rmtfs_files[f].data == NULL || rmtfs_files[f].size != size) {
			if (rmtfs_files[f].data != NULL) {
				kfree_data(rmtfs_files[f].data, rmtfs_files[f].size);
			}
			rmtfs_files[f].data = kalloc_data(size, Z_WAITOK | Z_ZERO);
			rmtfs_files[f].size = rmtfs_files[f].data == NULL ? 0 : size;
		}
		if (rmtfs_files[f].data == NULL) {
			printf("mpss: %u: no memory for %s (%llu bytes)\n", step, rmtfs_files[f].partition, size);
			continue;
		}
		error = hakios_ufs_read_partition(rmtfs_files[f].partition, 0, rmtfs_files[f].data, size, NULL);
		if (error != 0) {
			printf("mpss: %u: the partition %s: read error %d\n", step, rmtfs_files[f].partition, error);
			continue;
		}
		printf("mpss: %u: the partition %s (%s): %llu bytes read into memory of ours\n", step, rmtfs_files[f].partition,
		    rmtfs_files[f].path, size);
		loaded++;
	}
	return loaded;
}

/*
 * The buffer, once. The modem's requests carry addresses of 32 bits, so it has to be below 4 GB,
 * and the kernel's free memory there is not to be had in one piece of 2.5 MB; so it is the first
 * 2.5 MB of the memory reserved for the compute DSP (dt: reserved-memory cdsp@9c700000, 30 MB,
 * no-map: nothing of ours uses it, and the CDSP is never started here), through an uncached
 * window, as the modem reads it.
 */
#define RMTFS_BUFFER_PA_DEFAULT 0x9c700000ull   /* dt: a window into reserved-memory cdsp@... (the CDSP is not started here) */

static bool
rmtfs_buffer_make(void)
{
	if (rmtfs_window != NULL) {
		return true;
	}
	rmtfs_buffer_pa = RMTFS_BUFFER_PA_DEFAULT;
	PE_parse_boot_argn("modem_buffer", &rmtfs_buffer_pa, sizeof(rmtfs_buffer_pa));      /* (another region, to try) */
	rmtfs_window = (volatile uint8_t *)ml_io_map_unmappable((vm_offset_t)rmtfs_buffer_pa, RMTFS_BUFFER_SIZE, VM_WIMG_WCOMB);
	if (rmtfs_window == NULL) {
		printf("mpss: %u: the modem's buffer at %#llx cannot be mapped\n", step, rmtfs_buffer_pa);
		return false;
	}
	rmtfs_buffer = (vm_offset_t)rmtfs_window;
	for (uint32_t i = 0; i < RMTFS_BUFFER_SIZE; i += 8) {
		*(volatile uint64_t *)(rmtfs_window + i) = 0;
	}
	settle();
	printf("mpss: %u: the modem's buffer: %u bytes at %#llx (the compute DSP's memory, unused here)\n", step, RMTFS_BUFFER_SIZE,
	    rmtfs_buffer_pa);
	return true;
}

/*
 * TrustZone asked to let the modem into the buffer (Linux's qcom_scm_assign_mem(): the memory, who
 * has it now, who is to have it and how), or, with `in` false, out of it again.
 */
static bool
rmtfs_share(bool in)
{
	struct {
		uint64_t address, size;                 /* +0: the memory */
		uint64_t pad0[6];
		uint32_t sources[3];                    /* +64: who has it */
		uint32_t pad1[13];
		struct {                                /* +128: who is to have it */
			uint32_t vmid, permission;
			uint64_t context;
			uint32_t context_size, unused;
		} destinations[3];
		uint8_t pad2[40];
		uint64_t more[7];                       /* +240: the arguments beyond the registers */
	} *page;
	vm_offset_t memory = 0;
	uint64_t answer = ~0ull, pa;
	unsigned sources, destinations;
	int32_t result;

	if (rmtfs_buffer == 0 || rmtfs_shared == in) {
		return rmtfs_buffer != 0;
	}
	if (kmem_alloc(kernel_map, &memory, PAGE_SIZE, KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
		return false;
	}
	page = (void *)memory;
	page->address = rmtfs_buffer_pa;
	page->size = RMTFS_BUFFER_SIZE;
	if (in) {
		page->sources[0] = VMID_HLOS;
		sources = 1;
		page->destinations[0].vmid = VMID_HLOS;
		page->destinations[0].permission = VM_PERM_RW;
		page->destinations[1].vmid = VMID_MSS_MSA;
		page->destinations[1].permission = VM_PERM_RW;
		page->destinations[2].vmid = VMID_NAV;
		page->destinations[2].permission = VM_PERM_RW;
		destinations = option("modem_no_nav") ? 2 : 3;
	} else {
		page->sources[0] = VMID_HLOS;
		page->sources[1] = VMID_MSS_MSA;
		page->sources[2] = VMID_NAV;
		sources = option("modem_no_nav") ? 2 : 3;
		page->destinations[0].vmid = VMID_HLOS;
		page->destinations[0].permission = VM_PERM_RW;
		destinations = 1;
	}
	pa = kvtophys(memory);
	page->more[0] = sources * 4;                                    /* argument 3: the size of the sources */
	page->more[1] = pa + 128;                                       /* 4: the destinations */
	page->more[2] = destinations * sizeof(page->destinations[0]);   /* 5: their size */
	page->more[3] = 0;                                              /* 6: spare */
	for (vm_offset_t line = memory; line < memory + PAGE_SIZE; line += 64) {
		__asm__ volatile ("dc civac, %0" : : "r" (line));
	}
	settle();
	announce(in ? "ask TrustZone to let the modem into our buffer for its files (the memory assigned to HLOS and MSS, both "
	    "read and write)" : "ask TrustZone to take the buffer back from the modem (assigned to HLOS alone)");
	result = scm(in ? "to share the buffer with the modem" : "to take the buffer back", SCM_MP_ASSIGN, SCM_ASSIGN_ARGUMENTS, pa, 16,
	    pa + 64, pa + 240, &answer);
	kmem_free(kernel_map, memory, PAGE_SIZE);
	if (result != 0 || answer != 0) {
		printf("mpss: %u: TRUSTZONE DOES NOT %s: the call returns %d, its answer is %#llx\n", step, in ? "SHARE THE BUFFER" :
		    "TAKE THE BUFFER BACK", result, answer);
		return false;
	}
	rmtfs_shared = in;
	printf("mpss: %u: %s\n", step, in ? "THE MODEM MAY USE THE BUFFER" : "the buffer is ours alone again");
	if (in && option("modem_share_test")) {
		/* (an experiment: is the buffer still ours to write, now that it is shared? said aloud before and after) */
		tell("mpss: %u: about to write a word into the shared buffer at %#llx, and read it back\n", step, rmtfs_buffer_pa + 0x200);
		*(volatile uint32_t *)(rmtfs_window + 0x200) = 0x52545346;
		settle();
		tell("mpss: %u: the shared buffer takes our write: it reads %#x\n", step, *(volatile uint32_t *)(rmtfs_window + 0x200));
	}
	return true;
}

/* A TLV of a QMI message: its value and length, or NULL. */
static const uint8_t *
qmi_tlv(const uint8_t *m, size_t length, uint8_t type, uint16_t *size)
{
	for (size_t at = 7; at + 3 <= length;) {
		uint16_t n = (uint16_t)(m[at + 1] | m[at + 2] << 8);

		if (at + 3 + n > length) {
			return NULL;
		}
		if (m[at] == type) {
			*size = n;
			return m + at + 3;
		}
		at += 3 + n;
	}
	return NULL;
}

static uint32_t
le32(const uint8_t *p)
{
	return (uint32_t)(p[0] | p[1] << 8 | p[2] << 16 | p[3] << 24);
}

/* The modem's request for its files: answered, the one TLV the answer carries besides the result made here. From the link's thread. */
static void
rmtfs_receive(__unused void *context, uint32_t node, uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data, *value;
	uint8_t answer[64], *tlv;
	uint16_t message, size = 0, result = 0, error = 0, extra = 0;
	unsigned file = 4;

	if (length < 7 || m[0] != QMI_REQUEST) {
		return;
	}
	message = (uint16_t)(m[3] | m[4] << 8);
	rmtfs_requests++;
	bzero(answer, sizeof(answer));
	answer[0] = QMI_RESPONSE;
	answer[1] = m[1];
	answer[2] = m[2];
	answer[3] = m[3];
	answer[4] = m[4];
	tlv = answer + 7 + 7;                   /* (the result TLV, 7 bytes, comes first) */
	switch (message) {
	case RMTFS_OPEN:
		/* (the path: the TLV's value is the string itself, with no length in front -- the modem sends it so) */
		value = qmi_tlv(m, length, 1, &size);
		for (unsigned f = 0; value != NULL && f < 4; f++) {
			if (size == strlen(rmtfs_files[f].path) && memcmp(value, rmtfs_files[f].path, size) == 0) {
				file = f;
			}
		}
		if (file == 4 || rmtfs_files[file].data == NULL) {
			printf("mpss: rmtfs: the modem opens \"%.*s\": NOT HERE\n", value != NULL ? (int)(size < 64 ? size : 64) : 0,
			    value != NULL ? (const char *)value : "");
			result = QMI_RESULT_FAILURE;
			error = QMI_ERR_INTERNAL;
			rmtfs_refused++;
			break;
		}
		rmtfs_files[file].open = true;
		printf("mpss: rmtfs: the modem opens %s (%s, %llu bytes): caller %u\n", rmtfs_files[file].path, rmtfs_files[file].partition,
		    rmtfs_files[file].size, file + 1);
		tlv[0] = 0x10;                  /* the caller's id: the file's number plus one */
		tlv[1] = 4;
		tlv[2] = 0;
		tlv[3] = (uint8_t)(file + 1);
		extra = 7;
		break;
	case RMTFS_CLOSE:
		value = qmi_tlv(m, length, 1, &size);
		file = value != NULL && size == 4 && le32(value) >= 1 && le32(value) <= 4 ? le32(value) - 1 : 4;
		if (file < 4) {
			rmtfs_files[file].open = false;
			printf("mpss: rmtfs: the modem closes %s\n", rmtfs_files[file].path);
		}
		break;
	case RMTFS_ALLOC_BUFF:
		value = qmi_tlv(m, length, 2, &size);
		if (value == NULL || size != 4 || le32(value) > RMTFS_BUFFER_SIZE || rmtfs_buffer == 0 || !rmtfs_shared) {
			printf("mpss: rmtfs: the modem asks for a buffer of %u bytes: NOT GIVEN (ours is %u, %s)\n", value != NULL && size == 4 ?
			    le32(value) : 0, RMTFS_BUFFER_SIZE, rmtfs_shared ? "shared" : "NOT SHARED");
			result = QMI_RESULT_FAILURE;
			error = QMI_ERR_INTERNAL;
			rmtfs_refused++;
			break;
		}
		tell("mpss: rmtfs: the modem asks for a buffer of %u bytes: given ours at %#llx\n", le32(value), rmtfs_buffer_pa);
		tlv[0] = 0x10;
		tlv[1] = 8;
		tlv[2] = 0;
		for (unsigned b = 0; b < 8; b++) {
			tlv[3 + b] = (uint8_t)(rmtfs_buffer_pa >> (8 * b));
		}
		extra = 11;
		break;
	case RMTFS_RW_IOVEC: {
		const uint8_t *direction = qmi_tlv(m, length, 2, &size), *entries;
		uint16_t direction_size = size, entries_size = 0;
		bool write;
		unsigned count;

		value = qmi_tlv(m, length, 1, &size);
		entries = qmi_tlv(m, length, 3, &entries_size);
		file = value != NULL && size == 4 && le32(value) >= 1 && le32(value) <= 4 ? le32(value) - 1 : 4;
		if (file == 4 || !rmtfs_files[file].open || direction == NULL || direction_size != 1 || entries == NULL || entries_size < 1 ||
		    entries_size != 1 + 12 * entries[0] || rmtfs_window == NULL) {
			result = QMI_RESULT_FAILURE;
			error = QMI_ERR_MALFORMED;
			rmtfs_refused++;
			break;
		}
		write = direction[0] != 0;
		count = entries[0];
		{
			/* (an experiment: the first request waited on, said aloud before and after, for the log to catch what follows) */
			static bool first = true;
			uint32_t delay = 0;

			if (first && PE_parse_boot_argn("modem_rmtfs_delay", &delay, sizeof(delay)) && delay != 0) {
				tell("mpss: rmtfs: the modem's first request (%s %u pieces of %s, from sector %u, %u sectors, to %#x): waiting %u s "
				    "before it is served\n", write ? "write" : "read", count, rmtfs_files[file].partition, le32(entries + 1),
				    le32(entries + 9), le32(entries + 5), delay);
				for (uint32_t ms = 0; ms < delay * 1000; ms++) {
					sleep_ms(1);
				}
				tell("mpss: rmtfs: serving it now\n");
			}
			first = false;
		}
		if (option("modem_rmtfs_rw_off")) {
			/* (an experiment: the modem is told no, and the buffer is not touched from here) */
			printf("mpss: rmtfs: the modem %s %u pieces of %s: REFUSED (modem_rmtfs_rw_off)\n", write ? "writes" : "reads", count,
			    rmtfs_files[file].partition);
			result = QMI_RESULT_FAILURE;
			error = QMI_ERR_INTERNAL;
			rmtfs_refused++;
			break;
		}
		for (unsigned e = 0; e < count && result == 0; e++) {
			uint64_t sector = le32(entries + 1 + 12 * e), physical = le32(entries + 5 + 12 * e), sectors = le32(entries + 9 + 12 * e);
			uint64_t bytes = sectors * RMTFS_SECTOR;

			if (physical < rmtfs_buffer_pa || physical - rmtfs_buffer_pa + bytes > RMTFS_BUFFER_SIZE ||
			    sector * RMTFS_SECTOR + bytes > rmtfs_files[file].size) {
				printf("mpss: rmtfs: %s %llu sectors from %llu of %s at %#llx: OUTSIDE THE FILE OR THE BUFFER\n", write ? "write" :
				    "read", sectors, sector, rmtfs_files[file].partition, physical);
				result = QMI_RESULT_FAILURE;
				error = QMI_ERR_MALFORMED;
				rmtfs_refused++;
				break;
			}
			if (write) {
				for (uint64_t i = 0; i < bytes; i++) {
					rmtfs_files[file].data[sector * RMTFS_SECTOR + i] = rmtfs_window[physical - rmtfs_buffer_pa + i];
				}
				rmtfs_sectors_written += sectors;
			} else {
				for (uint64_t i = 0; i < bytes; i++) {
					rmtfs_window[physical - rmtfs_buffer_pa + i] = rmtfs_files[file].data[sector * RMTFS_SECTOR + i];
				}
				rmtfs_sectors_read += sectors;
			}
		}
		settle();
		if (option("modem_rmtfs_rw_fake")) {
			/* (an experiment: the buffer is filled as asked, but the modem is told no: does the filling alone end the run?) */
			tell("mpss: rmtfs: the buffer is filled for the modem's %s of %s, and it is told NO (modem_rmtfs_rw_fake)\n",
			    write ? "write" : "read", rmtfs_files[file].partition);
			result = QMI_RESULT_FAILURE;
			error = QMI_ERR_INTERNAL;
			break;
		}
		rmtfs_reads += !write;
		rmtfs_writes += write;
		if (rmtfs_reads + rmtfs_writes <= 12) {
			tell("mpss: rmtfs: the modem %s %u pieces of %s%s: served\n", write ? "writes" : "reads", count, rmtfs_files[file].partition,
			    write ? " (kept in memory, not on the storage)" : "");
		}
		break;
	}
	case RMTFS_GET_DEV_ERROR:
		tlv[0] = 0x10;                  /* the device's error status: none */
		tlv[1] = 1;
		tlv[2] = 0;
		tlv[3] = 0;
		extra = 4;
		break;
	default:
		printf("mpss: rmtfs: a request this does not know (%#x, %zu bytes)\n", message, length);
		result = QMI_RESULT_FAILURE;
		error = QMI_ERR_MALFORMED;
		break;
	}
	answer[5] = (uint8_t)(7 + extra);
	answer[6] = (uint8_t)((7 + extra) >> 8);
	answer[7] = 2;                          /* the result TLV */
	answer[8] = 4;
	answer[9] = 0;
	answer[10] = (uint8_t)result;
	answer[11] = (uint8_t)(result >> 8);
	answer[12] = (uint8_t)error;
	answer[13] = (uint8_t)(error >> 8);
	if (rmtfs_port != NULL) {
		int sent = qrtr_send(rmtfs_port, node, port, answer, 7 + 7 + extra);

		if (sent != 0) {
			printf("mpss: rmtfs: the answer to %#x is NOT SENT (%d)\n", message, sent);
		}
	}
}

/* Our service announced to a node (its control port): a "new server", or the empty one that ends an answer to a lookup. */
static void
qrtr_announce(uint32_t node, unsigned server, bool end)
{
	struct qrtr_control packet = { .command = QRTR_NEW_SERVER };

	if (!end) {
		packet.words[0] = our_servers[server].service;
		packet.words[1] = our_servers[server].instance;
		packet.words[2] = QRTR_US;
		packet.words[3] = our_servers[server].port;
	}
	qrtr_out_packet(QRTR_NEW_SERVER, QRTR_CONTROL, node, QRTR_CONTROL, &packet, sizeof(packet));
}

static void
qrtr_announce_all(uint32_t node)
{
	for (unsigned i = 0; i < sizeof(our_servers) / sizeof(our_servers[0]); i++) {
		if (our_servers[i].port != 0) {
			qrtr_announce(node, i, false);
		}
	}
}

static void tftp_receive(void *context, uint32_t node, uint32_t port, const void *data, size_t length);
static void servreg_receive(void *context, uint32_t node, uint32_t port, const void *data, size_t length);
static struct hakios_qrtr_port *tftp_port, *servreg_port;
static bool ipa_qmi_start(void);
static void ipa_qmi_stop(void);
static struct hakios_qrtr_port *ipa_host_port;

/* The port, and the service on it; the partitions are read in step 2, the buffer shared in step 7. */
static bool
rmtfs_start(void)
{
	if (rmtfs_port == NULL) {
		rmtfs_port = qrtr_open(rmtfs_receive, NULL, NULL);
	}
	if (rmtfs_port == NULL) {
		return false;
	}
	our_servers[0].service = RMTFS_SERVICE;
	our_servers[0].instance = RMTFS_VERSION | RMTFS_INSTANCE << 8;
	our_servers[0].port = rmtfs_port->number;
	if (!option("modem_no_tftp")) {
		if (tftp_port == NULL) {
			tftp_port = qrtr_open(tftp_receive, NULL, NULL);
		}
		if (tftp_port != NULL) {
			our_servers[1].service = 4096;          /* TQFTP (below) */
			our_servers[1].instance = 1;
			our_servers[1].port = tftp_port->number;
		}
		if (servreg_port == NULL) {
			servreg_port = qrtr_open(servreg_receive, NULL, NULL);
		}
		if (servreg_port != NULL) {
			our_servers[2].service = 64;            /* the service registry's locator (below) */
			our_servers[2].instance = 257;
			our_servers[2].port = servreg_port->number;
		}
		if (ipa_qmi_start()) {
			our_servers[3].service = 0x31;          /* the IPA's host service (below) */
			our_servers[3].instance = 0x101;
			our_servers[3].port = ipa_host_port->number;
		}
	}
	return true;
}

static void
rmtfs_stop(void)
{
	for (unsigned i = 0; i < sizeof(our_servers) / sizeof(our_servers[0]); i++) {
		our_servers[i].port = 0;
	}
	if (rmtfs_port != NULL) {
		qrtr_close(rmtfs_port);
		rmtfs_port = NULL;
	}
	if (tftp_port != NULL) {
		qrtr_close(tftp_port);
		tftp_port = NULL;
	}
	if (servreg_port != NULL) {
		qrtr_close(servreg_port);
		servreg_port = NULL;
	}
	ipa_qmi_stop();
	for (unsigned f = 0; f < 4; f++) {
		rmtfs_files[f].open = false;
	}
}

/* ---- TFTP over QRTR: the modem's files of configuration (tqftpserv) --------------------------------- */

/*
 * Besides its file systems, the modem reads files of its firmware partition -- the carrier
 * configurations (mcfg), shared objects -- through TFTP carried in QRTR datagrams, from a
 * service numbered 4096 that Linux's tqftpserv and Android's provide; until it has them it
 * stays offline ("device not ready" to every request to go online). Here the partition's
 * directory image/ is in the system image under /fw/image (userland/build.sh: fw-modem/image,
 * pulled from the phone's /vendor/firmware_mnt/image), and served read-only: a read request
 * names a file as /readonly/firmware/image/..., its options (RFC 2347: blksize, tsize, timeoutms;
 * Qualcomm's wsize, a window of blocks sent without waiting, rsize, how much of the file, and
 * seek, from where) are answered in an OACK, and the blocks go from a port of ours opened for the
 * transfer, as TFTP wants. Writes (/readwrite/...) are refused: nothing of the modem's is kept
 * here between runs.
 */
#define TQFTP_SERVICE           4096
#define TQFTP_INSTANCE          1               /* version 1, instance 0 */
#define TFTP_RRQ                1
#define TFTP_WRQ                2
#define TFTP_DATA               3
#define TFTP_ACK                4
#define TFTP_ERROR              5
#define TFTP_OACK               6
#define TFTP_ENOENT             1
#define TFTP_EACCESS            2
#define TFTP_EBADOP             4
#define TFTP_END_OF_TRANSFER    9
#define TFTP_MAX_BLOCK          8000            /* a datagram of ours holds 8192; the modem's intents 0x4400 */
#define TFTP_MAX_WINDOW         4

struct tftp_transfer {
	struct hakios_qrtr_port *port;
	uint32_t node, peer;
	const uint8_t *data;                    /* a read: the file, whole, in memory of ours (root_file(), or tftp_files) */
	uint64_t size;
	uint32_t blksize, wsize, rsize, seek;
	uint32_t sent;                          /* blocks sent so far */
	bool done, writing, borrowed;           /* borrowed: data is a tftp_files entry's, not to be freed */
	uint8_t *written;                       /* a write: what has come, up to TFTP_WRITE_MAX */
	uint32_t written_size, expected;
	char name[96];
};
static struct tftp_transfer tftp_transfers[4];

/* What the modem writes, kept for the run (it reads some of it back); not on the storage. */
#define TFTP_WRITE_MAX          (1u << 20)
static struct {
	char name[96];
	uint8_t *data;
	uint32_t size;
} tftp_files[8];

static void
tftp_error(struct hakios_qrtr_port *from, uint32_t node, uint32_t port, uint16_t code, const char *text)
{
	uint8_t packet[64];
	size_t n = strlen(text) + 1;

	if (n > sizeof(packet) - 4) {
		n = sizeof(packet) - 4;
	}
	packet[0] = 0;
	packet[1] = TFTP_ERROR;
	packet[2] = (uint8_t)(code >> 8);
	packet[3] = (uint8_t)code;
	memcpy(packet + 4, text, n);
	packet[4 + n - 1] = 0;
	qrtr_send(from, node, port, packet, 4 + n);
}

static void
tftp_transfer_end(struct tftp_transfer *t)
{
	if (t->data != NULL && !t->borrowed) {
		uint8_t *data = __DECONST(uint8_t *, t->data);

		kfree_data(data, t->size);
	}
	t->data = NULL;
	if (t->written != NULL) {
		kfree_data(t->written, TFTP_WRITE_MAX);
		t->written = NULL;
	}
	if (t->port != NULL) {
		qrtr_close(t->port);
		t->port = NULL;
	}
	t->done = true;
}

/* Blocks `from` + 1 onward, a window of them: TFTP's blocks are numbered from 1. */
static void
tftp_send_window(struct tftp_transfer *t, uint32_t from)
{
	uint8_t *packet = kalloc_data(t->blksize + 4, Z_WAITOK);

	if (packet == NULL) {
		return;
	}
	for (uint32_t w = 0; w < t->wsize; w++) {
		uint32_t block = from + 1 + w;
		uint64_t offset = (uint64_t)t->seek + (uint64_t)(block - 1) * t->blksize;
		uint32_t n = offset >= t->size ? 0 : (uint32_t)(t->size - offset < t->blksize ? t->size - offset : t->blksize);

		if (t->rsize != 0 && (uint64_t)(block - 1) * t->blksize >= t->rsize) {
			break;                          /* (as much as was asked for has gone) */
		}
		if (t->rsize != 0 && (uint64_t)block * t->blksize > t->rsize) {
			uint32_t left = (uint32_t)(t->rsize - (uint64_t)(block - 1) * t->blksize);

			n = left < n ? left : n;        /* (as much as was asked for, and no more than the file has) */
		}
		packet[0] = 0;
		packet[1] = TFTP_DATA;
		packet[2] = (uint8_t)(block >> 8);
		packet[3] = (uint8_t)block;
		if (n != 0) {
			memcpy(packet + 4, t->data + offset, n);
		}
		if (qrtr_send(t->port, t->node, t->peer, packet, 4 + n) != 0) {
			break;
		}
		t->sent = block;
		if (n < t->blksize) {
			break;                          /* the last block */
		}
	}
	kfree_data(packet, t->blksize + 4);
}

/* What the modem says to a transfer's port: acknowledgements, or an error that ends it. From the link's thread. */
static void
tftp_transfer_receive(void *context, uint32_t node, uint32_t port, const void *data, size_t length)
{
	struct tftp_transfer *t = context;
	const uint8_t *m = data;
	uint16_t opcode, block;

	if (t->done || node != t->node || port != t->peer || length < 4) {
		return;
	}
	opcode = (uint16_t)(m[0] << 8 | m[1]);
	block = (uint16_t)(m[2] << 8 | m[3]);
	if (t->writing && opcode == TFTP_DATA) {
		uint32_t n = (uint32_t)(length - 4);
		uint8_t ack[4] = { 0, TFTP_ACK, (uint8_t)(block >> 8), (uint8_t)block };

		if (block != t->expected) {
			tftp_error(t->port, node, port, TFTP_EBADOP, "block out of sequence");
			tftp_transfer_end(t);
			return;
		}
		if (t->written_size + n > TFTP_WRITE_MAX) {
			tftp_error(t->port, node, port, 3, "too much");
			tftp_transfer_end(t);
			return;
		}
		memcpy(t->written + t->written_size, m + 4, n);
		t->written_size += n;
		t->expected++;
		if (block % t->wsize == 0 || n < t->blksize) {
			qrtr_send(t->port, node, port, ack, sizeof(ack));
		}
		if (n < t->blksize) {
			/* the last block: the file is kept, under its name, for the run */
			unsigned f, free_slot = 8;

			for (f = 0; f < 8; f++) {
				if (tftp_files[f].data != NULL && strcmp(tftp_files[f].name, t->name) == 0) {
					break;
				}
				if (tftp_files[f].data == NULL && free_slot == 8) {
					free_slot = f;
				}
			}
			if (f == 8) {
				f = free_slot;
			}
			if (f < 8) {
				if (tftp_files[f].data != NULL) {
					kfree_data(tftp_files[f].data, tftp_files[f].size);
				}
				tftp_files[f].data = kalloc_data(t->written_size ? t->written_size : 1, Z_WAITOK);
				if (tftp_files[f].data != NULL) {
					memcpy(tftp_files[f].data, t->written, t->written_size);
					tftp_files[f].size = t->written_size;
					strlcpy(tftp_files[f].name, t->name, sizeof(tftp_files[f].name));
				}
			}
			printf("mpss: tftp: the modem has written \"%s\": %u bytes, kept%s\n", t->name, t->written_size, f < 8 ? "" :
			    " NOWHERE (no room)");
			tftp_served++;
			tftp_transfer_end(t);
		}
		return;
	}
	if (opcode == TFTP_ERROR) {
		if (block != TFTP_END_OF_TRANSFER) {
			printf("mpss: tftp: %s: the modem ends the transfer with error %u \"%.*s\"\n", t->name, block, (int)(length - 4), m + 4);
		}
		tftp_transfer_end(t);
		return;
	}
	if (opcode != TFTP_ACK) {
		tftp_error(t->port, node, port, TFTP_EBADOP, "expected ACK");
		tftp_transfer_end(t);
		return;
	}
	if ((uint64_t)t->seek + (uint64_t)block * t->blksize >= t->size || (t->rsize != 0 && (uint64_t)block * t->blksize >= t->rsize)) {
		/* the acknowledged block was the last: over */
		tftp_served++;
		tftp_transfer_end(t);
		return;
	}
	tftp_send_window(t, block);
}

static const char *
tftp_file_of(const char *path)
{
	static const char *const prefixes[] = { "/readonly/firmware/image/", "/readonly/vendor/firmware_mnt/image/",
		"/readonly/vendor/firmware/", "/readonly/firmware/" };

	for (unsigned i = 0; i < sizeof(prefixes) / sizeof(prefixes[0]); i++) {
		size_t n = strlen(prefixes[i]);

		if (strncmp(path, prefixes[i], n) == 0 && strnstr(path + n, "..", strlen(path + n)) == NULL) {
			return path + n;
		}
	}
	return NULL;
}

/* A request to our TFTP port: a read starts a transfer on a port of its own; anything else is refused. From the link's thread. */
static void
tftp_receive(__unused void *context, uint32_t node, uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data, *end = m + length;
	const char *name, *mode, *p;
	struct tftp_transfer *t = NULL;
	uint32_t blksize = 512, wsize = 1, rsize = 0, seek = 0;
	bool tsize = false, options = false;
	char fwname[160];
	uint64_t size = 0;
	const uint8_t *file;
	uint8_t oack[160];
	unsigned at;

	bool writing;

	if (length < 4) {
		return;
	}
	tftp_requests++;
	writing = (m[0] << 8 | m[1]) == TFTP_WRQ;
	if ((m[0] << 8 | m[1]) != TFTP_RRQ && !writing) {
		printf("mpss: tftp: the modem's request %u: not a read nor a write, refused\n", m[0] << 8 | m[1]);
		tftp_error(tftp_port, node, port, TFTP_EBADOP, "not a read nor a write");
		tftp_refused++;
		return;
	}
	name = (const char *)m + 2;
	p = name;
	while (p < (const char *)end && *p != 0) {
		p++;
	}
	if (p >= (const char *)end) {
		return;
	}
	mode = p + 1;
	p = mode;
	while (p < (const char *)end && *p != 0) {
		p++;
	}
	if (p >= (const char *)end) {
		return;
	}
	p++;
	/* the options: name, value, name, value..., each ended by a zero */
	while (p < (const char *)end) {
		const char *option = p, *value;

		while (p < (const char *)end && *p != 0) {
			p++;
		}
		if (p >= (const char *)end) {
			break;
		}
		value = ++p;
		while (p < (const char *)end && *p != 0) {
			p++;
		}
		if (p >= (const char *)end) {
			break;
		}
		p++;
		options = true;
		if (strcmp(option, "blksize") == 0) {
			blksize = (uint32_t)strtoul(value, NULL, 10);
		} else if (strcmp(option, "wsize") == 0) {
			wsize = (uint32_t)strtoul(value, NULL, 10);
		} else if (strcmp(option, "rsize") == 0) {
			rsize = (uint32_t)strtoul(value, NULL, 10);
		} else if (strcmp(option, "seek") == 0) {
			seek = (uint32_t)strtoul(value, NULL, 10);
		} else if (strcmp(option, "tsize") == 0) {
			tsize = true;
		}
	}
	if (blksize < 8) {
		blksize = 8;
	}
	if (blksize > TFTP_MAX_BLOCK) {
		blksize = TFTP_MAX_BLOCK;
	}
	if (wsize < 1) {
		wsize = 1;
	}
	if (wsize > TFTP_MAX_WINDOW) {
		wsize = TFTP_MAX_WINDOW;
	}
	for (unsigned i = 0; i < sizeof(tftp_transfers) / sizeof(tftp_transfers[0]) && t == NULL; i++) {
		if (tftp_transfers[i].port == NULL) {
			t = &tftp_transfers[i];
		}
	}
	if (t == NULL) {
		printf("mpss: tftp: the modem %s \"%s\": NO FREE TRANSFER\n", writing ? "writes" : "reads", name);
		tftp_error(tftp_port, node, port, TFTP_ENOENT, "busy");
		tftp_refused++;
		return;
	}
	if (writing) {
		bzero(t, sizeof(*t));
		t->written = kalloc_data(TFTP_WRITE_MAX, Z_WAITOK);
		t->port = t->written == NULL ? NULL : qrtr_open(tftp_transfer_receive, NULL, t);
		if (t->port == NULL) {
			tftp_transfer_end(t);
			tftp_error(tftp_port, node, port, TFTP_ENOENT, "no port");
			return;
		}
		t->node = node;
		t->peer = port;
		t->writing = true;
		t->expected = 1;
		t->blksize = blksize;
		t->wsize = wsize;
		strlcpy(t->name, name, sizeof(t->name));
		printf("mpss: tftp: the modem writes \"%s\": blocks of %u, window %u; taken into memory\n", name, blksize, wsize);
		if (!options) {
			uint8_t ack[4] = { 0, TFTP_ACK, 0, 0 };

			qrtr_send(t->port, node, port, ack, sizeof(ack));
			return;
		}
		size = 0;
		goto oack;
	}
	/* a read: what the modem wrote earlier under that name, else the firmware partition's file */
	file = NULL;
	for (unsigned f = 0; f < 8 && file == NULL; f++) {
		if (tftp_files[f].data != NULL && strcmp(tftp_files[f].name, name) == 0) {
			file = tftp_files[f].data;
			size = tftp_files[f].size;
			t->borrowed = true;
		}
	}
	if (file == NULL && tftp_file_of(name) == NULL) {
		printf("mpss: tftp: the modem reads \"%s\": NOT A PATH SERVED HERE\n", name);
		tftp_error(tftp_port, node, port, TFTP_ENOENT, "file not found");
		tftp_refused++;
		return;
	}
	if (file == NULL) {
		snprintf(fwname, sizeof(fwname), "image/%s", tftp_file_of(name));
		file = root_file(fwname, &size);
	} else {
		strlcpy(fwname, "(written by the modem)", sizeof(fwname));
	}
	if (file == NULL) {
		printf("mpss: tftp: the modem reads \"%s\" (fw/%s): NOT HERE\n", name, fwname);
		tftp_error(tftp_port, node, port, TFTP_ENOENT, "file not found");
		tftp_refused++;
		return;
	}
	{
		bool borrowed = t->borrowed;

		bzero(t, sizeof(*t));
		t->borrowed = borrowed;
	}
	t->port = qrtr_open(tftp_transfer_receive, NULL, t);
	if (t->port == NULL) {
		if (!t->borrowed) {
			uint8_t *gone = __DECONST(uint8_t *, file);

			kfree_data(gone, size);
		}
		tftp_error(tftp_port, node, port, TFTP_ENOENT, "no port");
		return;
	}
	t->node = node;
	t->peer = port;
	t->data = file;
	t->size = size;
	t->blksize = blksize;
	t->wsize = wsize;
	t->rsize = rsize;
	t->seek = seek;
	strlcpy(t->name, name, sizeof(t->name));
	if (tftp_requests <= 40) {
		printf("mpss: tftp: the modem reads \"%s\": %llu bytes; blocks of %u, window %u, %u asked for, from %u%s\n", name, size, blksize,
		    wsize, rsize, seek, options ? "" : " (no options)");
	}
	if (!options) {
		tftp_send_window(t, 0);
		return;
	}
oack:
	/* OACK: the options as taken, as tqftpserv answers them */
	oack[0] = 0;
	oack[1] = TFTP_OACK;
	at = 2;
	at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "blksize") + 1;
	at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "%u", blksize) + 1;
	at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "timeoutms") + 1;
	at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "1000") + 1;
	if (tsize && !t->writing) {
		at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "tsize") + 1;
		at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "%llu", size) + 1;
	}
	at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "wsize") + 1;
	at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "%u", wsize) + 1;
	if (rsize != 0) {
		at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "rsize") + 1;
		at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "%u", rsize) + 1;
	}
	if (seek != 0) {
		at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "seek") + 1;
		at += (unsigned)snprintf((char *)oack + at, sizeof(oack) - at, "%u", seek) + 1;
	}
	qrtr_send(t->port, node, port, oack, at);
}

/* ---- the service registry's locator (pd-mapper) ---------------------------------------------------- */

/*
 * The modem's protection domains register their services with a registry whose locator (QMI
 * service 64) is the application processor's: asked for a service by name ("tms/servreg",
 * "gps/gps_service"), it answers with the domains that provide it ("msm/modem/root_pd") and their
 * QMI instance. Linux's pd-mapper reads the table from the *.jsn files beside the firmware; here
 * they are read from /fw/image (the phone's own), each one domain with its services.
 */
#define SERVREG_SERVICE         64
#define SERVREG_INSTANCE        257             /* what pd-mapper publishes: version 257 */
#define SERVREG_GET_DOMAIN_LIST 0x21
#define SERVREG_PFR             0x24

static struct {
	char service[48];                       /* provider/service */
	char domain[48];                        /* soc/domain/subdomain */
	uint32_t instance;
} servreg_map[32];
static unsigned servreg_maps;

/* The text between the quotes after "key": in a JSON file of the registry's simple shape; NULL if there is none. */
static const char *
jsn_string(const char *from, const char *end, const char *key, char *out, size_t out_size)
{
	char quoted[48];
	const char *at;

	snprintf(quoted, sizeof(quoted), "\"%s\"", key);
	at = strnstr(from, quoted, (size_t)(end - from));
	if (at == NULL) {
		return NULL;
	}
	at += strlen(quoted);
	while (at < end && (*at == ':' || *at == ' ' || *at == '\t' || *at == '\n' || *at == '\r')) {
		at++;
	}
	if (at >= end) {
		return NULL;
	}
	if (*at == '"') {
		const char *close = at + 1;

		while (close < end && *close != '"') {
			close++;
		}
		if (close >= end) {
			return NULL;
		}
		snprintf(out, out_size, "%.*s", (int)(close - at - 1), at + 1);
		return close + 1;
	}
	/* a number */
	{
		const char *close = at;

		while (close < end && ((*close >= '0' && *close <= '9') || *close == '-')) {
			close++;
		}
		snprintf(out, out_size, "%.*s", (int)(close - at), at);
		return close;
	}
}

static void
servreg_load(void)
{
	static const char *const files[] = { "modemr.jsn", "adspr.jsn", "adsps.jsn", "adspua.jsn", "battmgr.jsn", "cdspr.jsn", "wpssr.jsn" };

	servreg_maps = 0;
	for (unsigned f = 0; f < sizeof(files) / sizeof(files[0]); f++) {
		char name[64], soc[24], domain[24], subdomain[24], instance[16], provider[24], service[24], full[48];
		uint64_t size = 0;
		const uint8_t *data;
		const char *text, *end, *at;

		snprintf(name, sizeof(name), "image/%s", files[f]);
		data = root_file(name, &size);
		if (data == NULL) {
			continue;
		}
		text = (const char *)data;
		end = text + size;
		if (jsn_string(text, end, "soc", soc, sizeof(soc)) == NULL || jsn_string(text, end, "domain", domain, sizeof(domain)) == NULL ||
		    jsn_string(text, end, "subdomain", subdomain, sizeof(subdomain)) == NULL ||
		    jsn_string(text, end, "qmi_instance_id", instance, sizeof(instance)) == NULL) {
			uint8_t *gone = __DECONST(uint8_t *, data);

			printf("mpss: servreg: fw/%s: not of the shape this reads\n", name);
			kfree_data(gone, size);
			continue;
		}
		snprintf(full, sizeof(full), "%s/%s/%s", soc, domain, subdomain);
		at = text;
		while (servreg_maps < sizeof(servreg_map) / sizeof(servreg_map[0]) &&
		    (at = jsn_string(at, end, "provider", provider, sizeof(provider))) != NULL &&
		    jsn_string(at, end, "service", service, sizeof(service)) != NULL) {
			snprintf(servreg_map[servreg_maps].service, sizeof(servreg_map[0].service), "%s/%s", provider, service);
			strlcpy(servreg_map[servreg_maps].domain, full, sizeof(servreg_map[0].domain));
			servreg_map[servreg_maps].instance = (uint32_t)strtoul(instance, NULL, 10);
			servreg_maps++;
		}
		{
			uint8_t *gone = __DECONST(uint8_t *, data);

			kfree_data(gone, size);
		}
	}
	printf("mpss: %u: the service registry: %u services of %u files\n", step, servreg_maps, (unsigned)(sizeof(files) / sizeof(files[0])));
	for (unsigned i = 0; i < servreg_maps; i++) {
		printf("mpss: %u:   %s at %s, instance %u\n", step, servreg_map[i].service, servreg_map[i].domain, servreg_map[i].instance);
	}
}

/* A question to the locator, answered. From the link's thread. */
static void
servreg_receive(__unused void *context, uint32_t node, uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data, *value;
	uint8_t answer[512];
	uint16_t message, size = 0;
	unsigned at = 7 + 7, found = 0;
	char wanted[64] = "";

	if (length < 7 || m[0] != QMI_REQUEST) {
		return;
	}
	message = (uint16_t)(m[3] | m[4] << 8);
	servreg_requests++;
	bzero(answer, sizeof(answer));
	answer[0] = QMI_RESPONSE;
	answer[1] = m[1];
	answer[2] = m[2];
	answer[3] = m[3];
	answer[4] = m[4];
	answer[7] = 2;                          /* result: success */
	answer[8] = 4;
	if (message == SERVREG_GET_DOMAIN_LIST) {
		value = qmi_tlv(m, length, 1, &size);
		if (value != NULL) {
			snprintf(wanted, sizeof(wanted), "%.*s", (int)(size < 63 ? size : 63), (const char *)value);
		}
		for (unsigned i = 0; i < servreg_maps; i++) {
			found += strcmp(servreg_map[i].service, wanted) == 0;
		}
		/* TLV 0x10: how many domains in all; 0x11: the table's revision; 0x12: the list, each a string with a
		 * byte of length, the instance (4 bytes), and the service data (a byte of validity, 4 bytes) */
		answer[at++] = 0x10;
		answer[at++] = 2;
		answer[at++] = 0;
		answer[at++] = (uint8_t)found;
		answer[at++] = (uint8_t)(found >> 8);
		answer[at++] = 0x11;
		answer[at++] = 2;
		answer[at++] = 0;
		answer[at++] = 1;
		answer[at++] = 0;
		if (found != 0) {
			unsigned list = at;

			answer[at++] = 0x12;
			at += 2;                        /* the length, filled in below */
			answer[at++] = (uint8_t)found;
			for (unsigned i = 0; i < servreg_maps && at + 64 < sizeof(answer); i++) {
				if (strcmp(servreg_map[i].service, wanted) != 0) {
					continue;
				}
				answer[at++] = (uint8_t)strlen(servreg_map[i].domain);
				memcpy(answer + at, servreg_map[i].domain, strlen(servreg_map[i].domain));
				at += (unsigned)strlen(servreg_map[i].domain);
				for (unsigned b = 0; b < 4; b++) {
					answer[at++] = (uint8_t)(servreg_map[i].instance >> (8 * b));
				}
				answer[at++] = 0;       /* no service data */
				at += 4;
			}
			answer[list + 1] = (uint8_t)(at - list - 3);
			answer[list + 2] = (uint8_t)((at - list - 3) >> 8);
		}
		printf("mpss: servreg: the modem asks where \"%s\" is: %u domain%s\n", wanted, found, found == 1 ? "" : "s");
	} else if (message == SERVREG_PFR) {
		value = qmi_tlv(m, length, 1, &size);
		printf("mpss: servreg: the modem reports a failure of \"%.*s\"\n", value ? (int)(size < 63 ? size : 63) : 0,
		    value ? (const char *)value : "");
	} else {
		printf("mpss: servreg: a request this does not know (%#x)\n", message);
		answer[10] = QMI_RESULT_FAILURE;
		answer[12] = QMI_ERR_MALFORMED;
	}
	answer[5] = (uint8_t)(at - 7);
	answer[6] = (uint8_t)((at - 7) >> 8);
	qrtr_send(servreg_port, node, port, answer, at);
}

/* ---- the IPA's QMI handshake with the modem (Linux's ipa_qmi.c) ------------------------------------- */

/*
 * The modem's IPA driver and the application processor's agree on the IPA's shared memory over
 * QMI before the modem uses the IPA: the application processor hosts service 0x31 instance 1 (the
 * modem registers for an indication there, 0x20, and says when its driver is up, 0x35), and asks
 * the modem's service 0x31 instance 2 to initialize its driver (0x21) with where the tables are in
 * the IPA's local memory -- the layout of IPA v4.11 (Linux: ipa_data-v4.11.c) --, then tells it
 * that all is complete (indication 0x22). Without this the modem's LTE downlink task dies on a
 * null pointer the moment the modem goes online (23 October: enl2_dl, three of three).
 */
#define IPA_QMI_SERVICE         0x31
#define IPA_HOST_INSTANCE       0x101           /* version 1, instance 1: ours */
#define IPA_MODEM_INSTANCE      0x201           /* version 1, instance 2: the modem's */
#define IPA_QMI_INDICATION_REGISTER 0x20
#define IPA_QMI_INIT_DRIVER     0x21
#define IPA_QMI_INIT_COMPLETE   0x22
#define IPA_QMI_INSTALL_FILTER_RULE 0x23
#define IPA_QMI_FILTER_INSTALLED_NOTIF 0x24
#define IPA_QMI_CONFIG          0x27
#define IPA_QMI_DRIVER_INIT_COMPLETE 0x35
#define QMI_INDICATION          4

static struct hakios_qrtr_port *ipa_client_port;
static uint32_t ipa_modem_node, ipa_modem_port;         /* the modem's IPA service; 0: not seen */
static uint32_t ipa_register_node, ipa_register_port;   /* who asked for the indication */
static bool ipa_indication_wanted, ipa_indication_sent, ipa_driver_ready, ipa_uc_ready;
static uint16_t ipa_transaction;

static unsigned
qmi_put_u32(uint8_t *at, uint8_t type, uint32_t value)
{
	at[0] = type;
	at[1] = 4;
	at[2] = 0;
	for (unsigned b = 0; b < 4; b++) {
		at[3 + b] = (uint8_t)(value >> (8 * b));
	}
	return 7;
}

static unsigned
qmi_put_pair(uint8_t *at, uint8_t type, uint32_t first, uint32_t second)
{
	at[0] = type;
	at[1] = 8;
	at[2] = 0;
	for (unsigned b = 0; b < 4; b++) {
		at[3 + b] = (uint8_t)(first >> (8 * b));
		at[7 + b] = (uint8_t)(second >> (8 * b));
	}
	return 11;
}

/* A QMI message with only the result TLV: an answer (flags 2, the transaction echoed) or an indication (flags 4). */
static void
ipa_qmi_result(struct hakios_qrtr_port *from, uint32_t node, uint32_t port, uint8_t flags, uint16_t transaction, uint16_t message)
{
	uint8_t packet[14] = { flags, (uint8_t)transaction, (uint8_t)(transaction >> 8), (uint8_t)message, (uint8_t)(message >> 8), 7, 0,
			       2, 4, 0, 0, 0, 0, 0 };

	qrtr_send(from, node, port, packet, sizeof(packet));
}

static void
ipa_init_complete(void)
{
	if (ipa_indication_wanted && !ipa_indication_sent && ipa_driver_ready) {
		ipa_qmi_result(ipa_host_port, ipa_register_node, ipa_register_port, QMI_INDICATION, 0, IPA_QMI_INIT_COMPLETE);
		ipa_indication_sent = true;
		tell("mpss: ipa: the modem is told that the IPA is all set up (indication %#x)\n", IPA_QMI_INIT_COMPLETE);
	}
}

/* The modem's requests to our IPA service. From the link's thread. */
static void
ipa_host_receive(__unused void *context, uint32_t node, uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint16_t message, transaction;

	if (length < 7 || m[0] != QMI_REQUEST) {
		return;
	}
	transaction = (uint16_t)(m[1] | m[2] << 8);
	message = (uint16_t)(m[3] | m[4] << 8);
	switch (message) {
	case IPA_QMI_INDICATION_REGISTER:
		ipa_qmi_result(ipa_host_port, node, port, QMI_RESPONSE, transaction, message);
		ipa_register_node = node;
		ipa_register_port = port;
		ipa_indication_wanted = true;
		printf("mpss: ipa: the modem registers for the IPA's indication (node %u port %u)\n", node, port);
		ipa_init_complete();
		break;
	case IPA_QMI_DRIVER_INIT_COMPLETE:
		ipa_qmi_result(ipa_host_port, node, port, QMI_RESPONSE, transaction, message);
		ipa_uc_ready = true;
		printf("mpss: ipa: the modem says its IPA driver is up\n");
		ipa_init_complete();
		break;
	case IPA_QMI_INSTALL_FILTER_RULE: {
		/*
		 * The modem's uplink filter rules, to be installed in the IPA by the application processor. Android
		 * answers with the rules' ids echoed (TLV 0x11: a count and the ids), which is what is done here; the
		 * rules themselves go nowhere yet. The rules come in TLV 0x15 as a count and that many structures of
		 * 173 bytes each on the wire (ipa_filter_spec_ex_type_v01 as ipa_qmi_service_v01.c encodes it: the
		 * arrays inside are of fixed length), the id at byte 168 of each (byte 163 in cycle 28 made the modem assert on the ids).
		 */
		uint16_t size = 0;
		const uint8_t *list = qmi_tlv(m, length, 0x15, &size);
		uint8_t answer[7 + 7 + 3 + 1 + 4 * 64];
		unsigned at = 7, count = list != NULL && size >= 1 ? list[0] : 0;

		if (list == NULL || size != 1 + count * 173 || count > 64) {
			ipa_qmi_result(ipa_host_port, node, port, QMI_RESPONSE, transaction, message);
			printf("mpss: ipa: the modem's filter rules (%u, %u bytes) are not of the shape this knows: answered with success alone\n",
			    count, size);
			break;
		}
		answer[0] = QMI_RESPONSE;
		answer[1] = (uint8_t)transaction;
		answer[2] = (uint8_t)(transaction >> 8);
		answer[3] = (uint8_t)message;
		answer[4] = (uint8_t)(message >> 8);
		answer[at++] = 2;                       /* the result: success */
		answer[at++] = 4;
		answer[at++] = 0;
		answer[at++] = 0;
		answer[at++] = 0;
		answer[at++] = 0;
		answer[at++] = 0;
		answer[at++] = 0x11;                    /* the ids */
		answer[at++] = (uint8_t)(1 + 4 * count);
		answer[at++] = (uint8_t)((1 + 4 * count) >> 8);
		answer[at++] = (uint8_t)count;
		for (unsigned r = 0; r < count; r++) {
			memcpy(answer + at, list + 1 + r * 173 + 168, 4);   /* ip_type 4, filter_rule 150, action 4, valid 1, route index 4, valid 1, mux id 4 */
			at += 4;
		}
		answer[5] = (uint8_t)(at - 7);
		answer[6] = (uint8_t)((at - 7) >> 8);
		qrtr_send(ipa_host_port, node, port, answer, at);
		printf("mpss: ipa: the modem's %u filter rules taken (not installed), their ids answered\n", count);
		break;
	}
	case IPA_QMI_CONFIG:
	case IPA_QMI_FILTER_INSTALLED_NOTIF:
	default:
		/*
		 * After the handshake the modem asks for the IPA's configuration (0x27), has its uplink filter rules
		 * installed (0x23) and tells that they are (0x24) -- Android's driver answers each with success and
		 * little else. Everything it asks is answered so here (the filter rules are not programmed: there is
		 * no data path yet).
		 */
		ipa_qmi_result(ipa_host_port, node, port, QMI_RESPONSE, transaction, message);
		printf("mpss: ipa: the modem's request %#x (%s) answered with success\n", message,
		    message == IPA_QMI_CONFIG ? "configuration" : message == IPA_QMI_INSTALL_FILTER_RULE ? "install filter rules" :
		    message == IPA_QMI_FILTER_INSTALLED_NOTIF ? "filters installed" : "not known here");
		break;
	}
}

/* The modem's answer to our INIT_DRIVER. From the link's thread. */
static void
ipa_client_receive(__unused void *context, uint32_t node, uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint16_t size = 0;
	const uint8_t *value;

	/* every datagram on this port is shown: on 24 October the modem's answer was waited for in vain, if it came at all */
	printf("mpss: ipa: from node %u port %u, %u bytes:", node, port, (unsigned)length);
	for (size_t i = 0; i < length && i < 24; i++) {
		printf(" %02x", m[i]);
	}
	printf("%s\n", length > 24 ? " ..." : "");
	if (length < 7 || m[0] != QMI_RESPONSE || node != ipa_modem_node || port != ipa_modem_port) {
		return;
	}
	value = qmi_tlv(m, length, 2, &size);
	if ((m[3] | m[4] << 8) == IPA_QMI_INIT_DRIVER && value != NULL && size == 4) {
		uint16_t result = (uint16_t)(value[0] | value[1] << 8), error = (uint16_t)(value[2] | value[3] << 8);

		tell("mpss: ipa: the modem %s our INIT_DRIVER (result %u, error %u)\n", result == 0 ? "ACCEPTS" : "REFUSES", result, error);
		ipa_driver_ready = result == 0;
		ipa_init_complete();
	}
}

/* The IPA's local memory as IPA v4.11 lays it out (Linux: ipa_data-v4.11.c), told to the modem. */
static void
ipa_init_driver(void)
{
	uint8_t packet[200];
	unsigned at = 7;
	uint32_t routes = 8;                    /* the route tables' entries the modem may use: indices 0..7 of 15 (Android's driver:
	                                         * v4_modem_rt_index_hi 7; its log "num_indices 7"); modem_ipa_routes=N */
	const uint32_t o = ipa_mem_offset;

	PE_parse_boot_argn("modem_ipa_routes", &routes, sizeof(routes));

	ipa_transaction = (uint16_t)(ipa_transaction + 1 == 0 ? 1 : ipa_transaction + 1);
	packet[0] = QMI_REQUEST;
	packet[1] = (uint8_t)ipa_transaction;
	packet[2] = (uint8_t)(ipa_transaction >> 8);
	packet[3] = IPA_QMI_INIT_DRIVER;
	packet[4] = 0;
	at += qmi_put_u32(packet + at, 0x10, 3);                                /* platform: 3 = MSM Android (Android's log: "platform_type 3"; 1 is a data card) */
	at += qmi_put_pair(packet + at, 0x11, o + 0x0688, o + 0x0688 + 0x0240 - 1); /* the modem's headers */
	at += qmi_put_pair(packet + at, 0x12, o + 0x0508, routes - 1);          /* IPv4 routes */
	at += qmi_put_pair(packet + at, 0x13, o + 0x0608, routes - 1);          /* IPv6 routes */
	at += qmi_put_u32(packet + at, 0x14, o + 0x0308);                       /* IPv4 filters */
	at += qmi_put_u32(packet + at, 0x15, o + 0x0408);                       /* IPv6 filters */
	at += qmi_put_pair(packet + at, 0x16, o + 0x27d8, 0x800);               /* the modem's own region (Android: modem_ofst 0x27d8, 0x800) */
	at += qmi_put_u32(packet + at, 0x17, 16);                               /* our endpoint for its control messages (AP_MODEM_RX) */
	{
		/*
		 * TLV 0x18 is "is_ssr_bootup" to Android's driver (skip_uc_load to Linux's): whether the modem is
		 * being restarted with the IPA's microcontroller already loaded. Android leaves it out on the first
		 * boot (its log: "is_ssr_bootup 0", not sent), and the modem then loads the microcontroller itself
		 * (its "booted" events follow the answer). Sent as 1 with modem_ipa_skip_uc=1 only.
		 */
		uint32_t skip = 0;

		PE_parse_boot_argn("modem_ipa_skip_uc", &skip, sizeof(skip));
		if (skip != 0) {
			packet[at++] = 0x18;
			packet[at++] = 1;
			packet[at++] = 0;
			packet[at++] = 1;
		}
	}
	at += qmi_put_pair(packet + at, 0x19, o + 0x0ad0, o + 0x0ad0 + 0x0ac0 - 1); /* the modem's processing contexts (Android: 0xad0, 0xac0) */
	at += qmi_put_pair(packet + at, 0x1b, o + 0x0488, routes - 1);          /* hashed IPv4 routes */
	at += qmi_put_pair(packet + at, 0x1c, o + 0x0588, routes - 1);          /* hashed IPv6 routes */
	at += qmi_put_u32(packet + at, 0x1d, o + 0x0288);                       /* hashed IPv4 filters */
	at += qmi_put_u32(packet + at, 0x1e, o + 0x0388);                       /* hashed IPv6 filters */
	at += qmi_put_u32(packet + at, 0x1f, o + 0x1d00);                       /* the modem's quota statistics (Android: stats_quota_q6 0x1d00, 0x30) */
	at += qmi_put_u32(packet + at, 0x20, 0x30);
	at += qmi_put_u32(packet + at, 0x21, o + 0x27b0);                       /* drop statistics (Android: stats_drop 0x27b0, 0x20) */
	at += qmi_put_u32(packet + at, 0x22, 0x20);
	packet[5] = (uint8_t)(at - 7);
	packet[6] = (uint8_t)((at - 7) >> 8);
	printf("mpss: ipa: about to ask the modem's IPA driver (node %u port %u) to initialize with the IPA's memory layout (%u bytes, "
	    "offset %#x)\n", ipa_modem_node, ipa_modem_port, at, o);
	if (qrtr_send(ipa_client_port, ipa_modem_node, ipa_modem_port, packet, at) != 0) {
		printf("mpss: ipa: INIT_DRIVER not sent\n");
	}
	if (!option("modem_ipa_no_probe")) {
		/*
		 * A probe: a request of a message the service has no handler for. A QMI service answers it with an
		 * error (invalid message id) at once -- or, if the modem's IPA driver is stuck before its service's
		 * loop, not at all. Whether anything comes back tells which (24 October: INIT_DRIVER got no answer).
		 */
		uint8_t probe[7] = { QMI_REQUEST, 0x7e, 0x00, 0xff, 0x7f, 0, 0 };

		if (qrtr_send(ipa_client_port, ipa_modem_node, ipa_modem_port, probe, sizeof(probe)) == 0) {
			printf("mpss: ipa: a probe sent to the modem's IPA service too (message 0x7fff, transaction 0x7e): an error back means it listens\n");
		}
	}
}

static void
ipa_client_server(__unused void *context, bool arrived, uint32_t service, __unused uint32_t version, uint32_t instance, uint32_t node,
    uint32_t port)
{
	if (service != IPA_QMI_SERVICE || instance != 2) {
		return;
	}
	if (arrived) {
		ipa_modem_node = node;
		ipa_modem_port = port;
		ipa_init_driver();
	} else {
		ipa_modem_node = ipa_modem_port = 0;
		ipa_driver_ready = ipa_indication_sent = ipa_indication_wanted = false;
	}
}

static bool
ipa_qmi_start(void)
{
	if (option("modem_no_ipa_qmi")) {
		return false;
	}
	if (ipa_host_port == NULL) {
		ipa_host_port = qrtr_open(ipa_host_receive, NULL, NULL);
	}
	if (ipa_client_port == NULL) {
		ipa_client_port = qrtr_open(ipa_client_receive, ipa_client_server, NULL);
		if (ipa_client_port != NULL) {
			qrtr_lookup(ipa_client_port, IPA_QMI_SERVICE, 1, 2);
		}
	}
	ipa_driver_ready = ipa_indication_sent = ipa_indication_wanted = ipa_uc_ready = false;
	return ipa_host_port != NULL && ipa_client_port != NULL;
}

static void
ipa_qmi_stop(void)
{
	if (ipa_host_port != NULL) {
		qrtr_close(ipa_host_port);
		ipa_host_port = NULL;
	}
	if (ipa_client_port != NULL) {
		qrtr_close(ipa_client_port);
		ipa_client_port = NULL;
	}
	ipa_modem_node = ipa_modem_port = 0;
}

/* ---- step 8: GLINK's queues, looked at -------------------------------------------------------------- */

static bool
step_queues(void)
{
	struct glink_message message;
	uint32_t head, tail;

	announce("find GLINK's descriptor and the two queues, and read what the modem has put into its own");
	if (!glink_find()) {
		printf("mpss: %u: GLINK's items are not all there, or not of a shape this knows (descriptor %#x, ours %#x of %u bytes, "
		    "the DSP's %#x of %u)\n", step, glink.descriptor, glink.ours, glink.ours_size, glink.theirs, glink.theirs_size);
		return false;
	}
	head = smem_word(glink.descriptor + GLINK_THEIR_HEAD);
	tail = smem_word(glink.descriptor + GLINK_THEIR_TAIL);
	printf("mpss: %u: our queue: %u bytes at %#lx, written to %#x, read by the DSP to %#x; the DSP's: %u bytes at %#lx, written "
	    "to %#x, read by us to %#x\n", step, glink.ours_size, smem_phys + glink.ours, smem_word(glink.descriptor + GLINK_OUR_HEAD),
	    smem_word(glink.descriptor + GLINK_OUR_TAIL), glink.theirs_size, smem_phys + glink.theirs, head, tail);
	for (uint32_t at = 0, shown = 0; at + sizeof(message) <= head && head < glink.theirs_size && tail == 0 && shown < 8; shown++) {
		char name[33] = "";
		uint32_t length = sizeof(message);

		glink_peek(&message, at, sizeof(message));
		if (message.command == GLINK_OPEN) {
			glink_peek(name, at + 8, message.param2 < 32 ? message.param2 : 32);
			length += message.param2;
		}
		printf("mpss: %u:   in the DSP's queue at %#x: %s %u %#x %s\n", step, at, glink_name(message.command), message.param1,
		    message.param2, name);
		if (message.command != GLINK_VERSION && message.command != GLINK_OPEN) {
			break;
		}
		at += (length + 7) & ~7u;
	}
	return true;
}

/* ---- step 9: GLINK, spoken -------------------------------------------------------------------------- */

static bool
step_link(void)
{
	thread_t thread;
	uint64_t began;

	if (!dsp_started || glink.descriptor == 0) {
		return false;
	}
	announce("take up GLINK with the modem: our version, then whatever its messages call for; a thread of ours listens "
	    "from here on, every millisecond");
	snprintf(last_write, sizeof(last_write), "step %u took up GLINK", step);
	if (!rmtfs_start()) {
		printf("mpss: %u: no port for the modem's file systems\n", step);
	}
	lck_mtx_lock(&glink_lock);
	glink.features = GLINK_INTENT_REUSE;
	glink.agreed = glink.broken = false;
	glink.said = glink.heard = 0;
	for (unsigned c = 0; c < CHANNELS; c++) {
		struct glink_channel *channel = &glink.channel[c];

		channel->asked = channel->acknowledged = channel->open = false;
		channel->theirs = channel->our_intent_count = 0;
		bzero(channel->their_intents, sizeof(channel->their_intents));
	}
	bzero(glink.others, sizeof(glink.others));
	glink.trace = 60;
	glink.up = true;
	qrtr_hellos = qrtr_in = qrtr_out = qrtr_lookups = 0;
	doorbell.since = began = now_us();
	glink_say(GLINK_VERSION, GLINK_VERSION_1, glink.features, NULL, 0);
	lck_mtx_unlock(&glink_lock);
	link_wanted = link_runs = true;
	if (kernel_thread_start(link_thread, NULL, &thread) != KERN_SUCCESS) {
		link_wanted = link_runs = false;
		printf("mpss: %u: no thread\n", step);
		return false;
	}
	link_thread_is = thread;
	thread_deallocate(thread);
	for (unsigned ms = 0; ms < 5000 && !glink.channel[CHANNEL_QRTR].open && !glink.broken; ms++) {
		sleep_ms(1);
	}
	sleep_ms(50);
	printf("mpss: %u: %llu ms after our version: versions %s (features %#x); the channel \"IPCRTR\" is %s; the modem also "
	    "opened \"%s\" \"%s\" \"%s\" \"%s\"\n", step, (now_us() - began) / 1000, glink.agreed ? "agreed" : "NOT AGREED", glink.features,
	    glink.channel[CHANNEL_QRTR].open ? "OPEN" : "NOT OPEN", glink.others[0], glink.others[1], glink.others[2], glink.others[3]);
	return glink.channel[CHANNEL_QRTR].open;
}

/* ---- step 10: QRTR ---------------------------------------------------------------------------------- */

static bool
step_qrtr(void)
{
	unsigned services = 0;

	if (!glink.channel[CHANNEL_QRTR].open) {
		return false;
	}
	PE_parse_boot_argn("modem_listen", &listen_seconds, sizeof(listen_seconds));
	announce("listen to QRTR on the channel: the DSP's hello (answered from the link's thread), and the services it tells of");
	for (unsigned ms = 0; ms < listen_seconds * 1000; ms++) {
		sleep_ms(1);
	}
	for (unsigned i = 0; i < sizeof(qrtr_servers) / sizeof(qrtr_servers[0]); i++) {
		services += qrtr_servers[i].there;
	}
	printf("mpss: %u: after %u s: %s; the DSP is node %u and has told of %u services; it looked for %u of ours; QRTR received %u "
	    "and sent %u datagrams\n", step, listen_seconds, qrtr_hellos ? "HELLO BOTH WAYS" : "NO HELLO FROM THE DSP", qrtr_peer,
	    services, qrtr_lookups, qrtr_in, qrtr_out);
	return qrtr_hellos != 0;
}

/* ---- step 11: the modem's services: online, and what the network says ------------------------------ */

/*
 * QMI, by hand: a message is a byte of flags (0 a request, 2 an answer, 4 a notice), a transaction
 * of two bytes, the message's number and the length of what follows, then TLVs (a type, a length
 * of two bytes, the value). The device management service (DMS, 2) is asked to put the modem
 * online -- message 0x2e, TLV 1 = 0 --: it starts in low power and stays there until somebody
 * does, which on Android is the RIL's first act. The network access service (NAS, 3) is then
 * asked, by hakios.nas.c, for the serving system (0x24: registered or not) and the signal (0x4f:
 * dBm, per radio); both are answers to questions, asked every second here and every two seconds
 * afterwards (modem=1), since no indication is asked for. What comes back is the kernel's to show:
 * sysctl kern.hakios.cellular_up, _registered, _bars, _dbm, _radio.
 */
#define QMI_DMS                 2
#define QMI_NAS                 3
#define DMS_SET_OPERATING_MODE  0x002e
#define DMS_GET_OPERATING_MODE  0x002d
#define DMS_ONLINE              0

static struct hakios_nas_client nas;
static bool nas_opened;
static struct hakios_qrtr_port *dms_port;
static uint32_t dms_node, dms_port_number;      /* where DMS is; 0: not seen */
static volatile uint32_t dms_answers;
static volatile uint32_t dms_result, dms_error;
static uint16_t dms_transaction;
static int cellular_up, cellular_registered, cellular_bars, cellular_dbm, cellular_radio;
static char cellular_operator[40];               /* "MCC MNC name" of the serving network; "" when none */
static char cellular_ip[20], cellular_gateway[20], cellular_dns[44];  /* the data session's addresses; "" until it is up */
static int cellular_data, cellular_mtu;         /* 1 once the data session is up */
static uint32_t nas_polls, nas_changes;

SYSCTL_DECL(_kern_hakios);
SYSCTL_INT(_kern_hakios, OID_AUTO, cellular_up, CTLFLAG_RD | CTLFLAG_LOCKED, &cellular_up, 0,
    "the modem runs and its network access service answers");
SYSCTL_INT(_kern_hakios, OID_AUTO, cellular_registered, CTLFLAG_RD | CTLFLAG_LOCKED, &cellular_registered, 0,
    "registered with a network");
SYSCTL_INT(_kern_hakios, OID_AUTO, cellular_bars, CTLFLAG_RD | CTLFLAG_LOCKED, &cellular_bars, 0, "signal, 0 to 4 bars");
SYSCTL_STRING(_kern_hakios, OID_AUTO, cellular_operator, CTLFLAG_RD | CTLFLAG_LOCKED, cellular_operator, 0,
    "the serving network: its name as the modem gives it; empty when none");
SYSCTL_INT(_kern_hakios, OID_AUTO, cellular_data, CTLFLAG_RD | CTLFLAG_LOCKED, &cellular_data, 0, "1: the data session is up");
SYSCTL_INT(_kern_hakios, OID_AUTO, cellular_mtu, CTLFLAG_RD | CTLFLAG_LOCKED, &cellular_mtu, 0, "the data session's MTU");
SYSCTL_STRING(_kern_hakios, OID_AUTO, cellular_ip, CTLFLAG_RD | CTLFLAG_LOCKED, cellular_ip, 0, "the data session's IPv4 address");
SYSCTL_STRING(_kern_hakios, OID_AUTO, cellular_gateway, CTLFLAG_RD | CTLFLAG_LOCKED, cellular_gateway, 0, "its gateway");
SYSCTL_STRING(_kern_hakios, OID_AUTO, cellular_dns, CTLFLAG_RD | CTLFLAG_LOCKED, cellular_dns, 0, "its DNS servers, space-separated");
SYSCTL_INT(_kern_hakios, OID_AUTO, cellular_dbm, CTLFLAG_RD | CTLFLAG_LOCKED, &cellular_dbm, 0, "signal, dBm; 0: none read");
SYSCTL_INT(_kern_hakios, OID_AUTO, cellular_radio, CTLFLAG_RD | CTLFLAG_LOCKED, &cellular_radio, 0,
    "the radio (QMI NAS): 4 GSM, 5 UMTS, 8 LTE, 12 NR");

static const char *
radio_name(uint8_t radio)
{
	return radio == 4 ? "GSM" : radio == 5 ? "UMTS" : radio == 8 ? "LTE" : radio == 12 ? "NR" : "?";
}

static void wds_advance(void);
static volatile bool wds_registered;      /* NAS says the modem is registered (set by nas_changed) */
static void probe_dump(const char *who, const uint8_t *m, size_t length);

/* From the link's thread, when what NAS says has changed. */
static void
nas_changed(__unused void *context, const struct hakios_nas_state *state)
{
	nas_changes++;
	cellular_registered = state->registered;
	snprintf(cellular_operator, sizeof(cellular_operator), "%s", state->registered ? state->operator_name : "");
	cellular_bars = state->bars;
	cellular_dbm = state->has_signal ? state->dbm : 0;
	cellular_radio = state->has_signal ? state->radio_interface : 0;
	wds_registered = state->registered;
	wds_advance();          /* bind once, start the session once registered, retry if refused */
	tell("mpss: nas: %s%s%s (%u-%u); signal %s%d dBm (%s): %u bars\n", state->registered ? "REGISTERED WITH A NETWORK" : "not registered",
	    state->operator_name[0] ? ", network " : "", state->operator_name, state->mcc, state->mnc,
	    state->has_signal ? "" : "none read, ", state->has_signal ? state->dbm : 0, radio_name(state->radio_interface), state->bars);
}

/* A datagram for our DMS port: the answer to our one question. From the link's thread. */
static void
dms_receive(__unused void *context, __unused uint32_t node, __unused uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint32_t result = ~0u, error = ~0u;

	if (length < 7 || m[0] != 2) {
		return;
	}
	for (size_t at = 7; at + 3 <= length;) {
		uint16_t size = (uint16_t)(m[at + 1] | m[at + 2] << 8);

		if (m[at] == 2 && size == 4 && at + 7 <= length) {
			result = (uint32_t)(m[at + 3] | m[at + 4] << 8);
			error = (uint32_t)(m[at + 5] | m[at + 6] << 8);
		}
		at += 3 + size;
	}
	printf("mpss: dms: < message %#x, transaction %u: result %u, error %u\n", m[3] | m[4] << 8, m[1] | m[2] << 8, result, error);
	if ((m[3] | m[4] << 8) == DMS_GET_OPERATING_MODE) {
		/* the mode (TLV 1: 0 online, 1 low power, 3 offline, 6 persistent low power, 7 mode-only low power), why offline (TLV 0x10,
		 * bits: host image, PRI image, PRI version, memory full), and whether the hardware restricts it (TLV 0x11) */
		static const char *const modes[] = { "ONLINE", "LOW POWER", "factory test", "OFFLINE", "resetting", "shutting down",
			"PERSISTENT LOW POWER", "mode-only low power", "network test" };
		uint32_t mode = ~0u, reason = ~0u, restricted = ~0u;

		for (size_t at = 7; at + 3 <= length;) {
			uint16_t size = (uint16_t)(m[at + 1] | m[at + 2] << 8);

			if (at + 3 + size > length) {
				break;
			}
			if (m[at] == 1 && size >= 1) {
				mode = m[at + 3];
			} else if (m[at] == 0x10 && size >= 2) {
				reason = (uint32_t)(m[at + 3] | m[at + 4] << 8);
			} else if (m[at] == 0x11 && size >= 1) {
				restricted = m[at + 3];
			}
			at += 3 + size;
		}
		tell("mpss: dms: THE MODEM'S OPERATING MODE IS %u (%s); offline reason %#x%s; hardware restricted %u\n", mode,
		    mode < sizeof(modes) / sizeof(modes[0]) ? modes[mode] : "?", reason, reason == ~0u ? " (not told)" : "", restricted);
	}
	dms_result = result;
	dms_error = error;
	dms_answers++;
}

static void
dms_server(__unused void *context, bool arrived, uint32_t service, __unused uint32_t version, __unused uint32_t instance,
    uint32_t node, uint32_t port)
{
	if (service != QMI_DMS) {
		return;
	}
	if (arrived) {
		dms_node = node;
		dms_port_number = port;
	} else if (node == dms_node && port == dms_port_number) {
		dms_node = dms_port_number = 0;
	}
}

/* DMS, once it is there, asked to put the modem online, and its answer waited for. */
static bool
dms_set_mode(uint32_t mode)
{
	uint8_t request[11] = { 0, 0, 0, DMS_SET_OPERATING_MODE & 0xff, DMS_SET_OPERATING_MODE >> 8, 4, 0, 1, 1, 0, (uint8_t)mode };
	uint32_t before;
	int error;

	if (dms_port == NULL || dms_port_number == 0) {
		return false;
	}
	{
		/* first, what mode the modem is in and why (the answer is said by dms_receive()) */
		uint8_t question[7] = { 0, 0, 0, DMS_GET_OPERATING_MODE & 0xff, DMS_GET_OPERATING_MODE >> 8, 0, 0 };

		dms_transaction = (uint16_t)(dms_transaction + 1 == 0 ? 1 : dms_transaction + 1);
		question[1] = (uint8_t)dms_transaction;
		question[2] = (uint8_t)(dms_transaction >> 8);
		before = dms_answers;
		if (qrtr_send(dms_port, dms_node, dms_port_number, question, sizeof(question)) == 0) {
			for (unsigned ms = 0; ms < 3000 && dms_answers == before; ms++) {
				sleep_ms(1);
			}
		}
	}
	dms_transaction = (uint16_t)(dms_transaction + 1 == 0 ? 1 : dms_transaction + 1);
	request[1] = (uint8_t)dms_transaction;
	request[2] = (uint8_t)(dms_transaction >> 8);
	before = dms_answers;
	printf("mpss: %u: about to ask DMS (node %u port %u) for operating mode %u (0 online, 1 low power): message %#x\n", step, dms_node,
	    dms_port_number, mode, DMS_SET_OPERATING_MODE);
	snprintf(last_write, sizeof(last_write), "step %u asked the modem to go online", step);
	hold();
	error = qrtr_send(dms_port, dms_node, dms_port_number, request, sizeof(request));
	if (error != 0) {
		printf("mpss: %u: NOT SENT (%d)\n", step, error);
		return false;
	}
	for (unsigned ms = 0; ms < 5000 && dms_answers == before; ms++) {
		sleep_ms(1);
	}
	if (dms_answers == before) {
		printf("mpss: %u: DMS DOES NOT ANSWER within 5 s\n", step);
		return false;
	}
	tell("mpss: %u: mode %u: %s (result %u, error %u)\n", step, mode, dms_result == 0 ? "THE MODEM AGREES" : "THE MODEM REFUSES",
	    dms_result, dms_error);
	return dms_result == 0;
}

/*
 * Online. A modem that boots offline (as this one does) does not go straight online: Qualcomm's
 * allow offline -> low power -> online (ModemManager and qmicli users meet the same refusal, error
 * 52, and go the long way), so that is tried when the short way is refused.
 */
static bool
dms_online(void)
{
	if (dms_set_mode(DMS_ONLINE)) {
		return true;
	}
	if (dms_error == 52 && dms_set_mode(1)) {
		sleep_ms(2000);
		return dms_set_mode(DMS_ONLINE);
	}
	return false;
}

/*
 * ---- the data session (QMI WDS, service 1) ----
 *
 * Once NAS says the modem is registered, a packet data session is started (Start Network
 * Interface 0x20: IPv4, the SIM's default profile -- what Android does for the default APN) and
 * its settings asked for (Get Current Settings 0x2d: the address, gateway, netmask, DNS, MTU),
 * which go to the log and to sysctl kern.hakios.cellular_ip, _gateway, _dns, _mtu, _data. The
 * packets themselves need the IPA's endpoints for the application processor, which is the next
 * step (the modem's side of the session is independent of them). modem_data=0: not started;
 * modem_apn=NAME: a session to that APN instead of the default profile.
 */
static bool keep;                       /* (defined below: the DSP and the link stay up with modem=1) */
#define QMI_WDS                 0x01
#define WDS_START_NETWORK_INTERFACE 0x0020
#define WDS_GET_CURRENT_SETTINGS 0x002d
#define WDS_BIND_MUX_DATA_PORT  0x00a2
#define WDS_SET_IP_FAMILY       0x004d
#define QMI_WDA                 0x1a
#define WDA_SET_DATA_FORMAT     0x0020
#define QMI_DPM                 0x2f            /* Data Port Mapper */
#define DPM_OPEN_PORT           0x0020
#define DATA_ENDPOINT_EMBEDDED  4               /* the IPA, "interface" 1: where the packets of this session go (as ModemManager binds it) */
#define DATA_ENDPOINT_INTERFACE 1
#define DATA_MUX                1               /* the QMAP mux id (hakios.cell.c carries mux 1) */
static struct hakios_qrtr_port *wds_port, *wda_port, *dpm_port;
static uint32_t wds_node, wds_port_number, wda_node, wda_port_number, dpm_node, dpm_port_number;
static volatile bool wds_started, wds_up, wds_bound, wda_set;
static volatile bool dpm_done, dpm_open, wda_sent, wda_done;  /* the port mapped, the data format asked and answered */
static uint64_t data_setup_began;       /* when WDS was first seen: the mapping and the format get 20 s before the bind goes ahead */
static volatile bool wds_bind_sent, wds_start_pending;
static uint64_t wds_last_try;           /* when the bind or the session was last sent, for the retry */
static uint32_t wds_handle;
static void wds_send_start(void);

/*
 * Before the session, the modem is told where its data goes and how it is framed, in the order
 * ModemManager follows for a modem behind the IPA (its port for the "ipa" network driver):
 *
 *   1. DPM Open Port (Data Port Mapper, service 0x2f, 0x20): the hardware data port -- the embedded
 *      endpoint, interface 1, and the IPA endpoints it runs on. The modem's receiving end is our
 *      transmit endpoint (2) and its transmitting end our receive endpoint (16), which is how Linux
 *      exports them (ipa_sysfs.c: modem/tx_endpoint_id, rx_endpoint_id) and MM hands them over.
 *      Without this the modem knows no embedded port: WDA answers InvalidOperation, the mux bind
 *      Internal, the session InvalidOperation -- what cycles 38 to 41 got.
 *   2. WDA Set Data Format (0x1a, 0x20) on that port: raw IP in QMAP (v1: no checksum headers, as
 *      hakios.ipa.c programs the endpoints), at most 16 packets or 14 KB per downlink aggregate.
 *   3. WDS Bind Mux Data Port, Set IP Family, Start Network Interface: wds_advance() below.
 *
 * Each step goes on when the one before has been answered, or after 20 s without an answer.
 * modem_dpm_off=1 / modem_wda_off=1: those steps left out.
 */
static void
wda_send_format(void)
{
	/* raw IP, QMAP both ways, downlink at most 16 packets or 14 KB per aggregate, on the embedded port */
	uint8_t request[7 + 7 + 7 + 7 + 7 + 7 + 11] = { 0, 0x43, 0, WDA_SET_DATA_FORMAT & 0xff, WDA_SET_DATA_FORMAT >> 8, 0, 0,
		0x11, 4, 0, 2, 0, 0, 0,
		0x12, 4, 0, 5, 0, 0, 0,
		0x13, 4, 0, 5, 0, 0, 0,
		0x15, 4, 0, 16, 0, 0, 0,
		0x16, 4, 0, 0x00, 0x38, 0, 0,
		0x17, 8, 0, DATA_ENDPOINT_EMBEDDED, 0, 0, 0, DATA_ENDPOINT_INTERFACE, 0, 0, 0 };

	if (wda_sent || wda_port == NULL || wda_port_number == 0) {
		return;
	}
	wda_sent = true;
	request[5] = sizeof(request) - 7;
	tell("mpss: wda: about to set the data format of the embedded port: raw IP in QMAP, no checksum offload\n");
	qrtr_send(wda_port, wda_node, wda_port_number, request, sizeof(request));
}

static void
wda_receive(__unused void *context, __unused uint32_t node, __unused uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint16_t size = 0;
	const uint8_t *v;
	uint32_t result = ~0u, error = ~0u, link = 0, ul = 0, dl = 0;

	if (length < 7 || m[0] != 2 || (uint16_t)(m[3] | m[4] << 8) != WDA_SET_DATA_FORMAT) {
		return;
	}
	probe_dump("wda", m, length);
	if ((v = qmi_tlv(m, length, 2, &size)) != NULL && size >= 4) {
		result = (uint32_t)(v[0] | v[1] << 8);
		error = (uint32_t)(v[2] | v[3] << 8);
	}
	if ((v = qmi_tlv(m, length, 0x11, &size)) != NULL && size >= 4) { link = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
	if ((v = qmi_tlv(m, length, 0x12, &size)) != NULL && size >= 4) { ul = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
	if ((v = qmi_tlv(m, length, 0x13, &size)) != NULL && size >= 4) { dl = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
	wda_set = result == 0;
	wda_done = true;
	tell("mpss: wda: THE DATA FORMAT %s: result %u error %u; link layer %u (2: raw IP), aggregation up %u down %u (5: QMAP)\n",
	    result == 0 ? "IS SET" : "IS REFUSED", result, error, link, ul, dl);
	wds_advance();
}

static void
wda_server(__unused void *context, bool arrived, uint32_t service, __unused uint32_t version, __unused uint32_t instance, uint32_t node,
    uint32_t port)
{
	if (service != QMI_WDA) {
		return;
	}
	if (arrived) {
		wda_node = node;
		wda_port_number = port;
		if (dpm_done || option("modem_dpm_off")) {
			wda_send_format();
		}
	} else if (node == wda_node && port == wda_port_number) {
		wda_node = wda_port_number = 0;
	}
}

static void
dpm_receive(__unused void *context, __unused uint32_t node, __unused uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint16_t size = 0;
	const uint8_t *v;
	uint32_t result = ~0u, error = ~0u;

	if (length < 7 || m[0] != 2 || (uint16_t)(m[3] | m[4] << 8) != DPM_OPEN_PORT) {
		return;
	}
	probe_dump("dpm", m, length);
	if ((v = qmi_tlv(m, length, 2, &size)) != NULL && size >= 4) {
		result = (uint32_t)(v[0] | v[1] << 8);
		error = (uint32_t)(v[2] | v[3] << 8);
	}
	dpm_open = result == 0;
	dpm_done = true;
	tell("mpss: dpm: THE EMBEDDED DATA PORT %s: result %u error %u\n", dpm_open ? "IS OPEN" : "IS NOT OPENED", result, error);
	wda_send_format();
}

static void
dpm_server(__unused void *context, bool arrived, uint32_t service, __unused uint32_t version, __unused uint32_t instance, uint32_t node,
    uint32_t port)
{
	if (service != QMI_DPM) {
		return;
	}
	if (arrived) {
		uint32_t to_modem = 2, from_modem = 16;
		/* TLV 0x11, the hardware data ports: a one-byte count, then per port the endpoint type, the interface
		 * number, the modem's receiving endpoint (ours that sends) and its transmitting one (ours that receives) */
		uint8_t request[7 + 3 + 1 + 16] = { 0, 0x45, 0, DPM_OPEN_PORT & 0xff, DPM_OPEN_PORT >> 8, 20, 0, 0x11, 17, 0, 1 };
		unsigned at = 11;

		hakios_ipa_modem_endpoints(&to_modem, &from_modem);
		dpm_node = node;
		dpm_port_number = port;
#define PUT32(x) do { request[at++] = (uint8_t)(x); request[at++] = (uint8_t)((x) >> 8); request[at++] = (uint8_t)((x) >> 16); request[at++] = (uint8_t)((x) >> 24); } while (0)
		PUT32(DATA_ENDPOINT_EMBEDDED);
		PUT32(DATA_ENDPOINT_INTERFACE);
		PUT32(to_modem);
		PUT32(from_modem);
#undef PUT32
		tell("mpss: dpm: about to open the embedded data port (interface %u, the modem receives on IPA endpoint %u and sends on %u)\n",
		    DATA_ENDPOINT_INTERFACE, to_modem, from_modem);
		qrtr_send(dpm_port, node, port, request, sizeof(request));
	} else if (node == dpm_node && port == dpm_port_number) {
		dpm_node = dpm_port_number = 0;
	}
}

static void
ip4_string(char *to, size_t size, uint32_t address)
{
	snprintf(to, size, "%u.%u.%u.%u", address >> 24, (address >> 16) & 0xff, (address >> 8) & 0xff, address & 0xff);
}

static void
wds_receive(__unused void *context, __unused uint32_t node, __unused uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint16_t size = 0, message;
	const uint8_t *v;

	if (length < 7) {
		return;
	}
	probe_dump("wds", m, length);
	message = (uint16_t)(m[3] | m[4] << 8);
	if (m[0] == 2 && message == WDS_START_NETWORK_INTERFACE) {
		uint32_t result = ~0u, error = ~0u, reason = ~0u;

		if ((v = qmi_tlv(m, length, 2, &size)) != NULL && size >= 4) {
			result = (uint32_t)(v[0] | v[1] << 8);
			error = (uint32_t)(v[2] | v[3] << 8);
		}
		if ((v = qmi_tlv(m, length, 0x10, &size)) != NULL && size >= 2) {
			reason = (uint32_t)(v[0] | v[1] << 8);
		}
		if (result == 0 && (v = qmi_tlv(m, length, 0x01, &size)) != NULL && size >= 4) {
			uint8_t ask[14] = { 0, 0x42, 0, WDS_GET_CURRENT_SETTINGS & 0xff, WDS_GET_CURRENT_SETTINGS >> 8, 7, 0, 0x10, 4, 0,
					    0x10, 0xa3, 0, 0 };      /* DNS, the address, the gateway, the MTU, the IP family */

			wds_handle = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24);
			tell("mpss: wds: THE DATA SESSION IS STARTED (handle %#x); its settings asked for\n", wds_handle);
			qrtr_send(wds_port, wds_node, wds_port_number, ask, sizeof(ask));
		} else {
			tell("mpss: wds: the data session is refused for now: result %u, error %u%s, call end reason %u (will retry)\n",
			    result, error, error == 70 ? " (invalid operation: the packet domain is not attached yet)" : "", reason);
			wds_start_pending = false;      /* retried by wds_advance() after a few seconds */
			wds_last_try = now_us();
		}
	} else if (m[0] == 2 && message == WDS_BIND_MUX_DATA_PORT) {
		uint32_t result = ~0u, error = ~0u;

		if ((v = qmi_tlv(m, length, 2, &size)) != NULL && size >= 4) {
			result = (uint32_t)(v[0] | v[1] << 8);
			error = (uint32_t)(v[2] | v[3] << 8);
		}
		wds_bound = result == 0;
		tell("mpss: wds: the client %s to mux %u of the embedded port (result %u, error %u); the session waits for registration%s\n",
		    wds_bound ? "is bound" : "IS NOT BOUND", DATA_MUX, result, error, wds_bound ? "" : " (started all the same)");
		{
			/* the client's IP family: IPv4 (WDS Set IP Family 0x4d), as MM does before starting */
			uint8_t family[11] = { 0, 0x46, 0, WDS_SET_IP_FAMILY & 0xff, WDS_SET_IP_FAMILY >> 8, 4, 0, 0x01, 1, 0, 4 };

			qrtr_send(wds_port, wds_node, wds_port_number, family, sizeof(family));
		}
		wds_last_try = now_us();        /* give the session a moment after the bind */
		wds_advance();
	} else if (m[0] == 2 && message == WDS_SET_IP_FAMILY) {
		uint32_t result = ~0u, error = ~0u;

		if ((v = qmi_tlv(m, length, 2, &size)) != NULL && size >= 4) {
			result = (uint32_t)(v[0] | v[1] << 8);
			error = (uint32_t)(v[2] | v[3] << 8);
		}
		tell("mpss: wds: the IP family (IPv4) %s (result %u, error %u)\n", result == 0 ? "is set" : "is not set", result, error);
	} else if (m[0] == 2 && message == WDS_GET_CURRENT_SETTINGS) {
		uint32_t ip = 0, gw = 0, mask = 0, dns1 = 0, dns2 = 0, mtu = 0;
		char a[20], g[20], d1[20], d2[20], mk[20];

		if ((v = qmi_tlv(m, length, 0x1e, &size)) != NULL && size >= 4) { ip = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
		if ((v = qmi_tlv(m, length, 0x20, &size)) != NULL && size >= 4) { gw = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
		if ((v = qmi_tlv(m, length, 0x21, &size)) != NULL && size >= 4) { mask = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
		if ((v = qmi_tlv(m, length, 0x15, &size)) != NULL && size >= 4) { dns1 = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
		if ((v = qmi_tlv(m, length, 0x16, &size)) != NULL && size >= 4) { dns2 = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
		if ((v = qmi_tlv(m, length, 0x29, &size)) != NULL && size >= 4) { mtu = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24); }
		ip4_string(a, sizeof(a), ip);
		ip4_string(g, sizeof(g), gw);
		ip4_string(mk, sizeof(mk), mask);
		ip4_string(d1, sizeof(d1), dns1);
		ip4_string(d2, sizeof(d2), dns2);
		if (ip != 0) {
			snprintf(cellular_ip, sizeof(cellular_ip), "%s", a);
			snprintf(cellular_gateway, sizeof(cellular_gateway), "%s", g);
			snprintf(cellular_dns, sizeof(cellular_dns), "%s%s%s", dns1 ? d1 : "", dns1 && dns2 ? " " : "", dns2 ? d2 : "");
			cellular_mtu = (int)mtu;
			cellular_data = 1;
			wds_up = true;
		}
		tell("mpss: wds: THE DATA SESSION'S SETTINGS: address %s, gateway %s, netmask %s, DNS %s %s, MTU %u\n", a, g, mk, d1, d2, mtu);
		if (ip != 0) {
			hakios_cell_session(true, ip, mask, gw, dns1, dns2, mtu);   /* en1 (hakios.cell.c) */
		}
	}
}

static void
wds_server(__unused void *context, bool arrived, uint32_t service, __unused uint32_t version, __unused uint32_t instance, uint32_t node,
    uint32_t port)
{
	if (service != QMI_WDS) {
		return;
	}
	if (arrived) {
		wds_node = node;
		wds_port_number = port;
	} else if (node == wds_node && port == wds_port_number) {
		wds_node = wds_port_number = 0;
	}
}

static void
wds_open(void)
{
	if (wds_port == NULL && !option("modem_data_off")) {
		wds_port = qrtr_open(wds_receive, wds_server, NULL);
		if (wds_port != NULL) {
			qrtr_lookup(wds_port, QMI_WDS, 0, 0);
		}
	}
	if (dpm_port == NULL && !option("modem_dpm_off") && !option("modem_data_off")) {
		dpm_port = qrtr_open(dpm_receive, dpm_server, NULL);
		if (dpm_port != NULL) {
			qrtr_lookup(dpm_port, QMI_DPM, 0, 0);
		}
	}
	if (wda_port == NULL && !option("modem_wda_off") && !option("modem_data_off")) {
		wda_port = qrtr_open(wda_receive, wda_server, NULL);
		if (wda_port != NULL) {
			qrtr_lookup(wda_port, QMI_WDA, 0, 0);
		}
	}
}

/*
 * The data session, step by step, idempotent and retry-aware: bind the client to the mux once,
 * then -- only once NAS is registered, and the packet domain has had a moment to attach -- start
 * the session, retrying every few seconds while it is refused (error 70 right after registration
 * means the packet domain is not attached yet). Called from nas_changed(), from the bind reply,
 * and every second from step_nas's loop. From the link's thread.
 */
#define WDS_RETRY_US    6000000ull
static void
wds_advance(void)
{
	uint8_t bind[7 + 11 + 4] = { 0, 0x44, 0, WDS_BIND_MUX_DATA_PORT & 0xff, WDS_BIND_MUX_DATA_PORT >> 8, 15, 0,
		0x10, 8, 0, DATA_ENDPOINT_EMBEDDED, 0, 0, 0, DATA_ENDPOINT_INTERFACE, 0, 0, 0,
		0x11, 1, 0, DATA_MUX };

	if (wds_port == NULL || wds_port_number == 0 || wds_up) {
		return;
	}
	if (data_setup_began == 0) {
		data_setup_began = now_us();
	}
	/* the port mapping and the data format first (or 20 s for them); a WDA that never came is not waited for */
	if (!wds_bind_sent && now_us() - data_setup_began < 20000000ull) {
		if (!option("modem_dpm_off") && !dpm_done) {
			return;
		}
		if (!option("modem_wda_off") && wda_port_number != 0 && !wda_done) {
			wda_send_format();      /* (if DPM answered before WDA was there) */
			return;
		}
	}
	/* bind once (unless modem_data_nobind), before or after registration */
	if (!wds_bind_sent && !option("modem_data_nobind")) {
		wds_bind_sent = true;
		wds_last_try = now_us();
		tell("mpss: wds: about to bind the client to mux %u of the embedded port (the IPA)\n", DATA_MUX);
		qrtr_send(wds_port, wds_node, wds_port_number, bind, sizeof(bind));
		return;
	}
	/* start the session only once registered, and retry while it is refused */
	if (wds_registered && !wds_start_pending && now_us() - wds_last_try >= WDS_RETRY_US) {
		wds_start_pending = true;
		wds_last_try = now_us();
		wds_send_start();
	}
}

static void
wds_send_start(void)
{
	uint8_t request[7 + 4 + 3 + 100];
	unsigned at = 7;
	char apn[64] = "";

	if (wds_port == NULL || wds_port_number == 0) {
		return;
	}
	request[0] = 0; request[1] = 0x41; request[2] = 0;
	request[3] = WDS_START_NETWORK_INTERFACE & 0xff; request[4] = WDS_START_NETWORK_INTERFACE >> 8;
	request[at++] = 0x19; request[at++] = 1; request[at++] = 0; request[at++] = 4;      /* IPv4 */
	if (PE_parse_boot_argn("modem_apn", apn, sizeof(apn)) && apn[0] != 0) {
		size_t n = strlen(apn);

		request[at++] = 0x14; request[at++] = (uint8_t)n; request[at++] = 0;
		memcpy(request + at, apn, n);
		at += n;
	}
	request[5] = (uint8_t)(at - 7); request[6] = (uint8_t)((at - 7) >> 8);
	tell("mpss: wds: about to start the data session (IPv4, %s%s)\n", apn[0] ? "APN " : "the default profile", apn);
	qrtr_send(wds_port, wds_node, wds_port_number, request, at);
}

static void
wds_close(void)
{
	if (wds_port != NULL) {
		qrtr_close(wds_port);
		wds_port = NULL;
	}
	if (wda_port != NULL) {
		qrtr_close(wda_port);
		wda_port = NULL;
	}
	if (dpm_port != NULL) {
		qrtr_close(dpm_port);
		dpm_port = NULL;
	}
	wds_node = wds_port_number = wda_node = wda_port_number = dpm_node = dpm_port_number = 0;
	dpm_done = dpm_open = wda_sent = wda_done = false;
	data_setup_began = 0;
	wds_started = wds_up = wds_bound = wda_set = false;
	wds_bind_sent = wds_start_pending = wds_registered = false;
	wds_last_try = 0;
	if (cellular_data) {
		hakios_cell_session(false, 0, 0, 0, 0, 0, 0);
	}
	cellular_data = 0;
	cellular_ip[0] = cellular_gateway[0] = cellular_dns[0] = 0;
}

/*
 * ---- the SIM and the network selection, looked at once the modem is online ----
 *
 * Online (cycle 29, 24 October) the modem reported no registration and no search: "registration 0,
 * CS 2, PS 2", no signal. What the modem thinks of the SIM (UIM, service 0xb: Get Card Status 0x2f:
 * the card's state and its applications' states) and of the network selection (NAS Get System
 * Selection Preference 0x34: mode preference, automatic or manual) is asked here, and the modem is
 * told to register automatically (NAS Initiate Network Register 0x22, action 1). Every answer is
 * shown as it comes (modem_sim_probe=0: not).
 */
#define QMI_UIM                 0x0b
#define UIM_GET_CARD_STATUS     0x002f
#define UIM_SWITCH_SLOT         0x0046
#define UIM_CHANGE_PROVISIONING_SESSION 0x0038
static volatile uint32_t sim_slot_ready;        /* the physical slot whose card is READY, found by uim_receive(); 0: none yet */
static volatile bool sim_slot_switched, sim_provisioned, sim_register_again;
#define NAS_INITIATE_NETWORK_REGISTER 0x0022
#define NAS_GET_SYSTEM_SELECTION_PREFERENCE 0x0034
#define NAS_GET_SYSTEM_INFO     0x004d
static struct hakios_qrtr_port *uim_port, *nas2_port;
static uint32_t uim_node, uim_port_number, nas2_node, nas2_port_number;

static void
probe_dump(const char *who, const uint8_t *m, size_t length)
{
	uint32_t result = ~0u, error = ~0u;

	for (size_t at = 7; at + 3 <= length;) {
		uint16_t size = (uint16_t)(m[at + 1] | m[at + 2] << 8);

		if (m[at] == 2 && size == 4 && at + 7 <= length) {
			result = (uint32_t)(m[at + 3] | m[at + 4] << 8);
			error = (uint32_t)(m[at + 5] | m[at + 6] << 8);
		}
		at += 3 + size;
	}
	printf("mpss: %s: < %s %#x, transaction %u, %u bytes: result %u, error %u;", who, m[0] == 2 ? "answer" : m[0] == 4 ? "indication" : "?",
	    m[3] | m[4] << 8, m[1] | m[2] << 8, (unsigned)length, result, error);
	for (size_t i = 0; i < length && i < 64; i++) {
		printf(" %02x", m[i]);
	}
	printf("%s\n", length > 64 ? " ..." : "");
}

static void
uim_receive(__unused void *context, __unused uint32_t node, __unused uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint16_t size = 0;
	const uint8_t *v;

	if (length < 7) {
		return;
	}
	probe_dump("uim", m, length);
	if ((m[3] | m[4] << 8) == UIM_SWITCH_SLOT) {
		tell("mpss: uim: the modem answers the slot switch\n");
		return;
	}
	if ((m[3] | m[4] << 8) == UIM_CHANGE_PROVISIONING_SESSION) {
		tell("mpss: uim: the modem answers the provisioning change: registration will be asked for again\n");
		sim_register_again = true;
		return;
	}
	if ((m[3] | m[4] << 8) != UIM_GET_CARD_STATUS || (v = qmi_tlv(m, length, 0x10, &size)) == NULL || size < 9) {
		return;
	}
	{
		/* card status: four indices of 16 bits, the number of slots, then per slot: the card's state (0 absent, 1 present, 2 error),
		 * UPIN state and retries, PUK retries, error code, the applications' count, and per application: type (1 SIM, 2 USIM, 3 RUIM,
		 * 4 CSIM, 5 ISIM), state (0 unknown, 1 detected, 2 PIN1 wanted, 3 PUK1 wanted, 4 personalization, 5 PIN1 blocked, 6 illegal,
		 * 7 READY), then 11 bytes of personalization and PIN states, and the AID */
		static const char *const card_states[] = { "ABSENT", "PRESENT", "ERROR" };
		static const char *const app_states[] = { "unknown", "detected", "PIN1 WANTED", "PUK1 wanted", "personalization", "PIN1 blocked",
			"illegal", "READY" };
		unsigned at = 8, slots = v[8], first_card = ~0u, primary_gw = (unsigned)(v[0] | v[1] << 8);
		uint8_t ready_aid[1 + 32] = { 0 };     /* the AID (a length, then the bytes) of a READY USIM in logical slot 1 */

		tell("mpss: uim: the primary GW application is slot %u application %u (0xff: none)\n", v[1], v[0]);
		at = 9;
		for (unsigned slot = 0; slot < slots && at + 6 <= size; slot++) {
			unsigned card = v[at], apps = v[at + 5];

			tell("mpss: uim: SLOT %u: THE CARD IS %s (%u), UPIN state %u, error %u, %u applications\n", slot + 1,
			    card < 3 ? card_states[card] : "?", card, v[at + 1], v[at + 4], apps);
			if (slot == 0) {
				first_card = card;
			}
			at += 6;
			for (unsigned a = 0; a < apps && at + 7 <= size; a++) {
				/* libqmi's "UIM Card Status": type, state, personalization state, feature, retries, unblock retries, the AID
				 * (a length and the bytes), UPIN replaces PIN1, PIN1 state, PIN1 retries, PUK1 retries, PIN2 state, retries, PUK2 */
				unsigned type = v[at], state = v[at + 1], aid = v[at + 6], pin = at + 7 + aid;

				tell("mpss: uim:   application %u: type %u, state %u (%s), PIN1 state %u (0 unknown, 1 enabled not verified, 2 enabled verified, "
				    "3 disabled, 4 blocked, 5 permanently blocked), retries %u\n", a, type, state, state < 8 ? app_states[state] : "?",
				    pin + 1 < size ? v[pin + 1] : 0, pin + 2 < size ? v[pin + 2] : 0);
				if (card == 1 && state == 7 && sim_slot_ready == 0) {
					sim_slot_ready = slot + 1;
				}
				/* a USIM (or SIM) in logical slot 1 that is READY, or only "detected" (cycle 37: the card sits in slot 1 with its
				 * application detected and never initialized, as no provisioning session names it): either is provisioned below */
				if (slot == 0 && card == 1 && (state == 7 || state == 1) && (type == 2 || type == 1) && aid <= 32 && at + 7 + aid <= size) {
					ready_aid[0] = (uint8_t)aid;
					memcpy(ready_aid + 1, v + at + 7, aid);
				}
				at += 7 + aid + 7;
			}
		}
		/*
		 * The first (logical) slot is the primary subscription's, the one NAS registers with. On this phone
		 * (cycle 30) the card there is "ERROR" while the SIM sits READY in slot 2: logical slot 1 is pointed at
		 * physical slot 2 (UIM Switch Slot: TLV 1 the logical slot, TLV 2 the physical one, both from 1), as
		 * Android's telephony does with its dual-SIM mapping (modem_sim_switch=0: not).
		 */
		/*
		 * The primary GW provisioning session: which application the subscription NAS registers with uses.
		 * After the slots are switched it is "none" (0xffff: cycle 32), and the modem sits in power save
		 * with no service: the READY USIM of logical slot 1 is made the primary GW application (UIM
		 * Change Provisioning Session: session type 0, activate 1, the slot and the AID).
		 */
		if (primary_gw == 0xffff && ready_aid[0] != 0 && !sim_provisioned && uim_port != NULL) {
			uint8_t pr[7 + 5 + 3 + 2 + 32];
			unsigned n = 0;

			pr[n++] = 0; pr[n++] = 0x37; pr[n++] = 0;
			pr[n++] = UIM_CHANGE_PROVISIONING_SESSION & 0xff; pr[n++] = UIM_CHANGE_PROVISIONING_SESSION >> 8;
			pr[n++] = 0; pr[n++] = 0;       /* the length, below */
			pr[n++] = 0x01; pr[n++] = 2; pr[n++] = 0; pr[n++] = 0; pr[n++] = 1;   /* primary GW, activate */
			pr[n++] = 0x10; pr[n++] = (uint8_t)(2 + ready_aid[0]); pr[n++] = 0;
			pr[n++] = 1;                    /* logical slot 1 */
			pr[n++] = ready_aid[0];
			memcpy(pr + n, ready_aid + 1, ready_aid[0]);
			n += ready_aid[0];
			pr[5] = (uint8_t)(n - 7); pr[6] = (uint8_t)((n - 7) >> 8);
			sim_provisioned = true;
			tell("mpss: uim: THE USIM OF SLOT 1 IS MADE THE PRIMARY GW APPLICATION (AID of %u bytes; its state is looked at again)\n",
			    ready_aid[0]);
			qrtr_send(uim_port, uim_node, uim_port_number, pr, n);
		}
		if (first_card != 1 && sim_slot_ready > 1 && !sim_slot_switched && !option("modem_sim_switch_off") && uim_port != NULL) {
			uint8_t sw[7 + 4 + 7] = { 0, 0x34, 0, UIM_SWITCH_SLOT & 0xff, UIM_SWITCH_SLOT >> 8, 11, 0,
						  1, 1, 0, 1, 2, 4, 0, (uint8_t)sim_slot_ready, 0, 0, 0 };

			sim_slot_switched = true;
			tell("mpss: uim: LOGICAL SLOT 1 IS POINTED AT PHYSICAL SLOT %u, where the card is ready\n", sim_slot_ready);
			qrtr_send(uim_port, uim_node, uim_port_number, sw, sizeof(sw));
		}
	}
}

static void
nas2_receive(__unused void *context, __unused uint32_t node, __unused uint32_t port, const void *data, size_t length)
{
	const uint8_t *m = data;
	uint16_t size = 0;
	const uint8_t *v;

	if (length < 7) {
		return;
	}
	probe_dump("nas", m, length);
	if ((m[3] | m[4] << 8) == NAS_GET_SYSTEM_SELECTION_PREFERENCE) {
		uint32_t mode = ~0u, selection = ~0u, emergency = ~0u, roaming = ~0u, domain = ~0u;

		if ((v = qmi_tlv(m, length, 0x11, &size)) != NULL && size >= 2) {
			mode = (uint32_t)(v[0] | v[1] << 8);
		}
		if ((v = qmi_tlv(m, length, 0x16, &size)) != NULL && size >= 1) {
			selection = v[0];
		}
		if ((v = qmi_tlv(m, length, 0x10, &size)) != NULL && size >= 1) {
			emergency = v[0];
		}
		if ((v = qmi_tlv(m, length, 0x14, &size)) != NULL && size >= 1) {
			roaming = v[0];
		}
		if ((v = qmi_tlv(m, length, 0x1c, &size)) != NULL && size >= 4) {
			domain = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24);
		}
		tell("mpss: nas: SYSTEM SELECTION: mode preference %#x (bits: 1 CDMA, 2 EVDO, 4 GSM, 8 UMTS, 0x10 LTE, 0x40 TD-SCDMA), network "
		    "selection %u (0 automatic, 1 manual), emergency mode %u, roaming %u, service domain %#x (0 CS, 1 PS, 2 both)\n", mode, selection,
		    emergency, roaming, domain);
	} else if ((m[3] | m[4] << 8) == NAS_INITIATE_NETWORK_REGISTER) {
		tell("mpss: nas: the modem answers INITIATE NETWORK REGISTER (automatic)\n");
	} else if ((m[3] | m[4] << 8) == NAS_GET_SYSTEM_INFO) {
		/* per radio: service status (0 none, 1 limited, 2 available, 3 limited regional, 4 power save), true status, data path;
		 * and the SIM reject reason (0x27: 0 available, 1 not available, 2 CS invalid, 3 CS+PS invalid, 4 PS invalid) */
		static const char *const radios[] = { "GSM", "WCDMA", "LTE" };
		static const uint8_t types[] = { 0x12, 0x13, 0x14 };
		uint32_t reject = ~0u;
		char line[160] = "";
		size_t at = 0;

		for (unsigned r = 0; r < 3; r++) {
			if ((v = qmi_tlv(m, length, types[r], &size)) != NULL && size >= 3) {
				at += (size_t)snprintf(line + at, sizeof(line) - at, "%s service %u (true %u, data path %u); ", radios[r], v[0], v[1], v[2]);
			}
		}
		if ((v = qmi_tlv(m, length, 0x27, &size)) != NULL && size >= 4) {
			reject = (uint32_t)(v[0] | v[1] << 8 | v[2] << 16 | v[3] << 24);
		}
		tell("mpss: nas: SYSTEM INFO: %sSIM reject %u (0 available, 1 not available, 2 CS invalid, 3 CS and PS invalid, 4 PS invalid)\n",
		    line, reject);
	}
}

static void
uim_server(__unused void *context, bool arrived, uint32_t service, __unused uint32_t version, __unused uint32_t instance, uint32_t node,
    uint32_t port)
{
	if (service != QMI_UIM) {
		return;
	}
	if (arrived) {
		uim_node = node;
		uim_port_number = port;
	} else if (node == uim_node && port == uim_port_number) {
		uim_node = uim_port_number = 0;
	}
}

static void
nas2_server(__unused void *context, bool arrived, uint32_t service, __unused uint32_t version, __unused uint32_t instance, uint32_t node,
    uint32_t port)
{
	if (service != QMI_NAS) {
		return;
	}
	if (arrived) {
		nas2_node = node;
		nas2_port_number = port;
	} else if (node == nas2_node && port == nas2_port_number) {
		nas2_node = nas2_port_number = 0;
	}
}

static void
sim_probe_open(void)
{
	if (uim_port == NULL) {
		uim_port = qrtr_open(uim_receive, uim_server, NULL);
		if (uim_port != NULL) {
			qrtr_lookup(uim_port, QMI_UIM, 0, 0);
		}
	}
	if (nas2_port == NULL) {
		nas2_port = qrtr_open(nas2_receive, nas2_server, NULL);
		if (nas2_port != NULL) {
			qrtr_lookup(nas2_port, QMI_NAS, 0, 0);
		}
	}
}

/* The three questions, sent; the answers come to uim_receive() and nas2_receive(). */
static void
sim_probe(void)
{
	uint8_t card[7] = { 0, 0x31, 0, UIM_GET_CARD_STATUS & 0xff, UIM_GET_CARD_STATUS >> 8, 0, 0 };
	uint8_t selection[7] = { 0, 0x32, 0, NAS_GET_SYSTEM_SELECTION_PREFERENCE & 0xff, NAS_GET_SYSTEM_SELECTION_PREFERENCE >> 8, 0, 0 };
	uint8_t reg[11] = { 0, 0x33, 0, NAS_INITIATE_NETWORK_REGISTER & 0xff, NAS_INITIATE_NETWORK_REGISTER >> 8, 4, 0, 1, 1, 0, 1 };

	if (uim_port != NULL && uim_port_number != 0) {
		printf("mpss: uim: about to ask for the card's status (node %u port %u)\n", uim_node, uim_port_number);
		qrtr_send(uim_port, uim_node, uim_port_number, card, sizeof(card));
	} else {
		printf("mpss: uim: NOT THERE: the card's status is not asked\n");
	}
	if (nas2_port != NULL && nas2_port_number != 0) {
		printf("mpss: nas: about to ask for the system selection preference, and for automatic registration (node %u port %u)\n",
		    nas2_node, nas2_port_number);
		qrtr_send(nas2_port, nas2_node, nas2_port_number, selection, sizeof(selection));
		qrtr_send(nas2_port, nas2_node, nas2_port_number, reg, sizeof(reg));
	}
}

static void
sim_probe_close(void)
{
	if (uim_port != NULL) {
		qrtr_close(uim_port);
		uim_port = NULL;
	}
	if (nas2_port != NULL) {
		qrtr_close(nas2_port);
		nas2_port = NULL;
	}
	uim_node = uim_port_number = nas2_node = nas2_port_number = 0;
	sim_slot_ready = 0;
	sim_slot_switched = sim_provisioned = sim_register_again = false;
}

static bool
step_nas(void)
{
	uint32_t wait_seconds = 90, online = 1;
	uint64_t began, asked_online = 0;
	bool registered = false, is_online = false, probed = option("modem_sim_probe_off"), probed_again = false;
	uint64_t probed_at = 0, watched_at = 0, data_deadline = 0;
	uint32_t data_wait = 40;        /* seconds to keep trying the data session after registration */

	if (!glink.channel[CHANNEL_QRTR].open) {
		return false;
	}
	PE_parse_boot_argn("modem_nas_wait", &wait_seconds, sizeof(wait_seconds));
	PE_parse_boot_argn("modem_online", &online, sizeof(online));
	PE_parse_boot_argn("modem_data_wait", &data_wait, sizeof(data_wait));
	announce("look the modem's services up as they come (they take some 30 s): DMS asked to put the modem online as soon as it is "
	    "there (modem_online=0: not), NAS asked for the serving system and the signal every second, until the modem is registered "
	    "with a signal read or modem_nas_wait seconds (default 90) are over");
	dms_port = qrtr_open(dms_receive, dms_server, NULL);
	if (dms_port == NULL || qrtr_lookup(dms_port, QMI_DMS, 0, 0) != 0) {
		printf("mpss: %u: no port for DMS\n", step);
		return false;
	}
	if (hakios_nas_client_open(&nas, &qrtr_ops, nas_changed, NULL) != 0) {
		printf("mpss: %u: no port for NAS\n", step);
		return false;
	}
	nas_opened = true;
	if (!probed) {
		sim_probe_open();
	}
	wds_open();
	began = now_us();
	while (now_us() - began < wait_seconds * 1000000ull) {
		if (online && !is_online && dms_port_number != 0 && now_us() - asked_online > 5000000ull) {
			asked_online = now_us();
			is_online = dms_online();       /* (asked again every five seconds until the modem agrees) */
		}
		if (!probed && (is_online || now_us() - began > 40000000ull) && uim_port_number != 0 && nas2_port_number != 0) {
			sleep_ms(2000);                 /* (the modem, just online, is given a moment) */
			sim_probe();
			probed = true;
			probed_at = now_us();
		}
		if (probed && sim_slot_switched && !probed_again && now_us() - probed_at > 6000000ull) {
			/* the slots switched: the card's status, the selection and the registration asked once more */
			probed_again = true;
			sim_probe();
		}
		if (sim_register_again && nas2_port_number != 0) {
			uint8_t reg[11] = { 0, 0x38, 0, NAS_INITIATE_NETWORK_REGISTER & 0xff, NAS_INITIATE_NETWORK_REGISTER >> 8, 4, 0, 1, 1, 0, 1 };

			sim_register_again = false;
			sleep_ms(3000);
			qrtr_send(nas2_port, nas2_node, nas2_port_number, reg, sizeof(reg));
			printf("mpss: nas: automatic registration asked for again after the provisioning\n");
		}
		if (probed && now_us() - watched_at > 10000000ull && uim_port_number != 0 && nas2_port_number != 0) {
			/* every ten seconds: the card's state as it initializes, and the system's service status */
			uint8_t card[7] = { 0, 0x35, 0, UIM_GET_CARD_STATUS & 0xff, UIM_GET_CARD_STATUS >> 8, 0, 0 };
			uint8_t info[7] = { 0, 0x36, 0, NAS_GET_SYSTEM_INFO & 0xff, NAS_GET_SYSTEM_INFO >> 8, 0, 0 };

			watched_at = now_us();
			qrtr_send(uim_port, uim_node, uim_port_number, card, sizeof(card));
			qrtr_send(nas2_port, nas2_node, nas2_port_number, info, sizeof(info));
		}
		if (nas.connected) {
			nas_polls += hakios_nas_client_poll(&nas) == 0;
		}
		wds_advance();          /* bind / start / retry the data session on its own clock */
		sleep_ms(1000);
		if (nas.state.registered && nas.state.has_signal) {
			if (!registered) {
				registered = true;
				data_deadline = now_us() + (uint64_t)data_wait * 1000000ull;
				tell("mpss: nas: registered; the data session is given up to %u s to come up\n", data_wait);
			}
			/* stay in the loop until the data session is up or the data deadline passes */
			if (wds_up || option("modem_data_off") || now_us() > data_deadline) {
				break;
			}
		}
	}
	cellular_up = nas.connected;
	sim_probe_close();
	if (!keep) {
		wds_close();
	}
	if (ipa_loaded && !option("modem_ipa_quiet")) {
		ipa_status("at the end of step 11");
		ipa_space_faults("at the end of step 11");
	}
	{
		struct smp2p words;

		if (dsp_words(&words)) {
			show_words("the modem's item now", &words);
		}
	}
	tell("mpss: %u: after %llu ms: DMS %s, the modem %s; NAS %s (node %u port %u), asked %u times, answered %u, %u changes; %s; "
	    "signal %s%d dBm (%s), %u bars\n", step, (now_us() - began) / 1000, dms_port_number ? "there" : "NOT THERE", is_online ?
	    "online" : "NOT PUT ONLINE", nas.connected ? "there" : "NOT THERE", nas.peer_node, nas.peer_port, nas_polls, nas.answers,
	    nas_changes, registered ? "REGISTERED WITH A NETWORK" : nas.state.registered ? "registered" : "not registered",
	    nas.state.has_signal ? "" : "none read, ", nas.state.has_signal ? nas.state.dbm : 0, radio_name(nas.state.radio_interface),
	    nas.state.bars);
	return nas.connected;
}

/* With modem=1: NAS asked every two seconds while the modem runs, on a thread of its own. */
static volatile bool nas_wanted, nas_runs;

static void
nas_thread(__unused void *parameter, __unused wait_result_t result)
{
	while (nas_wanted) {
		if (nas_opened && nas.connected) {
			nas_polls += hakios_nas_client_poll(&nas) == 0;
		}
		for (unsigned ms = 0; ms < 2000 && nas_wanted; ms += 100) {
			sleep_ms(100);
		}
	}
	nas_runs = false;
	thread_terminate(current_thread());
}

static void
nas_start(void)
{
	thread_t thread;

	nas_wanted = nas_runs = true;
	if (kernel_thread_start(nas_thread, NULL, &thread) != KERN_SUCCESS) {
		nas_wanted = nas_runs = false;
		return;
	}
	thread_deallocate(thread);
}

/* The questions end before the link does. */
static void
nas_stop(void)
{
	if (nas_runs) {
		nas_wanted = false;
		for (unsigned ms = 0; ms < 3000 && nas_runs; ms++) {
			sleep_ms(1);
		}
	}
	if (nas_opened) {
		hakios_nas_client_close(&nas);
		nas_opened = false;
	}
	if (dms_port != NULL) {
		qrtr_close(dms_port);
		dms_port = NULL;
	}
	rmtfs_stop();
	cellular_up = cellular_registered = cellular_bars = cellular_dbm = cellular_radio = 0;
}

/* ---- the steps, and their end ----------------------------------------------------------------------- */

static bool (*const steps[STEPS])(void) = {
	step_survey, step_firmware, step_mailbox, step_header, step_image, step_our_item, step_start, step_queues, step_link,
	step_qrtr, step_nas,
};

static bool keep;                       /* the DSP and the link stay up while the system runs (modem_keep, modem=1) */

static void
steps_over(uint32_t done)
{
	lck_mtx_lock(&dsp_lock);
	if (dsp_started && done == last_step && keep) {
		printf("mpss: THE MODEM STAYS ON while the system runs%s\n", link_runs ? ", and the link with it" : "");
	} else if (dsp_started) {
		dsp_shutdown("the steps are over");
	}
	lck_mtx_unlock(&dsp_lock);
}

/* When the run ends in order: a DSP that runs is shut down, for the next kernel to start again. */
void
hakios_mpss_stop(void)
{
	nas_stop();                             /* the questions first, while the modem still listens */
	lck_mtx_lock(&dsp_lock);
	if (dsp_started) {
		unsigned services = 0;
		uint32_t node = qrtr_peer;
		bool down;

		for (unsigned i = 0; i < sizeof(qrtr_servers) / sizeof(qrtr_servers[0]); i++) {
			services += qrtr_servers[i].there;
		}
		down = dsp_shutdown("the run is about to end");
		/* (said again at the end: a long run's log keeps its beginning and its end, and the middle may be gone) */
		tell("mpss: the modem is off: it had been up%s (node %u, %u services; NAS %s, %s, %u bars), it %s, and TrustZone %s\n",
		    node ? " and linked" : ", NOT LINKED", node, services, nas.connected ? "there" : "NOT THERE", nas.state.registered ?
		    "registered" : "not registered", nas.state.bars, stopped_by_its_word ? "stopped when asked" :
		    "DID NOT SAY IT HAD STOPPED", down ? "has shut it down" : "DID NOT SHUT IT DOWN");
	}
	lck_mtx_unlock(&dsp_lock);
}

/* The same on a panic's way back to the loader: one processor, interrupts off, no lock, no sleep. */
void
hakios_mpss_quiesce(void)
{
	if (!dsp_started) {
		return;
	}
	leaving = true;
	new_from = ~0u;
	dsp_shutdown("the kernel is on its way back to the loader");
}

static void
run_steps(void)
{
	uint32_t done = 0;

	for (step = 1; step <= last_step; step++) {
		if (!steps[step - 1]()) {
			printf("mpss: step %u: FAILED; the steps after it are not attempted\n", step);
			break;
		}
		printf("mpss: step %u: done\n", step);
		done = step;
	}
	step = done;
	new_from = STEPS + 1;
	steps_over(done);
	printf("mpss: over after step %u; %s\n", done, last_write);
}

/* modem_step=N: the steps once the root is there, then back to the loader if modem_return=1 says so. */
static void
steps_thread(__unused void *parameter, __unused wait_result_t result)
{
	for (unsigned waited = 0; waited < 300 && rootvnode == NULLVP; waited++) {
		sleep_ms(200);
	}
	if (rootvnode == NULLVP) {
		printf("mpss: no root file system within a minute: the steps are not performed\n");
	} else {
		run_steps();
	}
	if (option("modem_return")) {
		printf("mpss: back to the loader (modem_return)\n");
		loader_return();
	}
	thread_terminate(current_thread());
}

static void
whole_thread(__unused void *parameter, __unused wait_result_t result)
{
	uint64_t began = now_us();
	unsigned services = 0;

	/* (the firmware may be on the root file system, which is not mounted yet when this begins: a minute at most) */
	for (unsigned waited = 0; waited < 300 && rootvnode == NULLVP; waited++) {
		sleep_ms(200);
	}
	run_steps();
	for (unsigned i = 0; i < sizeof(qrtr_servers) / sizeof(qrtr_servers[0]); i++) {
		services += qrtr_servers[i].there;
	}
	if (step == STEPS && dsp_started) {
		tell("mpss: THE MODEM IS UP AND LINKED, %llu ms after the kernel began on it (the root's waited for included): node %u, %u "
		    "services; NAS %s, %s, %u bars (%d dBm, %s); it stays on, asked every two seconds\n", (now_us() - began) / 1000, qrtr_peer,
		    services, nas.connected ? "there" : "NOT THERE", nas.state.registered ? "REGISTERED" : "not registered", nas.state.bars,
		    nas.state.has_signal ? nas.state.dbm : 0, radio_name(nas.state.radio_interface));
		nas_start();
	} else {
		tell("mpss: THE MODEM DID NOT COME UP: step %u failed (%s); see the kernel's own log for the steps\n", step + 1,
		    last_write);
	}
	thread_terminate(current_thread());
}

/*
 * The SoC addresses of another Qualcomm phone, from boot-args, over the first phone's built-in
 * values. Everything the modem and the IPA touch is here: the register blocks mapped at run time
 * (the AOSS and IPCC mailboxes, the RPMh controller, the shared-memory hardware lock, the IPA and
 * GSI registers and the IPA's local memory, the application processors' SMMU) and the few
 * addresses kept as values (shared memory, the command DB, the modem's memory, the SoC's internal
 * memory). A port passes soc_<name>=0x... for each that differs; tools/soc-args.py reads them off
 * the phone's own device tree. Anything not given keeps the built-in value, so the first phone
 * needs no arguments. (The RMTFS buffer has its own argument, modem_buffer.)
 */
static void
soc_override(void)
{
	static const struct { const char *arg; struct block *block; } blocks[] = {
		{ "soc_aoss", &aoss }, { "soc_ipcc", &ipcc }, { "soc_rsc", &rsc }, { "soc_smem_lock", &smem_lock },
		{ "soc_ipa_regs", &ipa_regs }, { "soc_ipa_sram", &ipa_sram }, { "soc_gsi_ee0", &gsi_ee0 }, { "soc_apps_smmu", &apps_smmu },
	};
	unsigned said = 0;

	for (unsigned i = 0; i < sizeof(blocks) / sizeof(blocks[0]); i++) {
		uint32_t phys = 0;

		if (PE_parse_boot_argn(blocks[i].arg, &phys, sizeof(phys)) && phys != 0 && blocks[i].block->base == 0) {
			blocks[i].block->phys = phys;
			said++;
		}
	}
	PE_parse_boot_argn("soc_smem", &smem_phys, sizeof(smem_phys));
	PE_parse_boot_argn("soc_cmd_db", &cmd_db_phys, sizeof(cmd_db_phys));
	PE_parse_boot_argn("soc_mpss_mem", &mpss_memory, sizeof(mpss_memory));
	PE_parse_boot_argn("soc_imem", &ipa_imem, sizeof(ipa_imem));
	PE_parse_boot_argn("soc_ipa_stream", &ipa_stream, sizeof(ipa_stream));
	PE_parse_boot_argn("soc_ipa_stream_uc", &ipa_stream_uc, sizeof(ipa_stream_uc));
	PE_parse_boot_argn("soc_pas_mpss", &pas_mpss, sizeof(pas_mpss));
	PE_parse_boot_argn("soc_pas_ipa", &pas_ipa, sizeof(pas_ipa));
	if (said != 0 || smem_phys != SMEM_PHYS_DEFAULT || mpss_memory != MPSS_MEMORY_DEFAULT) {
		printf("mpss: SoC addresses from boot-args: %u register blocks; shared memory %#x, command DB %#x, the modem's "
		    "memory %#x, the SoC's internal memory %#x\n", said, smem_phys, cmd_db_phys, mpss_memory, ipa_imem);
	}
}

/* Called while the system starts (drivers/input/hakios.input.c), after the Wi-Fi's start has been set going. */
void
hakios_mpss_init(void)
{
	uint32_t on = 0;
	thread_t thread;

	PE_parse_boot_argn("modem_step", &last_step, sizeof(last_step));
	if (last_step == 0 && (!PE_parse_boot_argn("modem", &on, sizeof(on)) || on != 1)) {
		return;
	}
	if (pe_arm_get_soc_base_phys() != HAKIOS_SOC_BASE) {
		printf("mpss: modem=1 and modem_step=N are for the phone's modem, and this is not the phone\n");
		return;
	}
	soc_override();
	if (last_step == 0) {
		/* all of it, on a thread of its own, and left up */
		last_step = STEPS;
		keep = true;
		quiet = true;
		pause_ms = 0;
		listen_seconds = 1;
		new_from = STEPS + 1;
		if (kernel_thread_start(whole_thread, NULL, &thread) != KERN_SUCCESS) {
			printf("mpss: no thread: the modem is not brought up\n");
			return;
		}
		thread_deallocate(thread);
		tell("mpss: the modem is being brought up beside the system's start (modem=1)\n");
		return;
	}
	if (last_step > STEPS) {
		last_step = STEPS;
	}
	keep = option("modem_keep");
	new_from = last_step;
	PE_parse_boot_argn("modem_new", &new_from, sizeof(new_from));
	PE_parse_boot_argn("modem_pause", &pause_ms, sizeof(pause_ms));
	printf("mpss: steps up to %u; those from %u on are new and wait %u ms after each announcement\n", last_step, new_from,
	    pause_ms);
	/*
	 * The image is on the root (85 MB: too much for the small image), which is not mounted yet: the
	 * steps wait for it on a thread of their own, while the system goes on starting (sb_delay=N
	 * keeps SpringBoard out of the way meanwhile).
	 */
	if (kernel_thread_start(steps_thread, NULL, &thread) != KERN_SUCCESS) {
		printf("mpss: no thread: the steps are not performed\n");
		return;
	}
	thread_deallocate(thread);
	printf("mpss: the steps wait for the root file system, and the system goes on starting\n");
}