Skip to content

Apple's open-source XNU kernel on Android phones: its own loader and drivers, Metal on the Adreno. Runs from RAM; a reboot gives the phone back to Android.

C
33.2 KB
Raw Blame
/*
 * The IPA's data path for this processor: GSI rings, the IPA's endpoints for the application
 * processor, and the immediate commands that give the IPA its tables.
 *
 * The IPA (Qualcomm's "IP accelerator", here IPA 4.11 with GSI 2.11) moves packets between the
 * modem and this processor. Each side has "endpoints" (pipes) in the IPA and "channels" in the
 * GSI (the DMA engine beside it): a channel is a ring of transfer elements (TREs: address, length,
 * flags; 16 bytes) that this processor fills and the hardware consumes, and an event ring the
 * hardware fills with one event per finished TRE. Writing a doorbell register tells the hardware
 * how far the ring is filled; the event ring's write pointer, read from a register, tells us how
 * far the hardware got. No interrupt is used: the interface's thread polls (hakios.cell.c).
 *
 * What this processor needs, as Linux's driver sets them up for IPA 4.11 (drivers/net/ipa,
 * ipa_data-v4.11.c, gsi.c, ipa_endpoint.c, ipa_cmd.c, ipa_table.c; the register layout from
 * reg/gsi_reg-v4.11.c and reg/ipa_reg-v4.11.c):
 *   - the command endpoint (7, channel 5): immediate commands to the IPA itself, which is how it
 *     is told where its filter and route tables and its header table are;
 *   - the "LAN" receive endpoint (9, channel 14): the IPA's default route (what matches no rule)
 *     and its exceptions end here; it is looked at and dropped;
 *   - the modem transmit endpoint (2, channel 2): IP packets to the modem, each behind a QMAP
 *     header (4 bytes: pad length, mux id, packet length, big-endian);
 *   - the modem receive endpoint (16, channel 7): IP packets from the modem, QMAP-framed, several
 *     per buffer (the IPA aggregates them until 12 KB or 500 us).
 * No checksum offload, no status packets: the network stack computes its checksums itself and
 * the modem is told QMAP without checksum (hakios.mpss.c, WDA Set Data Format).
 *
 * All of this is done before the modem starts (Linux does the same): the tables the commands
 * write hold pointers to one "zero rule", and the modem then writes its own rules into its part
 * of them. The IPA reaches this memory through the SMMU context the modem driver gave it, which
 * shows DDR as it is (physical addresses). The memory is cached for us, so every buffer is
 * written back and dropped from the cache (dc civac) before the hardware looks at it and before
 * we look at what the hardware wrote.
 *
 * modem_ipa_data_off=1: none of this. modem_ipa_data_trace=1: a line per packet.
 */
#include <kern/clock.h>
#include <kern/kalloc.h>
#include <kern/locks.h>
#include <libkern/libkern.h>
#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <pexpert/pexpert.h>
#include <sys/systm.h>
#include <vm/vm_kern.h>
#include <sys/hakios.ipa.h>

extern vm_offset_t ml_io_map(vm_offset_t phys_addr, vm_size_t size);
extern uint64_t kvtophys(vm_offset_t va);
extern void FlushPoC_DcacheRegion(vm_offset_t va, size_t length);

static LCK_GRP_DECLARE(ipa_lock_group, "hakios-ipa");
static LCK_MTX_DECLARE(ipa_lock, &ipa_lock_group);

/* ---- the registers ---------------------------------------------------------------------------- */

/*
 * Where the IPA and the GSI are. These are the values of the first phone (POCO X5 Pro, SM7325),
 * which another Qualcomm phone overrides from its own device tree: the "reg" of the node whose
 * compatible is "qcom,ipa" (reg-names "ipa-base" and "gsi-base"), passed as boot-args soc_ipa_base
 * and soc_gsi_base (tools/soc-args.py reads them off the phone's fdt). The IPA driver has one
 * execution environment's worth of GSI registers mapped, from the inter-EE masks to the scratch.
 */
#define IPA_REGS_PHYS_DEFAULT   0x01e40000ull
#define GSI_PHYS_DEFAULT        0x01e04000ull
#define IPA_REGS_SIZE           0x1000
#define GSI_FROM                0xc000          /* what is used of the GSI's registers: from the inter-EE masks (0xc020) to the scratch (0x12400) */
#define GSI_SIZE                0x7000

static uint64_t ipa_regs_phys = IPA_REGS_PHYS_DEFAULT, gsi_phys = GSI_PHYS_DEFAULT;
static vm_offset_t ipa_regs, gsi_regs;         /* gsi_regs maps gsi_phys + GSI_FROM */

static inline uint32_t
ipa_rd(uint32_t offset)
{
	return *(volatile uint32_t *)(ipa_regs + offset);
}

static inline void
ipa_wr(uint32_t offset, uint32_t value)
{
	*(volatile uint32_t *)(ipa_regs + offset) = value;
	__asm__ volatile ("dsb sy" ::: "memory");
}

static inline uint32_t
gsi_rd(uint32_t offset)
{
	return *(volatile uint32_t *)(gsi_regs + (offset - GSI_FROM));
}

static inline void
gsi_wr(uint32_t offset, uint32_t value)
{
	*(volatile uint32_t *)(gsi_regs + (offset - GSI_FROM)) = value;
	__asm__ volatile ("dsb sy" ::: "memory");
}

/* The GSI's registers for our execution environment (EE 0; another EE's are 0x4000 further) */
#define CH_CNTXT(c, n)          (0xf000 + 0x80 * (c) + 4 * (n))        /* n 0..3: type/dir/event ring/state, length, address */
#define CH_QOS(c)               (0xf05c + 0x80 * (c))
#define CH_SCRATCH(c, n)        (0xf060 + 0x80 * (c) + 4 * (n))
#define EV_CNTXT(e, n)          (0x10000 + 0x80 * (e) + 4 * (n))       /* n 0..13: type/state, length, address, read/write pointers, moderation, MSI */
#define CH_DOORBELL(c)          (0x11000 + 8 * (c))
#define EV_DOORBELL(e)          (0x11100 + 8 * (e))
#define GSI_STATUS              0x12000
#define CH_CMD                  0x12008
#define EV_CH_CMD               0x12010
#define CNTXT_TYPE_IRQ          0x12080
#define CNTXT_TYPE_IRQ_MSK      0x12088
#define CNTXT_SRC_CH_IRQ        0x12090
#define CNTXT_SRC_EV_CH_IRQ     0x12094
#define CNTXT_SRC_CH_IRQ_MSK    0x12098
#define CNTXT_SRC_EV_CH_IRQ_MSK 0x1209c
#define CNTXT_SRC_CH_IRQ_CLR    0x120a0
#define CNTXT_SRC_EV_CH_IRQ_CLR 0x120a4
#define CNTXT_SRC_IEOB_IRQ      0x120b0
#define CNTXT_SRC_IEOB_IRQ_MSK  0x120b8
#define CNTXT_SRC_IEOB_IRQ_CLR  0x120c0
#define CNTXT_GLOB_IRQ_STTS     0x12100
#define CNTXT_GLOB_IRQ_EN       0x12108
#define CNTXT_GLOB_IRQ_CLR      0x12110
#define CNTXT_GSI_IRQ_STTS      0x12118
#define CNTXT_GSI_IRQ_EN        0x12120
#define CNTXT_GSI_IRQ_CLR       0x12128
#define CNTXT_INTSET            0x12180
#define ERROR_LOG               0x12200
#define ERROR_LOG_CLR           0x12210
#define INTER_EE_SRC_CH_IRQ_MSK 0xc020
#define INTER_EE_SRC_EV_CH_IRQ_MSK 0xc024

#define GSI_CH_ALLOCATE         0x0     /* channel commands (CH_CMD bits 31:24; the channel in 7:0) */
#define GSI_CH_START            0x1
#define GSI_CH_STOP             0x2
#define GSI_CH_RESET            0x9
#define GSI_CH_DE_ALLOC         0xa
#define GSI_EVT_ALLOCATE        0x0     /* event ring commands (EV_CH_CMD, the same layout) */
#define GSI_EVT_RESET           0x9
#define GSI_EVT_DE_ALLOC        0xa
#define CH_STATE_NOT_ALLOCATED  0       /* CH_CNTXT 0 bits 23:20 */
#define CH_STATE_ALLOCATED      1
#define CH_STATE_STARTED        2
#define CH_STATE_STOPPED        3
#define EV_STATE_NOT_ALLOCATED  0       /* EV_CNTXT 0 bits 23:20 */
#define EV_STATE_ALLOCATED      1
#define GSI_CHANNEL_TYPE_GPI    2
#define ELEMENT_SIZE            16

/* The IPA's endpoint registers (0x70 apart) */
#define ENDP_INIT_CFG(ep)       (0x808 + 0x70 * (ep))
#define ENDP_INIT_NAT(ep)       (0x80c + 0x70 * (ep))
#define ENDP_INIT_HDR(ep)       (0x810 + 0x70 * (ep))
#define ENDP_INIT_HDR_EXT(ep)   (0x814 + 0x70 * (ep))
#define ENDP_INIT_HDR_METADATA_MASK(ep) (0x818 + 0x70 * (ep))
#define ENDP_INIT_MODE(ep)      (0x820 + 0x70 * (ep))
#define ENDP_INIT_AGGR(ep)      (0x824 + 0x70 * (ep))
#define ENDP_INIT_HOL_BLOCK_EN(ep) (0x82c + 0x70 * (ep))
#define ENDP_INIT_DEAGGR(ep)    (0x834 + 0x70 * (ep))
#define ENDP_INIT_RSRC_GRP(ep)  (0x838 + 0x70 * (ep))
#define ENDP_INIT_SEQ(ep)       (0x83c + 0x70 * (ep))
#define ENDP_STATUS(ep)         (0x840 + 0x70 * (ep))
#define IPA_SHARED_MEM_SIZE     0x54            /* bits 31:16: the local memory's offset in 8-byte units */
#define IPA_FILT_ROUT_HASH_FLUSH 0x14c

/* ---- memory the hardware reads and writes ------------------------------------------------------ */

struct dma {
	vm_offset_t     virt;
	uint64_t        phys;
};

/* One page (16 KB), zeroed, written out of the cache; false if there is none. */
static bool
dma_page(struct dma *d)
{
	vm_offset_t virt = 0;

	if (kmem_alloc(kernel_map, &virt, PAGE_SIZE, KMA_KOBJECT | KMA_ZERO | KMA_DATA, VM_KERN_MEMORY_IOKIT) != KERN_SUCCESS) {
		return false;
	}
	d->virt = virt;
	d->phys = kvtophys(virt);
	FlushPoC_DcacheRegion(virt, PAGE_SIZE);
	return true;
}

/* Written back and dropped from the cache: before the hardware reads it, and before we read what the hardware wrote. */
static inline void
dma_sync(vm_offset_t virt, size_t size)
{
	vm_offset_t from = virt & ~63ul;

	FlushPoC_DcacheRegion(from, ((virt + size + 63) & ~63ul) - from);
	__asm__ volatile ("dsb sy" ::: "memory");
}

/* ---- the channels ------------------------------------------------------------------------------ */

struct tre {
	uint64_t        addr;
	uint16_t        len_opcode;     /* bytes to transfer, or the immediate command's opcode */
	uint16_t        reserved;
	uint32_t        flags;          /* TRE_* */
};
#define TRE_CHAIN               (1u << 0)
#define TRE_IEOT                (1u << 9)       /* an event when done */
#define TRE_BEI                 (1u << 10)      /* ... but no interrupt for it */
#define TRE_TYPE_XFER           (2u << 16)
#define TRE_TYPE_COMMAND        (3u << 16)

struct event {
	uint64_t        xfer;           /* the address of the TRE that finished */
	uint16_t        len;            /* bytes moved (what a receive buffer got) */
	uint8_t         reserved1;
	uint8_t         code;
	uint16_t        reserved2;
	uint8_t         type;
	uint8_t         chid;
};

struct channel {
	const char *    name;
	uint8_t         id, ring_id, endpoint;  /* the channel, its event ring, the IPA's endpoint */
	bool            to_ipa, command, qmap, aggregate, close_eof;
	uint32_t        slots;          /* TREs in the ring, and buffers (a power of two; they fit one page: at most 1024) */
	uint32_t        tlv;            /* the endpoint's TLV FIFO: how many TREs may be outstanding (ipa_data-v4.11.c tlv_count) */
	uint32_t        buffer_size;    /* bytes per slot (a divisor or a multiple of the page size) */
	struct dma      tre_ring, event_ring;
	struct dma *    pages;          /* the buffers, slot by slot */
	uint32_t        page_count;
	uint32_t        next;           /* the next TRE to fill (counts up; the slot is next % slots) */
	uint32_t        done;           /* the next TRE whose completion is awaited (transmit) */
	uint32_t        events;         /* the next event to read (counts up) */
	bool            started;
	uint64_t        packets, bytes, dropped, errors;
};

/*
 * The four: numbers from Linux's ipa_data-v4.11.c (channel, endpoint, TLV counts, which has
 * QMAP and aggregation). Rings here are smaller than Linux's; the buffers are a page for a receive
 * slot (the IPA aggregates into it) and 2 KB for a transmit slot (one packet and its header).
 */
static struct channel command = { "command", 5, 0, 7, true, true, false, false, false, 16, 20, 64 };
#define RECEIVE_BUFFER          16384           /* a page: the IPA aggregates into it */
static struct channel lan_rx = { "lan-rx", 14, 1, 9, false, false, false, true, false, 16, 9, RECEIVE_BUFFER };
static struct channel modem_tx = { "modem-tx", 2, 2, 2, true, false, true, false, false, 64, 16, 2048 };
static struct channel modem_rx = { "modem-rx", 7, 3, 16, false, false, true, true, true, 64, 9, RECEIVE_BUFFER };
static struct channel *channels[] = { &command, &lan_rx, &modem_tx, &modem_rx };
#define AGGR_LIMIT_KB           12              /* the IPA closes a receive buffer once this much is in it (a page holds another 4 KB packet) */
#define AGGR_TIME_TICKS         5               /* ... or after 5 ticks of the first pulse generator: 500 us (TIMERS_PULSE_GRAN_CFG gran 0 = 100 us) */

static bool ready, trace;
static uint32_t mem_offset;                     /* the local memory's offset within the IPA's shared memory */
static struct dma table, zero;                  /* the tables' page (below) and a page of zeros */

static inline vm_offset_t
slot_virt(const struct channel *ch, uint32_t slot)
{
	uint64_t at = (uint64_t)slot * ch->buffer_size;

	return ch->pages[at / PAGE_SIZE].virt + at % PAGE_SIZE;
}

static inline uint64_t
slot_phys(const struct channel *ch, uint32_t slot)
{
	uint64_t at = (uint64_t)slot * ch->buffer_size;

	return ch->pages[at / PAGE_SIZE].phys + at % PAGE_SIZE;
}

static inline struct tre *
tre_at(const struct channel *ch, uint32_t index)
{
	return (struct tre *)(ch->tre_ring.virt + (index % ch->slots) * ELEMENT_SIZE);
}

static bool
channel_memory(struct channel *ch)
{
	uint64_t bytes = (uint64_t)ch->slots * ch->buffer_size;

	ch->page_count = (uint32_t)((bytes + PAGE_SIZE - 1) / PAGE_SIZE);
	ch->pages = kalloc_type(struct dma, ch->page_count, Z_WAITOK | Z_ZERO);
	if (ch->pages == NULL || !dma_page(&ch->tre_ring) || !dma_page(&ch->event_ring)) {
		return false;
	}
	for (uint32_t p = 0; p < ch->page_count; p++) {
		if (!dma_page(&ch->pages[p])) {
			return false;
		}
	}
	return true;
}

static void
gsi_error(const char *what)
{
	uint32_t log = gsi_rd(ERROR_LOG);

	printf("ipa: %s: GSI status %#x, error log %#x (code %u, type %u, EE %u, index %u, args %#x), global %#x, general %#x\n", what,
	    gsi_rd(GSI_STATUS), log, (log >> 12) & 0xf, (log >> 24) & 0xf, log >> 28, (log >> 19) & 0x1f, log & 0xfff,
	    gsi_rd(CNTXT_GLOB_IRQ_STTS), gsi_rd(CNTXT_GSI_IRQ_STTS));
	if (log != 0) {
		gsi_wr(ERROR_LOG, 0);
		gsi_wr(ERROR_LOG_CLR, ~0u);
	}
}

static inline uint32_t
channel_state(const struct channel *ch)
{
	return (gsi_rd(CH_CNTXT(ch->id, 0)) >> 20) & 0xf;
}

static inline uint32_t
event_ring_state(const struct channel *ch)
{
	return (gsi_rd(EV_CNTXT(ch->ring_id, 0)) >> 20) & 0xf;
}

/* A command to the GSI about a channel or an event ring; true once the state is `wanted` (within 50 ms). */
static bool
gsi_command(struct channel *ch, bool event, uint32_t opcode, uint32_t wanted)
{
	uint32_t state = 0, number = event ? ch->ring_id : ch->id;

	gsi_wr(event ? EV_CH_CMD : CH_CMD, number | opcode << 24);
	for (unsigned us = 0; us < 50000; us += 100) {
		state = event ? event_ring_state(ch) : channel_state(ch);
		if (state == wanted) {
			break;
		}
		delay_for_interval(100, NSEC_PER_USEC);
	}
	/* (the completion is also flagged as an interrupt source, masked: cleared) */
	gsi_wr(event ? CNTXT_SRC_EV_CH_IRQ_CLR : CNTXT_SRC_CH_IRQ_CLR, 1u << number);
	if (state != wanted) {
		printf("ipa: %s: command %#x on %s %u leaves it in state %u, not %u\n", ch->name, opcode, event ? "event ring" : "channel", number,
		    state, wanted);
		gsi_error(ch->name);
		return false;
	}
	return true;
}

/* As Linux's gsi_irq_setup(): interrupts, not MSIs; every source masked (nothing is handled here: the rings are polled). */
static void
gsi_interrupts_off(void)
{
	gsi_wr(CNTXT_INTSET, 1);
	gsi_wr(CNTXT_TYPE_IRQ_MSK, 0);
	gsi_wr(CNTXT_SRC_CH_IRQ_MSK, 0);
	gsi_wr(CNTXT_SRC_EV_CH_IRQ_MSK, 0);
	gsi_wr(CNTXT_GLOB_IRQ_EN, 0);
	gsi_wr(CNTXT_SRC_IEOB_IRQ_MSK, 0);
	gsi_wr(INTER_EE_SRC_CH_IRQ_MSK, 0);
	gsi_wr(INTER_EE_SRC_EV_CH_IRQ_MSK, 0);
	gsi_wr(CNTXT_GSI_IRQ_EN, 0);
}

/* As Linux's gsi_evt_ring_program() and gsi_channel_program(), for the GPI protocol. */
static bool
channel_setup(struct channel *ch)
{
	uint32_t e = ch->ring_id, c = ch->id, value;

	if (!gsi_command(ch, true, GSI_EVT_ALLOCATE, EV_STATE_ALLOCATED)) {
		return false;
	}
	gsi_wr(EV_CNTXT(e, 0), GSI_CHANNEL_TYPE_GPI | 1u << 16 /* INTYPE: interrupt, not MSI */ | ELEMENT_SIZE << 24);
	gsi_wr(EV_CNTXT(e, 1), ch->slots * ELEMENT_SIZE);
	gsi_wr(EV_CNTXT(e, 2), (uint32_t)ch->event_ring.phys);
	gsi_wr(EV_CNTXT(e, 3), (uint32_t)(ch->event_ring.phys >> 32));
	gsi_wr(EV_CNTXT(e, 8), 32 /* moderation: 1 ms at 32 kHz */ | 1u << 16 /* MODC */);
	for (unsigned n = 9; n <= 13; n++) {
		gsi_wr(EV_CNTXT(e, n), 0);              /* no MSI, no read pointer updates */
	}
	/* "the last processed event": the ring's last entry, as nothing has been processed */
	gsi_wr(EV_DOORBELL(e), (uint32_t)(ch->event_ring.phys + (ch->slots - 1) * ELEMENT_SIZE));

	if (!gsi_command(ch, false, GSI_CH_ALLOCATE, CH_STATE_ALLOCATED)) {
		return false;
	}
	gsi_wr(CH_CNTXT(c, 0), GSI_CHANNEL_TYPE_GPI | (ch->to_ipa ? 1u << 3 : 0) | e << 14 | ELEMENT_SIZE << 24);
	gsi_wr(CH_CNTXT(c, 1), ch->slots * ELEMENT_SIZE);
	gsi_wr(CH_CNTXT(c, 2), (uint32_t)ch->tre_ring.phys);
	gsi_wr(CH_CNTXT(c, 3), (uint32_t)(ch->tre_ring.phys >> 32));
	/* QOS: the command channel gets the lowest round-robin weight and no escape buffer; the doorbell counts bytes */
	value = (ch->command ? 0xf : 0) | (ch->command ? 0 : 1u << 10 /* PREFETCH_MODE: escape buffer only */) | 1u << 24;
	gsi_wr(CH_QOS(c), value);
	/* the GPI scratch: the most outstanding TREs (in bytes), the threshold 2 TREs */
	gsi_wr(CH_SCRATCH(c, 0), (ch->tlv * ELEMENT_SIZE) << 16);
	gsi_wr(CH_SCRATCH(c, 1), (2 * ELEMENT_SIZE) << 16);
	gsi_wr(CH_SCRATCH(c, 2), 0);
	gsi_wr(CH_SCRATCH(c, 3), gsi_rd(CH_SCRATCH(c, 3)) & 0xffff);
	return true;
}

/* As Linux's ipa_endpoint_program() for IPA 4.11, with no checksum offload and no status. */
static void
endpoint_program(const struct channel *ch)
{
	uint32_t ep = ch->endpoint, value;

	ipa_wr(ENDP_INIT_CFG(ep), 0);                   /* no fragment or checksum offload */
	ipa_wr(ENDP_INIT_NAT(ep), 0);                   /* NAT bypassed */
	value = 0;
	if (ch->qmap) {
		value = 4 /* HDR_LEN */ | 1u << 6 /* HDR_OFST_METADATA_VALID, at 0 */;
		if (!ch->to_ipa) {
			/* where the IPA writes the mux id (byte 1) and the packet's length (bytes 2-3) of the header it makes */
			value |= 1u << 7 /* HDR_OFST_METADATA */ | 1u << 19 /* HDR_OFST_PKT_SIZE_VALID */ | 2u << 20 /* HDR_OFST_PKT_SIZE */;
		}
	}
	ipa_wr(ENDP_INIT_HDR(ep), value);
	value = 0;
	if (ch->qmap) {
		value = 1 /* HDR_ENDIANNESS: big */;
		if (!ch->to_ipa) {
			value |= 1u << 1 /* HDR_TOTAL_LEN_OR_PAD_VALID (pad) */ | 1u << 3 /* HDR_PAYLOAD_LEN_INC_PADDING */;
		}
	}
	ipa_wr(ENDP_INIT_HDR_EXT(ep), value);
	if (!ch->to_ipa) {
		ipa_wr(ENDP_INIT_HDR_METADATA_MASK(ep), ch->qmap ? 0xff000000 : 0);   /* the metadata's low byte, big-endian */
	} else {
		/* the command endpoint moves its data to the LAN receive endpoint (DMA mode); the rest is basic */
		ipa_wr(ENDP_INIT_MODE(ep), ch->command ? 3 | (uint32_t)lan_rx.endpoint << 4 : 0);
	}
	if (!ch->to_ipa && ch->aggregate) {
		value = 1 /* AGGR_EN: aggregate */ | 4u << 2 /* AGGR_TYPE: generic */ | AGGR_LIMIT_KB << 5 | AGGR_TIME_TICKS << 12 |
		    (ch->close_eof ? 1u << 23 /* SW_EOF_ACTIVE */ : 0);
	} else if (ch->to_ipa && ch->qmap) {
		value = 2 /* AGGR_EN: deaggregate */ | 6u << 2 /* AGGR_TYPE: QCMAP */;
	} else {
		value = 0;
	}
	ipa_wr(ENDP_INIT_AGGR(ep), value);
	if (!ch->to_ipa) {
		ipa_wr(ENDP_INIT_HOL_BLOCK_EN(ep), 0);  /* no head-of-line blocking drops */
	} else {
		ipa_wr(ENDP_INIT_DEAGGR(ep), 0);
	}
	ipa_wr(ENDP_INIT_RSRC_GRP(ep), 0);              /* resource group 0: UL/DL (the only one with limits) */
	if (ch->to_ipa) {
		ipa_wr(ENDP_INIT_SEQ(ep), ch->command ? 0 /* DMA */ : 4 /* 2_PASS_SKIP_LAST_UC */);
	}
	ipa_wr(ENDP_STATUS(ep), 0);                     /* no status packets */
}

/* The TRE at `index` of the channel's ring, filled and written out; the doorbell is rung separately. */
static void
tre_fill(struct channel *ch, uint32_t index, uint64_t addr, uint16_t len_opcode, uint32_t flags)
{
	struct tre *t = tre_at(ch, index);

	t->addr = addr;
	t->len_opcode = len_opcode;
	t->reserved = 0;
	t->flags = flags;
	dma_sync((vm_offset_t)t, sizeof(*t));
}

static inline void
channel_doorbell(struct channel *ch)
{
	gsi_wr(CH_DOORBELL(ch->id), (uint32_t)(ch->tre_ring.phys + (ch->next % ch->slots) * ELEMENT_SIZE));
}

/*
 * The events the hardware has written since last time, each to `each`. The event ring's write
 * pointer is in its context register 4; the doorbell afterwards tells the hardware the last event
 * read, which makes room.
 */
static unsigned
channel_events(struct channel *ch, void (*each)(struct channel *, const struct event *, void *), void *context)
{
	uint32_t wp = gsi_rd(EV_CNTXT(ch->ring_id, 4)), base = (uint32_t)ch->event_ring.phys, index;
	unsigned n = 0;

	if (wp < base || wp >= base + ch->slots * ELEMENT_SIZE) {
		if (ch->errors++ == 0) {
			printf("ipa: %s: the event ring's write pointer %#x is outside the ring at %#x\n", ch->name, wp, base);
		}
		return 0;
	}
	index = (wp - base) / ELEMENT_SIZE;
	while (ch->events % ch->slots != index) {
		struct event *e = (struct event *)(ch->event_ring.virt + (ch->events % ch->slots) * ELEMENT_SIZE);

		dma_sync((vm_offset_t)e, sizeof(*e));
		each(ch, e, context);
		ch->events++;
		n++;
	}
	if (n != 0) {
		gsi_wr(EV_DOORBELL(ch->ring_id), (uint32_t)(ch->event_ring.phys + ((ch->events - 1) % ch->slots) * ELEMENT_SIZE));
	}
	return n;
}

/* ---- immediate commands ------------------------------------------------------------------------- */

#define IPA_CMD_IP_V4_FILTER_INIT 0x3
#define IPA_CMD_IP_V6_FILTER_INIT 0x4
#define IPA_CMD_IP_V4_ROUTING_INIT 0x7
#define IPA_CMD_IP_V6_ROUTING_INIT 0x8
#define IPA_CMD_HDR_INIT_LOCAL  0x9
#define IPA_CMD_DMA_SHARED_MEM  0x13

struct command_done {
	uint64_t        tre;
	bool            done;
	uint8_t         code;
};

static void
command_event(__unused struct channel *ch, const struct event *e, void *context)
{
	struct command_done *waiting = context;

	if (e->xfer == waiting->tre) {
		waiting->done = true;
		waiting->code = e->code;
	}
	command.done++;
}

/* One immediate command, waited for (up to 200 ms). The payload is copied into the slot's buffer. */
static bool
ipa_command(const char *what, uint8_t opcode, const void *payload, size_t size)
{
	uint32_t slot = command.next % command.slots;
	struct command_done waiting = { .tre = command.tre_ring.phys + slot * ELEMENT_SIZE };

	if (size > command.buffer_size || command.next - command.done >= command.slots - 1) {
		return false;
	}
	memcpy((void *)slot_virt(&command, slot), payload, size);
	dma_sync(slot_virt(&command, slot), size);
	tre_fill(&command, command.next, slot_phys(&command, slot), opcode, TRE_TYPE_COMMAND | TRE_IEOT);
	command.next++;
	channel_doorbell(&command);
	for (unsigned us = 0; us < 200000 && !waiting.done; us += 200) {
		channel_events(&command, command_event, &waiting);
		if (!waiting.done) {
			delay_for_interval(200, NSEC_PER_USEC);
		}
	}
	if (!waiting.done) {
		printf("ipa: the command %s (opcode %#x) is not answered in 200 ms; channel state %u\n", what, opcode, channel_state(&command));
		gsi_error(what);
		return false;
	}
	if (trace) {
		printf("ipa: the command %s (opcode %#x) done, code %u\n", what, opcode, waiting.code);
	}
	return true;
}

/*
 * The tables, as Linux's ipa_table_init() lays them out in one buffer: a "zero rule" (8 bytes of
 * zero, which every table entry points at until somebody installs a rule), then the filter
 * table's bitmap of the endpoints that filter (bit n+1 for endpoint n: 2, 5 and 8 -- our modem
 * transmit endpoint and two of the modem's), then as many entries as the longest table (15: the
 * route tables' size, 0x78 bytes), each the zero rule's address. The filter table is the bitmap
 * and 3 entries from offset 8; a route table the 15 entries from offset 16.
 *
 * The IPA's local memory, in the layout the modem driver zeroed (Android's ipa_4_11_mem_part,
 * the same offsets as Linux's ipa_mem_local_data for these): IPv4 filter hashed 0x288, not
 * hashed 0x308, IPv6 0x388/0x408; IPv4 route 0x488/0x508, IPv6 0x588/0x608; the modem's and our
 * headers 0x688, 0x440 bytes in all.
 */
#define FILTERED_ENDPOINTS      ((1ull << 2) | (1ull << 5) | (1ull << 8))
#define FILTER_ENTRIES          4               /* the bitmap and one per filtering endpoint */
#define ROUTE_ENTRIES           15

static bool
table_init(const char *what, uint8_t opcode, bool filter, uint32_t hashed_offset, uint32_t offset)
{
	struct {
		uint64_t hash_rules_addr, flags, nhash_rules_addr;
	} payload;
	uint32_t entries = filter ? FILTER_ENTRIES : ROUTE_ENTRIES, size = entries * 8;
	uint64_t source = table.phys + (filter ? 8 : 16);

	payload.hash_rules_addr = source;
	payload.nhash_rules_addr = source;
	payload.flags = (uint64_t)size | (uint64_t)(mem_offset + hashed_offset) << 12 | (uint64_t)size << 28 |
	    (uint64_t)(mem_offset + offset) << 40;
	return ipa_command(what, opcode, &payload, sizeof(payload));
}

static bool
tables_setup(void)
{
	uint64_t *t = (uint64_t *)table.virt;
	struct {
		uint64_t hdr_table_addr;
		uint32_t flags, reserved;
	} headers;

	t[0] = 0;                                       /* the zero rule */
	t[1] = FILTERED_ENDPOINTS << 1;                 /* the filter bitmap */
	for (unsigned n = 0; n < ROUTE_ENTRIES; n++) {
		t[2 + n] = table.phys;
	}
	dma_sync(table.virt, (2 + ROUTE_ENTRIES) * 8);
	if (!table_init("IPv4 route table init", IPA_CMD_IP_V4_ROUTING_INIT, false, 0x488, 0x508) ||
	    !table_init("IPv6 route table init", IPA_CMD_IP_V6_ROUTING_INIT, false, 0x588, 0x608) ||
	    !table_init("IPv4 filter table init", IPA_CMD_IP_V4_FILTER_INIT, true, 0x288, 0x308) ||
	    !table_init("IPv6 filter table init", IPA_CMD_IP_V6_FILTER_INIT, true, 0x388, 0x408)) {
		return false;
	}
	headers.hdr_table_addr = zero.phys;
	headers.flags = 0x440 | (mem_offset + 0x688) << 12;
	headers.reserved = 0;
	if (!ipa_command("header table init", IPA_CMD_HDR_INIT_LOCAL, &headers, sizeof(headers))) {
		return false;
	}
	ipa_wr(IPA_FILT_ROUT_HASH_FLUSH, 0x1111);      /* the hashed tables' caches, flushed */
	return true;
}

/* ---- receiving and sending ---------------------------------------------------------------------- */

/* A receive slot handed to the hardware (one TRE for the whole buffer). */
static void
receive_post(struct channel *ch, uint32_t slot)
{
	tre_fill(ch, ch->next, slot_phys(ch, slot), (uint16_t)(ch->buffer_size > 0xffff ? 0xffff : ch->buffer_size), TRE_TYPE_XFER | TRE_IEOT);
	ch->next++;
}

struct receive_context {
	void            (*receive)(uint8_t mux, const uint8_t *packet, size_t length);
	unsigned        packets;
};

/* A receive buffer is full (or closed): the QMAP packets in it, one by one; then the slot is posted again. */
static void
receive_event(struct channel *ch, const struct event *e, void *context)
{
	struct receive_context *rc = context;
	uint32_t slot = (uint32_t)((e->xfer - ch->tre_ring.phys) / ELEMENT_SIZE), length = e->len;
	const uint8_t *buffer;

	if (slot >= ch->slots || e->xfer < ch->tre_ring.phys) {
		if (ch->errors++ < 4) {
			printf("ipa: %s: an event for TRE %#llx, outside the ring at %#llx\n", ch->name, e->xfer, ch->tre_ring.phys);
		}
		return;
	}
	/* receives complete in order; ch->done is the next expected slot, ch->next the next to post (slots-1 ahead) */
	if (slot != ch->done % ch->slots) {
		if (ch->errors++ < 4) {
			printf("ipa: %s: slot %u finished while slot %u was expected\n", ch->name, slot, ch->done % ch->slots);
		}
	}
	ch->done++;
	buffer = (const uint8_t *)slot_virt(ch, slot);
	if (length > ch->buffer_size) {
		length = ch->buffer_size;
	}
	dma_sync((vm_offset_t)buffer, length);
	ch->bytes += length;
	if (trace) {
		printf("ipa: %s: slot %u: %u bytes (code %u): %02x %02x %02x %02x %02x %02x %02x %02x ...\n", ch->name, slot, length, e->code,
		    buffer[0], buffer[1], buffer[2], buffer[3], buffer[4], buffer[5], buffer[6], buffer[7]);
	}
	if (ch->qmap) {
		for (uint32_t at = 0; at + 4 <= length;) {
			uint32_t pad = buffer[at] & 0x3f, cmd = buffer[at] & 0x80, mux = buffer[at + 1];
			uint32_t packet = (uint32_t)buffer[at + 2] << 8 | buffer[at + 3];

			if (packet == 0 || at + 4 + packet > length || pad > packet) {
				break;
			}
			if (cmd == 0) {
				ch->packets++;
				rc->packets++;
				if (rc->receive != NULL) {
					rc->receive((uint8_t)mux, buffer + at + 4, packet - pad);
				}
			} else if (trace) {
				printf("ipa: %s: a QMAP command (%u bytes) on mux %u\n", ch->name, packet, mux);
			}
			at += 4 + packet;
		}
	} else {
		ch->packets++;                          /* the LAN endpoint: looked at, dropped */
	}
	/* the slot posted again -- the one at the ring's next TRE, which is this one while the hardware finishes in order */
	receive_post(ch, ch->next % ch->slots);
	channel_doorbell(ch);
}

static void
transmit_event(struct channel *ch, const struct event *e, __unused void *context)
{
	uint32_t slot = (uint32_t)((e->xfer - ch->tre_ring.phys) / ELEMENT_SIZE);

	if (slot != ch->done % ch->slots && ch->errors++ < 4) {
		printf("ipa: %s: slot %u finished while slot %u was expected (code %u)\n", ch->name, slot, ch->done % ch->slots, e->code);
	}
	ch->done++;
}

bool
hakios_ipa_transmit(uint8_t mux, const void *packet, size_t length)
{
	struct channel *ch = &modem_tx;
	uint32_t slot;
	uint8_t *buffer;

	if (!ready || length == 0 || length + 4 > ch->buffer_size) {
		return false;
	}
	lck_mtx_lock(&ipa_lock);
	channel_events(ch, transmit_event, NULL);
	if (ch->next - ch->done >= ch->slots - 1) {     /* one slot always left free: write==read means empty */
		ch->dropped++;
		lck_mtx_unlock(&ipa_lock);
		return false;
	}
	slot = ch->next % ch->slots;
	buffer = (uint8_t *)slot_virt(ch, slot);
	buffer[0] = 0;                                  /* no pad, not a command */
	buffer[1] = mux;
	buffer[2] = (uint8_t)(length >> 8);
	buffer[3] = (uint8_t)length;
	memcpy(buffer + 4, packet, length);
	dma_sync((vm_offset_t)buffer, length + 4);
	tre_fill(ch, ch->next, slot_phys(ch, slot), (uint16_t)(length + 4), TRE_TYPE_XFER | TRE_IEOT | TRE_BEI);
	ch->next++;
	channel_doorbell(ch);
	ch->packets++;
	ch->bytes += length;
	if (trace) {
		printf("ipa: %s: slot %u: %zu bytes on mux %u\n", ch->name, slot, length, mux);
	}
	lck_mtx_unlock(&ipa_lock);
	return true;
}

unsigned
hakios_ipa_poll(void (*receive)(uint8_t mux, const uint8_t *packet, size_t length))
{
	struct receive_context rc = { receive, 0 };

	if (!ready) {
		return 0;
	}
	lck_mtx_lock(&ipa_lock);
	channel_events(&modem_rx, receive_event, &rc);
	channel_events(&lan_rx, receive_event, &rc);
	channel_events(&modem_tx, transmit_event, NULL);
	lck_mtx_unlock(&ipa_lock);
	return rc.packets;
}

void
hakios_ipa_modem_endpoints(uint32_t *to_modem, uint32_t *from_modem)
{
	*to_modem = modem_tx.endpoint;          /* 2: what Linux exports as modem/tx_endpoint_id */
	*from_modem = modem_rx.endpoint;        /* 16: modem/rx_endpoint_id */
}

bool
hakios_ipa_data_ready(void)
{
	return ready;
}

void
hakios_ipa_status(char *to, size_t size)
{
	snprintf(to, size, "to the modem %llu packets (%llu bytes, %llu dropped), from it %llu (%llu bytes), %llu to the LAN endpoint; "
	    "errors %llu/%llu/%llu; channels %u/%u/%u/%u", modem_tx.packets, modem_tx.bytes, modem_tx.dropped, modem_rx.packets, modem_rx.bytes,
	    lan_rx.packets, modem_tx.errors, modem_rx.errors, lan_rx.errors, channel_state(&command), channel_state(&lan_rx),
	    channel_state(&modem_tx), channel_state(&modem_rx));
}

/* ---- setting up ---------------------------------------------------------------------------------- */

bool
hakios_ipa_data_setup(void)
{
	uint32_t status, shared;
	char line[256];

	if (ready) {
		return true;
	}
	{
		uint32_t off = 0;

		if (PE_parse_boot_argn("modem_ipa_data_off", &off, sizeof(off)) && off != 0) {
			printf("ipa: no data path (modem_ipa_data_off)\n");
			return false;
		}
		PE_parse_boot_argn("modem_ipa_data_trace", &off, sizeof(off));
		trace = off != 0;
	}
	if (PAGE_SIZE != RECEIVE_BUFFER) {
		printf("ipa: pages of %lu bytes, not %u: no data path\n", (unsigned long)PAGE_SIZE, RECEIVE_BUFFER);
		return false;
	}
	if (ipa_regs == 0) {
		PE_parse_boot_argn("soc_ipa_base", &ipa_regs_phys, sizeof(ipa_regs_phys));
		PE_parse_boot_argn("soc_gsi_base", &gsi_phys, sizeof(gsi_phys));
		ipa_regs = ml_io_map(ipa_regs_phys, IPA_REGS_SIZE);
		gsi_regs = ml_io_map(gsi_phys + GSI_FROM, GSI_SIZE);
	}
	if (ipa_regs == 0 || gsi_regs == 0) {
		printf("ipa: the registers cannot be mapped: no data path\n");
		return false;
	}
	status = gsi_rd(GSI_STATUS);
	shared = ipa_rd(IPA_SHARED_MEM_SIZE);
	mem_offset = 8 * (shared >> 16);
	if ((status & 1) == 0) {
		printf("ipa: the GSI is not enabled (status %#x): no data path\n", status);
		return false;
	}
	if (!dma_page(&table) || !dma_page(&zero)) {
		printf("ipa: no memory for the tables: no data path\n");
		return false;
	}
	for (unsigned n = 0; n < 4; n++) {
		if (!channel_memory(channels[n])) {
			printf("ipa: no memory for the %s channel: no data path\n", channels[n]->name);
			return false;
		}
	}
	printf("ipa: setting up this processor's data path: GSI status %#x, local memory at %#x; rings and buffers in DDR from %#llx "
	    "(the tables at %#llx)\n", status, mem_offset, modem_rx.pages[0].phys, table.phys);
	gsi_interrupts_off();
	for (unsigned n = 0; n < 4; n++) {
		struct channel *ch = channels[n];

		if (!channel_setup(ch)) {
			printf("ipa: the %s channel (%u, event ring %u) cannot be set up: no data path\n", ch->name, ch->id, ch->ring_id);
			return false;
		}
		endpoint_program(ch);
	}
	for (unsigned n = 0; n < 4; n++) {
		struct channel *ch = channels[n];

		if (!gsi_command(ch, false, GSI_CH_START, CH_STATE_STARTED)) {
			printf("ipa: the %s channel does not start: no data path\n", ch->name);
			return false;
		}
		ch->started = true;
		if (!ch->to_ipa) {
			/* one short of the ring: the doorbell is the write pointer, and writing it level with
			 * the read pointer (base) would mean "empty", not "full" */
			for (uint32_t slot = 0; slot + 1 < ch->slots; slot++) {
				receive_post(ch, slot);
			}
			channel_doorbell(ch);
		}
	}
	if (!tables_setup()) {
		printf("ipa: the IPA does not take its tables: no data path (the channels stay)\n");
		return false;
	}
	ready = true;
	hakios_ipa_status(line, sizeof(line));
	printf("ipa: THE DATA PATH IS UP: endpoints %u (commands), %u (LAN in), %u (to the modem), %u (from the modem); the tables set; %s\n",
	    command.endpoint, lan_rx.endpoint, modem_tx.endpoint, modem_rx.endpoint, line);
	return true;
}