Skip to content

Your GeForce runs AI at half speed. This fixes it for 20/30/40 series cards. Up to 2× FP16 / BF16 / TF32 tensor throughput

C
15.7 KB
Raw Blame
// SPDX-License-Identifier: GPL-2.0-only
/* Falcon register transport. Firmware policy and ownership belong to the caller. */
#include <linux/delay.h>
#include <linux/dma-mapping.h>
#include <linux/device.h>
#include <linux/string.h>
#include <linux/timekeeping.h>

#include "falcon.h"
#include "prep_610.h"
#include "registers.h"

static int falcon_read(const struct gu_falcon_io *io, u32 offset, u32 *value)
{
	return io->read(io->context, offset, value);
}
static void falcon_write(const struct gu_falcon_io *io, u32 offset, u32 value)
{
	io->write(io->context, offset, value);
}
static bool inaccessible(u32 value)
{
	return value == ~0u || (value & 0xffff0000u) == 0xbadf0000u;
}

int gu_falcon_poll(const struct gu_falcon_io *io, u32 offset, u32 mask, u32 expect,
		   unsigned int budget_us, bool timeout_is_failure, u32 *last)
{
	ktime_t deadline = ktime_add_us(ktime_get(), budget_us);
	u32 value = 0;

	for (;;) {
		int ret = falcon_read(io, offset, &value);

		if (last)
			*last = value;
		if (ret || value == ~0u ||
		    (value & 0xffff0000u) == 0xbadf0000u) {
			dev_err(io->dev, "poll inaccessible: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
				offset, value, mask, expect, budget_us);
			return -EIO;
		}
		if ((value & mask) == expect)
			return 0;
		if (ktime_compare(ktime_get(), deadline) >= 0) {
			/* Console loglevel hides info. A tolerated timeout must not
			 * look like the failure that stopped the boot. */
			if (!timeout_is_failure) {
				dev_dbg(io->dev, "poll timeout tolerated: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
					offset, value, mask, expect, budget_us);
				return -ETIMEDOUT;
			}
			dev_err(io->dev, "poll timeout: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
				offset, value, mask, expect, budget_us);
			return -ETIMEDOUT;
		}
		usleep_range(50, 100);
	}
}

int gu_falcon_engine_reset(const struct gu_falcon_io *io, u32 base,
			   const char *phase, u32 *hwcfg_out,
			   struct gu_falcon_evidence *evidence)
{
	unsigned int j;
	u32 engine, before, first = 0, hwcfg, last = 0;
	int ret;

	memset(evidence, 0, sizeof(*evidence));
	if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
		return -ENODEV;
	before = engine;
	dev_dbg(io->dev, "engine reset assert: phase=%s base=%#x before=%#x write=%#x\n",
		 phase, base, before, before | 1);
	falcon_write(io, base + GU_FALCON_ENGINE_OFFSET, engine | 1);
	/* 610 ksec2ResetHw_TU102 requires ten register cycles for signal
	 * propagation. A stale zero during those cycles is not evidence that
	 * the assertion failed. No CPU start and no reset retry.
	 */
	for (j = 0; j < 10; j++) {
		if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) ||
		    inaccessible(engine)) {
			dev_err(io->dev, "engine reset inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
				phase, base, j + 1, engine);
			return -EIO;
		}
		if (!j)
			first = engine;
	}
	dev_dbg(io->dev, "engine reset propagation: phase=%s base=%#x first=%#x tenth=%#x\n",
		 phase, base, first, engine);
	ret = gu_falcon_poll(io, base + GU_FALCON_ENGINE_OFFSET, 1, 1, 1000, true, &last);
	dev_dbg(io->dev, "engine reset asserted: phase=%s base=%#x error=%d last=%#x\n",
		 phase, base, ret, last);
	if (ret)
		return ret;
	evidence->reset_asserted = true;
	if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
		return -EIO;
	dev_dbg(io->dev, "engine reset release: phase=%s base=%#x before=%#x write=%#x\n",
		 phase, base, engine, engine & ~1u);
	falcon_write(io, base + GU_FALCON_ENGINE_OFFSET, engine & ~1u);
	for (j = 0; j < 10; j++) {
		if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) ||
		    inaccessible(engine)) {
			dev_err(io->dev, "engine reset release inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
				phase, base, j + 1, engine);
			return -EIO;
		}
	}
	ret = gu_falcon_poll(io, base + GU_FALCON_ENGINE_OFFSET, 1, 0, 1000, true, NULL);
	if (ret)
		return ret;
	evidence->reset_released = true;
	/* FALCON_HWCFG2 bit 12 clears when memory scrubbing finishes.
	 * A PRI error fails this wait instead of being handed to RM.
	 */
	ret = gu_falcon_poll(io, base + GU_FALCON_HWCFG2_OFFSET, 1u << 12, 0, 1000000, true, NULL);
	if (ret)
		return ret;
	if (falcon_read(io, base + GU_FALCON_HWCFG2_OFFSET, &hwcfg) || inaccessible(hwcfg))
		return -EIO;
	dev_dbg(io->dev, "engine reset released: phase=%s base=%#x engine=%#x hwcfg2=%#x\n",
		 phase, base, engine, hwcfg);
	if (hwcfg_out)
		*hwcfg_out = hwcfg;
	return 0;
}

int gu_falcon_reset(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
		    struct gu_falcon_evidence *evidence)
{
	u32 hwcfg, bcr, chip;
	int ret;

	memset(evidence, 0, sizeof(*evidence));
	/* 610 kflcnPreResetWait_GA102 permits this timeout (HW bug 3419321). */
	dev_dbg(io->dev, "Falcon reset begin: base=%#x\n", eng->base);
	ret = gu_falcon_poll(io, eng->base + GU_FALCON_HWCFG2_OFFSET,
			     1u << 31, 1u << 31, 1000, false, NULL);
	if (ret && ret != -ETIMEDOUT)
		return ret;
	if (ret)
		dev_dbg(io->dev, "Falcon pre-reset timeout tolerated per HW bug 3419321\n");
	ret = gu_falcon_engine_reset(io, eng->base, "pre-launch", &hwcfg, evidence);
	if (ret)
		return ret;
	if (hwcfg & (1u << 10)) {
		if (falcon_read(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET, &bcr) || inaccessible(bcr))
			return -ENODEV;
		if (bcr & (1u << 4)) {
			falcon_write(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET, 0);
			ret = gu_falcon_poll(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET,
					     0x11, 1, 1000000, true, NULL);
			if (ret)
				return ret;
		}
	}
	if (falcon_read(io, GU_PMC_BOOT_0_OFFSET, &chip) || inaccessible(chip))
		return -ENODEV;
	falcon_write(io, eng->base + GU_FALCON_RM_OFFSET, chip);
	dev_dbg(io->dev, "Falcon reset done: base=%#x hwcfg=%#x chip=%#x\n", eng->base, hwcfg, chip);
	return 0;
}

/* DMATRFMOFFS is 24 bits, FBOFFS is 32 bits, and BASE is 49 bits.
 * Reject truncation/wrap before submitting any part of a transfer. */
static bool falcon_dma_valid(u32 dest, u32 mem_off, dma_addr_t src, u32 size)
{
	u64 rounded = ((u64)size + 255) & ~255ull;
	return size && !((dest | mem_off | src) & 255) &&
		(u64)dest + rounded <= (1ull << 24) &&
		(u64)mem_off + rounded <= (1ull << 32) &&
		src < (1ull << 49) &&
		(u64)mem_off + rounded <= (1ull << 49) - src;
}

int gu_falcon_dma(const struct gu_falcon_io *io, u32 base, u32 dest, u32 mem_off,
		      dma_addr_t src, u32 size, u32 cmd)
{
	u32 done = 0;
	int ret;

	dev_dbg(io->dev, "Falcon DMA: base=%#x dest=%#x offset=%#x src=%#llx size=%#x cmd=%#x\n",
		base, dest, mem_off, (unsigned long long)src, size, cmd);
	if (!falcon_dma_valid(dest, mem_off, src, size))
		return -EINVAL;
	ret = gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 1, 0, 1000000, true, NULL);
	if (ret)
		return ret;
	falcon_write(io, base + GU_FALCON_DMATRFBASE_OFFSET, lower_32_bits(src >> 8));
	falcon_write(io, base + GU_FALCON_DMATRFBASE1_OFFSET, upper_32_bits(src >> 8) & 0x1ff);
	while (done < size) {
		ret = gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 1, 0, 1000000, true, NULL);
		if (ret) {
			dev_err(io->dev, "Falcon DMA progress: done=%u total=%u\n", done, size);
			return ret;
		}
		falcon_write(io, base + GU_FALCON_DMATRFMOFFS_OFFSET, dest);
		falcon_write(io, base + GU_FALCON_DMATRFFBOFFS_OFFSET, mem_off);
		falcon_write(io, base + GU_FALCON_DMATRFCMD_OFFSET, cmd);
		done += 256;
		dest += 256;
		mem_off += 256;
	}
	return gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 2, 2, 1000000, true, NULL);
}

int gu_falcon_hs(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
		 const struct gu_falcon_hs_image *program,
		 struct gu_falcon_evidence *evidence)
{
	u32 trans, cpuctl;
	dma_addr_t code_src;
	int ret;

	memset(evidence, 0, sizeof(*evidence));
	if (program->dma > ~0ull - program->code_off ||
	    program->dma + program->code_off < program->code_va ||
	    program->dma > ~0ull - program->data_off)
		return -ERANGE;
	code_src = program->dma + program->code_off - program->code_va;
	if (!falcon_dma_valid(program->code_pa, program->code_va, code_src, program->code_size) ||
	    (program->data_size && !falcon_dma_valid(program->data_pa, 0,
						  program->dma + program->data_off, program->data_size)))
		return -EINVAL;
	ret = gu_falcon_reset(io, eng, evidence);
	if (ret)
		return ret;
	ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 0, 1000,
			     !program->stale_halt_ok, NULL);
	if (ret && !(ret == -ETIMEDOUT && program->stale_halt_ok))
		return ret;
	if (falcon_read(io, eng->fbif + GU_FBIF_CTL_OFFSET, &trans) || inaccessible(trans))
		return -EIO;
	trans |= 1u << 7;
	falcon_write(io, eng->fbif + GU_FBIF_CTL_OFFSET, trans);
	falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, 0);
	if (falcon_read(io, eng->fbif + GU_FBIF_TRANSCFG(0), &trans) || inaccessible(trans))
		return -ENODEV;
	trans = (trans & ~7u) | 5u;
	falcon_write(io, eng->fbif + GU_FBIF_TRANSCFG(0), trans);
	dma_wmb();
	if (program->os_code_size) {
		ret = gu_falcon_dma(io, eng->base, 0, 0,
				    program->dma + program->os_code_off, program->os_code_size,
				    gpu_unlock_dma_cmd(true, false));
		if (ret)
			return ret;
	}
	ret = gu_falcon_dma(io, eng->base, program->code_pa, program->code_va, code_src,
			 program->code_size, gpu_unlock_dma_cmd(true, true));
	if (ret)
		return ret;
	if (program->data_size) {
		ret = gu_falcon_dma(io, eng->base, program->data_pa, 0,
				    program->dma + program->data_off, program->data_size,
				    gpu_unlock_dma_cmd(false, false));
		if (ret)
			return ret;
	}
	evidence->reloaded = true;
	falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_PARAADDR_OFFSET, program->sig_dmem);
	falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_ENGIDMASK_OFFSET, program->engine_mask);
	falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_CURR_UCODE_ID_OFFSET, program->ucode_id & 0xff);
	falcon_write(io, eng->falcon2 + GU_FALCON2_MOD_SEL_OFFSET, 1);
	falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, program->code_va);
	falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, program->mailbox0);
	falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, program->mailbox1);
	if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &cpuctl) || inaccessible(cpuctl))
		return -EIO;
	dev_dbg(io->dev, "Falcon start: base=%#x cpuctl=%#x code_va=%#x sig_dmem=%#x engine_mask=%#x ucode=%u mailbox=%#x:%#x\n",
		eng->base, cpuctl, program->code_va, program->sig_dmem, program->engine_mask,
		program->ucode_id, program->mailbox1, program->mailbox0);
	evidence->started = true;
	if (cpuctl & (1u << 6))
		falcon_write(io, eng->base + GU_FALCON_CPUCTL_ALIAS_OFFSET, 1u << 1);
	else
		falcon_write(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 1u << 1);

	ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET,
			     1u << 4, 1u << 4, 5000000, true, NULL);
	if (ret)
		return ret;
	evidence->halted = true;
	return 0;
}

/* TU102/GA100 generic loader for board FWSEC v2. Descriptor offsets are
 * RM_FLCN_BL_DMEM_DESC (84 bytes); loader identity/geometry is pinned by the
 * extractor. The loader uses DMA index 4, coherent physical system memory.
 */
int gu_falcon_loader(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
		     const struct gu_falcon_loader_image *program,
		     struct gu_falcon_evidence *evidence)
{
	u32 desc[21] = { 0 }, hwcfg, trans, ctl, i, dst;
	const u32 *code;
	int ret;

	memset(evidence, 0, sizeof(*evidence));
	if (!program->loader || program->loader_size != 768 ||
	    (program->dma & 255) || program->dma > ~0ull - program->data_off ||
	    (program->data_off & 255))
		return -EINVAL;
	code = program->loader;
	ret = gu_falcon_reset(io, eng, evidence);
	if (ret)
		return ret;
	if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl) || (ctl & 16) ||
	    falcon_read(io, eng->base + GU_FALCON_HWCFG_OFFSET, &hwcfg) || inaccessible(hwcfg) ||
	    falcon_read(io, eng->base + GU_FALCON_DMACTL_OFFSET, &trans) || inaccessible(trans))
		return -ENODEV;
	if ((hwcfg & 0x1ff) < 2)
		return -ERANGE;
	dst = ((hwcfg & 0x1ff) - 2) << 8;
	falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, trans & ~1u);
	if (falcon_read(io, eng->fbif + GU_FBIF_TRANSCFG(4), &trans) || inaccessible(trans))
		return -ENODEV;
	falcon_write(io, eng->fbif + GU_FBIF_TRANSCFG(4), (trans & ~7u) | 5u);
	desc[8] = 4;
	desc[9] = lower_32_bits(program->dma);
	desc[10] = upper_32_bits(program->dma);
	desc[11] = program->imem_pa;
	desc[12] = program->imem_ns_size;
	desc[13] = program->imem_sec_pa;
	desc[14] = program->imem_sec_size;
	desc[16] = lower_32_bits(program->dma + program->data_off);
	desc[17] = upper_32_bits(program->dma + program->data_off);
	desc[18] = program->dmem_size;
	falcon_write(io, eng->base + GU_FALCON_DMEMC_OFFSET, 1u << 24);
	for (i = 0; i < 21; i++)
		falcon_write(io, eng->base + GU_FALCON_DMEMD_OFFSET, desc[i]);
	falcon_write(io, eng->base + GU_FALCON_IMEMC_OFFSET, dst | (1u << 24));
	for (i = 0; i < 512; i += 4) {
		if (!(i & 255))
			falcon_write(io, eng->base + GU_FALCON_IMEMT_OFFSET, 0xfd + (i >> 8));
		falcon_write(io, eng->base + GU_FALCON_IMEMD_OFFSET, code[i / 4]);
	}
	evidence->reloaded = true;
	falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, 0xfd00);
	falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, 0);
	falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, 0);
	dma_wmb();
	evidence->started = true;
	falcon_write(io, eng->base + ((ctl & 64) ? GU_FALCON_CPUCTL_ALIAS_OFFSET :
						GU_FALCON_CPUCTL_OFFSET), 2);
	ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 16, 5000000, true, NULL);
	evidence->halted = !ret;
	return ret;
}

/* GA100's direct loader: nonsecure OS stub, tagged secure app, DMEM.
 * GA10x uses the HS DMA/BROM loader with the same signed GA100 image. Its
 * cross-family acceptance is an explicit hardware gate, not an assumption.
 */
static void falcon_pio_imem(const struct gu_falcon_io *io, u32 base, const u32 *image,
			    u32 offset, u32 size, bool secure)
{
	u32 i;

	falcon_write(io, base + GU_FALCON_IMEMC_OFFSET, offset | (1u << 24) |
		  (secure ? 1u << 28 : 0));
	for (i = 0; i < size; i += 4) {
		if (!(i & 255))
			falcon_write(io, base + GU_FALCON_IMEMT_OFFSET, (offset + i) >> 8);
		falcon_write(io, base + GU_FALCON_IMEMD_OFFSET, image[(offset + i) / 4]);
	}
}

int gu_falcon_direct(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
		     const struct gu_falcon_direct_image *program,
		     struct gu_falcon_evidence *evidence)
{
	const u32 *image = program->words;
	u32 i, ctl;
	int ret;

	memset(evidence, 0, sizeof(*evidence));
	ret = gu_falcon_reset(io, eng, evidence);
	if (ret)
		return ret;
	ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 0, 1000, true, NULL);
	if (ret)
		return ret;
	/* REQUIRE_CTX=false, matching kflcnDisableCtxReq. */
	if (falcon_read(io, eng->base + GU_FALCON_DMACTL_OFFSET, &ctl) || inaccessible(ctl))
		return -ENODEV;
	falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, ctl & ~1u);
	falcon_pio_imem(io, eng->base, image, program->os_code_off, program->os_code_size, false);
	falcon_pio_imem(io, eng->base, image, program->app_code_off, program->app_code_size, true);
	falcon_write(io, eng->base + GU_FALCON_DMEMC_OFFSET, 1u << 24);
	for (i = 0; i < program->os_data_size; i += 4)
		falcon_write(io, eng->base + GU_FALCON_DMEMD_OFFSET, image[(program->os_data_off + i) / 4]);
	evidence->reloaded = true;
	falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, 0);
	falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, program->mailbox0);
	falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, program->mailbox1);
	if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl))
		return -ENODEV;
	dma_wmb();
	evidence->started = true;
	falcon_write(io, eng->base + ((ctl & 64) ? GU_FALCON_CPUCTL_ALIAS_OFFSET :
						GU_FALCON_CPUCTL_OFFSET), 2);
	ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 16, 5000000, true, NULL);
	evidence->halted = !ret;
	return ret;
}