Skip to content

Your GeForce runs AI at half speed. This fixes it for 20/30/40 series cards. Up to 2× FP16 / BF16 / TF32 tensor throughput

C

Initial public release

Zagitta0ebdb1c6308b
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..7438baf
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,13 @@
+# `make dist` ships only the source needed to build and test the packages.
+/* export-ignore
+/VERSION -export-ignore
+/Makefile -export-ignore
+/README.md -export-ignore
+/firmware -export-ignore
+/kernel -export-ignore
+/licenses -export-ignore
+/packaging -export-ignore
+/tests -export-ignore
+*.bin export-ignore
+*.cu export-ignore
+/debian -export-ignore
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ca57e86
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,26 @@
+/workbook/
+/scratchpad/
+/build/
+**/__pycache__/
+*.pyc
+*.ko
+*.o
+*.mod
+*.mod.c
+*.cmd
+*.tar.xz
+*.tar.zst
+*.deb
+*.rpm
+*.pkg.tar.zst
+/packaging/arch/src/
+/packaging/arch/pkg/
+/packaging/arch/gpu-unlock/
+/debian/.debhelper/
+/debian/tmp/
+/debian/files
+/debian/*.substvars
+/debian/*.debhelper.log
+/debian/gpu-unlock-*/
+modules.order
+Module.symvers
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..36f913c
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: GPL-2.0-only
+VERSION := $(shell cat VERSION)
+NV := 610.43.03
+PYTHON ?= python3
+DESTDIR ?=
+DISTDIR ?= build
+CACHE ?= $(HOME)/.cache/gpu-unlock
+INPUTS ?= $(CACHE)/inputs
+ASSETS ?= $(CACHE)/assets
+DKMSDIR ?= /usr/src/gpu-unlock-$(VERSION)
+FWDIR ?= /usr/lib/firmware/gpu-unlock/$(NV)
+LICENSEDIR ?= /usr/share/licenses
+PKG ?= gpu-unlock
+
+MODULE_FILES := $(filter-out %.mod.c,$(sort $(wildcard kernel/gpu_unlock/*.[ch]))) \
+ kernel/gpu_unlock/Kbuild kernel/gpu_unlock/Makefile kernel/gpu_unlock/dkms.conf
+FIRMWARE_FILES := booter_load_ga102.bin gsp_ga10x.bin gsp_boot_ga102.bin \
+ gsp_boot_desc_ga102.bin v67-ga102-$(NV).bin
+LICENSES := licenses/GPL-2.0.txt licenses/CMP40HX-MIT.txt licenses/NVIDIA-MIT.txt
+NV_SOURCE := NVIDIA-kernel-module-source-$(NV)
+NV_UTILS := nvidia-utils-$(NV)-1-x86_64_v3.pkg.tar.zst
+
+TARBALL := $(abspath $(DISTDIR))/gpu-unlock-$(VERSION).tar.xz
+DEBDIR := $(abspath $(DISTDIR))/deb
+RPMDIR := $(abspath $(DISTDIR))/rpm
+DEBS := $(foreach p,dkms firmware,$(DEBDIR)/gpu-unlock-$(p)_$(VERSION)_all.deb)
+
+.PHONY: all test dist inputs firmware assets install-dkms install-firmware \
+ install-mkinitcpio install-initramfs-tools install-dracut \
+ install-licenses arch deb rpm install-arch install-deb install-rpm
+
+all:
+
+# With generated or installed firmware, the tests also run the whole unlock
+# against mocked hardware.
+test:
+ GPU_UNLOCK_ASSETS=$(ASSETS) $(PYTHON) -m unittest discover -s tests -v
+
+# Source-only release tarball; see .gitattributes for what is left out.
+dist:
+ mkdir -p $(DISTDIR)
+ git archive --format=tar --prefix=gpu-unlock-$(VERSION)/ \
+ --output=$(DISTDIR)/gpu-unlock-$(VERSION).tar HEAD
+ xz -9 -f $(DISTDIR)/gpu-unlock-$(VERSION).tar
+
+# Download and SHA-256 check the pinned NVIDIA inputs. makepkg does this
+# itself from the PKGBUILD's source array.
+inputs:
+ $(PYTHON) firmware/fetch_artifacts.py $(INPUTS)
+
+# Generate firmware from $(INPUTS) into $(ASSETS); debian/rules and the RPM
+# spec call this from their build step.
+firmware:
+ rm -rf $(ASSETS).src
+ mkdir -p $(ASSETS).src
+ tar -xf $(INPUTS)/$(NV_SOURCE).tar.xz -C $(ASSETS).src
+ $(PYTHON) firmware/generate_v67.py "$$(realpath $(INPUTS)/$(NV_UTILS))" $(ASSETS)
+ $(PYTHON) firmware/extract_boot_assets.py $(ASSETS).src/$(NV_SOURCE) $(ASSETS)
+
+assets: inputs
+ $(MAKE) firmware
+
+# The module source. It does nothing unless gpu_unlock.targets= is on the
+# kernel command line.
+install-dkms:
+ grep -qx 'PACKAGE_VERSION="$(VERSION)"' kernel/gpu_unlock/dkms.conf
+ grep -q 'MODULE_VERSION("$(VERSION)")' kernel/gpu_unlock/main.c
+ install -Dm644 -t $(DESTDIR)$(DKMSDIR) $(MODULE_FILES)
+ install -Dm644 README.md $(DESTDIR)/usr/share/doc/gpu-unlock/README.md
+
+# The generators check every file against its pinned SHA-256 before writing it.
+install-firmware:
+ install -Dm644 -t $(DESTDIR)$(FWDIR) $(addprefix $(ASSETS)/,$(FIRMWARE_FILES))
+ install -Dm644 $(ASSETS)/NVIDIA-LICENSE $(DESTDIR)$(LICENSEDIR)/gpu-unlock-firmware/NVIDIA-LICENSE
+ install -Dm644 firmware/NOTICE.md $(DESTDIR)$(LICENSEDIR)/gpu-unlock-firmware/NOTICE.md
+
+# One file per initramfs generator: include gpu_unlock and its firmware in
+# the image so it runs before an nvidia module loaded from the initramfs.
+install-mkinitcpio:
+ install -Dm644 packaging/boot/mkinitcpio.conf $(DESTDIR)/etc/mkinitcpio.conf.d/zz-gpu-unlock.conf
+
+install-initramfs-tools:
+ install -Dm644 packaging/boot/initramfs-tools.conf \
+ $(DESTDIR)/usr/share/initramfs-tools/modules.d/gpu-unlock.conf
+
+install-dracut:
+ install -Dm644 packaging/boot/dracut.conf $(DESTDIR)/usr/lib/dracut/dracut.conf.d/gpu-unlock.conf
+
+install-licenses:
+ install -Dm644 -t $(DESTDIR)$(LICENSEDIR)/$(PKG) $(LICENSES)
+
+# Native packages, built from the committed HEAD like the PKGBUILD. The
+# install-* variants also pull build dependencies and install the result.
+arch:
+ cd packaging/arch && makepkg -f
+
+install-arch:
+ cd packaging/arch && makepkg -sif
+
+deb: dist inputs
+ rm -rf $(DEBDIR)
+ mkdir -p $(DEBDIR)
+ tar -xf $(TARBALL) -C $(DEBDIR)
+ cd $(DEBDIR)/gpu-unlock-$(VERSION) && \
+ GPU_UNLOCK_INPUTS=$(abspath $(INPUTS)) dpkg-buildpackage -us -uc
+
+install-deb:
+ sudo apt-get build-dep -y $(CURDIR)
+ $(MAKE) deb
+ sudo apt-get install -y $(DEBS)
+
+rpm: dist inputs
+ rm -rf $(RPMDIR)
+ mkdir -p $(RPMDIR)/SOURCES
+ cp $(TARBALL) $(RPMDIR)/SOURCES/
+ ln -s $(abspath $(INPUTS))/$(NV_SOURCE).tar.xz $(abspath $(INPUTS))/$(NV_UTILS) $(RPMDIR)/SOURCES/
+ rpmbuild -ba --define "_topdir $(RPMDIR)" packaging/rpm/gpu-unlock.spec
+
+install-rpm:
+ sudo dnf install -y rpm-build
+ sudo dnf builddep -y packaging/rpm/gpu-unlock.spec
+ $(MAKE) rpm
+ sudo dnf install -y $(RPMDIR)/RPMS/noarch/gpu-unlock-*-$(VERSION)-*.noarch.rpm
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..5854b4e
--- /dev/null
+++ b/README.md
@@ -0,0 +1,104 @@
+# GPU unlock 1.0.0 packaging candidate
+
+This Linux kernel module combines the CMP unlocker's V67/PLM technique with the
+pre-driver timing of the UEFI approach. It runs the unlock, writes
+the speed selectors (SS0 `0x88888888`, which selects the full BF16 rate, and
+the 4-bit SS1 `0x8`, whose effect is untested) and performs FLR before the unmodified
+NVIDIA driver binds the card.
+It is neither a UEFI application nor a patched NVIDIA driver.
+
+The observed result is GPU1, an RTX 3090 at `0000:21:00.0`, on the main
+CachyOS `7.2.8-1-cachyos` kernel with NVIDIA `615.71.09`. GPU0 at
+`0000:01:00.0` was an unmodified control. Other RTX 3090 boards and
+environments have not been verified. Other GA102 PCI device IDs are rejected.
+
+## Local build
+
+Every build downloads the pinned NVIDIA/CachyOS inputs over HTTPS, checks
+their SHA-256, and generates firmware only in the build directory and the
+binary package. All three formats install through the same `make install-*`
+targets. Packages are built from the committed `HEAD`.
+
+Build, install build dependencies, and install the packages in one step
+(asks for your sudo password):
+
+```sh
+make install-arch # makepkg -sif
+make install-deb # apt-get build-dep, dpkg-buildpackage, apt-get install
+make install-rpm # dnf builddep, rpmbuild, dnf install
+```
+
+`make arch`, `make deb` and `make rpm` only build. Debian and RPM packages go
+to `build/deb` and `build/rpm`. The NVIDIA inputs are cached in
+`~/.cache/gpu-unlock/inputs` (`make inputs`; override with `INPUTS=`).
+`dpkg-buildpackage` itself never uses the network, so a clean-chroot build (sbuild or
+pbuilder) needs `GPU_UNLOCK_INPUTS` pointing at that cache.
+The `make dist` archive, the Debian source package and the `.nosrc.rpm`
+contain no NVIDIA archives or generated firmware.
+
+## Boot flow and opt-in
+
+The packages put `gpu_unlock` and its firmware in the initramfs and load it
+there, before `nvidia` can bind a card:
+
+- **mkinitcpio:** `/etc/mkinitcpio.conf.d/zz-gpu-unlock.conf` inserts
+ `gpu_unlock` right before `nvidia` in `MODULES`, so a display card is blank
+ for as short a time as possible. Without `nvidia` in the image, it goes
+ last and still runs before an `nvidia` loaded from the root filesystem.
+ Images are rebuilt on install. mkinitcpio ignores drop-ins for an image
+ built with `-c <config>` or a preset `_config`; put `gpu_unlock` right
+ before `nvidia` in that config's `MODULES` yourself.
+- **initramfs-tools:** `/usr/share/initramfs-tools/modules.d/gpu-unlock.conf`.
+- **dracut:** `/usr/lib/dracut/dracut.conf.d/gpu-unlock.conf`.
+
+There is no `softdep nvidia pre:` rule: kmod honours only the first
+`softdep nvidia` line it reads, so other packages' rules (`nvidia-utils`,
+`gddr6_hwmon`) would silently override it, or it would override theirs.
+
+The module does nothing unless the kernel command line opts in:
+
+```
+gpu_unlock.targets=all # every unbound 10de:2204 card
+gpu_unlock.targets=0000:21:00.0 # only these cards
+gpu_unlock.targets=0000:21:00.0,0000:22:00.0 # comma-separated
+```
+
+Addresses are full PCI names as `lspci -D` prints them (hex digits in either
+case), independent of GPU numbering and display role. An entry that is not
+an RTX 3090, including `all` mixed with addresses, is logged and ignored.
+The cards are unlocked concurrently, about a second in all however many
+there are, and each on its own: a card that is already bound, powered down
+or bus mastering is skipped. Then `nvidia` loads and binds every card as
+usual.
+
+The display card works too. Its firmware framebuffer (the boot console and
+Plymouth) keeps running while the unlock's firmware steps run, and is
+removed just before the card's reset, as `nvidia-drm` would do a moment
+later anyway. Its screen is blank from that reset until `nvidia-drm` takes
+over.
+
+The unlock needs a cold card: power the machine off rather than rebooting.
+Warm reboots are not supported; with the option set, the unlock is expected
+to fail and leave the card as it was. Loading the module by hand after boot
+does nothing to a card that `nvidia` has bound. A card that a driver already
+used this power cycle is refused before any firmware runs (`card already
+initialised since power on`).
+
+Check the result with:
+
+```sh
+journalctl -k -b | grep gpu_unlock
+lspci -nnk -s 0000:21:00.0
+```
+
+To stop unlocking, remove `gpu_unlock.targets=` from the kernel command line;
+for a one-off boot, delete it in the bootloader's edit screen. Removing the
+packages also removes the module and the initramfs drop-ins.
+
+If the firmware on a card never stops, the module logs it and keeps the DMA
+buffers that firmware may still use until the next power off. Power off
+before trying again.
+
+The kernel module and local packaging are GPL-2.0-only. The generated payload
+derives from the pinned CMP reference; NVIDIA firmware remains under its own
+terms. See `firmware/NOTICE.md` and `licenses/` for attribution and notices.
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000..3eefcb9
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+1.0.0
diff --git a/debian/changelog b/debian/changelog
new file mode 100644
index 0000000..ee1bb2c
--- /dev/null
+++ b/debian/changelog
@@ -0,0 +1,5 @@
+gpu-unlock (1.0.0) UNRELEASED; urgency=medium
+
+ * Initial release.
+
+ -- Zagitta <[email protected]> Mon, 28 Sep 2026 00:00:00 +0000
diff --git a/debian/clean b/debian/clean
new file mode 100644
index 0000000..c6def0d
--- /dev/null
+++ b/debian/clean
@@ -0,0 +1 @@
+debian/gpu-unlock-build/
diff --git a/debian/control b/debian/control
new file mode 100644
index 0000000..5eae7f7
--- /dev/null
+++ b/debian/control
@@ -0,0 +1,21 @@
+Source: gpu-unlock
+Section: kernel
+Priority: optional
+Maintainer: Zagitta <[email protected]>
+Build-Depends: debhelper-compat (= 13), dh-sequence-dkms, python3, zstd
+Standards-Version: 4.7.0
+Rules-Requires-Root: no
+
+Package: gpu-unlock-dkms
+Architecture: all
+Depends: ${misc:Depends}, dkms,
+ gpu-unlock-firmware (= ${source:Version})
+Description: RTX 3090 BF16 unlock before the NVIDIA driver loads
+ DKMS source for gpu_unlock, loaded ahead of nvidia on every boot. It does
+ nothing unless gpu_unlock.targets= is on the kernel command line.
+
+Package: gpu-unlock-firmware
+Architecture: all
+Depends: ${misc:Depends}
+Description: locally generated pinned NVIDIA firmware for GPU unlock
+ This package is built for the local machine from verified upstream inputs.
diff --git a/debian/copyright b/debian/copyright
new file mode 100644
index 0000000..d0a9655
--- /dev/null
+++ b/debian/copyright
@@ -0,0 +1,11 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: gpu-unlock
+Source: local source checkout
+
+Files: *
+Copyright: See source file headers and licenses/ notices.
+License: GPL-2.0-only
+
+License: GPL-2.0-only
+ On Debian systems, the complete license text is in
+ /usr/share/common-licenses/GPL-2.
diff --git a/debian/gpu-unlock-dkms.dkms b/debian/gpu-unlock-dkms.dkms
new file mode 100644
index 0000000..f687470
--- /dev/null
+++ b/debian/gpu-unlock-dkms.dkms
@@ -0,0 +1 @@
+kernel/gpu_unlock/dkms.conf
diff --git a/debian/gpu-unlock-dkms.install b/debian/gpu-unlock-dkms.install
new file mode 100644
index 0000000..a11e6c7
--- /dev/null
+++ b/debian/gpu-unlock-dkms.install
@@ -0,0 +1,3 @@
+usr/src
+usr/share/initramfs-tools
+usr/share/doc/gpu-unlock
diff --git a/debian/gpu-unlock-firmware.install b/debian/gpu-unlock-firmware.install
new file mode 100644
index 0000000..616cd40
--- /dev/null
+++ b/debian/gpu-unlock-firmware.install
@@ -0,0 +1,2 @@
+usr/lib/firmware
+usr/share/licenses/gpu-unlock-firmware/* usr/share/doc/gpu-unlock-firmware
diff --git a/debian/gpu-unlock-firmware.lintian-overrides b/debian/gpu-unlock-firmware.lintian-overrides
new file mode 100644
index 0000000..d467ac7
--- /dev/null
+++ b/debian/gpu-unlock-firmware.lintian-overrides
@@ -0,0 +1,2 @@
+# Pinned NVIDIA GSP firmware is a device image, not a host ELF program.
+gpu-unlock-firmware: arch-independent-package-contains-binary-or-object [usr/lib/firmware/gpu-unlock/610.43.03/gsp_ga10x.bin]
diff --git a/debian/rules b/debian/rules
new file mode 100755
index 0000000..cc55b2e
--- /dev/null
+++ b/debian/rules
@@ -0,0 +1,23 @@
+#!/usr/bin/make -f
+include /usr/share/dpkg/pkg-info.mk
+
+# Pinned NVIDIA inputs, fetched beforehand with `make inputs`; the build
+# itself never touches the network.
+GPU_UNLOCK_INPUTS ?= $(HOME)/.cache/gpu-unlock/inputs
+ASSETS := $(CURDIR)/debian/gpu-unlock-build/assets
+
+%:
+ dh $@
+
+override_dh_auto_configure:
+ test "$$(cat VERSION)" = "$(DEB_VERSION_UPSTREAM)"
+
+override_dh_auto_build:
+ $(MAKE) INPUTS="$(GPU_UNLOCK_INPUTS)" ASSETS="$(ASSETS)" firmware
+
+override_dh_auto_test:
+ $(MAKE) test ASSETS="$(ASSETS)"
+
+override_dh_auto_install:
+ $(MAKE) DESTDIR=$(CURDIR)/debian/tmp ASSETS="$(ASSETS)" \
+ install-dkms install-firmware install-initramfs-tools
diff --git a/debian/source/format b/debian/source/format
new file mode 100644
index 0000000..89ae9db
--- /dev/null
+++ b/debian/source/format
@@ -0,0 +1 @@
+3.0 (native)
diff --git a/firmware/NOTICE.md b/firmware/NOTICE.md
new file mode 100644
index 0000000..eb14cc3
--- /dev/null
+++ b/firmware/NOTICE.md
@@ -0,0 +1,30 @@
+# Firmware provenance and distribution scope
+
+`gsp_ga10x.bin` is extracted byte-for-byte from the checksummed NVIDIA
+610.43.03 package in `artifacts-610.43.03.json`. The Booter and GSP boot images
+are decoded unchanged from the pinned 610.43.03 source archive. The extraction
+also preserves that package's complete `usr/share/licenses/nvidia-utils/LICENSE`
+as `NVIDIA-LICENSE`. It is installed with the firmware package.
+
+The source archive's open-module licensing does **not** license NVIDIA firmware.
+The binary package contains the NVIDIA Driver License Agreement. Its section
+1.1(d) addresses distribution for open source operating-system kernels with
+unchanged binary files and inclusion of the agreement. Additional limitations
+apply, including NVIDIA-platform use. Personal local packaging is the initial
+delivery target. Review the full pinned agreement and the separately generated
+payload before any public distribution; redistribution approval is not claimed.
+
+The V67 constants are adapted from WildFlash1st/cmp90hx-unlock at commit
+`76940ae583281daa2b883518a2e89b56c11d3a46`, patch
+`driver/patches/cmp90-rejoin15/0014-6104303-cmp90hx-stockflow-rejoin14-multigpu-state.patch`,
+function `_kgspCmp90FillV67Signature`. The repository's actual `LICENSE` is
+GPL version 2; the earlier manifest's MIT label was incorrect. That license is
+preserved in `licenses/GPL-2.0.txt`. The generated image is a separate file;
+the extracted NVIDIA images are never rewritten.
+
+NVIDIA source files inspected during the loader audit carry their own MIT
+copyright notices. The minimum helper's metadata layout and Falcon operations
+follow those source contracts; `licenses/NVIDIA-MIT.txt` preserves the notice.
+The inspected EFI reference's notice is in `licenses/CMP40HX-MIT.txt`.
+The reference changes a CMP 90HX, not an RTX 3090, and is not evidence of
+standalone boot or persistence through NVIDIA 615.71.09.
diff --git a/firmware/README.md b/firmware/README.md
new file mode 100644
index 0000000..32348bb
--- /dev/null
+++ b/firmware/README.md
@@ -0,0 +1,19 @@
+# Pinned 610.43.03 firmware
+
+`make inputs` runs `fetch_artifacts.py`, which downloads the NVIDIA open kernel
+module source archive and the exact CachyOS `nvidia-utils` package pinned in
+`artifacts-610.43.03.json`. Downloads use HTTPS and are checked against their
+SHA-256 before they enter the cache; cached files are rechecked on every run.
+Nothing downloaded is executed or installed.
+
+`make firmware` then builds the five files the kernel module loads from
+`gpu-unlock/610.43.03/`:
+
+- `generate_v67.py` extracts `gsp_ga10x.bin` and the NVIDIA license from the
+ package and generates the 64,000-byte `v67-ga102-610.43.03.bin`.
+- `extract_boot_assets.py` builds `booter_load_ga102.bin`,
+ `gsp_boot_ga102.bin` and `gsp_boot_desc_ga102.bin` from the source archive.
+
+Both scripts check every input and output against its pinned SHA-256 before
+writing anything. Python 3.14+ is required for zstd tar support. See
+[NOTICE.md](NOTICE.md) for provenance and licensing.
diff --git a/firmware/artifacts-610.43.03.json b/firmware/artifacts-610.43.03.json
new file mode 100644
index 0000000..d5405f1
--- /dev/null
+++ b/firmware/artifacts-610.43.03.json
@@ -0,0 +1,161 @@
+{
+ "version": 1,
+ "target": "GA102 / 610.43.03 rejoin15 reference path",
+ "source_artifacts": [
+ {
+ "name": "NVIDIA open kernel modules source archive",
+ "url": "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-610.43.03.tar.xz",
+ "checksum_url": "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-610.43.03.tar.xz.sha256sum",
+ "revision": "610.43.03 release source (NVIDIA tag 610.43.03)",
+ "sha256": "7e118923c7a23edc36114d63273a46e3e04e9af98695a42203e7ac2dfe9fc1dc",
+ "license": "GPL-2.0 OR MIT",
+ "filename": "NVIDIA-kernel-module-source-610.43.03.tar.xz"
+ },
+ {
+ "name": "GA102 CMP90 reference and rejoin15 patch set",
+ "url": "https://github.com/WildFlash1st/cmp90hx-unlock",
+ "revision": "76940ae583281daa2b883518a2e89b56c11d3a46",
+ "license": "GPL-2.0-only (repository LICENSE; previous MIT label was incorrect)",
+ "files": [
+ {
+ "path": "driver/patches/cmp90-rejoin15/0014-6104303-cmp90hx-stockflow-rejoin14-multigpu-state.patch",
+ "sha256": "c51805dc332c3b6be2e6dd34964a47ce6b1ea0ce73017e0351c216d319094bb1"
+ },
+ {
+ "path": "driver/patches/cmp90-rejoin15/0015-6104303-cmp90hx-stockflow-rejoin15-serialized-start.patch",
+ "sha256": "3dd3883479d10a822d54292db87a1552f15d6b144b8c42bfc88a519674836cb4"
+ },
+ {
+ "path": "LICENSE",
+ "sha256": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643"
+ }
+ ]
+ },
+ {
+ "name": "nvidia-utils binary package used for exact firmware extraction",
+ "version": "610.43.03-1",
+ "architecture": "x86_64_v3",
+ "license": "custom (package metadata)",
+ "sha256": "e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9",
+ "firmware": [
+ {
+ "path": "usr/lib/firmware/nvidia/610.43.03/gsp_ga10x.bin",
+ "size": 84277400,
+ "sha256": "572373620a37418f24dc16b5031c39338778c3257e48e8408de9a57291b24f3a"
+ }
+ ],
+ "filename": "nvidia-utils-610.43.03-1-x86_64_v3.pkg.tar.zst",
+ "url": "https://archive.cachyos.org/archive/cachyos-v3/nvidia-utils-610.43.03-1-x86_64_v3.pkg.tar.zst",
+ "notice": {
+ "path": "usr/share/licenses/nvidia-utils/LICENSE",
+ "sha256": "c692fb14ad4f499c6aee1b9e144e6905148c420e6225b17cb2937366c1e507ad"
+ }
+ }
+ ],
+ "generated_payload": {
+ "name": "rejoin15 GA102 610.43.03 V67 signature image",
+ "size": 64000,
+ "size_hex": "0x0000fa00",
+ "fill_dword_le": "0x0000019c",
+ "stack_map": {
+ "active_canary_dmem": "0x0000ff4c",
+ "initial_return_slot_dmem": "0x0000ff50"
+ },
+ "overrides": [
+ {
+ "offset": "0x00001100",
+ "value": "0x00000007"
+ },
+ {
+ "offset": "0x0000f948",
+ "value": "0xffffffff",
+ "role": "write value"
+ },
+ {
+ "offset": "0x0000f950",
+ "value": "0x00000d44"
+ },
+ {
+ "offset": "0x0000f960",
+ "value": "0x00823804",
+ "role": "FEAT_OVR_PLM address"
+ },
+ {
+ "offset": "0x0000f968",
+ "value": "0x00001fce"
+ },
+ {
+ "offset": "0x0000f974",
+ "value": "0x00000000"
+ },
+ {
+ "offset": "0x0000f97c",
+ "value": "0x00001101"
+ },
+ {
+ "offset": "0x0000f980",
+ "value": "0x000084c8"
+ },
+ {
+ "offset": "0x0000f984",
+ "value": "0x00008e18"
+ },
+ {
+ "offset": "0x0000f98c",
+ "value": "0x000084c8"
+ },
+ {
+ "offset": "0x0000f990",
+ "value": "0x00000000"
+ },
+ {
+ "offset": "0x0000f998",
+ "value": "0x00001fce"
+ },
+ {
+ "offset": "0x0000f9a4",
+ "value": "0x0000ffbc"
+ },
+ {
+ "offset": "0x0000f9ac",
+ "value": "0x00005789"
+ },
+ {
+ "offset": "0x0000f9bc",
+ "value": "0x00000d44"
+ },
+ {
+ "offset": "0x0000f9cc",
+ "value": "0x00000003"
+ },
+ {
+ "offset": "0x0000f9d4",
+ "value": "0x00001fce"
+ },
+ {
+ "offset": "0x0000f9e8",
+ "value": "0x00000d52"
+ },
+ {
+ "offset": "0x0000f9ec",
+ "value": "0x000081ee",
+ "role": "terminator"
+ }
+ ],
+ "write_value_offset": "0x0000f948",
+ "write_value": "0xffffffff",
+ "write_address_offset": "0x0000f960",
+ "write_address": "0x00823804",
+ "terminator_offset": "0x0000f9ec",
+ "terminator": "0x000081ee",
+ "sha256": "2a0aa1f86d0b6afff5214d641d87d46dc98ef06ed908ff05d0ddbad5ffb7e197",
+ "generator": "firmware/generate_v67.py",
+ "reference": "rejoin15 patch 0014, _kgspCmp90FillV67Signature"
+ },
+ "derived_assets": {
+ "booter_load_ga102.bin": "056cbc064a888f80717726c54d7f64fcd68044d123870c95a74f37f4975dcaa4",
+ "gsp_boot_ga102.bin": "3df651acb174fd5f5561a1996965b192f50c5d763168446ed0d4ead19d895768",
+ "gsp_boot_desc_ga102.bin": "cc5a3fc1f4c47f2fe107b1b68f0176d8a448e6dec84b575e5560ccef5262bf15"
+ },
+ "limitation": "The manifest and generator pin/extract firmware artifacts and reproduce the reference V67 image. They do not implement an independent GSP boot/BooterLoad path or establish applicability to RTX 3090 / driver 615.71.09."
+}
diff --git a/firmware/extract_boot_assets.py b/firmware/extract_boot_assets.py
new file mode 100644
index 0000000..eb277a5
--- /dev/null
+++ b/firmware/extract_boot_assets.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+"""Build the Booter and GSP boot assets from the pinned 610 generated source."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import zlib
+import re
+import struct
+import sys
+from pathlib import Path
+
+MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
+
+
+MAGIC = 0x314C5442
+LABELS = (
+ "IMAGE_PROD",
+ "HEADER_PROD",
+ "SIG_PROD",
+ "PATCH_LOC",
+ "PATCH_META",
+ "NUM_SIGS",
+)
+
+
+def _array_bytes(text: str, label: str) -> tuple[bytes, bool, int]:
+ marker = f'BINDATA_LABEL_{label}_data[]'
+ if text.count(marker) != 1:
+ raise ValueError(f"missing or duplicate {label}")
+ start = text.index(marker)
+ window = text[max(0, start - 400):start]
+ compression = re.findall(r"COMPRESSION: (YES|NO)", window)
+ sizes = re.findall(r"DATA SIZE \(bytes\): (\d+)", window)
+ if not compression or not sizes:
+ raise ValueError(f"missing storage declaration for {label}")
+ expected = int(sizes[-1]); compressed = compression[-1] == 'YES'
+ if not 0 < expected <= 16 * 1024 * 1024:
+ raise ValueError('asset exceeds extraction limit')
+ declaration = re.match(re.escape(marker) + r"\s*=\s*\{([^}]*)\};", text[start:], re.S)
+ if not declaration:
+ raise ValueError('malformed byte-array declaration')
+ tokens = declaration[1].strip().rstrip(',').split(',')
+ if any(not re.fullmatch(r"\s*0x[0-9a-fA-F]{2}\s*", x) for x in tokens):
+ raise ValueError('non-byte token in generated array')
+ raw = bytes(int(item, 16) for item in tokens)
+ if compressed:
+ stored = re.findall(r"COMPRESSED SIZE \(bytes\): (\d+)", window)
+ if not stored or len(raw) != int(stored[-1]):
+ raise ValueError('compressed storage size mismatch')
+ inflater = zlib.decompressobj(-15)
+ data = inflater.decompress(raw, expected + 1)
+ # NVIDIA BINDATA stores raw DEFLATE followed by the gzip CRC32/ISIZE
+ # trailer (the gzip header is omitted). Validate both trailer fields.
+ if not inflater.eof or len(inflater.unused_data) != 8 or inflater.unconsumed_tail:
+ raise ValueError('truncated, trailing or oversized compressed stream')
+ crc, size = struct.unpack('<II', inflater.unused_data)
+ if size != expected or crc != zlib.crc32(data):
+ raise ValueError('compressed checksum or size mismatch')
+ else:
+ data = raw
+ if len(data) != expected:
+ raise ValueError(f"{label}: inflated {len(data)} bytes, header says {expected}")
+ return data, compressed, expected
+
+
+def build_bundle(source_text: str) -> bytes:
+ image, _, _ = _array_bytes(source_text, "IMAGE_PROD")
+ header, _, _ = _array_bytes(source_text, "HEADER_PROD")
+ signatures, _, _ = _array_bytes(source_text, "SIG_PROD")
+ patch_loc, _, _ = _array_bytes(source_text, "PATCH_LOC")
+ patch_meta, _, _ = _array_bytes(source_text, "PATCH_META")
+ num_sigs, _, _ = _array_bytes(source_text, "NUM_SIGS")
+ if len(header) != 36 or len(patch_loc) != 4 or len(patch_meta) != 12 or len(num_sigs) != 4:
+ raise ValueError("unexpected BooterLoad record size")
+ fields = struct.unpack("<9I", header)
+ loc = struct.unpack("<I", patch_loc)[0]
+ _fuse, engine, ucode = struct.unpack("<3I", patch_meta)
+ count = struct.unpack("<I", num_sigs)[0]
+ if not count or len(signatures) != count * 384 or fields[4] != 1 or not ucode:
+ raise ValueError('unsupported production Booter signature/application table')
+ if loc < fields[2] or loc + 384 > fields[2] + fields[3] or fields[2] + fields[3] > len(image):
+ raise ValueError('signature patch is outside DMEM image')
+ for offset, size in ((fields[0],fields[1]), (fields[2],fields[3]), (fields[5],fields[6])):
+ if offset + size > len(image) or offset % 256 or size % 256:
+ raise ValueError('invalid Falcon transfer range')
+ hdr = struct.pack(
+ "<14I",
+ fields[0], fields[1], fields[2], fields[3], fields[4], fields[5], fields[6],
+ fields[7], fields[8], loc, ucode, engine, count, len(signatures),
+ )
+ prefix = struct.pack("<II", MAGIC, 1) + hdr + struct.pack("<II", len(image), len(signatures))
+ return prefix + image + signatures
+
+
+PINNED = {
+ 'g_bindata_kgspGetBinArchiveBooterLoadUcode_GA102.c': '04cd3db7f670adc4efd79d2e76217bb809fffddfbd6f1af884cde2af26dd39af',
+ 'g_bindata_kgspGetBinArchiveGspRmBoot_GA102.c': '35e5aa9075ad3898b6022ffd5820c953d809caa7dabd530cfc1a83dc2a67c5bd',
+}
+
+
+def extract_assets(source_root: Path, output: Path) -> None:
+ inputs = {}
+ for name, digest in PINNED.items():
+ data = (source_root / 'src/nvidia/generated' / name).read_bytes()
+ if hashlib.sha256(data).hexdigest() != digest:
+ raise ValueError('pinned generated source mismatch: ' + name)
+ inputs[name] = data.decode('ascii')
+ booter = build_bundle(inputs['g_bindata_kgspGetBinArchiveBooterLoadUcode_GA102.c'])
+ boot = inputs['g_bindata_kgspGetBinArchiveGspRmBoot_GA102.c']
+ image = _array_bytes(boot, 'UCODE_IMAGE_PROD')[0]
+ desc = _array_bytes(boot, 'UCODE_DESC_PROD')[0]
+ if len(desc) != 84:
+ raise ValueError('unexpected pinned bootloader descriptor length')
+ fields = struct.unpack('<21I', desc)
+ if fields[0] != 5 or fields[14] != 1 or fields[19] != len(image) or fields[20] != 0:
+ raise ValueError('unsupported bootloader descriptor')
+ for index in (1,3,8,10,12):
+ if not fields[index+1] or fields[index]+fields[index+1] > len(image):
+ raise ValueError('bootloader descriptor range outside image')
+ blobs = {'booter_load_ga102.bin': booter, 'gsp_boot_ga102.bin': image, 'gsp_boot_desc_ga102.bin': desc}
+ pinned = json.loads(MANIFEST.read_text())['derived_assets']
+ for name, data in blobs.items():
+ if hashlib.sha256(data).hexdigest() != pinned[name]:
+ raise ValueError('derived asset differs from its pinned SHA-256: ' + name)
+ output.mkdir(parents=True, exist_ok=True)
+ for name, data in blobs.items():
+ (output/name).write_bytes(data)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("source", type=Path, help='pinned NVIDIA source root')
+ parser.add_argument("output", type=Path)
+ args = parser.parse_args()
+ extract_assets(args.source, args.output)
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except (OSError, ValueError, zlib.error) as error:
+ sys.exit(f"extract failed: {error}")
diff --git a/firmware/fetch_artifacts.py b/firmware/fetch_artifacts.py
new file mode 100644
index 0000000..f8cc007
--- /dev/null
+++ b/firmware/fetch_artifacts.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+"""Fetch pinned inputs without executing/installing them; atomically verify cache."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import tempfile
+import urllib.request
+
+MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
+
+
+def digest(path: Path) -> str:
+ with path.open("rb") as stream:
+ return hashlib.file_digest(stream, "sha256").hexdigest()
+
+
+def inputs(manifest: dict) -> list[dict]:
+ """The NVIDIA source archive and the nvidia-utils package the build uses."""
+ source, _reference, package = manifest["source_artifacts"]
+ return [source, package]
+
+
+def fetch(entry: dict, output: Path, offline: bool = False) -> Path:
+ output.mkdir(parents=True, exist_ok=True)
+ destination = output / entry["filename"]
+ if destination.exists():
+ if destination.is_symlink() or digest(destination) != entry["sha256"]:
+ raise ValueError(f"invalid cached artifact: {destination}; remove it to retry")
+ return destination
+ if offline:
+ raise FileNotFoundError(f"missing cached artifact: {destination}")
+ if not entry["url"].startswith("https://"):
+ raise ValueError("artifact downloads require HTTPS")
+ temporary = None
+ try:
+ request = urllib.request.Request(entry["url"], headers={"User-Agent": "gpu-unlock-artifacts/0.1.0"})
+ with urllib.request.urlopen(request, timeout=60) as response:
+ if not response.url.startswith("https://"):
+ raise ValueError("refusing non-HTTPS redirect")
+ with tempfile.NamedTemporaryFile(dir=output, prefix=".download-", delete=False) as stream:
+ temporary = Path(stream.name)
+ total = 0
+ while block := response.read(1024 * 1024):
+ total += len(block)
+ if total > 1024 * 1024 * 1024:
+ raise ValueError("artifact exceeds 1 GiB download limit")
+ stream.write(block)
+ stream.flush()
+ os.fsync(stream.fileno())
+ if digest(temporary) != entry["sha256"]:
+ raise ValueError(f"SHA-256 mismatch: {entry['filename']}")
+ temporary.chmod(0o644)
+ temporary.replace(destination)
+ return destination
+ finally:
+ if temporary is not None:
+ temporary.unlink(missing_ok=True)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("output", type=Path, help="cache directory (for example scratchpad/artifacts)")
+ parser.add_argument("--offline", action="store_true", help="verify cached inputs without network")
+ args = parser.parse_args()
+ try:
+ for entry in inputs(json.loads(MANIFEST.read_text())):
+ print(fetch(entry, args.output, args.offline))
+ except (OSError, ValueError) as error:
+ parser.exit(1, f"fetch failed: {error}\n")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/firmware/generate_v67.py b/firmware/generate_v67.py
new file mode 100644
index 0000000..3275a6f
--- /dev/null
+++ b/firmware/generate_v67.py
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+"""Extract pinned 610.43.03 GA10x firmware and generate the rejoin15 V67 image."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import hashlib
+import subprocess
+import tarfile
+from pathlib import Path
+
+
+PACKAGE_SHA256 = "e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9"
+GSP_SHA256 = "572373620a37418f24dc16b5031c39338778c3257e48e8408de9a57291b24f3a"
+PAYLOAD_SHA256 = "2a0aa1f86d0b6afff5214d641d87d46dc98ef06ed908ff05d0ddbad5ffb7e197"
+PACKAGE_VERSION = "610.43.03-1"
+FIRMWARE_PREFIX = "usr/lib/firmware/nvidia/610.43.03/"
+MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
+
+PAYLOAD_SIZE = 0xFA00
+FILL_DWORD = 0x0000019C
+PATCHED_WORDS = {
+ 0x1100: 0x00000007,
+ 0xF948: 0xFFFFFFFF,
+ 0xF950: 0x00000D44,
+ 0xF960: 0x00823804,
+ 0xF968: 0x00001FCE,
+ 0xF974: 0x00000000,
+ 0xF97C: 0x00001101,
+ 0xF980: 0x000084C8,
+ 0xF984: 0x00008E18,
+ 0xF98C: 0x000084C8,
+ 0xF990: 0x00000000,
+ 0xF998: 0x00001FCE,
+ 0xF9A4: 0x0000FFBC,
+ 0xF9AC: 0x00005789,
+ 0xF9BC: 0x00000D44,
+ 0xF9CC: 0x00000003,
+ 0xF9D4: 0x00001FCE,
+ 0xF9E8: 0x00000D52,
+ 0xF9EC: 0x000081EE,
+}
+
+
+def sha256(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+def extract_member(package: Path, name: str) -> bytes:
+ if package.name.endswith(".zst"):
+ process = subprocess.Popen(["zstd", "-dc", str(package)], stdout=subprocess.PIPE)
+ try:
+ with tarfile.open(fileobj=process.stdout, mode="r|") as archive:
+ result = read_member(archive, name)
+ if process.wait() != 0:
+ raise ValueError("zstd could not read the pinned package")
+ return result
+ finally:
+ if process.stdout:
+ process.stdout.close()
+ if process.poll() is None:
+ process.kill()
+ process.wait()
+ with tarfile.open(package, mode="r:*") as archive:
+ return read_member(archive, name)
+
+
+def read_member(archive: tarfile.TarFile, name: str) -> bytes:
+ result = None
+ for member in archive:
+ if member.name != name:
+ continue
+ if result is not None:
+ raise ValueError(f"duplicate package member: {name}")
+ if not member.isfile() or member.size > 128 * 1024 * 1024:
+ raise ValueError(f"not a regular file in package: {name}")
+ stream = archive.extractfile(member)
+ if stream is None:
+ raise ValueError(f"could not read package member: {name}")
+ result = stream.read()
+ if result is None:
+ raise ValueError(f"missing package member: {name}")
+ return result
+
+
+def build_payload() -> bytes:
+ word = FILL_DWORD.to_bytes(4, "little")
+ payload = bytearray(word * (PAYLOAD_SIZE // len(word)))
+ for offset, value in PATCHED_WORDS.items():
+ payload[offset : offset + 4] = value.to_bytes(4, "little")
+ return bytes(payload)
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("package", type=Path, help="pinned nvidia-utils-610.43.03 package")
+ parser.add_argument("output", type=Path, help="directory for extracted firmware and payload")
+ args = parser.parse_args()
+
+ package_bytes_hash = hashlib.sha256()
+ with args.package.open("rb") as package_file:
+ for block in iter(lambda: package_file.read(1024 * 1024), b""):
+ package_bytes_hash.update(block)
+ if package_bytes_hash.hexdigest() != PACKAGE_SHA256:
+ raise SystemExit("package SHA-256 does not match the pinned nvidia-utils artifact")
+
+ pkginfo = extract_member(args.package, ".PKGINFO").decode("utf-8", errors="replace")
+ if f"pkgver = {PACKAGE_VERSION}" not in pkginfo.splitlines():
+ raise SystemExit("package metadata version does not match 610.43.03-1")
+
+ gsp = extract_member(args.package, FIRMWARE_PREFIX + "gsp_ga10x.bin")
+ if sha256(gsp) != GSP_SHA256:
+ raise SystemExit("GA10x GSP firmware SHA-256 does not match the pinned artifact")
+
+ payload = build_payload()
+ if sha256(payload) != PAYLOAD_SHA256:
+ raise SystemExit("generated V67 image differs from its pinned SHA-256")
+ manifest = json.loads(MANIFEST.read_text())
+ notice = manifest["source_artifacts"][2]["notice"]
+ license_text = extract_member(args.package, notice["path"])
+ if sha256(license_text) != notice["sha256"]:
+ raise SystemExit("NVIDIA license notice differs from pinned package")
+ args.output.mkdir(parents=True, exist_ok=True)
+ (args.output / "gsp_ga10x.bin").write_bytes(gsp)
+ (args.output / "v67-ga102-610.43.03.bin").write_bytes(payload)
+ (args.output / "NVIDIA-LICENSE").write_bytes(license_text)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/kernel/gpu_unlock/Kbuild b/kernel/gpu_unlock/Kbuild
new file mode 100644
index 0000000..1687c9f
--- /dev/null
+++ b/kernel/gpu_unlock/Kbuild
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0-only
+obj-m += gpu_unlock.o
+gpu_unlock-y := main.o gfw_eval.o prep_610.o layout_610.o \
+ vbios_610.o unlock_610.o
diff --git a/kernel/gpu_unlock/Makefile b/kernel/gpu_unlock/Makefile
new file mode 100644
index 0000000..1bccac4
--- /dev/null
+++ b/kernel/gpu_unlock/Makefile
@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Set KDIR to build for a kernel other than the running one.
+KDIR ?= /lib/modules/$(shell uname -r)/build
+.PHONY: all clean
+all clean:
+ $(MAKE) -C "$(KDIR)" M="$(CURDIR)" $(if $(filter clean,$@),clean,modules)
diff --git a/kernel/gpu_unlock/diagnostic.h b/kernel/gpu_unlock/diagnostic.h
new file mode 100644
index 0000000..57aa7de
--- /dev/null
+++ b/kernel/gpu_unlock/diagnostic.h
@@ -0,0 +1,24 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_DIAGNOSTIC_H
+#define GPU_UNLOCK_DIAGNOSTIC_H
+#ifdef __KERNEL__
+#include <linux/printk.h>
+#define GU_NOTE(fmt, ...) pr_info("gpu_unlock: " fmt, ##__VA_ARGS__)
+#define GU_ERROR(fmt, ...) pr_err("gpu_unlock: " fmt, ##__VA_ARGS__)
+#else
+#include <stdio.h>
+#define GU_NOTE(fmt, ...) fprintf(stderr, "gpu_unlock: " fmt, ##__VA_ARGS__)
+#define GU_ERROR(fmt, ...) fprintf(stderr, "gpu_unlock: " fmt, ##__VA_ARGS__)
+#endif
+
+/* No extra device reads. Preserve the errno and identify the exact guard in
+ * the matching source bundled with each kit, including pure parser failures. */
+static inline int gu_diagnostic_result(int error, const char *file,
+ const char *function, unsigned int line)
+{
+ if (error)
+ GU_ERROR("failure %s:%u %s error=%d\n", file, line, function, error);
+ return error;
+}
+#define GU_DIAG(result) gu_diagnostic_result((result), __FILE__, __func__, __LINE__)
+#endif
diff --git a/kernel/gpu_unlock/dkms.conf b/kernel/gpu_unlock/dkms.conf
new file mode 100644
index 0000000..f44ec11
--- /dev/null
+++ b/kernel/gpu_unlock/dkms.conf
@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: GPL-2.0-only
+PACKAGE_NAME="gpu-unlock"
+PACKAGE_VERSION="1.0.0"
+BUILT_MODULE_NAME[0]="gpu_unlock"
+DEST_MODULE_LOCATION[0]="/updates/dkms"
+AUTOINSTALL="yes"
+BUILD_EXCLUSIVE_CONFIG="CONFIG_PCI CONFIG_FW_LOADER"
+MAKE[0]="make KDIR=${kernel_source_dir}"
diff --git a/kernel/gpu_unlock/gfw_eval.c b/kernel/gpu_unlock/gfw_eval.c
new file mode 100644
index 0000000..d30c860
--- /dev/null
+++ b/kernel/gpu_unlock/gfw_eval.c
@@ -0,0 +1,32 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#ifdef __KERNEL__
+#include <linux/errno.h>
+#else
+#include <errno.h>
+#endif
+
+#include "gfw_eval.h"
+
+int gpu_unlock_gfw_assess(const struct gpu_unlock_gfw_sample *sample)
+{
+ if (!sample || !sample->config_ok)
+ return -ENODEV;
+ /* All-ones satisfies the bit tests below and is not a live device. */
+ if (sample->plm == (u32)~0U || sample->boot == (u32)~0U ||
+ sample->cpuctl == (u32)~0U)
+ return -ENODEV;
+ if (!(sample->cpuctl & GU_GFW_CPUCTL_HALTED))
+ return -EAGAIN;
+ if (!(sample->plm & GU_GFW_PLM_LEVEL0))
+ return -EAGAIN;
+ if ((sample->boot & 0xffu) != GU_GFW_BOOT_COMPLETED)
+ return -EAGAIN;
+ return 0;
+}
+
+u16 gpu_unlock_command_restore(u16 original, u16 live)
+{
+ u16 mask = GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER;
+
+ return (live & (u16)~mask) | (original & mask);
+}
diff --git a/kernel/gpu_unlock/gfw_eval.h b/kernel/gpu_unlock/gfw_eval.h
new file mode 100644
index 0000000..f721971
--- /dev/null
+++ b/kernel/gpu_unlock/gfw_eval.h
@@ -0,0 +1,46 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_GFW_EVAL_H
+#define GPU_UNLOCK_GFW_EVAL_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdbool.h>
+#include <stdint.h>
+typedef uint16_t u16;
+typedef uint32_t u32;
+#endif
+
+/*
+ * Offsets and fields from the pinned 610.43.03 GA102 headers:
+ * dev_gc6_island.h, dev_gc6_island_addendum.h, dev_gsp.h, dev_falcon_v4.h.
+ * GFW completion uses kflcnWaitForHalt_TU102 (Falcon CPUCTL), not RISC-V CPUCTL.
+ */
+#define GU_GFW_PLM_OFFSET 0x00118128u
+#define GU_GFW_BOOT_OFFSET 0x00118234u
+#define GU_GFW_CPUCTL_OFFSET 0x00110100u
+#define GU_GFW_TIMEOUT_US 2050000u
+#define GU_FEAT_PLM_OFFSET 0x00823804u
+#define GU_FEAT_SS0_OFFSET 0x0082381cu
+#define GU_FEAT_SS1_OFFSET 0x00823820u
+#define GU_GFW_PLM_LEVEL0 0x1u
+#define GU_GFW_BOOT_COMPLETED 0xffu
+#define GU_GFW_CPUCTL_HALTED 0x10u
+#define GU_PCI_COMMAND_IO 0x1u
+#define GU_PCI_COMMAND_MEMORY 0x2u
+#define GU_PCI_COMMAND_MASTER 0x4u
+
+struct gpu_unlock_gfw_sample {
+ bool config_ok;
+ u32 plm;
+ u32 boot;
+ u32 cpuctl;
+};
+
+/* 0 complete, -EAGAIN still running, -ENODEV inaccessible. */
+int gpu_unlock_gfw_assess(const struct gpu_unlock_gfw_sample *sample);
+
+/* Put memory and bus-master bits back. Leave every other command bit alone. */
+u16 gpu_unlock_command_restore(u16 original, u16 live);
+
+#endif
diff --git a/kernel/gpu_unlock/layout_610.c b/kernel/gpu_unlock/layout_610.c
new file mode 100644
index 0000000..0c91328
--- /dev/null
+++ b/kernel/gpu_unlock/layout_610.c
@@ -0,0 +1,108 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "diagnostic.h"
+#ifdef __KERNEL__
+#include <linux/errno.h>
+#else
+#include <errno.h>
+#endif
+
+#include "layout_610.h"
+
+static int align_down(u64 value, u64 align, u64 *out)
+{
+ if (!align || (align & (align - 1)) || value < align)
+ return GU_DIAG(-EINVAL);
+ *out = value & ~(align - 1);
+ return 0;
+}
+
+static int align_up(u64 value, u64 align, u64 *out)
+{
+ u64 down;
+
+ if (!align || (align & (align - 1)))
+ return GU_DIAG(-EINVAL);
+ if (value > ~(u64)0 - (align - 1))
+ return GU_DIAG(-EOVERFLOW);
+ if (align_down(value + align - 1, align, &down))
+ return GU_DIAG(-EINVAL);
+ *out = down;
+ return 0;
+}
+
+static int sub_u64(u64 value, u64 amount, u64 *out)
+{
+ if (value < amount)
+ return GU_DIAG(-ERANGE);
+ *out = value - amount;
+ return 0;
+}
+
+int gpu_unlock_layout(const struct gpu_unlock_layout_in *in,
+ struct gpu_unlock_layout_out *out)
+{
+ u64 reserved, wpr_end, frts, boot, gsp, nonwpr, meta, heap;
+ u64 heap_off, wpr_start, nonwpr_start;
+ int ret;
+
+ if (!in || !out)
+ return GU_DIAG(-EINVAL);
+ if (!in->usable_fb || !in->vga_base || !in->frts_size || !in->boot_size ||
+ !in->image_size)
+ return GU_DIAG(-EINVAL);
+ reserved = in->vga_base;
+ if (in->mmu_lock_valid && in->mmu_lock_low < reserved)
+ reserved = in->mmu_lock_low;
+ ret = align_down(reserved, 128ull * 1024, &wpr_end);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = sub_u64(wpr_end, in->frts_size, &frts);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = sub_u64(frts, in->boot_size, &boot);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = align_down(boot, 4096, &boot);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = sub_u64(boot, in->image_size, &gsp);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = align_down(gsp, 64ull * 1024, &gsp);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = align_up(in->nonwpr_size, 1024ull * 1024, &nonwpr);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = align_up(256, 1024ull * 1024, &meta);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = align_up(in->heap_request, 1024ull * 1024, &heap);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = sub_u64(gsp, heap, &heap_off);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = align_down(heap_off, 1024ull * 1024, &heap_off);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = sub_u64(heap_off, meta, &wpr_start);
+ if (ret)
+ return GU_DIAG(ret);
+ ret = sub_u64(wpr_start, nonwpr, &nonwpr_start);
+ if (ret)
+ return GU_DIAG(ret);
+ if (nonwpr_start >= in->usable_fb || wpr_end > in->usable_fb ||
+ wpr_start >= wpr_end || gsp >= boot || boot >= frts || frts >= wpr_end)
+ return GU_DIAG(-ERANGE);
+ out->reserved_top = reserved;
+ out->wpr_end = wpr_end;
+ out->frts_offset = frts;
+ out->boot_offset = boot;
+ out->gsp_offset = gsp;
+ out->heap_offset = heap_off;
+ out->heap_size = (gsp - heap_off) & ~((1ull << 20) - 1);
+ out->wpr_start = wpr_start;
+ out->nonwpr_start = nonwpr_start;
+ return 0;
+}
diff --git a/kernel/gpu_unlock/layout_610.h b/kernel/gpu_unlock/layout_610.h
new file mode 100644
index 0000000..67d9203
--- /dev/null
+++ b/kernel/gpu_unlock/layout_610.h
@@ -0,0 +1,44 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_LAYOUT_610_H
+#define GPU_UNLOCK_LAYOUT_610_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdbool.h>
+#include <stdint.h>
+#ifndef GPU_UNLOCK_U32_DEFINED
+#define GPU_UNLOCK_U32_DEFINED
+typedef uint32_t u32;
+typedef uint64_t u64;
+#endif
+#endif
+
+struct gpu_unlock_layout_in {
+ u64 usable_fb;
+ u64 vga_base;
+ bool mmu_lock_valid;
+ u64 mmu_lock_low;
+ u64 frts_size;
+ u64 boot_size;
+ u64 image_size;
+ u64 nonwpr_size;
+ u64 heap_request;
+};
+
+struct gpu_unlock_layout_out {
+ u64 reserved_top;
+ u64 wpr_end;
+ u64 frts_offset;
+ u64 boot_offset;
+ u64 gsp_offset;
+ u64 heap_offset;
+ u64 heap_size;
+ u64 wpr_start;
+ u64 nonwpr_start;
+};
+
+int gpu_unlock_layout(const struct gpu_unlock_layout_in *in,
+ struct gpu_unlock_layout_out *out);
+
+#endif
diff --git a/kernel/gpu_unlock/main.c b/kernel/gpu_unlock/main.c
new file mode 100644
index 0000000..c84abbc
--- /dev/null
+++ b/kernel/gpu_unlock/main.c
@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/async.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/string.h>
+
+#include "unlock_610.h"
+
+static char *targets;
+module_param(targets, charp, 0444);
+MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of RTX 3090 cards to unlock");
+
+#define RTX_3090 0x2204
+
+static ASYNC_DOMAIN_EXCLUSIVE(unlock_domain);
+static struct gpu_unlock_610_shared shared;
+
+/* Runs concurrently for each card; consumes the reference it is given. */
+static void unlock_one(void *data, async_cookie_t cookie)
+{
+ struct pci_dev *pdev = data;
+ int ret;
+
+ pci_dev_lock(pdev);
+ ret = pdev->driver ? -EBUSY : gpu_unlock_610_unlock(pdev, &shared);
+ pci_dev_unlock(pdev);
+ if (ret)
+ pci_err(pdev, "gpu_unlock: unlock failed (%d)\n", ret);
+ else
+ pci_info(pdev, "gpu_unlock: unlocked\n");
+ pci_dev_put(pdev);
+}
+
+/* True when the comma-separated targets list names this PCI device. */
+static bool listed(const char *name)
+{
+ const char *entry, *end;
+
+ for (entry = targets; *entry; entry = *end ? end + 1 : end) {
+ end = strchrnul(entry, ',');
+ if (strlen(name) == end - entry && !strncasecmp(name, entry, end - entry))
+ return true;
+ }
+ return false;
+}
+
+static void report_unmatched(void)
+{
+ const char *entry, *end;
+ struct pci_dev *pdev;
+
+ for (entry = targets; *entry; entry = *end ? end + 1 : end) {
+ end = strchrnul(entry, ',');
+ pdev = NULL;
+ while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, RTX_3090, pdev)))
+ if (strlen(pci_name(pdev)) == end - entry &&
+ !strncasecmp(pci_name(pdev), entry, end - entry))
+ break;
+ if (pdev)
+ pci_dev_put(pdev);
+ else
+ pr_err("gpu_unlock: no RTX 3090 at \"%.*s\"\n",
+ (int)(end - entry), entry);
+ }
+}
+
+static int __init gpu_unlock_init(void)
+{
+ struct pci_dev *pdev = NULL;
+ bool all;
+ int ret;
+
+ /* Loaded ahead of nvidia on every boot; the command line is the opt-in. */
+ if (!targets || !*targets)
+ return 0;
+ all = !strcmp(targets, "all");
+ if (!all)
+ report_unmatched();
+ while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, RTX_3090, pdev))) {
+ if (!all && !listed(pci_name(pdev)))
+ continue;
+ if (!shared.gsp) {
+ ret = gpu_unlock_610_load_shared(pdev, &shared);
+ if (ret) {
+ pci_err(pdev, "gpu_unlock: firmware load failed (%d)\n", ret);
+ pci_dev_put(pdev);
+ return 0;
+ }
+ }
+ async_schedule_domain(unlock_one, pci_dev_get(pdev), &unlock_domain);
+ }
+ /* nvidia loads once this returns: every card must be finished by then. */
+ async_synchronize_full_domain(&unlock_domain);
+ gpu_unlock_610_release_shared(&shared);
+ return 0;
+}
+
+static void __exit gpu_unlock_exit(void)
+{
+}
+
+module_init(gpu_unlock_init);
+module_exit(gpu_unlock_exit);
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("RTX 3090 pre-driver BF16 unlock");
+MODULE_VERSION("1.0.0");
diff --git a/kernel/gpu_unlock/prep_610.c b/kernel/gpu_unlock/prep_610.c
new file mode 100644
index 0000000..6013f57
--- /dev/null
+++ b/kernel/gpu_unlock/prep_610.c
@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "diagnostic.h"
+#ifdef __KERNEL__
+#include <linux/errno.h>
+#include <linux/string.h>
+#else
+#include <errno.h>
+#include <stdbool.h>
+#include <string.h>
+#endif
+
+#include "prep_610.h"
+
+u32 gpu_unlock_fuse_version(u32 raw)
+{
+ u32 index = 0;
+ u32 value = raw & 0xffffu;
+
+ if (!value)
+ return 0;
+ while (value > 1) {
+ value >>= 1;
+ index++;
+ }
+ return index + 1;
+}
+
+int gpu_unlock_booter_patch(u8 *image, u32 image_size,
+ const struct gpu_unlock_booter_hdr *hdr,
+ const u8 *signatures, u32 sig_bytes, u32 fuse_ver)
+{
+ u32 index, sig_size;
+
+ if (!image || !hdr || !signatures)
+ return GU_DIAG(-EINVAL);
+ if (hdr->num_apps != 1 || !hdr->num_sigs)
+ return GU_DIAG(-EINVAL);
+ if (sig_bytes != hdr->sig_bytes || sig_bytes % hdr->num_sigs)
+ return GU_DIAG(-EINVAL);
+ sig_size = sig_bytes / hdr->num_sigs;
+ if (!sig_size || image_size < hdr->patch_loc ||
+ image_size - hdr->patch_loc < sig_size)
+ return GU_DIAG(-EINVAL);
+ if (hdr->num_sigs > 1) {
+ if (fuse_ver >= hdr->num_sigs)
+ return GU_DIAG(-ERANGE);
+ index = hdr->num_sigs - 1 - fuse_ver;
+ } else {
+ index = 0;
+ }
+ memcpy(image + hdr->patch_loc, signatures + (size_t)index * sig_size,
+ sig_size);
+ return 0;
+}
+
+static u16 rd16(const u8 *p)
+{
+ return (u16)p[0] | ((u16)p[1] << 8);
+}
+
+static u32 rd32(const u8 *p)
+{
+ return (u32)p[0] | ((u32)p[1] << 8) | ((u32)p[2] << 16) | ((u32)p[3] << 24);
+}
+
+static u64 rd64(const u8 *p)
+{
+ return (u64)rd32(p) | ((u64)rd32(p + 4) << 32);
+}
+
+int gpu_unlock_elf_section(const u8 *elf, size_t length, const char *name,
+ struct gpu_unlock_span *span)
+{
+ u64 shoff;
+ u16 shentsize, shnum, shstrndx, i;
+ u32 name_off;
+ const u8 *shstr;
+ u64 shstr_off, shstr_size;
+ bool found = false;
+
+ if (!elf || !name || !span || length < 64)
+ return GU_DIAG(-EINVAL);
+ if (elf[0] != 0x7f || elf[1] != 'E' || elf[2] != 'L' || elf[3] != 'F')
+ return GU_DIAG(-EINVAL);
+ if (elf[4] != 2 || elf[5] != 1 || elf[6] != 1 ||
+ rd32(elf + 20) != 1 || rd16(elf + 52) != 64)
+ return GU_DIAG(-EINVAL);
+ shoff = rd64(elf + 40);
+ shentsize = rd16(elf + 58);
+ shnum = rd16(elf + 60);
+ shstrndx = rd16(elf + 62);
+ if (shentsize != 64 || !shnum || shstrndx >= shnum)
+ return GU_DIAG(-EINVAL);
+ if (shoff > length || (u64)shnum * shentsize > length - shoff)
+ return GU_DIAG(-EINVAL);
+ shstr_off = rd64(elf + shoff + (u64)shstrndx * shentsize + 24);
+ shstr_size = rd64(elf + shoff + (u64)shstrndx * shentsize + 32);
+ if (shstr_off > length || shstr_size > length - shstr_off)
+ return GU_DIAG(-EINVAL);
+ shstr = elf + shstr_off;
+ for (i = 0; i < shnum; i++) {
+ const u8 *sh = elf + shoff + (u64)i * shentsize;
+ u64 off = rd64(sh + 24);
+ u64 size = rd64(sh + 32);
+ size_t namelen = strlen(name);
+
+ name_off = rd32(sh);
+ if ((u64)name_off >= shstr_size)
+ return GU_DIAG(-EINVAL);
+ if (!memchr(shstr + name_off, 0, shstr_size - name_off))
+ return GU_DIAG(-EINVAL);
+ if (namelen >= shstr_size - name_off ||
+ memcmp(shstr + name_off, name, namelen) ||
+ shstr[name_off + namelen] != '\0')
+ continue;
+ if (found || off > length || size > length - off)
+ return GU_DIAG(-EINVAL);
+ span->offset = off;
+ span->size = size;
+ found = true;
+ }
+ return GU_DIAG(found ? 0 : -ENOENT);
+}
+
+int gpu_unlock_radix_plan(u64 size, struct gpu_unlock_radix_plan *plan)
+{
+ u64 pages[4];
+ unsigned int level;
+ u64 tables = 0;
+
+ if (!plan || !size)
+ return GU_DIAG(-EINVAL);
+ if (size > ~(u64)0 - (GU_RADIX_PAGE - 1))
+ return GU_DIAG(-EOVERFLOW);
+ memset(plan, 0, sizeof(*plan));
+ pages[3] = (size + GU_RADIX_PAGE - 1) >> 12;
+ for (level = 3; level > 0; level--)
+ pages[level - 1] = ((pages[level] - 1) >> 9) + 1;
+ if (pages[0] != 1)
+ return GU_DIAG(-ERANGE);
+ for (level = 0; level < 3; level++)
+ tables += pages[level];
+ plan->pages[0] = pages[0];
+ plan->pages[1] = pages[1];
+ plan->pages[2] = pages[2];
+ plan->pages[3] = pages[3];
+ plan->table_pages = tables;
+ plan->table_bytes = tables << 12;
+ plan->data_bytes = pages[3] << 12;
+ return 0;
+}
+
+static void wr64(u8 *p, u64 value)
+{
+ unsigned int i;
+
+ for (i = 0; i < 8; i++)
+ p[i] = (u8)(value >> (8 * i));
+}
+
+int gpu_unlock_boot_desc_parse(const u8 *data, size_t size, u32 image_size,
+ struct gpu_unlock_boot_desc *out)
+{
+ static const unsigned int ranges[] = { 1, 3, 8, 10, 12 };
+ unsigned int i;
+
+ if (!data || !out || size != 84 || !image_size ||
+ rd32(data) != 5 || rd32(data + 56) != 1 ||
+ rd32(data + 76) != image_size || rd32(data + 80))
+ return GU_DIAG(-EINVAL);
+ for (i = 0; i < sizeof(ranges) / sizeof(ranges[0]); i++) {
+ u32 off = rd32(data + ranges[i] * 4);
+ u32 len = rd32(data + (ranges[i] + 1) * 4);
+
+ if (!len || off > image_size || len > image_size - off)
+ return GU_DIAG(-ERANGE);
+ }
+ /* This path doesn't boot a secure partition, load ITCM or enable SMP. */
+ for (i = 5; i <= 7; i++)
+ if (rd32(data + i * 4))
+ return GU_DIAG(-EOPNOTSUPP);
+ for (i = 15; i <= 18; i++)
+ if (rd32(data + i * 4))
+ return GU_DIAG(-EOPNOTSUPP);
+ out->manifest_offset = rd32(data + 32);
+ out->data_offset = rd32(data + 40);
+ out->code_offset = rd32(data + 48);
+ out->image_size = image_size;
+ return 0;
+}
+
+int gpu_unlock_radix_page(u8 out[GU_RADIX_PAGE], const u64 *children, size_t count)
+{
+ size_t i;
+
+ if (!out || !children || !count || count > GU_RADIX_PAGE / 8)
+ return GU_DIAG(-EINVAL);
+ for (i = 0; i < count; i++)
+ if (children[i] & (GU_RADIX_PAGE - 1))
+ return GU_DIAG(-EINVAL);
+ memset(out, 0, GU_RADIX_PAGE);
+ for (i = 0; i < count; i++)
+ wr64(out + i * 8, children[i]);
+ return 0;
+}
+
+int gpu_unlock_wpr_meta_pack(u8 out[GU_WPR_META_SIZE],
+ const struct gpu_unlock_layout_out *layout,
+ const struct gpu_unlock_wpr_args *a)
+{
+ if (!out || !layout || !a || !a->signature_size || !a->image_size ||
+ !a->boot.image_size || !a->fb_size || !a->frts_size ||
+ a->vga_offset >= a->fb_size ||
+ a->boot.code_offset >= a->boot.image_size ||
+ a->boot.data_offset >= a->boot.image_size ||
+ a->boot.manifest_offset >= a->boot.image_size ||
+ (a->radix_dma & 4095) || (a->boot_dma & 4095) ||
+ layout->nonwpr_start >= layout->wpr_start ||
+ layout->wpr_start >= layout->heap_offset ||
+ layout->heap_offset >= layout->gsp_offset ||
+ layout->heap_size > layout->gsp_offset - layout->heap_offset ||
+ layout->gsp_offset >= layout->boot_offset ||
+ a->image_size > layout->boot_offset - layout->gsp_offset ||
+ layout->boot_offset >= layout->frts_offset ||
+ a->boot.image_size > layout->frts_offset - layout->boot_offset ||
+ layout->frts_offset >= layout->wpr_end ||
+ a->frts_size != layout->wpr_end - layout->frts_offset ||
+ layout->wpr_end > a->vga_offset)
+ return GU_DIAG(-EINVAL);
+ memset(out, 0, GU_WPR_META_SIZE);
+ wr64(out + 0, GU_WPR_META_MAGIC);
+ wr64(out + 8, 1);
+ wr64(out + 16, a->radix_dma);
+ wr64(out + 24, a->image_size);
+ wr64(out + 32, a->boot_dma);
+ wr64(out + 40, a->boot.image_size);
+ wr64(out + 48, a->boot.code_offset);
+ wr64(out + 56, a->boot.data_offset);
+ wr64(out + 64, a->boot.manifest_offset);
+ wr64(out + 72, a->signature_dma);
+ wr64(out + 80, a->signature_size);
+ wr64(out + 88, layout->nonwpr_start);
+ wr64(out + 96, layout->nonwpr_start);
+ wr64(out + 104, layout->wpr_start - layout->nonwpr_start);
+ wr64(out + 112, layout->wpr_start);
+ wr64(out + 120, layout->heap_offset);
+ wr64(out + 128, layout->heap_size);
+ wr64(out + 136, layout->gsp_offset);
+ wr64(out + 144, layout->boot_offset);
+ wr64(out + 152, layout->frts_offset);
+ wr64(out + 160, a->frts_size);
+ wr64(out + 168, layout->wpr_end);
+ wr64(out + 176, a->fb_size);
+ wr64(out + 184, a->vga_offset);
+ wr64(out + 192, a->fb_size - a->vga_offset);
+ return 0;
+}
+
+u32 gpu_unlock_dma_cmd(bool imem)
+{
+ u32 cmd = 6u << 8;
+
+ if (imem)
+ cmd |= (1u << 4) | (1u << 2);
+ return cmd;
+}
diff --git a/kernel/gpu_unlock/prep_610.h b/kernel/gpu_unlock/prep_610.h
new file mode 100644
index 0000000..273f345
--- /dev/null
+++ b/kernel/gpu_unlock/prep_610.h
@@ -0,0 +1,97 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_PREP_610_H
+#define GPU_UNLOCK_PREP_610_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+typedef uint8_t u8;
+typedef uint16_t u16;
+#define GPU_UNLOCK_U32_DEFINED
+typedef uint32_t u32;
+typedef uint64_t u64;
+#endif
+
+#include "layout_610.h"
+
+#define GU_BOOTER_IMAGE_SIZE 60416u
+#define GU_BOOTER_SIG_TOTAL 768u
+#define GU_V67_SIZE 0xfa00u
+#define GU_WPR_META_SIZE 256u
+#define GU_WPR_META_MAGIC 0xdc3aae21371a60b3ull
+#define GU_RADIX_PAGE 4096u
+#define GU_FUSE_SEC2_UCODE1 0x00824140u
+
+struct gpu_unlock_booter_hdr {
+ u32 os_code_off;
+ u32 os_code_size;
+ u32 os_data_off;
+ u32 os_data_size;
+ u32 num_apps;
+ u32 app_code_off;
+ u32 app_code_size;
+ u32 app_data_off;
+ u32 app_data_size;
+ u32 patch_loc;
+ u32 ucode_id;
+ u32 engine_id;
+ u32 num_sigs;
+ u32 sig_bytes;
+};
+
+struct gpu_unlock_span {
+ u64 offset;
+ u64 size;
+};
+
+struct gpu_unlock_radix_plan {
+ u64 pages[4];
+ u64 table_pages;
+ u64 table_bytes;
+ u64 data_bytes;
+};
+
+/* Version-5 production boot descriptor, decoded from the pinned wire format. */
+struct gpu_unlock_boot_desc {
+ u32 code_offset;
+ u32 data_offset;
+ u32 manifest_offset;
+ u32 image_size;
+};
+
+struct gpu_unlock_wpr_args {
+ u64 fb_size, vga_offset, frts_size;
+ u64 signature_dma, signature_size;
+ u64 radix_dma, image_size;
+ u64 boot_dma;
+ struct gpu_unlock_boot_desc boot;
+};
+
+int gpu_unlock_boot_desc_parse(const u8 *data, size_t size, u32 image_size,
+ struct gpu_unlock_boot_desc *out);
+
+/* One table page; children are device DMA addresses, never CPU pointers. */
+int gpu_unlock_radix_page(u8 out[GU_RADIX_PAGE], const u64 *children, size_t count);
+
+/* SEC2 fuse register value to the Booter signature index input. */
+u32 gpu_unlock_fuse_version(u32 raw);
+
+int gpu_unlock_booter_patch(u8 *image, u32 image_size,
+ const struct gpu_unlock_booter_hdr *hdr,
+ const u8 *signatures, u32 sig_bytes, u32 fuse_ver);
+
+int gpu_unlock_elf_section(const u8 *elf, size_t length, const char *name,
+ struct gpu_unlock_span *span);
+
+int gpu_unlock_radix_plan(u64 size, struct gpu_unlock_radix_plan *plan);
+
+int gpu_unlock_wpr_meta_pack(u8 out[GU_WPR_META_SIZE],
+ const struct gpu_unlock_layout_out *layout,
+ const struct gpu_unlock_wpr_args *args);
+
+u32 gpu_unlock_dma_cmd(bool imem);
+
+#endif
diff --git a/kernel/gpu_unlock/unlock_610.c b/kernel/gpu_unlock/unlock_610.c
new file mode 100644
index 0000000..79c39cc
--- /dev/null
+++ b/kernel/gpu_unlock/unlock_610.c
@@ -0,0 +1,1421 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Pre-driver V67 unlock, run before the stock driver binds: FWSEC-FRTS from
+ * the board ROM, Booter with the V67 signature, the SS0/SS1 selector writes, then
+ * FLR so the stock driver starts from a clean, halted GSP.
+ */
+#include <linux/aperture.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/firmware.h>
+#include <linux/io.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+#include <linux/timekeeping.h>
+
+#include "diagnostic.h"
+#include "gfw_eval.h"
+#include "prep_610.h"
+#include "unlock_610.h"
+#include "vbios_610.h"
+
+#define SEC2_BASE 0x00840000u
+#define SEC2_FBIF 0x00840600u
+#define SEC2_FALCON2 0x00841000u
+/* SS0 holds eight 4-bit fields; SS1 stores only its low 4 bits (0x2 when
+ * cold). Both get the same per-field value 8. */
+#define GU_SS0_UNLOCK 0x88888888u
+#define GU_SS1_UNLOCK 0x8u
+#define BOOTER_MAGIC 0x314c5442u
+#define BOOTER_FILE "gpu-unlock/610.43.03/booter_load_ga102.bin"
+#define V67_FILE "gpu-unlock/610.43.03/v67-ga102-610.43.03.bin"
+#define GSP_FILE "gpu-unlock/610.43.03/gsp_ga10x.bin"
+#define BOOT_FILE "gpu-unlock/610.43.03/gsp_boot_ga102.bin"
+#define BOOT_DESC_FILE "gpu-unlock/610.43.03/gsp_boot_desc_ga102.bin"
+
+struct unlock_transaction_page {
+ void *cpu;
+ dma_addr_t dma;
+};
+
+struct unlock_transaction_work {
+ struct pci_dev *pdev;
+ void __iomem *bar;
+ resource_size_t bar_len;
+ u16 original_cmd;
+ bool cmd_valid;
+ bool enabled;
+ bool region;
+ bool mapped;
+ bool master;
+ const struct firmware *booter_fw;
+ const struct firmware *v67_fw;
+ const struct firmware *gsp_fw; /* borrowed from gpu_unlock_610_shared */
+ const struct firmware *boot_fw;
+ struct gpu_unlock_span image_span, sig_span;
+ struct gpu_unlock_layout_in layout_in;
+ struct gpu_unlock_layout_out layout;
+ struct gpu_unlock_boot_desc boot_desc;
+ struct gpu_unlock_radix_plan radix;
+ struct unlock_transaction_page *pages;
+ u32 page_count, pages_allocated;
+ void *boot_cpu;
+ dma_addr_t boot_dma;
+ void *booter_cpu;
+ dma_addr_t booter_dma;
+ u32 booter_image_size;
+ struct gpu_unlock_booter_hdr booter_hdr;
+ void *v67_cpu;
+ dma_addr_t v67_dma;
+ void *stock_cpu;
+ dma_addr_t stock_dma;
+ u64 stock_size;
+ void *meta_cpu;
+ dma_addr_t meta_dma;
+ u64 *streaming_ptr;
+ u64 streaming_before;
+ u64 coherent_before;
+ bool streaming_set;
+ bool coherent_set;
+ unsigned int signature_calls;
+ bool published;
+ bool masks_saved;
+ bool firmware_started;
+ bool quiesced;
+ bool retained;
+ u32 plm_entry;
+ u32 bars[6];
+ void *fwsec_cpu;
+ dma_addr_t fwsec_dma;
+ u32 fwsec_size;
+ u32 last_offset, last_value;
+ bool last_valid;
+ u32 last_pci_offset, last_pci_value;
+ int last_pci_error;
+ bool last_pci_cap;
+};
+
+static int unlock_transaction_result(struct unlock_transaction_work *work, int error,
+ const char *function, unsigned int line)
+{
+ if (error)
+ {
+ dev_err(&work->pdev->dev,
+ "failure unlock_610.c:%u %s error=%d last_mmio=%#x value=%#x valid=%u mapped=%u master=%u firmware_started=%u quiesced=%u pages=%u/%u\n",
+ line, function, error, work->last_offset, work->last_value,
+ work->last_valid, work->mapped, work->master, work->firmware_started,
+ work->quiesced, work->pages_allocated, work->page_count);
+ dev_err(&work->pdev->dev, "last PCI sample: capability=%u offset=%#x value=%#x result=%d\n",
+ work->last_pci_cap, work->last_pci_offset, work->last_pci_value, work->last_pci_error);
+ }
+ return error;
+}
+#define GU_RESULT(result) unlock_transaction_result(work, (result), __func__, __LINE__)
+
+static int diag_pci_word(struct unlock_transaction_work *work, bool cap, int offset, u16 *value)
+{
+ int ret;
+ *value = 0xffff;
+ ret = cap ? pcie_capability_read_word(work->pdev, offset, value) :
+ pci_read_config_word(work->pdev, offset, value);
+ work->last_pci_cap = cap;
+ work->last_pci_offset = offset;
+ work->last_pci_value = *value;
+ work->last_pci_error = ret;
+ return ret;
+}
+
+static int diag_pci_dword(struct unlock_transaction_work *work, bool cap, int offset, u32 *value)
+{
+ int ret;
+ *value = ~0u;
+ ret = cap ? pcie_capability_read_dword(work->pdev, offset, value) :
+ pci_read_config_dword(work->pdev, offset, value);
+ work->last_pci_cap = cap;
+ work->last_pci_offset = offset;
+ work->last_pci_value = *value;
+ work->last_pci_error = ret;
+ return ret;
+}
+
+static void *diag_dma_alloc(struct unlock_transaction_work *work, const char *name,
+ size_t size, dma_addr_t *dma, unsigned int flags)
+{
+ void *memory = dma_alloc_coherent(&work->pdev->dev, size, dma, flags);
+ if (!memory)
+ dev_err(&work->pdev->dev, "DMA allocation failed: asset=%s bytes=%zu pages_allocated=%u requested=%u\n",
+ name, size, work->pages_allocated, work->page_count);
+ return memory;
+}
+
+/* CONFIG_FW_CACHE attaches name records to devres even though this is not
+ * devm_request_firmware(). release_firmware() does not remove them, and
+ * really_probe() refuses a device with any existing devres. Scope each
+ * synchronous request so only its cache records are released; the returned
+ * firmware remains owned by the caller.
+ */
+static int request_scoped(struct device *dev, const struct firmware **fw, const char *name)
+{
+ void *group;
+ int ret, released;
+
+ group = devres_open_group(dev, NULL, GFP_KERNEL);
+ if (!group)
+ return -ENOMEM;
+ dev_info(dev, "asset request: %s\n", name);
+ ret = request_firmware(fw, name, dev);
+ released = devres_release_group(dev, group);
+ dev_info(dev, "asset result: %s error=%d bytes=%zu\n",
+ name, ret, ret ? (size_t)0 : (*fw)->size);
+ dev_info(dev, "asset cache cleanup: %s devres_released=%d\n", name, released);
+ return ret;
+}
+
+static int request_asset(struct unlock_transaction_work *work, const struct firmware **fw,
+ const char *name)
+{
+ return GU_RESULT(request_scoped(&work->pdev->dev, fw, name));
+}
+
+static int check_devres_empty(struct unlock_transaction_work *work, const char *phase)
+{
+ struct device *dev = &work->pdev->dev;
+ unsigned long flags;
+ bool empty;
+
+ /* The caller also holds the device lock across the whole transaction. */
+ spin_lock_irqsave(&dev->devres_lock, flags);
+ empty = list_empty(&dev->devres_head);
+ spin_unlock_irqrestore(&dev->devres_lock, flags);
+ dev_info(dev, "driver probe resources: phase=%s empty=%u\n", phase, empty);
+ return GU_RESULT(empty ? 0 : -EBUSY);
+}
+
+static int read_bar(struct unlock_transaction_work *work, u32 offset, u32 *value)
+{
+ work->last_offset = offset;
+ work->last_valid = false;
+ if (!work->mapped || offset > work->bar_len || work->bar_len - offset < 4)
+ return GU_RESULT(-EINVAL);
+ *value = readl(work->bar + offset);
+ work->last_value = *value;
+ work->last_valid = true;
+ return 0;
+}
+
+static void write_bar(struct unlock_transaction_work *work, u32 offset, u32 value)
+{
+ writel(value, work->bar + offset);
+}
+
+static int poll_us(struct unlock_transaction_work *work, u32 offset, u32 mask, u32 expect,
+ unsigned int budget_us, bool timeout_is_failure)
+{
+ ktime_t deadline = ktime_add_us(ktime_get(), budget_us);
+ u32 value = 0;
+
+ for (;;) {
+ if (read_bar(work, offset, &value) || value == ~0u ||
+ (value & 0xffff0000u) == 0xbadf0000u) {
+ dev_err(&work->pdev->dev, "poll inaccessible: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return GU_RESULT(-EIO);
+ }
+ if ((value & mask) == expect)
+ return 0;
+ if (ktime_compare(ktime_get(), deadline) >= 0) {
+ /* Console loglevel hides info. A tolerated timeout must not
+ * look like the failure that stopped the boot. */
+ if (!timeout_is_failure) {
+ dev_info(&work->pdev->dev, "poll timeout tolerated: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return -ETIMEDOUT;
+ }
+ dev_err(&work->pdev->dev, "poll timeout: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return GU_RESULT(-ETIMEDOUT);
+ }
+ usleep_range(50, 100);
+ }
+}
+
+struct falcon_eng {
+ u32 base;
+ u32 fbif;
+ u32 falcon2;
+};
+
+static bool inaccessible(u32 value)
+{
+ return value == ~0u || (value & 0xffff0000u) == 0xbadf0000u;
+}
+
+static int falcon_reset(struct unlock_transaction_work *work, const struct falcon_eng *eng)
+{
+ u32 engine, hwcfg, bcr, chip;
+ unsigned int i;
+ int ret;
+
+ /* 610 kflcnPreResetWait_GA102 permits this timeout (HW bug 3419321). */
+ dev_info(&work->pdev->dev, "Falcon reset begin: base=%#x\n", eng->base);
+ ret = poll_us(work, eng->base + 0xf4, 1u << 31, 1u << 31, 1000, false);
+ if (ret && ret != -ETIMEDOUT)
+ return GU_RESULT(ret);
+ if (ret)
+ dev_info(&work->pdev->dev, "Falcon pre-reset timeout tolerated per HW bug 3419321\n");
+ if (read_bar(work, eng->base + 0x3c0, &engine) || inaccessible(engine))
+ return GU_RESULT(-ENODEV);
+ write_bar(work, eng->base + 0x3c0, engine | 1);
+ for (i = 0; i < 10; i++)
+ readl(work->bar + eng->base + 0x3c0);
+ write_bar(work, eng->base + 0x3c0, engine & ~1u);
+ for (i = 0; i < 10; i++)
+ readl(work->bar + eng->base + 0x3c0);
+ ret = poll_us(work, eng->base + 0xf4, 1u << 12, 0, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, eng->base + 0xf4, &hwcfg) || inaccessible(hwcfg))
+ return GU_RESULT(-EIO);
+ if (hwcfg & (1u << 10)) {
+ if (read_bar(work, eng->falcon2 + 0x668, &bcr) || inaccessible(bcr))
+ return GU_RESULT(-ENODEV);
+ if (bcr & (1u << 4)) {
+ write_bar(work, eng->falcon2 + 0x668, 0);
+ ret = poll_us(work, eng->falcon2 + 0x668, 0x11, 1, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ }
+ }
+ if (read_bar(work, 0, &chip) || inaccessible(chip))
+ return GU_RESULT(-ENODEV);
+ write_bar(work, eng->base + 0x84, chip);
+ dev_info(&work->pdev->dev, "Falcon reset done: base=%#x hwcfg=%#x chip=%#x\n", eng->base, hwcfg, chip);
+ return 0;
+}
+
+/* DMATRFMOFFS is 24 bits, FBOFFS is 32 bits, and BASE is 49 bits.
+ * Reject truncation/wrap before submitting any part of a transfer. */
+static bool falcon_dma_valid(u32 dest, u32 mem_off, dma_addr_t src, u32 size)
+{
+ return size && !((size | dest | mem_off | src) & 255) &&
+ (u64)dest + size <= (1ull << 24) &&
+ (u64)mem_off + size <= (1ull << 32) &&
+ src < (1ull << 49) &&
+ (u64)mem_off + size <= (1ull << 49) - src;
+}
+
+static int falcon_dma(struct unlock_transaction_work *work, u32 base, u32 dest, u32 mem_off,
+ dma_addr_t src, u32 size, u32 cmd)
+{
+ u32 done = 0;
+ int ret;
+
+ dev_info(&work->pdev->dev, "Falcon DMA: base=%#x dest=%#x offset=%#x src=%#llx size=%#x cmd=%#x\n",
+ base, dest, mem_off, (unsigned long long)src, size, cmd);
+ if (!falcon_dma_valid(dest, mem_off, src, size))
+ return GU_RESULT(-EINVAL);
+ ret = poll_us(work, base + 0x118, 1, 0, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ write_bar(work, base + 0x110, lower_32_bits(src >> 8));
+ write_bar(work, base + 0x128, upper_32_bits(src >> 8) & 0x1ff);
+ while (done < size) {
+ ret = poll_us(work, base + 0x118, 1, 0, 1000000, true);
+ if (ret) {
+ dev_err(&work->pdev->dev, "Falcon DMA progress: done=%u total=%u\n", done, size);
+ return GU_RESULT(ret);
+ }
+ write_bar(work, base + 0x114, dest);
+ write_bar(work, base + 0x11c, mem_off);
+ write_bar(work, base + 0x118, cmd);
+ done += 256;
+ dest += 256;
+ mem_off += 256;
+ }
+ return GU_RESULT(poll_us(work, base + 0x118, 2, 2, 1000000, true));
+}
+
+static int falcon_hs(struct unlock_transaction_work *work, const struct falcon_eng *eng,
+ u32 code_off, u32 code_size, u32 code_va, u32 code_pa,
+ u32 data_off, u32 data_size, u32 data_pa,
+ u32 sig_dmem, u32 engine_mask, u32 ucode_id,
+ dma_addr_t image, u32 mailbox0, u32 mailbox1,
+ bool wait_plm)
+{
+ u32 trans, cpuctl;
+ dma_addr_t code_src;
+ int ret;
+
+ if (image > ~0ull - code_off || image + code_off < code_va ||
+ image > ~0ull - data_off)
+ return GU_RESULT(-ERANGE);
+ code_src = image + code_off - code_va;
+ if (!falcon_dma_valid(code_pa, code_va, code_src, code_size) ||
+ (data_size && !falcon_dma_valid(data_pa, 0, image + data_off, data_size)))
+ return GU_RESULT(-EINVAL);
+ ret = falcon_reset(work, eng);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, eng->fbif + 0x24, &trans) || inaccessible(trans))
+ return GU_RESULT(-EIO);
+ trans |= 1u << 7;
+ write_bar(work, eng->fbif + 0x24, trans);
+ write_bar(work, eng->base + 0x10c, 0);
+ if (read_bar(work, eng->fbif, &trans) || inaccessible(trans))
+ return GU_RESULT(-ENODEV);
+ trans = (trans & ~7u) | 5u;
+ write_bar(work, eng->fbif, trans);
+ dma_wmb();
+ ret = falcon_dma(work, eng->base, code_pa, code_va, code_src,
+ code_size, gpu_unlock_dma_cmd(true));
+ if (ret)
+ return GU_RESULT(ret);
+ if (data_size) {
+ ret = falcon_dma(work, eng->base, data_pa, 0, image + data_off, data_size,
+ gpu_unlock_dma_cmd(false));
+ if (ret)
+ return GU_RESULT(ret);
+ }
+ write_bar(work, eng->falcon2 + 0x210, sig_dmem);
+ write_bar(work, eng->falcon2 + 0x19c, engine_mask);
+ write_bar(work, eng->falcon2 + 0x198, ucode_id & 0xff);
+ write_bar(work, eng->falcon2 + 0x180, 1);
+ write_bar(work, eng->base + 0x104, code_va);
+ write_bar(work, eng->base + 0x40, mailbox0);
+ write_bar(work, eng->base + 0x44, mailbox1);
+ if (read_bar(work, eng->base + 0x100, &cpuctl) || inaccessible(cpuctl))
+ return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev, "Falcon start: base=%#x cpuctl=%#x code_va=%#x sig_dmem=%#x engine_mask=%#x ucode=%u mailbox=%#x:%#x\n",
+ eng->base, cpuctl, code_va, sig_dmem, engine_mask, ucode_id, mailbox1, mailbox0);
+ if (cpuctl & (1u << 6))
+ write_bar(work, eng->base + 0x130, 1u << 1);
+ else
+ write_bar(work, eng->base + 0x100, 1u << 1);
+ if (wait_plm) {
+ ktime_t deadline = ktime_add_us(ktime_get(), 5000000);
+
+ for (;;) {
+ u32 plm, engine, identity;
+
+ if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
+ identity != 0x220410de ||
+ read_bar(work, eng->base + 0x100, &engine) || engine == ~0u ||
+ (engine & 0xffff0000u) == 0xbadf0000u ||
+ read_bar(work, GU_FEAT_PLM_OFFSET, &plm))
+ return GU_RESULT(-ENODEV);
+ /* Cold entry was closed: this transition proves payload execution. */
+ if (plm == ~0u && work->plm_entry != ~0u)
+ return 0;
+ if ((engine & (1u << 4)) || ktime_compare(ktime_get(), deadline) >= 0)
+ dev_err(&work->pdev->dev, "early PLM failed: pci=%#x cpuctl=%#x plm=%#x entry=%#x budget_us=5000000\n",
+ identity, engine, plm, work->plm_entry);
+ if (engine & (1u << 4))
+ return GU_RESULT(-EIO);
+ if (ktime_compare(ktime_get(), deadline) >= 0)
+ return GU_RESULT(-ETIMEDOUT);
+ usleep_range(50, 100);
+ }
+ }
+ ret = poll_us(work, eng->base + 0x100, 1u << 4, 1u << 4, 5000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ return 0;
+}
+
+static int load_booter(struct unlock_transaction_work *work)
+{
+ const u8 *data;
+ u32 magic, version, image_size, sig_bytes;
+ struct gpu_unlock_booter_hdr hdr;
+ int ret;
+
+ ret = request_asset(work, &work->booter_fw, BOOTER_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ data = work->booter_fw->data;
+ if (work->booter_fw->size < 72)
+ return GU_RESULT(-EINVAL);
+ memcpy(&magic, data, sizeof(magic));
+ memcpy(&version, data + 4, sizeof(version));
+ if (magic != BOOTER_MAGIC || version != 1)
+ return GU_RESULT(-EINVAL);
+ memcpy(&hdr, data + 8, sizeof(hdr));
+ memcpy(&image_size, data + 8 + sizeof(hdr), sizeof(image_size));
+ memcpy(&sig_bytes, data + 12 + sizeof(hdr), sizeof(sig_bytes));
+ if (!hdr.ucode_id || hdr.num_apps != 1 || sig_bytes != hdr.sig_bytes ||
+ image_size != GU_BOOTER_IMAGE_SIZE ||
+ work->booter_fw->size != 72u + image_size + sig_bytes)
+ return GU_RESULT(-EINVAL);
+ /* Only this pinned production BOOT_FROM_HS geometry is implemented. */
+ if (hdr.ucode_id != 3 || hdr.engine_id != 1 || hdr.num_sigs != 2 ||
+ sig_bytes != GU_BOOTER_SIG_TOTAL || hdr.os_code_off != 0 ||
+ hdr.os_code_size != 256 || hdr.os_data_off != 35328 ||
+ hdr.os_data_size != 25088 || hdr.app_code_off != 256 ||
+ hdr.app_code_size != 35072 || hdr.app_data_off != 256 ||
+ hdr.app_data_size != 0 || hdr.patch_loc != 35344)
+ return GU_RESULT(-EINVAL);
+ work->booter_hdr = hdr;
+ work->booter_image_size = image_size;
+ /* Allocation follows the ownership and DMA-mask snapshot, not asset lookup. */
+ return 0;
+}
+
+static int patch_booter(struct unlock_transaction_work *work)
+{
+ const u8 *data = work->booter_fw->data;
+ u32 fuse_raw, fuse_ver;
+ u32 index = work->booter_hdr.ucode_id - 1;
+
+ if (read_bar(work, GU_FUSE_SEC2_UCODE1 + index * 4, &fuse_raw))
+ return GU_RESULT(-EIO);
+ fuse_ver = gpu_unlock_fuse_version(fuse_raw);
+ dev_info(&work->pdev->dev, "Booter signature: fuse_raw=%#x version=%u signatures=%u size=%u\n",
+ fuse_raw, fuse_ver, work->booter_hdr.num_sigs, work->booter_hdr.sig_bytes);
+ return GU_RESULT(gpu_unlock_booter_patch(work->booter_cpu, work->booter_image_size,
+ &work->booter_hdr,
+ data + 72 + work->booter_image_size,
+ work->booter_hdr.sig_bytes, fuse_ver));
+}
+
+static void free_coherent(struct unlock_transaction_work *work)
+{
+ u32 i;
+
+ for (i = 0; i < work->pages_allocated; i++)
+ dma_free_coherent(&work->pdev->dev, GU_RADIX_PAGE,
+ work->pages[i].cpu, work->pages[i].dma);
+ kvfree(work->pages);
+ work->pages = NULL;
+ work->pages_allocated = 0;
+ if (work->boot_cpu)
+ dma_free_coherent(&work->pdev->dev, work->boot_desc.image_size,
+ work->boot_cpu, work->boot_dma);
+ work->boot_cpu = NULL;
+ if (work->fwsec_cpu) {
+ dma_free_coherent(&work->pdev->dev, work->fwsec_size,
+ work->fwsec_cpu, work->fwsec_dma);
+ work->fwsec_cpu = NULL;
+ }
+ if (work->booter_cpu)
+ dma_free_coherent(&work->pdev->dev, GU_BOOTER_IMAGE_SIZE,
+ work->booter_cpu, work->booter_dma);
+ if (work->v67_cpu)
+ dma_free_coherent(&work->pdev->dev, GU_V67_SIZE, work->v67_cpu,
+ work->v67_dma);
+ if (work->stock_cpu)
+ dma_free_coherent(&work->pdev->dev, work->stock_size,
+ work->stock_cpu, work->stock_dma);
+ if (work->meta_cpu)
+ dma_free_coherent(&work->pdev->dev, GU_WPR_META_SIZE,
+ work->meta_cpu, work->meta_dma);
+ work->booter_cpu = NULL;
+ work->v67_cpu = NULL;
+ work->stock_cpu = NULL;
+ work->meta_cpu = NULL;
+}
+
+static int unlock_transaction_unwind(struct unlock_transaction_work *work)
+{
+ u16 now, want;
+ int ret = 0;
+
+ /* A timeout never authorizes freeing firmware-visible memory. */
+ if (work->firmware_started && !work->quiesced) {
+ work->retained = true;
+ return GU_RESULT(-EBUSY);
+ }
+ if (work->master) {
+ pci_clear_master(work->pdev);
+ work->master = false;
+ }
+ if (work->mapped) {
+ pci_iounmap(work->pdev, work->bar);
+ work->bar = NULL;
+ work->mapped = false;
+ }
+ if (work->region) {
+ pci_release_region(work->pdev, 0);
+ work->region = false;
+ }
+ if (work->enabled) {
+ pci_disable_device(work->pdev);
+ work->enabled = false;
+ }
+ free_coherent(work);
+ if (work->booter_fw) {
+ release_firmware(work->booter_fw);
+ work->booter_fw = NULL;
+ }
+ if (work->v67_fw) {
+ release_firmware(work->v67_fw);
+ work->v67_fw = NULL;
+ }
+ if (work->boot_fw) {
+ release_firmware(work->boot_fw);
+ work->boot_fw = NULL;
+ }
+ /* Match the driver core's entry condition, not just BAR/mask cleanup.
+ * Never release unrelated resources to make this check pass.
+ */
+ if (check_devres_empty(work, "handoff"))
+ ret = -EBUSY;
+ if (work->streaming_set && work->streaming_ptr) {
+ if (work->pdev->dev.dma_mask != work->streaming_ptr)
+ return GU_RESULT(-EIO);
+ if (dma_set_mask(&work->pdev->dev, work->streaming_before) ||
+ *work->streaming_ptr != work->streaming_before)
+ ret = -EIO;
+ dev_info(&work->pdev->dev, "restore streaming mask: actual=%#llx expected=%#llx\n",
+ (unsigned long long)*work->streaming_ptr, (unsigned long long)work->streaming_before);
+ }
+ if (work->coherent_set) {
+ if (dma_set_coherent_mask(&work->pdev->dev, work->coherent_before) ||
+ work->pdev->dev.coherent_dma_mask != work->coherent_before)
+ ret = -EIO;
+ dev_info(&work->pdev->dev, "restore coherent mask: actual=%#llx expected=%#llx\n",
+ (unsigned long long)work->pdev->dev.coherent_dma_mask,
+ (unsigned long long)work->coherent_before);
+ }
+ if (!work->cmd_valid)
+ return GU_RESULT(ret);
+ if (diag_pci_word(work, false, PCI_COMMAND, &now) || now == 0xffff)
+ return GU_RESULT(-ENODEV);
+ want = gpu_unlock_command_restore(work->original_cmd, now);
+ if (want != now && pci_write_config_word(work->pdev, PCI_COMMAND, want))
+ return GU_RESULT(-EIO);
+ if (diag_pci_word(work, false, PCI_COMMAND, &now))
+ return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev, "restore PCI command: actual=%#x original=%#x mask=%#x\n",
+ now, work->original_cmd, GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER);
+ if ((now & (GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER)) !=
+ (work->original_cmd & (GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER)))
+ return GU_RESULT(-EIO);
+ return GU_RESULT(ret);
+}
+
+static int unlock_transaction_assets(struct unlock_transaction_work *work)
+{
+ const struct firmware *desc;
+ int ret;
+
+ ret = check_devres_empty(work, "entry");
+ if (ret)
+ return ret;
+ ret = request_asset(work, &work->v67_fw, V67_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ if (work->v67_fw->size != GU_V67_SIZE)
+ return GU_RESULT(-EBADMSG);
+ ret = gpu_unlock_elf_section(work->gsp_fw->data, work->gsp_fw->size,
+ ".fwimage", &work->image_span);
+ if (ret)
+ return GU_RESULT(ret);
+ ret = gpu_unlock_elf_section(work->gsp_fw->data, work->gsp_fw->size,
+ ".fwsignature_ga10x", &work->sig_span);
+ if (ret)
+ return GU_RESULT(ret);
+ if (!work->image_span.size || work->image_span.size > 128ull << 20 ||
+ work->sig_span.size != 4096)
+ return GU_RESULT(-EINVAL);
+ ret = gpu_unlock_radix_plan(work->image_span.size, &work->radix);
+ if (ret)
+ return GU_RESULT(ret);
+ ret = request_asset(work, &work->boot_fw, BOOT_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ if (work->boot_fw->size != 24576)
+ return GU_RESULT(-EINVAL);
+ ret = request_asset(work, &desc, BOOT_DESC_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ ret = gpu_unlock_boot_desc_parse(desc->data, desc->size,
+ work->boot_fw->size, &work->boot_desc);
+ dev_info(&work->pdev->dev, "asset geometry: image=%llu signature=%llu boot=%u code=%#x data=%#x manifest=%#x parse=%d\n",
+ (unsigned long long)work->image_span.size, (unsigned long long)work->sig_span.size,
+ work->boot_desc.image_size, work->boot_desc.code_offset, work->boot_desc.data_offset,
+ work->boot_desc.manifest_offset, ret);
+ release_firmware(desc);
+ return GU_RESULT(ret ? ret : load_booter(work));
+}
+
+static int unlock_transaction_layout(struct unlock_transaction_work *work)
+{
+ struct gpu_unlock_layout_in *in = &work->layout_in;
+ u32 mb, vga, lo, hi, plm;
+ int ret;
+
+ if (read_bar(work, 0x001183a4, &mb) || !mb || mb == ~0u ||
+ read_bar(work, 0x00625f04, &vga) || vga == ~0u ||
+ read_bar(work, 0x001fa7c8, &plm) || plm == ~0u || !(plm & 1) ||
+ read_bar(work, 0x001fa82c, &lo) || lo == ~0u ||
+ read_bar(work, 0x001fa830, &hi) || hi == ~0u)
+ return GU_RESULT(-ENODEV);
+ /* Keep the cold board inputs in the USB kernel log even on layout failure.
+ * Register values after NVIDIA binds cannot reconstruct this state. */
+ dev_info(&work->pdev->dev,
+ "layout inputs: fb_mb=%#x vga=%#x mmu_plm=%#x mmu_lo=%#x mmu_hi=%#x image=%llu boot=%u\n",
+ mb, vga, plm, lo, hi, (unsigned long long)work->image_span.size,
+ work->boot_desc.image_size);
+ in->usable_fb = (u64)mb << 20;
+ if (in->usable_fb < 256ull << 20) {
+ dev_err(&work->pdev->dev, "layout rejected: usable FB below 256 MiB\n");
+ return GU_RESULT(-ERANGE);
+ }
+ /* kgspPopulateWprMeta_TU102: PRAMIN is the top 1 MiB fallback. */
+ in->vga_base = in->usable_fb - (1ull << 20);
+ if (vga & (1u << 3)) {
+ in->vga_base = (u64)(vga >> 8) << 16;
+ if (in->vga_base < in->usable_fb - (1ull << 20))
+ in->vga_base = in->usable_fb - 0x20000;
+ }
+ if (in->vga_base >= in->usable_fb) {
+ dev_err(&work->pdev->dev, "layout rejected: VGA base at or above usable FB\n");
+ return GU_RESULT(-ERANGE);
+ }
+ in->mmu_lock_low = (u64)(lo >> 4) << 12;
+ {
+ /* VAL is bits 31:4; the address is that field shifted by 12.
+ * memmgrBlockMemLockedMemory_GA100 only treats the lock as the
+ * VBIOS end-of-FB reservation when its exclusive high address is
+ * the FB top. A lock elsewhere is not that reservation. GA102
+ * has no scrubber, so WPR must stay in the pre-scrubbed top. */
+ u64 mmu_hi = (u64)(hi >> 4) << 12;
+ u64 mmu_top = mmu_hi > ~0ull - 0xffffull ? ~0ull :
+ (mmu_hi + 0xffffull) & ~0xffffull;
+ bool top_lock;
+
+ in->mmu_lock_valid = mmu_hi >= in->mmu_lock_low;
+ top_lock = in->mmu_lock_valid && (mmu_top >> 20) == (in->usable_fb >> 20);
+ dev_info(&work->pdev->dev,
+ "layout mmu: lo=%#llx hi=%#llx hi_exclusive=%#llx constrain=%u\n",
+ (unsigned long long)in->mmu_lock_low, (unsigned long long)mmu_hi,
+ (unsigned long long)mmu_top, top_lock);
+ if (!top_lock) {
+ dev_info(&work->pdev->dev,
+ "mmu lock ignored: high address is not the FB top\n");
+ in->mmu_lock_valid = false;
+ }
+ }
+ in->frts_size = 1ull << 20;
+ in->boot_size = work->boot_desc.image_size;
+ in->image_size = work->image_span.size;
+ in->nonwpr_size = 1ull << 20;
+ /* GA102's supported minimum heap; no GSP-RM clients are started here. */
+ in->heap_request = 88ull << 20;
+ ret = gpu_unlock_layout(in, &work->layout);
+ dev_info(&work->pdev->dev,
+ "layout decoded: fb=%#llx vga=%#llx mmu_valid=%u mmu_low=%#llx frts=%#llx heap=%#llx result=%d\n",
+ (unsigned long long)in->usable_fb, (unsigned long long)in->vga_base,
+ in->mmu_lock_valid, (unsigned long long)in->mmu_lock_low,
+ (unsigned long long)in->frts_size, (unsigned long long)in->heap_request, ret);
+ if (ret) {
+ dev_err(&work->pdev->dev, "layout rejected: reservation calculation (%d)\n", ret);
+ return GU_RESULT(ret);
+ }
+ dev_info(&work->pdev->dev,
+ "layout ranges: nonwpr=%#llx wpr=%#llx heap=%#llx gsp=%#llx boot=%#llx frts=%#llx end=%#llx scrub_floor=%#llx\n",
+ (unsigned long long)work->layout.nonwpr_start,
+ (unsigned long long)work->layout.wpr_start,
+ (unsigned long long)work->layout.heap_offset,
+ (unsigned long long)work->layout.gsp_offset,
+ (unsigned long long)work->layout.boot_offset,
+ (unsigned long long)work->layout.frts_offset,
+ (unsigned long long)work->layout.wpr_end,
+ (unsigned long long)(in->usable_fb - (256ull << 20)));
+ /* GA102 has no scrubber: all reservations must fit the pre-scrubbed top. */
+ if (work->layout.nonwpr_start < in->usable_fb - (256ull << 20)) {
+ dev_err(&work->pdev->dev, "layout rejected: reservations below pre-scrubbed top 256 MiB\n");
+ return GU_RESULT(-ERANGE);
+ }
+ return 0;
+}
+
+static int unlock_transaction_radix(struct unlock_transaction_work *work)
+{
+ const struct gpu_unlock_radix_plan *r = &work->radix;
+ u64 *addresses;
+ u32 i, level, first = 0;
+ int ret = 0;
+
+ work->page_count = r->table_pages + r->pages[3];
+ work->pages = kvcalloc(work->page_count, sizeof(*work->pages), GFP_KERNEL);
+ addresses = kvcalloc(work->page_count, sizeof(*addresses), GFP_KERNEL);
+ if (!work->pages || !addresses) {
+ kvfree(addresses);
+ return GU_RESULT(-ENOMEM);
+ }
+ for (i = 0; i < work->page_count; i++) {
+ struct unlock_transaction_page *page = &work->pages[i];
+
+ page->cpu = diag_dma_alloc(work, "radix-page", GU_RADIX_PAGE,
+ &page->dma, GFP_KERNEL);
+ if (!page->cpu) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ work->pages_allocated++;
+ addresses[i] = page->dma;
+ memset(page->cpu, 0, GU_RADIX_PAGE);
+ if (i >= r->table_pages) {
+ u64 offset = (i - r->table_pages) * GU_RADIX_PAGE;
+ size_t n = min_t(u64, GU_RADIX_PAGE, work->image_span.size - offset);
+
+ memcpy(page->cpu, work->gsp_fw->data + work->image_span.offset + offset, n);
+ }
+ }
+ for (level = 0; level < 3; level++) {
+ u32 child = first + r->pages[level];
+
+ for (i = 0; i < r->pages[level]; i++) {
+ u32 at = i * (GU_RADIX_PAGE / 8);
+ u32 n = min_t(u64, GU_RADIX_PAGE / 8, r->pages[level + 1] - at);
+
+ ret = gpu_unlock_radix_page(work->pages[first + i].cpu,
+ addresses + child + at, n);
+ if (ret)
+ goto out;
+ }
+ first = child;
+ }
+out:
+ kvfree(addresses);
+ return GU_RESULT(ret);
+}
+
+static int unlock_transaction_claim(struct unlock_transaction_work *work)
+{
+ u16 command, pmcsr;
+ u32 cap;
+ unsigned int i;
+ int ret;
+
+ dev_info(&work->pdev->dev, "claim: power=%d pm_cap=%#x flags=%#x\n",
+ work->pdev->current_state, work->pdev->pm_cap, work->pdev->dev_flags);
+ if (work->pdev->current_state != PCI_D0 ||
+ (work->pdev->pm_cap &&
+ (diag_pci_word(work, false, work->pdev->pm_cap + PCI_PM_CTRL, &pmcsr) ||
+ (pmcsr & PCI_PM_CTRL_STATE_MASK) != PCI_D0)))
+ return GU_RESULT(-EOPNOTSUPP);
+ if ((work->pdev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET) ||
+ !pci_is_pcie(work->pdev) ||
+ diag_pci_dword(work, true, PCI_EXP_DEVCAP, &cap) ||
+ !(cap & PCI_EXP_DEVCAP_FLR))
+ return GU_RESULT(-EOPNOTSUPP);
+ if (diag_pci_word(work, false, PCI_COMMAND, &command) || command == 0xffff)
+ return GU_RESULT(-ENODEV);
+ work->original_cmd = command;
+ work->cmd_valid = true;
+ if (command & PCI_COMMAND_MASTER)
+ return GU_RESULT(-EBUSY);
+ dev_info(&work->pdev->dev, "claim: command=%#x capability=%#x BAR_mask=%#x\n",
+ command, cap, pci_select_bars(work->pdev, IORESOURCE_MEM));
+ /* Only BAR0 is mapped. On the display card the firmware framebuffer keeps
+ * BAR1 until just before the FLR, as it does while the stock driver boots.
+ */
+ ret = pci_request_region(work->pdev, 0, "gpu_unlock");
+ if (ret)
+ return GU_RESULT(ret);
+ work->region = true;
+ if (!(command & PCI_COMMAND_MEMORY)) {
+ ret = pci_enable_device_mem(work->pdev);
+ if (ret)
+ return GU_RESULT(ret);
+ work->enabled = true;
+ }
+ work->bar_len = pci_resource_len(work->pdev, 0);
+ if (work->bar_len < SEC2_FALCON2 + 0x1000)
+ return GU_RESULT(-EINVAL);
+ for (i = 0; i < ARRAY_SIZE(work->bars); i++)
+ if (diag_pci_dword(work, false, PCI_BASE_ADDRESS_0 + i * 4, &work->bars[i]))
+ return GU_RESULT(-EIO);
+ ret = pci_save_state(work->pdev);
+ if (ret)
+ return GU_RESULT(ret);
+ work->bar = pci_iomap(work->pdev, 0, 0);
+ if (!work->bar)
+ return GU_RESULT(-ENOMEM);
+ work->mapped = true;
+ return 0;
+}
+
+static int unlock_transaction_dma(struct unlock_transaction_work *work)
+{
+ struct device *dev = &work->pdev->dev;
+
+ if (!dev->dma_mask || work->masks_saved)
+ return GU_RESULT(-EINVAL);
+ work->masks_saved = true;
+ work->streaming_ptr = dev->dma_mask;
+ work->streaming_before = *dev->dma_mask;
+ work->coherent_before = dev->coherent_dma_mask;
+ dev_info(dev, "DMA masks: streaming=%#llx coherent=%#llx requested=%#llx\n",
+ (unsigned long long)work->streaming_before, (unsigned long long)work->coherent_before,
+ (unsigned long long)DMA_BIT_MASK(32));
+ work->streaming_set = true;
+ if (GU_DIAG(dma_set_mask(dev, DMA_BIT_MASK(32))))
+ return GU_RESULT(-EIO);
+ work->coherent_set = true;
+ if (GU_DIAG(dma_set_coherent_mask(dev, DMA_BIT_MASK(32))))
+ return GU_RESULT(-EIO);
+ work->booter_cpu = diag_dma_alloc(work, "booter", work->booter_image_size,
+ &work->booter_dma, GFP_KERNEL);
+ if (!work->booter_cpu)
+ return GU_RESULT(-ENOMEM);
+ memcpy(work->booter_cpu, work->booter_fw->data + 72, work->booter_image_size);
+ work->v67_cpu = diag_dma_alloc(work, "v67", GU_V67_SIZE, &work->v67_dma, GFP_KERNEL);
+ work->meta_cpu = diag_dma_alloc(work, "metadata", GU_WPR_META_SIZE, &work->meta_dma, GFP_KERNEL);
+ if (!work->v67_cpu || !work->meta_cpu)
+ return GU_RESULT(-ENOMEM);
+ memcpy(work->v67_cpu, work->v67_fw->data, GU_V67_SIZE);
+ work->stock_size = work->sig_span.size;
+ work->stock_cpu = diag_dma_alloc(work, "stock-signature", work->stock_size, &work->stock_dma, GFP_KERNEL);
+ work->boot_cpu = diag_dma_alloc(work, "boot-image", work->boot_desc.image_size, &work->boot_dma, GFP_KERNEL);
+ if (!work->stock_cpu || !work->boot_cpu)
+ return GU_RESULT(-ENOMEM);
+ memcpy(work->stock_cpu, work->gsp_fw->data + work->sig_span.offset, work->stock_size);
+ memcpy(work->boot_cpu, work->boot_fw->data, work->boot_desc.image_size);
+ {
+ int ret = patch_booter(work);
+
+ return GU_RESULT(ret ? ret : unlock_transaction_radix(work));
+ }
+}
+
+static int unlock_transaction_fwsec(struct unlock_transaction_work *work)
+{
+ static const struct falcon_eng gsp = {
+ .base = 0x00110000, .fbif = 0x00110600, .falcon2 = 0x00111000,
+ };
+ size_t mapped = 0;
+ u8 *rom = NULL;
+ struct gpu_unlock_fwsec fw;
+ u32 fuse_raw, version, scratch, wpr_lo, wpr_hi;
+ u16 command;
+ u64 frts = work->layout.frts_offset;
+ void *image = NULL;
+ dma_addr_t image_dma = 0;
+ int ret;
+
+ /* kgspExtractVbiosFromRom_TU102 reads NV_PROM_DATA, up to 1 MiB.
+ * pci_map_rom stops at the PCI last-image flag and hides the VN extension. */
+ dev_info(&work->pdev->dev, "FWSEC: reading PROM at %#x\n", 0x300000);
+ if (work->bar_len < 0x300000 + 0x1a)
+ return GU_RESULT(-ENODEV);
+ mapped = min_t(size_t, 1024 * 1024, work->bar_len - 0x300000);
+ rom = kvmalloc(mapped, GFP_KERNEL);
+ if (!rom)
+ return GU_RESULT(-ENOMEM);
+ memcpy_fromio(rom, work->bar + 0x300000, mapped);
+ dev_info(&work->pdev->dev, "FWSEC: PROM bytes=%zu signature=%#x\n",
+ mapped, (unsigned int)(rom[0] | (rom[1] << 8)));
+ ret = gpu_unlock_vbios_fwsec(rom, mapped, &fw);
+ if (ret) {
+ dev_err(&work->pdev->dev, "FWSEC-FRTS not found in board ROM (%d)\n", ret);
+ goto out;
+ }
+ if (fw.engine_mask != 0x400) {
+ dev_err(&work->pdev->dev, "FWSEC: unsupported GSP engine mask=%#x\n", fw.engine_mask);
+ ret = -EOPNOTSUPP;
+ goto out;
+ }
+ image = diag_dma_alloc(work, "fwsec", fw.image_size, &image_dma, GFP_KERNEL);
+ if (!image) {
+ ret = -ENOMEM;
+ goto out;
+ }
+ work->fwsec_cpu = image;
+ work->fwsec_dma = image_dma;
+ work->fwsec_size = fw.image_size;
+ memcpy(image, rom + fw.image_off, fw.image_size);
+ ret = gpu_unlock_fwsec_patch_frts(image, &fw, frts);
+ if (ret)
+ goto out;
+ if (!fw.ucode_id ||
+ read_bar(work, 0x008241c0 + (fw.ucode_id - 1) * 4, &fuse_raw)) {
+ ret = -EIO;
+ goto out;
+ }
+ version = gpu_unlock_fuse_version(fuse_raw);
+ dev_info(&work->pdev->dev, "FWSEC fuse: raw=%#x version=%u\n", fuse_raw, version);
+ {
+ u32 sig_at = 0;
+
+ ret = gpu_unlock_fwsec_sig_offset(fw.sig_versions, version, 384, &sig_at);
+ if (ret)
+ goto out;
+ memcpy((u8 *)image + fw.data_off + fw.pkc_off, rom + fw.sig_off + sig_at, 384);
+ }
+ pci_set_master(work->pdev);
+ work->master = true;
+ if (diag_pci_word(work, false, PCI_COMMAND, &command) ||
+ command == 0xffff || !(command & PCI_COMMAND_MASTER)) {
+ ret = -EIO;
+ goto out;
+ }
+ work->firmware_started = true;
+ ret = falcon_hs(work, &gsp, 0, fw.imem_size, fw.imem_va, fw.imem_pa,
+ fw.data_off, fw.dmem_size, fw.dmem_pa, fw.pkc_off,
+ fw.engine_mask, fw.ucode_id,
+ image_dma, 0, 0, false);
+ if (ret)
+ goto out;
+ if (read_bar(work, 0x00001400 + 0x0e * 4, &scratch)) {
+ ret = -EIO;
+ goto out;
+ }
+ dev_info(&work->pdev->dev, "FWSEC result: scratch=%#x expected_hi16=0\n", scratch);
+ if (scratch >> 16) {
+ ret = -EIO;
+ goto out;
+ }
+ if (read_bar(work, 0x001fa824, &wpr_lo) || wpr_lo == ~0u ||
+ read_bar(work, 0x001fa828, &wpr_hi) || wpr_hi == ~0u) {
+ ret = -EIO;
+ goto out;
+ }
+ dev_info(&work->pdev->dev, "FWSEC WPR2: low=%#x high=%#x expected_page=%#llx end=%#llx\n",
+ wpr_lo, wpr_hi, (unsigned long long)(frts >> 12),
+ (unsigned long long)work->layout.wpr_end);
+ /* Match NVIDIA's initialized-WPR2 check as well as the expected low
+ * address. Never publish Booter metadata for disabled/alien WPR2. */
+ if (!(wpr_hi >> 4) || (wpr_hi >> 4) < (wpr_lo >> 4) ||
+ ((u64)(wpr_hi >> 4) << 12) >= work->layout.wpr_end ||
+ (wpr_lo >> 4) != (u32)(frts >> 12))
+ ret = -EIO;
+out:
+ /* Ownership transfers to work before publication, including failure paths. */
+ kvfree(rom);
+ return GU_RESULT(ret);
+}
+
+static int unlock_transaction_gfw(struct unlock_transaction_work *work)
+{
+ ktime_t deadline = ktime_add_us(ktime_get(), GU_GFW_TIMEOUT_US);
+ int assessed = -EAGAIN;
+
+ for (;;) {
+ struct gpu_unlock_gfw_sample sample = { };
+ u16 vendor = 0xffff;
+
+ if (read_bar(work, GU_GFW_PLM_OFFSET, &sample.plm) ||
+ read_bar(work, GU_GFW_BOOT_OFFSET, &sample.boot) ||
+ read_bar(work, GU_GFW_CPUCTL_OFFSET, &sample.cpuctl))
+ return GU_RESULT(-EIO);
+ if (!diag_pci_word(work, false, PCI_VENDOR_ID, &vendor) &&
+ vendor != 0xffff)
+ sample.config_ok = true;
+ assessed = gpu_unlock_gfw_assess(&sample);
+ if (!assessed) {
+ u32 ss0, ss1, wpr_hi;
+
+ if (read_bar(work, GU_FEAT_PLM_OFFSET, &work->plm_entry) ||
+ read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || ss0 == ~0u ||
+ read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || ss1 == ~0u ||
+ read_bar(work, 0x001fa828, &wpr_hi) || inaccessible(wpr_hi))
+ return GU_RESULT(-ENODEV);
+ dev_info(&work->pdev->dev, "entry: plm=%#x ss0=%#x ss1=%#x wpr2_hi=%#x\n",
+ work->plm_entry, ss0, ss1, wpr_hi);
+ /* An open PLM means the card was already unlocked or not cold. */
+ if (work->plm_entry == ~0u)
+ return GU_RESULT(-EALREADY);
+ /* WPR2 is down from power-on until FWSEC-FRTS runs. An enabled
+ * WPR2 means a driver has already used the card this power
+ * cycle; starting FWSEC and Booter again from here is untested.
+ */
+ if (wpr_hi >> 4) {
+ dev_err(&work->pdev->dev, "card already initialised since power on (WPR2 enabled); power off first\n");
+ return GU_RESULT(-EBUSY);
+ }
+ return 0;
+ }
+ if (assessed != -EAGAIN)
+ return GU_RESULT(assessed);
+ if (ktime_compare(ktime_get(), deadline) >= 0)
+ return GU_RESULT(-ETIMEDOUT);
+ usleep_range(1000, 2000);
+ }
+}
+
+static int unlock_transaction_boot(struct unlock_transaction_work *work)
+{
+ static const struct falcon_eng sec2 = {
+ .base = SEC2_BASE, .fbif = SEC2_FBIF, .falcon2 = SEC2_FALCON2,
+ };
+ const struct gpu_unlock_booter_hdr *h = &work->booter_hdr;
+
+ if (!work->published || !work->master || !work->firmware_started)
+ return GU_RESULT(-EINVAL);
+ return GU_RESULT(falcon_hs(work, &sec2, h->app_code_off, h->app_code_size,
+ h->app_code_off, 0, h->os_data_off, h->os_data_size, 0,
+ h->patch_loc - h->os_data_off, h->engine_id, h->ucode_id,
+ work->booter_dma, lower_32_bits(work->meta_dma),
+ upper_32_bits(work->meta_dma), true));
+}
+
+static void put_le64(u8 *dest, u64 value)
+{
+ unsigned int i;
+
+ for (i = 0; i < 8; i++)
+ dest[i] = (u8)(value >> (8 * i));
+}
+
+static int unlock_transaction_signature(struct unlock_transaction_work *work)
+{
+ u8 *meta = work->meta_cpu;
+
+ if (!meta || !work->stock_cpu || !work->pages_allocated ||
+ work->signature_calls >= 2)
+ return GU_RESULT(-EINVAL);
+ if (work->signature_calls == 0) {
+ struct gpu_unlock_wpr_args args = {
+ .fb_size = work->layout_in.usable_fb,
+ .vga_offset = work->layout_in.vga_base,
+ .frts_size = work->layout_in.frts_size,
+ .signature_dma = work->v67_dma, .signature_size = GU_V67_SIZE,
+ .radix_dma = work->pages[0].dma,
+ .image_size = work->image_span.size,
+ .boot_dma = work->boot_dma, .boot = work->boot_desc,
+ };
+ int ret = gpu_unlock_wpr_meta_pack(meta, &work->layout, &args);
+
+ if (ret)
+ return GU_RESULT(ret);
+ dma_wmb();
+ work->published = true;
+ } else {
+ put_le64(meta + 72, work->stock_dma);
+ put_le64(meta + 80, work->stock_size);
+ dma_wmb();
+ }
+ work->signature_calls++;
+ return 0;
+}
+
+/* Write the unlock value to one speed selector and read it back. */
+static int write_selector(struct unlock_transaction_work *work, const char *name,
+ u32 offset, u32 unlock)
+{
+ u32 value;
+
+ if (read_bar(work, offset, &value) || inaccessible(value))
+ return GU_RESULT(-ENODEV);
+ dev_info(&work->pdev->dev, "selector read: %s=%#x expected_unlock=%#x\n",
+ name, value, unlock);
+ if (value == unlock)
+ return 0;
+ dev_info(&work->pdev->dev, "selector write: %s=%#x\n", name, unlock);
+ write_bar(work, offset, unlock);
+ if (read_bar(work, offset, &value) || value == ~0u)
+ return GU_RESULT(-ENODEV);
+ return GU_RESULT(value == unlock ? 0 : -EIO);
+}
+
+/* SS0 selects the full BF16 rate; SS1 gets the same field value. FLR keeps both. */
+static int unlock_transaction_selector(struct unlock_transaction_work *work)
+{
+ int ret = write_selector(work, "SS0", GU_FEAT_SS0_OFFSET, GU_SS0_UNLOCK);
+
+ return ret ? ret : write_selector(work, "SS1", GU_FEAT_SS1_OFFSET, GU_SS1_UNLOCK);
+}
+
+/* Reset SEC2 after FLR and leave it halted with reset released. Do not reset
+ * GSP here: stock RM first requires completed GFW and a halted GSP Falcon.
+ * ksec2ResetHw_TU102 ends with RESET_FALSE. Leaving RESET_TRUE makes later
+ * host reads return FECS_PRI_RESET (0xbadf12 in bits 31:8). RM's scrub wait
+ * treats that value as scrubbing-pending and does not finish.
+ */
+static int hold_sec2_reset(struct unlock_transaction_work *work, const char *phase)
+{
+ unsigned int j;
+ u32 engine, before, first = 0, hwcfg;
+ int ret;
+
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) || inaccessible(engine))
+ return GU_RESULT(-ENODEV);
+ before = engine;
+ dev_info(&work->pdev->dev, "engine reset assert: phase=%s base=%#x before=%#x write=%#x\n",
+ phase, SEC2_BASE, before, before | 1);
+ write_bar(work, SEC2_BASE + 0x3c0, engine | 1);
+ /* 610 ksec2ResetHw_TU102 requires ten register cycles for signal
+ * propagation. A stale zero during those cycles is not evidence that
+ * the assertion failed. No CPU start and no reset retry.
+ */
+ for (j = 0; j < 10; j++) {
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) ||
+ inaccessible(engine)) {
+ dev_err(&work->pdev->dev, "engine reset inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
+ phase, SEC2_BASE, j + 1, engine);
+ return GU_RESULT(-EIO);
+ }
+ if (!j)
+ first = engine;
+ }
+ dev_info(&work->pdev->dev, "engine reset propagation: phase=%s base=%#x first=%#x tenth=%#x\n",
+ phase, SEC2_BASE, first, engine);
+ ret = poll_us(work, SEC2_BASE + 0x3c0, 1, 1, 1000, true);
+ dev_info(&work->pdev->dev, "engine reset asserted: phase=%s base=%#x error=%d last=%#x\n",
+ phase, SEC2_BASE, ret, work->last_value);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) || inaccessible(engine))
+ return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev, "engine reset release: phase=%s base=%#x before=%#x write=%#x\n",
+ phase, SEC2_BASE, engine, engine & ~1u);
+ write_bar(work, SEC2_BASE + 0x3c0, engine & ~1u);
+ for (j = 0; j < 10; j++) {
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) ||
+ inaccessible(engine)) {
+ dev_err(&work->pdev->dev, "engine reset release inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
+ phase, SEC2_BASE, j + 1, engine);
+ return GU_RESULT(-EIO);
+ }
+ }
+ ret = poll_us(work, SEC2_BASE + 0x3c0, 1, 0, 1000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ /* NV_PFALCON_FALCON_HWCFG2 bit 12 clears when memory scrubbing finishes.
+ * A PRI error fails this wait instead of being handed to RM.
+ */
+ ret = poll_us(work, SEC2_BASE + 0xf4, 1u << 12, 0, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, SEC2_BASE + 0xf4, &hwcfg) || inaccessible(hwcfg))
+ return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev, "engine reset released: phase=%s base=%#x engine=%#x hwcfg2=%#x\n",
+ phase, SEC2_BASE, engine, hwcfg);
+ return 0;
+}
+
+/* PCI configuration can be ready before BAR0. The 610 BIF restore path waits
+ * for PMC_BOOT_0 to match the saved chip ID before touching engine registers.
+ * Transient badf/all-ones reads are expected only inside this bounded wait.
+ */
+static int wait_post_flr_bar(struct unlock_transaction_work *work, u32 chip)
+{
+ ktime_t deadline = ktime_add_us(ktime_get(), 1000000);
+ u32 value = 0;
+
+ for (;;) {
+ if (read_bar(work, 0, &value))
+ return GU_RESULT(-EIO);
+ if (value == chip) {
+ dev_info(&work->pdev->dev, "post-FLR BAR ready: chip=%#x\n", value);
+ return 0;
+ }
+ if (ktime_compare(ktime_get(), deadline) >= 0) {
+ dev_err(&work->pdev->dev, "post-FLR BAR timeout: chip=%#x expected=%#x budget_us=1000000\n",
+ value, chip);
+ return GU_RESULT(-ETIMEDOUT);
+ }
+ usleep_range(1000, 2000);
+ }
+}
+
+/* Match gpuWaitForGfwBootComplete_TU102, without launching any firmware or
+ * overwriting the cold-entry evidence. The post-FLR ROM boot may still be busy.
+ */
+static int wait_post_flr_gfw(struct unlock_transaction_work *work)
+{
+ ktime_t deadline = ktime_add_us(ktime_get(), GU_GFW_TIMEOUT_US);
+ struct gpu_unlock_gfw_sample sample = { };
+ u32 identity = 0;
+ int ret;
+
+ for (;;) {
+ sample = (struct gpu_unlock_gfw_sample){ };
+ ret = diag_pci_dword(work, false, PCI_VENDOR_ID, &identity);
+ sample.config_ok = !ret && identity == 0x220410de;
+ if (!sample.config_ok) {
+ ret = -ENODEV;
+ break;
+ }
+ if (read_bar(work, GU_GFW_CPUCTL_OFFSET, &sample.cpuctl) ||
+ read_bar(work, GU_GFW_PLM_OFFSET, &sample.plm)) {
+ ret = -EIO;
+ break;
+ }
+ /* Avoid the protected scratch read until PL0 can read it. */
+ if (!inaccessible(sample.plm) && (sample.plm & GU_GFW_PLM_LEVEL0) &&
+ read_bar(work, GU_GFW_BOOT_OFFSET, &sample.boot)) {
+ ret = -EIO;
+ break;
+ }
+ ret = inaccessible(sample.plm) || inaccessible(sample.boot) ||
+ inaccessible(sample.cpuctl) ? -EAGAIN : gpu_unlock_gfw_assess(&sample);
+ if (!ret)
+ break;
+ if (ktime_compare(ktime_get(), deadline) >= 0) {
+ ret = -ETIMEDOUT;
+ break;
+ }
+ usleep_range(1000, 2000);
+ }
+ dev_info(&work->pdev->dev, "post-FLR GFW result: error=%d pci=%#x plm=%#x boot=%#x cpuctl=%#x budget_us=%u\n",
+ ret, identity, sample.plm, sample.boot, sample.cpuctl, GU_GFW_TIMEOUT_US);
+ return GU_RESULT(ret);
+}
+
+static int unlock_transaction_reset(struct unlock_transaction_work *work)
+{
+ u32 identity, bar, lo, hi, ss0, ss1, plm, chip, cpuctl;
+ u16 command, pending;
+ unsigned int i;
+ int ret;
+
+ if (work->signature_calls != 2)
+ return GU_RESULT(-EPROTO);
+ if (read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || ss0 == ~0u ||
+ read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || ss1 == ~0u)
+ return GU_RESULT(-ENODEV);
+ dev_info(&work->pdev->dev, "pre-FLR selectors: ss0=%#x expected=%#x ss1=%#x expected=%#x\n",
+ ss0, GU_SS0_UNLOCK, ss1, GU_SS1_UNLOCK);
+ if (ss0 != GU_SS0_UNLOCK || ss1 != GU_SS1_UNLOCK)
+ return GU_RESULT(-EIO);
+ pci_clear_master(work->pdev);
+ work->master = false;
+ if (diag_pci_word(work, false, PCI_COMMAND, &command) ||
+ command == 0xffff || (command & PCI_COMMAND_MASTER))
+ return GU_RESULT(-EIO);
+ if (read_bar(work, 0, &chip) || !chip || inaccessible(chip))
+ return GU_RESULT(-ENODEV);
+ /* Ampere PF-FLR requires no engine-reset preconditioning in this cold,
+ * unbound path (no initialized NVLink/P2P). The pinned GA102 rejoin14
+ * sequence also goes from the early PLM result to FLR. A pre-FLR engine
+ * hold is not a prerequisite: secure firmware can reject that write.
+ * Keep EVERY allocation alive, BME off, until post-FLR containment.
+ */
+ /* FLR resets the display engine, so a firmware framebuffer (simpledrm,
+ * efifb) on this card must go first; nvidia-drm replaces it later. This
+ * does nothing on a card without one.
+ */
+ ret = aperture_remove_conflicting_pci_devices(work->pdev, "gpu_unlock");
+ if (ret)
+ return GU_RESULT(ret);
+ dev_info(&work->pdev->dev, "PCIe FLR begin: BME=0 DMA retained, chip=%#x\n", chip);
+ ret = pcie_flr(work->pdev);
+ dev_info(&work->pdev->dev, "PCIe FLR returned: result=%d\n", ret);
+ if (ret)
+ return GU_RESULT(ret);
+ /* The saved image predates BME. No transient restore with BME set. */
+ pci_restore_state(work->pdev);
+ if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
+ identity != 0x220410de ||
+ diag_pci_word(work, false, PCI_COMMAND, &command) ||
+ command == 0xffff || (command & PCI_COMMAND_MASTER) ||
+ !(command & PCI_COMMAND_MEMORY))
+ return GU_RESULT(-ENODEV);
+ for (i = 0; i < ARRAY_SIZE(work->bars); i++) {
+ if (diag_pci_dword(work, false, PCI_BASE_ADDRESS_0 + i * 4, &bar) ||
+ bar != work->bars[i]) {
+ dev_err(&work->pdev->dev, "post-FLR BAR%u: actual=%#x expected=%#x\n", i, bar, work->bars[i]);
+ return GU_RESULT(-EIO);
+ }
+ }
+ ret = wait_post_flr_bar(work, chip);
+ if (ret)
+ return GU_RESULT(ret);
+ /* ROM GFW completion/halt plus FLR disposes of the old GSP execution.
+ * Stock RM checks this halt before resetting GSP with its own buffers.
+ */
+ ret = wait_post_flr_gfw(work);
+ if (ret)
+ return GU_RESULT(ret);
+ ret = hold_sec2_reset(work, "post-FLR");
+ if (ret)
+ return GU_RESULT(ret);
+ if (diag_pci_word(work, true, PCI_EXP_DEVSTA, &pending) ||
+ pending == 0xffff || (pending & PCI_EXP_DEVSTA_TRPND))
+ return GU_RESULT(-EBUSY);
+ if (read_bar(work, GU_GFW_CPUCTL_OFFSET, &cpuctl) || inaccessible(cpuctl) ||
+ !(cpuctl & GU_GFW_CPUCTL_HALTED) ||
+ read_bar(work, 0x001fa824, &lo) || inaccessible(lo) ||
+ read_bar(work, 0x001fa828, &hi) || inaccessible(hi) ||
+ read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || inaccessible(ss0) ||
+ read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || inaccessible(ss1) ||
+ read_bar(work, GU_FEAT_PLM_OFFSET, &plm) ||
+ (plm & 0xffff0000u) == 0xbadf0000u)
+ return GU_RESULT(-ENODEV);
+ dev_info(&work->pdev->dev,
+ "post-FLR SEC2 reset released, GSP halted, BME=0 pending=0 WPR2=%08x:%08x SS0=%08x SS1=%08x\n",
+ lo, hi, ss0, ss1);
+ /* Both 610 and installed 615 _kgspBootGspRm reject preexisting WPR2
+ * on this cold dGPU path, even our old in-bounds FRTS reservation.
+ * kgspIsWpr2Up_TU102 uses HI.VAL != 0, not a LO/HI range comparison.
+ */
+ if (hi >> 4) {
+ dev_err(&work->pdev->dev, "post-FLR WPR2 remains enabled: lo=%#x hi=%#x; stock driver requires HI.VAL=0\n", lo, hi);
+ return GU_RESULT(-EBUSY);
+ }
+ work->quiesced = true;
+ if (ss0 != GU_SS0_UNLOCK || ss1 != GU_SS1_UNLOCK)
+ return GU_RESULT(-EIO);
+ return 0;
+}
+
+int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared)
+{
+ return request_scoped(&pdev->dev, &shared->gsp, GSP_FILE);
+}
+
+void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
+{
+ release_firmware(shared->gsp);
+ shared->gsp = NULL;
+}
+
+int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared)
+{
+ struct unlock_transaction_work *work;
+ int ret, cleanup;
+
+ work = kzalloc(sizeof(*work), GFP_KERNEL);
+ if (!work)
+ return -ENOMEM;
+ work->pdev = pci_dev_get(pdev);
+ work->gsp_fw = shared->gsp;
+ ret = unlock_transaction_assets(work);
+ if (!ret)
+ ret = unlock_transaction_claim(work);
+ if (!ret)
+ ret = unlock_transaction_layout(work);
+ if (!ret)
+ ret = unlock_transaction_dma(work);
+ if (!ret)
+ ret = unlock_transaction_gfw(work);
+ if (!ret)
+ ret = unlock_transaction_fwsec(work);
+ if (!ret)
+ ret = unlock_transaction_signature(work); /* publish V67 */
+ if (!ret)
+ ret = unlock_transaction_boot(work);
+ if (!ret)
+ ret = unlock_transaction_signature(work); /* restore stock */
+ if (!ret)
+ ret = unlock_transaction_selector(work);
+ if (!ret)
+ ret = unlock_transaction_reset(work);
+ /* Never leave the V67 signature published for a later stock boot. */
+ if (ret && work->signature_calls == 1)
+ unlock_transaction_signature(work);
+ cleanup = unlock_transaction_unwind(work);
+ if (!ret)
+ ret = cleanup;
+ if (work->retained) {
+ /* Firmware may still DMA into these buffers: keep them, the device
+ * reference and this module until the next power off. */
+ dev_err(&pdev->dev, "firmware did not stop; keeping its DMA buffers until power off\n");
+ __module_get(THIS_MODULE);
+ return ret;
+ }
+ pci_dev_put(work->pdev);
+ kfree(work);
+ return ret;
+}
+
+MODULE_FIRMWARE(BOOTER_FILE);
+MODULE_FIRMWARE(V67_FILE);
+MODULE_FIRMWARE(GSP_FILE);
+MODULE_FIRMWARE(BOOT_FILE);
+MODULE_FIRMWARE(BOOT_DESC_FILE);
diff --git a/kernel/gpu_unlock/unlock_610.h b/kernel/gpu_unlock/unlock_610.h
new file mode 100644
index 0000000..dbca2d9
--- /dev/null
+++ b/kernel/gpu_unlock/unlock_610.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_UNLOCK_610_H
+#define GPU_UNLOCK_UNLOCK_610_H
+
+struct firmware;
+struct pci_dev;
+
+/* Read-only firmware every unlock uses: loaded once, shared by concurrent unlocks. */
+struct gpu_unlock_610_shared {
+ const struct firmware *gsp;
+};
+
+/*
+ * Load the shared firmware. pdev is any target card; the request leaves no
+ * device resources behind on it. Returns 0 or a negative errno.
+ */
+int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared);
+
+/* Release the shared firmware once every unlock using it has returned. */
+void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared);
+
+/*
+ * Unlock one cold, unbound RTX 3090. The caller holds pci_dev_lock. Returns 0
+ * when the card is unlocked and handed back as found; otherwise a negative
+ * errno, with the card left locked. Unlocks of different cards may run
+ * concurrently with the same shared firmware.
+ */
+int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared);
+
+#endif
diff --git a/kernel/gpu_unlock/vbios_610.c b/kernel/gpu_unlock/vbios_610.c
new file mode 100644
index 0000000..87c173b
--- /dev/null
+++ b/kernel/gpu_unlock/vbios_610.c
@@ -0,0 +1,304 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "diagnostic.h"
+/* Formats: NVIDIA 610.43.03 kernel_gsp_fwsec.c and kernel_gsp_vbios_tu102.c.
+ * Independent byte parser; no firmware-owned pointer is trusted as a host pointer.
+ */
+#ifdef __KERNEL__
+#include <linux/errno.h>
+#include <linux/string.h>
+#else
+#include <errno.h>
+#include <string.h>
+#endif
+#include "vbios_610.h"
+
+static u16 r16(const u8 *p)
+{
+ return (u16)p[0] | ((u16)p[1] << 8);
+}
+
+static u32 r32(const u8 *p)
+{
+ return (u32)p[0] | ((u32)p[1] << 8) | ((u32)p[2] << 16) | ((u32)p[3] << 24);
+}
+
+static void w32(u8 *p, u32 value)
+{
+ p[0] = (u8)value;
+ p[1] = (u8)(value >> 8);
+ p[2] = (u8)(value >> 16);
+ p[3] = (u8)(value >> 24);
+}
+
+static int range(size_t length, u64 off, u64 bytes)
+{
+ return off > length || bytes > length - off ? -EINVAL : 0;
+}
+
+/* kgspExtractVbiosFromRom_TU102 / s_locateExpansionRoms. Image signatures are
+ * 0xAA55, 0x4E56 ("VN") and 0xBB77. Data signatures are PCIR, NPDS and RGIS.
+ */
+static int rom_signature(u16 sig)
+{
+ return sig == 0xaa55 || sig == 0x4e56 || sig == 0xbb77;
+}
+
+static int data_signature(u32 sig)
+{
+ return sig == 0x52494350 || sig == 0x5344504e || sig == 0x53494752;
+}
+
+static int expansion(const u8 *rom, size_t length, u32 *adjust)
+{
+ u32 pos = 0, base = 0, ext = 0;
+ unsigned int images = 0;
+
+ if (range(length, 0, 2) || !rom_signature(r16(rom)))
+ return GU_DIAG(-EINVAL);
+ for (;;) {
+ u64 pcir, npde;
+ u32 bytes, last, sig;
+ u16 ext_rev, ext_len;
+
+ if (++images > 8 || range(length, pos, 0x1a) || !rom_signature(r16(rom + pos)))
+ return GU_DIAG(-EINVAL);
+ pcir = (u64)pos + r16(rom + pos + 0x18);
+ if (range(length, pcir, 0x16))
+ return GU_DIAG(-EINVAL);
+ sig = r32(rom + pcir);
+ if (!data_signature(sig))
+ return GU_DIAG(-EINVAL);
+ bytes = (u32)r16(rom + pcir + 0x10) * 512;
+ last = rom[pcir + 0x15] & 0x80;
+ npde = (pcir + r16(rom + pcir + 0x0a) + 15) & ~(u64)15;
+ if (!range(length, npde, 4) && r32(rom + npde) == 0x4544504e) {
+ if (range(length, npde, 10))
+ return GU_DIAG(-EINVAL);
+ ext_rev = r16(rom + npde + 4);
+ ext_len = r16(rom + npde + 6);
+ if (ext_rev == 0x100 || ext_rev == 0x101) {
+ u32 sub = (u32)r16(rom + npde + 8) * 512;
+
+ if (ext_len < 10 || range(length, npde, ext_len))
+ return GU_DIAG(-EINVAL);
+ if (ext_len > 10)
+ last = rom[npde + 10] & 0x80;
+ else if (sub < bytes)
+ last = 0;
+ bytes = sub;
+ }
+ }
+ GU_NOTE("ROM chain: pos=%#x data=%#x bytes=%u type=%#x last=%u\n",
+ pos, sig, bytes, rom[pcir + 0x14], last);
+ if (!bytes || range(length, pos, bytes))
+ return GU_DIAG(-EINVAL);
+ if (!base && rom[pcir + 0x14] == 0)
+ base = bytes;
+ if (!ext && rom[pcir + 0x14] == 0xe0)
+ ext = pos;
+ if (last)
+ break;
+ pos += bytes;
+ }
+ if (ext && base && ext < base)
+ return GU_DIAG(-EINVAL);
+ *adjust = ext && base ? ext - base : 0;
+ GU_NOTE("ROM expansion: base=%#x ext=%#x adjust=%#x\n", base, ext, *adjust);
+ return 0;
+}
+
+static int descriptor(const u8 *rom, size_t length, u64 off,
+ struct gpu_unlock_fwsec *out)
+{
+ struct gpu_unlock_fwsec fw = { 0 };
+ u32 hdr, size, stored, bits, count = 0;
+
+ if (range(length, off, 44))
+ return GU_DIAG(-EINVAL);
+ hdr = r32(rom + off);
+ size = hdr >> 16;
+ GU_NOTE("FWSEC descriptor: offset=%#llx ROM_bytes=%zu header=%#x\n",
+ (unsigned long long)off, length, hdr);
+ /* vDesc: bit 0 set, version in bits 15:8, size in bits 31:16. */
+ if (!(hdr & 1) || ((hdr >> 8) & 0xff) != 3 || size < 44)
+ return GU_DIAG(-EOPNOTSUPP);
+ if (range(length, off, size))
+ return GU_DIAG(-EINVAL);
+ stored = r32(rom + off + 4);
+ if (!stored || stored > ~(u32)0 - 255)
+ return GU_DIAG(-EINVAL);
+ fw.image_size = (stored + 255) & ~255u;
+ if (range(length, off + size, fw.image_size))
+ return GU_DIAG(-EINVAL);
+ fw.image_off = off + size;
+ fw.pkc_off = r32(rom + off + 8);
+ fw.interface_off = r32(rom + off + 12);
+ fw.imem_pa = r32(rom + off + 16);
+ fw.imem_size = r32(rom + off + 20);
+ fw.imem_va = r32(rom + off + 24);
+ fw.dmem_pa = r32(rom + off + 28);
+ fw.dmem_size = r32(rom + off + 32);
+ fw.data_off = fw.imem_size;
+ fw.engine_mask = r16(rom + off + 36);
+ fw.ucode_id = rom[off + 38];
+ fw.sig_count = rom[off + 39];
+ fw.sig_versions = r16(rom + off + 40);
+ fw.sig_off = off + 44;
+ fw.sig_bytes = size - 44;
+ GU_NOTE("FWSEC geometry: image_off=%#x size=%#x imem=%#x va=%#x pa=%#x dmem=%#x pa=%#x pkc=%#x interface=%#x engine=%#x ucode=%u signatures=%u versions=%#x bytes=%u\n",
+ fw.image_off, fw.image_size, fw.imem_size, fw.imem_va, fw.imem_pa,
+ fw.dmem_size, fw.dmem_pa, fw.pkc_off, fw.interface_off, fw.engine_mask,
+ fw.ucode_id, fw.sig_count, fw.sig_versions, fw.sig_bytes);
+ bits = fw.sig_versions;
+ while (bits) {
+ count += bits & 1;
+ bits >>= 1;
+ }
+ if (!fw.ucode_id || fw.ucode_id > 16 || !fw.engine_mask || !count ||
+ count != fw.sig_count || fw.sig_bytes != count * 384 ||
+ !fw.imem_size || !fw.dmem_size ||
+ range(fw.image_size, fw.data_off, fw.dmem_size) ||
+ range(fw.dmem_size, fw.pkc_off, 384) ||
+ range(fw.dmem_size, fw.interface_off, 4))
+ return GU_DIAG(-EINVAL);
+ *out = fw;
+ return 0;
+}
+
+int gpu_unlock_vbios_fwsec(const u8 *rom, size_t length, struct gpu_unlock_fwsec *out)
+{
+ u32 bit, adjust;
+ int ret;
+
+ if (!rom || !out || length > 1024 * 1024)
+ return GU_DIAG(-EINVAL);
+ memset(out, 0, sizeof(*out));
+ ret = expansion(rom, length, &adjust);
+ if (ret)
+ return GU_DIAG(ret);
+ for (bit = 0; bit + 12 <= length; bit++) {
+ u32 size, stride, tokens, sum = 0, i;
+
+ if (r16(rom + bit) != 0xb8ff || r32(rom + bit + 2) != 0x00544942)
+ continue;
+ size = rom[bit + 8];
+ stride = rom[bit + 9];
+ tokens = rom[bit + 10];
+ if (size < 12 || stride < 6 || range(length, bit, size))
+ return GU_DIAG(-EINVAL);
+ for (i = 0; i < size; i++)
+ sum += rom[bit + i];
+ if (sum & 0xff)
+ continue;
+ GU_NOTE("ROM BIT: offset=%#x size=%u stride=%u tokens=%u adjust=%#x\n",
+ bit, size, stride, tokens, adjust);
+ if (range(length, (u64)bit + size, (u64)tokens * stride))
+ return GU_DIAG(-EINVAL);
+ for (i = 0; i < tokens; i++) {
+ const u8 *token = rom + bit + size + i * stride;
+ u32 ptr, j;
+ u64 table;
+
+ if (token[0] != 0x70 || token[1] != 2 || r16(token + 2) < 4)
+ continue;
+ ptr = stride >= 8 ? r32(token + 4) : r16(token + 4);
+ if (range(length, ptr, r16(token + 2)))
+ return GU_DIAG(-EINVAL);
+ table = (u64)adjust + r32(rom + ptr);
+ if (range(length, table, 6))
+ return GU_DIAG(-EINVAL);
+ GU_NOTE("ROM ucode table: offset=%#llx version=%u header=%u stride=%u entries=%u\n",
+ (unsigned long long)table, rom[table], rom[table+1], rom[table+2], rom[table+3]);
+ if (rom[table] != 1 || rom[table + 1] < 6 || rom[table + 2] < 6)
+ return GU_DIAG(-EOPNOTSUPP);
+ if (range(length, table + rom[table + 1],
+ (u64)rom[table + 2] * rom[table + 3]))
+ return GU_DIAG(-EINVAL);
+ for (j = 0; j < rom[table + 3]; j++) {
+ const u8 *entry = rom + table + rom[table + 1] + j * rom[table + 2];
+
+ if (entry[0] == 0x85)
+ return GU_DIAG(descriptor(rom, length,
+ (u64)adjust + r32(entry + 2), out));
+ }
+ }
+ }
+ return GU_DIAG(-ENOENT);
+}
+
+int gpu_unlock_fwsec_sig_offset(u32 versions, u32 fuse_version, u32 sig_size, u32 *offset)
+{
+ u32 mask, n = 0;
+
+ GU_NOTE("FWSEC signature selection: versions=%#x fuse_version=%u sig_bytes=%u\n",
+ versions, fuse_version, sig_size);
+ if (!offset || !sig_size || fuse_version >= 16 || versions > 0xffff)
+ return GU_DIAG(-EINVAL);
+ if (!(versions & (1u << fuse_version)))
+ return GU_DIAG(-ERANGE);
+ mask = versions & ((1u << fuse_version) - 1);
+ while (mask) {
+ n += mask & 1;
+ mask >>= 1;
+ }
+ if (n && sig_size > ~(u32)0 / n)
+ return GU_DIAG(-EOVERFLOW);
+ *offset = n * sig_size;
+ return 0;
+}
+
+int gpu_unlock_fwsec_patch_frts(u8 *image, const struct gpu_unlock_fwsec *fw, u64 frts_offset)
+{
+ u8 *dmem, *mapper = NULL;
+ u32 entries, i, cmd_off, cmd_size;
+ u64 off;
+ u8 cmd[44] = { 0 };
+
+ if (!image || !fw || !frts_offset || (frts_offset & 0xfff) ||
+ (frts_offset >> 12) > ~(u32)0 ||
+ range(fw->image_size, fw->data_off, fw->dmem_size) ||
+ range(fw->dmem_size, fw->interface_off, 4))
+ return GU_DIAG(-EINVAL);
+ dmem = image + fw->data_off;
+ GU_NOTE("FWSEC interface: offset=%#x version=%u header=%u stride=%u entries=%u FRTS=%#llx\n",
+ fw->interface_off, dmem[fw->interface_off], dmem[fw->interface_off+1],
+ dmem[fw->interface_off+2], dmem[fw->interface_off+3], (unsigned long long)frts_offset);
+ if (dmem[fw->interface_off] != 1 || dmem[fw->interface_off + 1] != 4 ||
+ dmem[fw->interface_off + 2] != 8)
+ return GU_DIAG(-EOPNOTSUPP);
+ entries = dmem[fw->interface_off + 3];
+ off = (u64)fw->interface_off + 4;
+ if (entries < 2 || range(fw->dmem_size, off, (u64)entries * 8))
+ return GU_DIAG(-EINVAL);
+ for (i = 0; i < entries; i++, off += 8) {
+ u32 dest = r32(dmem + off + 4);
+
+ if (r32(dmem + off) == 4) {
+ if (mapper || range(fw->dmem_size, dest, 64))
+ return GU_DIAG(-EINVAL);
+ mapper = dmem + dest;
+ }
+ }
+ if (!mapper)
+ return GU_DIAG(-ENOENT);
+ GU_NOTE("FWSEC mapper: version=%u size=%u cmd_offset=%#x cmd_size=%u\n",
+ r16(mapper+4), r16(mapper+6), r32(mapper+8), r32(mapper+12));
+ /* Only the audited V3 mapper layout is supported. */
+ if (r16(mapper + 4) != 3 || r16(mapper + 6) < 64)
+ return GU_DIAG(-EOPNOTSUPP);
+ cmd_off = r32(mapper + 8);
+ cmd_size = r32(mapper + 12);
+ if (cmd_size < sizeof(cmd) || range(fw->dmem_size, cmd_off, cmd_size))
+ return GU_DIAG(-EINVAL);
+ w32(cmd, 1);
+ w32(cmd + 4, 24);
+ w32(cmd + 20, 2);
+ w32(cmd + 24, 1);
+ w32(cmd + 28, 20);
+ w32(cmd + 32, frts_offset >> 12);
+ w32(cmd + 36, 0x100);
+ w32(cmd + 40, 2);
+ memcpy(dmem + cmd_off, cmd, sizeof(cmd));
+ w32(mapper + 44, 0x15);
+ return 0;
+}
diff --git a/kernel/gpu_unlock/vbios_610.h b/kernel/gpu_unlock/vbios_610.h
new file mode 100644
index 0000000..89413d6
--- /dev/null
+++ b/kernel/gpu_unlock/vbios_610.h
@@ -0,0 +1,42 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_VBIOS_610_H
+#define GPU_UNLOCK_VBIOS_610_H
+
+#ifdef __KERNEL__
+#include <linux/types.h>
+#else
+#include <stddef.h>
+#include <stdint.h>
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+#endif
+
+/* Production FWSEC descriptor extracted from a VBIOS image. */
+struct gpu_unlock_fwsec {
+ u32 image_off;
+ u32 image_size;
+ u32 imem_pa;
+ u32 imem_size;
+ u32 imem_va;
+ u32 dmem_pa;
+ u32 data_off;
+ u32 dmem_size;
+ u32 pkc_off;
+ u32 interface_off;
+ u32 ucode_id;
+ u32 engine_mask;
+ u32 sig_count;
+ u32 sig_versions;
+ u32 sig_off;
+ u32 sig_bytes;
+};
+
+int gpu_unlock_vbios_fwsec(const u8 *rom, size_t length, struct gpu_unlock_fwsec *out);
+int gpu_unlock_fwsec_patch_frts(u8 *image, const struct gpu_unlock_fwsec *fw,
+ u64 frts_offset);
+int gpu_unlock_fwsec_sig_offset(u32 versions, u32 fuse_version, u32 sig_size,
+ u32 *offset);
+
+#endif
diff --git a/licenses/CMP40HX-MIT.txt b/licenses/CMP40HX-MIT.txt
new file mode 100644
index 0000000..e8a9ddc
--- /dev/null
+++ b/licenses/CMP40HX-MIT.txt
@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2026
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
diff --git a/licenses/GPL-2.0.txt b/licenses/GPL-2.0.txt
new file mode 100644
index 0000000..d159169
--- /dev/null
+++ b/licenses/GPL-2.0.txt
@@ -0,0 +1,339 @@
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+freedom to share and change it. By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users. This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it. (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.) You can apply it to
+your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have. You must make sure that they, too, receive or can get the
+source code. And you must show them these terms so they know their
+rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software. If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+patents. We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary. To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License. The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language. (Hereinafter, translation is included without limitation in
+the term "modification".) Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope. The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole. If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works. But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it. For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable. However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License. Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+signed it. However, nothing else grants you permission to modify or
+distribute the Program or its derivative works. These actions are
+prohibited by law if you do not accept this License. Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions. You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all. For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices. Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded. In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time. Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number. If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation. If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission. For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this. Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License. Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs. If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library. If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
diff --git a/licenses/NVIDIA-MIT.txt b/licenses/NVIDIA-MIT.txt
new file mode 100644
index 0000000..56baef3
--- /dev/null
+++ b/licenses/NVIDIA-MIT.txt
@@ -0,0 +1,22 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD
new file mode 100644
index 0000000..ca3b2de
--- /dev/null
+++ b/packaging/arch/PKGBUILD
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: GPL-2.0-only
+# Builds from the committed HEAD of the checkout this file lives in:
+# cd packaging/arch && makepkg -si
+# For a published release, point the first source at the tagged tarball.
+pkgbase=gpu-unlock
+pkgname=(gpu-unlock-dkms gpu-unlock-firmware)
+pkgver=1.0.0
+pkgrel=1
+_nv=610.43.03
+_utils=nvidia-utils-$_nv-1-x86_64_v3.pkg.tar.zst
+pkgdesc='RTX 3090 BF16 unlock before the NVIDIA driver loads'
+arch=('x86_64')
+url=''
+license=('GPL-2.0-only')
+makedepends=('git' 'python' 'zstd')
+options=('!strip' '!debug')
+source=("$pkgbase::git+file://${startdir%/packaging/arch}"
+ "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-$_nv.tar.xz"
+ "https://archive.cachyos.org/archive/cachyos-v3/$_utils")
+noextract=("$_utils")
+sha256sums=('SKIP'
+ '7e118923c7a23edc36114d63273a46e3e04e9af98695a42203e7ac2dfe9fc1dc'
+ 'e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9')
+
+prepare() {
+ cd $pkgbase
+ test "$(cat VERSION)" = "$pkgver"
+}
+
+build() {
+ cd $pkgbase
+ python firmware/generate_v67.py "$(realpath "$srcdir/$_utils")" "$srcdir/assets"
+ python firmware/extract_boot_assets.py "$srcdir/NVIDIA-kernel-module-source-$_nv" "$srcdir/assets"
+}
+
+check() {
+ cd $pkgbase
+ make test ASSETS="$srcdir/assets"
+}
+
+package_gpu-unlock-dkms() {
+ arch=('any')
+ depends=('dkms' "gpu-unlock-firmware=$pkgver-$pkgrel")
+ backup=('etc/mkinitcpio.conf.d/zz-gpu-unlock.conf')
+ conflicts=('gpu-unlock-mkinitcpio')
+ replaces=('gpu-unlock-mkinitcpio')
+ install=gpu-unlock.install
+ cd $pkgbase
+ make DESTDIR="$pkgdir" install-dkms install-mkinitcpio
+ make DESTDIR="$pkgdir" PKG=$pkgname install-licenses
+}
+
+package_gpu-unlock-firmware() {
+ pkgdesc='Locally generated, pinned NVIDIA 610.43.03 firmware assets for gpu-unlock'
+ license=('LicenseRef-NVIDIA' 'GPL-2.0-only')
+ cd $pkgbase
+ make DESTDIR="$pkgdir" ASSETS="$srcdir/assets" install-firmware
+}
diff --git a/packaging/arch/gpu-unlock.install b/packaging/arch/gpu-unlock.install
new file mode 100644
index 0000000..1ea86c0
--- /dev/null
+++ b/packaging/arch/gpu-unlock.install
@@ -0,0 +1,10 @@
+post_install() {
+ cat <<'EOF'
+==> gpu_unlock now loads ahead of nvidia on every boot but does nothing until
+==> you opt in on the kernel command line:
+==> gpu_unlock.targets=all every unbound RTX 3090
+==> gpu_unlock.targets=0000:21:00.0 only these PCI addresses (comma-separated)
+==> Remove the option in your bootloader to boot without the unlock.
+==> See /usr/share/doc/gpu-unlock/README.md.
+EOF
+}
diff --git a/packaging/boot/dracut.conf b/packaging/boot/dracut.conf
new file mode 100644
index 0000000..58e46e3
--- /dev/null
+++ b/packaging/boot/dracut.conf
@@ -0,0 +1,4 @@
+# Put gpu_unlock and its firmware in the initramfs and load it before udev
+# can load nvidia. It does nothing unless gpu_unlock.targets= is on the
+# kernel command line.
+force_drivers+=" gpu_unlock "
diff --git a/packaging/boot/initramfs-tools.conf b/packaging/boot/initramfs-tools.conf
new file mode 100644
index 0000000..98b30c7
--- /dev/null
+++ b/packaging/boot/initramfs-tools.conf
@@ -0,0 +1,3 @@
+# Put gpu_unlock and its firmware in the initramfs. It does nothing unless
+# gpu_unlock.targets= is on the kernel command line.
+gpu_unlock
diff --git a/packaging/boot/mkinitcpio.conf b/packaging/boot/mkinitcpio.conf
new file mode 100644
index 0000000..a7027d8
--- /dev/null
+++ b/packaging/boot/mkinitcpio.conf
@@ -0,0 +1,16 @@
+# Put gpu_unlock and its firmware in every image and load it right before
+# nvidia, so a display card's screen is blank only from its unlock until
+# nvidia-drm takes over. Without nvidia in the image it loads last, still
+# before any nvidia loaded from the root filesystem. Named to be read after
+# every other drop-in. It does nothing unless gpu_unlock.targets= is on the
+# kernel command line.
+[[ ${MODULES@a} == *a* ]] || read -ra MODULES <<<"$MODULES"
+_gu_modules=()
+for _gu_module in "${MODULES[@]}"; do
+ [[ ${_gu_module%\?} == gpu_unlock ]] && continue
+ [[ ${_gu_module%\?} == nvidia ]] && _gu_modules+=(gpu_unlock?)
+ _gu_modules+=("$_gu_module")
+done
+[[ " ${_gu_modules[*]} " == *' gpu_unlock? '* ]] || _gu_modules+=(gpu_unlock?)
+MODULES=("${_gu_modules[@]}")
+unset _gu_modules _gu_module
diff --git a/packaging/rpm/gpu-unlock.rpmlintrc b/packaging/rpm/gpu-unlock.rpmlintrc
new file mode 100644
index 0000000..3907866
--- /dev/null
+++ b/packaging/rpm/gpu-unlock.rpmlintrc
@@ -0,0 +1,6 @@
+# Pinned GPU firmware is a device image, not a host program.
+addFilter(r"^gpu-unlock-firmware.*: [WE]: (unstripped-binary-or-object|statically-linked-binary|missing-PT_GNU_STACK-section)")
+# The DKMS package intentionally ships source and configuration only.
+addFilter(r"^gpu-unlock-dkms.*: E: no-binary")
+# Retain the unmodified GPL-2.0 notice required by the source provenance.
+addFilter(r"^gpu-unlock-dkms.*: E: incorrect-fsf-address")
diff --git a/packaging/rpm/gpu-unlock.spec b/packaging/rpm/gpu-unlock.spec
new file mode 100644
index 0000000..a17273d
--- /dev/null
+++ b/packaging/rpm/gpu-unlock.spec
@@ -0,0 +1,78 @@
+%global nv 610.43.03
+
+Name: gpu-unlock
+Version: 1.0.0
+Release: 1%{?dist}
+Summary: RTX 3090 BF16 unlock before the NVIDIA driver loads
+License: GPL-2.0-only
+BuildArch: noarch
+Source0: gpu-unlock-%{version}.tar.xz
+Source1: https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-%{nv}.tar.xz
+Source2: https://archive.cachyos.org/archive/cachyos-v3/nvidia-utils-%{nv}-1-x86_64_v3.pkg.tar.zst
+# NVIDIA inputs stay out of the source RPM (it becomes a .nosrc.rpm).
+NoSource: 1
+NoSource: 2
+BuildRequires: make
+BuildRequires: gcc
+BuildRequires: python3
+BuildRequires: zstd
+
+%description
+Kernel module that unlocks RTX 3090 BF16 throughput before the unmodified
+NVIDIA driver binds, opted in on the kernel command line.
+
+%package dkms
+Summary: RTX 3090 BF16 unlock kernel source
+Requires: dkms
+Requires: gpu-unlock-firmware = %{version}-%{release}
+
+%description dkms
+DKMS source for gpu_unlock, loaded ahead of nvidia on every boot. It does
+nothing unless gpu_unlock.targets= is on the kernel command line.
+
+%package firmware
+Summary: Locally generated pinned NVIDIA firmware for GPU unlock
+License: LicenseRef-NVIDIA AND GPL-2.0-only
+
+%description firmware
+Firmware assets generated locally from verified upstream inputs.
+
+%prep
+sha256sum -c - <<EOF
+7e118923c7a23edc36114d63273a46e3e04e9af98695a42203e7ac2dfe9fc1dc %{SOURCE1}
+e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9 %{SOURCE2}
+EOF
+%autosetup
+test "$(cat VERSION)" = "%{version}"
+
+%build
+%make_build INPUTS=%{_sourcedir} ASSETS="$PWD/assets" firmware
+
+%check
+make test ASSETS="$PWD/assets"
+
+%install
+make DESTDIR=%{buildroot} ASSETS="$PWD/assets" \
+ install-dkms install-firmware install-dracut
+
+%files dkms
+%license licenses/GPL-2.0.txt licenses/CMP40HX-MIT.txt licenses/NVIDIA-MIT.txt
+%doc %{_docdir}/gpu-unlock
+%{_usrsrc}/gpu-unlock-%{version}
+%{_prefix}/lib/dracut/dracut.conf.d/gpu-unlock.conf
+
+%files firmware
+%license %{_defaultlicensedir}/gpu-unlock-firmware
+%{_prefix}/lib/firmware/gpu-unlock
+
+%post dkms
+dkms add -m gpu-unlock -v %{version} --rpm_safe_upgrade || :
+
+%preun dkms
+if [ "$1" -eq 0 ] && [ -d /var/lib/dkms/gpu-unlock/%{version} ]; then
+ dkms remove -m gpu-unlock -v %{version} --all --rpm_safe_upgrade || :
+fi
+
+%changelog
+* Mon Sep 28 2026 Zagitta <[email protected]> - 1.0.0-1
+- Initial release.
diff --git a/tests/main_shim/compat.h b/tests/main_shim/compat.h
new file mode 100644
index 0000000..0e78a16
--- /dev/null
+++ b/tests/main_shim/compat.h
@@ -0,0 +1,55 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Just enough kernel for main.c. Async work runs on real threads, so the
+ * test sees true concurrency and main.c's own synchronization. */
+#ifndef MAIN_SHIM_H
+#define MAIN_SHIM_H
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+
+#define __init
+#define __exit
+#define module_param(name, type, perm)
+#define MODULE_PARM_DESC(name, desc)
+#define MODULE_LICENSE(x)
+#define MODULE_DESCRIPTION(x)
+#define MODULE_VERSION(x)
+/* Keep the entry points reachable so the test calls what the kernel would. */
+#define module_init(fn) int (*test_module_init)(void) = fn;
+#define module_exit(fn) void (*test_module_exit)(void) = fn;
+
+#define PCI_VENDOR_ID_NVIDIA 0x10de
+struct pci_dev {
+ const char *name;
+ unsigned short vendor, device;
+ void *driver;
+ int refs;
+ pthread_mutex_t lock;
+ pthread_t owner;
+ bool locked;
+};
+struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, struct pci_dev *from);
+struct pci_dev *pci_dev_get(struct pci_dev *);
+void pci_dev_put(struct pci_dev *);
+void pci_dev_lock(struct pci_dev *);
+void pci_dev_unlock(struct pci_dev *);
+static inline const char *pci_name(const struct pci_dev *pdev) { return pdev->name; }
+
+void test_log(const struct pci_dev *, const char *, ...) __attribute__((format(printf, 2, 3)));
+#define pci_err(pdev, ...) test_log(pdev, __VA_ARGS__)
+#define pci_info(pdev, ...) test_log(pdev, __VA_ARGS__)
+#define pr_err(...) test_log(NULL, __VA_ARGS__)
+
+typedef uint64_t async_cookie_t;
+typedef void (*async_func_t)(void *data, async_cookie_t cookie);
+struct async_domain {
+ pthread_mutex_t lock;
+ pthread_t threads[16];
+ unsigned int count;
+};
+#define ASYNC_DOMAIN_EXCLUSIVE(name) struct async_domain name = { .lock = PTHREAD_MUTEX_INITIALIZER }
+async_cookie_t async_schedule_domain(async_func_t, void *, struct async_domain *);
+void async_synchronize_full_domain(struct async_domain *);
+#endif
diff --git a/tests/main_shim/linux/async.h b/tests/main_shim/linux/async.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/main_shim/linux/async.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/main_shim/linux/init.h b/tests/main_shim/linux/init.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/main_shim/linux/init.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/main_shim/linux/module.h b/tests/main_shim/linux/module.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/main_shim/linux/module.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/main_shim/linux/pci.h b/tests/main_shim/linux/pci.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/main_shim/linux/pci.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/main_shim/linux/string.h b/tests/main_shim/linux/string.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/main_shim/linux/string.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/test_gfw_eval.c b/tests/test_gfw_eval.c
new file mode 100644
index 0000000..4596e75
--- /dev/null
+++ b/tests/test_gfw_eval.c
@@ -0,0 +1,48 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+
+#include "gfw_eval.h"
+
+int main(void)
+{
+ struct gpu_unlock_gfw_sample sample = {
+ .config_ok = true,
+ .plm = GU_GFW_PLM_LEVEL0,
+ .boot = GU_GFW_BOOT_COMPLETED,
+ .cpuctl = GU_GFW_CPUCTL_HALTED,
+ };
+
+ assert(gpu_unlock_gfw_assess(&sample) == 0);
+ sample.cpuctl = 0;
+ assert(gpu_unlock_gfw_assess(&sample) == -EAGAIN);
+ sample.cpuctl = GU_GFW_CPUCTL_HALTED;
+ sample.plm = 0;
+ assert(gpu_unlock_gfw_assess(&sample) == -EAGAIN);
+ sample.plm = GU_GFW_PLM_LEVEL0;
+ sample.boot = 0xfe;
+ assert(gpu_unlock_gfw_assess(&sample) == -EAGAIN);
+ sample.boot = (u32)~0U;
+ sample.plm = (u32)~0U;
+ sample.cpuctl = (u32)~0U;
+ assert(gpu_unlock_gfw_assess(&sample) == -ENODEV);
+ sample.boot = GU_GFW_BOOT_COMPLETED;
+ sample.plm = GU_GFW_PLM_LEVEL0;
+ sample.cpuctl = GU_GFW_CPUCTL_HALTED;
+ sample.config_ok = false;
+ assert(gpu_unlock_gfw_assess(&sample) == -ENODEV);
+
+ assert(gpu_unlock_command_restore(0, GU_PCI_COMMAND_MEMORY |
+ GU_PCI_COMMAND_MASTER) == 0);
+ assert(gpu_unlock_command_restore(GU_PCI_COMMAND_MEMORY,
+ GU_PCI_COMMAND_MEMORY |
+ GU_PCI_COMMAND_MASTER) ==
+ GU_PCI_COMMAND_MEMORY);
+ assert(gpu_unlock_command_restore(GU_PCI_COMMAND_IO,
+ GU_PCI_COMMAND_IO | GU_PCI_COMMAND_MEMORY) ==
+ GU_PCI_COMMAND_IO);
+
+ puts("gfw predicate and command restore checks passed");
+ return 0;
+}
diff --git a/tests/test_layout_610.c b/tests/test_layout_610.c
new file mode 100644
index 0000000..c59e589
--- /dev/null
+++ b/tests/test_layout_610.c
@@ -0,0 +1,35 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "layout_610.h"
+
+int main(void)
+{
+ struct gpu_unlock_layout_in in = {
+ .usable_fb = 24ull << 30,
+ .vga_base = (24ull << 30) - (1024ull * 1024),
+ .frts_size = 1024ull * 1024,
+ .boot_size = 4096,
+ .image_size = 64ull * 1024,
+ .nonwpr_size = 1,
+ .heap_request = 1024ull * 1024,
+ };
+ struct gpu_unlock_layout_out out;
+
+ assert(gpu_unlock_layout(NULL, &out) == -EINVAL);
+ assert(!gpu_unlock_layout(&in, &out));
+ assert(out.wpr_end > out.frts_offset && out.frts_offset > out.boot_offset);
+ assert(out.boot_offset > out.gsp_offset && out.gsp_offset > out.heap_offset);
+ assert(out.heap_offset > out.wpr_start && out.wpr_start > out.nonwpr_start);
+ assert((out.wpr_end & ((128ull * 1024) - 1)) == 0);
+ assert(out.heap_size >= in.heap_request);
+ /* Reservations that do not fit below the VGA workspace fail cleanly. */
+ in.frts_size = in.vga_base;
+ assert(gpu_unlock_layout(&in, &out) == -EINVAL ||
+ gpu_unlock_layout(&in, &out) == -ERANGE);
+ puts("layout checks passed");
+ return 0;
+}
diff --git a/tests/test_main.c b/tests/test_main.c
new file mode 100644
index 0000000..52f75e6
--- /dev/null
+++ b/tests/test_main.c
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* main.c against mocked PCI and a mocked unlock: which cards are unlocked,
+ * that they run concurrently, and that the shared firmware outlives them. */
+#define _GNU_SOURCE
+#include <assert.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+#include "../kernel/gpu_unlock/main.c"
+
+#define MAX_DEVICES 8
+
+static struct pci_dev devices[MAX_DEVICES];
+static unsigned int device_count;
+static char gsp_marker;
+#define fake_gsp ((const struct firmware *)&gsp_marker)
+static pthread_mutex_t fake_lock = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t fake_cond = PTHREAD_COND_INITIALIZER;
+static struct {
+ int loads, releases, fail_load;
+ struct pci_dev *load_dev;
+ int started, finished, in_flight, max_in_flight;
+ int expected_concurrent; /* unlocks that must overlap */
+ int unlocks[MAX_DEVICES], fail_unlock[MAX_DEVICES];
+ char log[16384];
+} fake;
+
+void test_log(const struct pci_dev *pdev, const char *format, ...)
+{
+ va_list args;
+ size_t at;
+
+ pthread_mutex_lock(&fake_lock);
+ at = strlen(fake.log);
+ if (pdev)
+ at += snprintf(fake.log + at, sizeof(fake.log) - at, "%s: ", pdev->name);
+ va_start(args, format);
+ vsnprintf(fake.log + at, sizeof(fake.log) - at, format, args);
+ va_end(args);
+ pthread_mutex_unlock(&fake_lock);
+}
+
+struct pci_dev *pci_dev_get(struct pci_dev *pdev)
+{
+ __atomic_add_fetch(&pdev->refs, 1, __ATOMIC_SEQ_CST);
+ return pdev;
+}
+
+void pci_dev_put(struct pci_dev *pdev)
+{
+ if (pdev)
+ assert(__atomic_sub_fetch(&pdev->refs, 1, __ATOMIC_SEQ_CST) >= 1);
+}
+
+/* Like the kernel: returns the next match with a reference, drops from's. */
+struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, struct pci_dev *from)
+{
+ unsigned int i = from ? (unsigned int)(from - devices) + 1 : 0;
+
+ pci_dev_put(from);
+ for (; i < device_count; i++)
+ if (devices[i].vendor == vendor && devices[i].device == device)
+ return pci_dev_get(&devices[i]);
+ return NULL;
+}
+
+void pci_dev_lock(struct pci_dev *pdev)
+{
+ pthread_mutex_lock(&pdev->lock);
+ pdev->owner = pthread_self();
+ pdev->locked = true;
+}
+
+void pci_dev_unlock(struct pci_dev *pdev)
+{
+ assert(pdev->locked && pthread_equal(pdev->owner, pthread_self()));
+ pdev->locked = false;
+ pthread_mutex_unlock(&pdev->lock);
+}
+
+struct async_entry {
+ async_func_t fn;
+ void *data;
+ async_cookie_t cookie;
+};
+
+static void *async_thread(void *arg)
+{
+ struct async_entry *entry = arg;
+
+ entry->fn(entry->data, entry->cookie);
+ free(entry);
+ return NULL;
+}
+
+async_cookie_t async_schedule_domain(async_func_t fn, void *data, struct async_domain *domain)
+{
+ struct async_entry *entry = malloc(sizeof(*entry));
+ async_cookie_t cookie;
+
+ assert(entry);
+ pthread_mutex_lock(&domain->lock);
+ assert(domain->count < sizeof(domain->threads) / sizeof(domain->threads[0]));
+ cookie = domain->count + 1;
+ *entry = (struct async_entry){ fn, data, cookie };
+ assert(!pthread_create(&domain->threads[domain->count++], NULL, async_thread, entry));
+ pthread_mutex_unlock(&domain->lock);
+ return cookie;
+}
+
+void async_synchronize_full_domain(struct async_domain *domain)
+{
+ pthread_mutex_lock(&domain->lock);
+ for (unsigned int i = 0; i < domain->count; i++)
+ assert(!pthread_join(domain->threads[i], NULL));
+ domain->count = 0;
+ pthread_mutex_unlock(&domain->lock);
+}
+
+int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared)
+{
+ pthread_mutex_lock(&fake_lock);
+ /* Loaded once, before any unlock can use it. */
+ assert(!fake.started && !shared->gsp);
+ fake.loads++;
+ fake.load_dev = pdev;
+ pthread_mutex_unlock(&fake_lock);
+ if (fake.fail_load)
+ return fake.fail_load;
+ shared->gsp = fake_gsp;
+ return 0;
+}
+
+void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
+{
+ pthread_mutex_lock(&fake_lock);
+ /* Only after every unlock has returned. */
+ assert(!fake.in_flight && fake.finished == fake.started);
+ assert(shared->gsp == (fake.loads && !fake.fail_load ? fake_gsp : NULL));
+ fake.releases++;
+ shared->gsp = NULL;
+ pthread_mutex_unlock(&fake_lock);
+}
+
+int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared)
+{
+ unsigned int index = pdev - devices;
+ struct timespec deadline;
+ int ret;
+
+ assert(shared->gsp == fake_gsp);
+ assert(pdev->locked && pthread_equal(pdev->owner, pthread_self()) && !pdev->driver);
+ /* The initial reference, the worker's, and nothing dropped early. */
+ assert(__atomic_load_n(&pdev->refs, __ATOMIC_SEQ_CST) >= 2);
+ pthread_mutex_lock(&fake_lock);
+ fake.unlocks[index]++;
+ fake.started++;
+ if (++fake.in_flight > fake.max_in_flight)
+ fake.max_in_flight = fake.in_flight;
+ pthread_cond_broadcast(&fake_cond);
+ /* Wait until every expected card is inside an unlock at once. Serial
+ * unlocking never gets there and falls through on the timeout. */
+ clock_gettime(CLOCK_REALTIME, &deadline);
+ deadline.tv_sec += 2;
+ while (fake.max_in_flight < fake.expected_concurrent &&
+ pthread_cond_timedwait(&fake_cond, &fake_lock, &deadline) != ETIMEDOUT)
+ ;
+ pthread_mutex_unlock(&fake_lock);
+ /* Still busy after the rendezvous, so an unsynchronized init would
+ * return, and release the firmware, while this card is mid-unlock. */
+ usleep(20000);
+ pthread_mutex_lock(&fake_lock);
+ fake.in_flight--;
+ fake.finished++;
+ ret = fake.fail_unlock[index];
+ pthread_mutex_unlock(&fake_lock);
+ return ret;
+}
+
+static int driver_marker;
+
+static void add(const char *name, unsigned short device, bool bound)
+{
+ struct pci_dev *pdev = &devices[device_count++];
+
+ *pdev = (struct pci_dev){ .name = name, .vendor = PCI_VENDOR_ID_NVIDIA, .device = device,
+ .driver = bound ? &driver_marker : NULL, .refs = 1 };
+ pthread_mutex_init(&pdev->lock, NULL);
+}
+
+static void reset(char *list)
+{
+ memset(&fake, 0, sizeof(fake));
+ device_count = 0;
+ targets = list;
+}
+
+/* Runs init, then checks what holds for every case. */
+static void run(void)
+{
+ assert(!test_module_init());
+ test_module_exit();
+ assert(!fake.in_flight && fake.finished == fake.started);
+ for (unsigned int i = 0; i < device_count; i++) {
+ assert(devices[i].refs == 1 && !devices[i].locked);
+ assert(fake.unlocks[i] <= 1);
+ }
+ assert(fake.loads <= 1 && fake.releases <= 1);
+}
+
+static bool logged(const char *text)
+{
+ return strstr(fake.log, text);
+}
+
+int main(void)
+{
+ /* Every unbound RTX 3090 at once; other models and bound cards left alone. */
+ reset("all");
+ add("0000:01:00.0", RTX_3090, false);
+ add("0000:21:00.0", RTX_3090, false);
+ add("0000:31:00.0", 0x2208, false);
+ add("0000:41:00.0", RTX_3090, true);
+ add("0000:61:00.0", RTX_3090, false);
+ fake.expected_concurrent = 3;
+ run();
+ assert(fake.max_in_flight == 3 && fake.started == 3);
+ assert(fake.unlocks[0] && fake.unlocks[1] && !fake.unlocks[2] && !fake.unlocks[3] && fake.unlocks[4]);
+ assert(fake.loads == 1 && fake.load_dev == &devices[0] && fake.releases == 1);
+ assert(logged("0000:01:00.0: gpu_unlock: unlocked") && logged("0000:61:00.0: gpu_unlock: unlocked"));
+ assert(logged("0000:41:00.0: gpu_unlock: unlock failed (-16)"));
+ puts("all: three cards unlocked concurrently, bound card skipped PASS");
+
+ /* A list picks cards in any case; the rest are not touched. */
+ reset("0000:61:00.0,0000:0A:00.0");
+ add("0000:01:00.0", RTX_3090, false);
+ add("0000:0a:00.0", RTX_3090, false);
+ add("0000:61:00.0", RTX_3090, false);
+ fake.expected_concurrent = 2;
+ run();
+ assert(fake.max_in_flight == 2 && !fake.unlocks[0] && fake.unlocks[1] && fake.unlocks[2]);
+ assert(fake.loads == 1 && fake.load_dev == &devices[1] && fake.releases == 1);
+ assert(!logged("no RTX 3090"));
+ puts("list: only listed cards, concurrently PASS");
+
+ /* One card failing leaves the others unlocked. */
+ reset("all");
+ add("0000:01:00.0", RTX_3090, false);
+ add("0000:21:00.0", RTX_3090, false);
+ fake.fail_unlock[0] = -EIO;
+ fake.expected_concurrent = 2;
+ run();
+ assert(fake.max_in_flight == 2);
+ assert(logged("0000:01:00.0: gpu_unlock: unlock failed (-5)"));
+ assert(logged("0000:21:00.0: gpu_unlock: unlocked"));
+ puts("failure: other cards unaffected PASS");
+
+ /* Without the shared firmware no card is attempted. */
+ reset("all");
+ add("0000:01:00.0", RTX_3090, false);
+ add("0000:21:00.0", RTX_3090, false);
+ fake.fail_load = -ENOENT;
+ run();
+ assert(fake.loads == 1 && !fake.started && !fake.releases);
+ assert(logged("0000:01:00.0: gpu_unlock: firmware load failed (-2)"));
+ puts("firmware load failure: nothing attempted PASS");
+
+ /* Unmatched entries are reported; nothing to load for no cards. */
+ reset("0000:99:00.0");
+ add("0000:01:00.0", RTX_3090, false);
+ run();
+ assert(!fake.loads && !fake.started && fake.releases == 1);
+ assert(logged("gpu_unlock: no RTX 3090 at \"0000:99:00.0\""));
+ puts("unmatched target: reported, nothing loaded PASS");
+
+ /* No opt-in on the command line: the module does nothing. */
+ for (int empty = 0; empty < 2; empty++) {
+ reset(empty ? "" : NULL);
+ add("0000:01:00.0", RTX_3090, false);
+ run();
+ assert(!fake.loads && !fake.started && !fake.releases && !fake.log[0]);
+ }
+ puts("no targets: idle PASS");
+ return 0;
+}
diff --git a/tests/test_packaging.py b/tests/test_packaging.py
new file mode 100644
index 0000000..4b161c4
--- /dev/null
+++ b/tests/test_packaging.py
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Check the generated payload and the shared install layout."""
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "firmware"))
+import generate_v67 # noqa: E402
+
+
+class PackageTests(unittest.TestCase):
+ def test_v67_payload_matches_pin(self):
+ digest = hashlib.sha256(generate_v67.build_payload()).hexdigest()
+ self.assertEqual(digest, generate_v67.PAYLOAD_SHA256)
+ self.assertEqual(digest, self.manifest()["generated_payload"]["sha256"])
+
+ def test_install_uses_standard_paths(self):
+ version = (ROOT / "VERSION").read_text().strip()
+ with tempfile.TemporaryDirectory() as directory:
+ subprocess.run(["make", "-s", f"DESTDIR={directory}", "install-dkms",
+ "install-mkinitcpio", "install-initramfs-tools",
+ "install-dracut"],
+ cwd=ROOT, check=True)
+ root = Path(directory)
+ paths = {path.relative_to(root).as_posix() for path in root.rglob("*")
+ if path.is_file()}
+ for expected in (f"usr/src/gpu-unlock-{version}/dkms.conf",
+ f"usr/src/gpu-unlock-{version}/main.c",
+ "etc/mkinitcpio.conf.d/zz-gpu-unlock.conf",
+ "usr/share/initramfs-tools/modules.d/gpu-unlock.conf",
+ "usr/lib/dracut/dracut.conf.d/gpu-unlock.conf",
+ "usr/share/doc/gpu-unlock/README.md"):
+ self.assertIn(expected, paths)
+ self.assertFalse(any(path.endswith(".mod.c") for path in paths))
+ # Another package's "softdep nvidia" line would silently win over
+ # ours (kmod honours only the first), so the order comes from the
+ # initramfs instead: gpu_unlock goes right before nvidia.
+ self.assertFalse(any("modprobe.d" in path for path in paths))
+ dropin = root / "etc/mkinitcpio.conf.d/zz-gpu-unlock.conf"
+ for before, after in (("zfs nvidia nvidia_drm", "zfs gpu_unlock? nvidia nvidia_drm"),
+ ("zfs", "zfs gpu_unlock?")):
+ result = subprocess.run(["bash", "-c", f"MODULES=({before}); . {dropin}; echo ${{MODULES[*]}}"],
+ capture_output=True, text=True, check=True)
+ self.assertEqual(result.stdout.strip(), after)
+
+ def test_pkgbuild_pins_match_manifest(self):
+ manifest = json.loads((ROOT / "firmware/artifacts-610.43.03.json").read_text())
+ pkgbuild = (ROOT / "packaging/arch/PKGBUILD").read_text()
+ for index in (0, 2):
+ artifact = manifest["source_artifacts"][index]
+ self.assertIn(artifact["sha256"], pkgbuild)
+ self.assertIn(artifact["url"].rsplit("/", 1)[0], pkgbuild)
+ self.assertIn(f"pkgver={(ROOT / 'VERSION').read_text().strip()}\n", pkgbuild)
+
+ def test_native_versions_match(self):
+ version = (ROOT / "VERSION").read_text().strip()
+ spec = (ROOT / "packaging/rpm/gpu-unlock.spec").read_text()
+ self.assertIn(f"Version: {version}\n", spec)
+ for index in (0, 2):
+ self.assertIn(self.manifest()["source_artifacts"][index]["sha256"], spec)
+ changelog = (ROOT / "debian/changelog").read_text()
+ self.assertTrue(changelog.startswith(f"gpu-unlock ({version}) "))
+
+ @staticmethod
+ def manifest():
+ return json.loads((ROOT / "firmware/artifacts-610.43.03.json").read_text())
diff --git a/tests/test_prep_610.c b/tests/test_prep_610.c
new file mode 100644
index 0000000..49abe19
--- /dev/null
+++ b/tests/test_prep_610.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "layout_610.h"
+#include "prep_610.h"
+
+static void put16(unsigned char *p, unsigned int v)
+{
+ p[0] = v & 0xff;
+ p[1] = (v >> 8) & 0xff;
+}
+
+static void put32(unsigned char *p, unsigned int v)
+{
+ p[0] = v & 0xff;
+ p[1] = (v >> 8) & 0xff;
+ p[2] = (v >> 16) & 0xff;
+ p[3] = (v >> 24) & 0xff;
+}
+
+static void put64(unsigned char *p, unsigned long long v)
+{
+ unsigned int i;
+
+ for (i = 0; i < 8; i++)
+ p[i] = (unsigned char)(v >> (8 * i));
+}
+
+int main(void)
+{
+ unsigned char elf[256];
+ unsigned char image[64];
+ unsigned char sigs[8] = { 1, 2, 3, 4, 5, 6, 7, 8 };
+ unsigned char meta[GU_WPR_META_SIZE];
+ struct gpu_unlock_span span;
+ struct gpu_unlock_radix_plan radix;
+ struct gpu_unlock_booter_hdr hdr = {
+ .num_apps = 1,
+ .patch_loc = 16,
+ .num_sigs = 2,
+ .sig_bytes = 8,
+ };
+ struct gpu_unlock_layout_in in = {
+ .usable_fb = 24ull << 30,
+ .vga_base = (24ull << 30) - (1024ull * 1024),
+ .frts_size = 1024ull * 1024,
+ .boot_size = 4096,
+ .image_size = 64ull * 1024,
+ .nonwpr_size = 1,
+ .heap_request = 1024ull * 1024,
+ };
+ struct gpu_unlock_layout_out layout;
+ struct gpu_unlock_wpr_args args = {
+ .fb_size = in.usable_fb, .vga_offset = in.vga_base,
+ .frts_size = in.frts_size, .signature_dma = 0x1000,
+ .signature_size = 0xfa00, .radix_dma = 0x2000,
+ .image_size = in.image_size, .boot_dma = 0x3000,
+ .boot = { .code_offset = 256, .data_offset = 512, .image_size = 4096 },
+ };
+
+ assert(gpu_unlock_fuse_version(0) == 0);
+ assert(gpu_unlock_fuse_version(1) == 1);
+ assert(gpu_unlock_fuse_version(0x8) == 4);
+
+ memset(image, 0, sizeof(image));
+ assert(!gpu_unlock_booter_patch(image, sizeof(image), &hdr, sigs,
+ sizeof(sigs), 0));
+ assert(!memcmp(image + 16, sigs + 4, 4));
+ assert(gpu_unlock_booter_patch(image, sizeof(image), &hdr, sigs,
+ sizeof(sigs), 2) == -ERANGE);
+
+ memset(elf, 0, sizeof(elf));
+ elf[0] = 0x7f;
+ elf[1] = 'E';
+ elf[2] = 'L';
+ elf[3] = 'F';
+ elf[4] = 2;
+ elf[5] = 1;
+ elf[6] = 1;
+ put32(elf + 20, 1);
+ put16(elf + 52, 64);
+ put64(elf + 40, 64);
+ put16(elf + 58, 64);
+ put16(elf + 60, 2);
+ put16(elf + 62, 1);
+ put32(elf + 64, 0);
+ put64(elf + 64 + 24, 220);
+ put64(elf + 64 + 32, 4);
+ put32(elf + 128, 9);
+ put64(elf + 128 + 24, 192);
+ put64(elf + 128 + 32, 16);
+ memcpy(elf + 192, ".fwimage", 8);
+ elf[200] = 0;
+ memcpy(elf + 220, "name", 4);
+ assert(!gpu_unlock_elf_section(elf, sizeof(elf), ".fwimage", &span));
+ assert(span.offset == 220 && span.size == 4);
+ assert(gpu_unlock_elf_section(elf, sizeof(elf), ".missing", &span) == -ENOENT);
+
+ assert(!gpu_unlock_radix_plan(4096, &radix));
+ assert(radix.pages[0] == 1 && radix.pages[3] == 1);
+ assert(!gpu_unlock_radix_plan(4097, &radix));
+ assert(radix.pages[3] == 2);
+ assert(gpu_unlock_radix_plan(~(u64)0, &radix) == -EOVERFLOW);
+ /* A name that ends at the string-table boundary must include its NUL. */
+ put64(elf + 128 + 32, 8);
+ assert(gpu_unlock_elf_section(elf, sizeof(elf), ".fwimage", &span) == -EINVAL);
+ put64(elf + 128 + 32, 16);
+ put32(elf + 128, 0);
+ assert(gpu_unlock_elf_section(elf, sizeof(elf), ".fwimage", &span) == -EINVAL);
+ assert(gpu_unlock_dma_cmd(true) == ((6u << 8) | (1u << 4) | (1u << 2)));
+ assert(gpu_unlock_dma_cmd(false) == (6u << 8));
+
+ assert(!gpu_unlock_layout(&in, &layout));
+ assert(!gpu_unlock_wpr_meta_pack(meta, &layout, &args));
+ assert(meta[0] == 0xb3 && meta[7] == 0xdc);
+ assert(meta[72] == 0x00 && meta[73] == 0x10);
+ assert(meta[49] == 1 && meta[57] == 2);
+ assert(!memcmp(meta + 184, (unsigned char[]){0,0,0xf0,0xff,5,0,0,0}, 8));
+ args.image_size = ~0ull;
+ assert(gpu_unlock_wpr_meta_pack(meta, &layout, &args) == -EINVAL);
+ {
+ unsigned char desc[84] = {0}, page[GU_RADIX_PAGE];
+ u64 addresses[] = {0x123456000ull, 0xdeadbe000ull};
+ struct gpu_unlock_boot_desc boot;
+ const u32 words[] = {5,20480,2176,22656,16,0,0,0,0,2048,2048,4096,6144,10496,1,0,0,0,0,24576,0};
+
+ for (unsigned int i=0; i<21; i++) put32(desc+i*4,words[i]);
+ assert(!gpu_unlock_boot_desc_parse(desc, sizeof(desc), 24576, &boot));
+ assert(boot.code_offset == 6144 && boot.data_offset == 2048 && !boot.manifest_offset);
+ assert(gpu_unlock_boot_desc_parse(desc, 83, 24576, &boot) == -EINVAL);
+ put32(desc+48, ~0u);
+ assert(gpu_unlock_boot_desc_parse(desc, 84, 24576, &boot) == -ERANGE);
+ assert(!gpu_unlock_radix_page(page, addresses, 2));
+ assert(page[3] == 0x23 && page[4] == 1 && page[12] == 0xd);
+ for (unsigned int i=16; i<sizeof(page); i++) assert(page[i] == 0);
+ addresses[0]++;
+ assert(gpu_unlock_radix_page(page, addresses, 2) == -EINVAL);
+ assert(gpu_unlock_radix_page(page, addresses, 513) == -EINVAL);
+ }
+
+ puts("prep checks passed");
+ return 0;
+}
diff --git a/tests/test_release.py b/tests/test_release.py
new file mode 100644
index 0000000..f5586a7
--- /dev/null
+++ b/tests/test_release.py
@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Host checks for the retained module components."""
+from pathlib import Path
+import os
+import subprocess
+import tempfile
+import unittest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+KERNEL = ROOT / "kernel/gpu_unlock"
+# Generated or installed firmware lets the adapter run the whole unlock
+# against mocked hardware.
+ASSETS = next((path for path in (Path(os.environ.get("GPU_UNLOCK_ASSETS", "")),
+ Path("/usr/lib/firmware/gpu-unlock/610.43.03"))
+ if (path / "gsp_ga10x.bin").is_file()), None)
+
+CASES = {
+ "gfw_eval": ["gfw_eval.c"],
+ "layout_610": ["layout_610.c"],
+ "prep_610": ["layout_610.c", "prep_610.c"],
+ "vbios_610": ["vbios_610.c"],
+ "unlock_adapter": ["gfw_eval.c", "prep_610.c", "layout_610.c", "vbios_610.c"],
+ "main": [],
+}
+# Tests that include a kernel source directly build it against a shim.
+SHIMS = {
+ "unlock_adapter": ["-I", str(ROOT / "tests/unlock_shim")],
+ # The kernel build does not warn on sign-compare; match it for main.c.
+ "main": ["-I", str(ROOT / "tests/main_shim"), "-pthread", "-Wno-sign-compare"],
+}
+
+
+class ComponentTests(unittest.TestCase):
+ def test_components(self):
+ with tempfile.TemporaryDirectory() as directory:
+ for name, sources in CASES.items():
+ with self.subTest(name=name):
+ output = Path(directory) / name
+ shim = SHIMS.get(name, [])
+ command = [os.environ.get("CC", "cc"), "-std=c11", "-Wall",
+ "-Wextra", "-Werror", "-Wno-unused-parameter",
+ "-I", str(KERNEL), *shim,
+ *(str(KERNEL / source) for source in sources),
+ str(ROOT / "tests" / f"test_{name}.c"),
+ "-o", str(output)]
+ subprocess.run(command, check=True)
+ args = [str(output)]
+ if name == "unlock_adapter" and ASSETS:
+ args += ["-", str(ASSETS)]
+ result = subprocess.run(args, capture_output=True, text=True)
+ self.assertEqual(result.returncode, 0,
+ (result.stdout + result.stderr)[-4000:])
diff --git a/tests/test_unlock_adapter.c b/tests/test_unlock_adapter.c
new file mode 100644
index 0000000..59ca342
--- /dev/null
+++ b/tests/test_unlock_adapter.c
@@ -0,0 +1,671 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <assert.h>
+#include <stdarg.h>
+#include <sys/stat.h>
+#define __KERNEL__
+#include "../kernel/gpu_unlock/unlock_610.c"
+#include "vbios_fixture.h"
+static u32 regs[0x900000/4];
+static const char *asset_root;
+static struct unlock_transaction_work *last_work;
+static bool full_active;
+static void check_published_image(void);
+static void check_restored_signature(void);
+static struct {
+ u16 cmd,saved;
+ unsigned long now;
+ int allocs,live,frees,fail_alloc,fail_mask,flrs,fail_reset,starts,boot_mode,bar_bad,barriers,writes;
+ int gsp_starts,fwsec_mode,master_fail,selector_writes,ss1_writes,fw_live;
+ int pending,lost_identity,corrupt_selector,alien_wpr;
+ int fw_requests,fail_fw,fail_claim,fail_map,fail_save,fail_context,pins,selector_bad;
+ int restore_mask_fail,late_devres;
+ int group_requests,fail_group,group_live,group_before,cache_records,cache_released;
+ int switch_stuck,dma_commands,dma_stall_after;
+ int reset_delay,reset_reads[2],reset_pending[2],reset_post_fault;
+ int reset_locked_until_flr,hold_writes,bar_ready_after,bar_ready_reads,wpr_after;
+ int post_gfw,post_gfw_reads,gsp_post_reset_writes;
+ int fwsec_wpr;u32 fwsec_wpr_lo,fwsec_wpr_hi; /* WPR2 that FWSEC-FRTS enables */
+ int apertures,regions;
+ char log[32768];
+} fake;
+void *kzalloc(size_t n,int flags) {
+ if(fake.fail_context)return NULL;
+ last_work=calloc(1,n);return last_work;
+}
+void test_device_log(const char *format, ...) {
+ va_list args;
+ size_t at=strlen(fake.log);
+ va_start(args,format);
+ vsnprintf(fake.log+at,sizeof(fake.log)-at,format,args);
+ va_end(args);
+}
+static u64 mask;
+static struct pci_dev dev;
+static u8 fw_image[4097];
+static struct firmware fw={sizeof(fw_image),fw_image};
+static void setup(struct unlock_transaction_work *w) {
+ memset(&fake,0,sizeof(fake)); memset(regs,0,sizeof(regs)); memset(w,0,sizeof(*w));
+ mask=0xffffffffffffull; dev=(struct pci_dev){.dev={.dma_mask=&mask,.coherent_dma_mask=0xfffffffffull},.refs=1};
+ w->pdev=&dev; w->bar=regs; w->bar_len=sizeof(regs); w->mapped=true;
+ fake.cmd=2;fake.saved=2;regs[(SEC2_BASE+0xf4)/4]=1u<<31;regs[(0x110000+0xf4)/4]=1u<<31;
+ regs[(SEC2_BASE+0x118)/4]=2;regs[(0x110000+0x118)/4]=2;
+ regs[GU_FEAT_SS0_OFFSET/4]=0x88888888;regs[GU_FEAT_SS1_OFFSET/4]=0;
+ regs[0x1fa824/4]=0xfffffff0;regs[0x1fa828/4]=0;
+ regs[0]=0x172000a1;
+}
+void *dma_alloc_coherent(struct device *d,size_t n,dma_addr_t *a,int flags) {
+ fake.allocs++; if(fake.fail_alloc==fake.allocs)return NULL;
+ void *p=calloc(1,n); assert(p);
+ *a=(d->coherent_dma_mask<=0xffffffffull?0x100000ull:0x100000000ull)+fake.allocs*0x10000ull;
+ assert(*a+n-1<=d->coherent_dma_mask);fake.live++;return p;
+}
+void dma_free_coherent(struct device *d,size_t n,void *p,dma_addr_t a) {
+ assert(p && a>=0x100000ull);free(p);fake.live--;fake.frees++;
+}
+int dma_set_mask(struct device *d,u64 v) {
+ *d->dma_mask=v;return fake.fail_mask==1 || (fake.restore_mask_fail && v>0xffffffffull)?-EIO:0;
+}
+int dma_set_coherent_mask(struct device *d,u64 v) { d->coherent_dma_mask=v;return fake.fail_mask==2?-EIO:0; }
+void dma_wmb(void) { fake.barriers++; }
+void usleep_range(unsigned long a,unsigned long b) {fake.now+=a;}
+ktime_t ktime_get(void) {return fake.now;}
+u32 readl(const void *p) {
+ size_t n=((const u8 *)p-(const u8 *)regs)/4;assert(n<ARRAY_SIZE(regs));
+ if(fake.flrs && !n && fake.bar_ready_after) {
+ fake.bar_ready_reads++;
+ if(fake.bar_ready_after<0 || fake.bar_ready_reads<fake.bar_ready_after)
+ return fake.bar_ready_after==-2?0x12345678:0xbadf0000;
+ }
+ if(fake.flrs && n==GU_GFW_CPUCTL_OFFSET/4 && fake.post_gfw==1 && ++fake.post_gfw_reads==5) {
+ regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
+ }
+ int eng=n==(SEC2_BASE+0x3c0)/4?0:n==0x1103c0/4?1:-1;
+ if(eng>=0 && fake.reset_pending[eng] && ++fake.reset_reads[eng]>=fake.reset_delay) {
+ regs[n]=1;fake.reset_pending[eng]=0;
+ }
+ return regs[n];
+}
+void writel(u32 v,void *p) {
+ fake.writes++;
+ size_t off=(u8 *)p-(u8 *)regs;assert(off<sizeof(regs)); regs[off/4]=v;
+ if(fake.flrs && off==0x1103c0) {
+ fake.gsp_post_reset_writes++;
+ regs[GU_GFW_CPUCTL_OFFSET/4]=0; /* Holding GSP reset breaks RM's entry wait. */
+ }
+ if(off==GU_FEAT_SS0_OFFSET)fake.selector_writes++;
+ if(off==GU_FEAT_SS0_OFFSET && fake.selector_bad==1)regs[off/4]=0;
+ if(off==GU_FEAT_SS1_OFFSET){fake.ss1_writes++;regs[off/4]=v&0xf;} /* 4-bit register, as on the card */
+ if(off==GU_FEAT_SS1_OFFSET && fake.selector_bad==2)regs[off/4]=0;
+ if(off==SEC2_BASE+0x118 || off==0x110118) {
+ fake.dma_commands++;
+ regs[off/4]=fake.dma_stall_after && fake.dma_commands>=fake.dma_stall_after?1:2;
+ }
+ if(off==SEC2_FALCON2+0x668 || off==0x111668)regs[off/4]=fake.switch_stuck?0:1;
+ if(off==0x110100 || off==0x110130) {
+ fake.gsp_starts++;
+ if(fake.fwsec_wpr){regs[0x1fa824/4]=fake.fwsec_wpr_lo;regs[0x1fa828/4]=fake.fwsec_wpr_hi;}
+ regs[0x110100/4]=fake.fwsec_mode==1?2:0x10;
+ }
+ if(off==SEC2_BASE+0x100 || off==SEC2_BASE+0x130) {
+ if(full_active)check_published_image();
+ fake.starts++;
+ if(fake.boot_mode==0)regs[GU_FEAT_PLM_OFFSET/4]=~0u;
+ if(fake.boot_mode==2)regs[(SEC2_BASE+0x100)/4]=~0u;
+ if(fake.boot_mode==3)regs[(SEC2_BASE+0x100)/4]=0x10;
+ }
+ if(fake.fail_reset==2 && (off==SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0;
+ if(fake.fail_reset==8 && (off==SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0xbadf0001;
+ if((off==SEC2_BASE+0x3c0 || off==0x1103c0) && (v&1) && !(fake.cmd&4)) {
+ fake.hold_writes++;
+ int eng=off==SEC2_BASE+0x3c0?0:1;
+ if(fake.reset_delay) {
+ regs[off/4]=0;fake.reset_reads[eng]=0;fake.reset_pending[eng]=1;
+ }
+ if(fake.flrs && fake.reset_post_fault) {
+ regs[off/4]=fake.reset_post_fault==1?0:0xbadf0001;
+ fake.reset_pending[eng]=0;
+ }
+ if(fake.reset_locked_until_flr && !fake.flrs) {
+ regs[off/4]=0;fake.reset_pending[eng]=0;
+ }
+ if(fake.flrs && fake.post_gfw==4)regs[GU_GFW_CPUCTL_OFFSET/4]=0;
+ }
+}
+void *devres_open_group(struct device *d,void *id,int flags) {
+ if(++fake.group_requests==fake.fail_group)return NULL;
+ assert(!fake.group_live && !id);
+ fake.group_live=1;fake.group_before=d->devres_head.count;
+ d->devres_head.count++; /* open group marker */
+ return &fake.group_live;
+}
+int devres_release_group(struct device *d,void *id) {
+ assert(fake.group_live && id==&fake.group_live);
+ int released=d->devres_head.count-fake.group_before-1;
+ fake.cache_released+=released;d->devres_head.count=fake.group_before;
+ fake.group_live=0;return released;
+}
+int request_firmware(const struct firmware **f,const char *n,struct device *d) {
+ if(++fake.fw_requests==fake.fail_fw)return -ENOENT;
+ if(!asset_root)return -ENOENT;
+ char path[4096];
+ int length=snprintf(path,sizeof(path),"%s/%s",asset_root,strrchr(n,'/')+1);
+ assert(length>0 && (size_t)length<sizeof(path));
+ FILE *file=fopen(path,"rb");assert(file);
+ assert(!fseek(file,0,SEEK_END));long size=ftell(file);assert(size>0);assert(!fseek(file,0,SEEK_SET));
+ struct firmware *loaded=malloc(sizeof(*loaded));u8 *data=malloc(size);assert(loaded && data);
+ assert(fread(data,1,size,file)==(size_t)size);fclose(file);
+ *loaded=(struct firmware){size,data};*f=loaded;fake.fw_live++;
+ /* Linux CONFIG_FW_CACHE: successful request adds a device resource;
+ * release_firmware frees the image, not this suspend-cache record. */
+ d->devres_head.count++;fake.cache_records++;return 0;
+}
+void release_firmware(const struct firmware *f) {if(f){free((void *)f->data);free((void *)f);fake.fw_live--;}}
+void __module_get(void *p) {fake.pins++;}
+int pci_read_config_word(struct pci_dev *p,int r,u16 *v) {*v=r==PCI_COMMAND?fake.cmd:r==PCI_VENDOR_ID?0x10de:0;return 0;}
+int pci_read_config_dword(struct pci_dev *p,int r,u32 *v) {
+ *v=r==PCI_VENDOR_ID?(fake.flrs && fake.lost_identity?~0u:0x220410de):(u32)fake.bar_bad;return 0;
+}
+int pci_write_config_word(struct pci_dev *p,int r,u16 v) {assert(r==PCI_COMMAND);fake.cmd=v;return 0;}
+int pcie_capability_read_word(struct pci_dev *p,int r,u16 *v) {*v=r==PCI_EXP_DEVSTA?fake.pending:0;return 0;}
+int pcie_capability_read_dword(struct pci_dev *p,int r,u32 *v) {*v=PCI_EXP_DEVCAP_FLR;return 0;}
+int pci_is_pcie(struct pci_dev *p) {return 1;}
+int pci_select_bars(struct pci_dev *p,int r) {return 7;}
+int pci_request_region(struct pci_dev *p,int bar,const char *n) {
+ assert(!bar); /* only BAR0: the display card's framebuffer keeps BAR1 */
+ if(fake.fail_claim)return -EBUSY;
+ fake.regions++;return 0;
+}
+void pci_release_region(struct pci_dev *p,int bar) {assert(!bar && fake.regions==1);fake.regions--;}
+int aperture_remove_conflicting_pci_devices(struct pci_dev *p,const char *n) {
+ /* The framebuffer stays through FWSEC and Booter; it goes right before FLR. */
+ assert(!fake.flrs && !(fake.cmd&4));
+ if(full_active)assert(fake.starts==1);
+ fake.apertures++;return 0;
+}
+int pci_enable_device_mem(struct pci_dev *p) {fake.cmd|=2;return 0;}
+void pci_disable_device(struct pci_dev *p) {}
+resource_size_t pci_resource_len(struct pci_dev *p,int r) {return sizeof(regs);}
+void *pci_iomap(struct pci_dev *p,int r,unsigned long n) {return fake.fail_map?NULL:regs;}
+void pci_iounmap(struct pci_dev *p,void *v) {}
+void *pci_map_rom(struct pci_dev *p,size_t *n) {return NULL;}
+void pci_unmap_rom(struct pci_dev *p,void *v) {}
+int pci_save_state(struct pci_dev *p) {fake.saved=fake.cmd;return fake.fail_save?-EIO:0;}
+void pci_restore_state(struct pci_dev *p) {assert(!(fake.saved&4));fake.cmd=fake.saved;}
+void pci_set_master(struct pci_dev *p) {if(!fake.master_fail)fake.cmd|=4;}
+void pci_clear_master(struct pci_dev *p) {fake.cmd&=~4;}
+int pcie_flr(struct pci_dev *p) {
+ if(full_active)check_restored_signature();
+ if(full_active)assert(fake.apertures==1);
+ fake.flrs++;
+ if(fake.late_devres)p->dev.devres_head.count++;
+ /* FLR releases engine reset; post-FLR assertion must work independently. */
+ regs[(SEC2_BASE+0x3c0)/4]=regs[0x1103c0/4]=0;
+ regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;
+ regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
+ if(fake.post_gfw==1 || fake.post_gfw==2) {
+ regs[GU_GFW_CPUCTL_OFFSET/4]=0;regs[GU_GFW_BOOT_OFFSET/4]=0;
+ }
+ if(fake.post_gfw==3)regs[GU_GFW_PLM_OFFSET/4]=0xbadf0001;
+ if(fake.wpr_after!=4)regs[0x1fa824/4]=regs[0x1fa828/4]=0;
+ if(fake.corrupt_selector==1)regs[GU_FEAT_SS0_OFFSET/4]=0;
+ if(fake.corrupt_selector==2)regs[GU_FEAT_SS1_OFFSET/4]=0;
+ if(fake.alien_wpr){regs[0x1fa824/4]=0x1000;regs[0x1fa828/4]=0x2000;}
+ if(fake.wpr_after==1)regs[0x1fa824/4]=regs[0x1fa828/4]=0;
+ if(fake.wpr_after==2){regs[0x1fa824/4]=0x2000;regs[0x1fa828/4]=0x1000;}
+ if(fake.wpr_after==3)regs[0x1fa824/4]=0xbadf0000;
+ return fake.fail_reset==1?-EIO:0;
+}
+struct pci_dev *pci_dev_get(struct pci_dev *p) {p->refs++;return p;}
+void pci_dev_put(struct pci_dev *p) {p->refs--;}
+static u64 get64(const void *p) {const u8 *s=p;u64 v=0;for(int i=7;i>=0;i--)v=(v<<8)|s[i];return v;}
+static const u8 *dma_page(u64 address) {
+ struct unlock_transaction_work *w=last_work;
+ assert(address>=w->pages[0].dma && !((address-w->pages[0].dma)%0x10000));
+ u64 index=(address-w->pages[0].dma)/0x10000;
+ assert(index<w->pages_allocated && w->pages[index].dma==address);
+ return w->pages[index].cpu;
+}
+static void check_published_image(void) {
+ struct unlock_transaction_work *w=last_work;const u8 *meta=w->meta_cpu;
+ assert(w->published && !fake.pins && (fake.cmd&4));
+ assert(get64(meta)==GU_WPR_META_MAGIC && get64(meta+8)==1);
+ assert(get64(meta+72)==w->v67_dma && get64(meta+80)==GU_V67_SIZE);
+ assert(!memcmp(w->v67_cpu,w->v67_fw->data,GU_V67_SIZE));
+ assert(get64(meta+112)==0x5f5400000ull && get64(meta+136)==0x5fada0000ull);
+ assert(get64(meta+144)==0x5ffdfa000ull && get64(meta+152)==0x5ffe00000ull);
+ assert(get64(meta+168)==0x5fff00000ull && get64(meta+176)==0x600000000ull);
+ /* Independently walk the radix the way a firmware consumer does. Compare
+ * every published image byte, including transitions across 512-entry tables. */
+ u64 image_bytes=get64(meta+24);assert(image_bytes==84226048);
+ for(u64 page=0,off=0;off<image_bytes;page++,off+=4096) {
+ const u8 *level0=dma_page(get64(meta+16));
+ const u8 *level1=dma_page(get64(level0+(page>>18)*8));
+ const u8 *level2=dma_page(get64(level1+((page>>9)&511)*8));
+ const u8 *leaf=dma_page(get64(level2+(page&511)*8));
+ size_t bytes=image_bytes-off<4096?image_bytes-off:4096;
+ assert(!memcmp(leaf,w->gsp_fw->data+w->image_span.offset+off,bytes));
+ for(size_t i=bytes;i<4096;i++)assert(!leaf[i]);
+ }
+ assert(!memcmp(w->boot_cpu,w->boot_fw->data,w->boot_desc.image_size));
+ /* SEC2 fuse v1 selects slot 0 of the two-signature Booter asset. */
+ assert(!memcmp((u8 *)w->booter_cpu+35344,w->booter_fw->data+72+60416,384));
+}
+static void check_restored_signature(void) {
+ struct unlock_transaction_work *w=last_work;const u8 *meta=w->meta_cpu;
+ assert(w->signature_calls==2 && get64(meta+72)==w->stock_dma && get64(meta+80)==4096);
+ assert(!memcmp(w->stock_cpu,w->gsp_fw->data+w->sig_span.offset,4096));
+ assert(!(fake.cmd&4));
+ /* All published buffers remain alive; FLR must precede engine holds. */
+ assert(fake.live==fake.allocs && !fake.frees && !fake.hold_writes);
+}
+static void synthetic_rom(u8 *rom) {
+ vbios_fixture(rom,6);
+ rom[292]=0;rom[293]=4;rom[294]=9; /* GSP engine and FWSEC ucode */
+ u8 *dmem=rom+684+256;
+ dmem[0]=1;dmem[1]=4;dmem[2]=8;dmem[3]=2;
+ vbios_w32(dmem+4,4);vbios_w32(dmem+8,32);
+ dmem[36]=3;dmem[38]=64;
+ vbios_w32(dmem+40,128);vbios_w32(dmem+44,44);
+}
+static void load_rom(const char *path) {
+ u8 *rom=(u8 *)regs+0x300000;
+ if(!path || !strcmp(path,"-")) { synthetic_rom(rom); return; }
+ FILE *prom=fopen(path,"rb");
+ assert(prom && fread(rom,1,1024*1024,prom)==1024*1024);
+ assert(fgetc(prom)==EOF);fclose(prom);
+}
+int main(int argc, char **argv) {
+ struct unlock_transaction_work w;
+ setup(&w);
+ regs[0x110118/4]=0x1230;
+ assert(poll_us(&w,0x110118,2,2,100,true)==-ETIMEDOUT);
+ assert(strstr(fake.log,"poll timeout: reg=0x110118 last=0x1230 mask=0x2 expected=0x2 budget_us=100"));
+ assert(strstr(fake.log,"poll_us error=-110"));
+ assert(!fake.writes);
+ setup(&w);
+ regs[0x110118/4]=~0u;
+ assert(poll_us(&w,0x110118,2,2,100,true)==-EIO);
+ assert(strstr(fake.log,"poll inaccessible: reg=0x110118 last=0xffffffff"));
+ /* Exercise the dual-core reset path, including RISC-V -> Falcon switching
+ * and the documented tolerated pre-reset timeout. */
+ for(int mode=0;mode<8;mode++) {
+ setup(&w);
+ const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
+ regs[(SEC2_BASE+0xf4)/4]|=1u<<10;
+ regs[(SEC2_FALCON2+0x668)/4]=mode==0?1:0x10;
+ if(mode==2)regs[(SEC2_FALCON2+0x668)/4]=0xbadf0001;
+ if(mode==3)fake.switch_stuck=1;
+ if(mode==4)regs[(SEC2_BASE+0xf4)/4]|=1u<<12;
+ if(mode==5)regs[(SEC2_BASE+0xf4)/4]&=~(1u<<31);
+ if(mode==6)regs[0]=~0u;
+ if(mode==7)regs[(SEC2_BASE+0x3c0)/4]=0xbadf0001;
+ int ret=falcon_reset(&w,&eng);
+ assert((!ret)==(mode==0 || mode==1 || mode==5));
+ if(mode==1)assert(regs[(SEC2_FALCON2+0x668)/4]==1);
+ if(mode==5){
+ assert(strstr(fake.log,"pre-reset timeout tolerated"));
+ assert(strstr(fake.log,"poll timeout tolerated:"));
+ assert(!strstr(fake.log,"poll timeout:"));
+ assert(!strstr(fake.log,"poll_us error=-110"));
+ assert(!strstr(fake.log,"failure unlock_610.c"));
+ }
+ assert(!fake.gsp_starts && !fake.starts && !fake.dma_commands);
+ }
+ for(int mode=0;mode<3;mode++) {
+ setup(&w);
+ if(mode==0)regs[(SEC2_BASE+0x118)/4]=1;
+ else fake.dma_stall_after=1;
+ assert(falcon_dma(&w,SEC2_BASE,0,0,0x100000,mode==2?256:512,0x614)==-ETIMEDOUT);
+ assert(fake.dma_commands==(mode?1:0) && fake.now<=1000100);
+ if(mode==1)assert(strstr(fake.log,"Falcon DMA progress: done=256 total=512"));
+ }
+ for(int mode=0;mode<4;mode++) {
+ setup(&w);
+ const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
+ w.plm_entry=0x8f;
+ if(mode==0)regs[(SEC2_FBIF+0x24)/4]=0xbadf0001;
+ if(mode==1)regs[SEC2_FBIF/4]=~0u;
+ if(mode==2)regs[(SEC2_BASE+0x100)/4]=~0u;
+ if(mode==3)regs[(SEC2_BASE+0x100)/4]=1u<<6; /* CPUCTL alias path */
+ int ret=falcon_hs(&w,&eng,0,256,0,0,256,256,0,0,1,3,0x100000,0,0,true);
+ assert((!ret)==(mode==3));
+ assert(fake.starts==(mode==3?1:0));
+ if(mode<2)assert(!fake.dma_commands);
+ }
+ /* Reject DMA address truncation/wrap before any reset or transfer. */
+ for(int bad=0;bad<5;bad++) {
+ setup(&w);
+ u32 dest=bad==0?0xffff00:0, off=bad==1?0xffffff00:0;
+ dma_addr_t src=bad==2?(1ull<<49)-256:0x100000;
+ u32 size=bad==3?257:512;
+ if(bad==4)src++;
+ assert(falcon_dma(&w,SEC2_BASE,dest,off,src,size,0x614)==-EINVAL);
+ assert(!fake.writes);
+ }
+ {
+ const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
+ setup(&w);
+ assert(falcon_hs(&w,&eng,0,256,0x200000,0,256,256,0,0,1,3,
+ 0x100000,0,0,false)==-ERANGE);
+ assert(!fake.writes);
+ /* Bad DMEM geometry must not leave an already-submitted IMEM DMA. */
+ assert(falcon_hs(&w,&eng,0,256,0,0,256,257,0,0,1,3,
+ 0x100000,0,0,false)==-EINVAL);
+ assert(!fake.writes);
+ }
+ /* Actual PROM -> descriptor -> patch -> GSP DMA/start -> WPR validation.
+ * Faults after firmware access retain DMA, without starting SEC2. */
+ for(int mode=0;mode<10;mode++) {
+ setup(&w);
+ u8 *rom=(u8 *)regs+0x300000;
+ load_rom(argc>=2?argv[1]:NULL);
+ struct gpu_unlock_fwsec parsed;
+ assert(!gpu_unlock_vbios_fwsec(rom,1024*1024,&parsed));
+ w.layout.frts_offset=0x5ffe00000ull;w.layout.wpr_end=0x5fff00000ull;
+ regs[0x1fa824/4]=(u32)(w.layout.frts_offset>>12)<<4;
+ regs[0x1fa828/4]=(u32)((w.layout.wpr_end-4096)>>12)<<4;
+ fake.fwsec_mode=mode;
+ if(mode==2)regs[(0x1400+0x0e*4)/4]=1u<<16;
+ if(mode==3)regs[0x1fa828/4]=0; /* disabled, low alone matched */
+ if(mode==4)regs[0x1fa828/4]+=16; /* outside reservation */
+ if(mode==5)regs[0x1fa824/4]-=16;
+ if(mode==6)regs[(0x8241c0+8*4)/4]=0xffff; /* unavailable fuse version */
+ if(mode==7)fake.fail_alloc=1;
+ if(mode==8)fake.master_fail=1;
+ if(mode==9)rom[parsed.sig_off-44+37]=1; /* wrong engine mask */
+ int ret=unlock_transaction_fwsec(&w);
+ assert(!fake.starts && !fake.flrs);
+ if(mode==0) {
+ assert(!ret && fake.gsp_starts==1);
+ assert(strstr(fake.log,"FWSEC WPR2:"));
+ } else {
+ assert(ret);
+ if(mode>=6)assert(!fake.gsp_starts && !w.firmware_started);
+ else assert(fake.gsp_starts==1 && w.firmware_started);
+ }
+ if(w.firmware_started) {
+ assert(unlock_transaction_unwind(&w)==-EBUSY && w.retained && fake.live==1);
+ free_coherent(&w); /* harness disposal only */
+ } else assert(!unlock_transaction_unwind(&w) && !fake.live);
+ }
+ /* Exercise the register-to-layout adapter with the pinned image geometry.
+ * A range error is pre-DMA/pre-firmware; never "fix" it by skipping bounds. */
+ for(int scenario=0;scenario<8;scenario++) {
+ setup(&w);
+ regs[0x1183a4/4]=24*1024;
+ regs[0x1fa7c8/4]=1;
+ regs[0x1fa82c/4]=0xfffffff0; /* disabled MMU lock: hi < lo */
+ regs[0x1fa830/4]=0;
+ w.image_span.size=0x5053000;
+ w.boot_desc.image_size=24576;
+ if(scenario==1)regs[0x625f04/4]=(u32)(((24ull<<30)-(1ull<<20))>>16)<<8 | 8;
+ if(scenario==2)regs[0x625f04/4]=8; /* valid old low workspace -> VBIOS fallback */
+ if(scenario==3 || scenario==4) {
+ u64 low=(24ull<<30)-((scenario==3?2ull:200ull)<<20);
+ regs[0x1fa82c/4]=(u32)(low>>12)<<4;
+ regs[0x1fa830/4]=(u32)(((24ull<<30)-4096)>>12)<<4;
+ }
+ if(scenario==5)regs[0x1183a4/4]=128; /* insufficient usable FB */
+ if(scenario==6)regs[0x625f04/4]=(u32)((24ull<<30)>>16)<<8 | 8; /* beyond FB */
+ /* Captured cold GA102 lock: lo=256 MiB, hi exclusive is not FB top. */
+ if(scenario==7) {
+ regs[0x1183a4/4]=0x6000; regs[0x625f04/4]=1; regs[0x1fa7c8/4]=0x45cb8f;
+ regs[0x1fa82c/4]=0x100000; regs[0x1fa830/4]=0x5cffff0;
+ w.image_span.size=84226048;
+ }
+ int ret=unlock_transaction_layout(&w);
+ assert(ret==(scenario>=4 && scenario<=6?-ERANGE:0));
+ assert(strstr(fake.log,"layout inputs: fb_mb="));
+ if(scenario==4) assert(strstr(fake.log,"reservations below pre-scrubbed"));
+ if(scenario==5) assert(strstr(fake.log,"usable FB below 256 MiB"));
+ if(scenario==6) assert(strstr(fake.log,"VGA base at or above usable FB"));
+ if(scenario==7) assert(strstr(fake.log,"mmu lock ignored: high address is not the FB top"));
+ if(scenario<4 || scenario==7) {
+ assert(w.layout.nonwpr_start>=w.layout_in.usable_fb-(256ull<<20));
+ assert(w.layout.wpr_end<=w.layout_in.vga_base);
+ assert(w.layout.heap_size>=88ull<<20);
+ }
+ assert(!fake.writes && !fake.allocs && !fake.starts && !fake.flrs);
+ }
+ /* Cold entry: completed GFW, closed PLM and WPR2 down. A used card is refused. */
+ for(int mode=0;mode<3;mode++) {
+ setup(&w);
+ regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;
+ regs[GU_FEAT_PLM_OFFSET/4]=mode==1?~0u:0x8f;
+ if(mode==2)regs[0x1fa828/4]=0x5ffef0;
+ int ret=unlock_transaction_gfw(&w);
+ assert(ret==(mode==0?0:mode==1?-EALREADY:-EBUSY));
+ assert(!fake.writes && !fake.gsp_starts);
+ if(mode==2)assert(strstr(fake.log,"card already initialised since power on"));
+ }
+ setup(&w); dev.dev_flags=PCI_DEV_FLAGS_NO_FLR_RESET;
+ assert(unlock_transaction_claim(&w)==-EOPNOTSUPP && !fake.flrs && !fake.allocs);
+ /* A setter may mutate a mask before returning failure; restore both. */
+ for(int fail=1;fail<=2;fail++) {
+ setup(&w); fake.fail_mask=fail;
+ assert(unlock_transaction_dma(&w)==-EIO);
+ fake.fail_mask=0;
+ assert(!unlock_transaction_unwind(&w));
+ assert(mask==0xffffffffffffull && dev.dev.coherent_dma_mask==0xfffffffffull);
+ }
+ /* Stock restoration must preserve layout and must not republish V67. */
+ {
+ setup(&w);
+ struct unlock_transaction_page page={.dma=0x200000};
+ u8 metadata[GU_WPR_META_SIZE], stock[4096], before[GU_WPR_META_SIZE];
+ w.meta_cpu=metadata;w.stock_cpu=stock;w.stock_dma=0x400000;w.stock_size=4096;
+ w.v67_dma=0x300000;w.boot_dma=0x500000;w.pages=&page;w.pages_allocated=1;
+ w.layout_in=(struct gpu_unlock_layout_in){.usable_fb=24ull<<30,
+ .vga_base=(24ull<<30)-(1ull<<20),.frts_size=1ull<<20,
+ .image_size=4097,.boot_size=24576,.nonwpr_size=1ull<<20,.heap_request=88ull<<20};
+ assert(!gpu_unlock_layout(&w.layout_in,&w.layout));w.image_span.size=4097;
+ w.boot_desc=(struct gpu_unlock_boot_desc){.code_offset=6144,.data_offset=2048,.image_size=24576};
+ assert(!unlock_transaction_signature(&w) && w.published && w.signature_calls==1);
+ assert(get64(metadata+72)==w.v67_dma && get64(metadata+80)==GU_V67_SIZE);
+ memcpy(before,metadata,sizeof(before));
+ assert(!unlock_transaction_signature(&w) && w.signature_calls==2);
+ assert(get64(metadata+72)==w.stock_dma && get64(metadata+80)==4096);
+ assert(!memcmp(before,metadata,72) && !memcmp(before+88,metadata+88,sizeof(before)-88));
+ assert(unlock_transaction_signature(&w)==-EINVAL && fake.barriers==2);
+ }
+ /* Exercise every coherent-allocation failure in a real multi-page radix. */
+ for(int fail=0;fail<=5;fail++) {
+ setup(&w);w.gsp_fw=&fw;w.image_span.size=sizeof(fw_image);
+ assert(!gpu_unlock_radix_plan(w.image_span.size,&w.radix));fake.fail_alloc=fail;
+ int ret=unlock_transaction_radix(&w);
+ if(fail) {
+ assert(ret==-ENOMEM);
+ assert(strstr(fake.log,"DMA allocation failed: asset=radix-page bytes=4096"));
+ } else {
+ assert(ret==0 && w.pages_allocated==5);
+ assert(get64(w.pages[0].cpu)==w.pages[1].dma);
+ assert(get64(w.pages[1].cpu)==w.pages[2].dma);
+ assert(get64(w.pages[2].cpu)==w.pages[3].dma);
+ assert(get64((u8 *)w.pages[2].cpu+8)==w.pages[4].dma);
+ assert(get64((u8 *)w.pages[2].cpu+16)==0);
+ }
+ free_coherent(&w);assert(fake.live==0);
+ }
+ /* Early PLM polling must not wait for HALT or accept inaccessible engines. */
+ for(int mode=0;mode<4;mode++) {
+ setup(&w);fake.boot_mode=mode;w.master=w.firmware_started=w.published=true;
+ w.booter_hdr=(struct gpu_unlock_booter_hdr){.app_code_off=256,.app_code_size=256,
+ .os_data_off=512,.os_data_size=256,.patch_loc=512,.engine_id=1,.ucode_id=3};
+ w.booter_dma=0x100000;w.meta_dma=0x200000;w.plm_entry=0x8f;
+ int ret=unlock_transaction_boot(&w);
+ assert(fake.starts==1);
+ if(mode==0)assert(!ret);
+ if(mode==1)assert(ret==-ETIMEDOUT && fake.now<=5000100);
+ if(mode==1)assert(strstr(fake.log,"early PLM failed: pci=0x220410de"));
+ if(mode==2)assert(ret==-ENODEV);
+ if(mode==3)assert(ret==-EIO);
+ }
+ /* Failed FLR/reset/BAR checks retain allocations; success excludes fetches. */
+ for(int failure=0;failure<10;failure++) {
+ setup(&w);w.firmware_started=true;w.original_cmd=2;w.cmd_valid=true;
+ w.v67_cpu=dma_alloc_coherent(&dev.dev,GU_V67_SIZE,&w.v67_dma,0);
+ fake.fail_reset=(failure<3 || failure==8)?failure:0;fake.bar_bad=failure==3;
+ fake.pending=failure==4?PCI_EXP_DEVSTA_TRPND:0;
+ fake.lost_identity=failure==5;fake.corrupt_selector=failure==6?1:failure==9?2:0;fake.alien_wpr=failure==7;
+ w.signature_calls=2;regs[GU_FEAT_SS1_OFFSET/4]=0x8;
+ int ret=unlock_transaction_reset(&w);
+ if(failure==6 || failure==9) {
+ /* Lost selector state fails the unlock, but proven reset permits cleanup. */
+ assert(ret==-EIO && w.quiesced);
+ assert(!unlock_transaction_unwind(&w) && !fake.live && !w.retained);
+ } else if(failure) {
+ assert(ret && !w.quiesced);
+ assert(unlock_transaction_unwind(&w)==-EBUSY && w.retained && fake.live==1);
+ assert(strstr(fake.log,"failure unlock_610.c:"));
+ if(failure==1)assert(strstr(fake.log,"PCIe FLR returned: result=-5"));
+ if(failure==3)assert(strstr(fake.log,"post-FLR BAR0: actual=0x1 expected=0"));
+ /* Host harness disposal only; the kernel intentionally retains. */
+ free_coherent(&w);
+ } else {
+ assert(!ret && w.quiesced && fake.flrs==1);
+ assert(!(regs[(SEC2_BASE+0x3c0)/4]&1) && !regs[0x1103c0/4]);
+ assert(!unlock_transaction_unwind(&w) && fake.live==0);
+ }
+ }
+ /* Entire production transaction with pinned assets and a board ROM (a real
+ * dump, or "-" for the synthetic fixture). Only PCI/MMIO/DMA are mocked;
+ * parsers, patching and ordering are real. */
+ if(argc==3) {
+ asset_root=argv[2];
+ full_active=true;
+ static const char *cases[]={"success","already-full","context-oom","asset-1","asset-2",
+ "asset-3","asset-4","asset-5","claim-busy","map-failed","save-failed",
+ "dma-1","dma-6","dma-7","dma-8","dma-48","dma-last",
+ "gfw-timeout","preexisting-plm","fwsec-timeout","fwsec-error","fwsec-wpr",
+ "booter-timeout","booter-inaccessible","booter-halted","selector-readback",
+ "flr-failed","reset-inaccessible","pci-lost","pending","wpr-alien","selector-lost","mask-restore","decode-off",
+ "reset-delay10","reset-delay12","reset-never","reset-post-never","reset-post-inaccessible",
+ "reset-locked-before-flr","bar-ready-late","bar-inaccessible","bar-wrong-chip",
+ "wpr-disabled-zero","wpr-inverted","wpr-inaccessible",
+ "post-gfw-late","post-gfw-timeout","post-gfw-inaccessible","post-gsp-resumed",
+ "wpr-frts-retained","firmware-group-oom-first","firmware-group-oom-later","devres-preexisting","devres-at-handoff",
+ "warm-wpr2","ss1-readback","ss1-lost"};
+ int total_allocations=0;
+ for(unsigned scenario=0;scenario<ARRAY_SIZE(cases);scenario++) {
+ setup(&w);
+ load_rom(argv[1]);
+ regs[0x1183a4/4]=0x6000;regs[0x625f04/4]=1;regs[0x1fa7c8/4]=0x45cb8f;
+ regs[0x1fa82c/4]=0x100000;regs[0x1fa830/4]=0x5cffff0;
+ regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
+ regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;
+ /* Cold values captured from the Palit card at 0000:21:00.0. */
+ regs[GU_FEAT_PLM_OFFSET/4]=0xffffff8f;regs[GU_FEAT_SS0_OFFSET/4]=0x40171022;
+ regs[GU_FEAT_SS1_OFFSET/4]=0x2;
+ fake.fwsec_wpr=1;
+ fake.fwsec_wpr_lo=(u32)(0x5ffe00000ull>>12)<<4;
+ fake.fwsec_wpr_hi=(u32)(0x5ffeff000ull>>12)<<4;
+ regs[(GU_FUSE_SEC2_UCODE1+2*4)/4]=1;
+ /* FWSEC fuse: the real board ROM needs version 1, the fixture 0. */
+ if(strcmp(argv[1],"-"))regs[(0x8241c0+8*4)/4]=1;
+ if(scenario==1){regs[GU_FEAT_SS0_OFFSET/4]=0x88888888;regs[GU_FEAT_SS1_OFFSET/4]=0x8;}
+ if(scenario==2)fake.fail_context=1;
+ if(scenario>=3 && scenario<=7)fake.fail_fw=scenario-2;
+ if(scenario==8)fake.fail_claim=1;
+ if(scenario==9)fake.fail_map=1;
+ if(scenario==10)fake.fail_save=1;
+ if(scenario>=11 && scenario<=16) {
+ const int failures[]={1,6,7,8,48,total_allocations};
+ fake.fail_alloc=failures[scenario-11];
+ }
+ if(scenario==17)regs[GU_GFW_BOOT_OFFSET/4]=0;
+ if(scenario==18)regs[GU_FEAT_PLM_OFFSET/4]=~0u;
+ if(scenario==19)fake.fwsec_mode=1;
+ if(scenario==20)regs[(0x1400+0x0e*4)/4]=1u<<16;
+ if(scenario==21)fake.fwsec_wpr_hi=0;
+ if(scenario>=22 && scenario<=24)fake.boot_mode=scenario-21;
+ if(scenario==25)fake.selector_bad=1;
+ if(scenario==26)fake.fail_reset=1;
+ if(scenario==27)fake.fail_reset=8;
+ if(scenario==28)fake.lost_identity=1;
+ if(scenario==29)fake.pending=PCI_EXP_DEVSTA_TRPND;
+ if(scenario==30)fake.alien_wpr=1;
+ if(scenario==31)fake.corrupt_selector=1;
+ if(scenario==32)fake.restore_mask_fail=1;
+ if(scenario==33)fake.cmd=0;
+ if(scenario==34)fake.reset_delay=10;
+ if(scenario==35)fake.reset_delay=12;
+ if(scenario==36)fake.fail_reset=2;
+ if(scenario==37)fake.reset_post_fault=1;
+ if(scenario==38)fake.reset_post_fault=2;
+ if(scenario==39)fake.reset_locked_until_flr=1;
+ if(scenario==40)fake.bar_ready_after=5;
+ if(scenario==41)fake.bar_ready_after=-1;
+ if(scenario==42)fake.bar_ready_after=-2;
+ if(scenario>=43 && scenario<=45)fake.wpr_after=scenario-42;
+ if(scenario>=46 && scenario<=49)fake.post_gfw=scenario-45;
+ if(scenario==50)fake.wpr_after=4;
+ if(scenario==51)fake.fail_group=1;
+ if(scenario==52)fake.fail_group=3;
+ if(scenario==53)dev.dev.devres_head.count=1;
+ if(scenario==54)fake.late_devres=1;
+ if(scenario==55)regs[0x1fa828/4]=fake.fwsec_wpr_hi; /* a driver already ran FWSEC */
+ if(scenario==56)fake.selector_bad=2;
+ if(scenario==57)fake.corrupt_selector=2;
+ /* As main.c: the GSP file is loaded once, ahead of every card. */
+ struct gpu_unlock_610_shared shared={0};last_work=NULL;
+ int ret=gpu_unlock_610_load_shared(&dev,&shared);
+ if(!ret)ret=gpu_unlock_610_unlock(&dev,&shared);
+ gpu_unlock_610_release_shared(&shared);
+ bool expected=scenario<2 || (scenario>=33 && scenario<=35) || scenario==39 || scenario==40 || scenario==43 || scenario==46;
+ bool retained=fake.pins;
+ if((!ret)!=expected)fprintf(stderr,"%s\n%s: error=%d\n",fake.log,cases[scenario],ret);
+ assert((!ret)==expected);
+ if(expected) {
+ assert(fake.gsp_starts==1 && fake.starts==1 && fake.flrs==1 && !fake.pins);
+ assert(fake.apertures==1 && !fake.regions);
+ assert(fake.selector_writes==(scenario==1?0:1) && fake.ss1_writes==(scenario==1?0:1));
+ assert(regs[GU_FEAT_SS0_OFFSET/4]==0x88888888 && regs[GU_FEAT_SS1_OFFSET/4]==0x8);
+ assert(mask==0xffffffffffffull && dev.dev.coherent_dma_mask==0xfffffffffull);
+ /* Mirror Linux really_probe: driver entry rejects nonempty devres. */
+ assert(list_empty(&dev.dev.devres_head));
+ assert(fake.cache_records==5 && fake.cache_released==5);
+ /* Pinned RM checks GSP halt + readable completed GFW BEFORE
+ * its own reset/bootstrap. Preserve that entry contract. */
+ assert((regs[GU_GFW_CPUCTL_OFFSET/4]&0x10) && (regs[GU_GFW_PLM_OFFSET/4]&1));
+ assert((regs[GU_GFW_BOOT_OFFSET/4]&0xff)==0xff && !regs[0x1103c0/4]);
+ assert(!(regs[0x1fa828/4]>>4)); /* Stock RM rejects any active WPR2. */
+ if(!scenario)total_allocations=fake.allocs;
+ }
+ if(scenario>=34 && scenario<=50) {
+ assert(fake.starts==1 && fake.flrs==1);
+ if(!expected)assert(retained && !last_work->quiesced);
+ if(scenario==41 || scenario==42)assert(!fake.hold_writes && ret==-ETIMEDOUT);
+ if(scenario==39)assert(fake.hold_writes==1);
+ if(scenario==47 || scenario==48)assert(!fake.hold_writes && ret==-ETIMEDOUT);
+ }
+ assert(!fake.gsp_post_reset_writes);
+ assert(fake.gsp_starts<=1 && fake.starts<=1 && fake.flrs<=1 && fake.pins<=1);
+ assert(fake.selector_writes<=1 && fake.ss1_writes<=1);
+ if(retained) {
+ assert(last_work->retained && fake.live>0 && dev.refs==2);
+ /* Harness-only disposal of intentionally retained production state. */
+ last_work->firmware_started=false;fake.restore_mask_fail=0;
+ assert(!unlock_transaction_unwind(last_work));
+ pci_dev_put(last_work->pdev);free(last_work);
+ }
+ assert(!fake.live && !fake.fw_live && dev.refs==1 && !(fake.cmd&4));
+ assert(!fake.group_live && fake.cache_records==fake.cache_released);
+ assert(dev.dev.devres_head.count==(scenario==53 || scenario==54?1u:0u));
+ if(scenario>=51 && scenario<=53)assert(!fake.writes && !fake.allocs && !fake.flrs);
+ if(scenario==53)assert(fake.fw_requests==1 && ret==-EBUSY); /* only the shared load */
+ if(scenario==3 || scenario==51)assert(!last_work && fake.fw_requests==(scenario==3));
+ if(scenario==54)assert(fake.flrs==1 && ret==-EBUSY);
+ if(scenario==53 || scenario==54)assert(strstr(fake.log,"driver probe resources: phase=handoff empty=0"));
+ if(scenario==33)assert(fake.cmd==0);
+ if(scenario==56)assert(ret==-EIO && retained && fake.ss1_writes==1 && !fake.flrs);
+ if(scenario==57)assert(ret==-EIO && !retained && fake.flrs==1);
+ if(scenario==55) {
+ assert(ret==-EBUSY && !retained && !fake.gsp_starts && !fake.starts && !fake.flrs);
+ assert(strstr(fake.log,"card already initialised since power on"));
+ }
+ printf("full transaction %s: error=%d retained=%u PASS\n",cases[scenario],ret,retained);
+ }
+ puts("pinned firmware full BF16 transactions passed under mocked hardware");
+ }
+ puts("actual unlock_transaction radix, early-PLM and reset/retention checks passed");
+}
diff --git a/tests/test_vbios_610.c b/tests/test_vbios_610.c
new file mode 100644
index 0000000..2276b7e
--- /dev/null
+++ b/tests/test_vbios_610.c
@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <assert.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "vbios_610.h"
+
+static void w32(u8 *p, u32 v)
+{
+ for (int i = 0; i < 4; i++)
+ p[i] = v >> (8 * i);
+}
+
+#include "vbios_fixture.h"
+
+int main(int argc, char **argv)
+{
+ u8 rom[4096], image[1280] = {0};
+ struct gpu_unlock_fwsec fw;
+ u32 offset;
+
+ for (int stride = 6; stride <= 8; stride += 2) {
+ vbios_fixture(rom, stride);
+ assert(!gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw));
+ assert(fw.image_off == 684 && fw.image_size == 1280);
+ }
+ /* Allocate the actual short length: a larger backing array hides overreads. */
+ for (size_t len = 0; len < 4096; len++) {
+ u8 *short_rom = malloc(len ? len : 1);
+ assert(short_rom);
+ memcpy(short_rom, rom, len);
+ assert(gpu_unlock_vbios_fwsec(short_rom, len, &fw));
+ free(short_rom);
+ }
+ /* Legacy + EFI with NPDE continuation + VN/NPDS extension. The BIT
+ * pointers address the extension as if it followed the legacy image. */
+ for (int ext_len = 10; ext_len <= 11; ext_len++) {
+ u8 chain[8192] = {0};
+ vbios_fixture(rom, 6);
+ memcpy(chain, rom, 4096);
+ chain[53] = 0;
+ chain[4096] = 0x55; chain[4097] = 0xaa; chain[4096+0x18] = 32;
+ memcpy(chain+4096+32, "PCIR", 4);
+ chain[4096+42] = 24; chain[4096+48] = 8;
+ chain[4096+52] = 3; chain[4096+53] = 0x80;
+ memcpy(chain+4096+64, "NPDE", 4);
+ chain[4096+69] = 1; chain[4096+70] = ext_len;
+ chain[4096+72] = 1; /* one 512-byte subimage */
+ chain[4608] = 'V'; chain[4609] = 'N'; chain[4608+0x18] = 32;
+ memcpy(chain+4608+32, "NPDS", 4);
+ chain[4608+42] = 24; chain[4608+48] = 7;
+ chain[4608+52] = 0xe0; chain[4608+53] = 0x80;
+ w32(chain+100, 4096+128);
+ memcpy(chain+4608+128, rom+128, 18);
+ w32(chain+4608+142, 4096+256);
+ memcpy(chain+4608+256, rom+256, 428+1280);
+ assert(!gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw));
+ assert(fw.image_off == 4608+684);
+ assert(gpu_unlock_vbios_fwsec(chain, 4608, &fw) == -EINVAL);
+ /* RGIS/BB77 is also supported, malformed data signatures are not. */
+ chain[4608] = 0x77; chain[4609] = 0xbb;
+ memcpy(chain+4608+32, "RGIS", 4);
+ assert(!gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw));
+ chain[4096+70] = 9;
+ assert(gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw) == -EINVAL);
+ chain[4096+70] = ext_len; chain[4096+72] = 0;
+ assert(gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw) == -EINVAL);
+ }
+ vbios_fixture(rom, 8);
+ w32(rom + 264, 0xffffffff);
+ assert(gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw) == -EINVAL);
+ vbios_fixture(rom, 8);
+ w32(rom + 288, 0xffffffff);
+ assert(gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw) == -EINVAL);
+ vbios_fixture(rom, 8);
+ assert(!gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw));
+ image[256] = 1; image[257] = 4; image[258] = 8; image[259] = 2;
+ w32(image + 260, 4); w32(image + 264, 32);
+ image[292] = 3; image[294] = 64;
+ w32(image + 296, 128); w32(image + 300, 44);
+ assert(!gpu_unlock_fwsec_patch_frts(image, &fw, 0x100000));
+ assert(image[332] == 0x15 && image[384] == 1);
+ fw.interface_off = fw.dmem_size - 1;
+ assert(gpu_unlock_fwsec_patch_frts(image, &fw, 0x100000) == -EINVAL);
+ fw.interface_off = 0;
+ w32(image + 264, 0xfffffff0);
+ assert(gpu_unlock_fwsec_patch_frts(image, &fw, 0x100000) == -EINVAL);
+ assert(!gpu_unlock_fwsec_sig_offset(7, 0, 384, &offset) && offset == 0);
+ assert(!gpu_unlock_fwsec_sig_offset(5, 2, 384, &offset) && offset == 384);
+ assert(gpu_unlock_fwsec_sig_offset(1, 1, 384, &offset) == -ERANGE);
+ /* Optional integration check; normal tests contain no board-owned blobs. */
+ if (argc == 2) {
+ FILE *prom = fopen(argv[1], "rb");
+ u8 *image;
+ size_t got;
+
+ assert(prom);
+ assert(!fseek(prom, 0, SEEK_END));
+ got = ftell(prom);
+ assert(got == 1024 * 1024);
+ rewind(prom);
+ image = malloc(got);
+ assert(image && fread(image, 1, got, prom) == got);
+ fclose(prom);
+ assert(!gpu_unlock_vbios_fwsec(image, got, &fw));
+ assert(fw.ucode_id == 9 && fw.sig_count == 3 && fw.sig_versions == 7);
+ assert(fw.engine_mask == 0x400 && fw.imem_size == 0xda00 && fw.dmem_size == 0x800);
+ assert(fw.image_off == 0x4289c && fw.image_size == 0xe200);
+ assert(fw.pkc_off == 0x5a4 && fw.interface_off == 0x1c);
+ /* Revision 05 recorded Booter fuse version 1. That selects signature slot 1. */
+ assert(!gpu_unlock_fwsec_sig_offset(fw.sig_versions, 1, 384, &offset) && offset == 384);
+ assert(!gpu_unlock_fwsec_patch_frts(image + fw.image_off, &fw, 0x5ffe00000ull));
+ {
+ const u8 *cmd = image + fw.image_off + fw.data_off + 0x7c0;
+ u32 frts4k = (u32)cmd[32] | ((u32)cmd[33] << 8) | ((u32)cmd[34] << 16) | ((u32)cmd[35] << 24);
+
+ assert(frts4k == (0x5ffe00000ull >> 12));
+ assert(cmd[36] == 0 && cmd[37] == 1 && cmd[40] == 2);
+ }
+ free(image);
+ }
+ puts("vbios valid-chain, malformed range, patch and signature checks passed");
+ return 0;
+}
diff --git a/tests/unlock_shim/compat.h b/tests/unlock_shim/compat.h
new file mode 100644
index 0000000..cb2f2a3
--- /dev/null
+++ b/tests/unlock_shim/compat.h
@@ -0,0 +1,98 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef UNLOCK_TRANSACTION_SHIM_H
+#define UNLOCK_TRANSACTION_SHIM_H
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+#include <stdio.h>
+#include <asm-generic/errno.h>
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int32_t s32;
+typedef uint64_t dma_addr_t;
+typedef uint64_t resource_size_t;
+typedef uint64_t ktime_t;
+#define __iomem
+#define GFP_KERNEL 0
+#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
+#define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b))
+#define lower_32_bits(v) ((u32)(v))
+#define upper_32_bits(v) ((u32)((u64)(v)>>32))
+#define DMA_BIT_MASK(n) ((1ull<<(n))-1)
+#define MODULE_FIRMWARE(x)
+#define THIS_MODULE NULL
+void test_device_log(const char *, ...) __attribute__((format(printf, 1, 2)));
+#define dev_err(dev, ...) test_device_log(__VA_ARGS__)
+#define dev_info(dev, ...) test_device_log(__VA_ARGS__)
+#define PCI_D0 0
+#define PCI_DEV_FLAGS_NO_FLR_RESET (1u << 10)
+#define PCI_VENDOR_ID 0
+#define PCI_COMMAND 4
+#define PCI_COMMAND_MEMORY 2
+#define PCI_COMMAND_MASTER 4
+#define PCI_PM_CTRL 4
+#define PCI_PM_CTRL_STATE_MASK 3
+#define PCI_EXP_DEVCAP 4
+#define PCI_EXP_DEVCAP_FLR (1u<<28)
+#define PCI_EXP_DEVSTA 10
+#define PCI_EXP_DEVSTA_TRPND 32
+#define PCI_BASE_ADDRESS_0 16
+#define IORESOURCE_MEM 0x200
+struct list_head { unsigned int count; };
+static inline bool list_empty(const struct list_head *h) { return !h->count; }
+#define spin_lock_irqsave(lock, flags) do { (void)(lock); (flags)=0; } while (0)
+#define spin_unlock_irqrestore(lock, flags) do { (void)(lock); (void)(flags); } while (0)
+struct device { u64 *dma_mask; u64 coherent_dma_mask; struct list_head devres_head; int devres_lock; };
+void *devres_open_group(struct device *,void *,int);
+int devres_release_group(struct device *,void *);
+struct pci_dev { struct device dev; int current_state, pm_cap, refs; unsigned int dev_flags; };
+struct firmware { size_t size; const u8 *data; };
+void usleep_range(unsigned long, unsigned long);
+ktime_t ktime_get(void);
+static inline ktime_t ktime_add_us(ktime_t t, unsigned long u) { return t+u; }
+static inline int ktime_compare(ktime_t a,ktime_t b) { return a>b?1:a<b?-1:0; }
+u32 readl(const void *);
+void writel(u32,void *);
+#define memcpy_fromio memcpy
+void *dma_alloc_coherent(struct device *,size_t,dma_addr_t *,int);
+void dma_free_coherent(struct device *,size_t,void *,dma_addr_t);
+int dma_set_mask(struct device *,u64);
+int dma_set_coherent_mask(struct device *,u64);
+void dma_wmb(void);
+void *kzalloc(size_t n,int flags);
+static inline void *kvcalloc(size_t n,size_t s,int flags) { return calloc(n,s); }
+static inline void *kvmalloc(size_t n,int flags) { return malloc(n); }
+#define kfree free
+#define kvfree free
+int request_firmware(const struct firmware **,const char *,struct device *);
+void release_firmware(const struct firmware *);
+void __module_get(void *);
+int pci_read_config_word(struct pci_dev *,int,u16 *);
+int pci_read_config_dword(struct pci_dev *,int,u32 *);
+int pci_write_config_word(struct pci_dev *,int,u16);
+int pcie_capability_read_word(struct pci_dev *,int,u16 *);
+int pcie_capability_read_dword(struct pci_dev *,int,u32 *);
+int pci_is_pcie(struct pci_dev *);
+int pci_select_bars(struct pci_dev *,int);
+int pci_request_region(struct pci_dev *,int,const char *);
+void pci_release_region(struct pci_dev *,int);
+int aperture_remove_conflicting_pci_devices(struct pci_dev *,const char *);
+int pci_enable_device_mem(struct pci_dev *);
+void pci_disable_device(struct pci_dev *);
+resource_size_t pci_resource_len(struct pci_dev *,int);
+void *pci_iomap(struct pci_dev *,int,unsigned long);
+void pci_iounmap(struct pci_dev *,void *);
+void *pci_map_rom(struct pci_dev *,size_t *);
+void pci_unmap_rom(struct pci_dev *,void *);
+int pci_save_state(struct pci_dev *);
+void pci_restore_state(struct pci_dev *);
+void pci_set_master(struct pci_dev *);
+void pci_clear_master(struct pci_dev *);
+int pcie_flr(struct pci_dev *);
+struct pci_dev *pci_dev_get(struct pci_dev *);
+void pci_dev_put(struct pci_dev *);
+#endif
diff --git a/tests/unlock_shim/linux/aperture.h b/tests/unlock_shim/linux/aperture.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/aperture.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/delay.h b/tests/unlock_shim/linux/delay.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/delay.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/dma-mapping.h b/tests/unlock_shim/linux/dma-mapping.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/dma-mapping.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/firmware.h b/tests/unlock_shim/linux/firmware.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/firmware.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/io.h b/tests/unlock_shim/linux/io.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/io.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/module.h b/tests/unlock_shim/linux/module.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/module.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/pci.h b/tests/unlock_shim/linux/pci.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/pci.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/printk.h b/tests/unlock_shim/linux/printk.h
new file mode 100644
index 0000000..4ac5987
--- /dev/null
+++ b/tests/unlock_shim/linux/printk.h
@@ -0,0 +1,3 @@
+#include <stdio.h>
+#define pr_info(...) fprintf(stderr, __VA_ARGS__)
+#define pr_err(...) fprintf(stderr, __VA_ARGS__)
diff --git a/tests/unlock_shim/linux/slab.h b/tests/unlock_shim/linux/slab.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/slab.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/timekeeping.h b/tests/unlock_shim/linux/timekeeping.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/timekeeping.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/unlock_shim/linux/types.h b/tests/unlock_shim/linux/types.h
new file mode 100644
index 0000000..4715430
--- /dev/null
+++ b/tests/unlock_shim/linux/types.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#include "../compat.h"
diff --git a/tests/vbios_fixture.h b/tests/vbios_fixture.h
new file mode 100644
index 0000000..7951214
--- /dev/null
+++ b/tests/vbios_fixture.h
@@ -0,0 +1,33 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_TEST_VBIOS_FIXTURE_H
+#define GPU_UNLOCK_TEST_VBIOS_FIXTURE_H
+/* Synthetic, redistributable ROM; no board firmware contents. */
+static void vbios_w32(u8 *p, u32 v)
+{
+ for (int i = 0; i < 4; i++) p[i] = v >> (8 * i);
+}
+static void vbios_fixture(u8 *rom, int stride)
+{
+ unsigned int sum = 0;
+
+ memset(rom, 0, 4096);
+ rom[0] = 0x55; rom[1] = 0xaa; rom[0x18] = 32;
+ memcpy(rom + 32, "PCIR", 4);
+ rom[42] = 24; rom[48] = 8; rom[53] = 0x80;
+ rom[64] = 0xff; rom[65] = 0xb8; memcpy(rom + 66, "BIT", 3);
+ rom[71] = 1; rom[72] = 12; rom[73] = stride; rom[74] = 1;
+ for (int i = 64; i < 75; i++)
+ sum += rom[i];
+ rom[75] = -sum;
+ rom[76] = 0x70; rom[77] = 2; rom[78] = 4; vbios_w32(rom + 80, 100);
+ vbios_w32(rom + 100, 128);
+ rom[128] = 1; rom[129] = 6; rom[130] = 6; rom[131] = 2;
+ /* FWSEC need not be the first entry. */
+ rom[134] = 0x45; rom[140] = 0x85; vbios_w32(rom + 142, 256);
+ vbios_w32(rom + 256, (428u << 16) | 0x301);
+ vbios_w32(rom + 260, 1280); vbios_w32(rom + 264, 512); vbios_w32(rom + 268, 0);
+ vbios_w32(rom + 276, 256); vbios_w32(rom + 288, 1024);
+ rom[292] = 1; rom[294] = 1; rom[295] = 1; rom[296] = 1;
+}
+
+#endif