Skip to content

Your GeForce runs AI at half speed. This fixes it for 20/30/40 series cards. Up to 2× FP16 / BF16 / TF32 tensor throughput

C

Refactor driver to be recipe driven.

Zagitta09305c468f7e
diff --git a/.gitattributes b/.gitattributes
index 7438baf..3d013ab 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,4 +1,4 @@
-# `make dist` ships only the source needed to build and test the packages.
+# `make dist` ships only the source needed to build the packages.
/* export-ignore
/VERSION -export-ignore
/Makefile -export-ignore
@@ -7,7 +7,7 @@
/kernel -export-ignore
/licenses -export-ignore
/packaging -export-ignore
-/tests -export-ignore
*.bin export-ignore
+*.rom export-ignore
*.cu export-ignore
/debian -export-ignore
diff --git a/.gitignore b/.gitignore
index 99f5e55..981cef4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,4 @@
/debian/gpu-unlock-*/
modules.order
Module.symvers
+packaging/*/NVIDIA-Linux-x86_64-610.43.03-no-compat32.run
diff --git a/AGENTS.md b/AGENTS.md
new file mode 100644
index 0000000..8603c8b
--- /dev/null
+++ b/AGENTS.md
@@ -0,0 +1,3 @@
+Don't touch documentation or README unless explicitly asked to.
+Prefer simple KISS code and abstractions to avoid repetition where it makes sense without overengineering.
+Code should be straight forward to follow.
\ No newline at end of file
diff --git a/Makefile b/Makefile
index d6ac3a0..a37ef9c 100644
--- a/Makefile
+++ b/Makefile
@@ -14,37 +14,34 @@ PKG ?= gpu-unlock
MODULE_FILES := $(filter-out %.mod.c,$(sort $(wildcard kernel/gpu_unlock/*.[ch]))) \
kernel/gpu_unlock/Kbuild kernel/gpu_unlock/Makefile kernel/gpu_unlock/dkms.conf
-FIRMWARE_FILES := booter_load_ga102.bin gsp_ga10x.bin gsp_boot_ga102.bin \
- gsp_boot_desc_ga102.bin v67-ga102-$(NV).bin
+FIRMWARE_FILES := booter_load_ad102.bin booter_load_ga102.bin gsp_boot_ad102.bin gsp_boot_desc_ad102.bin \
+ booter_load_ga100.bin falcon_loader.bin gsp_ga10x.bin gsp_tu10x.bin \
+ gsp_boot_ga102.bin gsp_boot_desc_ga102.bin \
+ gsp_boot_ga100.bin gsp_boot_desc_ga100.bin
LICENSES := licenses/GPL-2.0.txt licenses/CMP40HX-MIT.txt licenses/NVIDIA-MIT.txt
NV_SOURCE := NVIDIA-kernel-module-source-$(NV)
-NV_UTILS := nvidia-utils-$(NV)-1-x86_64_v3.pkg.tar.zst
+NV_INSTALLER := NVIDIA-Linux-x86_64-$(NV)-no-compat32.run
TARBALL := $(abspath $(DISTDIR))/gpu-unlock-$(VERSION).tar.xz
DEBDIR := $(abspath $(DISTDIR))/deb
RPMDIR := $(abspath $(DISTDIR))/rpm
DEBS := $(foreach p,dkms firmware,$(DEBDIR)/gpu-unlock-$(p)_$(VERSION)_all.deb)
-.PHONY: all test dist inputs firmware assets install-dkms install-firmware \
+.PHONY: all dist inputs firmware assets install-dkms install-firmware \
install-mkinitcpio install-initramfs-tools install-dracut \
- install-licenses arch deb rpm install install-arch install-deb install-rpm
+ install-licenses arch deb rpm install install-arch install-deb install-rpm
all:
-# With generated or installed firmware, the tests also run the whole unlock
-# against mocked hardware.
-test:
- GPU_UNLOCK_ASSETS=$(ASSETS) $(PYTHON) -m unittest discover -s tests -v
-
# Source-only release tarball; see .gitattributes for what is left out.
dist:
mkdir -p $(DISTDIR)
git archive --format=tar --prefix=gpu-unlock-$(VERSION)/ \
- --output=$(DISTDIR)/gpu-unlock-$(VERSION).tar HEAD
+ --output=$(DISTDIR)/gpu-unlock-$(VERSION).tar $$(packaging/snapshot.sh)
xz -9 -f $(DISTDIR)/gpu-unlock-$(VERSION).tar
-# Download and SHA-256 check the pinned NVIDIA inputs. makepkg does this
-# itself from the PKGBUILD's source array.
+# Download the NVIDIA inputs. makepkg does this itself from the PKGBUILD's
+# source array.
inputs:
$(PYTHON) firmware/fetch_artifacts.py $(INPUTS)
@@ -54,7 +51,7 @@ firmware:
rm -rf $(ASSETS).src
mkdir -p $(ASSETS).src
tar -xf $(INPUTS)/$(NV_SOURCE).tar.xz -C $(ASSETS).src
- $(PYTHON) firmware/generate_v67.py "$$(realpath $(INPUTS)/$(NV_UTILS))" $(ASSETS)
+ $(PYTHON) firmware/extract_gsp.py "$$(realpath $(INPUTS)/$(NV_INSTALLER))" $(ASSETS)
$(PYTHON) firmware/extract_boot_assets.py $(ASSETS).src/$(NV_SOURCE) $(ASSETS)
assets: inputs
@@ -68,7 +65,6 @@ install-dkms:
install -Dm644 -t $(DESTDIR)$(DKMSDIR) $(MODULE_FILES)
install -Dm644 README.md $(DESTDIR)/usr/share/doc/gpu-unlock/README.md
-# The generators check every file against its pinned SHA-256 before writing it.
install-firmware:
install -Dm644 -t $(DESTDIR)$(FWDIR) $(addprefix $(ASSETS)/,$(FIRMWARE_FILES))
install -Dm644 $(ASSETS)/NVIDIA-LICENSE $(DESTDIR)$(LICENSEDIR)/gpu-unlock-firmware/NVIDIA-LICENSE
@@ -89,7 +85,7 @@ install-dracut:
install-licenses:
install -Dm644 -t $(DESTDIR)$(LICENSEDIR)/$(PKG) $(LICENSES)
-# Native packages, built from the committed HEAD like the PKGBUILD. The
+# Native packages, built from the working tree like the PKGBUILD. The
# install-* variants also pull build dependencies and install the result.
install:
@set -eu; \
@@ -134,11 +130,11 @@ rpm: dist inputs
rm -rf $(RPMDIR)
mkdir -p $(RPMDIR)/SOURCES
cp $(TARBALL) $(RPMDIR)/SOURCES/
- ln -s $(abspath $(INPUTS))/$(NV_SOURCE).tar.xz $(abspath $(INPUTS))/$(NV_UTILS) $(RPMDIR)/SOURCES/
+ ln -s $(abspath $(INPUTS))/$(NV_SOURCE).tar.xz $(abspath $(INPUTS))/$(NV_INSTALLER) $(RPMDIR)/SOURCES/
rpmbuild -ba --define "_topdir $(RPMDIR)" packaging/rpm/gpu-unlock.spec
install-rpm:
sudo dnf install -y rpm-build
sudo dnf builddep -y packaging/rpm/gpu-unlock.spec
$(MAKE) rpm
- sudo dnf install -y $(RPMDIR)/RPMS/noarch/gpu-unlock-*-$(VERSION)-*.noarch.rpm
+ sudo dnf install -y $(RPMDIR)/RPMS/noarch/gpu-unlock-*-$(VERSION)-*.noarch.rpm
\ No newline at end of file
diff --git a/README.md b/README.md
index ae48fa0..9cb5f6b 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,8 @@
-# GPU unlock 1.0.0
+# GPU unlock 1.1.0
-Unlock FP/BF16 instruction issue rate on Nvidia 3000's cards to their professional equivalents
+Unlock FP/BF16 instruction issue rate on consumer Nvidia 30 series cards to their professional equivalents
-This Linux kernel module combines the CMP unlocker's V67/PLM technique with the pre-driver timing of the UEFI approach. It runs the unlock, writes the speed selectors (SS0 `0x88888888`, which selects the full BF16 rate, and the 4-bit SS1 `0x8` IMLA4) and performs FLR before the unmodified NVIDIA driver binds the card. It is neither a UEFI application nor a patched NVIDIA driver.
+This Linux kernel module combines the cmpunlocker's V67/PLM technique with the pre-driver timing of the UEFI approach. It runs the unlock, writes the speed selectors (SS0 `0x88888888`, which selects the full BF16 rate, and the 4-bit SS1 `0x8` IMLA4) and performs FLR before the unmodified NVIDIA driver binds the card. It is neither a UEFI application nor a patched NVIDIA driver.
The benefit of this early kmod driver is that it does not require manual patching each time nvidia releases a new driver version.
@@ -10,55 +10,12 @@ The effect of this unlock is that it doubles the instruction issuing speed of FP
[3090 Ti dense 8192³ GEMM at 450 W, five-sample medians with FP32 accumulation](https://github.com/Zagitta/gpu-unlock/issues/1#issue-5905622687):
-
| Input | Before | After |
| ----- | ------------ | ------------- |
| FP16 | 87.46 TFLOPS | 118.39 TFLOPS |
| BF16 | 87.50 TFLOPS | 157.41 TFLOPS |
| TF32 | 42.58 TFLOPS | 82.61 TFLOPS |
-## Tested cards
-
-Results reported by users. The module accepts every GeForce ID listed in
-`kernel/gpu_unlock/main.c`, but the cards below are the ones someone has
-reported on. Please add yours with a
-[compatibility report](https://github.com/Zagitta/gpu-unlock/issues/new?template=compatibility-report.yml),
-whether it worked or not.
-
-| Chip | Card | PCI ID | Result | BF16 GEMM gain | Reports |
-| ----- | --------------------------------- | ----------- | ----------------------------------- | -------------- | ------- |
-| GA102 | RTX 3090 Ti | `10de:2203` | Unlocked | +68–80% | [#1](https://github.com/Zagitta/gpu-unlock/issues/1) |
-| GA102 | RTX 3090 | `10de:2204` | Unlocked | +25–67% | [#5](https://github.com/Zagitta/gpu-unlock/issues/5), [#9](https://github.com/Zagitta/gpu-unlock/issues/9), [#10](https://github.com/Zagitta/gpu-unlock/issues/10), [#17](https://github.com/Zagitta/gpu-unlock/issues/17), [#25](https://github.com/Zagitta/gpu-unlock/issues/25), [#27](https://github.com/Zagitta/gpu-unlock/issues/27), [#33](https://github.com/Zagitta/gpu-unlock/issues/33), [#148](https://github.com/Zagitta/gpu-unlock/issues/148), [#152](https://github.com/Zagitta/gpu-unlock/issues/152), [#155](https://github.com/Zagitta/gpu-unlock/issues/155), [#158](https://github.com/Zagitta/gpu-unlock/issues/158) |
-| GA102 | RTX 3080 Ti 20 GB (OEM, GA102-250) | `10de:2205` | Unlock completes, BF16 stays at half rate | — | [#28](https://github.com/Zagitta/gpu-unlock/issues/28) |
-| GA102 | RTX 3080 10 GB | `10de:2206` | Unlocked | +70–73% | [#14](https://github.com/Zagitta/gpu-unlock/issues/14) |
-| GA102 | RTX 3080 20 GB (memory upgrade) | `10de:2206` | Unlocked | +56–59% | [#10](https://github.com/Zagitta/gpu-unlock/issues/10), [#15](https://github.com/Zagitta/gpu-unlock/issues/15) |
-| GA104 | RTX 3070 | `10de:2484` | Unlocked | +73–81% | [#10](https://github.com/Zagitta/gpu-unlock/issues/10), [#12](https://github.com/Zagitta/gpu-unlock/issues/12) |
-| GA104 | RTX 3070 LHR | `10de:2488` | Unlocked | +77–84% | [#74](https://github.com/Zagitta/gpu-unlock/issues/74), [#137](https://github.com/Zagitta/gpu-unlock/issues/137) |
-| GA104 | RTX 3070 Ti | `10de:2482` | Not tested with this module (the same unlock worked from a UEFI port on Windows) | — | [#32](https://github.com/Zagitta/gpu-unlock/issues/32) |
-| GA106 | RTX 3060 12 GB | `10de:2504` | Unlocked | about +80% | [#13](https://github.com/Zagitta/gpu-unlock/pull/13), [#31](https://github.com/Zagitta/gpu-unlock/issues/31) |
-| GA106M | RTX 3060 Laptop 6 GB | `10de:2560` | Unlocked | +56–60% | [#86](https://github.com/Zagitta/gpu-unlock/issues/86) |
-| GA103 | any | `10de:2414` | No reports yet | — | — |
-
-The gains are dense GEMM throughput from each report's own benchmark, so
-they are not directly comparable. The power limit matters a lot: the lowest
-figure (+25%) is from cards capped at 225 W.
-
-Matrix-heavy work speeds up: LLM prefill (+14–36% in
-[#9](https://github.com/Zagitta/gpu-unlock/issues/9),
-[#15](https://github.com/Zagitta/gpu-unlock/issues/15),
-[#25](https://github.com/Zagitta/gpu-unlock/issues/25) and
-[#33](https://github.com/Zagitta/gpu-unlock/issues/33)), diffusion and
-FlashAttention. LLM decode, which is limited by memory bandwidth, and INT8
-work stay about the same. The reports that compared outputs found them
-bit-identical before and after.
-
-Reports also cover up to four cards in one machine, the card driving the
-display, and cards passed through to a VM with vfio-pci
-([#17](https://github.com/Zagitta/gpu-unlock/issues/17)).
-
-
-
-
## How to install
First run:
@@ -83,14 +40,24 @@ in `/etc/default/grub`, then run `sudo update-grub`.
- **UKI / generated entries:** add it to `/etc/kernel/cmdline` and regenerate
your boot images/entries (e.g. `sudo mkinitcpio -P` for mkinitcpio UKIs).
+# FAQ
+- Q: Does this depend on a specific driver version? I see `610.43.03` is pinned.
+A: No the driver is only fetched to extract firmware this unlock depends on
+- Q: What about Windows?
+A: I don't run windows and don't plan to maintain it but [https://github.com/impacifist/Windows-GPU-UNLOCK](https://github.com/impacifist/Windows-GPU-UNLOCK) has built a port, YMMV.
+- Q: What about Turing and Ada?
+A: There are forks with support already, I will integrate this soon.
+- Q: Does this hurt my GPU?
+A: It's very unlikely, this only unlocks a small handicap Nvidia has put in place to differentiate datacenter and professional cards from consumer versions. It may heat up your GPU slightly because it now does more compute but the standard throttling and power management remains completely intact and should downclock.
+- Q: How can I assist?
+A: Please consider [donating](#support-me) so I can purchase more GPUs <3
## Local build
-Every build downloads the pinned NVIDIA/CachyOS inputs over HTTPS, checks
-their SHA-256, and generates firmware only in the build directory and the
-binary package. All three formats install through the same `make install-*`
-targets. Packages are built from the committed `HEAD`.
+Every build downloads the NVIDIA/CachyOS inputs over HTTPS and generates
+firmware only in the build directory and the binary package. All three formats install through the same `make install-*`
+targets. Packages are built from the working tree, uncommitted changes included.
Build, install build dependencies, and install the packages in one step
(asks for your sudo password):
@@ -141,14 +108,14 @@ There is no `softdep nvidia pre:` rule: kmod honours only the first
The module does nothing unless the kernel command line opts in:
```
-gpu_unlock.targets=all # every unbound GA102/GA103/GA104/GA106 card
+gpu_unlock.targets=all # every supported unbound card
gpu_unlock.targets=0000:21:00.0 # only these cards
gpu_unlock.targets=0000:21:00.0,0000:22:00.0 # comma-separated
```
Addresses are full PCI names as `lspci -D` prints them (hex digits in either
case), independent of GPU numbering and display role. An entry that is not
-a GA102/GA103/GA104/GA106 card, including `all` mixed with addresses, is logged and ignored.
+a supported recipe (GA100/GA102/GA103/GA104/GA106 or RTX 4090), including `all` mixed with addresses, is logged and ignored.
The cards are unlocked concurrently, about a second in all however many
there are, and each on its own: a card that is already bound or powered
down is skipped. Bus mastering that the board firmware left on (Supermicro
@@ -181,7 +148,7 @@ Check the result with:
```sh
journalctl -k -b | grep gpu_unlock
-lspci -nnk -s 0000:21:00.0
+lspci -nnk -s 0000:[ID]:00.0
```
To stop unlocking, remove `gpu_unlock.targets=` from the kernel command line;
@@ -192,7 +159,7 @@ If the firmware on a card never stops, the module logs it and keeps the DMA
buffers that firmware may still use until the next power off. Power off
before trying again.
-The kernel module and local packaging are GPL-2.0-only. The generated payload
+The kernel module and local packaging are GPL-2.0-only. The generated buffer
derives from the pinned CMP reference; NVIDIA firmware remains under its own
terms. See `firmware/NOTICE.md` and `licenses/` for attribution and notices.
@@ -200,7 +167,9 @@ terms. See `firmware/NOTICE.md` and `licenses/` for attribution and notices.
Help me buy more GPUs <3
-[https://buymeacoffee.com/zagitta](https://buymeacoffee.com/zagitta)
+[Github Sponsor](https://github.com/sponsors/Zagitta)
+
+[https://buymeacoffee.com/zagitta](https://buymeacoffee.com/zagitta)
ETH: 0xf4eee937937f92038D70B16AbD7736D8a53a1c5c
diff --git a/VERSION b/VERSION
index 3eefcb9..9084fa2 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.0.0
+1.1.0
diff --git a/debian/changelog b/debian/changelog
index ee1bb2c..8342a27 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-gpu-unlock (1.0.0) UNRELEASED; urgency=medium
+gpu-unlock (1.1.0) UNRELEASED; urgency=medium
* Initial release.
diff --git a/debian/rules b/debian/rules
index cc55b2e..299630a 100755
--- a/debian/rules
+++ b/debian/rules
@@ -15,9 +15,6 @@ override_dh_auto_configure:
override_dh_auto_build:
$(MAKE) INPUTS="$(GPU_UNLOCK_INPUTS)" ASSETS="$(ASSETS)" firmware
-override_dh_auto_test:
- $(MAKE) test ASSETS="$(ASSETS)"
-
override_dh_auto_install:
$(MAKE) DESTDIR=$(CURDIR)/debian/tmp ASSETS="$(ASSETS)" \
install-dkms install-firmware install-initramfs-tools
diff --git a/firmware/NOTICE.md b/firmware/NOTICE.md
index eb14cc3..e95e6fb 100644
--- a/firmware/NOTICE.md
+++ b/firmware/NOTICE.md
@@ -1,7 +1,7 @@
# Firmware provenance and distribution scope
-`gsp_ga10x.bin` is extracted byte-for-byte from the checksummed NVIDIA
-610.43.03 package in `artifacts-610.43.03.json`. The Booter and GSP boot images
+`gsp_ga10x.bin` and `gsp_tu10x.bin` are extracted byte-for-byte from the NVIDIA
+610.43.03 package by `extract_gsp.py`. The Booter and GSP boot images
are decoded unchanged from the pinned 610.43.03 source archive. The extraction
also preserves that package's complete `usr/share/licenses/nvidia-utils/LICENSE`
as `NVIDIA-LICENSE`. It is installed with the firmware package.
@@ -14,17 +14,25 @@ apply, including NVIDIA-platform use. Personal local packaging is the initial
delivery target. Review the full pinned agreement and the separately generated
payload before any public distribution; redistribution approval is not claimed.
-The V67 constants are adapted from WildFlash1st/cmp90hx-unlock at commit
-`76940ae583281daa2b883518a2e89b56c11d3a46`, patch
-`driver/patches/cmp90-rejoin15/0014-6104303-cmp90hx-stockflow-rejoin14-multigpu-state.patch`,
-function `_kgspCmp90FillV67Signature`. The repository's actual `LICENSE` is
-GPL version 2; the earlier manifest's MIT label was incorrect. That license is
-preserved in `licenses/GPL-2.0.txt`. The generated image is a separate file;
-the extracted NVIDIA images are never rewritten.
+The GA102 re-entrant payload is derived from the live 610.43.03 Booter
+behavior (its OS-block validation, PLM pair writes and exit path), not from
+the cmpunlocker template. The Stage 1 reference checkout,
+amoghmunikote/cmpunlocker at commit
+`2d10f891a58824f0591c3332c4eccf37ff36b086`, is the recorded origin of the
+initial payload constants and PLM group. Its repository license is GPL version
+2, preserved in `licenses/GPL-2.0.txt`. The payload is source code in this
+repository, separate from the unchanged NVIDIA firmware.
NVIDIA source files inspected during the loader audit carry their own MIT
copyright notices. The minimum helper's metadata layout and Falcon operations
follow those source contracts; `licenses/NVIDIA-MIT.txt` preserves the notice.
The inspected EFI reference's notice is in `licenses/CMP40HX-MIT.txt`.
-The reference changes a CMP 90HX, not an RTX 3090, and is not evidence of
-standalone boot or persistence through NVIDIA 615.71.09.
+The payload reference targets GA100 CMP 170HX. It does not establish GA10x
+firmware acceptance, standalone multiwrite, final FLR retention or stock-driver
+handover. Those require the Stage 1 hardware experiments.
+
+The Linux loading geometry, register sequence and success gates are ported from
+[elias-gaeros/gpu-unlock's RTX 4090 work](https://github.com/elias-gaeros/gpu-unlock/blob/rtx4090/rtx4090-register-report.md).
+That report records one successful RTX 4090 production-booter transaction.
+A hardware repeat of the refactored path is still needed. AD102 uses the same per-entry signature image as GA10x. Its
+privileged stores are WPR_CFG, WPR and FEAT.
diff --git a/firmware/README.md b/firmware/README.md
index 32348bb..e318f65 100644
--- a/firmware/README.md
+++ b/firmware/README.md
@@ -1,19 +1,20 @@
# Pinned 610.43.03 firmware
-`make inputs` runs `fetch_artifacts.py`, which downloads the NVIDIA open kernel
-module source archive and the exact CachyOS `nvidia-utils` package pinned in
-`artifacts-610.43.03.json`. Downloads use HTTPS and are checked against their
-SHA-256 before they enter the cache; cached files are rechecked on every run.
-Nothing downloaded is executed or installed.
+`make inputs` downloads the two NVIDIA inputs into the cache.
+`make firmware` runs `extract_gsp.py` for the two GSP ELFs and the NVIDIA
+license, and `extract_boot_assets.py` for the signed SEC2 Booter bundles,
+generic Falcon loader, and per-family GSP boot images/descriptors.
+Extracted NVIDIA bytes are unchanged.
-`make firmware` then builds the five files the kernel module loads from
-`gpu-unlock/610.43.03/`:
+The payload is ordinary source code: the V67 call table in
+`kernel/gpu_unlock/recipes.c`, built per write by `payload.c`. There is no
+generator, no external `dmem.bin` override, and no reference image on disk.
-- `generate_v67.py` extracts `gsp_ga10x.bin` and the NVIDIA license from the
- package and generates the 64,000-byte `v67-ga102-610.43.03.bin`.
-- `extract_boot_assets.py` builds `booter_load_ga102.bin`,
- `gsp_boot_ga102.bin` and `gsp_boot_desc_ga102.bin` from the source archive.
+The SEC2 Booter executes on Falcon and loads the separate GSP boot image.
+GA100 uses `gsp_tu10x.bin` with `.fwsignature_ga100`; GA10x uses
+`gsp_ga10x.bin` with `.fwsignature_ga10x`. The selected card determines its
+GSP assets once, separately from register tables. All new hardware contexts
+remain experimental; see [Stage 1 evidence](../docs/STAGE1.md).
-Both scripts check every input and output against its pinned SHA-256 before
-writing anything. Python 3.14+ is required for zstd tar support. See
-[NOTICE.md](NOTICE.md) for provenance and licensing.
+Python 3, zstd, and the pinned inputs are sufficient for extraction.
+See [NOTICE.md](NOTICE.md) for provenance and licensing.
diff --git a/firmware/artifacts-610.43.03.json b/firmware/artifacts-610.43.03.json
deleted file mode 100644
index d5405f1..0000000
--- a/firmware/artifacts-610.43.03.json
+++ /dev/null
@@ -1,161 +0,0 @@
-{
- "version": 1,
- "target": "GA102 / 610.43.03 rejoin15 reference path",
- "source_artifacts": [
- {
- "name": "NVIDIA open kernel modules source archive",
- "url": "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-610.43.03.tar.xz",
- "checksum_url": "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-610.43.03.tar.xz.sha256sum",
- "revision": "610.43.03 release source (NVIDIA tag 610.43.03)",
- "sha256": "7e118923c7a23edc36114d63273a46e3e04e9af98695a42203e7ac2dfe9fc1dc",
- "license": "GPL-2.0 OR MIT",
- "filename": "NVIDIA-kernel-module-source-610.43.03.tar.xz"
- },
- {
- "name": "GA102 CMP90 reference and rejoin15 patch set",
- "url": "https://github.com/WildFlash1st/cmp90hx-unlock",
- "revision": "76940ae583281daa2b883518a2e89b56c11d3a46",
- "license": "GPL-2.0-only (repository LICENSE; previous MIT label was incorrect)",
- "files": [
- {
- "path": "driver/patches/cmp90-rejoin15/0014-6104303-cmp90hx-stockflow-rejoin14-multigpu-state.patch",
- "sha256": "c51805dc332c3b6be2e6dd34964a47ce6b1ea0ce73017e0351c216d319094bb1"
- },
- {
- "path": "driver/patches/cmp90-rejoin15/0015-6104303-cmp90hx-stockflow-rejoin15-serialized-start.patch",
- "sha256": "3dd3883479d10a822d54292db87a1552f15d6b144b8c42bfc88a519674836cb4"
- },
- {
- "path": "LICENSE",
- "sha256": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643"
- }
- ]
- },
- {
- "name": "nvidia-utils binary package used for exact firmware extraction",
- "version": "610.43.03-1",
- "architecture": "x86_64_v3",
- "license": "custom (package metadata)",
- "sha256": "e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9",
- "firmware": [
- {
- "path": "usr/lib/firmware/nvidia/610.43.03/gsp_ga10x.bin",
- "size": 84277400,
- "sha256": "572373620a37418f24dc16b5031c39338778c3257e48e8408de9a57291b24f3a"
- }
- ],
- "filename": "nvidia-utils-610.43.03-1-x86_64_v3.pkg.tar.zst",
- "url": "https://archive.cachyos.org/archive/cachyos-v3/nvidia-utils-610.43.03-1-x86_64_v3.pkg.tar.zst",
- "notice": {
- "path": "usr/share/licenses/nvidia-utils/LICENSE",
- "sha256": "c692fb14ad4f499c6aee1b9e144e6905148c420e6225b17cb2937366c1e507ad"
- }
- }
- ],
- "generated_payload": {
- "name": "rejoin15 GA102 610.43.03 V67 signature image",
- "size": 64000,
- "size_hex": "0x0000fa00",
- "fill_dword_le": "0x0000019c",
- "stack_map": {
- "active_canary_dmem": "0x0000ff4c",
- "initial_return_slot_dmem": "0x0000ff50"
- },
- "overrides": [
- {
- "offset": "0x00001100",
- "value": "0x00000007"
- },
- {
- "offset": "0x0000f948",
- "value": "0xffffffff",
- "role": "write value"
- },
- {
- "offset": "0x0000f950",
- "value": "0x00000d44"
- },
- {
- "offset": "0x0000f960",
- "value": "0x00823804",
- "role": "FEAT_OVR_PLM address"
- },
- {
- "offset": "0x0000f968",
- "value": "0x00001fce"
- },
- {
- "offset": "0x0000f974",
- "value": "0x00000000"
- },
- {
- "offset": "0x0000f97c",
- "value": "0x00001101"
- },
- {
- "offset": "0x0000f980",
- "value": "0x000084c8"
- },
- {
- "offset": "0x0000f984",
- "value": "0x00008e18"
- },
- {
- "offset": "0x0000f98c",
- "value": "0x000084c8"
- },
- {
- "offset": "0x0000f990",
- "value": "0x00000000"
- },
- {
- "offset": "0x0000f998",
- "value": "0x00001fce"
- },
- {
- "offset": "0x0000f9a4",
- "value": "0x0000ffbc"
- },
- {
- "offset": "0x0000f9ac",
- "value": "0x00005789"
- },
- {
- "offset": "0x0000f9bc",
- "value": "0x00000d44"
- },
- {
- "offset": "0x0000f9cc",
- "value": "0x00000003"
- },
- {
- "offset": "0x0000f9d4",
- "value": "0x00001fce"
- },
- {
- "offset": "0x0000f9e8",
- "value": "0x00000d52"
- },
- {
- "offset": "0x0000f9ec",
- "value": "0x000081ee",
- "role": "terminator"
- }
- ],
- "write_value_offset": "0x0000f948",
- "write_value": "0xffffffff",
- "write_address_offset": "0x0000f960",
- "write_address": "0x00823804",
- "terminator_offset": "0x0000f9ec",
- "terminator": "0x000081ee",
- "sha256": "2a0aa1f86d0b6afff5214d641d87d46dc98ef06ed908ff05d0ddbad5ffb7e197",
- "generator": "firmware/generate_v67.py",
- "reference": "rejoin15 patch 0014, _kgspCmp90FillV67Signature"
- },
- "derived_assets": {
- "booter_load_ga102.bin": "056cbc064a888f80717726c54d7f64fcd68044d123870c95a74f37f4975dcaa4",
- "gsp_boot_ga102.bin": "3df651acb174fd5f5561a1996965b192f50c5d763168446ed0d4ead19d895768",
- "gsp_boot_desc_ga102.bin": "cc5a3fc1f4c47f2fe107b1b68f0176d8a448e6dec84b575e5560ccef5262bf15"
- },
- "limitation": "The manifest and generator pin/extract firmware artifacts and reproduce the reference V67 image. They do not implement an independent GSP boot/BooterLoad path or establish applicability to RTX 3090 / driver 615.71.09."
-}
diff --git a/firmware/extract_boot_assets.py b/firmware/extract_boot_assets.py
index eb277a5..2fcbffa 100644
--- a/firmware/extract_boot_assets.py
+++ b/firmware/extract_boot_assets.py
@@ -1,31 +1,30 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-only
-"""Build the Booter and GSP boot assets from the pinned 610 generated source."""
+"""Build the Booter and GSP boot assets from the NVIDIA 610 generated source."""
from __future__ import annotations
import argparse
-import hashlib
-import json
import zlib
import re
import struct
import sys
from pathlib import Path
-MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
-
-
MAGIC = 0x314C5442
-LABELS = (
- "IMAGE_PROD",
- "HEADER_PROD",
- "SIG_PROD",
- "PATCH_LOC",
- "PATCH_META",
- "NUM_SIGS",
+
+# Generated NVIDIA sources this script reads.
+SOURCES = (
+ 'g_bindata_kgspGetBinArchiveBooterLoadUcode_AD102.c',
+ 'g_bindata_kgspGetBinArchiveBooterLoadUcode_GA100.c',
+ 'g_bindata_kgspGetBinArchiveBooterLoadUcode_GA102.c',
+ 'g_bindata_kgspGetBinArchiveGspRmBoot_AD102.c',
+ 'g_bindata_kgspGetBinArchiveGspRmBoot_GA100.c',
+ 'g_bindata_kgspGetBinArchiveGspRmBoot_GA102.c',
+ 'g_bindata_ksec2GetBinArchiveBlUcode_TU102.c',
)
+
def _array_bytes(text: str, label: str) -> tuple[bytes, bool, int]:
marker = f'BINDATA_LABEL_{label}_data[]'
if text.count(marker) != 1:
@@ -95,36 +94,37 @@ def build_bundle(source_text: str) -> bytes:
return prefix + image + signatures
-PINNED = {
- 'g_bindata_kgspGetBinArchiveBooterLoadUcode_GA102.c': '04cd3db7f670adc4efd79d2e76217bb809fffddfbd6f1af884cde2af26dd39af',
- 'g_bindata_kgspGetBinArchiveGspRmBoot_GA102.c': '35e5aa9075ad3898b6022ffd5820c953d809caa7dabd530cfc1a83dc2a67c5bd',
-}
-
-
def extract_assets(source_root: Path, output: Path) -> None:
inputs = {}
- for name, digest in PINNED.items():
+ for name in SOURCES:
data = (source_root / 'src/nvidia/generated' / name).read_bytes()
- if hashlib.sha256(data).hexdigest() != digest:
- raise ValueError('pinned generated source mismatch: ' + name)
inputs[name] = data.decode('ascii')
- booter = build_bundle(inputs['g_bindata_kgspGetBinArchiveBooterLoadUcode_GA102.c'])
boot = inputs['g_bindata_kgspGetBinArchiveGspRmBoot_GA102.c']
image = _array_bytes(boot, 'UCODE_IMAGE_PROD')[0]
desc = _array_bytes(boot, 'UCODE_DESC_PROD')[0]
if len(desc) != 84:
- raise ValueError('unexpected pinned bootloader descriptor length')
+ raise ValueError('unexpected bootloader descriptor length')
fields = struct.unpack('<21I', desc)
if fields[0] != 5 or fields[14] != 1 or fields[19] != len(image) or fields[20] != 0:
raise ValueError('unsupported bootloader descriptor')
for index in (1,3,8,10,12):
if not fields[index+1] or fields[index]+fields[index+1] > len(image):
raise ValueError('bootloader descriptor range outside image')
- blobs = {'booter_load_ga102.bin': booter, 'gsp_boot_ga102.bin': image, 'gsp_boot_desc_ga102.bin': desc}
- pinned = json.loads(MANIFEST.read_text())['derived_assets']
- for name, data in blobs.items():
- if hashlib.sha256(data).hexdigest() != pinned[name]:
- raise ValueError('derived asset differs from its pinned SHA-256: ' + name)
+ blobs = {'gsp_boot_ga102.bin': image, 'gsp_boot_desc_ga102.bin': desc}
+ for chip in ('GA100', 'GA102'):
+ blobs[f'booter_load_{chip.lower()}.bin'] = build_bundle(
+ inputs[f'g_bindata_kgspGetBinArchiveBooterLoadUcode_{chip}.c'])
+ ga100_boot = inputs['g_bindata_kgspGetBinArchiveGspRmBoot_GA100.c']
+ blobs['gsp_boot_ga100.bin'] = _array_bytes(ga100_boot, 'UCODE_IMAGE')[0]
+ blobs['gsp_boot_desc_ga100.bin'] = _array_bytes(ga100_boot, 'UCODE_DESC')[0]
+ loader = inputs['g_bindata_ksec2GetBinArchiveBlUcode_TU102.c']
+ if not re.search(r'UCODE_DESC_data\s*=\s*\{\s*0xfd,\s*0,\s*\{\s*0x0,\s*0x200,\s*0x200,\s*0x100\s*}', loader):
+ raise ValueError('unsupported generic loader descriptor')
+ blobs['falcon_loader.bin'] = _array_bytes(loader, 'UCODE_IMAGE')[0]
+ ad102_boot = inputs['g_bindata_kgspGetBinArchiveGspRmBoot_AD102.c']
+ blobs['booter_load_ad102.bin'] = build_bundle(inputs['g_bindata_kgspGetBinArchiveBooterLoadUcode_AD102.c'])
+ blobs['gsp_boot_ad102.bin'] = _array_bytes(ad102_boot, 'UCODE_IMAGE_PROD')[0]
+ blobs['gsp_boot_desc_ad102.bin'] = _array_bytes(ad102_boot, 'UCODE_DESC_PROD')[0]
output.mkdir(parents=True, exist_ok=True)
for name, data in blobs.items():
(output/name).write_bytes(data)
@@ -132,7 +132,7 @@ def extract_assets(source_root: Path, output: Path) -> None:
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("source", type=Path, help='pinned NVIDIA source root')
+ parser.add_argument("source", type=Path, help='NVIDIA source root')
parser.add_argument("output", type=Path)
args = parser.parse_args()
extract_assets(args.source, args.output)
diff --git a/firmware/extract_gsp.py b/firmware/extract_gsp.py
new file mode 100644
index 0000000..fdc6698
--- /dev/null
+++ b/firmware/extract_gsp.py
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+"""Extract the GSP firmware and NVIDIA license from the driver installer."""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import tarfile
+from pathlib import Path
+
+FIRMWARE = (
+ "firmware/gsp_ga10x.bin",
+ "firmware/gsp_tu10x.bin",
+)
+NOTICE = "LICENSE"
+VERSION = "610.43.03"
+HEADER_BYTES = 1 << 20
+
+
+def payload_offset(installer: Path) -> int:
+ """Offset of the makeself payload after its shell header."""
+ with installer.open("rb") as stream:
+ header = stream.read(HEADER_BYTES)
+ version = re.search(rb"^version_string=([0-9.]+)$", header, re.M)
+ skip = re.search(rb"^skip=(\d+)$", header, re.M)
+ if not version or not skip:
+ raise ValueError("not an NVIDIA makeself installer")
+ if version[1].decode() != VERSION:
+ raise ValueError(f"installer version is {version[1].decode()}, expected {VERSION}")
+ # The installer extracts with `tail -n +$skip`, so the payload starts
+ # after skip - 1 header lines.
+ offset = 0
+ for _ in range(int(skip[1]) - 1):
+ newline = header.find(b"\n", offset)
+ if newline < 0:
+ raise ValueError("installer header is shorter than its skip count")
+ offset = newline + 1
+ return offset
+
+
+def read_members(archive: tarfile.TarFile, names: tuple[str, ...]) -> dict[str, bytes]:
+ wanted = set(names)
+ result = {}
+ for member in archive:
+ if member.name not in wanted:
+ continue
+ if member.name in result:
+ raise ValueError(f"duplicate installer member: {member.name}")
+ if not member.isfile() or member.size > 128 * 1024 * 1024:
+ raise ValueError(f"not a regular file in installer: {member.name}")
+ stream = archive.extractfile(member)
+ if stream is None:
+ raise ValueError(f"could not read installer member: {member.name}")
+ result[member.name] = stream.read()
+ missing = sorted(wanted - result.keys())
+ if missing:
+ raise ValueError(f"missing installer member: {', '.join(missing)}")
+ return result
+
+
+def extract_members(installer: Path, names: tuple[str, ...]) -> dict[str, bytes]:
+ offset = payload_offset(installer)
+ with installer.open("rb") as stream:
+ stream.seek(offset)
+ process = subprocess.Popen(["zstd", "-dc"], stdin=stream, stdout=subprocess.PIPE)
+ try:
+ with tarfile.open(fileobj=process.stdout, mode="r|") as archive:
+ result = read_members(archive, names)
+ if process.wait() != 0:
+ raise ValueError("zstd could not read the installer payload")
+ return result
+ finally:
+ if process.stdout:
+ process.stdout.close()
+ if process.poll() is None:
+ process.kill()
+ process.wait()
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("installer", type=Path, help="NVIDIA-Linux-x86_64-610.43.03-no-compat32.run")
+ parser.add_argument("output", type=Path, help="directory for extracted firmware")
+ args = parser.parse_args()
+
+ members = extract_members(args.installer, (*FIRMWARE, NOTICE))
+ args.output.mkdir(parents=True, exist_ok=True)
+ for path in FIRMWARE:
+ (args.output / Path(path).name).write_bytes(members[path])
+ (args.output / "NVIDIA-LICENSE").write_bytes(members[NOTICE])
+ return 0
+
+
+if __name__ == "__main__":
+ try:
+ raise SystemExit(main())
+ except (OSError, ValueError) as error:
+ raise SystemExit(f"extraction failed: {error}")
diff --git a/firmware/fetch_artifacts.py b/firmware/fetch_artifacts.py
index f8cc007..74553ad 100644
--- a/firmware/fetch_artifacts.py
+++ b/firmware/fetch_artifacts.py
@@ -1,41 +1,35 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-only
-"""Fetch pinned inputs without executing/installing them; atomically verify cache."""
+"""Fetch the NVIDIA inputs without executing or installing them."""
from __future__ import annotations
import argparse
-import hashlib
-import json
import os
from pathlib import Path
import tempfile
import urllib.request
-MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
-
-
-def digest(path: Path) -> str:
- with path.open("rb") as stream:
- return hashlib.file_digest(stream, "sha256").hexdigest()
-
-
-def inputs(manifest: dict) -> list[dict]:
- """The NVIDIA source archive and the nvidia-utils package the build uses."""
- source, _reference, package = manifest["source_artifacts"]
- return [source, package]
+INPUTS = (
+ {
+ "filename": "NVIDIA-kernel-module-source-610.43.03.tar.xz",
+ "url": "https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/"
+ "NVIDIA-kernel-module-source-610.43.03.tar.xz",
+ },
+ {
+ "filename": "NVIDIA-Linux-x86_64-610.43.03-no-compat32.run",
+ "url": "https://download.nvidia.com/XFree86/Linux-x86_64/610.43.03/"
+ "NVIDIA-Linux-x86_64-610.43.03-no-compat32.run",
+ },
+)
def fetch(entry: dict, output: Path, offline: bool = False) -> Path:
output.mkdir(parents=True, exist_ok=True)
destination = output / entry["filename"]
if destination.exists():
- if destination.is_symlink() or digest(destination) != entry["sha256"]:
- raise ValueError(f"invalid cached artifact: {destination}; remove it to retry")
return destination
if offline:
raise FileNotFoundError(f"missing cached artifact: {destination}")
- if not entry["url"].startswith("https://"):
- raise ValueError("artifact downloads require HTTPS")
temporary = None
try:
request = urllib.request.Request(entry["url"], headers={"User-Agent": "gpu-unlock-artifacts/0.1.0"})
@@ -52,8 +46,6 @@ def fetch(entry: dict, output: Path, offline: bool = False) -> Path:
stream.write(block)
stream.flush()
os.fsync(stream.fileno())
- if digest(temporary) != entry["sha256"]:
- raise ValueError(f"SHA-256 mismatch: {entry['filename']}")
temporary.chmod(0o644)
temporary.replace(destination)
return destination
@@ -65,10 +57,10 @@ def fetch(entry: dict, output: Path, offline: bool = False) -> Path:
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("output", type=Path, help="cache directory (for example scratchpad/artifacts)")
- parser.add_argument("--offline", action="store_true", help="verify cached inputs without network")
+ parser.add_argument("--offline", action="store_true", help="use cached inputs without network")
args = parser.parse_args()
try:
- for entry in inputs(json.loads(MANIFEST.read_text())):
+ for entry in INPUTS:
print(fetch(entry, args.output, args.offline))
except (OSError, ValueError) as error:
parser.exit(1, f"fetch failed: {error}\n")
diff --git a/firmware/generate_v67.py b/firmware/generate_v67.py
deleted file mode 100644
index 3275a6f..0000000
--- a/firmware/generate_v67.py
+++ /dev/null
@@ -1,133 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-License-Identifier: GPL-2.0-only
-"""Extract pinned 610.43.03 GA10x firmware and generate the rejoin15 V67 image."""
-
-from __future__ import annotations
-
-import argparse
-import json
-import hashlib
-import subprocess
-import tarfile
-from pathlib import Path
-
-
-PACKAGE_SHA256 = "e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9"
-GSP_SHA256 = "572373620a37418f24dc16b5031c39338778c3257e48e8408de9a57291b24f3a"
-PAYLOAD_SHA256 = "2a0aa1f86d0b6afff5214d641d87d46dc98ef06ed908ff05d0ddbad5ffb7e197"
-PACKAGE_VERSION = "610.43.03-1"
-FIRMWARE_PREFIX = "usr/lib/firmware/nvidia/610.43.03/"
-MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
-
-PAYLOAD_SIZE = 0xFA00
-FILL_DWORD = 0x0000019C
-PATCHED_WORDS = {
- 0x1100: 0x00000007,
- 0xF948: 0xFFFFFFFF,
- 0xF950: 0x00000D44,
- 0xF960: 0x00823804,
- 0xF968: 0x00001FCE,
- 0xF974: 0x00000000,
- 0xF97C: 0x00001101,
- 0xF980: 0x000084C8,
- 0xF984: 0x00008E18,
- 0xF98C: 0x000084C8,
- 0xF990: 0x00000000,
- 0xF998: 0x00001FCE,
- 0xF9A4: 0x0000FFBC,
- 0xF9AC: 0x00005789,
- 0xF9BC: 0x00000D44,
- 0xF9CC: 0x00000003,
- 0xF9D4: 0x00001FCE,
- 0xF9E8: 0x00000D52,
- 0xF9EC: 0x000081EE,
-}
-
-
-def sha256(data: bytes) -> str:
- return hashlib.sha256(data).hexdigest()
-
-
-def extract_member(package: Path, name: str) -> bytes:
- if package.name.endswith(".zst"):
- process = subprocess.Popen(["zstd", "-dc", str(package)], stdout=subprocess.PIPE)
- try:
- with tarfile.open(fileobj=process.stdout, mode="r|") as archive:
- result = read_member(archive, name)
- if process.wait() != 0:
- raise ValueError("zstd could not read the pinned package")
- return result
- finally:
- if process.stdout:
- process.stdout.close()
- if process.poll() is None:
- process.kill()
- process.wait()
- with tarfile.open(package, mode="r:*") as archive:
- return read_member(archive, name)
-
-
-def read_member(archive: tarfile.TarFile, name: str) -> bytes:
- result = None
- for member in archive:
- if member.name != name:
- continue
- if result is not None:
- raise ValueError(f"duplicate package member: {name}")
- if not member.isfile() or member.size > 128 * 1024 * 1024:
- raise ValueError(f"not a regular file in package: {name}")
- stream = archive.extractfile(member)
- if stream is None:
- raise ValueError(f"could not read package member: {name}")
- result = stream.read()
- if result is None:
- raise ValueError(f"missing package member: {name}")
- return result
-
-
-def build_payload() -> bytes:
- word = FILL_DWORD.to_bytes(4, "little")
- payload = bytearray(word * (PAYLOAD_SIZE // len(word)))
- for offset, value in PATCHED_WORDS.items():
- payload[offset : offset + 4] = value.to_bytes(4, "little")
- return bytes(payload)
-
-
-def main() -> int:
- parser = argparse.ArgumentParser()
- parser.add_argument("package", type=Path, help="pinned nvidia-utils-610.43.03 package")
- parser.add_argument("output", type=Path, help="directory for extracted firmware and payload")
- args = parser.parse_args()
-
- package_bytes_hash = hashlib.sha256()
- with args.package.open("rb") as package_file:
- for block in iter(lambda: package_file.read(1024 * 1024), b""):
- package_bytes_hash.update(block)
- if package_bytes_hash.hexdigest() != PACKAGE_SHA256:
- raise SystemExit("package SHA-256 does not match the pinned nvidia-utils artifact")
-
- pkginfo = extract_member(args.package, ".PKGINFO").decode("utf-8", errors="replace")
- if f"pkgver = {PACKAGE_VERSION}" not in pkginfo.splitlines():
- raise SystemExit("package metadata version does not match 610.43.03-1")
-
- gsp = extract_member(args.package, FIRMWARE_PREFIX + "gsp_ga10x.bin")
- if sha256(gsp) != GSP_SHA256:
- raise SystemExit("GA10x GSP firmware SHA-256 does not match the pinned artifact")
-
- payload = build_payload()
- if sha256(payload) != PAYLOAD_SHA256:
- raise SystemExit("generated V67 image differs from its pinned SHA-256")
- manifest = json.loads(MANIFEST.read_text())
- notice = manifest["source_artifacts"][2]["notice"]
- license_text = extract_member(args.package, notice["path"])
- if sha256(license_text) != notice["sha256"]:
- raise SystemExit("NVIDIA license notice differs from pinned package")
- args.output.mkdir(parents=True, exist_ok=True)
- (args.output / "gsp_ga10x.bin").write_bytes(gsp)
- (args.output / "v67-ga102-610.43.03.bin").write_bytes(payload)
- (args.output / "NVIDIA-LICENSE").write_bytes(license_text)
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/kernel/gpu_unlock/Kbuild b/kernel/gpu_unlock/Kbuild
index 1687c9f..c0ce572 100644
--- a/kernel/gpu_unlock/Kbuild
+++ b/kernel/gpu_unlock/Kbuild
@@ -1,4 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-m += gpu_unlock.o
gpu_unlock-y := main.o gfw_eval.o prep_610.o layout_610.o \
- vbios_610.o unlock_610.o
+ vbios_610.o recipes.o payload.o firmware_610.o falcon.o unlock_610.o
diff --git a/kernel/gpu_unlock/device.h b/kernel/gpu_unlock/device.h
new file mode 100644
index 0000000..36186b8
--- /dev/null
+++ b/kernel/gpu_unlock/device.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_DEVICE_H
+#define GPU_UNLOCK_DEVICE_H
+
+struct pci_dev;
+struct gu_card;
+
+/* Resolved once during enumeration. The coordinator owns the PCI reference.
+ * Firmware files and launch procedure come from recipe->family. Round-scoped
+ * firmware assets are never stored here. */
+struct gu_device {
+ struct pci_dev *pdev;
+ const struct gu_card *recipe;
+};
+
+#endif
diff --git a/kernel/gpu_unlock/diagnostic.h b/kernel/gpu_unlock/diagnostic.h
index 57aa7de..c1db4a9 100644
--- a/kernel/gpu_unlock/diagnostic.h
+++ b/kernel/gpu_unlock/diagnostic.h
@@ -3,12 +3,21 @@
#define GPU_UNLOCK_DIAGNOSTIC_H
#ifdef __KERNEL__
#include <linux/printk.h>
-#define GU_NOTE(fmt, ...) pr_info("gpu_unlock: " fmt, ##__VA_ARGS__)
-#define GU_ERROR(fmt, ...) pr_err("gpu_unlock: " fmt, ##__VA_ARGS__)
+#define GU_NOTE(fmt, ...) pr_debug("gpu_unlock: " fmt, ##__VA_ARGS__)
#else
#include <stdio.h>
-#define GU_NOTE(fmt, ...) fprintf(stderr, "gpu_unlock: " fmt, ##__VA_ARGS__)
-#define GU_ERROR(fmt, ...) fprintf(stderr, "gpu_unlock: " fmt, ##__VA_ARGS__)
+#include <stdlib.h>
+#include <string.h>
+/* Userspace builds are the integration harness; keep diagnostics opt-in so
+ * the acceptance output is readable. The kernel build is unchanged. */
+static inline int gu_diag_enabled(void)
+{
+ const char *on = getenv("GPU_UNLOCK_DIAG");
+ return on && *on && strcmp(on, "0");
+}
+#define GU_NOTE(fmt, ...) do { \
+ if (gu_diag_enabled()) fprintf(stderr, "gpu_unlock: " fmt, ##__VA_ARGS__); \
+} while (0)
#endif
/* No extra device reads. Preserve the errno and identify the exact guard in
@@ -17,7 +26,7 @@ static inline int gu_diagnostic_result(int error, const char *file,
const char *function, unsigned int line)
{
if (error)
- GU_ERROR("failure %s:%u %s error=%d\n", file, line, function, error);
+ GU_NOTE("failure %s:%u %s error=%d\n", file, line, function, error);
return error;
}
#define GU_DIAG(result) gu_diagnostic_result((result), __FILE__, __func__, __LINE__)
diff --git a/kernel/gpu_unlock/dkms.conf b/kernel/gpu_unlock/dkms.conf
index 8ac6e9a..817b1fe 100644
--- a/kernel/gpu_unlock/dkms.conf
+++ b/kernel/gpu_unlock/dkms.conf
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
PACKAGE_NAME="gpu-unlock"
-PACKAGE_VERSION="1.0.0"
+PACKAGE_VERSION="1.1.0"
BUILT_MODULE_NAME[0]="gpu_unlock"
DEST_MODULE_LOCATION[0]="/updates/dkms"
AUTOINSTALL="yes"
diff --git a/kernel/gpu_unlock/falcon.c b/kernel/gpu_unlock/falcon.c
new file mode 100644
index 0000000..6abf17d
--- /dev/null
+++ b/kernel/gpu_unlock/falcon.c
@@ -0,0 +1,408 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Falcon register transport. Firmware policy and ownership belong to the caller. */
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/device.h>
+#include <linux/string.h>
+#include <linux/timekeeping.h>
+
+#include "falcon.h"
+#include "prep_610.h"
+#include "registers.h"
+
+static int falcon_read(const struct gu_falcon_io *io, u32 offset, u32 *value)
+{
+ return io->read(io->context, offset, value);
+}
+static void falcon_write(const struct gu_falcon_io *io, u32 offset, u32 value)
+{
+ io->write(io->context, offset, value);
+}
+static bool inaccessible(u32 value)
+{
+ return value == ~0u || (value & 0xffff0000u) == 0xbadf0000u;
+}
+
+int gu_falcon_poll(const struct gu_falcon_io *io, u32 offset, u32 mask, u32 expect,
+ unsigned int budget_us, bool timeout_is_failure, u32 *last)
+{
+ ktime_t deadline = ktime_add_us(ktime_get(), budget_us);
+ u32 value = 0;
+
+ for (;;) {
+ int ret = falcon_read(io, offset, &value);
+
+ if (last)
+ *last = value;
+ if (ret || value == ~0u ||
+ (value & 0xffff0000u) == 0xbadf0000u) {
+ dev_err(io->dev, "poll inaccessible: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return -EIO;
+ }
+ if ((value & mask) == expect)
+ return 0;
+ if (ktime_compare(ktime_get(), deadline) >= 0) {
+ /* Console loglevel hides info. A tolerated timeout must not
+ * look like the failure that stopped the boot. */
+ if (!timeout_is_failure) {
+ dev_dbg(io->dev, "poll timeout tolerated: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return -ETIMEDOUT;
+ }
+ dev_err(io->dev, "poll timeout: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return -ETIMEDOUT;
+ }
+ usleep_range(50, 100);
+ }
+}
+
+int gu_falcon_engine_reset(const struct gu_falcon_io *io, u32 base,
+ const char *phase, u32 *hwcfg_out,
+ struct gu_falcon_evidence *evidence)
+{
+ unsigned int j;
+ u32 engine, before, first = 0, hwcfg, last = 0;
+ int ret;
+
+ memset(evidence, 0, sizeof(*evidence));
+ if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
+ return -ENODEV;
+ before = engine;
+ dev_dbg(io->dev, "engine reset assert: phase=%s base=%#x before=%#x write=%#x\n",
+ phase, base, before, before | 1);
+ falcon_write(io, base + GU_FALCON_ENGINE_OFFSET, engine | 1);
+ /* 610 ksec2ResetHw_TU102 requires ten register cycles for signal
+ * propagation. A stale zero during those cycles is not evidence that
+ * the assertion failed. No CPU start and no reset retry.
+ */
+ for (j = 0; j < 10; j++) {
+ if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) ||
+ inaccessible(engine)) {
+ dev_err(io->dev, "engine reset inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
+ phase, base, j + 1, engine);
+ return -EIO;
+ }
+ if (!j)
+ first = engine;
+ }
+ dev_dbg(io->dev, "engine reset propagation: phase=%s base=%#x first=%#x tenth=%#x\n",
+ phase, base, first, engine);
+ ret = gu_falcon_poll(io, base + GU_FALCON_ENGINE_OFFSET, 1, 1, 1000, true, &last);
+ dev_dbg(io->dev, "engine reset asserted: phase=%s base=%#x error=%d last=%#x\n",
+ phase, base, ret, last);
+ if (ret)
+ return ret;
+ evidence->reset_asserted = true;
+ if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
+ return -EIO;
+ dev_dbg(io->dev, "engine reset release: phase=%s base=%#x before=%#x write=%#x\n",
+ phase, base, engine, engine & ~1u);
+ falcon_write(io, base + GU_FALCON_ENGINE_OFFSET, engine & ~1u);
+ for (j = 0; j < 10; j++) {
+ if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) ||
+ inaccessible(engine)) {
+ dev_err(io->dev, "engine reset release inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
+ phase, base, j + 1, engine);
+ return -EIO;
+ }
+ }
+ ret = gu_falcon_poll(io, base + GU_FALCON_ENGINE_OFFSET, 1, 0, 1000, true, NULL);
+ if (ret)
+ return ret;
+ evidence->reset_released = true;
+ /* FALCON_HWCFG2 bit 12 clears when memory scrubbing finishes.
+ * A PRI error fails this wait instead of being handed to RM.
+ */
+ ret = gu_falcon_poll(io, base + GU_FALCON_HWCFG2_OFFSET, 1u << 12, 0, 1000000, true, NULL);
+ if (ret)
+ return ret;
+ if (falcon_read(io, base + GU_FALCON_HWCFG2_OFFSET, &hwcfg) || inaccessible(hwcfg))
+ return -EIO;
+ dev_dbg(io->dev, "engine reset released: phase=%s base=%#x engine=%#x hwcfg2=%#x\n",
+ phase, base, engine, hwcfg);
+ if (hwcfg_out)
+ *hwcfg_out = hwcfg;
+ return 0;
+}
+
+int gu_falcon_reset(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ struct gu_falcon_evidence *evidence)
+{
+ u32 hwcfg, bcr, chip;
+ int ret;
+
+ memset(evidence, 0, sizeof(*evidence));
+ /* 610 kflcnPreResetWait_GA102 permits this timeout (HW bug 3419321). */
+ dev_dbg(io->dev, "Falcon reset begin: base=%#x\n", eng->base);
+ ret = gu_falcon_poll(io, eng->base + GU_FALCON_HWCFG2_OFFSET,
+ 1u << 31, 1u << 31, 1000, false, NULL);
+ if (ret && ret != -ETIMEDOUT)
+ return ret;
+ if (ret)
+ dev_dbg(io->dev, "Falcon pre-reset timeout tolerated per HW bug 3419321\n");
+ ret = gu_falcon_engine_reset(io, eng->base, "pre-launch", &hwcfg, evidence);
+ if (ret)
+ return ret;
+ if (hwcfg & (1u << 10)) {
+ if (falcon_read(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET, &bcr) || inaccessible(bcr))
+ return -ENODEV;
+ if (bcr & (1u << 4)) {
+ falcon_write(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET, 0);
+ ret = gu_falcon_poll(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET,
+ 0x11, 1, 1000000, true, NULL);
+ if (ret)
+ return ret;
+ }
+ }
+ if (falcon_read(io, GU_PMC_BOOT_0_OFFSET, &chip) || inaccessible(chip))
+ return -ENODEV;
+ falcon_write(io, eng->base + GU_FALCON_RM_OFFSET, chip);
+ dev_dbg(io->dev, "Falcon reset done: base=%#x hwcfg=%#x chip=%#x\n", eng->base, hwcfg, chip);
+ return 0;
+}
+
+/* DMATRFMOFFS is 24 bits, FBOFFS is 32 bits, and BASE is 49 bits.
+ * Reject truncation/wrap before submitting any part of a transfer. */
+static bool falcon_dma_valid(u32 dest, u32 mem_off, dma_addr_t src, u32 size)
+{
+ u64 rounded = ((u64)size + 255) & ~255ull;
+ return size && !((dest | mem_off | src) & 255) &&
+ (u64)dest + rounded <= (1ull << 24) &&
+ (u64)mem_off + rounded <= (1ull << 32) &&
+ src < (1ull << 49) &&
+ (u64)mem_off + rounded <= (1ull << 49) - src;
+}
+
+int gu_falcon_dma(const struct gu_falcon_io *io, u32 base, u32 dest, u32 mem_off,
+ dma_addr_t src, u32 size, u32 cmd)
+{
+ u32 done = 0;
+ int ret;
+
+ dev_dbg(io->dev, "Falcon DMA: base=%#x dest=%#x offset=%#x src=%#llx size=%#x cmd=%#x\n",
+ base, dest, mem_off, (unsigned long long)src, size, cmd);
+ if (!falcon_dma_valid(dest, mem_off, src, size))
+ return -EINVAL;
+ ret = gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 1, 0, 1000000, true, NULL);
+ if (ret)
+ return ret;
+ falcon_write(io, base + GU_FALCON_DMATRFBASE_OFFSET, lower_32_bits(src >> 8));
+ falcon_write(io, base + GU_FALCON_DMATRFBASE1_OFFSET, upper_32_bits(src >> 8) & 0x1ff);
+ while (done < size) {
+ ret = gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 1, 0, 1000000, true, NULL);
+ if (ret) {
+ dev_err(io->dev, "Falcon DMA progress: done=%u total=%u\n", done, size);
+ return ret;
+ }
+ falcon_write(io, base + GU_FALCON_DMATRFMOFFS_OFFSET, dest);
+ falcon_write(io, base + GU_FALCON_DMATRFFBOFFS_OFFSET, mem_off);
+ falcon_write(io, base + GU_FALCON_DMATRFCMD_OFFSET, cmd);
+ done += 256;
+ dest += 256;
+ mem_off += 256;
+ }
+ return gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 2, 2, 1000000, true, NULL);
+}
+
+int gu_falcon_hs(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ const struct gu_falcon_hs_image *program,
+ struct gu_falcon_evidence *evidence)
+{
+ u32 trans, cpuctl;
+ dma_addr_t code_src;
+ int ret;
+
+ memset(evidence, 0, sizeof(*evidence));
+ if (program->dma > ~0ull - program->code_off ||
+ program->dma + program->code_off < program->code_va ||
+ program->dma > ~0ull - program->data_off)
+ return -ERANGE;
+ code_src = program->dma + program->code_off - program->code_va;
+ if (!falcon_dma_valid(program->code_pa, program->code_va, code_src, program->code_size) ||
+ (program->data_size && !falcon_dma_valid(program->data_pa, 0,
+ program->dma + program->data_off, program->data_size)))
+ return -EINVAL;
+ ret = gu_falcon_reset(io, eng, evidence);
+ if (ret)
+ return ret;
+ ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 0, 1000,
+ !program->stale_halt_ok, NULL);
+ if (ret && !(ret == -ETIMEDOUT && program->stale_halt_ok))
+ return ret;
+ if (falcon_read(io, eng->fbif + GU_FBIF_CTL_OFFSET, &trans) || inaccessible(trans))
+ return -EIO;
+ trans |= 1u << 7;
+ falcon_write(io, eng->fbif + GU_FBIF_CTL_OFFSET, trans);
+ falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, 0);
+ if (falcon_read(io, eng->fbif + GU_FBIF_TRANSCFG(0), &trans) || inaccessible(trans))
+ return -ENODEV;
+ trans = (trans & ~7u) | 5u;
+ falcon_write(io, eng->fbif + GU_FBIF_TRANSCFG(0), trans);
+ dma_wmb();
+ if (program->os_code_size) {
+ ret = gu_falcon_dma(io, eng->base, 0, 0,
+ program->dma + program->os_code_off, program->os_code_size,
+ gpu_unlock_dma_cmd(true, false));
+ if (ret)
+ return ret;
+ }
+ ret = gu_falcon_dma(io, eng->base, program->code_pa, program->code_va, code_src,
+ program->code_size, gpu_unlock_dma_cmd(true, true));
+ if (ret)
+ return ret;
+ if (program->data_size) {
+ ret = gu_falcon_dma(io, eng->base, program->data_pa, 0,
+ program->dma + program->data_off, program->data_size,
+ gpu_unlock_dma_cmd(false, false));
+ if (ret)
+ return ret;
+ }
+ evidence->reloaded = true;
+ falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_PARAADDR_OFFSET, program->sig_dmem);
+ falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_ENGIDMASK_OFFSET, program->engine_mask);
+ falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_CURR_UCODE_ID_OFFSET, program->ucode_id & 0xff);
+ falcon_write(io, eng->falcon2 + GU_FALCON2_MOD_SEL_OFFSET, 1);
+ falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, program->code_va);
+ falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, program->mailbox0);
+ falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, program->mailbox1);
+ if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &cpuctl) || inaccessible(cpuctl))
+ return -EIO;
+ dev_dbg(io->dev, "Falcon start: base=%#x cpuctl=%#x code_va=%#x sig_dmem=%#x engine_mask=%#x ucode=%u mailbox=%#x:%#x\n",
+ eng->base, cpuctl, program->code_va, program->sig_dmem, program->engine_mask,
+ program->ucode_id, program->mailbox1, program->mailbox0);
+ evidence->started = true;
+ if (cpuctl & (1u << 6))
+ falcon_write(io, eng->base + GU_FALCON_CPUCTL_ALIAS_OFFSET, 1u << 1);
+ else
+ falcon_write(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 1u << 1);
+
+ ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET,
+ 1u << 4, 1u << 4, 5000000, true, NULL);
+ if (ret)
+ return ret;
+ evidence->halted = true;
+ return 0;
+}
+
+/* TU102/GA100 generic loader for board FWSEC v2. Descriptor offsets are
+ * RM_FLCN_BL_DMEM_DESC (84 bytes); loader identity/geometry is pinned by the
+ * extractor. The loader uses DMA index 4, coherent physical system memory.
+ */
+int gu_falcon_loader(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ const struct gu_falcon_loader_image *program,
+ struct gu_falcon_evidence *evidence)
+{
+ u32 desc[21] = { 0 }, hwcfg, trans, ctl, i, dst;
+ const u32 *code;
+ int ret;
+
+ memset(evidence, 0, sizeof(*evidence));
+ if (!program->loader || program->loader_size != 768 ||
+ (program->dma & 255) || program->dma > ~0ull - program->data_off ||
+ (program->data_off & 255))
+ return -EINVAL;
+ code = program->loader;
+ ret = gu_falcon_reset(io, eng, evidence);
+ if (ret)
+ return ret;
+ if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl) || (ctl & 16) ||
+ falcon_read(io, eng->base + GU_FALCON_HWCFG_OFFSET, &hwcfg) || inaccessible(hwcfg) ||
+ falcon_read(io, eng->base + GU_FALCON_DMACTL_OFFSET, &trans) || inaccessible(trans))
+ return -ENODEV;
+ if ((hwcfg & 0x1ff) < 2)
+ return -ERANGE;
+ dst = ((hwcfg & 0x1ff) - 2) << 8;
+ falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, trans & ~1u);
+ if (falcon_read(io, eng->fbif + GU_FBIF_TRANSCFG(4), &trans) || inaccessible(trans))
+ return -ENODEV;
+ falcon_write(io, eng->fbif + GU_FBIF_TRANSCFG(4), (trans & ~7u) | 5u);
+ desc[8] = 4;
+ desc[9] = lower_32_bits(program->dma);
+ desc[10] = upper_32_bits(program->dma);
+ desc[11] = program->imem_pa;
+ desc[12] = program->imem_ns_size;
+ desc[13] = program->imem_sec_pa;
+ desc[14] = program->imem_sec_size;
+ desc[16] = lower_32_bits(program->dma + program->data_off);
+ desc[17] = upper_32_bits(program->dma + program->data_off);
+ desc[18] = program->dmem_size;
+ falcon_write(io, eng->base + GU_FALCON_DMEMC_OFFSET, 1u << 24);
+ for (i = 0; i < 21; i++)
+ falcon_write(io, eng->base + GU_FALCON_DMEMD_OFFSET, desc[i]);
+ falcon_write(io, eng->base + GU_FALCON_IMEMC_OFFSET, dst | (1u << 24));
+ for (i = 0; i < 512; i += 4) {
+ if (!(i & 255))
+ falcon_write(io, eng->base + GU_FALCON_IMEMT_OFFSET, 0xfd + (i >> 8));
+ falcon_write(io, eng->base + GU_FALCON_IMEMD_OFFSET, code[i / 4]);
+ }
+ evidence->reloaded = true;
+ falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, 0xfd00);
+ falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, 0);
+ falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, 0);
+ dma_wmb();
+ evidence->started = true;
+ falcon_write(io, eng->base + ((ctl & 64) ? GU_FALCON_CPUCTL_ALIAS_OFFSET :
+ GU_FALCON_CPUCTL_OFFSET), 2);
+ ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 16, 5000000, true, NULL);
+ evidence->halted = !ret;
+ return ret;
+}
+
+/* GA100's direct loader: nonsecure OS stub, tagged secure app, DMEM.
+ * GA10x uses the HS DMA/BROM loader with the same signed GA100 image. Its
+ * cross-family acceptance is an explicit hardware gate, not an assumption.
+ */
+static void falcon_pio_imem(const struct gu_falcon_io *io, u32 base, const u32 *image,
+ u32 offset, u32 size, bool secure)
+{
+ u32 i;
+
+ falcon_write(io, base + GU_FALCON_IMEMC_OFFSET, offset | (1u << 24) |
+ (secure ? 1u << 28 : 0));
+ for (i = 0; i < size; i += 4) {
+ if (!(i & 255))
+ falcon_write(io, base + GU_FALCON_IMEMT_OFFSET, (offset + i) >> 8);
+ falcon_write(io, base + GU_FALCON_IMEMD_OFFSET, image[(offset + i) / 4]);
+ }
+}
+
+int gu_falcon_direct(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ const struct gu_falcon_direct_image *program,
+ struct gu_falcon_evidence *evidence)
+{
+ const u32 *image = program->words;
+ u32 i, ctl;
+ int ret;
+
+ memset(evidence, 0, sizeof(*evidence));
+ ret = gu_falcon_reset(io, eng, evidence);
+ if (ret)
+ return ret;
+ ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 0, 1000, true, NULL);
+ if (ret)
+ return ret;
+ /* REQUIRE_CTX=false, matching kflcnDisableCtxReq. */
+ if (falcon_read(io, eng->base + GU_FALCON_DMACTL_OFFSET, &ctl) || inaccessible(ctl))
+ return -ENODEV;
+ falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, ctl & ~1u);
+ falcon_pio_imem(io, eng->base, image, program->os_code_off, program->os_code_size, false);
+ falcon_pio_imem(io, eng->base, image, program->app_code_off, program->app_code_size, true);
+ falcon_write(io, eng->base + GU_FALCON_DMEMC_OFFSET, 1u << 24);
+ for (i = 0; i < program->os_data_size; i += 4)
+ falcon_write(io, eng->base + GU_FALCON_DMEMD_OFFSET, image[(program->os_data_off + i) / 4]);
+ evidence->reloaded = true;
+ falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, 0);
+ falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, program->mailbox0);
+ falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, program->mailbox1);
+ if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl))
+ return -ENODEV;
+ dma_wmb();
+ evidence->started = true;
+ falcon_write(io, eng->base + ((ctl & 64) ? GU_FALCON_CPUCTL_ALIAS_OFFSET :
+ GU_FALCON_CPUCTL_OFFSET), 2);
+ ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 16, 5000000, true, NULL);
+ evidence->halted = !ret;
+ return ret;
+}
diff --git a/kernel/gpu_unlock/falcon.h b/kernel/gpu_unlock/falcon.h
new file mode 100644
index 0000000..b079edc
--- /dev/null
+++ b/kernel/gpu_unlock/falcon.h
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_FALCON_H
+#define GPU_UNLOCK_FALCON_H
+
+#include <linux/types.h>
+
+struct device;
+
+/* Borrowed hardware access only: no PCI, allocation or transaction ownership. */
+struct gu_falcon_io {
+ struct device *dev;
+ void *context;
+ int (*read)(void *context, u32 offset, u32 *value);
+ void (*write)(void *context, u32 offset, u32 value);
+};
+struct gu_falcon_engine { u32 base, fbif, falcon2; };
+
+/* Reset for every operation, including validation failures. Available on error.
+ * started records a submitted start; halted records a successful halt wait.
+ * Neither proves firmware-specific success, cleanup or safe DMA disposal. */
+struct gu_falcon_evidence {
+ bool reset_asserted, reset_released, reloaded, started, halted;
+};
+
+/* Image spans/firmware geometry are validated by preparation before calling.
+ * Transport additionally rejects unrepresentable HS/DMA addresses before reset. */
+struct gu_falcon_hs_image {
+ dma_addr_t dma;
+ u32 code_off, code_size, code_va, code_pa;
+ /* Non-secure OS stub loaded at IMEM 0, used by the Booter bundle. */
+ u32 os_code_off, os_code_size;
+ u32 data_off, data_size, data_pa;
+ u32 sig_dmem, engine_mask, ucode_id;
+ u32 mailbox0, mailbox1;
+ /* The caller verifies the result independently, so HALTED left set
+ * across reset is tolerated, as RM does. GA102 GSP and SEC2 keep it. */
+ bool stale_halt_ok;
+};
+struct gu_falcon_direct_image {
+ const u32 *words;
+ u32 os_code_off, os_code_size, app_code_off, app_code_size;
+ u32 os_data_off, os_data_size;
+ u32 mailbox0, mailbox1;
+};
+struct gu_falcon_loader_image {
+ const u32 *loader;
+ size_t loader_size;
+ dma_addr_t dma;
+ u32 data_off, imem_pa, imem_ns_size, imem_sec_pa, imem_sec_size, dmem_size;
+};
+
+int gu_falcon_poll(const struct gu_falcon_io *io, u32 offset, u32 mask, u32 expect,
+ unsigned int budget_us, bool timeout_is_failure, u32 *last);
+int gu_falcon_dma(const struct gu_falcon_io *io, u32 base, u32 dest, u32 mem_off,
+ dma_addr_t src, u32 size, u32 cmd);
+int gu_falcon_engine_reset(const struct gu_falcon_io *io, u32 base, const char *phase,
+ u32 *hwcfg, struct gu_falcon_evidence *evidence);
+int gu_falcon_reset(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ struct gu_falcon_evidence *evidence);
+int gu_falcon_hs(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ const struct gu_falcon_hs_image *image, struct gu_falcon_evidence *evidence);
+int gu_falcon_direct(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ const struct gu_falcon_direct_image *image, struct gu_falcon_evidence *evidence);
+int gu_falcon_loader(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
+ const struct gu_falcon_loader_image *image, struct gu_falcon_evidence *evidence);
+
+#endif
diff --git a/kernel/gpu_unlock/firmware_610.c b/kernel/gpu_unlock/firmware_610.c
new file mode 100644
index 0000000..d723aaa
--- /dev/null
+++ b/kernel/gpu_unlock/firmware_610.c
@@ -0,0 +1,260 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Firmware format validation and immutable assets for one coordination round. */
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+
+#include "diagnostic.h"
+#include "firmware_610.h"
+#include "recipes.h"
+
+#define BOOTER_MAGIC 0x314c5442u
+
+/* CONFIG_FW_CACHE attaches name records to devres even though this is not
+ * devm_request_firmware(). release_firmware() does not remove them, and
+ * really_probe() refuses a device with any existing devres. Scope each
+ * synchronous request so only its cache records are released; the returned
+ * firmware remains owned by the caller.
+ */
+static int request_scoped(struct device *dev, const struct firmware **fw, const char *name)
+{
+ void *group;
+ int ret, released;
+
+ group = devres_open_group(dev, NULL, GFP_KERNEL);
+ if (!group) {
+ dev_err(dev, "asset request failed: %s error=%d\n", name, -ENOMEM);
+ return -ENOMEM;
+ }
+ dev_dbg(dev, "asset request: %s\n", name);
+ ret = request_firmware(fw, name, dev);
+ released = devres_release_group(dev, group);
+ if (ret)
+ dev_err(dev, "asset request failed: %s error=%d\n", name, ret);
+ dev_dbg(dev, "asset result: %s error=%d bytes=%zu\n",
+ name, ret, ret ? (size_t)0 : (*fw)->size);
+ dev_dbg(dev, "asset cache cleanup: %s devres_released=%d\n", name, released);
+ return ret;
+}
+
+static int validate_booter(const struct firmware *fw, struct gpu_unlock_booter_hdr *out,
+ const struct gu_booter_expect *expect)
+{
+ const u8 *data;
+ u32 magic, version, image_size, sig_bytes;
+ struct gpu_unlock_booter_hdr hdr;
+
+ data = fw->data;
+ if (fw->size < 72)
+ return GU_DIAG(-EINVAL);
+ memcpy(&magic, data, sizeof(magic));
+ memcpy(&version, data + 4, sizeof(version));
+ if (magic != BOOTER_MAGIC || version != 1)
+ return GU_DIAG(-EINVAL);
+ memcpy(&hdr, data + 8, sizeof(hdr));
+ memcpy(&image_size, data + 8 + sizeof(hdr), sizeof(image_size));
+ memcpy(&sig_bytes, data + 12 + sizeof(hdr), sizeof(sig_bytes));
+ if (!hdr.ucode_id || hdr.num_apps != 1 || sig_bytes != hdr.sig_bytes ||
+ image_size != expect->image_size ||
+ fw->size != 72u + image_size + sig_bytes)
+ return GU_DIAG(-EINVAL);
+ /* Keep the signed GA100 and AD102 production geometries distinct. */
+ if (hdr.ucode_id != 3 || hdr.engine_id != 1 || hdr.num_sigs != expect->num_sigs ||
+ sig_bytes != expect->sig_bytes || hdr.os_code_off != 0 ||
+ hdr.os_code_size != 256 || hdr.os_data_off != expect->os_data_off ||
+ hdr.os_data_size != expect->os_data_size || hdr.app_code_off != 256 ||
+ hdr.app_code_size != expect->app_code_size || hdr.app_data_off != 256 ||
+ hdr.app_data_size != 0 || hdr.patch_loc != expect->patch_loc)
+ return GU_DIAG(-EINVAL);
+ *out = hdr;
+ return 0;
+}
+
+static void release_bundle(struct gpu_unlock_610_firmware *assets)
+{
+ release_firmware(assets->gsp);
+ release_firmware(assets->boot);
+ memset(assets, 0, sizeof(*assets));
+}
+
+static int prepare_family(struct pci_dev *pdev, struct gpu_unlock_610_firmware *assets,
+ const struct gu_fw_desc *load)
+{
+ const struct firmware *desc = NULL;
+ int ret;
+
+ ret = request_scoped(&pdev->dev, &assets->gsp,
+ load->gsp);
+ if (ret)
+ return ret;
+ ret = gpu_unlock_elf_section(assets->gsp->data, assets->gsp->size,
+ ".fwimage", &assets->image_span);
+ if (ret)
+ return ret;
+ ret = gpu_unlock_elf_section(assets->gsp->data, assets->gsp->size,
+ load->signature_section, &assets->sig_span);
+ if (ret)
+ return ret;
+ if (!assets->image_span.size || assets->image_span.size > 128ull << 20 ||
+ assets->sig_span.size != 4096)
+ return -EINVAL;
+ ret = gpu_unlock_radix_plan(assets->image_span.size, &assets->radix);
+ if (ret)
+ return ret;
+ ret = request_scoped(&pdev->dev, &assets->boot, load->boot);
+ if (ret)
+ return ret;
+ if (assets->boot->size != load->boot_size)
+ return -EINVAL;
+ ret = request_scoped(&pdev->dev, &desc, load->desc);
+ if (ret)
+ return ret;
+ ret = gpu_unlock_boot_desc_parse(desc->data, desc->size,
+ assets->boot->size, &assets->boot_desc);
+ dev_dbg(&pdev->dev, "asset geometry: image=%llu signature=%llu boot=%u code=%#x data=%#x manifest=%#x parse=%d\n",
+ (unsigned long long)assets->image_span.size, (unsigned long long)assets->sig_span.size,
+ assets->boot_desc.image_size, assets->boot_desc.code_offset,
+ assets->boot_desc.data_offset, assets->boot_desc.manifest_offset, ret);
+ release_firmware(desc);
+ return ret;
+}
+
+static struct gpu_unlock_610_firmware *
+claim_bundle(struct gpu_unlock_610_shared *shared, const struct gu_fw_desc *fw)
+{
+ struct gpu_unlock_610_firmware *slot;
+ unsigned int i;
+
+ slot = (struct gpu_unlock_610_firmware *)gpu_unlock_610_bundle_find(shared, fw);
+ if (slot)
+ return slot;
+ for (i = 0; i < GU_SHARED_BUNDLES; i++) {
+ if (!shared->bundle[i].key) {
+ shared->bundle[i].key = fw;
+ return &shared->bundle[i];
+ }
+ }
+ return NULL;
+}
+
+static int ensure_booter(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared,
+ const struct gu_booter *desc)
+{
+ struct gpu_unlock_610_booter *slot = NULL;
+ const struct firmware *fw = NULL;
+ struct gpu_unlock_booter_hdr hdr;
+ unsigned int i;
+ int ret;
+
+ if (gpu_unlock_610_booter_find(shared, desc))
+ return 0;
+ for (i = 0; i < GU_SHARED_BOOTERS; i++) {
+ if (!shared->booter[i].key) {
+ slot = &shared->booter[i];
+ break;
+ }
+ }
+ if (!slot)
+ return -ENOSPC;
+ ret = request_scoped(&pdev->dev, &fw, desc->file);
+ if (ret)
+ return ret;
+ ret = validate_booter(fw, &hdr, &desc->expect);
+ if (ret) {
+ dev_err(&pdev->dev, "invalid SEC2 Booter bundle: error=%d\n", ret);
+ release_firmware(fw);
+ return ret;
+ }
+ slot->key = desc;
+ slot->fw = fw;
+ slot->hdr = hdr;
+ return 0;
+}
+
+static int ensure_loader(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared,
+ const char *name)
+{
+ const struct firmware *loader = NULL;
+ int ret;
+
+ if (!name)
+ return 0;
+ if (shared->loader)
+ return shared->loader_name == name ? 0 : -EINVAL;
+ ret = request_scoped(&pdev->dev, &loader, name);
+ if (ret)
+ return ret;
+ if (loader->size != 768) {
+ dev_err(&pdev->dev, "invalid generic Falcon loader: bytes=%zu\n", loader->size);
+ release_firmware(loader);
+ return -EINVAL;
+ }
+ shared->loader = loader;
+ shared->loader_name = name;
+ return 0;
+}
+
+int gpu_unlock_610_load_shared(const struct gu_device *device, struct gpu_unlock_610_shared *shared)
+{
+ struct pci_dev *pdev;
+ const struct gu_family *family;
+ const struct gu_fw_desc *load;
+ struct gpu_unlock_610_firmware *assets;
+ int ret;
+
+ if (!device->recipe || !device->recipe->family || !device->recipe->family->fw ||
+ !device->recipe->family->booter)
+ return -ENODEV;
+ pdev = device->pdev;
+ family = device->recipe->family;
+ /* The payload belongs to the Booter image; refuse a pairing the runner
+ * could not use instead of tearing it apart per card later. */
+ if (!family->booter->payload || gu_payload_check(family->booter->payload))
+ return -EINVAL;
+ load = family->fw;
+ assets = claim_bundle(shared, load);
+ if (!assets)
+ return -ENOSPC;
+ if (assets->prepared)
+ return assets->error;
+ ret = prepare_family(pdev, assets, load);
+ if (ret)
+ goto out;
+ ret = ensure_booter(pdev, shared, family->booter);
+ if (ret)
+ goto out;
+ ret = ensure_loader(pdev, shared, family->loader);
+out:
+ if (ret)
+ release_bundle(assets);
+ assets->key = load;
+ assets->error = ret;
+ assets->prepared = true;
+ return ret;
+}
+
+void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
+{
+ unsigned int i;
+
+ for (i = 0; i < GU_SHARED_BUNDLES; i++)
+ release_bundle(&shared->bundle[i]);
+ for (i = 0; i < GU_SHARED_BOOTERS; i++)
+ release_firmware(shared->booter[i].fw);
+ release_firmware(shared->loader);
+ memset(shared, 0, sizeof(*shared));
+}
+
+MODULE_FIRMWARE(GU_FW_BOOTER_GA100);
+MODULE_FIRMWARE(GU_FW_BOOTER_GA102);
+MODULE_FIRMWARE(GU_FW_LOADER);
+MODULE_FIRMWARE(GU_FW_GSP_GA10X);
+MODULE_FIRMWARE(GU_FW_BOOT_GA102);
+MODULE_FIRMWARE(GU_FW_DESC_GA102);
+MODULE_FIRMWARE(GU_FW_GSP_GA100);
+MODULE_FIRMWARE(GU_FW_BOOT_GA100);
+MODULE_FIRMWARE(GU_FW_DESC_GA100);
+MODULE_FIRMWARE(GU_FW_BOOTER_AD102);
+MODULE_FIRMWARE(GU_FW_BOOT_AD102);
+MODULE_FIRMWARE(GU_FW_DESC_AD102);
diff --git a/kernel/gpu_unlock/firmware_610.h b/kernel/gpu_unlock/firmware_610.h
new file mode 100644
index 0000000..6532333
--- /dev/null
+++ b/kernel/gpu_unlock/firmware_610.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_FIRMWARE_610_H
+#define GPU_UNLOCK_FIRMWARE_610_H
+
+#include "prep_610.h"
+#include "device.h"
+#include "recipes.h"
+
+struct firmware;
+
+/* Immutable after preparation, including decoded metadata. Owns gsp and boot.
+ * key is the family file set this slot was claimed for. */
+struct gpu_unlock_610_firmware {
+ const struct gu_fw_desc *key;
+ const struct firmware *gsp, *boot;
+ struct gpu_unlock_span image_span, sig_span;
+ struct gpu_unlock_boot_desc boot_desc;
+ struct gpu_unlock_radix_plan radix;
+ bool prepared;
+ int error;
+};
+
+/* A loaded Booter image. Families that share a gu_booter share one slot. */
+struct gpu_unlock_610_booter {
+ const struct gu_booter *key;
+ const struct firmware *fw;
+ struct gpu_unlock_booter_hdr hdr;
+};
+
+#define GU_SHARED_BUNDLES 3
+#define GU_SHARED_BOOTERS 3
+
+/* One coordination round. Bundles and Booters are interned by descriptor
+ * pointer, so families that share a Booter load it once. */
+struct gpu_unlock_610_shared {
+ struct gpu_unlock_610_firmware bundle[GU_SHARED_BUNDLES];
+ struct gpu_unlock_610_booter booter[GU_SHARED_BOOTERS];
+ const struct firmware *loader;
+ const char *loader_name;
+};
+
+static inline const struct gpu_unlock_610_firmware *
+gpu_unlock_610_bundle_find(const struct gpu_unlock_610_shared *shared,
+ const struct gu_fw_desc *fw)
+{
+ unsigned int i;
+
+ if (!shared || !fw)
+ return NULL;
+ for (i = 0; i < GU_SHARED_BUNDLES; i++)
+ if (shared->bundle[i].key == fw)
+ return &shared->bundle[i];
+ return NULL;
+}
+
+static inline const struct gpu_unlock_610_booter *
+gpu_unlock_610_booter_find(const struct gpu_unlock_610_shared *shared,
+ const struct gu_booter *desc)
+{
+ unsigned int i;
+
+ if (!shared || !desc)
+ return NULL;
+ for (i = 0; i < GU_SHARED_BOOTERS; i++)
+ if (shared->booter[i].key == desc)
+ return &shared->booter[i];
+ return NULL;
+}
+
+/* Serial preparation only; no workers may run until all calls have returned.
+ * Cache failures for this round without invalidating another family's assets.
+ * A fresh round starts after release_shared(), including on resume. */
+int gpu_unlock_610_load_shared(const struct gu_device *device, struct gpu_unlock_610_shared *shared);
+void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared);
+
+#endif
diff --git a/kernel/gpu_unlock/gfw_eval.h b/kernel/gpu_unlock/gfw_eval.h
index f721971..81d853c 100644
--- a/kernel/gpu_unlock/gfw_eval.h
+++ b/kernel/gpu_unlock/gfw_eval.h
@@ -2,6 +2,8 @@
#ifndef GPU_UNLOCK_GFW_EVAL_H
#define GPU_UNLOCK_GFW_EVAL_H
+#include "registers.h"
+
#ifdef __KERNEL__
#include <linux/types.h>
#else
@@ -16,13 +18,7 @@ typedef uint32_t u32;
* dev_gc6_island.h, dev_gc6_island_addendum.h, dev_gsp.h, dev_falcon_v4.h.
* GFW completion uses kflcnWaitForHalt_TU102 (Falcon CPUCTL), not RISC-V CPUCTL.
*/
-#define GU_GFW_PLM_OFFSET 0x00118128u
-#define GU_GFW_BOOT_OFFSET 0x00118234u
-#define GU_GFW_CPUCTL_OFFSET 0x00110100u
#define GU_GFW_TIMEOUT_US 2050000u
-#define GU_FEAT_PLM_OFFSET 0x00823804u
-#define GU_FEAT_SS0_OFFSET 0x0082381cu
-#define GU_FEAT_SS1_OFFSET 0x00823820u
#define GU_GFW_PLM_LEVEL0 0x1u
#define GU_GFW_BOOT_COMPLETED 0xffu
#define GU_GFW_CPUCTL_HALTED 0x10u
diff --git a/kernel/gpu_unlock/main.c b/kernel/gpu_unlock/main.c
index 26208e8..de32b25 100644
--- a/kernel/gpu_unlock/main.c
+++ b/kernel/gpu_unlock/main.c
@@ -9,43 +9,23 @@
#include <linux/suspend.h>
#include "unlock_610.h"
+#include "recipes.h"
static char *targets;
module_param(targets, charp, 0444);
-MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of GA102/GA103/GA104/GA106 cards to unlock");
+MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of supported Ampere cards to unlock");
static bool resume = true;
module_param(resume, bool, 0644);
MODULE_PARM_DESC(resume, "Unlock the cards again after resume from suspend, which powers them off (default: on)");
-/* GA102, GA103, GA104 and GA106 GPU functions in pci.ids, excluding audio
- * functions and the professional (GL) parts, which already run BF16 at full
- * rate. GA103 and GA106 also exclude mobile parts.
- */
-static const unsigned short supported_ids[] = {
- /* GA102 */
- 0x2200, 0x2203, 0x2204, 0x2205, 0x2206, 0x2207, 0x2208, 0x220a,
- 0x220d, 0x2216, 0x222b, 0x222f,
- /* GA104 */
- 0x2480, 0x2482, 0x2483, 0x2484, 0x2486, 0x2487, 0x2488, 0x2489,
- 0x248a, 0x248c, 0x248d, 0x248e, 0x249c, 0x249d, 0x249f, 0x24a0,
- 0x24a4, 0x24ac, 0x24ad, 0x24af, 0x24bf, 0x24c0, 0x24c7, 0x24c8,
- 0x24c9, 0x24dc, 0x24dd, 0x24df, 0x24e0,
- /* GA103 (untested) */
- 0x2414,
- /* GA106 (validated only on RTX 3060 12GB 0x2504) */
- 0x2501, 0x2503, 0x2504, 0x2505, 0x2507, 0x2508, 0x2509, 0x252f,
- 0x2544,
- /* GA106M RTX 3060 Laptop/Max-Q */
- 0x2520, 0x2560,
-};
-
/* A target card. Those unlocked at boot stay listed, with a reference, to be
* unlocked again after each resume. */
struct card {
struct list_head node;
- struct pci_dev *pdev;
+ struct gu_device device;
bool unlocked;
+ bool prepared;
};
static LIST_HEAD(cards);
@@ -53,15 +33,13 @@ static ASYNC_DOMAIN_EXCLUSIVE(unlock_domain);
static struct gpu_unlock_610_shared shared;
/* The next supported NVIDIA card after from, like pci_get_device(). */
-static struct pci_dev *next_supported(struct pci_dev *from)
+static struct pci_dev *next_supported(struct pci_dev *from, const struct gu_card **recipe)
{
struct pci_dev *pdev = from;
- unsigned int i;
while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, pdev)))
- for (i = 0; i < ARRAY_SIZE(supported_ids); i++)
- if (pdev->device == supported_ids[i])
- return pdev;
+ if ((*recipe = gu_card_find(pdev->device)))
+ return pdev;
return NULL;
}
@@ -69,11 +47,11 @@ static struct pci_dev *next_supported(struct pci_dev *from)
static void unlock_one(void *data, async_cookie_t cookie)
{
struct card *card = data;
- struct pci_dev *pdev = card->pdev;
+ struct pci_dev *pdev = card->device.pdev;
int ret;
pci_dev_lock(pdev);
- ret = pdev->driver ? -EBUSY : gpu_unlock_610_unlock(pdev, &shared, false);
+ ret = pdev->driver ? -EBUSY : gpu_unlock_610_unlock(&card->device, &shared, false);
pci_dev_unlock(pdev);
if (ret)
pci_err(pdev, "gpu_unlock: unlock failed (%d)\n", ret);
@@ -87,14 +65,14 @@ static void unlock_one(void *data, async_cookie_t cookie)
static void unlock_again(void *data, async_cookie_t cookie)
{
struct card *card = data;
- struct pci_dev *pdev = card->pdev;
+ struct pci_dev *pdev = card->device.pdev;
bool nvidia;
int ret = 0;
pci_dev_lock(pdev);
nvidia = pdev->driver && !strcmp(pdev->driver->name, "nvidia");
if (nvidia)
- ret = gpu_unlock_610_unlock(pdev, &shared, true);
+ ret = gpu_unlock_610_unlock(&card->device, &shared, true);
pci_dev_unlock(pdev);
if (!nvidia)
pci_info(pdev, "gpu_unlock: not bound to nvidia, left as is after resume\n");
@@ -106,21 +84,22 @@ static void unlock_again(void *data, async_cookie_t cookie)
pci_info(pdev, "gpu_unlock: unlocked again after resume\n");
}
-/* Unlock every listed card concurrently, with the GSP firmware loaded once. */
+/* Prepare all selected firmware families, then run the ready cards concurrently. */
static void unlock_cards(async_func_t unlock)
{
struct card *card;
int ret;
list_for_each_entry(card, &cards, node) {
- if (!shared.gsp) {
- ret = gpu_unlock_610_load_shared(card->pdev, &shared);
- if (ret) {
- pci_err(card->pdev, "gpu_unlock: firmware load failed (%d)\n", ret);
- break;
- }
- }
- async_schedule_domain(unlock, card, &unlock_domain);
+ ret = gpu_unlock_610_load_shared(&card->device, &shared);
+ card->prepared = !ret;
+ if (ret)
+ pci_err(card->device.pdev, "gpu_unlock: firmware load failed (%d)\n", ret);
+ }
+ /* Publish the completed cache only after every selected family is prepared. */
+ list_for_each_entry(card, &cards, node) {
+ if (card->prepared)
+ async_schedule_domain(unlock, card, &unlock_domain);
}
async_synchronize_full_domain(&unlock_domain);
gpu_unlock_610_release_shared(&shared);
@@ -129,7 +108,7 @@ static void unlock_cards(async_func_t unlock)
static void drop_card(struct card *card)
{
list_del(&card->node);
- pci_dev_put(card->pdev);
+ pci_dev_put(card->device.pdev);
kfree(card);
}
@@ -163,18 +142,21 @@ static bool listed(const char *name)
static void report_unmatched(void)
{
const char *entry, *end;
- struct pci_dev *pdev;
+ struct card *card;
+ bool matched;
for (entry = targets; *entry; entry = *end ? end + 1 : end) {
end = strchrnul(entry, ',');
- pdev = NULL;
- while ((pdev = next_supported(pdev)))
- if (strlen(pci_name(pdev)) == end - entry &&
- !strncasecmp(pci_name(pdev), entry, end - entry))
+ matched = false;
+ list_for_each_entry(card, &cards, node) {
+ const char *name = pci_name(card->device.pdev);
+
+ if (strlen(name) == end - entry && !strncasecmp(name, entry, end - entry)) {
+ matched = true;
break;
- if (pdev)
- pci_dev_put(pdev);
- else
+ }
+ }
+ if (!matched)
pr_err("gpu_unlock: no supported card at \"%.*s\"\n",
(int)(end - entry), entry);
}
@@ -184,25 +166,29 @@ static int __init gpu_unlock_init(void)
{
struct pci_dev *pdev = NULL;
struct card *card, *next;
- bool all;
+ const struct gu_card *recipe;
+ bool all, enumerated = true;
/* Loaded ahead of nvidia on every boot; the command line is the opt-in. */
if (!targets || !*targets)
return 0;
all = !strcmp(targets, "all");
- if (!all)
- report_unmatched();
- while ((pdev = next_supported(pdev))) {
+ while ((pdev = next_supported(pdev, &recipe))) {
if (!all && !listed(pci_name(pdev)))
continue;
card = kzalloc(sizeof(*card), GFP_KERNEL);
if (!card) {
+ pci_err(pdev, "gpu_unlock: card allocation failed\n");
+ enumerated = false;
pci_dev_put(pdev);
break;
}
- card->pdev = pci_dev_get(pdev);
+ card->device.pdev = pci_dev_get(pdev);
+ card->device.recipe = recipe;
list_add_tail(&card->node, &cards);
}
+ if (!all && enumerated)
+ report_unmatched();
/* nvidia loads once this returns: every card must be finished by then. */
unlock_cards(unlock_one);
list_for_each_entry_safe(card, next, &cards, node)
@@ -226,5 +212,5 @@ static void __exit gpu_unlock_exit(void)
module_init(gpu_unlock_init);
module_exit(gpu_unlock_exit);
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Ampere pre-driver BF16 unlock");
-MODULE_VERSION("1.0.0");
+MODULE_DESCRIPTION("Ampere pre-driver hardware unlock");
+MODULE_VERSION("1.1.0");
diff --git a/kernel/gpu_unlock/payload.c b/kernel/gpu_unlock/payload.c
new file mode 100644
index 0000000..9b031ae
--- /dev/null
+++ b/kernel/gpu_unlock/payload.c
@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Signature image construction. The image is data the signed Booter reads, so
+ * every offset is validated against the selected descriptor before writing. */
+#include "diagnostic.h"
+#ifdef __KERNEL__
+#include <linux/errno.h>
+#else
+#include <errno.h>
+#endif
+
+#include "payload.h"
+
+static void put_word(u8 *image, u32 offset, u32 value)
+{
+ unsigned int i;
+
+ for (i = 0; i < 4; i++)
+ image[offset + i] = value >> (8 * i);
+}
+
+int gu_payload_check(const struct gu_payload *payload)
+{
+ size_t i;
+
+ if (!payload || !payload->name || !payload->size || (payload->size & 3) ||
+ payload->value_off > payload->size - 4 ||
+ payload->address_off > payload->size - 4 ||
+ (payload->word_count && !payload->words))
+ return GU_DIAG(-EINVAL);
+ for (i = 0; i < payload->word_count; i++)
+ if (payload->words[i].offset > payload->size - 4)
+ return GU_DIAG(-EINVAL);
+ return 0;
+}
+
+int gu_payload_build(const struct gu_payload *payload, u8 *image,
+ u32 address, u32 value)
+{
+ u32 offset;
+ size_t i;
+ int ret;
+
+ if (!image)
+ return GU_DIAG(-EINVAL);
+ ret = gu_payload_check(payload);
+ if (ret)
+ return ret;
+ for (offset = 0; offset < payload->size; offset += 4)
+ put_word(image, offset, payload->fill);
+ for (i = 0; i < payload->word_count; i++)
+ put_word(image, payload->words[i].offset, payload->words[i].value);
+ put_word(image, payload->value_off, value);
+ put_word(image, payload->address_off, address);
+ return 0;
+}
diff --git a/kernel/gpu_unlock/payload.h b/kernel/gpu_unlock/payload.h
new file mode 100644
index 0000000..87f9e1b
--- /dev/null
+++ b/kernel/gpu_unlock/payload.h
@@ -0,0 +1,40 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_PAYLOAD_H
+#define GPU_UNLOCK_PAYLOAD_H
+
+#include "prep_610.h"
+
+/* One word baked into the signature image at a fixed DMEM-relative offset.
+ * These are image offsets, not BAR0 registers. */
+struct gu_payload_word {
+ u32 offset, value;
+};
+
+/* What a completed SEC2 launch must show besides a halt and a reopened
+ * RESET_PLM. Payloads whose exit returns the input metadata pointer in
+ * MAILBOX0/1 use the echo variant; anything unmeasured stays on the generic
+ * predicate rather than inventing a return convention. */
+enum gu_payload_completion {
+ GU_PAYLOAD_HALT_RESET_PLM,
+ GU_PAYLOAD_HALT_RESET_PLM_ECHO,
+};
+
+/* One signed SEC2 Booter's DMEM program. Selected by the Booter, never by the
+ * family: the gadget and frame offsets are relative to that image's layout, so
+ * another Booter cannot use them. The address and value slots carry the single
+ * privileged write of one invocation. */
+struct gu_payload {
+ const char *name;
+ u32 size; /* signature allocation in bytes */
+ u32 fill; /* body dword */
+ u32 value_off, address_off;
+ const struct gu_payload_word *words;
+ size_t word_count;
+ enum gu_payload_completion completion;
+};
+
+int gu_payload_check(const struct gu_payload *payload);
+int gu_payload_build(const struct gu_payload *payload, u8 *image,
+ u32 address, u32 value);
+
+#endif
diff --git a/kernel/gpu_unlock/prep_610.c b/kernel/gpu_unlock/prep_610.c
index 6013f57..e1a37f3 100644
--- a/kernel/gpu_unlock/prep_610.c
+++ b/kernel/gpu_unlock/prep_610.c
@@ -256,11 +256,11 @@ int gpu_unlock_wpr_meta_pack(u8 out[GU_WPR_META_SIZE],
return 0;
}
-u32 gpu_unlock_dma_cmd(bool imem)
+u32 gpu_unlock_dma_cmd(bool imem, bool secure)
{
u32 cmd = 6u << 8;
if (imem)
- cmd |= (1u << 4) | (1u << 2);
+ cmd |= (1u << 4) | (secure ? (1u << 2) : 0);
return cmd;
}
diff --git a/kernel/gpu_unlock/prep_610.h b/kernel/gpu_unlock/prep_610.h
index 273f345..0e54ae2 100644
--- a/kernel/gpu_unlock/prep_610.h
+++ b/kernel/gpu_unlock/prep_610.h
@@ -17,13 +17,9 @@ typedef uint64_t u64;
#include "layout_610.h"
-#define GU_BOOTER_IMAGE_SIZE 60416u
-#define GU_BOOTER_SIG_TOTAL 768u
-#define GU_V67_SIZE 0xfa00u
#define GU_WPR_META_SIZE 256u
#define GU_WPR_META_MAGIC 0xdc3aae21371a60b3ull
#define GU_RADIX_PAGE 4096u
-#define GU_FUSE_SEC2_UCODE1 0x00824140u
struct gpu_unlock_booter_hdr {
u32 os_code_off;
@@ -92,6 +88,6 @@ int gpu_unlock_wpr_meta_pack(u8 out[GU_WPR_META_SIZE],
const struct gpu_unlock_layout_out *layout,
const struct gpu_unlock_wpr_args *args);
-u32 gpu_unlock_dma_cmd(bool imem);
+u32 gpu_unlock_dma_cmd(bool imem, bool secure);
#endif
diff --git a/kernel/gpu_unlock/recipes.c b/kernel/gpu_unlock/recipes.c
new file mode 100644
index 0000000..a6fbbf4
--- /dev/null
+++ b/kernel/gpu_unlock/recipes.c
@@ -0,0 +1,213 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include "recipes.h"
+#include "registers.h"
+
+/* GA102 re-entrant V67 call table. The Booter dispatches the frame RET into
+ * these gadgets and performs the privileged write from the value/address
+ * slots, so one invocation applies one table entry. Offsets are relative to
+ * the signature image, not to BAR0. */
+static const struct gu_payload_word ga102_v67_words[] = {
+ { 0x1100, 0x00000007 },
+ { 0xf950, 0x00000d44 },
+ { 0xf968, 0x00001fce },
+ { 0xf974, 0x00000000 },
+ { 0xf97c, 0x00001101 },
+ { 0xf980, 0x000084c8 },
+ { 0xf984, 0x00008e18 },
+ { 0xf98c, 0x000084c8 },
+ { 0xf990, 0x00000000 },
+ { 0xf998, 0x00001fce },
+ { 0xf9a4, 0x0000ffb0 },
+ { 0xf9ac, 0x00005789 },
+ { 0xf9bc, 0x00000d44 },
+ { 0xf9cc, 0x00000003 },
+ { 0xf9d4, 0x00001fce },
+ { 0xf9e8, 0x00000d52 },
+ { 0xf9ec, 0x000081ee },
+};
+#define GU_PAYLOAD_V67(label) { \
+ .name = label, \
+ .size = 0xfa00, \
+ .fill = 0x0000019c, \
+ .value_off = 0xf948, \
+ .address_off = 0xf960, \
+ .words = ga102_v67_words, \
+ .word_count = sizeof(ga102_v67_words) / sizeof(ga102_v67_words[0]), \
+ .completion = GU_PAYLOAD_HALT_RESET_PLM, \
+}
+
+static const struct gu_payload ga102_v67 = GU_PAYLOAD_V67("ga102-v67");
+/* No payload has been derived for the GA100 or AD102 signed Booter yet; both
+ * reuse the GA102 V67 body until one is. Replace the descriptor when it lands. */
+static const struct gu_payload ga100_v67 = GU_PAYLOAD_V67("ga100-v67");
+static const struct gu_payload ad102_v67 = GU_PAYLOAD_V67("ad102-v67");
+
+/* Same per-entry image as GA10x. These are the stores, in order. */
+static const struct gu_reg_write ad102_plm[] = {
+ { GU_WPR_CFG_PLM_OFFSET, 0xfffff0ff },
+ { GU_WPR_PLM_OFFSET, 0xffffffff },
+ { GU_FEAT_PLM_OFFSET, 0xffffffff },
+};
+
+/* Only the masks needed by the GA10x compute recipe. */
+static const struct gu_reg_write ga10x_plm[] = {
+ { GU_WPR_CFG_PLM_OFFSET, 0xfffff0ff },
+ { GU_FEAT_PLM_OFFSET, 0xffffffff },
+};
+
+/* cmpunlocker 2d10f891, sec2-postbl-plm-ss-cfg.patch, plmTable.
+ * Order and WPR_CFG's source-control bits are significant. No geometry,
+ * link retraining, ECC or GPC/TPC configuration is performed in Stage 1.
+ */
+static const struct gu_reg_write ga100_plm[] = {
+ { GU_WPR_CFG_PLM_OFFSET, 0xfffff0ff },
+ { GU_FBPA_PLM_OFFSET, 0xffffffff },
+ { GU_WPR_PLM_OFFSET, 0xffffffff },
+ { GU_FEAT_PLM_OFFSET, 0xffffffff },
+ { GU_XVE_PLM_OFFSET, 0xffffffff },
+ { GU_XVE_B_PLM_OFFSET, 0xffffffff },
+ { GU_XVE_C_PLM_OFFSET, 0xffffffff },
+ { GU_FEAT2_PLM_OFFSET, 0xffffffff },
+ { GU_OPT_PLM_OFFSET, 0xffffffff },
+ { GU_RECONFIG_PLM_OFFSET, 0xffffffff },
+ { GU_PJTAG_PLM_OFFSET, 0xffffffff },
+ { GU_PJTAG_SEC_PLM_OFFSET, 0xffffffff },
+};
+
+/* Host side writes for speed unlocks */
+static const struct gu_reg_write selectors[] = {
+ { GU_FEAT_SS0_OFFSET, GU_SS0_UNLOCK },
+ { GU_FEAT_SS1_OFFSET, GU_SS1_UNLOCK },
+};
+
+static const struct gu_booter sec2_ga100 = {
+ .file = GU_FW_BOOTER_GA100,
+ .launch = GU_SEC2_DIRECT,
+ .payload = &ga100_v67,
+ .expect = {
+ .image_size = 60160,
+ .num_sigs = 1,
+ .sig_bytes = 384,
+ .os_data_off = 34560,
+ .os_data_size = 25600,
+ .app_code_size = 34304,
+ .patch_loc = 35072,
+ },
+};
+/* The GA10x SEC2 Booter. Its signature patch location is 0x8a10. */
+static const struct gu_booter sec2_ga102 = {
+ .file = GU_FW_BOOTER_GA102,
+ .launch = GU_SEC2_HS,
+ .payload = &ga102_v67,
+ .expect = {
+ .image_size = 60416,
+ .num_sigs = 2,
+ .sig_bytes = 768,
+ .os_data_off = 35328,
+ .os_data_size = 25088,
+ .app_code_size = 35072,
+ .patch_loc = 35344,
+ },
+};
+static const struct gu_booter sec2_ad102 = {
+ .file = GU_FW_BOOTER_AD102,
+ .launch = GU_SEC2_HS,
+ .payload = &ad102_v67,
+ .expect = {
+ .image_size = 0xde00,
+ .num_sigs = 2,
+ .sig_bytes = 768,
+ .os_data_off = 0x7e00,
+ .os_data_size = 0x6000,
+ .app_code_size = 0x7d00,
+ .patch_loc = 0x7e10,
+ },
+};
+
+static const struct gu_fw_desc ga100_fw = {
+ .gsp = GU_FW_GSP_GA100,
+ .boot = GU_FW_BOOT_GA100,
+ .desc = GU_FW_DESC_GA100,
+ .signature_section = ".fwsignature_ga100",
+ .boot_size = 4096,
+};
+static const struct gu_fw_desc ga10x_fw = {
+ .gsp = GU_FW_GSP_GA10X,
+ .boot = GU_FW_BOOT_GA102,
+ .desc = GU_FW_DESC_GA102,
+ .signature_section = ".fwsignature_ga10x",
+ .boot_size = 24576,
+};
+static const struct gu_fw_desc ad102_fw = {
+ .gsp = GU_FW_GSP_GA10X,
+ .boot = GU_FW_BOOT_AD102,
+ .desc = GU_FW_DESC_AD102,
+ .signature_section = ".fwsignature_ad10x",
+ .boot_size = 36864,
+};
+
+static const struct gu_family ga100 = {
+ .name = "GA100",
+ .privileged = GU_WRITE_TABLE(ga100_plm),
+ .host = GU_WRITE_TABLE(selectors),
+ .fw = &ga100_fw,
+ .booter = &sec2_ga100,
+ .loader = GU_FW_LOADER,
+ .fwsec_version = 2,
+};
+static const struct gu_family ga10x = {
+ .name = "GA102/GA103/GA104/GA106",
+ .privileged = GU_WRITE_TABLE(ga10x_plm),
+ .host = GU_WRITE_TABLE(selectors),
+ .fw = &ga10x_fw,
+ .booter = &sec2_ga102,
+ .loader = NULL,
+ .fwsec_version = 3,
+};
+static const struct gu_family ad102 = {
+ .name = "AD102",
+ .privileged = GU_WRITE_TABLE(ad102_plm),
+ .host = GU_WRITE_TABLE(selectors),
+ .fw = &ad102_fw,
+ .booter = &sec2_ad102,
+ .loader = NULL,
+ .fwsec_version = 3,
+};
+
+#define AD102(id) { id, 0x192, &ad102 }
+
+#define GA100(id) { id, 0x170, &ga100 }
+#define GA102(id) { id, 0x172, &ga10x }
+#define GA103(id) { id, 0x173, &ga10x }
+#define GA104(id) { id, 0x174, &ga10x }
+#define GA106(id) { id, 0x176, &ga10x }
+
+static const struct gu_card cards[] = {
+ GA102(0x2200), GA102(0x2203), GA102(0x2204), GA102(0x2205),
+ GA102(0x2206), GA102(0x2207), GA102(0x2208), GA102(0x220a),
+ GA102(0x220d), GA102(0x2216), GA102(0x222b), GA102(0x222f),
+
+ GA103(0x2414),
+
+ GA104(0x2480), GA104(0x2482), GA104(0x2483), GA104(0x2484),
+ GA104(0x2486), GA104(0x2487), GA104(0x2488), GA104(0x2489),
+ GA104(0x248a), GA104(0x248c), GA104(0x248d), GA104(0x248e),
+ GA104(0x249c), GA104(0x249d), GA104(0x249f), GA104(0x24a0),
+ GA104(0x24a4), GA104(0x24ac), GA104(0x24ad), GA104(0x24af),
+ GA104(0x24bf), GA104(0x24c0), GA104(0x24c7), GA104(0x24c8),
+ GA104(0x24c9), GA104(0x24dc), GA104(0x24dd), GA104(0x24df),
+ GA104(0x24e0),
+
+ GA106(0x2501), GA106(0x2503), GA106(0x2504), GA106(0x2505),
+ GA106(0x2507), GA106(0x2508), GA106(0x2509), GA106(0x252f),
+ GA106(0x2544),
+};
+
+const struct gu_card *gu_card_find(u16 device)
+{
+ size_t i;
+ for (i = 0; i < sizeof(cards) / sizeof(cards[0]); i++)
+ if (cards[i].device == device)
+ return &cards[i];
+ return NULL;
+}
diff --git a/kernel/gpu_unlock/recipes.h b/kernel/gpu_unlock/recipes.h
new file mode 100644
index 0000000..21db7a3
--- /dev/null
+++ b/kernel/gpu_unlock/recipes.h
@@ -0,0 +1,68 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_RECIPES_H
+#define GPU_UNLOCK_RECIPES_H
+#include "payload.h"
+
+#define GU_FW_GSP_GA10X "gpu-unlock/610.43.03/gsp_ga10x.bin"
+#define GU_FW_BOOT_GA102 "gpu-unlock/610.43.03/gsp_boot_ga102.bin"
+#define GU_FW_DESC_GA102 "gpu-unlock/610.43.03/gsp_boot_desc_ga102.bin"
+#define GU_FW_GSP_GA100 "gpu-unlock/610.43.03/gsp_tu10x.bin"
+#define GU_FW_BOOT_GA100 "gpu-unlock/610.43.03/gsp_boot_ga100.bin"
+#define GU_FW_DESC_GA100 "gpu-unlock/610.43.03/gsp_boot_desc_ga100.bin"
+#define GU_FW_LOADER "gpu-unlock/610.43.03/falcon_loader.bin"
+#define GU_FW_BOOTER_GA100 "gpu-unlock/610.43.03/booter_load_ga100.bin"
+#define GU_FW_BOOTER_GA102 "gpu-unlock/610.43.03/booter_load_ga102.bin"
+#define GU_FW_BOOTER_AD102 "gpu-unlock/610.43.03/booter_load_ad102.bin"
+#define GU_FW_BOOT_AD102 "gpu-unlock/610.43.03/gsp_boot_ad102.bin"
+#define GU_FW_DESC_AD102 "gpu-unlock/610.43.03/gsp_boot_desc_ad102.bin"
+
+struct gu_reg_write { u32 addr, value; };
+struct gu_write_table { const struct gu_reg_write *entries; size_t count; };
+
+/* Header fields that differ between signed Booter images. The common checks
+ * stay in the validator. */
+struct gu_booter_expect {
+ u32 image_size, num_sigs, sig_bytes;
+ u32 os_data_off, os_data_size, app_code_size, patch_loc;
+};
+
+enum gu_sec2_launch {
+ GU_SEC2_DIRECT, /* IMEM copy. FWSEC uses the generic Falcon loader. */
+ GU_SEC2_HS,
+};
+
+/* One signed SEC2 Booter image. The family selects the Booter, and the Booter
+ * selects its payload: the payload's offsets only mean something for the DMEM
+ * frame layout of this image. Families that share a pointer share one loaded
+ * slot. */
+struct gu_booter {
+ const char *file;
+ struct gu_booter_expect expect;
+ enum gu_sec2_launch launch; /* how this image is started */
+ const struct gu_payload *payload;
+};
+
+/* Immutable file set for one GSP image. Families that share a pointer share
+ * one loaded bundle. */
+struct gu_fw_desc {
+ const char *gsp, *boot, *desc, *signature_section;
+ u32 boot_size;
+};
+
+struct gu_family {
+ const char *name;
+ struct gu_write_table privileged, host;
+ const struct gu_fw_desc *fw;
+ const struct gu_booter *booter;
+ const char *loader; /* NULL when this family has no generic loader */
+ u8 fwsec_version;
+};
+struct gu_card {
+ u16 device;
+ u32 chip; /* PMC_BOOT_0 architecture + implementation */
+ const struct gu_family *family;
+};
+#define GU_WRITE_TABLE(t) { .entries = (t), .count = sizeof(t) / sizeof((t)[0]) }
+
+const struct gu_card *gu_card_find(u16 device);
+#endif
diff --git a/kernel/gpu_unlock/registers.h b/kernel/gpu_unlock/registers.h
new file mode 100644
index 0000000..48cdc5f
--- /dev/null
+++ b/kernel/gpu_unlock/registers.h
@@ -0,0 +1,93 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GPU_UNLOCK_REGISTERS_H
+#define GPU_UNLOCK_REGISTERS_H
+
+/* BAR0 registers used by the Ampere recipes and firmware transaction.
+ * Names follow the pinned NVIDIA 610.43.03 register headers and cmpunlocker
+ * PLM table. Falcon/FBIF/Falcon2 offsets are relative to their engine bases.
+ */
+#define GU_PMC_BOOT_0_OFFSET 0x00000000u
+#define GU_PROM_DATA_OFFSET 0x00300000u
+#define GU_BUS_SW_SCRATCH(index) (0x00001400u + (index) * 4u)
+#define GU_ACR_SCRATCH_OFFSET GU_BUS_SW_SCRATCH(14)
+
+#define GU_GSP_BASE 0x00110000u
+#define GU_GSP_FBIF_BASE 0x00110600u
+#define GU_GSP_FALCON2_BASE 0x00111000u
+#define GU_SEC2_BASE 0x00840000u
+#define GU_SEC2_FBIF_BASE 0x00840600u
+#define GU_SEC2_FALCON2_BASE 0x00841000u
+#define GU_FALCON2_WINDOW_SIZE 0x00001000u
+
+#define GU_FALCON_MAILBOX0_OFFSET 0x00000040u
+#define GU_FALCON_MAILBOX1_OFFSET 0x00000044u
+#define GU_FALCON_RM_OFFSET 0x00000084u
+#define GU_FALCON_HWCFG2_OFFSET 0x000000f4u
+#define GU_FALCON_CPUCTL_OFFSET 0x00000100u
+#define GU_FALCON_BOOTVEC_OFFSET 0x00000104u
+#define GU_FALCON_HWCFG_OFFSET 0x00000108u
+#define GU_FALCON_DMACTL_OFFSET 0x0000010cu
+#define GU_FALCON_DMATRFBASE_OFFSET 0x00000110u
+#define GU_FALCON_DMATRFMOFFS_OFFSET 0x00000114u
+#define GU_FALCON_DMATRFCMD_OFFSET 0x00000118u
+#define GU_FALCON_DMATRFFBOFFS_OFFSET 0x0000011cu
+#define GU_FALCON_DMATRFBASE1_OFFSET 0x00000128u
+#define GU_FALCON_CPUCTL_ALIAS_OFFSET 0x00000130u
+/* Host IMEM/DMEM access port 0. */
+#define GU_FALCON_IMEMC_OFFSET 0x00000180u
+#define GU_FALCON_IMEMD_OFFSET 0x00000184u
+#define GU_FALCON_IMEMT_OFFSET 0x00000188u
+#define GU_FALCON_DMEMC_OFFSET 0x000001c0u
+#define GU_FALCON_DMEMD_OFFSET 0x000001c4u
+#define GU_FALCON_ENGINE_OFFSET 0x000003c0u
+#define GU_FALCON_FHSTATE_OFFSET 0x0000005cu
+#define GU_SEC2_CSBERRSTAT_OFFSET 0x00000244u
+#define GU_FALCON_RESET_PLM_OFFSET 0x000003c4u
+#define GU_SEC2_RESET_PLM_OFFSET (GU_SEC2_BASE + GU_FALCON_RESET_PLM_OFFSET)
+
+#define GU_FBIF_TRANSCFG(index) (0x00000000u + (index) * 4u)
+#define GU_FBIF_CTL_OFFSET 0x00000024u
+#define GU_FALCON2_MOD_SEL_OFFSET 0x00000180u
+#define GU_FALCON2_BROM_CURR_UCODE_ID_OFFSET 0x00000198u
+#define GU_FALCON2_BROM_ENGIDMASK_OFFSET 0x0000019cu
+/* BROM parameter-address register 0 and RISC-V boot control. */
+#define GU_FALCON2_BROM_PARAADDR_OFFSET 0x00000210u
+#define GU_RISCV_BCR_CTRL_OFFSET 0x00000668u
+
+#define GU_FUSE_SEC2_UCODE1 0x00824140u
+#define GU_FUSE_GSP_UCODE1 0x008241c0u
+
+#define GU_GFW_PLM_OFFSET 0x00118128u
+#define GU_GFW_BOOT_OFFSET 0x00118234u
+#define GU_GFW_CPUCTL_OFFSET (GU_GSP_BASE + GU_FALCON_CPUCTL_OFFSET)
+/* PGC6 AON_SECURE_SCRATCH_GROUP_42 reports framebuffer size in MiB. */
+#define GU_GFW_FB_SIZE_OFFSET 0x001183a4u
+#define GU_VGA_WORKSPACE_BASE_OFFSET 0x00625f04u
+#define GU_MMU_LOCK_CFG_PLM_OFFSET 0x001fa7c8u
+#define GU_MMU_LOCK_ADDR_LO_OFFSET 0x001fa82cu
+#define GU_MMU_LOCK_ADDR_HI_OFFSET 0x001fa830u
+#define GU_WPR2_ADDR_LO_OFFSET 0x001fa824u
+#define GU_WPR2_ADDR_HI_OFFSET 0x001fa828u
+#define GU_WPR2_ALIGN 0x00020000u
+
+#define GU_WPR_CFG_PLM_OFFSET 0x001fa7ccu
+#define GU_FBPA_PLM_OFFSET 0x009a0148u
+#define GU_WPR_PLM_OFFSET 0x001fa7c4u
+#define GU_FEAT_PLM_OFFSET 0x00823804u
+#define GU_XVE_PLM_OFFSET 0x00088ff4u
+#define GU_XVE_B_PLM_OFFSET 0x00088ab4u
+#define GU_XVE_C_PLM_OFFSET 0x00088ff8u
+#define GU_FEAT2_PLM_OFFSET 0x00823b00u
+#define GU_OPT_PLM_OFFSET 0x008200fcu
+#define GU_RECONFIG_PLM_OFFSET 0x008200d8u
+#define GU_PJTAG_PLM_OFFSET 0x0000c840u
+#define GU_PJTAG_SEC_PLM_OFFSET 0x0000c848u
+#define GU_FEAT_READOUT_1_OFFSET 0x00823818u
+#define GU_SPEED_READOUT_MASK 0x07ffffffu
+#define GU_FEAT_SS0_OFFSET 0x0082381cu
+#define GU_FEAT_SS1_OFFSET 0x00823820u
+/* SS0 has eight 4-bit fields; SS1 has one. Value 8 selects full rate. */
+#define GU_SS0_UNLOCK 0x88888888u
+#define GU_SS1_UNLOCK 0x8u
+
+#endif
diff --git a/kernel/gpu_unlock/unlock_610.c b/kernel/gpu_unlock/unlock_610.c
index 2d45133..fe91abc 100644
--- a/kernel/gpu_unlock/unlock_610.c
+++ b/kernel/gpu_unlock/unlock_610.c
@@ -16,30 +16,27 @@
#include <linux/timekeeping.h>
#include "diagnostic.h"
+#include "falcon.h"
#include "gfw_eval.h"
#include "prep_610.h"
+#include "recipes.h"
+#include "registers.h"
#include "unlock_610.h"
#include "vbios_610.h"
-#define SEC2_BASE 0x00840000u
-#define SEC2_FBIF 0x00840600u
-#define SEC2_FALCON2 0x00841000u
-/* SS0 holds eight 4-bit fields; SS1 stores only its low 4 bits (0x2 when
- * cold). Both get the same per-field value 8. */
-#define GU_SS0_UNLOCK 0x88888888u
-#define GU_SS1_UNLOCK 0x8u
-#define BOOTER_MAGIC 0x314c5442u
-#define BOOTER_FILE "gpu-unlock/610.43.03/booter_load_ga102.bin"
-#define V67_FILE "gpu-unlock/610.43.03/v67-ga102-610.43.03.bin"
-#define GSP_FILE "gpu-unlock/610.43.03/gsp_ga10x.bin"
-#define BOOT_FILE "gpu-unlock/610.43.03/gsp_boot_ga102.bin"
-#define BOOT_DESC_FILE "gpu-unlock/610.43.03/gsp_boot_desc_ga102.bin"
-
+enum gu_lifecycle { GU_ALLOCATED, GU_PUBLISHED, GU_ACTIVE, GU_STOCK, GU_QUIESCED };
struct unlock_transaction_page {
void *cpu;
dma_addr_t dma;
};
+/* One coherent allocation. size is the length passed to dma_alloc_coherent. */
+struct unlock_dma {
+ void *cpu;
+ dma_addr_t dma;
+ size_t size;
+};
+
struct unlock_transaction_work {
struct pci_dev *pdev;
bool resume; /* bound to nvidia, which suspended it before sleep */
@@ -49,67 +46,53 @@ struct unlock_transaction_work {
bool cmd_valid;
bool enabled;
bool region;
- bool mapped;
bool master;
- const struct firmware *booter_fw;
- const struct firmware *v67_fw;
- const struct firmware *gsp_fw; /* borrowed from gpu_unlock_610_shared */
- const struct firmware *boot_fw;
- struct gpu_unlock_span image_span, sig_span;
- struct gpu_unlock_layout_in layout_in;
+ const struct gpu_unlock_610_firmware *bundle; /* borrowed immutable shared assets */
+ const struct gpu_unlock_610_booter *booter;
+ const struct gu_payload *payload; /* borrowed, selected by the Booter */
+ const struct firmware *loader_fw;
+ u64 usable_fb, vga_base, frts_size;
struct gpu_unlock_layout_out layout;
- struct gpu_unlock_boot_desc boot_desc;
- struct gpu_unlock_radix_plan radix;
struct unlock_transaction_page *pages;
u32 page_count, pages_allocated;
- void *boot_cpu;
- dma_addr_t boot_dma;
- void *booter_cpu;
- dma_addr_t booter_dma;
- u32 booter_image_size;
- struct gpu_unlock_booter_hdr booter_hdr;
- void *v67_cpu;
- dma_addr_t v67_dma;
- void *stock_cpu;
- dma_addr_t stock_dma;
- u64 stock_size;
- void *meta_cpu;
- dma_addr_t meta_dma;
+ struct unlock_dma boot, booter_buf, buffer, stock, meta, fwsec;
u64 *streaming_ptr;
u64 streaming_before;
u64 coherent_before;
- bool streaming_set;
bool coherent_set;
- unsigned int signature_calls;
- bool published;
- bool masks_saved;
+ enum gu_lifecycle state;
+ const struct gu_card *card;
+ size_t write_index;
+ unsigned int attempt;
+ u32 frts_lo, frts_hi;
+ bool frts_saved, invocation_halted;
bool firmware_started;
- bool quiesced;
bool retained;
- u32 plm_entry;
+ bool error_reported;
u32 bars[6];
- void *fwsec_cpu;
- dma_addr_t fwsec_dma;
- u32 fwsec_size;
u32 last_offset, last_value;
bool last_valid;
u32 last_pci_offset, last_pci_value;
int last_pci_error;
bool last_pci_cap;
+ u32 target_addr, target_value;
+};
+
+struct gu_boot_state {
+ u32 ctl, engine, reset_plm, mailbox0, mailbox1, acr, csberr, fhstate;
+ u32 wpr_lo, wpr_hi;
+ bool ok;
};
+static void gu_snapshot(struct unlock_transaction_work *work, const char *function,
+ unsigned int line, int error);
+
static int unlock_transaction_result(struct unlock_transaction_work *work, int error,
const char *function, unsigned int line)
{
- if (error)
- {
- dev_err(&work->pdev->dev,
- "failure unlock_610.c:%u %s error=%d last_mmio=%#x value=%#x valid=%u mapped=%u master=%u firmware_started=%u quiesced=%u pages=%u/%u\n",
- line, function, error, work->last_offset, work->last_value,
- work->last_valid, work->mapped, work->master, work->firmware_started,
- work->quiesced, work->pages_allocated, work->page_count);
- dev_err(&work->pdev->dev, "last PCI sample: capability=%u offset=%#x value=%#x result=%d\n",
- work->last_pci_cap, work->last_pci_offset, work->last_pci_value, work->last_pci_error);
+ if (error && !work->error_reported) {
+ work->error_reported = true;
+ gu_snapshot(work, function, line, error);
}
return error;
}
@@ -157,35 +140,6 @@ static void *diag_dma_alloc(struct unlock_transaction_work *work, const char *na
return memory;
}
-/* CONFIG_FW_CACHE attaches name records to devres even though this is not
- * devm_request_firmware(). release_firmware() does not remove them, and
- * really_probe() refuses a device with any existing devres. Scope each
- * synchronous request so only its cache records are released; the returned
- * firmware remains owned by the caller.
- */
-static int request_scoped(struct device *dev, const struct firmware **fw, const char *name)
-{
- void *group;
- int ret, released;
-
- group = devres_open_group(dev, NULL, GFP_KERNEL);
- if (!group)
- return -ENOMEM;
- dev_info(dev, "asset request: %s\n", name);
- ret = request_firmware(fw, name, dev);
- released = devres_release_group(dev, group);
- dev_info(dev, "asset result: %s error=%d bytes=%zu\n",
- name, ret, ret ? (size_t)0 : (*fw)->size);
- dev_info(dev, "asset cache cleanup: %s devres_released=%d\n", name, released);
- return ret;
-}
-
-static int request_asset(struct unlock_transaction_work *work, const struct firmware **fw,
- const char *name)
-{
- return GU_RESULT(request_scoped(&work->pdev->dev, fw, name));
-}
-
static int check_devres_empty(struct unlock_transaction_work *work, const char *phase)
{
struct device *dev = &work->pdev->dev;
@@ -196,7 +150,7 @@ static int check_devres_empty(struct unlock_transaction_work *work, const char *
spin_lock_irqsave(&dev->devres_lock, flags);
empty = list_empty(&dev->devres_head);
spin_unlock_irqrestore(&dev->devres_lock, flags);
- dev_info(dev, "driver probe resources: phase=%s empty=%u\n", phase, empty);
+ dev_dbg(dev, "driver probe resources: phase=%s empty=%u\n", phase, empty);
return GU_RESULT(empty ? 0 : -EBUSY);
}
@@ -204,7 +158,7 @@ static int read_bar(struct unlock_transaction_work *work, u32 offset, u32 *value
{
work->last_offset = offset;
work->last_valid = false;
- if (!work->mapped || offset > work->bar_len || work->bar_len - offset < 4)
+ if (!work->bar || offset > work->bar_len || work->bar_len - offset < 4)
return GU_RESULT(-EINVAL);
*value = readl(work->bar + offset);
work->last_value = *value;
@@ -217,277 +171,172 @@ static void write_bar(struct unlock_transaction_work *work, u32 offset, u32 valu
writel(value, work->bar + offset);
}
-static int poll_us(struct unlock_transaction_work *work, u32 offset, u32 mask, u32 expect,
- unsigned int budget_us, bool timeout_is_failure)
+static int transport_read(void *context, u32 offset, u32 *value)
{
- ktime_t deadline = ktime_add_us(ktime_get(), budget_us);
- u32 value = 0;
-
- for (;;) {
- if (read_bar(work, offset, &value) || value == ~0u ||
- (value & 0xffff0000u) == 0xbadf0000u) {
- dev_err(&work->pdev->dev, "poll inaccessible: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
- offset, value, mask, expect, budget_us);
- return GU_RESULT(-EIO);
- }
- if ((value & mask) == expect)
- return 0;
- if (ktime_compare(ktime_get(), deadline) >= 0) {
- /* Console loglevel hides info. A tolerated timeout must not
- * look like the failure that stopped the boot. */
- if (!timeout_is_failure) {
- dev_info(&work->pdev->dev, "poll timeout tolerated: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
- offset, value, mask, expect, budget_us);
- return -ETIMEDOUT;
- }
- dev_err(&work->pdev->dev, "poll timeout: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
- offset, value, mask, expect, budget_us);
- return GU_RESULT(-ETIMEDOUT);
- }
- usleep_range(50, 100);
- }
+ return read_bar(context, offset, value);
+}
+static void transport_write(void *context, u32 offset, u32 value)
+{
+ write_bar(context, offset, value);
+}
+static struct gu_falcon_io falcon_io(struct unlock_transaction_work *work)
+{
+ return (struct gu_falcon_io) {
+ .dev = &work->pdev->dev, .context = work,
+ .read = transport_read, .write = transport_write,
+ };
}
-
-struct falcon_eng {
- u32 base;
- u32 fbif;
- u32 falcon2;
-};
static bool inaccessible(u32 value)
{
return value == ~0u || (value & 0xffff0000u) == 0xbadf0000u;
}
-static int falcon_reset(struct unlock_transaction_work *work, const struct falcon_eng *eng)
+static u32 booter_image_bytes(const struct gpu_unlock_booter_hdr *hdr)
{
- u32 engine, hwcfg, bcr, chip;
- unsigned int i;
- int ret;
-
- /* 610 kflcnPreResetWait_GA102 permits this timeout (HW bug 3419321). */
- dev_info(&work->pdev->dev, "Falcon reset begin: base=%#x\n", eng->base);
- ret = poll_us(work, eng->base + 0xf4, 1u << 31, 1u << 31, 1000, false);
- if (ret && ret != -ETIMEDOUT)
- return GU_RESULT(ret);
- if (ret)
- dev_info(&work->pdev->dev, "Falcon pre-reset timeout tolerated per HW bug 3419321\n");
- if (read_bar(work, eng->base + 0x3c0, &engine) || inaccessible(engine))
- return GU_RESULT(-ENODEV);
- write_bar(work, eng->base + 0x3c0, engine | 1);
- for (i = 0; i < 10; i++)
- readl(work->bar + eng->base + 0x3c0);
- write_bar(work, eng->base + 0x3c0, engine & ~1u);
- for (i = 0; i < 10; i++)
- readl(work->bar + eng->base + 0x3c0);
- ret = poll_us(work, eng->base + 0xf4, 1u << 12, 0, 1000000, true);
- if (ret)
- return GU_RESULT(ret);
- if (read_bar(work, eng->base + 0xf4, &hwcfg) || inaccessible(hwcfg))
- return GU_RESULT(-EIO);
- if (hwcfg & (1u << 10)) {
- if (read_bar(work, eng->falcon2 + 0x668, &bcr) || inaccessible(bcr))
- return GU_RESULT(-ENODEV);
- if (bcr & (1u << 4)) {
- write_bar(work, eng->falcon2 + 0x668, 0);
- ret = poll_us(work, eng->falcon2 + 0x668, 0x11, 1, 1000000, true);
- if (ret)
- return GU_RESULT(ret);
- }
- }
- if (read_bar(work, 0, &chip) || inaccessible(chip))
- return GU_RESULT(-ENODEV);
- write_bar(work, eng->base + 0x84, chip);
- dev_info(&work->pdev->dev, "Falcon reset done: base=%#x hwcfg=%#x chip=%#x\n", eng->base, hwcfg, chip);
- return 0;
+ return hdr->os_data_off + hdr->os_data_size;
}
-/* DMATRFMOFFS is 24 bits, FBOFFS is 32 bits, and BASE is 49 bits.
- * Reject truncation/wrap before submitting any part of a transfer. */
-static bool falcon_dma_valid(u32 dest, u32 mem_off, dma_addr_t src, u32 size)
+static void gu_read_boot_state(struct unlock_transaction_work *work,
+ struct gu_boot_state *state)
{
- return size && !((size | dest | mem_off | src) & 255) &&
- (u64)dest + size <= (1ull << 24) &&
- (u64)mem_off + size <= (1ull << 32) &&
- src < (1ull << 49) &&
- (u64)mem_off + size <= (1ull << 49) - src;
+ u32 value;
+
+ memset(state, 0, sizeof(*state));
+ if (!work->bar)
+ return;
+ state->ok = true;
+#define GU_BOOT_RD(field, offset) \
+ do { \
+ if (read_bar(work, (offset), &value)) \
+ state->ok = false; \
+ else \
+ state->field = value; \
+ } while (0)
+ GU_BOOT_RD(ctl, GU_SEC2_BASE + GU_FALCON_CPUCTL_OFFSET);
+ GU_BOOT_RD(engine, GU_SEC2_BASE + GU_FALCON_ENGINE_OFFSET);
+ GU_BOOT_RD(reset_plm, GU_SEC2_RESET_PLM_OFFSET);
+ GU_BOOT_RD(mailbox0, GU_SEC2_BASE + GU_FALCON_MAILBOX0_OFFSET);
+ GU_BOOT_RD(mailbox1, GU_SEC2_BASE + GU_FALCON_MAILBOX1_OFFSET);
+ GU_BOOT_RD(acr, GU_ACR_SCRATCH_OFFSET);
+ GU_BOOT_RD(csberr, GU_SEC2_BASE + GU_SEC2_CSBERRSTAT_OFFSET);
+ GU_BOOT_RD(fhstate, GU_SEC2_BASE + GU_FALCON_FHSTATE_OFFSET);
+ GU_BOOT_RD(wpr_lo, GU_WPR2_ADDR_LO_OFFSET);
+ GU_BOOT_RD(wpr_hi, GU_WPR2_ADDR_HI_OFFSET);
+#undef GU_BOOT_RD
}
-static int falcon_dma(struct unlock_transaction_work *work, u32 base, u32 dest, u32 mem_off,
- dma_addr_t src, u32 size, u32 cmd)
+/* Preserve the first failure state before unwind/FLR can hide it. Every
+ * read here is diagnostic: errors are ignored and must not replace the
+ * original result. */
+static void gu_snapshot(struct unlock_transaction_work *work, const char *function,
+ unsigned int line, int error)
{
- u32 done = 0;
- int ret;
-
- dev_info(&work->pdev->dev, "Falcon DMA: base=%#x dest=%#x offset=%#x src=%#llx size=%#x cmd=%#x\n",
- base, dest, mem_off, (unsigned long long)src, size, cmd);
- if (!falcon_dma_valid(dest, mem_off, src, size))
- return GU_RESULT(-EINVAL);
- ret = poll_us(work, base + 0x118, 1, 0, 1000000, true);
- if (ret)
- return GU_RESULT(ret);
- write_bar(work, base + 0x110, lower_32_bits(src >> 8));
- write_bar(work, base + 0x128, upper_32_bits(src >> 8) & 0x1ff);
- while (done < size) {
- ret = poll_us(work, base + 0x118, 1, 0, 1000000, true);
- if (ret) {
- dev_err(&work->pdev->dev, "Falcon DMA progress: done=%u total=%u\n", done, size);
- return GU_RESULT(ret);
- }
- write_bar(work, base + 0x114, dest);
- write_bar(work, base + 0x11c, mem_off);
- write_bar(work, base + 0x118, cmd);
- done += 256;
- dest += 256;
- mem_off += 256;
- }
- return GU_RESULT(poll_us(work, base + 0x118, 2, 2, 1000000, true));
+ const char *family = "?";
+ const char *payload = "?";
+ struct gu_boot_state b;
+ u32 last_off = work->last_offset, last_val = work->last_value;
+ bool last_valid = work->last_valid;
+ u32 last_pci_off = work->last_pci_offset, last_pci_val = work->last_pci_value;
+ int last_pci_err = work->last_pci_error;
+ bool last_pci_cap = work->last_pci_cap;
+ u32 feat = 0, wpr_cfg = 0, wpr = 0, lock = 0, ss0 = 0, ss1 = 0;
+ u16 command = 0xffff, devsta = 0xffff;
+ u64 start = 0, end = 0;
+ u32 dma_now = 0, coherent_now = 0;
+
+ if (work->card && work->card->family)
+ family = work->card->family->name;
+ if (work->payload)
+ payload = work->payload->name;
+ gu_read_boot_state(work, &b);
+ read_bar(work, GU_FEAT_PLM_OFFSET, &feat);
+ read_bar(work, GU_WPR_CFG_PLM_OFFSET, &wpr_cfg);
+ read_bar(work, GU_WPR_PLM_OFFSET, &wpr);
+ read_bar(work, GU_MMU_LOCK_CFG_PLM_OFFSET, &lock);
+ read_bar(work, GU_FEAT_SS0_OFFSET, &ss0);
+ read_bar(work, GU_FEAT_SS1_OFFSET, &ss1);
+ diag_pci_word(work, false, PCI_COMMAND, &command);
+ diag_pci_word(work, true, PCI_EXP_DEVSTA, &devsta);
+ if (work->streaming_ptr)
+ dma_now = (u32)*work->streaming_ptr;
+ coherent_now = (u32)work->pdev->dev.coherent_dma_mask;
+ start = (u64)(b.wpr_lo >> 4) << 12;
+ end = ((u64)(b.wpr_hi >> 4) << 12) + GU_WPR2_ALIGN;
+
+ dev_err(&work->pdev->dev,
+ "gpu_unlock: failure %s:%u error=%d family=%s payload=%s state=%u index=%zu attempt=%u target=%#x value=%#x last_mmio=%#x:%#x valid=%u bar=%u\n",
+ function, line, error, family, payload, work->state, work->write_index,
+ work->attempt, work->target_addr, work->target_value,
+ last_off, last_val, last_valid, !!work->bar);
+ dev_err(&work->pdev->dev,
+ "gpu_unlock: boot reset_plm=%#x ctl=%#x engine=%#x fhstate=%#x mbox=%#x:%#x acr=%#x csberr=%#x ok=%u\n",
+ b.reset_plm, b.ctl, b.engine, b.fhstate, b.mailbox1, b.mailbox0,
+ b.acr, b.csberr, b.ok);
+ dev_err(&work->pdev->dev,
+ "gpu_unlock: wpr2 lo=%#x hi=%#x start=%#llx end=%#llx frts=%#x:%#x saved=%u\n",
+ b.wpr_lo, b.wpr_hi, (unsigned long long)start,
+ (unsigned long long)end, work->frts_lo, work->frts_hi,
+ work->frts_saved);
+ dev_err(&work->pdev->dev,
+ "gpu_unlock: plm feat=%#x rd=%#x wr=%#x src=%#x rv=%u wv=%u wpr_cfg=%#x wpr=%#x lock=%#x ss0=%#x ss1=%#x\n",
+ feat, feat & 0xfu, (feat >> 4) & 0xfu, feat >> 12,
+ (feat >> 8) & 1u, (feat >> 9) & 1u, wpr_cfg, wpr, lock, ss0, ss1);
+ dev_err(&work->pdev->dev,
+ "gpu_unlock: pci cmd=%#x original=%#x pm=%d devsta=%#x trpnd=%u bars=%#x,%#x,%#x,%#x,%#x,%#x last_pci=%u:%#x:%#x:%d\n",
+ command, work->original_cmd, work->pdev->current_state, devsta,
+ !!(devsta & PCI_EXP_DEVSTA_TRPND), work->bars[0], work->bars[1],
+ work->bars[2], work->bars[3], work->bars[4], work->bars[5],
+ last_pci_cap, last_pci_off, last_pci_val, last_pci_err);
+ dev_err(&work->pdev->dev,
+ "gpu_unlock: resources retained=%u pages=%u/%u dma_saved=%#llx/%#llx dma_now=%#x/%#x firmware_started=%u master=%u quiesced=%u\n",
+ work->retained, work->pages_allocated, work->page_count,
+ (unsigned long long)work->streaming_before,
+ (unsigned long long)work->coherent_before, dma_now, coherent_now,
+ work->firmware_started, work->master, work->state == GU_QUIESCED);
}
-static int falcon_hs(struct unlock_transaction_work *work, const struct falcon_eng *eng,
- u32 code_off, u32 code_size, u32 code_va, u32 code_pa,
- u32 data_off, u32 data_size, u32 data_pa,
- u32 sig_dmem, u32 engine_mask, u32 ucode_id,
- dma_addr_t image, u32 mailbox0, u32 mailbox1,
- bool wait_plm)
+static int alloc_dma(struct unlock_transaction_work *work, const char *name,
+ struct unlock_dma *buf, size_t size)
{
- u32 trans, cpuctl;
- dma_addr_t code_src;
- int ret;
-
- if (image > ~0ull - code_off || image + code_off < code_va ||
- image > ~0ull - data_off)
- return GU_RESULT(-ERANGE);
- code_src = image + code_off - code_va;
- if (!falcon_dma_valid(code_pa, code_va, code_src, code_size) ||
- (data_size && !falcon_dma_valid(data_pa, 0, image + data_off, data_size)))
- return GU_RESULT(-EINVAL);
- ret = falcon_reset(work, eng);
- if (ret)
- return GU_RESULT(ret);
- if (read_bar(work, eng->fbif + 0x24, &trans) || inaccessible(trans))
- return GU_RESULT(-EIO);
- trans |= 1u << 7;
- write_bar(work, eng->fbif + 0x24, trans);
- write_bar(work, eng->base + 0x10c, 0);
- if (read_bar(work, eng->fbif, &trans) || inaccessible(trans))
- return GU_RESULT(-ENODEV);
- trans = (trans & ~7u) | 5u;
- write_bar(work, eng->fbif, trans);
- dma_wmb();
- ret = falcon_dma(work, eng->base, code_pa, code_va, code_src,
- code_size, gpu_unlock_dma_cmd(true));
- if (ret)
- return GU_RESULT(ret);
- if (data_size) {
- ret = falcon_dma(work, eng->base, data_pa, 0, image + data_off, data_size,
- gpu_unlock_dma_cmd(false));
- if (ret)
- return GU_RESULT(ret);
- }
- write_bar(work, eng->falcon2 + 0x210, sig_dmem);
- write_bar(work, eng->falcon2 + 0x19c, engine_mask);
- write_bar(work, eng->falcon2 + 0x198, ucode_id & 0xff);
- write_bar(work, eng->falcon2 + 0x180, 1);
- write_bar(work, eng->base + 0x104, code_va);
- write_bar(work, eng->base + 0x40, mailbox0);
- write_bar(work, eng->base + 0x44, mailbox1);
- if (read_bar(work, eng->base + 0x100, &cpuctl) || inaccessible(cpuctl))
- return GU_RESULT(-EIO);
- dev_info(&work->pdev->dev, "Falcon start: base=%#x cpuctl=%#x code_va=%#x sig_dmem=%#x engine_mask=%#x ucode=%u mailbox=%#x:%#x\n",
- eng->base, cpuctl, code_va, sig_dmem, engine_mask, ucode_id, mailbox1, mailbox0);
- if (cpuctl & (1u << 6))
- write_bar(work, eng->base + 0x130, 1u << 1);
- else
- write_bar(work, eng->base + 0x100, 1u << 1);
- if (wait_plm) {
- ktime_t deadline = ktime_add_us(ktime_get(), 5000000);
-
- for (;;) {
- u32 plm, engine, identity;
-
- if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
- identity != pci_identity(work->pdev) ||
- read_bar(work, eng->base + 0x100, &engine) || engine == ~0u ||
- (engine & 0xffff0000u) == 0xbadf0000u ||
- read_bar(work, GU_FEAT_PLM_OFFSET, &plm))
- return GU_RESULT(-ENODEV);
- /* Cold entry was closed: this transition proves payload execution. */
- if (plm == ~0u && work->plm_entry != ~0u)
- return 0;
- if ((engine & (1u << 4)) || ktime_compare(ktime_get(), deadline) >= 0)
- dev_err(&work->pdev->dev, "early PLM failed: pci=%#x cpuctl=%#x plm=%#x entry=%#x budget_us=5000000\n",
- identity, engine, plm, work->plm_entry);
- if (engine & (1u << 4))
- return GU_RESULT(-EIO);
- if (ktime_compare(ktime_get(), deadline) >= 0)
- return GU_RESULT(-ETIMEDOUT);
- usleep_range(50, 100);
- }
- }
- ret = poll_us(work, eng->base + 0x100, 1u << 4, 1u << 4, 5000000, true);
- if (ret)
- return GU_RESULT(ret);
+ buf->cpu = diag_dma_alloc(work, name, size, &buf->dma, GFP_KERNEL);
+ if (!buf->cpu)
+ return -ENOMEM;
+ buf->size = size;
return 0;
}
-static int load_booter(struct unlock_transaction_work *work)
+static void free_dma(struct device *dev, struct unlock_dma *buf)
{
- const u8 *data;
- u32 magic, version, image_size, sig_bytes;
- struct gpu_unlock_booter_hdr hdr;
- int ret;
-
- ret = request_asset(work, &work->booter_fw, BOOTER_FILE);
- if (ret)
- return GU_RESULT(ret);
- data = work->booter_fw->data;
- if (work->booter_fw->size < 72)
- return GU_RESULT(-EINVAL);
- memcpy(&magic, data, sizeof(magic));
- memcpy(&version, data + 4, sizeof(version));
- if (magic != BOOTER_MAGIC || version != 1)
- return GU_RESULT(-EINVAL);
- memcpy(&hdr, data + 8, sizeof(hdr));
- memcpy(&image_size, data + 8 + sizeof(hdr), sizeof(image_size));
- memcpy(&sig_bytes, data + 12 + sizeof(hdr), sizeof(sig_bytes));
- if (!hdr.ucode_id || hdr.num_apps != 1 || sig_bytes != hdr.sig_bytes ||
- image_size != GU_BOOTER_IMAGE_SIZE ||
- work->booter_fw->size != 72u + image_size + sig_bytes)
- return GU_RESULT(-EINVAL);
- /* Only this pinned production BOOT_FROM_HS geometry is implemented. */
- if (hdr.ucode_id != 3 || hdr.engine_id != 1 || hdr.num_sigs != 2 ||
- sig_bytes != GU_BOOTER_SIG_TOTAL || hdr.os_code_off != 0 ||
- hdr.os_code_size != 256 || hdr.os_data_off != 35328 ||
- hdr.os_data_size != 25088 || hdr.app_code_off != 256 ||
- hdr.app_code_size != 35072 || hdr.app_data_off != 256 ||
- hdr.app_data_size != 0 || hdr.patch_loc != 35344)
- return GU_RESULT(-EINVAL);
- work->booter_hdr = hdr;
- work->booter_image_size = image_size;
- /* Allocation follows the ownership and DMA-mask snapshot, not asset lookup. */
- return 0;
+ if (!buf->cpu)
+ return;
+ dma_free_coherent(dev, buf->size, buf->cpu, buf->dma);
+ *buf = (struct unlock_dma){ };
}
static int patch_booter(struct unlock_transaction_work *work)
{
- const u8 *data = work->booter_fw->data;
+ const struct gpu_unlock_booter_hdr *hdr = &work->booter->hdr;
+ const u8 *data = work->booter->fw->data;
+ u32 image = booter_image_bytes(hdr);
u32 fuse_raw, fuse_ver;
- u32 index = work->booter_hdr.ucode_id - 1;
+ u32 index = hdr->ucode_id - 1;
if (read_bar(work, GU_FUSE_SEC2_UCODE1 + index * 4, &fuse_raw))
return GU_RESULT(-EIO);
fuse_ver = gpu_unlock_fuse_version(fuse_raw);
- dev_info(&work->pdev->dev, "Booter signature: fuse_raw=%#x version=%u signatures=%u size=%u\n",
- fuse_raw, fuse_ver, work->booter_hdr.num_sigs, work->booter_hdr.sig_bytes);
- return GU_RESULT(gpu_unlock_booter_patch(work->booter_cpu, work->booter_image_size,
- &work->booter_hdr,
- data + 72 + work->booter_image_size,
- work->booter_hdr.sig_bytes, fuse_ver));
+ dev_dbg(&work->pdev->dev, "Booter signature: fuse_raw=%#x version=%u signatures=%u size=%u\n",
+ fuse_raw, fuse_ver, hdr->num_sigs, hdr->sig_bytes);
+ return GU_RESULT(gpu_unlock_booter_patch(work->booter_buf.cpu, image, hdr,
+ data + 72 + image, hdr->sig_bytes, fuse_ver));
}
static void free_coherent(struct unlock_transaction_work *work)
{
+ struct unlock_dma *bufs[] = {
+ &work->boot, &work->fwsec, &work->booter_buf,
+ &work->buffer, &work->stock, &work->meta,
+ };
u32 i;
for (i = 0; i < work->pages_allocated; i++)
@@ -496,31 +345,8 @@ static void free_coherent(struct unlock_transaction_work *work)
kvfree(work->pages);
work->pages = NULL;
work->pages_allocated = 0;
- if (work->boot_cpu)
- dma_free_coherent(&work->pdev->dev, work->boot_desc.image_size,
- work->boot_cpu, work->boot_dma);
- work->boot_cpu = NULL;
- if (work->fwsec_cpu) {
- dma_free_coherent(&work->pdev->dev, work->fwsec_size,
- work->fwsec_cpu, work->fwsec_dma);
- work->fwsec_cpu = NULL;
- }
- if (work->booter_cpu)
- dma_free_coherent(&work->pdev->dev, GU_BOOTER_IMAGE_SIZE,
- work->booter_cpu, work->booter_dma);
- if (work->v67_cpu)
- dma_free_coherent(&work->pdev->dev, GU_V67_SIZE, work->v67_cpu,
- work->v67_dma);
- if (work->stock_cpu)
- dma_free_coherent(&work->pdev->dev, work->stock_size,
- work->stock_cpu, work->stock_dma);
- if (work->meta_cpu)
- dma_free_coherent(&work->pdev->dev, GU_WPR_META_SIZE,
- work->meta_cpu, work->meta_dma);
- work->booter_cpu = NULL;
- work->v67_cpu = NULL;
- work->stock_cpu = NULL;
- work->meta_cpu = NULL;
+ for (i = 0; i < ARRAY_SIZE(bufs); i++)
+ free_dma(&work->pdev->dev, bufs[i]);
}
static int unlock_transaction_unwind(struct unlock_transaction_work *work)
@@ -529,18 +355,22 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
int ret = 0;
/* A timeout never authorizes freeing firmware-visible memory. */
- if (work->firmware_started && !work->quiesced) {
+ if (work->firmware_started && work->state != GU_QUIESCED) {
+ pci_clear_master(work->pdev);
+ work->master = false;
work->retained = true;
+ ret = diag_pci_word(work, false, PCI_COMMAND, &now);
+ dev_err(&work->pdev->dev, "handover refused: incomplete transaction; BME_off=%u command=%#x config_result=%d; BAR ownership and DMA buffers retained until power off\n",
+ !ret && now != 0xffff && !(now & PCI_COMMAND_MASTER), now, ret);
return GU_RESULT(-EBUSY);
}
if (work->master) {
pci_clear_master(work->pdev);
work->master = false;
}
- if (work->mapped) {
+ if (work->bar) {
pci_iounmap(work->pdev, work->bar);
work->bar = NULL;
- work->mapped = false;
}
if (work->region) {
pci_release_region(work->pdev, 0);
@@ -551,38 +381,26 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
work->enabled = false;
}
free_coherent(work);
- if (work->booter_fw) {
- release_firmware(work->booter_fw);
- work->booter_fw = NULL;
- }
- if (work->v67_fw) {
- release_firmware(work->v67_fw);
- work->v67_fw = NULL;
- }
- if (work->boot_fw) {
- release_firmware(work->boot_fw);
- work->boot_fw = NULL;
- }
/* Match the driver core's entry condition, not just BAR/mask cleanup.
* Never release unrelated resources to make this check pass. A bound
* driver keeps its own resources and is not probed again.
*/
if (!work->resume && check_devres_empty(work, "handoff"))
ret = -EBUSY;
- if (work->streaming_set && work->streaming_ptr) {
+ if (work->streaming_ptr) {
if (work->pdev->dev.dma_mask != work->streaming_ptr)
return GU_RESULT(-EIO);
if (dma_set_mask(&work->pdev->dev, work->streaming_before) ||
*work->streaming_ptr != work->streaming_before)
ret = -EIO;
- dev_info(&work->pdev->dev, "restore streaming mask: actual=%#llx expected=%#llx\n",
+ dev_dbg(&work->pdev->dev, "restore streaming mask: actual=%#llx expected=%#llx\n",
(unsigned long long)*work->streaming_ptr, (unsigned long long)work->streaming_before);
}
if (work->coherent_set) {
if (dma_set_coherent_mask(&work->pdev->dev, work->coherent_before) ||
work->pdev->dev.coherent_dma_mask != work->coherent_before)
ret = -EIO;
- dev_info(&work->pdev->dev, "restore coherent mask: actual=%#llx expected=%#llx\n",
+ dev_dbg(&work->pdev->dev, "restore coherent mask: actual=%#llx expected=%#llx\n",
(unsigned long long)work->pdev->dev.coherent_dma_mask,
(unsigned long long)work->coherent_before);
}
@@ -599,7 +417,7 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
pci_set_master(work->pdev);
if (diag_pci_word(work, false, PCI_COMMAND, &now))
return GU_RESULT(-EIO);
- dev_info(&work->pdev->dev, "restore PCI command: actual=%#x original=%#x mask=%#x\n",
+ dev_dbg(&work->pdev->dev, "restore PCI command: actual=%#x original=%#x mask=%#x\n",
now, work->original_cmd, GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER);
if ((now & (GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER)) !=
(work->original_cmd & (GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER)))
@@ -607,71 +425,25 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
return GU_RESULT(ret);
}
-static int unlock_transaction_assets(struct unlock_transaction_work *work)
-{
- const struct firmware *desc;
- int ret;
-
- if (!work->resume) {
- ret = check_devres_empty(work, "entry");
- if (ret)
- return ret;
- }
- ret = request_asset(work, &work->v67_fw, V67_FILE);
- if (ret)
- return GU_RESULT(ret);
- if (work->v67_fw->size != GU_V67_SIZE)
- return GU_RESULT(-EBADMSG);
- ret = gpu_unlock_elf_section(work->gsp_fw->data, work->gsp_fw->size,
- ".fwimage", &work->image_span);
- if (ret)
- return GU_RESULT(ret);
- ret = gpu_unlock_elf_section(work->gsp_fw->data, work->gsp_fw->size,
- ".fwsignature_ga10x", &work->sig_span);
- if (ret)
- return GU_RESULT(ret);
- if (!work->image_span.size || work->image_span.size > 128ull << 20 ||
- work->sig_span.size != 4096)
- return GU_RESULT(-EINVAL);
- ret = gpu_unlock_radix_plan(work->image_span.size, &work->radix);
- if (ret)
- return GU_RESULT(ret);
- ret = request_asset(work, &work->boot_fw, BOOT_FILE);
- if (ret)
- return GU_RESULT(ret);
- if (work->boot_fw->size != 24576)
- return GU_RESULT(-EINVAL);
- ret = request_asset(work, &desc, BOOT_DESC_FILE);
- if (ret)
- return GU_RESULT(ret);
- ret = gpu_unlock_boot_desc_parse(desc->data, desc->size,
- work->boot_fw->size, &work->boot_desc);
- dev_info(&work->pdev->dev, "asset geometry: image=%llu signature=%llu boot=%u code=%#x data=%#x manifest=%#x parse=%d\n",
- (unsigned long long)work->image_span.size, (unsigned long long)work->sig_span.size,
- work->boot_desc.image_size, work->boot_desc.code_offset, work->boot_desc.data_offset,
- work->boot_desc.manifest_offset, ret);
- release_firmware(desc);
- return GU_RESULT(ret ? ret : load_booter(work));
-}
-
static int unlock_transaction_layout(struct unlock_transaction_work *work)
{
- struct gpu_unlock_layout_in *in = &work->layout_in;
+ struct gpu_unlock_layout_in input = { };
+ struct gpu_unlock_layout_in *in = &input;
u32 mb, vga, lo, hi, plm;
int ret;
- if (read_bar(work, 0x001183a4, &mb) || !mb || mb == ~0u ||
- read_bar(work, 0x00625f04, &vga) || vga == ~0u ||
- read_bar(work, 0x001fa7c8, &plm) || plm == ~0u || !(plm & 1) ||
- read_bar(work, 0x001fa82c, &lo) || lo == ~0u ||
- read_bar(work, 0x001fa830, &hi) || hi == ~0u)
+ if (read_bar(work, GU_GFW_FB_SIZE_OFFSET, &mb) || !mb || mb == ~0u ||
+ read_bar(work, GU_VGA_WORKSPACE_BASE_OFFSET, &vga) || vga == ~0u ||
+ read_bar(work, GU_MMU_LOCK_CFG_PLM_OFFSET, &plm) || plm == ~0u || !(plm & 1) ||
+ read_bar(work, GU_MMU_LOCK_ADDR_LO_OFFSET, &lo) || lo == ~0u ||
+ read_bar(work, GU_MMU_LOCK_ADDR_HI_OFFSET, &hi) || hi == ~0u)
return GU_RESULT(-ENODEV);
/* Keep the cold board inputs in the USB kernel log even on layout failure.
* Register values after NVIDIA binds cannot reconstruct this state. */
- dev_info(&work->pdev->dev,
+ dev_dbg(&work->pdev->dev,
"layout inputs: fb_mb=%#x vga=%#x mmu_plm=%#x mmu_lo=%#x mmu_hi=%#x image=%llu boot=%u\n",
- mb, vga, plm, lo, hi, (unsigned long long)work->image_span.size,
- work->boot_desc.image_size);
+ mb, vga, plm, lo, hi, (unsigned long long)work->bundle->image_span.size,
+ work->bundle->boot_desc.image_size);
in->usable_fb = (u64)mb << 20;
if (in->usable_fb < 256ull << 20) {
dev_err(&work->pdev->dev, "layout rejected: usable FB below 256 MiB\n");
@@ -702,24 +474,24 @@ static int unlock_transaction_layout(struct unlock_transaction_work *work)
in->mmu_lock_valid = mmu_hi >= in->mmu_lock_low;
top_lock = in->mmu_lock_valid && (mmu_top >> 20) == (in->usable_fb >> 20);
- dev_info(&work->pdev->dev,
+ dev_dbg(&work->pdev->dev,
"layout mmu: lo=%#llx hi=%#llx hi_exclusive=%#llx constrain=%u\n",
(unsigned long long)in->mmu_lock_low, (unsigned long long)mmu_hi,
(unsigned long long)mmu_top, top_lock);
if (!top_lock) {
- dev_info(&work->pdev->dev,
+ dev_dbg(&work->pdev->dev,
"mmu lock ignored: high address is not the FB top\n");
in->mmu_lock_valid = false;
}
}
in->frts_size = 1ull << 20;
- in->boot_size = work->boot_desc.image_size;
- in->image_size = work->image_span.size;
+ in->boot_size = work->bundle->boot_desc.image_size;
+ in->image_size = work->bundle->image_span.size;
in->nonwpr_size = 1ull << 20;
/* GA102's supported minimum heap; no GSP-RM clients are started here. */
in->heap_request = 88ull << 20;
ret = gpu_unlock_layout(in, &work->layout);
- dev_info(&work->pdev->dev,
+ dev_dbg(&work->pdev->dev,
"layout decoded: fb=%#llx vga=%#llx mmu_valid=%u mmu_low=%#llx frts=%#llx heap=%#llx result=%d\n",
(unsigned long long)in->usable_fb, (unsigned long long)in->vga_base,
in->mmu_lock_valid, (unsigned long long)in->mmu_lock_low,
@@ -728,7 +500,7 @@ static int unlock_transaction_layout(struct unlock_transaction_work *work)
dev_err(&work->pdev->dev, "layout rejected: reservation calculation (%d)\n", ret);
return GU_RESULT(ret);
}
- dev_info(&work->pdev->dev,
+ dev_dbg(&work->pdev->dev,
"layout ranges: nonwpr=%#llx wpr=%#llx heap=%#llx gsp=%#llx boot=%#llx frts=%#llx end=%#llx scrub_floor=%#llx\n",
(unsigned long long)work->layout.nonwpr_start,
(unsigned long long)work->layout.wpr_start,
@@ -743,12 +515,24 @@ static int unlock_transaction_layout(struct unlock_transaction_work *work)
dev_err(&work->pdev->dev, "layout rejected: reservations below pre-scrubbed top 256 MiB\n");
return GU_RESULT(-ERANGE);
}
+ work->usable_fb = in->usable_fb;
+ work->vga_base = in->vga_base;
+ work->frts_size = in->frts_size;
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=layout fb=%#llx vga=%#llx frts=%#llx wpr=%#llx..%#llx boot=%#llx gsp=%#llx heap=%#llx\n",
+ (unsigned long long)work->usable_fb, (unsigned long long)work->vga_base,
+ (unsigned long long)work->frts_size,
+ (unsigned long long)work->layout.wpr_start,
+ (unsigned long long)work->layout.wpr_end,
+ (unsigned long long)work->layout.boot_offset,
+ (unsigned long long)work->layout.gsp_offset,
+ (unsigned long long)work->layout.heap_offset);
return 0;
}
static int unlock_transaction_radix(struct unlock_transaction_work *work)
{
- const struct gpu_unlock_radix_plan *r = &work->radix;
+ const struct gpu_unlock_radix_plan *r = &work->bundle->radix;
u64 *addresses;
u32 i, level, first = 0;
int ret = 0;
@@ -774,9 +558,9 @@ static int unlock_transaction_radix(struct unlock_transaction_work *work)
memset(page->cpu, 0, GU_RADIX_PAGE);
if (i >= r->table_pages) {
u64 offset = (i - r->table_pages) * GU_RADIX_PAGE;
- size_t n = min_t(u64, GU_RADIX_PAGE, work->image_span.size - offset);
+ size_t n = min_t(u64, GU_RADIX_PAGE, work->bundle->image_span.size - offset);
- memcpy(page->cpu, work->gsp_fw->data + work->image_span.offset + offset, n);
+ memcpy(page->cpu, work->bundle->gsp->data + work->bundle->image_span.offset + offset, n);
}
}
for (level = 0; level < 3; level++) {
@@ -805,7 +589,7 @@ static int unlock_transaction_claim(struct unlock_transaction_work *work)
unsigned int i;
int ret;
- dev_info(&work->pdev->dev, "claim: power=%d pm_cap=%#x flags=%#x\n",
+ dev_dbg(&work->pdev->dev, "claim: power=%d pm_cap=%#x flags=%#x\n",
work->pdev->current_state, work->pdev->pm_cap, work->pdev->dev_flags);
/* Older kernels (seen on 6.8 and 6.14) leave a device no driver has
* enabled at PCI_UNKNOWN. Accept that only when PMCSR confirms D0. */
@@ -833,13 +617,13 @@ static int unlock_transaction_claim(struct unlock_transaction_work *work)
diag_pci_word(work, false, PCI_COMMAND, &command) ||
command == 0xffff || (command & PCI_COMMAND_MASTER))
return GU_RESULT(-EBUSY);
- dev_info(&work->pdev->dev, "claim: cleared %s BME\n",
+ dev_dbg(&work->pdev->dev, "claim: cleared %s BME\n",
work->resume ? "driver" : "firmware");
/* Firmware's BME stays off; the suspended driver gets its own back. */
if (!work->resume)
work->original_cmd = command;
}
- dev_info(&work->pdev->dev, "claim: command=%#x capability=%#x BAR_mask=%#x\n",
+ dev_dbg(&work->pdev->dev, "claim: command=%#x capability=%#x BAR_mask=%#x\n",
command, cap, pci_select_bars(work->pdev, IORESOURCE_MEM));
/* Only BAR0 is mapped. On the display card the firmware framebuffer keeps
* BAR1 until just before the FLR, as it does while the stock driver boots.
@@ -858,7 +642,7 @@ static int unlock_transaction_claim(struct unlock_transaction_work *work)
work->enabled = true;
}
work->bar_len = pci_resource_len(work->pdev, 0);
- if (work->bar_len < SEC2_FALCON2 + 0x1000)
+ if (work->bar_len < GU_SEC2_FALCON2_BASE + GU_FALCON2_WINDOW_SIZE)
return GU_RESULT(-EINVAL);
for (i = 0; i < ARRAY_SIZE(work->bars); i++)
if (diag_pci_dword(work, false, PCI_BASE_ADDRESS_0 + i * 4, &work->bars[i]))
@@ -866,12 +650,16 @@ static int unlock_transaction_claim(struct unlock_transaction_work *work)
ret = pci_save_state(work->pdev);
if (ret)
return GU_RESULT(ret);
- if (work->mapped)
+ if (work->bar)
return 0;
work->bar = pci_iomap(work->pdev, 0, 0);
if (!work->bar)
return GU_RESULT(-ENOMEM);
- work->mapped = true;
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=claim device=%#x family=%s cmd=%#x original=%#x bar_len=%#llx region=%u enabled=%u master=%u\n",
+ work->pdev->device, work->card->family->name, command,
+ work->original_cmd, (unsigned long long)work->bar_len,
+ work->region, work->enabled, work->master);
return 0;
}
@@ -879,108 +667,114 @@ static int unlock_transaction_dma(struct unlock_transaction_work *work)
{
struct device *dev = &work->pdev->dev;
- if (!dev->dma_mask || work->masks_saved)
+ if (!dev->dma_mask || work->streaming_ptr)
return GU_RESULT(-EINVAL);
- work->masks_saved = true;
work->streaming_ptr = dev->dma_mask;
work->streaming_before = *dev->dma_mask;
work->coherent_before = dev->coherent_dma_mask;
- dev_info(dev, "DMA masks: streaming=%#llx coherent=%#llx requested=%#llx\n",
+ dev_dbg(dev, "DMA masks: streaming=%#llx coherent=%#llx requested=%#llx\n",
(unsigned long long)work->streaming_before, (unsigned long long)work->coherent_before,
(unsigned long long)DMA_BIT_MASK(32));
- work->streaming_set = true;
if (GU_DIAG(dma_set_mask(dev, DMA_BIT_MASK(32))))
return GU_RESULT(-EIO);
work->coherent_set = true;
if (GU_DIAG(dma_set_coherent_mask(dev, DMA_BIT_MASK(32))))
return GU_RESULT(-EIO);
- work->booter_cpu = diag_dma_alloc(work, "booter", work->booter_image_size,
- &work->booter_dma, GFP_KERNEL);
- if (!work->booter_cpu)
+ if (alloc_dma(work, "booter", &work->booter_buf, booter_image_bytes(&work->booter->hdr)))
return GU_RESULT(-ENOMEM);
- memcpy(work->booter_cpu, work->booter_fw->data + 72, work->booter_image_size);
- work->v67_cpu = diag_dma_alloc(work, "v67", GU_V67_SIZE, &work->v67_dma, GFP_KERNEL);
- work->meta_cpu = diag_dma_alloc(work, "metadata", GU_WPR_META_SIZE, &work->meta_dma, GFP_KERNEL);
- if (!work->v67_cpu || !work->meta_cpu)
+ memcpy(work->booter_buf.cpu, work->booter->fw->data + 72, work->booter_buf.size);
+ if (alloc_dma(work, "buffer", &work->buffer, work->payload->size) ||
+ alloc_dma(work, "metadata", &work->meta, GU_WPR_META_SIZE))
return GU_RESULT(-ENOMEM);
- memcpy(work->v67_cpu, work->v67_fw->data, GU_V67_SIZE);
- work->stock_size = work->sig_span.size;
- work->stock_cpu = diag_dma_alloc(work, "stock-signature", work->stock_size, &work->stock_dma, GFP_KERNEL);
- work->boot_cpu = diag_dma_alloc(work, "boot-image", work->boot_desc.image_size, &work->boot_dma, GFP_KERNEL);
- if (!work->stock_cpu || !work->boot_cpu)
+ /* No launch precedes the first table entry, so the slots stay unused
+ * until run_privileged_writes() builds the image for its write. */
+ {
+ int ret = gu_payload_build(work->payload, work->buffer.cpu, 0, 0);
+
+ if (ret)
+ return GU_RESULT(ret);
+ }
+ if (alloc_dma(work, "stock-signature", &work->stock, work->bundle->sig_span.size) ||
+ alloc_dma(work, "boot-image", &work->boot, work->bundle->boot_desc.image_size))
return GU_RESULT(-ENOMEM);
- memcpy(work->stock_cpu, work->gsp_fw->data + work->sig_span.offset, work->stock_size);
- memcpy(work->boot_cpu, work->boot_fw->data, work->boot_desc.image_size);
+ memcpy(work->stock.cpu, work->bundle->gsp->data + work->bundle->sig_span.offset,
+ work->stock.size);
+ memcpy(work->boot.cpu, work->bundle->boot->data, work->boot.size);
{
int ret = patch_booter(work);
- return GU_RESULT(ret ? ret : unlock_transaction_radix(work));
+ ret = ret ? ret : unlock_transaction_radix(work);
+ if (!ret)
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=dma booter=%zu payload=%#x pages=%u meta=%#llx sig=%#llx stock=%#llx boot=%#llx\n",
+ work->booter_buf.size, work->payload->size,
+ work->page_count, (unsigned long long)work->meta.dma,
+ (unsigned long long)work->buffer.dma,
+ (unsigned long long)work->stock.dma,
+ (unsigned long long)work->boot.dma);
+ return GU_RESULT(ret);
}
}
static int unlock_transaction_fwsec(struct unlock_transaction_work *work)
{
- static const struct falcon_eng gsp = {
- .base = 0x00110000, .fbif = 0x00110600, .falcon2 = 0x00111000,
+ static const struct gu_falcon_engine gsp = {
+ .base = GU_GSP_BASE, .fbif = GU_GSP_FBIF_BASE, .falcon2 = GU_GSP_FALCON2_BASE,
};
+ struct gu_falcon_io io = falcon_io(work);
size_t mapped = 0;
u8 *rom = NULL;
struct gpu_unlock_fwsec fw;
- u32 fuse_raw, version, scratch, wpr_lo, wpr_hi;
+ u32 fuse_raw = 0, version = 0, scratch = 0, wpr_lo = 0, wpr_hi = 0;
u16 command;
+ struct gu_falcon_evidence launch = { };
u64 frts = work->layout.frts_offset;
- void *image = NULL;
- dma_addr_t image_dma = 0;
int ret;
/* kgspExtractVbiosFromRom_TU102 reads NV_PROM_DATA, up to 1 MiB.
* pci_map_rom stops at the PCI last-image flag and hides the VN extension. */
- dev_info(&work->pdev->dev, "FWSEC: reading PROM at %#x\n", 0x300000);
- if (work->bar_len < 0x300000 + 0x1a)
+ dev_dbg(&work->pdev->dev, "FWSEC: reading PROM at %#x\n", GU_PROM_DATA_OFFSET);
+ if (work->bar_len < GU_PROM_DATA_OFFSET + 0x1a)
return GU_RESULT(-ENODEV);
- mapped = min_t(size_t, 1024 * 1024, work->bar_len - 0x300000);
+ mapped = min_t(size_t, 1024 * 1024, work->bar_len - GU_PROM_DATA_OFFSET);
rom = kvmalloc(mapped, GFP_KERNEL);
if (!rom)
return GU_RESULT(-ENOMEM);
- memcpy_fromio(rom, work->bar + 0x300000, mapped);
- dev_info(&work->pdev->dev, "FWSEC: PROM bytes=%zu signature=%#x\n",
+ memcpy_fromio(rom, work->bar + GU_PROM_DATA_OFFSET, mapped);
+ dev_dbg(&work->pdev->dev, "FWSEC: PROM bytes=%zu signature=%#x\n",
mapped, (unsigned int)(rom[0] | (rom[1] << 8)));
ret = gpu_unlock_vbios_fwsec(rom, mapped, &fw);
if (ret) {
dev_err(&work->pdev->dev, "FWSEC-FRTS not found in board ROM (%d)\n", ret);
goto out;
}
- if (fw.engine_mask != 0x400) {
+ if (fw.version != work->card->family->fwsec_version ||
+ (fw.version == 3 && fw.engine_mask != 0x400)) {
dev_err(&work->pdev->dev, "FWSEC: unsupported GSP engine mask=%#x\n", fw.engine_mask);
ret = -EOPNOTSUPP;
goto out;
}
- image = diag_dma_alloc(work, "fwsec", fw.image_size, &image_dma, GFP_KERNEL);
- if (!image) {
+ if (alloc_dma(work, "fwsec", &work->fwsec, fw.image_size)) {
ret = -ENOMEM;
goto out;
}
- work->fwsec_cpu = image;
- work->fwsec_dma = image_dma;
- work->fwsec_size = fw.image_size;
- memcpy(image, rom + fw.image_off, fw.image_size);
- ret = gpu_unlock_fwsec_patch_frts(image, &fw, frts);
+ memcpy(work->fwsec.cpu, rom + fw.image_off, fw.image_size);
+ ret = gpu_unlock_fwsec_patch_frts(work->fwsec.cpu, &fw, frts);
if (ret)
goto out;
- if (!fw.ucode_id ||
- read_bar(work, 0x008241c0 + (fw.ucode_id - 1) * 4, &fuse_raw)) {
- ret = -EIO;
- goto out;
- }
- version = gpu_unlock_fuse_version(fuse_raw);
- dev_info(&work->pdev->dev, "FWSEC fuse: raw=%#x version=%u\n", fuse_raw, version);
- {
+ if (fw.version == 3) {
u32 sig_at = 0;
-
+ if (!fw.ucode_id ||
+ read_bar(work, GU_FUSE_GSP_UCODE1 + (fw.ucode_id - 1) * 4, &fuse_raw)) {
+ ret = -EIO;
+ goto out;
+ }
+ version = gpu_unlock_fuse_version(fuse_raw);
+ dev_dbg(&work->pdev->dev, "FWSEC fuse: raw=%#x version=%u\n", fuse_raw, version);
ret = gpu_unlock_fwsec_sig_offset(fw.sig_versions, version, 384, &sig_at);
if (ret)
goto out;
- memcpy((u8 *)image + fw.data_off + fw.pkc_off, rom + fw.sig_off + sig_at, 384);
+ memcpy((u8 *)work->fwsec.cpu + fw.data_off + fw.pkc_off, rom + fw.sig_off + sig_at, 384);
}
pci_set_master(work->pdev);
work->master = true;
@@ -989,37 +783,87 @@ static int unlock_transaction_fwsec(struct unlock_transaction_work *work)
ret = -EIO;
goto out;
}
+ /* Preserve conservative DMA retention from the moment FWSEC may access it,
+ * including reset/load failures. Launch evidence is observed separately. */
work->firmware_started = true;
- ret = falcon_hs(work, &gsp, 0, fw.imem_size, fw.imem_va, fw.imem_pa,
- fw.data_off, fw.dmem_size, fw.dmem_pa, fw.pkc_off,
- fw.engine_mask, fw.ucode_id,
- image_dma, 0, 0, false);
+ if (fw.version == 2) {
+ const struct gu_falcon_loader_image program = {
+ .loader = work->loader_fw ? (const u32 *)work->loader_fw->data : NULL,
+ .loader_size = work->loader_fw ? work->loader_fw->size : 0,
+ .dma = work->fwsec.dma, .data_off = fw.data_off,
+ .imem_pa = fw.imem_pa, .imem_ns_size = fw.imem_ns_size,
+ .imem_sec_pa = fw.imem_sec_pa, .imem_sec_size = fw.imem_sec_size,
+ .dmem_size = fw.dmem_size,
+ };
+ ret = gu_falcon_loader(&io, &gsp, &program, &launch);
+ } else {
+ const struct gu_falcon_hs_image program = {
+ .dma = work->fwsec.dma, .code_off = 0, .code_size = fw.imem_size,
+ .code_va = fw.imem_va, .code_pa = fw.imem_pa,
+ .data_off = fw.data_off, .data_size = fw.dmem_size,
+ .data_pa = fw.dmem_pa, .sig_dmem = fw.pkc_off,
+ .engine_mask = fw.engine_mask, .ucode_id = fw.ucode_id,
+ /* The WPR2 check below proves FWSEC-FRTS ran. */
+ .stale_halt_ok = true,
+ };
+ ret = gu_falcon_hs(&io, &gsp, &program, &launch);
+ }
if (ret)
goto out;
- if (read_bar(work, 0x00001400 + 0x0e * 4, &scratch)) {
+ if (read_bar(work, GU_ACR_SCRATCH_OFFSET, &scratch)) {
ret = -EIO;
goto out;
}
- dev_info(&work->pdev->dev, "FWSEC result: scratch=%#x expected_hi16=0\n", scratch);
+ dev_dbg(&work->pdev->dev,
+ "FWSEC result: scratch=%#x expected_hi16=0 started=%u halted=%u reset=%u/%u reloaded=%u\n",
+ scratch, launch.started, launch.halted, launch.reset_asserted,
+ launch.reset_released, launch.reloaded);
if (scratch >> 16) {
ret = -EIO;
goto out;
}
- if (read_bar(work, 0x001fa824, &wpr_lo) || wpr_lo == ~0u ||
- read_bar(work, 0x001fa828, &wpr_hi) || wpr_hi == ~0u) {
+ if (read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &wpr_lo) || wpr_lo == ~0u ||
+ read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &wpr_hi) || wpr_hi == ~0u) {
ret = -EIO;
goto out;
}
- dev_info(&work->pdev->dev, "FWSEC WPR2: low=%#x high=%#x expected_page=%#llx end=%#llx\n",
+ dev_dbg(&work->pdev->dev, "FWSEC WPR2: low=%#x high=%#x expected_page=%#llx end=%#llx\n",
wpr_lo, wpr_hi, (unsigned long long)(frts >> 12),
(unsigned long long)work->layout.wpr_end);
/* Match NVIDIA's initialized-WPR2 check as well as the expected low
- * address. Never publish Booter metadata for disabled/alien WPR2. */
+ * address. Never publish Booter metadata for disabled/alien WPR2. FWSEC
+ * sizes the region itself (0xe1000 bytes on GA102), so the high end is
+ * only bounded by the reservation. */
if (!(wpr_hi >> 4) || (wpr_hi >> 4) < (wpr_lo >> 4) ||
((u64)(wpr_hi >> 4) << 12) >= work->layout.wpr_end ||
(wpr_lo >> 4) != (u32)(frts >> 12))
ret = -EIO;
+ if (!ret) {
+ u64 frts_end = work->layout.frts_offset + work->frts_size;
+ u32 booter_hi;
+
+ /* FWSEC publishes the inclusive 4 KiB end; the Booter decodes
+ * WPR2_ADDR_HI with a 128 KiB alignment term. Store the Booter
+ * encoding so restore_frts() writes what the next HS launch
+ * re-validates, not the FWSEC convention. */
+ if (frts_end < GU_WPR2_ALIGN || (frts_end & (GU_WPR2_ALIGN - 1))) {
+ ret = -EINVAL;
+ } else {
+ booter_hi = (u32)(((frts_end - GU_WPR2_ALIGN) >> 12) << 4);
+ dev_dbg(&work->pdev->dev,
+ "FWSEC WPR2 restore encoding: raw_hi=%#x booter_hi=%#x frts_end=%#llx\n",
+ wpr_hi, booter_hi, (unsigned long long)frts_end);
+ work->frts_lo = wpr_lo;
+ work->frts_hi = booter_hi;
+ work->frts_saved = true;
+ }
+ }
out:
+ if (!ret)
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=fwsec version=%u fuse=%#x ver=%u scratch=%#x wpr2_raw=%#x:%#x frts=%#x:%#x\n",
+ fw.version, fuse_raw, version, scratch, wpr_lo, wpr_hi,
+ work->frts_lo, work->frts_hi);
/* Ownership transfers to work before publication, including failure paths. */
kvfree(rom);
return GU_RESULT(ret);
@@ -1043,17 +887,20 @@ static int unlock_transaction_gfw(struct unlock_transaction_work *work)
sample.config_ok = true;
assessed = gpu_unlock_gfw_assess(&sample);
if (!assessed) {
- u32 ss0, ss1, wpr_hi;
+ u32 ss0, ss1, wpr_hi, plm;
- if (read_bar(work, GU_FEAT_PLM_OFFSET, &work->plm_entry) ||
+ if (read_bar(work, GU_FEAT_PLM_OFFSET, &plm) ||
read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || ss0 == ~0u ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || ss1 == ~0u ||
- read_bar(work, 0x001fa828, &wpr_hi) || inaccessible(wpr_hi))
+ read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &wpr_hi) || inaccessible(wpr_hi))
return GU_RESULT(-ENODEV);
- dev_info(&work->pdev->dev, "entry: plm=%#x ss0=%#x ss1=%#x wpr2_hi=%#x\n",
- work->plm_entry, ss0, ss1, wpr_hi);
+ dev_dbg(&work->pdev->dev, "entry: plm=%#x ss0=%#x ss1=%#x wpr2_hi=%#x\n",
+ plm, ss0, ss1, wpr_hi);
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=gfw plm=%#x ss0=%#x ss1=%#x wpr2_hi=%#x\n",
+ plm, ss0, ss1, wpr_hi);
/* An open PLM means the card was already unlocked or not cold. */
- if (work->plm_entry == ~0u)
+ if (plm == ~0u)
return GU_RESULT(-EALREADY);
/* WPR2 is down from power-on until FWSEC-FRTS runs. An enabled
* WPR2 means a driver has already used the card this power
@@ -1087,21 +934,23 @@ static int unlock_transaction_suspended(struct unlock_transaction_work *work)
!(command & PCI_COMMAND_MEMORY))
return GU_RESULT(-ENODEV);
work->bar_len = pci_resource_len(work->pdev, 0);
- if (work->bar_len < SEC2_FALCON2 + 0x1000)
+ if (work->bar_len < GU_SEC2_FALCON2_BASE + GU_FALCON2_WINDOW_SIZE)
return GU_RESULT(-EINVAL);
work->bar = pci_iomap(work->pdev, 0, 0);
if (!work->bar)
return GU_RESULT(-ENOMEM);
- work->mapped = true;
/* Read before GFW completes, so an unreadable value proves nothing yet. */
- if (read_bar(work, 0x001fa828, &wpr_hi) ||
+ if (read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &wpr_hi) ||
read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1))
return GU_RESULT(-EIO);
- dev_info(&work->pdev->dev, "resume entry: wpr2_hi=%#x ss0=%#x ss1=%#x\n",
+ dev_dbg(&work->pdev->dev, "resume entry: wpr2_hi=%#x ss0=%#x ss1=%#x\n",
wpr_hi, ss0, ss1);
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=resume-entry wpr2_hi=%#x ss0=%#x ss1=%#x cmd=%#x\n",
+ wpr_hi, ss0, ss1, command);
if (!inaccessible(wpr_hi) && (wpr_hi >> 4)) {
- dev_info(&work->pdev->dev, "resume: GSP-RM already running (WPR2 enabled); card left as is\n");
+ dev_dbg(&work->pdev->dev, "resume: GSP-RM already running (WPR2 enabled); card left as is\n");
return -EBUSY;
}
/* Sleep that kept the card powered (suspend to idle, an aborted suspend). */
@@ -1112,18 +961,95 @@ static int unlock_transaction_suspended(struct unlock_transaction_work *work)
static int unlock_transaction_boot(struct unlock_transaction_work *work)
{
- static const struct falcon_eng sec2 = {
- .base = SEC2_BASE, .fbif = SEC2_FBIF, .falcon2 = SEC2_FALCON2,
+ static const struct gu_falcon_engine sec2 = {
+ .base = GU_SEC2_BASE, .fbif = GU_SEC2_FBIF_BASE, .falcon2 = GU_SEC2_FALCON2_BASE,
};
- const struct gpu_unlock_booter_hdr *h = &work->booter_hdr;
-
- if (!work->published || !work->master || !work->firmware_started)
+ const struct gpu_unlock_booter_hdr *h = &work->booter->hdr;
+ struct gu_falcon_io io = falcon_io(work);
+ struct gu_falcon_evidence launch = { };
+ int ret;
+ u32 ctl, reset_plm, engine, scratch, identity, mailbox0, mailbox1;
+ u32 wpr_lo = 0, wpr_hi = 0, feat = 0, wpr_cfg = 0, csberr = 0, fhstate = 0;
+ bool fresh, cleanup_complete;
+ if ((work->state != GU_PUBLISHED && work->state != GU_ACTIVE) || !work->master || !work->firmware_started)
return GU_RESULT(-EINVAL);
- return GU_RESULT(falcon_hs(work, &sec2, h->app_code_off, h->app_code_size,
- h->app_code_off, 0, h->os_data_off, h->os_data_size, 0,
- h->patch_loc - h->os_data_off, h->engine_id, h->ucode_id,
- work->booter_dma, lower_32_bits(work->meta_dma),
- upper_32_bits(work->meta_dma), true));
+ /* This Booter build consumes the WprMeta sysmem address as a 64-bit
+ * mailbox pair: MAILBOX0 low, MAILBOX1 high. The reference exit leaves
+ * the same pair in the mailboxes, which is the completion echo. */
+ if (work->card->family->booter->launch == GU_SEC2_DIRECT) {
+ const struct gu_falcon_direct_image program = {
+ .words = work->booter_buf.cpu,
+ .os_code_off = h->os_code_off, .os_code_size = h->os_code_size,
+ .app_code_off = h->app_code_off, .app_code_size = h->app_code_size,
+ .os_data_off = h->os_data_off, .os_data_size = h->os_data_size,
+ .mailbox0 = lower_32_bits(work->meta.dma),
+ .mailbox1 = upper_32_bits(work->meta.dma),
+ };
+ ret = gu_falcon_direct(&io, &sec2, &program, &launch);
+ } else {
+ const struct gu_falcon_hs_image program = {
+ .dma = work->booter_buf.dma,
+ .code_off = h->app_code_off, .code_size = h->app_code_size,
+ .code_va = h->app_code_off, .code_pa = 0,
+ .os_code_off = h->os_code_off, .os_code_size = h->os_code_size,
+ .data_off = h->os_data_off, .data_size = h->os_data_size, .data_pa = 0,
+ .sig_dmem = h->patch_loc - h->os_data_off,
+ .engine_mask = h->engine_id, .ucode_id = h->ucode_id,
+ .mailbox0 = lower_32_bits(work->meta.dma),
+ .mailbox1 = upper_32_bits(work->meta.dma),
+ /* The runner reads back each target register. */
+ .stale_halt_ok = true,
+ };
+ ret = gu_falcon_hs(&io, &sec2, &program, &launch);
+ }
+ /* Always capture the first failure, including successful write/no halt. */
+ if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
+ identity != pci_identity(work->pdev) ||
+ read_bar(work, GU_SEC2_BASE + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl) ||
+ read_bar(work, GU_SEC2_BASE + GU_FALCON_MAILBOX0_OFFSET, &mailbox0) ||
+ read_bar(work, GU_SEC2_BASE + GU_FALCON_MAILBOX1_OFFSET, &mailbox1) ||
+ read_bar(work, GU_SEC2_RESET_PLM_OFFSET, &reset_plm) ||
+ read_bar(work, GU_SEC2_BASE + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine) ||
+ read_bar(work, GU_ACR_SCRATCH_OFFSET, &scratch))
+ return GU_RESULT(-ENODEV);
+ read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &wpr_lo);
+ read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &wpr_hi);
+ read_bar(work, GU_FEAT_PLM_OFFSET, &feat);
+ read_bar(work, GU_WPR_CFG_PLM_OFFSET, &wpr_cfg);
+ read_bar(work, GU_SEC2_BASE + GU_SEC2_CSBERRSTAT_OFFSET, &csberr);
+ read_bar(work, GU_SEC2_BASE + GU_FALCON_FHSTATE_OFFSET, &fhstate);
+ dev_dbg(&work->pdev->dev,
+ "completion diagnostics: engine=%#x acr_scratch=%#x csberr=%#x fhstate=%#x wpr2=%#x:%#x plm_feat=%#x plm_wpr_cfg=%#x target=%#x:%#x secure_bus_owner=unavailable token_return=unobserved dmem_cleanup=unobserved\n",
+ engine, scratch, csberr, fhstate, wpr_hi, wpr_lo, feat, wpr_cfg,
+ work->target_addr, work->target_value);
+ work->invocation_halted = !!(ctl & 16);
+ fresh = launch.reset_asserted && launch.reset_released &&
+ launch.reloaded && launch.started && work->invocation_halted;
+ /* The V67 exit returns the input metadata pointer in the mailboxes; the
+ * descriptor records that because it is a payload property, not a Booter
+ * or hardware convention. Payloads without that evidence are accepted on
+ * the generic halt/RESET_PLM predicate and their mailboxes are logged. */
+ cleanup_complete = fresh && (reset_plm & 0xff) == 0xff;
+ if (cleanup_complete &&
+ work->payload->completion == GU_PAYLOAD_HALT_RESET_PLM_ECHO)
+ cleanup_complete = mailbox0 == lower_32_bits(work->meta.dma) &&
+ mailbox1 == upper_32_bits(work->meta.dma);
+ dev_dbg(&work->pdev->dev,
+ "invocation: index=%zu attempt=%u reset=%u/%u reload=%u start=%u cpuctl=%#x engine=%#x reset_plm@%#x=%#x mailbox=%#x:%#x acr=%#x csberr=%#x fhstate=%#x wpr2=%#x:%#x plm_feat=%#x plm_wpr_cfg=%#x fresh=%u cleanup_observed=%u result=%d\n",
+ work->write_index, work->attempt, launch.reset_asserted,
+ launch.reset_released, launch.reloaded, launch.started, ctl,
+ engine, GU_SEC2_RESET_PLM_OFFSET, reset_plm, mailbox1, mailbox0,
+ scratch, csberr, fhstate, wpr_hi, wpr_lo, feat, wpr_cfg,
+ fresh, cleanup_complete, ret);
+ if (ret || !cleanup_complete)
+ dev_err(&work->pdev->dev,
+ "invocation failed: index=%zu attempt=%u cpuctl=%#x engine=%#x reset_plm=%#x mailbox=%#x:%#x acr=%#x csberr=%#x fhstate=%#x wpr2=%#x:%#x plm_feat=%#x plm_wpr_cfg=%#x target=%#x:%#x reset=%u/%u reload=%u start=%u fresh=%u cleanup=%u error=%d\n",
+ work->write_index, work->attempt, ctl, engine, reset_plm,
+ mailbox1, mailbox0, scratch, csberr, fhstate, wpr_hi, wpr_lo,
+ feat, wpr_cfg, work->target_addr, work->target_value,
+ launch.reset_asserted, launch.reset_released, launch.reloaded,
+ launch.started, fresh, cleanup_complete, ret ? ret : -EIO);
+ return GU_RESULT(ret ? ret : cleanup_complete ? 0 : -EIO);
}
static void put_le64(u8 *dest, u64 value)
@@ -1134,38 +1060,187 @@ static void put_le64(u8 *dest, u64 value)
dest[i] = (u8)(value >> (8 * i));
}
-static int unlock_transaction_signature(struct unlock_transaction_work *work)
+static int publish_signature(struct unlock_transaction_work *work)
{
- u8 *meta = work->meta_cpu;
+ struct gpu_unlock_wpr_args args = {
+ .fb_size = work->usable_fb,
+ .vga_offset = work->vga_base,
+ .frts_size = work->frts_size,
+ .image_size = work->bundle->image_span.size,
+ .boot_dma = work->boot.dma, .boot = work->bundle->boot_desc,
+ };
+ int ret;
- if (!meta || !work->stock_cpu || !work->pages_allocated ||
- work->signature_calls >= 2)
+ args.signature_dma = work->buffer.dma;
+ args.signature_size = work->payload->size;
+ if (!work->meta.cpu || !work->stock.cpu || !work->pages_allocated ||
+ ((work->buffer.dma | work->stock.dma | work->meta.dma) & 255) ||
+ work->state != GU_ALLOCATED)
return GU_RESULT(-EINVAL);
- if (work->signature_calls == 0) {
- struct gpu_unlock_wpr_args args = {
- .fb_size = work->layout_in.usable_fb,
- .vga_offset = work->layout_in.vga_base,
- .frts_size = work->layout_in.frts_size,
- .signature_dma = work->v67_dma, .signature_size = GU_V67_SIZE,
- .radix_dma = work->pages[0].dma,
- .image_size = work->image_span.size,
- .boot_dma = work->boot_dma, .boot = work->boot_desc,
- };
- int ret = gpu_unlock_wpr_meta_pack(meta, &work->layout, &args);
+ args.radix_dma = work->pages[0].dma;
+ ret = gpu_unlock_wpr_meta_pack(work->meta.cpu, &work->layout, &args);
+ if (ret)
+ return GU_RESULT(ret);
+ dma_wmb();
+ work->state = GU_PUBLISHED;
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=publish meta=%#llx sig=%#llx size=%#x radix=%#llx\n",
+ (unsigned long long)work->meta.dma,
+ (unsigned long long)work->buffer.dma, work->payload->size,
+ (unsigned long long)work->pages[0].dma);
+ return 0;
+}
- if (ret)
- return GU_RESULT(ret);
- dma_wmb();
- work->published = true;
- } else {
- put_le64(meta + 72, work->stock_dma);
- put_le64(meta + 80, work->stock_size);
- dma_wmb();
- }
- work->signature_calls++;
+static int restore_stock_signature(struct unlock_transaction_work *work)
+{
+ if (!work->meta.cpu || !work->stock.cpu ||
+ (work->state != GU_PUBLISHED && work->state != GU_ACTIVE))
+ return GU_RESULT(-EINVAL);
+ put_le64((u8 *)work->meta.cpu + 72, work->stock.dma);
+ put_le64((u8 *)work->meta.cpu + 80, work->bundle->sig_span.size);
+ dma_wmb();
+ work->state = GU_STOCK;
return 0;
}
+static int restore_frts(struct unlock_transaction_work *work)
+{
+ u32 lo = 0, hi = 0;
+ if (!work->frts_saved)
+ return GU_RESULT(-EPROTO);
+ read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo);
+ read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi);
+ dev_dbg(&work->pdev->dev, "WPR2 before restore: index=%zu attempt=%u lo=%#x hi=%#x\n",
+ work->write_index, work->attempt, lo, hi);
+ dev_dbg(&work->pdev->dev,
+ "WPR2 target: expected=%#x:%#x start=%#llx end=%#llx raw_before=%#x:%#x\n",
+ work->frts_lo, work->frts_hi,
+ (unsigned long long)((u64)(work->frts_lo >> 4) << 12),
+ (unsigned long long)(((u64)(work->frts_hi >> 4) << 12) + GU_WPR2_ALIGN),
+ work->frts_lo, work->frts_hi);
+ write_bar(work, GU_WPR2_ADDR_LO_OFFSET, work->frts_lo);
+ write_bar(work, GU_WPR2_ADDR_HI_OFFSET, work->frts_hi);
+ if (read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo) || read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi))
+ return GU_RESULT(-EIO);
+ {
+ u32 feat = 0, wpr_cfg = 0, wpr = 0, lock = 0;
+
+ read_bar(work, GU_FEAT_PLM_OFFSET, &feat);
+ read_bar(work, GU_WPR_CFG_PLM_OFFSET, &wpr_cfg);
+ read_bar(work, GU_WPR_PLM_OFFSET, &wpr);
+ read_bar(work, GU_MMU_LOCK_CFG_PLM_OFFSET, &lock);
+ dev_dbg(&work->pdev->dev,
+ "WPR2 restored: lo=%#x hi=%#x expected=%#x:%#x plm feat=%#x wpr_cfg=%#x wpr=%#x lock=%#x\n",
+ lo, hi, work->frts_lo, work->frts_hi, feat, wpr_cfg, wpr, lock);
+ }
+ return GU_RESULT(lo == work->frts_lo && hi == work->frts_hi ? 0 : -EIO);
+}
+static int table_failure(struct unlock_transaction_work *work, const char *phase,
+ const struct gu_reg_write *entry, u32 observed, int ret)
+{
+ dev_err(&work->pdev->dev,
+ "table failure: family=%s payload=%s phase=%s index=%zu attempt=%u address=%#x expected=%#x observed=%#x error=%d\n",
+ work->card->family->name, work->payload->name, phase, work->write_index,
+ work->attempt, entry->addr, entry->value, observed, ret);
+ return GU_RESULT(ret);
+}
+static int run_privileged_writes(struct unlock_transaction_work *work,
+ const struct gu_write_table *table)
+{
+ size_t i;
+ u32 after = 0;
+ int ret;
+ if (work->state != GU_PUBLISHED && work->state != GU_ACTIVE)
+ return GU_RESULT(-EPROTO);
+ work->state = GU_ACTIVE;
+ for (i = 0; i < table->count; i++) {
+ const struct gu_reg_write *entry = &table->entries[i];
+ work->write_index = i;
+ for (work->attempt = 1; work->attempt <= 2; work->attempt++) {
+ u32 before, lo = 0, hi = 0, ctl = 0, engine = 0;
+ struct gu_boot_state st;
+ work->error_reported = false;
+ work->target_addr = entry->addr;
+ work->target_value = entry->value;
+ after = 0;
+ /* FWSEC completed before the first attempt. Every subsequent
+ * refill requires the prior invocation to have stopped. */
+ if ((i || work->attempt > 1) && !work->invocation_halted)
+ return table_failure(work, "prior-halt", entry, after, -EBUSY);
+ /* FWSEC already left WPR2 at the FRTS carve for the first
+ * launch, and WPR_CFG is not open yet, so a host write here
+ * would be a denied PLM access. Every later launch has
+ * extended WPR2 and needs the carve restored. */
+ if (i || work->attempt > 1) {
+ ret = restore_frts(work);
+ if (ret)
+ return table_failure(work, "restore", entry, after, ret);
+ }
+ /* CPUCTL is diagnostic only: after FWSEC, GA102 SEC2 CPUCTL reads
+ * 0xbadf5620 until the launch resets the engine. */
+ if (read_bar(work, entry->addr, &before) ||
+ (before & 0xffff0000u) == 0xbadf0000u ||
+ read_bar(work, GU_SEC2_BASE + GU_FALCON_CPUCTL_OFFSET, &ctl) ||
+ read_bar(work, GU_SEC2_BASE + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
+ return table_failure(work, "prelaunch", entry, after, -ENODEV);
+ read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo);
+ read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi);
+ gu_read_boot_state(work, &st);
+ dev_dbg(&work->pdev->dev,
+ "prelaunch: index=%zu/%zu attempt=%u target=%#x value=%#x before=%#x cpuctl=%#x engine=%#x reset_plm=%#x wpr2=%#x:%#x fhstate=%#x\n",
+ i, table->count, work->attempt, entry->addr, entry->value,
+ before, ctl, engine, st.reset_plm, lo, hi, st.fhstate);
+ ret = gu_payload_build(work->payload, work->buffer.cpu,
+ entry->addr, entry->value);
+ if (ret)
+ return table_failure(work, "payload", entry, after, ret);
+ put_le64((u8 *)work->meta.cpu + 72, work->buffer.dma);
+ put_le64((u8 *)work->meta.cpu + 80, work->payload->size);
+ dma_wmb();
+ dev_dbg(&work->pdev->dev,
+ "payload: name=%s size=%#x dma=%#llx index=%zu/%zu attempt=%u address=%#x value=%#x mode=%s\n",
+ work->payload->name, work->payload->size,
+ (unsigned long long)work->buffer.dma, i, table->count,
+ work->attempt, entry->addr, entry->value,
+ work->attempt > 1 ? "retry" : "first");
+ ret = unlock_transaction_boot(work);
+ if (read_bar(work, entry->addr, &after) ||
+ (after & 0xffff0000u) == 0xbadf0000u ||
+ read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo) || read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi))
+ return table_failure(work, "result-inaccessible", entry, after, -ENODEV);
+ gu_read_boot_state(work, &st);
+ dev_dbg(&work->pdev->dev,
+ "write result: index=%zu/%zu attempt=%u address=%#x expected=%#x observed=%#x wpr2=%#x:%#x write=%u reset_plm=%#x ctl=%#x engine=%#x fhstate=%#x mbox=%#x:%#x acr=%#x csberr=%#x\n",
+ i, table->count, work->attempt, entry->addr, entry->value,
+ after, lo, hi, after == entry->value, st.reset_plm,
+ st.ctl, st.engine, st.fhstate, st.mailbox1, st.mailbox0,
+ st.acr, st.csberr);
+ if (ret) {
+ dev_err(&work->pdev->dev, "failed invocation WPR2: actual=%#x:%#x saved_FRTS=%#x:%#x\n",
+ lo, hi, work->frts_lo, work->frts_hi);
+ return table_failure(work, "completion", entry, after, ret);
+ }
+ if (after == entry->value) {
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: pass=%zu/%zu attempt=%u target=%#x value=%#x before=%#x after=%#x wpr2=%#x:%#x reset_plm=%#x mbox=%#x:%#x ret=0\n",
+ i + 1, table->count, work->attempt, entry->addr,
+ entry->value, before, after, lo, hi,
+ st.reset_plm, st.mailbox1, st.mailbox0);
+ break;
+ }
+ /* Only literal idempotent PLMs belong here. A mismatching
+ * readback after proven halt/cleanup may be retried once.
+ * Errors/timeouts never replay an uncertain side effect. */
+ table_failure(work, "readback", entry, after, -EIO);
+ }
+ if (after != entry->value)
+ return table_failure(work, "readback-exhausted", entry, after, -EIO);
+ }
+ ret = restore_frts(work);
+ return ret && table->count ?
+ table_failure(work, "final-FRTS", &table->entries[table->count - 1], after, ret) : ret;
+}
+
/* Write the unlock value to one speed selector and read it back. */
static int write_selector(struct unlock_transaction_work *work, const char *name,
u32 offset, u32 unlock)
@@ -1174,90 +1249,41 @@ static int write_selector(struct unlock_transaction_work *work, const char *name
if (read_bar(work, offset, &value) || inaccessible(value))
return GU_RESULT(-ENODEV);
- dev_info(&work->pdev->dev, "selector read: %s=%#x expected_unlock=%#x\n",
+ dev_dbg(&work->pdev->dev, "selector read: %s=%#x expected_unlock=%#x\n",
name, value, unlock);
- if (value == unlock)
- return 0;
- dev_info(&work->pdev->dev, "selector write: %s=%#x\n", name, unlock);
+ dev_dbg(&work->pdev->dev, "selector write: %s=%#x\n", name, unlock);
write_bar(work, offset, unlock);
if (read_bar(work, offset, &value) || value == ~0u)
return GU_RESULT(-ENODEV);
return GU_RESULT(value == unlock ? 0 : -EIO);
}
-/* SS0 selects the full BF16 rate; SS1 gets the same field value. FLR keeps both. */
-static int unlock_transaction_selector(struct unlock_transaction_work *work)
+static int run_host_writes(struct unlock_transaction_work *work,
+ const struct gu_write_table *table)
{
- int ret = write_selector(work, "SS0", GU_FEAT_SS0_OFFSET, GU_SS0_UNLOCK);
-
- return ret ? ret : write_selector(work, "SS1", GU_FEAT_SS1_OFFSET, GU_SS1_UNLOCK);
-}
-
-/* Reset SEC2 after FLR and leave it halted with reset released. Do not reset
- * GSP here: stock RM first requires completed GFW and a halted GSP Falcon.
- * ksec2ResetHw_TU102 ends with RESET_FALSE. Leaving RESET_TRUE makes later
- * host reads return FECS_PRI_RESET (0xbadf12 in bits 31:8). RM's scrub wait
- * treats that value as scrubbing-pending and does not finish.
- */
-static int hold_sec2_reset(struct unlock_transaction_work *work, const char *phase)
-{
- unsigned int j;
- u32 engine, before, first = 0, hwcfg;
- int ret;
-
- if (read_bar(work, SEC2_BASE + 0x3c0, &engine) || inaccessible(engine))
- return GU_RESULT(-ENODEV);
- before = engine;
- dev_info(&work->pdev->dev, "engine reset assert: phase=%s base=%#x before=%#x write=%#x\n",
- phase, SEC2_BASE, before, before | 1);
- write_bar(work, SEC2_BASE + 0x3c0, engine | 1);
- /* 610 ksec2ResetHw_TU102 requires ten register cycles for signal
- * propagation. A stale zero during those cycles is not evidence that
- * the assertion failed. No CPU start and no reset retry.
- */
- for (j = 0; j < 10; j++) {
- if (read_bar(work, SEC2_BASE + 0x3c0, &engine) ||
- inaccessible(engine)) {
- dev_err(&work->pdev->dev, "engine reset inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
- phase, SEC2_BASE, j + 1, engine);
- return GU_RESULT(-EIO);
+ size_t i;
+ for (i = 0; i < table->count; i++) {
+ const struct gu_reg_write *entry = &table->entries[i];
+ u32 observed = 0;
+ int ret;
+ work->write_index = i;
+ ret = write_selector(work, "host", entry->addr, entry->value);
+ if (ret) {
+ read_bar(work, entry->addr, &observed);
+ return table_failure(work, "host", entry, observed, ret);
}
- if (!j)
- first = engine;
}
- dev_info(&work->pdev->dev, "engine reset propagation: phase=%s base=%#x first=%#x tenth=%#x\n",
- phase, SEC2_BASE, first, engine);
- ret = poll_us(work, SEC2_BASE + 0x3c0, 1, 1, 1000, true);
- dev_info(&work->pdev->dev, "engine reset asserted: phase=%s base=%#x error=%d last=%#x\n",
- phase, SEC2_BASE, ret, work->last_value);
- if (ret)
- return GU_RESULT(ret);
- if (read_bar(work, SEC2_BASE + 0x3c0, &engine) || inaccessible(engine))
- return GU_RESULT(-EIO);
- dev_info(&work->pdev->dev, "engine reset release: phase=%s base=%#x before=%#x write=%#x\n",
- phase, SEC2_BASE, engine, engine & ~1u);
- write_bar(work, SEC2_BASE + 0x3c0, engine & ~1u);
- for (j = 0; j < 10; j++) {
- if (read_bar(work, SEC2_BASE + 0x3c0, &engine) ||
- inaccessible(engine)) {
- dev_err(&work->pdev->dev, "engine reset release inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
- phase, SEC2_BASE, j + 1, engine);
- return GU_RESULT(-EIO);
- }
+ {
+ u32 ss0 = 0, ss1 = 0, feat = 0, wpr_cfg = 0;
+
+ read_bar(work, GU_FEAT_SS0_OFFSET, &ss0);
+ read_bar(work, GU_FEAT_SS1_OFFSET, &ss1);
+ read_bar(work, GU_FEAT_PLM_OFFSET, &feat);
+ read_bar(work, GU_WPR_CFG_PLM_OFFSET, &wpr_cfg);
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=host ss0=%#x ss1=%#x plm_feat=%#x plm_wpr_cfg=%#x\n",
+ ss0, ss1, feat, wpr_cfg);
}
- ret = poll_us(work, SEC2_BASE + 0x3c0, 1, 0, 1000, true);
- if (ret)
- return GU_RESULT(ret);
- /* NV_PFALCON_FALCON_HWCFG2 bit 12 clears when memory scrubbing finishes.
- * A PRI error fails this wait instead of being handed to RM.
- */
- ret = poll_us(work, SEC2_BASE + 0xf4, 1u << 12, 0, 1000000, true);
- if (ret)
- return GU_RESULT(ret);
- if (read_bar(work, SEC2_BASE + 0xf4, &hwcfg) || inaccessible(hwcfg))
- return GU_RESULT(-EIO);
- dev_info(&work->pdev->dev, "engine reset released: phase=%s base=%#x engine=%#x hwcfg2=%#x\n",
- phase, SEC2_BASE, engine, hwcfg);
return 0;
}
@@ -1271,10 +1297,10 @@ static int wait_post_flr_bar(struct unlock_transaction_work *work, u32 chip)
u32 value = 0;
for (;;) {
- if (read_bar(work, 0, &value))
+ if (read_bar(work, GU_PMC_BOOT_0_OFFSET, &value))
return GU_RESULT(-EIO);
if (value == chip) {
- dev_info(&work->pdev->dev, "post-FLR BAR ready: chip=%#x\n", value);
+ dev_dbg(&work->pdev->dev, "post-FLR BAR ready: chip=%#x\n", value);
return 0;
}
if (ktime_compare(ktime_get(), deadline) >= 0) {
@@ -1325,7 +1351,7 @@ static int wait_post_flr_gfw(struct unlock_transaction_work *work)
}
usleep_range(1000, 2000);
}
- dev_info(&work->pdev->dev, "post-FLR GFW result: error=%d pci=%#x plm=%#x boot=%#x cpuctl=%#x budget_us=%u\n",
+ dev_dbg(&work->pdev->dev, "post-FLR GFW result: error=%d pci=%#x plm=%#x boot=%#x cpuctl=%#x budget_us=%u\n",
ret, identity, sample.plm, sample.boot, sample.cpuctl, GU_GFW_TIMEOUT_US);
return GU_RESULT(ret);
}
@@ -1337,12 +1363,12 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
unsigned int i;
int ret;
- if (work->signature_calls != 2)
+ if (work->state != GU_STOCK)
return GU_RESULT(-EPROTO);
if (read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || ss0 == ~0u ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || ss1 == ~0u)
return GU_RESULT(-ENODEV);
- dev_info(&work->pdev->dev, "pre-FLR selectors: ss0=%#x expected=%#x ss1=%#x expected=%#x\n",
+ dev_dbg(&work->pdev->dev, "pre-FLR selectors: ss0=%#x expected=%#x ss1=%#x expected=%#x\n",
ss0, GU_SS0_UNLOCK, ss1, GU_SS1_UNLOCK);
if (ss0 != GU_SS0_UNLOCK || ss1 != GU_SS1_UNLOCK)
return GU_RESULT(-EIO);
@@ -1351,7 +1377,7 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
if (diag_pci_word(work, false, PCI_COMMAND, &command) ||
command == 0xffff || (command & PCI_COMMAND_MASTER))
return GU_RESULT(-EIO);
- if (read_bar(work, 0, &chip) || !chip || inaccessible(chip))
+ if (read_bar(work, GU_PMC_BOOT_0_OFFSET, &chip) || !chip || inaccessible(chip))
return GU_RESULT(-ENODEV);
/* Ampere PF-FLR requires no engine-reset preconditioning in this cold,
* unbound path (no initialized NVLink/P2P). The pinned GA102 rejoin14
@@ -1368,9 +1394,9 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
if (ret)
return GU_RESULT(ret);
}
- dev_info(&work->pdev->dev, "PCIe FLR begin: BME=0 DMA retained, chip=%#x\n", chip);
+ dev_dbg(&work->pdev->dev, "PCIe FLR begin: BME=0 DMA retained, chip=%#x\n", chip);
ret = pcie_flr(work->pdev);
- dev_info(&work->pdev->dev, "PCIe FLR returned: result=%d\n", ret);
+ dev_dbg(&work->pdev->dev, "PCIe FLR returned: result=%d\n", ret);
if (ret)
return GU_RESULT(ret);
/* The saved image predates BME. No transient restore with BME set. */
@@ -1397,7 +1423,11 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
ret = wait_post_flr_gfw(work);
if (ret)
return GU_RESULT(ret);
- ret = hold_sec2_reset(work, "post-FLR");
+ {
+ struct gu_falcon_io io = falcon_io(work);
+ struct gu_falcon_evidence reset;
+ ret = gu_falcon_engine_reset(&io, GU_SEC2_BASE, "post-FLR", NULL, &reset);
+ }
if (ret)
return GU_RESULT(ret);
if (diag_pci_word(work, true, PCI_EXP_DEVSTA, &pending) ||
@@ -1405,16 +1435,16 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
return GU_RESULT(-EBUSY);
if (read_bar(work, GU_GFW_CPUCTL_OFFSET, &cpuctl) || inaccessible(cpuctl) ||
!(cpuctl & GU_GFW_CPUCTL_HALTED) ||
- read_bar(work, 0x001fa824, &lo) || inaccessible(lo) ||
- read_bar(work, 0x001fa828, &hi) || inaccessible(hi) ||
+ read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo) || inaccessible(lo) ||
+ read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi) || inaccessible(hi) ||
read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || inaccessible(ss0) ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || inaccessible(ss1) ||
read_bar(work, GU_FEAT_PLM_OFFSET, &plm) ||
(plm & 0xffff0000u) == 0xbadf0000u)
return GU_RESULT(-ENODEV);
- dev_info(&work->pdev->dev,
- "post-FLR SEC2 reset released, GSP halted, BME=0 pending=0 WPR2=%08x:%08x SS0=%08x SS1=%08x\n",
- lo, hi, ss0, ss1);
+ dev_dbg(&work->pdev->dev,
+ "post-FLR SEC2 reset released, GSP halted, BME=0 pending=%u WPR2=%08x:%08x SS0=%08x SS1=%08x PLM=%08x CMD=%#x CHIP=%#x\n",
+ pending, lo, hi, ss0, ss1, plm, command, chip);
/* Both 610 and installed 615 _kgspBootGspRm reject preexisting WPR2
* on this cold dGPU path, even our old in-bounds FRTS reservation.
* kgspIsWpr2Up_TU102 uses HI.VAL != 0, not a LO/HI range comparison.
@@ -1423,42 +1453,70 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
dev_err(&work->pdev->dev, "post-FLR WPR2 remains enabled: lo=%#x hi=%#x; stock driver requires HI.VAL=0\n", lo, hi);
return GU_RESULT(-EBUSY);
}
- work->quiesced = true;
+ work->state = GU_QUIESCED;
if (ss0 != GU_SS0_UNLOCK || ss1 != GU_SS1_UNLOCK)
return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev,
+ "gpu_unlock: stage=flr post_wpr2=%#x:%#x ss0=%#x ss1=%#x plm=%#x cmd=%#x devsta=%#x chip=%#x\n",
+ lo, hi, ss0, ss1, plm, command, pending, chip);
return 0;
}
-int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared)
-{
- return request_scoped(&pdev->dev, &shared->gsp, GSP_FILE);
-}
-
-void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
-{
- release_firmware(shared->gsp);
- shared->gsp = NULL;
-}
-
-int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared,
+int gpu_unlock_610_unlock(const struct gu_device *device, const struct gpu_unlock_610_shared *shared,
bool resume)
{
+ struct pci_dev *pdev = device->pdev;
struct unlock_transaction_work *work;
+ const struct gpu_unlock_610_firmware *assets;
+ const struct gpu_unlock_610_booter *booter;
int ret, cleanup;
+ if (!device->recipe || !device->recipe->family)
+ return -ENODEV;
work = kzalloc(sizeof(*work), GFP_KERNEL);
if (!work)
return -ENOMEM;
work->pdev = pci_dev_get(pdev);
- work->gsp_fw = shared->gsp;
+ work->card = device->recipe;
+ assets = gpu_unlock_610_bundle_find(shared, work->card->family->fw);
+ booter = gpu_unlock_610_booter_find(shared, work->card->family->booter);
+ if (!assets || !assets->prepared || assets->error || !booter || !booter->fw ||
+ !work->card->family->booter->payload ||
+ gu_payload_check(work->card->family->booter->payload) ||
+ (work->card->family->loader && !shared->loader)) {
+ ret = assets && assets->prepared && assets->error ? assets->error : -EINVAL;
+ pci_dev_put(work->pdev);
+ kfree(work);
+ return ret;
+ }
+ work->bundle = assets;
+ work->booter = booter;
+ work->payload = work->card->family->booter->payload;
+ work->loader_fw = work->card->family->loader ? shared->loader : NULL;
+ dev_dbg(&pdev->dev, "recipe selection: device=%#x family=%s payload=%s privileged=%zu host=%zu launch=%u\n",
+ pdev->device, work->card->family->name, work->payload->name,
+ work->card->family->privileged.count, work->card->family->host.count,
+ work->card->family->booter->launch);
work->resume = resume;
/* After resume GFW may still be running: check and wait before reading
- * the layout, and before loading assets for a card that kept power. */
+ * the layout. Firmware preparation has already completed for the round. */
ret = resume ? unlock_transaction_suspended(work) : 0;
- if (!ret)
- ret = unlock_transaction_assets(work);
+ if (!ret && !resume)
+ ret = check_devres_empty(work, "entry");
if (!ret)
ret = unlock_transaction_claim(work);
+ if (!ret) {
+ u32 chip;
+ if (read_bar(work, GU_PMC_BOOT_0_OFFSET, &chip) || inaccessible(chip) ||
+ ((chip >> 20) & 0x1ff) != work->card->chip)
+ ret = GU_RESULT(-ENODEV);
+ }
+ if (!ret) {
+ size_t i;
+ for (i = 0; i < work->card->family->privileged.count; i++)
+ if ((u64)work->card->family->privileged.entries[i].addr + 4 > work->bar_len)
+ ret = GU_RESULT(-ERANGE);
+ }
if (!ret)
ret = unlock_transaction_layout(work);
if (!ret)
@@ -1468,21 +1526,26 @@ int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shar
if (!ret)
ret = unlock_transaction_fwsec(work);
if (!ret)
- ret = unlock_transaction_signature(work); /* publish V67 */
+ ret = publish_signature(work);
if (!ret)
- ret = unlock_transaction_boot(work);
+ ret = run_privileged_writes(work, &work->card->family->privileged);
if (!ret)
- ret = unlock_transaction_signature(work); /* restore stock */
+ ret = restore_stock_signature(work);
if (!ret)
- ret = unlock_transaction_selector(work);
+ ret = run_host_writes(work, &work->card->family->host);
if (!ret)
ret = unlock_transaction_reset(work);
/* Never leave the V67 signature published for a later stock boot. */
- if (ret && work->signature_calls == 1)
- unlock_transaction_signature(work);
+ if (ret && (work->state == GU_PUBLISHED || work->state == GU_ACTIVE))
+ restore_stock_signature(work);
+ work->error_reported = false;
cleanup = unlock_transaction_unwind(work);
if (!ret)
ret = cleanup;
+ dev_info(&pdev->dev,
+ "gpu_unlock: stage=handoff ret=%d state=%u retained=%u pages=%u/%u cleanup=%d\n",
+ ret, work->state, work->retained, work->pages_allocated,
+ work->page_count, cleanup);
if (work->retained) {
/* Firmware may still DMA into these buffers: keep them, the device
* reference and this module until the next power off. */
@@ -1494,9 +1557,3 @@ int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shar
kfree(work);
return ret;
}
-
-MODULE_FIRMWARE(BOOTER_FILE);
-MODULE_FIRMWARE(V67_FILE);
-MODULE_FIRMWARE(GSP_FILE);
-MODULE_FIRMWARE(BOOT_FILE);
-MODULE_FIRMWARE(BOOT_DESC_FILE);
diff --git a/kernel/gpu_unlock/unlock_610.h b/kernel/gpu_unlock/unlock_610.h
index ff70a76..f94a146 100644
--- a/kernel/gpu_unlock/unlock_610.h
+++ b/kernel/gpu_unlock/unlock_610.h
@@ -2,35 +2,22 @@
#ifndef GPU_UNLOCK_UNLOCK_610_H
#define GPU_UNLOCK_UNLOCK_610_H
-#include <linux/types.h>
-
-struct firmware;
-struct pci_dev;
-
-/* Read-only firmware every unlock uses: loaded once, shared by concurrent unlocks. */
-struct gpu_unlock_610_shared {
- const struct firmware *gsp;
-};
-
-/*
- * Load the shared firmware. pdev is any target card; the request leaves no
- * device resources behind on it. Returns 0 or a negative errno.
- */
-int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared);
-
-/* Release the shared firmware once every unlock using it has returned. */
-void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared);
+#include "firmware_610.h"
/*
- * Unlock one cold, unbound RTX 3090, or with resume one that nvidia suspended
+ * Run the selected family recipe on a cold, unbound supported card, or one nvidia suspended
* before a suspend to RAM and resumes after this returns. The caller holds
- * pci_dev_lock. Returns 0 when the card is unlocked and handed back as found;
- * otherwise a negative errno, with the card left locked. After resume,
+ * pci_dev_lock and supplies the context resolved during enumeration. Returns
+ * 0 when the card is unlocked and handed back as found;
+ * otherwise a negative errno. A failed transaction may have applied a prefix
+ * of the table: it must not be reported as ready. If quiescence cannot be
+ * established, DMA buffers and BAR ownership are retained until power off.
+ * After resume,
* -EALREADY means the card kept power and is still unlocked, and nothing was
* written. Unlocks of different cards may run concurrently with the same
* shared firmware.
*/
-int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared,
+int gpu_unlock_610_unlock(const struct gu_device *device, const struct gpu_unlock_610_shared *shared,
bool resume);
#endif
diff --git a/kernel/gpu_unlock/vbios_610.c b/kernel/gpu_unlock/vbios_610.c
index 87c173b..4e2c1b1 100644
--- a/kernel/gpu_unlock/vbios_610.c
+++ b/kernel/gpu_unlock/vbios_610.c
@@ -1,6 +1,8 @@
// SPDX-License-Identifier: GPL-2.0-only
#include "diagnostic.h"
/* Formats: NVIDIA 610.43.03 kernel_gsp_fwsec.c and kernel_gsp_vbios_tu102.c.
+ * The PROM contains the full VBIOS container; locate the IFR image offset the
+ * same way kgspExtractVbiosFromRom_TU102 does before parsing the legacy image.
* Independent byte parser; no firmware-owned pointer is trusted as a host pointer.
*/
#ifdef __KERNEL__
@@ -12,6 +14,12 @@
#endif
#include "vbios_610.h"
+/* Production selects 0x85. The integration build selects the debug app
+ * 0x45; this is a compile-time seam, never a runtime knob. */
+#ifndef GPU_UNLOCK_FWSEC_APP
+#define GPU_UNLOCK_FWSEC_APP 0x85
+#endif
+
static u16 r16(const u8 *p)
{
return (u16)p[0] | ((u16)p[1] << 8);
@@ -119,6 +127,50 @@ static int descriptor(const u8 *rom, size_t length, u64 off,
size = hdr >> 16;
GU_NOTE("FWSEC descriptor: offset=%#llx ROM_bytes=%zu header=%#x\n",
(unsigned long long)off, length, hdr);
+ fw.version = (hdr >> 8) & 0xff;
+ /* GA100 uses the v2 generic-loader descriptor, with embedded signatures.
+ * Its image is kept contiguous here; the loader receives separate code
+ * and data DMA bases within that allocation. */
+ if ((hdr & 1) && fw.version == 2) {
+ u32 sec_base, sec_size;
+ if (size < 60 || range(length, off, size))
+ return GU_DIAG(-EINVAL);
+ fw.image_off = off + size;
+ fw.image_size = r32(rom + off + 4);
+ fw.interface_off = r32(rom + off + 16);
+ fw.imem_pa = r32(rom + off + 20);
+ fw.imem_size = r32(rom + off + 24);
+ fw.imem_va = r32(rom + off + 28);
+ sec_base = r32(rom + off + 32);
+ sec_size = r32(rom + off + 36);
+ fw.data_off = r32(rom + off + 40);
+ fw.dmem_pa = r32(rom + off + 44);
+ fw.dmem_size = r32(rom + off + 48);
+ /* Restrict to the layout this contiguous adapter implements; refuse
+ * alternate images, relocation, compression and nonzero entry. */
+ if (!fw.image_size || !fw.imem_size || !fw.dmem_size ||
+ r32(rom + off + 8) != fw.image_size || r32(rom + off + 12) ||
+ r32(rom + off + 52) || r32(rom + off + 56) ||
+ fw.imem_pa || fw.dmem_pa || sec_size > fw.imem_size ||
+ sec_base < fw.imem_va || sec_size > ~(u32)0 - 255 ||
+ (fw.data_off & 255) || (fw.imem_size & 255) ||
+ (fw.dmem_size & 255) || (fw.image_size & 255) ||
+ range(length, fw.image_off, fw.image_size) ||
+ range(fw.image_size, 0, fw.imem_size) ||
+ fw.data_off < fw.imem_size ||
+ range(fw.image_size, fw.data_off, fw.dmem_size) ||
+ range(fw.image_size, fw.data_off, ((u64)fw.dmem_size + 255) & ~255ull) ||
+ range(fw.dmem_size, fw.interface_off, 4))
+ return GU_DIAG(-EINVAL);
+ fw.imem_ns_size = fw.imem_size - sec_size;
+ fw.imem_sec_pa = sec_base - fw.imem_va;
+ fw.imem_sec_size = (sec_size + 255) & ~255u;
+ if ((fw.imem_ns_size & 255) || (fw.imem_sec_pa & 255) ||
+ range(fw.imem_size, fw.imem_sec_pa, fw.imem_sec_size))
+ return GU_DIAG(-EINVAL);
+ *out = fw;
+ return 0;
+ }
/* vDesc: bit 0 set, version in bits 15:8, size in bits 31:16. */
if (!(hdr & 1) || ((hdr >> 8) & 0xff) != 3 || size < 44)
return GU_DIAG(-EOPNOTSUPP);
@@ -158,6 +210,7 @@ static int descriptor(const u8 *rom, size_t length, u64 off,
count != fw.sig_count || fw.sig_bytes != count * 384 ||
!fw.imem_size || !fw.dmem_size ||
range(fw.image_size, fw.data_off, fw.dmem_size) ||
+ range(fw.image_size, fw.data_off, ((u64)fw.dmem_size + 255) & ~255ull) ||
range(fw.dmem_size, fw.pkc_off, 384) ||
range(fw.dmem_size, fw.interface_off, 4))
return GU_DIAG(-EINVAL);
@@ -167,12 +220,40 @@ static int descriptor(const u8 *rom, size_t length, u64 off,
int gpu_unlock_vbios_fwsec(const u8 *rom, size_t length, struct gpu_unlock_fwsec *out)
{
- u32 bit, adjust;
+ u32 bit, adjust, image = 0;
int ret;
if (!rom || !out || length > 1024 * 1024)
return GU_DIAG(-EINVAL);
memset(out, 0, sizeof(*out));
+
+ /* The GPU exposes the complete VBIOS container through the PROM
+ * window. NVIDIA's kgspExtractVbiosFromRom_TU102 walks the IFR
+ * header to the first PCI expansion image; match that here instead of
+ * requiring the container to start with 0xaa55. */
+ if (length >= 12 && r32(rom) == 0x4947564eu) {
+ u32 version = (r32(rom + 4) >> 8) & 0xffu;
+ if (version != 3)
+ return GU_DIAG(-EOPNOTSUPP);
+ {
+ u32 total = r32(rom + 8) & 0xfffffu;
+ u32 flash, dir;
+ if (range(length, total, 4))
+ return GU_DIAG(-EINVAL);
+ flash = r32(rom + total);
+ dir = flash + 0x1000u;
+ if (range(length, dir, 12) || r32(rom + dir) != 0x44524652u)
+ return GU_DIAG(-EINVAL);
+ image = r32(rom + dir + 8);
+ }
+ if (!image || (image & 3u) || range(length, image, 2) ||
+ r16(rom + image) != 0xaa55u)
+ return GU_DIAG(-EINVAL);
+ GU_NOTE("ROM image: container NVGI version 3 image=%#x\n", image);
+ rom += image;
+ length -= image;
+ }
+
ret = expansion(rom, length, &adjust);
if (ret)
return GU_DIAG(ret);
@@ -217,9 +298,16 @@ int gpu_unlock_vbios_fwsec(const u8 *rom, size_t length, struct gpu_unlock_fwsec
for (j = 0; j < rom[table + 3]; j++) {
const u8 *entry = rom + table + rom[table + 1] + j * rom[table + 2];
- if (entry[0] == 0x85)
- return GU_DIAG(descriptor(rom, length,
- (u64)adjust + r32(entry + 2), out));
+ if (entry[0] == GPU_UNLOCK_FWSEC_APP) {
+ ret = descriptor(rom, length,
+ (u64)adjust + r32(entry + 2), out);
+ if (!ret) {
+ out->image_off += image;
+ if (out->sig_off)
+ out->sig_off += image;
+ }
+ return GU_DIAG(ret);
+ }
}
}
}
diff --git a/kernel/gpu_unlock/vbios_610.h b/kernel/gpu_unlock/vbios_610.h
index 89413d6..05b4f33 100644
--- a/kernel/gpu_unlock/vbios_610.h
+++ b/kernel/gpu_unlock/vbios_610.h
@@ -15,11 +15,13 @@ typedef uint64_t u64;
/* Production FWSEC descriptor extracted from a VBIOS image. */
struct gpu_unlock_fwsec {
+ u32 version;
u32 image_off;
u32 image_size;
u32 imem_pa;
u32 imem_size;
u32 imem_va;
+ u32 imem_ns_size, imem_sec_pa, imem_sec_size;
u32 dmem_pa;
u32 data_off;
u32 dmem_size;
diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD
index ca3b2de..0378b84 100644
--- a/packaging/arch/PKGBUILD
+++ b/packaging/arch/PKGBUILD
@@ -1,43 +1,47 @@
# SPDX-License-Identifier: GPL-2.0-only
-# Builds from the committed HEAD of the checkout this file lives in:
+# Builds the working tree of the checkout this file lives in, uncommitted
+# changes included. snapshot.sh publishes that tree; the git source is pinned
+# to it. An unpinned source checks out the mirror's origin/HEAD, which stays
+# on the branch the mirror was first cloned from.
# cd packaging/arch && makepkg -si
# For a published release, point the first source at the tagged tarball.
pkgbase=gpu-unlock
pkgname=(gpu-unlock-dkms gpu-unlock-firmware)
-pkgver=1.0.0
+pkgver=1.1.0
pkgrel=1
_nv=610.43.03
-_utils=nvidia-utils-$_nv-1-x86_64_v3.pkg.tar.zst
-pkgdesc='RTX 3090 BF16 unlock before the NVIDIA driver loads'
+_installer=NVIDIA-Linux-x86_64-$_nv-no-compat32.run
+pkgdesc='Ampere pre-driver hardware unlock'
arch=('x86_64')
url=''
license=('GPL-2.0-only')
-makedepends=('git' 'python' 'zstd')
+makedepends=('git' 'python' 'zstd' 'openssl')
options=('!strip' '!debug')
-source=("$pkgbase::git+file://${startdir%/packaging/arch}"
+_root=${startdir%/packaging/arch}
+_commit=$("$_root/packaging/snapshot.sh")
+source=("$pkgbase::git+file://${_root}#commit=${_commit}"
"https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-$_nv.tar.xz"
- "https://archive.cachyos.org/archive/cachyos-v3/$_utils")
-noextract=("$_utils")
+ "https://download.nvidia.com/XFree86/Linux-x86_64/$_nv/$_installer")
sha256sums=('SKIP'
'7e118923c7a23edc36114d63273a46e3e04e9af98695a42203e7ac2dfe9fc1dc'
- 'e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9')
+ 'e084451db730a6df29d3d74f655749c4d236e3c945f7d94b5205e84318fe2f90')
prepare() {
cd $pkgbase
- test "$(cat VERSION)" = "$pkgver"
+ got=$(cat VERSION)
+ [ "$got" = "$pkgver" ] || {
+ printf 'VERSION is %s, pkgver is %s (checked out %s)\n' \
+ "$got" "$pkgver" "$(git rev-parse --short HEAD)" >&2
+ return 1
+ }
}
build() {
cd $pkgbase
- python firmware/generate_v67.py "$(realpath "$srcdir/$_utils")" "$srcdir/assets"
+ python firmware/extract_gsp.py "$(realpath "$srcdir/$_installer")" "$srcdir/assets"
python firmware/extract_boot_assets.py "$srcdir/NVIDIA-kernel-module-source-$_nv" "$srcdir/assets"
}
-check() {
- cd $pkgbase
- make test ASSETS="$srcdir/assets"
-}
-
package_gpu-unlock-dkms() {
arch=('any')
depends=('dkms' "gpu-unlock-firmware=$pkgver-$pkgrel")
diff --git a/packaging/rpm/gpu-unlock.spec b/packaging/rpm/gpu-unlock.spec
index 108fad1..13f5d43 100644
--- a/packaging/rpm/gpu-unlock.spec
+++ b/packaging/rpm/gpu-unlock.spec
@@ -1,14 +1,14 @@
%global nv 610.43.03
Name: gpu-unlock
-Version: 1.0.0
+Version: 1.1.0
Release: 1%{?dist}
Summary: RTX 3090 BF16 unlock before the NVIDIA driver loads
License: GPL-2.0-only
BuildArch: noarch
Source0: gpu-unlock-%{version}.tar.xz
Source1: https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-%{nv}.tar.xz
-Source2: https://archive.cachyos.org/archive/cachyos-v3/nvidia-utils-%{nv}-1-x86_64_v3.pkg.tar.zst
+Source2: https://download.nvidia.com/XFree86/Linux-x86_64/%{nv}/NVIDIA-Linux-x86_64-%{nv}-no-compat32.run
# NVIDIA inputs stay out of the source RPM (it becomes a .nosrc.rpm).
NoSource: 1
NoSource: 2
@@ -42,7 +42,7 @@ Firmware assets generated locally from verified upstream inputs.
%prep
sha256sum -c - <<EOF
7e118923c7a23edc36114d63273a46e3e04e9af98695a42203e7ac2dfe9fc1dc %{SOURCE1}
-e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9 %{SOURCE2}
+e084451db730a6df29d3d74f655749c4d236e3c945f7d94b5205e84318fe2f90 %{SOURCE2}
EOF
%autosetup
test "$(cat VERSION)" = "%{version}"
@@ -50,9 +50,6 @@ test "$(cat VERSION)" = "%{version}"
%build
%make_build INPUTS=%{_sourcedir} ASSETS="$PWD/assets" firmware
-%check
-make test ASSETS="$PWD/assets"
-
%install
make DESTDIR=%{buildroot} ASSETS="$PWD/assets" \
install-dkms install-firmware install-dracut
@@ -131,5 +128,5 @@ if [ "$1" -eq 0 ]; then
fi
%changelog
-* Mon Sep 28 2026 Zagitta <[email protected]> - 1.0.0-1
+* Mon Sep 28 2026 Zagitta <[email protected]> - 1.1.0-1
- Initial release.
diff --git a/packaging/snapshot.sh b/packaging/snapshot.sh
new file mode 100755
index 0000000..6422aa6
--- /dev/null
+++ b/packaging/snapshot.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0-only
+# Commit the working tree, uncommitted and untracked files included, on top
+# of HEAD and print its id. The checkout, index and branches are untouched;
+# refs/gpu-unlock/snapshot keeps the commit reachable for makepkg's clone.
+# A clean tree prints HEAD. The same tree and HEAD give the same id.
+set -eu
+cd "$(dirname "$0")/.."
+index=$(mktemp)
+trap 'rm -f "$index"' EXIT
+cp "$(git rev-parse --git-path index)" "$index"
+# .claude holds Claude Code session state and worktrees, not source.
+GIT_INDEX_FILE=$index git add -A -- . ':!.claude'
+tree=$(GIT_INDEX_FILE=$index git write-tree)
+head=$(git rev-parse HEAD)
+if [ "$tree" = "$(git rev-parse "HEAD^{tree}")" ]; then
+ commit=$head
+else
+ date=$(git log -1 --format=%cd --date=raw HEAD)
+ commit=$(GIT_AUTHOR_NAME=gpu-unlock GIT_AUTHOR_EMAIL=snapshot@localhost \
+ GIT_COMMITTER_NAME=gpu-unlock GIT_COMMITTER_EMAIL=snapshot@localhost \
+ GIT_AUTHOR_DATE=$date GIT_COMMITTER_DATE=$date \
+ git commit-tree --no-gpg-sign "$tree" -p "$head" -m "Working tree snapshot")
+fi
+git update-ref refs/gpu-unlock/snapshot "$commit"
+echo "$commit"
diff --git a/tests/integration/README.md b/tests/integration/README.md
new file mode 100644
index 0000000..a4e28a6
--- /dev/null
+++ b/tests/integration/README.md
@@ -0,0 +1,74 @@
+# Driver integration harness
+
+Native runner for the `kernel/gpu_unlock` transaction against the GA102/AD102
+device model. The driver, the kernel shim and the device library are compiled
+into one shared object; Python only calls `gd_run_transaction` once per run and
+decodes the trace afterwards.
+
+## Run
+
+```sh
+python3 -m tests.integration.run GA10x --threads
+GPU_UNLOCK_DEBUG_KEY=<32 hex chars> python3 -m tests.integration.test_integration
+make integration-test
+```
+
+`GPU_UNLOCK_DEBUG_KEY` is required for the real path. The test executes the
+real FWSEC and SEC2 Booter with the real V67 payload; the final clean-halt,
+target-write and unlock checks currently fail and are reported as failures.
+
+The runner reads firmware from `$GPU_UNLOCK_ASSETS` (default
+`~/.cache/gpu-unlock/assets`) and prepares the committed VBIOS containers under
+`emulator/vbios/`. `GPU_UNLOCK_DEBUG_KEY` (32 hex chars) supplies the external
+AES-128 debug key; it is never written to the run directory.
+
+The Booter bundle copied from the cache has HS-encrypted application code. For
+keyed execution the harness replaces only that code range with the external
+encrypted debug image (`$GPU_UNLOCK_BOOTERS`, default `~/Projects/booters`:
+`ga100_secure.enc`, `ga102_secure.enc`, `ad102_secure.enc`). Header, DMEM data
+and signature area remain the production bundle; the driver still loads one
+ordinary driver-format bundle and performs the same secure load. The emulator
+decrypts at the secure DMA/PIO boundary, and the Python code-table builder
+performs the same AES-ECB transform. The debug image is not committed.
+
+Outputs per run: `rom.bin`, `assets/`, `manifest.json`, `result.json`,
+`trace.bin` and the decoded `trace.jsonl`.
+
+## Stages
+
+| Stage | State |
+| --- | --- |
+| 1. Host shim + end-to-end driver | Done. The driver parses the prepared ROM, patches FRTS, publishes the V67 signature and runs the real FWSEC/Booter transaction. |
+| 2. Free-running engines | Done. SEC2/GSP pthreads, per-engine locks, a BAR lock, DMA queue draining and the FLR quiesce barrier. The real-path test runs with free-running engines; traces are not compared for equality. |
+| 3. Execute FWSEC | Partial. Debug application selection (`GPU_UNLOCK_FWSEC_APP=0x45`), AES-128-ECB IMEM loading, a table-driven C interpreter, full-VBIOS `NVGI`/IFR scanning, the HS_UCODE_DMEM op4 SCP KDF, and the GSP SHA-256 operation/mutex path are implemented. With the supplied key the device executes the real FWSEC through the VBIOS `NVGI` check and SHA digest comparisons, then reaches and completes the DIO/SE PKA RSA path; the driver now reaches the real Booter. FWSEC and Booter each have their own pre-decoded instruction table, so the Booter no longer runs against FWSEC records. Neither engine reaches a clean halt yet (`fwsec_executed=false`, `booter_executed=false`). The secure-bus window, GSP identity values, AES/SHA registers, SE PKA modexp and the WPR2-visible register path are modeled. |
+| 4. Hooks and tracing | Done for the actual `booter.toml` inventory (storage, the two compiled CSB operations) with binary per-run trace rings and Python decoding. |
+| 5. Behavioral gates | Partial. GA10x real-path execution is asserted; the clean-halt/target/unlock gates fail with the current payload. |
+
+## Verdicts
+
+`result.json` now separates:
+
+- `driver_result` — the driver's return value.
+- `fwsec_executed`, `booter_executed` — real instruction execution only.
+- `plm_opened`, `target_written` — latched before FLR, not post-reset state.
+- `unlock_verified` — true only when the real FWSEC and Booter ran, the PLM
+ opened, the target mask was written, both selectors read back, and SEC2 left
+ `RESET_PLM=0xffffffff`.
+
+PLM enforcement is active: a host write to a protected target is denied while
+its mask still has the matching write level closed. The real firmware performs
+the privileged writes and then must pass the same host-side gate.
+
+The real keyed run asserts `unlock_verified`. It currently reaches the Booter,
+but the V67 chain faults at `pc=1` before the clean halt and target write, so
+the integration test reports failures rather than a synthesized success.
+
+## Deliberate limits
+
+- Production firmware remains unexecutable. The real path requires the external
+ debug FWSEC/Booter assets and a complete Falcon core; neither is in this repository.
+- The current V67 payload does not reach a clean Booter halt; the final gates fail.
+- GA100 stays blocked by the v2 descriptor gap described in
+ `emulator/PLAN_GSP_FWSEC.md`.
+- Traces are not compared for equality. Acceptance uses driver return values,
+ execution evidence, halt/fault PCs, WPR2 state and unknown-register evidence.
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/integration/assets.py b/tests/integration/assets.py
new file mode 100644
index 0000000..6ebcf74
--- /dev/null
+++ b/tests/integration/assets.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Prepare the ROM and asset directory for one family.
+
+The committed VBIOS files are full containers. The device exposes the same
+1 MiB PROM window to the host driver and to the GSP secure bus; the driver
+walks the IFR header to the embedded PCI/legacy image exactly as
+kgspExtractVbiosFromRom_TU102 does. Assets come from the existing gpu-unlock
+cache (or GPU_UNLOCK_ASSETS) and are copied into the run directory; nothing is
+fetched.
+"""
+from __future__ import annotations
+
+import os
+import shutil
+import struct
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+VBIOS = ROOT / "emulator" / "vbios"
+CACHE = Path(os.environ.get("GPU_UNLOCK_ASSETS", Path.home() / ".cache" / "gpu-unlock" / "assets"))
+BOOTERS = Path(os.environ.get("GPU_UNLOCK_BOOTERS", Path.home() / "Projects" / "booters"))
+
+PROM_SIZE = 0x100000
+PROM_OFFSET = 0x300000
+
+FAMILIES = {
+ "GA100": {
+ "device": 0x20C2,
+ "chip": 0x170,
+ "rom": VBIOS / "257744.rom",
+ "offset": 0x0,
+ "fwsec_version": 2,
+ "booter": "booter_load_ga100.bin",
+ "debug_booter": "ga100_secure.enc",
+ },
+ "GA10x": {
+ "device": 0x2204,
+ "chip": 0x172,
+ "rom": VBIOS / "Palit.RTX3090.24576.210310_3.rom",
+ "offset": 0x0,
+ "fwsec_version": 3,
+ "booter": "booter_load_ga102.bin",
+ "debug_booter": "ga102_secure.enc",
+ },
+ "AD102": {
+ "device": 0x2684,
+ "chip": 0x192,
+ "rom": VBIOS / "Palit.RTX4090.24576.221012.rom",
+ "offset": 0x0,
+ "fwsec_version": 3,
+ "booter": "booter_load_ad102.bin",
+ "debug_booter": "ad102_secure.enc",
+ },
+}
+
+ASSETS = [
+ "gsp_ga10x.bin", "gsp_boot_ga102.bin", "gsp_boot_desc_ga102.bin",
+ "gsp_tu10x.bin", "gsp_boot_ga100.bin", "gsp_boot_desc_ga100.bin",
+ "gsp_boot_ad102.bin", "gsp_boot_desc_ad102.bin",
+ "falcon_loader.bin", "booter_load_ga100.bin", "booter_load_ga102.bin",
+ "booter_load_ad102.bin",
+]
+
+
+def prepare_rom(family: str, output: Path) -> dict:
+ spec = FAMILIES[family]
+ data = spec["rom"].read_bytes()
+ offset = spec["offset"]
+ image = data[offset:offset + PROM_SIZE]
+ if len(image) < 2 or not ((image[0] == 0x55 and image[1] == 0xAA) or
+ (image[0] == 0x4E and image[1] == 0x56)):
+ raise SystemExit(f"{family}: no VBIOS image at {offset:#x} in {spec['rom']}")
+ image = image.ljust(PROM_SIZE, b"\xff")
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_bytes(image)
+ return {"family": family, "rom": str(spec["rom"]), "offset": offset,
+ "size": PROM_SIZE, "device": spec["device"], "chip": spec["chip"]}
+
+
+def prepare_assets(family: str, destination: Path) -> list[str]:
+ destination.mkdir(parents=True, exist_ok=True)
+ copied = []
+ for name in ASSETS:
+ source = CACHE / name
+ if not source.exists():
+ raise SystemExit(f"missing asset {source}; run firmware/fetch_artifacts.py first")
+ shutil.copyfile(source, destination / name)
+ copied.append(name)
+ _install_debug_booter(family, destination / FAMILIES[family]["booter"])
+ return copied
+
+
+def _install_debug_booter(family: str, bundle: Path) -> None:
+ """Install the external encrypted debug Booter IMEM.
+
+ The driver loads one driver-format bundle and the HS loader decrypts the
+ code. The debug image is AES-ECB encrypted with the same debug key; the
+ emulator performs that transform at the secure DMA/PIO boundary. Header,
+ DMEM data and signature area stay from the production bundle.
+ """
+ debug = BOOTERS / FAMILIES[family]["debug_booter"]
+ if not debug.is_file():
+ return
+ data = bytearray(bundle.read_bytes())
+ if len(data) < 80:
+ raise SystemExit(f"{family}: short Booter bundle {bundle}")
+ fields = struct.unpack_from("<14I", data, 8)
+ code_off, code_size = fields[5], fields[6]
+ start = 72 + code_off
+ end = start + code_size
+ code = debug.read_bytes()
+ if len(code) != code_size:
+ raise SystemExit(
+ f"{family}: debug Booter {debug} is {len(code)} bytes; bundle needs {code_size}")
+ if len(code) % 16:
+ raise SystemExit(f"{family}: debug Booter {debug} is not AES block aligned")
+ if end > len(data):
+ raise SystemExit(f"{family}: Booter code range outside bundle")
+ data[start:end] = code
+ bundle.write_bytes(data)
+
+
+def prepare_run(family: str, run_dir: Path) -> dict:
+ run_dir.mkdir(parents=True, exist_ok=True)
+ rom = run_dir / "rom.bin"
+ manifest = prepare_rom(family, rom)
+ assets = prepare_assets(family, run_dir / "assets")
+ manifest["assets"] = assets
+ manifest["run_dir"] = str(run_dir)
+ return manifest
diff --git a/tests/integration/build.py b/tests/integration/build.py
new file mode 100644
index 0000000..4266af2
--- /dev/null
+++ b/tests/integration/build.py
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Build the runner shared object from the working tree.
+
+The runner links the real driver sources, the kernel shim and the native
+device library into one object. Python loads it with ctypes and calls
+gd_run_transaction once per transaction.
+"""
+from __future__ import annotations
+
+import os
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+HERE = Path(__file__).resolve().parent
+DRIVER = ROOT / "kernel" / "gpu_unlock"
+
+SOURCES = [
+ HERE / "runner.c",
+ HERE / "shim" / "shim.c",
+ HERE / "device" / "gd.c",
+ HERE / "device" / "aes.c",
+ HERE / "device" / "wpr2.c",
+ HERE / "device" / "services.c",
+ DRIVER / "firmware_610.c",
+ DRIVER / "unlock_610.c",
+ DRIVER / "falcon.c",
+ DRIVER / "vbios_610.c",
+ DRIVER / "layout_610.c",
+ DRIVER / "prep_610.c",
+ DRIVER / "payload.c",
+ DRIVER / "recipes.c",
+ DRIVER / "gfw_eval.c",
+]
+
+FLAGS = [
+ "-std=gnu11", "-O2", "-g", "-fPIC", "-shared", "-pthread",
+ "-Wall", "-Wextra", "-Wno-unused-parameter", "-Wno-sign-compare",
+ "-I" + str(HERE / "shim"),
+ "-I" + str(ROOT),
+ "-DGPU_UNLOCK_FWSEC_APP=0x45",
+]
+
+
+def build(output: Path | None = None, *, sanitize: bool = False,
+ verbose: bool = False) -> Path:
+ output = Path(output or ROOT / "build" / "integration")
+ output.mkdir(parents=True, exist_ok=True)
+ library = output / "libgdrunner.so"
+ flags = list(FLAGS)
+ if sanitize:
+ flags[1] = "-O1"
+ flags += ["-fsanitize=address,undefined", "-fno-omit-frame-pointer"]
+ command = shlex.split(os.environ.get("CC", "cc")) + flags + [
+ str(source) for source in SOURCES
+ ] + ["-o", str(library)]
+ if verbose:
+ print("building:", shlex.join(command), file=sys.stderr)
+ subprocess.run(command, check=True, cwd=ROOT)
+ return library
+
+
+if __name__ == "__main__":
+ print(build(verbose=bool(os.environ.get("GPU_UNLOCK_DIAG"))))
diff --git a/tests/integration/codetable.py b/tests/integration/codetable.py
new file mode 100644
index 0000000..4959990
--- /dev/null
+++ b/tests/integration/codetable.py
@@ -0,0 +1,120 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Pre-decode the debug FWSEC IMEM with the Python reference decoder.
+
+The device still loads and decrypts the image itself. This table is input
+data for the C interpreter, so no Python runs after the transaction starts.
+"""
+from __future__ import annotations
+
+import ctypes as C
+import struct
+from pathlib import Path
+
+from emulator.falcon_emu.decode import _decode
+
+OPS = {
+ "fallback": 0, "mov": 1, "sethi": 2, "add": 3, "adc": 4, "sub": 5,
+ "sbb": 6, "cmpu": 7, "cmps": 8, "cmp": 9, "shl": 10, "shr": 11,
+ "sar": 12, "shlc": 13, "shrc": 14, "not": 15, "neg": 16, "hswap": 17,
+ "clear": 18, "setf": 19, "ld": 20, "st": 21, "mulu": 22, "muls": 23,
+ "sext": 24, "extr": 25, "extrs": 26, "ins": 27, "and": 28, "or": 29,
+ "xor": 30, "xbit": 31, "xbitf": 32, "bset": 33, "bclr": 34, "btgl": 35,
+ "div": 36, "mod": 37, "bra": 38, "brcmp": 39, "jmp": 40, "addsp": 41,
+ "nop": 42, "movsr": 43, "movrs": 44, "call": 45, "ret": 46, "push": 47,
+ "pop": 48, "mpush": 49, "mpop": 50, "mpopret": 51, "mpopadd": 52,
+ "mpopaddret": 53, "lbra": 65, "lcall": 66, "exit": 63, "trap": 64,
+ "iord": 60, "iords": 61, "iowr": 62, "iowrs": 62,
+ "sleep": 67, "xdwait": 78, "xdfence": 79, "xcwait": 80,
+ "csecret": 70, "cenc": 71, "cxor": 72, "crnd": 73, "ckeyreg": 74,
+ "cmov": 75, "xdst": 76, "xdld": 77, "cxsin": 81, "cxsout": 82,
+ "vtlb": 83, "itlb": 84, "ptlb": 85, "cs0begin": 86, "cs0exec": 87,
+ "csigclr": 88, "bsetf": 89, "bclrf": 90, "btglf": 91, "setp": 92, "cxset": 93,
+ "iret": 94, "xcld": 95,
+ "cadd": 96, "cand": 97, "crev": 98, "cgfmul": 99, "cchmod": 100,
+ "ckexp": 101, "ckrexp": 102, "cdec": 103, "csigcmp": 104, "csigenc": 105,
+ "cs1begin": 106, "cs1exec": 107,
+ "cicmov": 120, "cixsin": 121, "cixsout": 122, "cicrnd": 123,
+ "cics0begin": 124, "cics0exec": 125, "cics1begin": 126, "cics1exec": 127,
+ "cichmod": 128, "cixor": 129, "cicadd": 130, "cicand": 131,
+ "cicrev": 132, "cicgfmul": 133, "cicsecret": 134, "cikeyreg": 135,
+ "cikexp": 136, "cikrexp": 137, "cicenc": 138, "cidec": 139,
+ "cisigcmp": 140, "cisigenc": 141, "cisigclr": 142,
+}
+
+
+class Info(C.Structure):
+ _fields_ = [("version", C.c_uint32), ("image_off", C.c_uint32),
+ ("image_size", C.c_uint32), ("data_off", C.c_uint32),
+ ("imem_size", C.c_uint32), ("dmem_size", C.c_uint32),
+ ("interface_off", C.c_uint32)]
+
+
+def build(library: Path, rom: Path, key: bytes, output: Path) -> int:
+ lib = C.CDLL(str(library))
+ lib.gd_fwsec_extract.argtypes = [C.c_char_p, C.POINTER(Info)]
+ info = Info()
+ if lib.gd_fwsec_extract(str(rom).encode(), C.byref(info)) != 0:
+ raise SystemExit("code table: FWSEC descriptor not found")
+ image = bytearray(Path(rom).read_bytes()[info.image_off:info.image_off + info.image_size])
+ if len(key) != 16:
+ raise SystemExit("code table: key must be 16 bytes")
+ imem = image[:info.imem_size]
+ buffer = (C.c_ubyte * len(imem)).from_buffer(imem)
+ if lib.gd_aes_decrypt_buffer(key, buffer, len(imem)) != 0:
+ raise SystemExit("code table: AES failed")
+ return _write_table(imem, output, 0)
+
+
+def _write_table(imem: bytes | bytearray, output: Path, base: int) -> int:
+ records = []
+ pc = base
+ while pc < base + len(imem):
+ rel = pc - base
+ ins = _decode(pc, bytes(imem[rel:rel + 6]).ljust(6, b"\0"))
+ if ins.op == "invalid":
+ break
+ target = ins.mem if isinstance(ins.mem, int) else 0
+ if ins.op in ("bra", "jmp", "lbra", "lcall", "call") and target == 0:
+ target = ins.a if isinstance(ins.a, int) else 0
+ mem = ins.mem if hasattr(ins.mem, "base") else None
+ mbase = 16 if mem is not None and mem.base == "sp" else (mem.base if mem is not None else 0)
+ idx = mem.idx if mem is not None and mem.idx is not None else -1
+ off = mem.off if mem is not None else 0
+ scale = mem.scale if mem is not None else 1
+ space = 1 if mem is not None and mem.space == "I" else 0
+ records.append(struct.pack(
+ "<IIiIIIiiiBBBBBB2x",
+ pc & 0xffffffff, target & 0xffffffff, off,
+ (ins.a & 0xffffffff) if isinstance(ins.a, int) else 0,
+ (ins.b & 0xffffffff) if isinstance(ins.b, int) else 0,
+ (ins.d & 0xffffffff) if isinstance(ins.d, int) else 0,
+ mbase, idx, scale,
+ ins.length, OPS.get(ins.op, 0), ins.sz, 1 if ins.sized else 0,
+ 1 if ins.bimm else 0, space))
+ pc += ins.length
+ output.write_bytes(struct.pack("<4sII", b"GDC1", len(records), len(imem)) + b"".join(records))
+ return len(records)
+
+
+def build_booter(library: Path, bundle: Path, key: bytes, output: Path) -> int:
+ """Decode the SEC2 Booter application image, not the FWSEC image."""
+ data = Path(bundle).read_bytes()
+ if len(data) < 80:
+ raise SystemExit("code table: short Booter bundle")
+ fields = struct.unpack_from("<14I", data, 8)
+ app_code_off, app_code_size = fields[5], fields[6]
+ image = data[72:]
+ if app_code_off + app_code_size > len(image):
+ raise SystemExit("code table: Booter app code outside image")
+ if len(key) != 16:
+ raise SystemExit("code table: key must be 16 bytes")
+ code = bytearray(image[app_code_off:app_code_off + app_code_size])
+ lib = C.CDLL(str(library))
+ lib.gd_aes_decrypt_buffer.argtypes = [C.c_char_p, C.c_void_p, C.c_size_t]
+ buffer = (C.c_ubyte * len(code)).from_buffer(code)
+ if lib.gd_aes_decrypt_buffer(key, buffer, len(code)) != 0:
+ raise SystemExit("code table: Booter AES decrypt failed")
+ count = _write_table(code, output, app_code_off)
+ if not count:
+ raise SystemExit("code table: Booter code is not decodable; install the external debug image")
+ return count
diff --git a/tests/integration/device/aes.c b/tests/integration/device/aes.c
new file mode 100644
index 0000000..fc5505c
--- /dev/null
+++ b/tests/integration/device/aes.c
@@ -0,0 +1,252 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* AES-128-ECB for the FWSEC load boundary and the crypto KDF. No padding: the
+ * caller supplies whole blocks. */
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "aes.h"
+
+static const uint8_t sbox[256] = {
+ 0x63,0x7c,0x77,0x7b,0xf2,0x6b,0x6f,0xc5,0x30,0x01,0x67,0x2b,0xfe,0xd7,0xab,0x76,
+ 0xca,0x82,0xc9,0x7d,0xfa,0x59,0x47,0xf0,0xad,0xd4,0xa2,0xaf,0x9c,0xa4,0x72,0xc0,
+ 0xb7,0xfd,0x93,0x26,0x36,0x3f,0xf7,0xcc,0x34,0xa5,0xe5,0xf1,0x71,0xd8,0x31,0x15,
+ 0x04,0xc7,0x23,0xc3,0x18,0x96,0x05,0x9a,0x07,0x12,0x80,0xe2,0xeb,0x27,0xb2,0x75,
+ 0x09,0x83,0x2c,0x1a,0x1b,0x6e,0x5a,0xa0,0x52,0x3b,0xd6,0xb3,0x29,0xe3,0x2f,0x84,
+ 0x53,0xd1,0x00,0xed,0x20,0xfc,0xb1,0x5b,0x6a,0xcb,0xbe,0x39,0x4a,0x4c,0x58,0xcf,
+ 0xd0,0xef,0xaa,0xfb,0x43,0x4d,0x33,0x85,0x45,0xf9,0x02,0x7f,0x50,0x3c,0x9f,0xa8,
+ 0x51,0xa3,0x40,0x8f,0x92,0x9d,0x38,0xf5,0xbc,0xb6,0xda,0x21,0x10,0xff,0xf3,0xd2,
+ 0xcd,0x0c,0x13,0xec,0x5f,0x97,0x44,0x17,0xc4,0xa7,0x7e,0x3d,0x64,0x5d,0x19,0x73,
+ 0x60,0x81,0x4f,0xdc,0x22,0x2a,0x90,0x88,0x46,0xee,0xb8,0x14,0xde,0x5e,0x0b,0xdb,
+ 0xe0,0x32,0x3a,0x0a,0x49,0x06,0x24,0x5c,0xc2,0xd3,0xac,0x62,0x91,0x95,0xe4,0x79,
+ 0xe7,0xc8,0x37,0x6d,0x8d,0xd5,0x4e,0xa9,0x6c,0x56,0xf4,0xea,0x65,0x7a,0xae,0x08,
+ 0xba,0x78,0x25,0x2e,0x1c,0xa6,0xb4,0xc6,0xe8,0xdd,0x74,0x1f,0x4b,0xbd,0x8b,0x8a,
+ 0x70,0x3e,0xb5,0x66,0x48,0x03,0xf6,0x0e,0x61,0x35,0x57,0xb9,0x86,0xc1,0x1d,0x9e,
+ 0xe1,0xf8,0x98,0x11,0x69,0xd9,0x8e,0x94,0x9b,0x1e,0x87,0xe9,0xce,0x55,0x28,0xdf,
+ 0x8c,0xa1,0x89,0x0d,0xbf,0xe6,0x42,0x68,0x41,0x99,0x2d,0x0f,0xb0,0x54,0xbb,0x16
+};
+
+static const uint8_t sbox_inv[256] = {
+ 0x52,0x09,0x6a,0xd5,0x30,0x36,0xa5,0x38,0xbf,0x40,0xa3,0x9e,0x81,0xf3,0xd7,0xfb,
+ 0x7c,0xe3,0x39,0x82,0x9b,0x2f,0xff,0x87,0x34,0x8e,0x43,0x44,0xc4,0xde,0xe9,0xcb,
+ 0x54,0x7b,0x94,0x32,0xa6,0xc2,0x23,0x3d,0xee,0x4c,0x95,0x0b,0x42,0xfa,0xc3,0x4e,
+ 0x08,0x2e,0xa1,0x66,0x28,0xd9,0x24,0xb2,0x76,0x5b,0xa2,0x49,0x6d,0x8b,0xd1,0x25,
+ 0x72,0xf8,0xf6,0x64,0x86,0x68,0x98,0x16,0xd4,0xa4,0x5c,0xcc,0x5d,0x65,0xb6,0x92,
+ 0x6c,0x70,0x48,0x50,0xfd,0xed,0xb9,0xda,0x5e,0x15,0x46,0x57,0xa7,0x8d,0x9d,0x84,
+ 0x90,0xd8,0xab,0x00,0x8c,0xbc,0xd3,0x0a,0xf7,0xe4,0x58,0x05,0xb8,0xb3,0x45,0x06,
+ 0xd0,0x2c,0x1e,0x8f,0xca,0x3f,0x0f,0x02,0xc1,0xaf,0xbd,0x03,0x01,0x13,0x8a,0x6b,
+ 0x3a,0x91,0x11,0x41,0x4f,0x67,0xdc,0xea,0x97,0xf2,0xcf,0xce,0xf0,0xb4,0xe6,0x73,
+ 0x96,0xac,0x74,0x22,0xe7,0xad,0x35,0x85,0xe2,0xf9,0x37,0xe8,0x1c,0x75,0xdf,0x6e,
+ 0x47,0xf1,0x1a,0x71,0x1d,0x29,0xc5,0x89,0x6f,0xb7,0x62,0x0e,0xaa,0x18,0xbe,0x1b,
+ 0xfc,0x56,0x3e,0x4b,0xc6,0xd2,0x79,0x20,0x9a,0xdb,0xc0,0xfe,0x78,0xcd,0x5a,0xf4,
+ 0x1f,0xdd,0xa8,0x33,0x88,0x07,0xc7,0x31,0xb1,0x12,0x10,0x59,0x27,0x80,0xec,0x5f,
+ 0x60,0x51,0x7f,0xa9,0x19,0xb5,0x4a,0x0d,0x2d,0xe5,0x7a,0x9f,0x93,0xc9,0x9c,0xef,
+ 0xa0,0xe0,0x3b,0x4d,0xae,0x2a,0xf5,0xb0,0xc8,0xeb,0xbb,0x3c,0x83,0x53,0x99,0x61,
+ 0x17,0x2b,0x04,0x7e,0xba,0x77,0xd6,0x26,0xe1,0x69,0x14,0x63,0x55,0x21,0x0c,0x7d
+};
+
+static uint8_t xtime(uint8_t x)
+{
+ return (uint8_t)((x << 1) ^ ((x >> 7) * 0x1b));
+}
+
+static uint8_t mul(uint8_t a, uint8_t b)
+{
+ uint8_t result = 0;
+ while (b) {
+ if (b & 1)
+ result ^= a;
+ a = xtime(a);
+ b >>= 1;
+ }
+ return result;
+}
+
+static void add_round_key(uint8_t state[16], const uint8_t *key)
+{
+ int i;
+ for (i = 0; i < 16; i++)
+ state[i] ^= key[i];
+}
+
+static void inv_shift_rows(uint8_t s[16])
+{
+ uint8_t t;
+ t = s[13]; s[13] = s[9]; s[9] = s[5]; s[5] = s[1]; s[1] = t;
+ t = s[2]; s[2] = s[10]; s[10] = t;
+ t = s[6]; s[6] = s[14]; s[14] = t;
+ t = s[3]; s[3] = s[7]; s[7] = s[11]; s[11] = s[15]; s[15] = t;
+}
+
+static void inv_mix_columns(uint8_t s[16])
+{
+ int c;
+ for (c = 0; c < 4; c++) {
+ uint8_t *p = s + c * 4;
+ uint8_t a = p[0], b = p[1], d = p[2], e = p[3];
+ p[0] = mul(a, 14) ^ mul(b, 11) ^ mul(d, 13) ^ mul(e, 9);
+ p[1] = mul(a, 9) ^ mul(b, 14) ^ mul(d, 11) ^ mul(e, 13);
+ p[2] = mul(a, 13) ^ mul(b, 9) ^ mul(d, 14) ^ mul(e, 11);
+ p[3] = mul(a, 11) ^ mul(b, 13) ^ mul(d, 9) ^ mul(e, 14);
+ }
+}
+
+static void expand_key(const uint8_t key[16], uint8_t round[176])
+{
+ static const uint8_t rcon[10] = {1,2,4,8,16,32,64,128,27,54};
+ uint8_t temp[4];
+ int i, j;
+ memcpy(round, key, 16);
+ for (i = 16; i < 176; i += 4) {
+ memcpy(temp, round + i - 4, 4);
+ if (i % 16 == 0) {
+ uint8_t t = temp[0];
+ temp[0] = sbox[temp[1]];
+ temp[1] = sbox[temp[2]];
+ temp[2] = sbox[temp[3]];
+ temp[3] = sbox[t];
+ temp[0] ^= rcon[i / 16 - 1];
+ }
+ for (j = 0; j < 4; j++)
+ round[i + j] = round[i - 16 + j] ^ temp[j];
+ }
+}
+
+static void shift_rows(uint8_t s[16])
+{
+ uint8_t t;
+ t = s[1]; s[1] = s[5]; s[5] = s[9]; s[9] = s[13]; s[13] = t;
+ t = s[2]; s[2] = s[10]; s[10] = t;
+ t = s[6]; s[6] = s[14]; s[14] = t;
+ t = s[3]; s[3] = s[15]; s[15] = s[11]; s[11] = s[7]; s[7] = t;
+}
+
+static void mix_columns(uint8_t s[16])
+{
+ int c;
+ for (c = 0; c < 4; c++) {
+ uint8_t *p = s + c * 4;
+ uint8_t a = p[0], b = p[1], d = p[2], e = p[3];
+ p[0] = mul(a, 2) ^ mul(b, 3) ^ d ^ e;
+ p[1] = a ^ mul(b, 2) ^ mul(d, 3) ^ e;
+ p[2] = a ^ b ^ mul(d, 2) ^ mul(e, 3);
+ p[3] = mul(a, 3) ^ b ^ d ^ mul(e, 2);
+ }
+}
+
+void gd_aes128_expand_key(const uint8_t key[16], uint8_t round_keys[176])
+{
+ expand_key(key, round_keys);
+}
+
+void gd_aes128_reverse_key(const uint8_t last_round[16], uint8_t key[16])
+{
+ static const uint8_t rcon[10] = {1,2,4,8,16,32,64,128,27,54};
+ uint8_t w[176], t[4];
+ int i, j;
+
+ memcpy(w + 160, last_round, 16);
+ for (i = 43; i >= 4; i--) {
+ memcpy(t, w + (i - 1) * 4, 4);
+ if (i % 4 == 0) {
+ uint8_t x = t[0];
+ t[0] = sbox[t[1]];
+ t[1] = sbox[t[2]];
+ t[2] = sbox[t[3]];
+ t[3] = sbox[x];
+ t[0] ^= rcon[i / 4 - 1];
+ }
+ for (j = 0; j < 4; j++)
+ w[(i - 4) * 4 + j] = w[i * 4 + j] ^ t[j];
+ }
+ memcpy(key, w, 16);
+}
+
+void gd_aes128_encrypt_block(const uint8_t key[16], const uint8_t in[16], uint8_t out[16])
+{
+ uint8_t round[176], state[16];
+ int r, i;
+ expand_key(key, round);
+ memcpy(state, in, 16);
+ add_round_key(state, round);
+ for (r = 1; r <= 10; r++) {
+ for (i = 0; i < 16; i++)
+ state[i] = sbox[state[i]];
+ shift_rows(state);
+ if (r != 10)
+ mix_columns(state);
+ add_round_key(state, round + r * 16);
+ }
+ memcpy(out, state, 16);
+}
+
+int gd_aes128_encrypt(const uint8_t key[16], uint8_t *data, size_t size)
+{
+ uint8_t block[16];
+ size_t at;
+ if (size % 16)
+ return -1;
+ for (at = 0; at < size; at += 16) {
+ gd_aes128_encrypt_block(key, data + at, block);
+ memcpy(data + at, block, 16);
+ }
+ return 0;
+}
+
+void gd_aes128_decrypt_block(const uint8_t key[16], const uint8_t in[16], uint8_t out[16])
+{
+ uint8_t round[176], state[16];
+ int r, i;
+ expand_key(key, round);
+ memcpy(state, in, 16);
+ add_round_key(state, round + 160);
+ for (r = 9; r > 0; r--) {
+ inv_shift_rows(state);
+ for (i = 0; i < 16; i++)
+ state[i] = sbox_inv[state[i]];
+ add_round_key(state, round + r * 16);
+ inv_mix_columns(state);
+ }
+ inv_shift_rows(state);
+ for (i = 0; i < 16; i++)
+ state[i] = sbox_inv[state[i]];
+ add_round_key(state, round);
+ memcpy(out, state, 16);
+}
+
+int gd_aes128_decrypt(const uint8_t key[16], uint8_t *data, size_t size)
+{
+ uint8_t block[16];
+ size_t at;
+ if (size % 16)
+ return -1;
+ for (at = 0; at < size; at += 16) {
+ gd_aes128_decrypt_block(key, data + at, block);
+ memcpy(data + at, block, 16);
+ }
+ return 0;
+}
+
+bool gd_aes128_selftest(void)
+{
+ /* NIST SP 800-38A F.1.2, first ECB block. */
+ static const uint8_t key[16] = {
+ 0x2b,0x7e,0x15,0x16,0x28,0xae,0xd2,0xa6,
+ 0xab,0xf7,0x15,0x88,0x09,0xcf,0x4f,0x3c,
+ };
+ static const uint8_t cipher[16] = {
+ 0x3a,0xd7,0x7b,0xb4,0x0d,0x7a,0x36,0x60,
+ 0xa8,0x9e,0xca,0xf3,0x24,0x66,0xef,0x97,
+ };
+ static const uint8_t plain[16] = {
+ 0x6b,0xc1,0xbe,0xe2,0x2e,0x40,0x9f,0x96,
+ 0xe9,0x3d,0x7e,0x11,0x73,0x93,0x17,0x2a,
+ };
+ uint8_t out[16];
+ gd_aes128_encrypt_block(key, plain, out);
+ if (memcmp(out, cipher, 16) != 0)
+ return false;
+ gd_aes128_decrypt_block(key, cipher, out);
+ return memcmp(out, plain, 16) == 0;
+}
diff --git a/tests/integration/device/aes.h b/tests/integration/device/aes.h
new file mode 100644
index 0000000..1c1e299
--- /dev/null
+++ b/tests/integration/device/aes.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+#ifndef GD_AES_H
+#define GD_AES_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+void gd_aes128_decrypt_block(const uint8_t key[16], const uint8_t in[16], uint8_t out[16]);
+void gd_aes128_encrypt_block(const uint8_t key[16], const uint8_t in[16], uint8_t out[16]);
+int gd_aes128_encrypt(const uint8_t key[16], uint8_t *data, size_t size);
+int gd_aes128_decrypt(const uint8_t key[16], uint8_t *data, size_t size);
+void gd_aes128_expand_key(const uint8_t key[16], uint8_t round_keys[176]);
+void gd_aes128_reverse_key(const uint8_t last_round[16], uint8_t key[16]);
+bool gd_aes128_selftest(void);
+
+#endif
diff --git a/tests/integration/device/booter_regs.h b/tests/integration/device/booter_regs.h
new file mode 100644
index 0000000..6924b40
--- /dev/null
+++ b/tests/integration/device/booter_regs.h
@@ -0,0 +1,238 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Generated from emulator/hw/booter.toml by tests/integration/hooks.py.
+ * Reset values only; behavior lives in gd.c and the compiled hook table. */
+#ifndef GD_BOOTER_REGS_H
+#define GD_BOOTER_REGS_H
+struct gd_reg_default { unsigned addr, reset, mask; };
+static const struct gd_reg_default gd_bar0_defaults[] = {
+ { 0x00000000u, 0x172000a1u, 0xffffffffu },
+ { 0x00000200u, 0x00000002u, 0xffffffffu },
+ { 0x00000a00u, 0x17600000u, 0xffffffffu },
+ { 0x00001070u, 0x00000000u, 0x00000000u },
+ { 0x00001074u, 0x00000000u, 0x00000000u },
+ { 0x00001078u, 0x00000000u, 0x00000000u },
+ { 0x00001438u, 0x00000000u, 0xffffffffu },
+ { 0x00022430u, 0x00000007u, 0xffffffffu },
+ { 0x00022434u, 0x00000006u, 0xffffffffu },
+ { 0x00100ce0u, 0x00000000u, 0x7fffffffu },
+ { 0x00110084u, 0x00000000u, 0xffffffffu },
+ { 0x001100f4u, 0x80000000u, 0xffffffffu },
+ { 0x00110100u, 0x00000010u, 0xffffffffu },
+ { 0x0011010cu, 0x00000000u, 0xffffffffu },
+ { 0x00110118u, 0x00000002u, 0xffffffffu },
+ { 0x001103c0u, 0x00000000u, 0xffffffffu },
+ { 0x001103c4u, 0xffffffffu, 0xffffffffu },
+ { 0x00110600u, 0x00000000u, 0xffffffffu },
+ { 0x00110624u, 0x00000000u, 0xffffffffu },
+ { 0x00111180u, 0x00000000u, 0xffffffffu },
+ { 0x00111198u, 0x00000000u, 0xffffffffu },
+ { 0x0011119cu, 0x00000000u, 0xffffffffu },
+ { 0x00111210u, 0x00000000u, 0xffffffffu },
+ { 0x00111668u, 0x00000000u, 0xffffffffu },
+ { 0x001180f8u, 0x00000000u, 0xffffffffu },
+ { 0x00118128u, 0xffffff8fu, 0xffffffffu },
+ { 0x0011812cu, 0xffffff88u, 0xffffffffu },
+ { 0x0011821cu, 0x00000000u, 0xffffffffu },
+ { 0x00118220u, 0x00000000u, 0xffffffffu },
+ { 0x00118234u, 0x000000ffu, 0xffffffffu },
+ { 0x00118244u, 0x00000000u, 0xffffffffu },
+ { 0x00118248u, 0x00000000u, 0xffffffffu },
+ { 0x0011824cu, 0x00000000u, 0xffffffffu },
+ { 0x00118250u, 0x00000000u, 0xffffffffu },
+ { 0x00118388u, 0xffffff88u, 0xffffffffu },
+ { 0x0011838cu, 0xffffff88u, 0xffffffffu },
+ { 0x00118390u, 0xffffff8fu, 0xffffffffu },
+ { 0x0011839cu, 0x00000000u, 0xffffffffu },
+ { 0x001183a0u, 0x00000000u, 0xffffffffu },
+ { 0x001183a4u, 0x00006000u, 0xffffffffu },
+ { 0x00118c6cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9000u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9004u, 0x00000000u, 0xffffffffu },
+ { 0x001f9008u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f900cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9010u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9014u, 0x00000000u, 0xffffffffu },
+ { 0x001f9018u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f901cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9020u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9024u, 0x00000000u, 0xffffffffu },
+ { 0x001f9028u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f902cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9030u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9034u, 0x00000000u, 0xffffffffu },
+ { 0x001f9038u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f903cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9040u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9044u, 0x00000000u, 0xffffffffu },
+ { 0x001f9048u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f904cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9050u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9054u, 0x00000000u, 0xffffffffu },
+ { 0x001f9058u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f905cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9060u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9064u, 0x00000000u, 0xffffffffu },
+ { 0x001f9068u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f906cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9070u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9074u, 0x00000000u, 0xffffffffu },
+ { 0x001f9078u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f907cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9080u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9084u, 0x00000000u, 0xffffffffu },
+ { 0x001f9088u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f908cu, 0x00000000u, 0xffffffffu },
+ { 0x001f9090u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f9094u, 0x00000000u, 0xffffffffu },
+ { 0x001f9098u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f909cu, 0x00000000u, 0xffffffffu },
+ { 0x001f90a0u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f90a4u, 0x00000000u, 0xffffffffu },
+ { 0x001f90a8u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f90acu, 0x00000000u, 0xffffffffu },
+ { 0x001f90b0u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f90b4u, 0x00000000u, 0xffffffffu },
+ { 0x001f90b8u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f90bcu, 0x00000000u, 0xffffffffu },
+ { 0x001f91c0u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f91c4u, 0x00000000u, 0xffffffffu },
+ { 0x001f91c8u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f91ccu, 0x00000000u, 0xffffffffu },
+ { 0x001f91d0u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f91d4u, 0x00000000u, 0xffffffffu },
+ { 0x001f91d8u, 0xfffffff0u, 0xffffffffu },
+ { 0x001f91dcu, 0x00000000u, 0xffffffffu },
+ { 0x001fa7c4u, 0xfffffbffu, 0xffffffffu },
+ { 0x001fa7c8u, 0x0045cb8fu, 0xffffffffu },
+ { 0x001fa7ccu, 0x0004c38fu, 0xffffffffu },
+ { 0x001fa7d8u, 0xfffffbffu, 0xffffffffu },
+ { 0x001fa7dcu, 0xfffffbffu, 0xffffffffu },
+ { 0x001fa7e0u, 0xfffffbffu, 0xffffffffu },
+ { 0x001fa7ecu, 0xfffffbffu, 0xffffffffu },
+ { 0x001fa814u, 0x00000000u, 0xffffffffu },
+ { 0x001fa818u, 0x00000000u, 0xffffffffu },
+ { 0x001fa81cu, 0x1ffffe00u, 0xffffffffu },
+ { 0x001fa820u, 0x00000000u, 0xffffffffu },
+ { 0x001fa824u, 0x00000000u, 0xffffffffu },
+ { 0x001fa828u, 0x00000000u, 0xffffffffu },
+ { 0x001fa82cu, 0x00100000u, 0xffffffffu },
+ { 0x001fa830u, 0x05cffff0u, 0xffffffffu },
+ { 0x00625f04u, 0x00000001u, 0xffffffffu },
+ { 0x00820000u, 0x00000000u, 0xffffffffu },
+ { 0x00820004u, 0x0000023fu, 0x000003ffu },
+ { 0x00820008u, 0x00000000u, 0xffffffffu },
+ { 0x0082000cu, 0x12345678u, 0xffffffffu },
+ { 0x00820028u, 0x00000000u, 0xffffffffu },
+ { 0x00820040u, 0x00000001u, 0xffffffffu },
+ { 0x00820050u, 0x00000000u, 0x03ff03ffu },
+ { 0x00820060u, 0x00000000u, 0xffffffffu },
+ { 0x00820078u, 0x00000000u, 0xffffffffu },
+ { 0x0082007cu, 0x00000000u, 0xffffffffu },
+ { 0x00820084u, 0x00000001u, 0xffffffffu },
+ { 0x00820088u, 0x00000000u, 0x00000001u },
+ { 0x008200d0u, 0xffffff8fu, 0xffffffffu },
+ { 0x008200d8u, 0xffffff8fu, 0xffffffffu },
+ { 0x008200dcu, 0xffffff8fu, 0xffffffffu },
+ { 0x008200e0u, 0xffffff8fu, 0xffffffffu },
+ { 0x008200f4u, 0xffffff8fu, 0xffffffffu },
+ { 0x008200f8u, 0xffffff8fu, 0xffffffffu },
+ { 0x008200fcu, 0xffffffffu, 0xffffffffu },
+ { 0x00820350u, 0x00000000u, 0xffffffffu },
+ { 0x00820508u, 0x00000000u, 0xffffffffu },
+ { 0x0082050cu, 0x00000000u, 0xffffffffu },
+ { 0x00820514u, 0x00000000u, 0xffffffffu },
+ { 0x00820564u, 0x00000000u, 0xffffffffu },
+ { 0x008205acu, 0x00000000u, 0xffffffffu },
+ { 0x008205c8u, 0x00000000u, 0xffffffffu },
+ { 0x00820760u, 0x00000001u, 0xffffffffu },
+ { 0x00820764u, 0x00000000u, 0xffffffffu },
+ { 0x00820768u, 0x00000000u, 0xffffffffu },
+ { 0x0082076cu, 0x00000000u, 0xffffffffu },
+ { 0x00820770u, 0x00000000u, 0xffffffffu },
+ { 0x00820774u, 0x00000000u, 0xffffffffu },
+ { 0x00820778u, 0x00000000u, 0xffffffffu },
+ { 0x00820838u, 0x00000000u, 0xffffffffu },
+ { 0x0082083cu, 0x00000000u, 0xffffffffu },
+ { 0x00820840u, 0x00000000u, 0xffffffffu },
+ { 0x00820844u, 0x00000000u, 0xffffffffu },
+ { 0x00820848u, 0x00000000u, 0xffffffffu },
+ { 0x0082084cu, 0x00000000u, 0xffffffffu },
+ { 0x00820850u, 0x00000000u, 0xffffffffu },
+ { 0x00820a40u, 0x00000000u, 0xffffffffu },
+ { 0x00820a44u, 0x00000000u, 0xffffffffu },
+ { 0x00820a48u, 0x00000000u, 0xffffffffu },
+ { 0x00820a4cu, 0x00000000u, 0xffffffffu },
+ { 0x00820a50u, 0x00000000u, 0xffffffffu },
+ { 0x00820a54u, 0x00000000u, 0xffffffffu },
+ { 0x00820a58u, 0x00000000u, 0xffffffffu },
+ { 0x00820b84u, 0x00000000u, 0xffffffffu },
+ { 0x00820c1cu, 0x00000000u, 0xffffffffu },
+ { 0x00820c38u, 0x00000001u, 0xffffffffu },
+ { 0x00820c3cu, 0x00000000u, 0xffffffffu },
+ { 0x00820c40u, 0x00000000u, 0xffffffffu },
+ { 0x00820c44u, 0x00000000u, 0xffffffffu },
+ { 0x00820c48u, 0x00000000u, 0xffffffffu },
+ { 0x00820c4cu, 0x00000000u, 0xffffffffu },
+ { 0x00820c50u, 0x00000000u, 0xffffffffu },
+ { 0x00823804u, 0xffffff8fu, 0xffffffffu },
+ { 0x0082381cu, 0x40171022u, 0xffffffffu },
+ { 0x00823820u, 0x00000002u, 0x0000000fu },
+ { 0x00824148u, 0x00000001u, 0xffffffffu },
+ { 0x008241e0u, 0x00000000u, 0xffffffffu },
+ { 0x00824310u, 0x00000000u, 0xffffffffu },
+ { 0x00824314u, 0x00000000u, 0xffffffffu },
+ { 0x008400f4u, 0x80000000u, 0xffffffffu },
+ { 0x0084010cu, 0x00000000u, 0xffffffffu },
+ { 0x008403c0u, 0x00000000u, 0xffffffffu },
+ { 0x008403c4u, 0xffffffffu, 0xffffffffu },
+ { 0x00840600u, 0x00000000u, 0xffffffffu },
+ { 0x00840624u, 0x00000000u, 0xffffffffu },
+ { 0x00841180u, 0x00000000u, 0xffffffffu },
+ { 0x00841198u, 0x00000000u, 0xffffffffu },
+ { 0x0084119cu, 0x00000000u, 0xffffffffu },
+ { 0x00841210u, 0x00000000u, 0xffffffffu },
+ { 0x00841668u, 0x00000000u, 0xffffffffu },
+};
+static const struct gd_reg_default gd_io_defaults[] = {
+ { 0x00000098u, 0x00000000u, 0xffffffffu },
+ { 0x0000009cu, 0x00000000u, 0xffffffffu },
+ { 0x000000b0u, 0x00000000u, 0xffffffffu },
+ { 0x000000b4u, 0x00000000u, 0xffffffffu },
+ { 0x000000b8u, 0x00000000u, 0xffffffffu },
+ { 0x000000ecu, 0xffffff8fu, 0xffffffffu },
+ { 0x00000128u, 0x00000000u, 0xffffffffu },
+ { 0x00000138u, 0x00000000u, 0xffffffffu },
+ { 0x0000013cu, 0x00000100u, 0xffffffffu },
+ { 0x00000160u, 0x00000000u, 0xffffffffu },
+ { 0x0000022cu, 0xffffffffu, 0xffffffffu },
+ { 0x00000240u, 0x00003000u, 0xffffffffu },
+ { 0x00000244u, 0x40000000u, 0xc0000000u },
+ { 0x00000248u, 0x00000000u, 0x00000000u },
+ { 0x0000024cu, 0x00000000u, 0x00000000u },
+ { 0x00000340u, 0x001c0000u, 0xffffffffu },
+ { 0x00000344u, 0x00000000u, 0xffffffffu },
+ { 0x00000348u, 0x00000000u, 0xffffffffu },
+ { 0x00000350u, 0x00000000u, 0xffffffffu },
+ { 0x00000354u, 0x00000000u, 0xffffffffu },
+ { 0x00000404u, 0x00000000u, 0xffffffffu },
+ { 0x00000408u, 0x00100000u, 0xffffffffu },
+ { 0x00000480u, 0x00000000u, 0xffffffffu },
+ { 0x00000484u, 0x00000000u, 0xffffffffu },
+ { 0x00000490u, 0x00000000u, 0xffffffffu },
+ { 0x00000498u, 0x00000000u, 0xffffffffu },
+ { 0x00000500u, 0x00000000u, 0xffffffffu },
+ { 0x00000504u, 0x00000000u, 0xffffffffu },
+ { 0x0000052cu, 0x00000000u, 0xffffffffu },
+ { 0x00000600u, 0x00000000u, 0xffffffffu },
+ { 0x00000604u, 0x00000000u, 0xffffffffu },
+ { 0x00000608u, 0x00000000u, 0xffffffffu },
+ { 0x00000624u, 0x00000000u, 0xffffffffu },
+ { 0x0000066cu, 0x00000000u, 0xffffffffu },
+ { 0x00000700u, 0x00000000u, 0x803000f3u },
+ { 0x00000704u, 0x00000000u, 0xffffffffu },
+ { 0x00000708u, 0x00000000u, 0xffffffffu },
+ { 0x0000070cu, 0x00000000u, 0xffffffffu },
+ { 0x00000b84u, 0x00000000u, 0xffffffffu },
+ { 0x00000cc0u, 0x00000000u, 0xffffffffu },
+};
+#endif
diff --git a/tests/integration/device/gd.c b/tests/integration/device/gd.c
new file mode 100644
index 0000000..318e339
--- /dev/null
+++ b/tests/integration/device/gd.c
@@ -0,0 +1,3306 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Native GA102 device model. Stage 1 models the host-visible register and
+ * DMA behavior; execution is always by the Falcon core. */
+#include <errno.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "gd.h"
+#include "aes.h"
+#include "booter_regs.h"
+#include "wpr2.h"
+#include "services.h"
+
+#define SEC2_BASE 0x00840000u
+#define GSP_BASE 0x00110000u
+#define ENGINE_WINDOW 0x00004000u
+#define GD_IO_SIZE 0x02000000u
+/* The layout places every reservation below the top 256 MiB of usable
+ * framebuffer; model that whole window so WPR2, the GSP image, the BL and
+ * FRTS all have backing. The base is derived from the FB-size register. */
+#define GD_FB_SIZE (256u << 20)
+#define GD_DEVMAP_BASE 0x01000000u
+#define GD_DEVMAP_SIZE 0x00001000u
+#define PROM_OFFSET 0x00300000u
+#define PROM_SIZE 0x00100000u
+
+#define OFF_MAILBOX0 0x040u
+#define OFF_MAILBOX1 0x044u
+#define OFF_HWCFG2 0x0f4u
+#define OFF_CPUCTL 0x100u
+#define OFF_BOOTVEC 0x104u
+#define OFF_HWCFG 0x108u
+#define OFF_DMACTL 0x10cu
+#define OFF_DMABASE 0x110u
+#define OFF_DMAMOFFS 0x114u
+#define OFF_DMACMD 0x118u
+#define OFF_DMAFBOFFS 0x11cu
+#define OFF_DMABASE1 0x128u
+#define OFF_CPUCTL_ALT 0x130u
+#define OFF_IMEMC 0x180u
+#define OFF_IMEMD 0x184u
+#define OFF_IMEMT 0x188u
+#define OFF_DMEMC 0x1c0u
+#define OFF_DMEMD 0x1c4u
+#define OFF_ENGINE 0x3c0u
+#define OFF_RESET_PLM 0x3c4u
+
+#define GD_DIAG(...) do { } while (0)
+
+#define CPUCTL_START 0x2u
+#define CPUCTL_HALTED 0x10u
+#define HWCFG2_SCRUB (1u << 12)
+#define HWCFG2_RESET (1u << 31)
+
+struct gd_code_rec {
+ uint32_t pc, target;
+ int32_t off, a, b, d, base, idx, scale;
+ uint8_t len, op, sz, sized, bimm, space;
+};
+
+struct gd_engine {
+ struct gd_device *dev;
+ int index;
+ uint32_t base;
+ bool enabled;
+ bool started, halted, faulted;
+ bool scrub_pending;
+ uint64_t scrub_done_us;
+ uint32_t cpuctl, hwcfg2, bootvec, hwcfg, dmactl;
+ uint32_t dma_base, dma_base1, dma_moffs, dma_fboffs, dma_cmd, dma_stat;
+ uint32_t mailbox[2], rm, scratch[2], reset_plm, engine_reg;
+ uint32_t imemc, dmemc, pio_imem, pio_dmem;
+ uint32_t pio_secure_lo, pio_secure_hi;
+ bool csb_enable, csb_valid;
+ uint32_t csb_pc;
+ uint32_t imem_tag;
+ uint8_t imem[0x10000];
+ uint8_t dmem[0x10000];
+ uint8_t devmap[GD_DEVMAP_SIZE];
+ uint32_t io[GD_IO_SIZE / 4];
+ uint8_t io_valid[GD_IO_SIZE / 4];
+ pthread_t thread;
+ pthread_mutex_t lock;
+ pthread_cond_t wake;
+ bool thread_up, stop, busy, pending_kick;
+ pthread_cond_t idle;
+ uint32_t dma_q[64];
+ unsigned qhead, qtail;
+ uint64_t instructions;
+ uint32_t r[16], pc, sp, flags;
+ bool core_fault;
+ uint32_t sr[16];
+ uint32_t call_depth;
+ uint32_t call_pc[64];
+ uint32_t pc_ring[64], pc_ring_at;
+ uint32_t br_pc[64], br_tgt[64], br_sp[64], br_at;
+ uint32_t ret_pc[64], ret_val[64], ret_sp[64], ret_at;
+ uint32_t sh_ret[128], sh_sp[128], sh_at, sh_bad;
+ uint32_t halt_pc, halt_reason;
+ uint32_t token, owner, aux_owner, response, response_ready;
+ uint8_t c[8][16];
+ bool xfer_trap, xfer_shortcut;
+ uint32_t seq_pending;
+ bool crypto_recording;
+ uint8_t crypto_record_slot, crypto_record_left;
+ uint8_t crypto_script_op[2][64], crypto_script_d[2][64], crypto_script_a[2][64], crypto_script_imm[2][64];
+ uint8_t crypto_script_count[2];
+ uint8_t crypto_stream_in[64][16], crypto_stream_out[64][16];
+ uint8_t fwsec_kdf_out[16], fwsec_expect[32], fwsec_digest[16];
+ bool fwsec_kdf_seen, fwsec_digest_seen;
+ uint8_t crypto_stream_in_count, crypto_stream_out_count;
+ uint64_t crypto_rng;
+ uint8_t tlb_flags[256], tlb_virt[256];
+ uint32_t keyreg;
+ uint32_t xfer_count;
+ uint32_t sha_op, sha_status;
+ uint32_t sha_mutex;
+ uint32_t sha_h[8], sha_buf_len, sha_update_bits;
+ uint8_t sha_buf[64];
+ uint64_t sha_total_bits;
+ bool sha_started;
+ bool sha_stream;
+ uint32_t sha_in_addr, sha_in_hi, sha_msg_len;
+ uint32_t sha_result[8];
+ uint32_t doc_addr, doc_data[3];
+ uint32_t dic_count, dic_valid, dic_data[3];
+ uint32_t se_regs[0x8000];
+ struct gd_csec csec;
+ struct gd_doc doc;
+ struct gd_trng trng;
+ struct gd_code_rec *code;
+ uint32_t code_count;
+ int32_t *code_index;
+ bool c_hidden[8];
+};
+
+struct gd_csb {
+ /* NV_SSE_SE_COMMON_MUTEX: 3 groups x 0x50 bytes. ID is an
+ * 8-bit hardware-assigned token, MUTEX has 16 slots, and
+ * ID_RELEASE returns the token. */
+ uint32_t plm[3];
+ uint8_t id[3];
+ uint8_t mutex[3][16];
+ bool released[3];
+};
+
+struct gd_device {
+ const struct gd_board *board;
+ struct gd_run_config config;
+ uint32_t *bar;
+ uint8_t *bar_valid;
+ uint8_t *prom;
+ size_t prom_size;
+ struct gd_engine engine[2];
+ pthread_mutex_t lock;
+ pthread_mutex_t bar_lock;
+ pthread_mutex_t dma_lock;
+ struct gd_dma_span dma[GD_MAX_DMA];
+ uint32_t dma_count;
+ struct gd_csb csb;
+ uint8_t *fb;
+ size_t fb_size;
+ uint64_t fb_base;
+ uint32_t pci_command;
+ uint32_t pci_saved_command;
+ uint32_t devices;
+ uint64_t time_us;
+ int device_error;
+ uint32_t fault_pc, fault_opcode;
+ uint32_t denied_writes;
+ uint32_t plm_violations;
+ bool plm_opened_seen, target_write_seen;
+ uint32_t plm_feat_latched, plm_wpr_cfg_latched;
+ uint8_t secrets[256][16];
+ bool secret_set[256];
+ uint32_t unknown_count;
+ struct gd_unknown unknown[GD_MAX_UNKNOWN];
+ uint64_t host_writes, host_reads, engine_writes, engine_reads;
+ uint32_t launches[2];
+ uint32_t resets;
+ bool flr_seen;
+ bool fwsec_launch_reached, booter_launch_reached;
+ bool fwsec_kdf_ok;
+ bool fwsec_mac_ok;
+ bool wpr2_normalized;
+ uint32_t feats[8];
+ uint32_t feat_count;
+ uint32_t wpr2_lo, wpr2_hi;
+ uint32_t ss0, ss1;
+ struct gd_trace_record trace[GD_MAX_TRACE];
+ uint32_t trace_head, trace_count;
+ pthread_mutex_t trace_lock;
+};
+
+static uint64_t now_us(void);
+
+static void trace_add(struct gd_device *dev, enum gd_trace_kind kind, int source,
+ uint32_t address, uint32_t value, uint32_t pc)
+{
+ uint32_t slot;
+ pthread_mutex_lock(&dev->trace_lock);
+ slot = dev->trace_head++ % GD_MAX_TRACE;
+ if (dev->trace_count < GD_MAX_TRACE)
+ dev->trace_count++;
+ dev->trace[slot].time_us = now_us();
+ dev->trace[slot].kind = (uint16_t)kind;
+ dev->trace[slot].source = (uint16_t)source;
+ dev->trace[slot].address = address;
+ dev->trace[slot].value = value;
+ dev->trace[slot].pc = pc;
+ dev->trace[slot].size = 4;
+ pthread_mutex_unlock(&dev->trace_lock);
+}
+
+static uint64_t now_us(void)
+{
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (uint64_t)ts.tv_sec * 1000000u + ts.tv_nsec / 1000u;
+}
+
+void gd_log(struct gd_device *dev, enum gd_log_level level, const char *format, ...)
+{
+ va_list args;
+ (void)level;
+ if (!dev || !dev->config.verbose)
+ return;
+ va_start(args, format);
+ vfprintf(stderr, format, args);
+ va_end(args);
+}
+
+static void device_error(struct gd_device *dev, const char *what, uint32_t address, uint32_t value)
+{
+ if (!dev->device_error)
+ dev->device_error = -ENODEV;
+ gd_log(dev, GD_LOG_ERROR, "device: %s at %#x value=%#x\n", what, address, value);
+}
+
+static void record_unknown(struct gd_device *dev, bool write, uint32_t addr, uint32_t value, uint32_t pc)
+{
+ if (dev->unknown_count < GD_MAX_UNKNOWN) {
+ struct gd_unknown *u = &dev->unknown[dev->unknown_count++];
+ u->write = write;
+ u->address = addr;
+ u->value = value;
+ u->pc = pc;
+ }
+ device_error(dev, write ? "unknown write" : "unknown read", addr, value);
+}
+
+static struct gd_engine *engine_for(struct gd_device *dev, uint32_t address, uint32_t *offset)
+{
+ uint32_t off;
+
+ (void)off;
+ if (address >= SEC2_BASE && address < SEC2_BASE + ENGINE_WINDOW) {
+ *offset = address - SEC2_BASE;
+ return &dev->engine[0];
+ }
+ if (address >= GSP_BASE && address < GSP_BASE + ENGINE_WINDOW) {
+ *offset = address - GSP_BASE;
+ return &dev->engine[1];
+ }
+ return NULL;
+}
+
+static struct gd_dma_span *dma_find(struct gd_device *dev, uint64_t address, size_t size)
+{
+ uint32_t i;
+ for (i = 0; i < dev->dma_count; i++) {
+ struct gd_dma_span *s = &dev->dma[i];
+ if (s->live && address >= s->address && address + size <= s->address + s->size)
+ return s;
+ }
+ return NULL;
+}
+
+static struct gd_dma_span *dma_find_size(struct gd_device *dev, size_t size)
+{
+ uint32_t i;
+ for (i = 0; i < dev->dma_count; i++)
+ if (dev->dma[i].live && dev->dma[i].size == size)
+ return &dev->dma[i];
+ return NULL;
+}
+
+static void plm_latch_write(struct gd_device *dev, uint32_t address, uint32_t value);
+
+static void set_bar_locked(struct gd_device *dev, uint32_t address, uint32_t value)
+{
+ if (address + 4 > GD_BAR_SIZE) {
+ record_unknown(dev, true, address, value, 0);
+ return;
+ }
+ dev->bar[address / 4] = value;
+ dev->bar_valid[address / 4] = 1;
+ plm_latch_write(dev, address, value);
+}
+
+static void settle_scrub(struct gd_device *dev, struct gd_engine *eng)
+{
+ if (!eng->scrub_pending)
+ return;
+ if (dev->time_us < eng->scrub_done_us)
+ return;
+ eng->scrub_pending = false;
+ eng->hwcfg2 &= ~HWCFG2_SCRUB;
+ memset(eng->imem, 0, sizeof(eng->imem));
+ memset(eng->dmem, 0, sizeof(eng->dmem));
+ gd_log(dev, GD_LOG_DEBUG, "device: engine %d scrub complete\n", eng->index);
+}
+
+static void engine_reset_state(struct gd_device *dev, struct gd_engine *eng)
+{
+ eng->started = eng->faulted = false;
+ /* A CPU reset leaves the core halted and not started; the next
+ * CPUCTL.STARTCPU write clears HALTED. */
+ eng->halted = true;
+ eng->cpuctl = CPUCTL_HALTED;
+ eng->bootvec = 0;
+ eng->dmactl = 0;
+ eng->dma_stat = 2;
+ eng->rm = 0;
+ eng->reset_plm = 0xffffffffu; /* all levels enabled, per RESET_PLM init */
+ /* The CPU reset restores the per-register masks to all-levels; the
+ * Booter locks them again before it starts. */
+ eng->io[0x288 / 4] = 0xffffffffu;
+ eng->io[0x194 / 4] = 0xffffffffu;
+ eng->io[0x28c / 4] = 0xffffffffu;
+ eng->io[0x190 / 4] = 0xffffffffu;
+ eng->io[0x18c / 4] = 0xffffffffu;
+ eng->io[0x2b0 / 4] = 0xffffffffu;
+ eng->io[0x3c4 / 4] = eng->reset_plm;
+ eng->engine_reg = 0;
+ eng->pio_imem = eng->pio_dmem = 0;
+ eng->pio_secure_lo = eng->pio_secure_hi = 0;
+ eng->csb_enable = true;
+ eng->csb_valid = false;
+ eng->csb_pc = 0;
+ eng->sha_started = false;
+ eng->sha_buf_len = 0;
+ eng->sha_update_bits = 0;
+ eng->sha_total_bits = 0;
+ eng->imemc = eng->dmemc = 0;
+ eng->imem_tag = 0;
+ eng->hwcfg2 = HWCFG2_RESET;
+ eng->scrub_pending = false;
+ eng->scrub_done_us = 0;
+ memset(eng->imem, 0, sizeof(eng->imem));
+ memset(eng->dmem, 0, sizeof(eng->dmem));
+ memset(eng->se_regs, 0, sizeof(eng->se_regs));
+ eng->se_regs[0x144 / 4] = 0xffffffffu; /* SE mutex timeout default */
+ eng->se_regs[0x114 / 4] = 1; /* PKA mutex free/available */
+ eng->se_regs[0x4020 / 4] = 1u << 30; /* PKA status IRQ */
+}
+
+static uint32_t mask_of(int sz) { return sz == 32 ? 0xffffffffu : (1u << sz) - 1; }
+
+static void set_r(struct gd_engine *eng, int d, uint32_t value, int sz)
+{
+ uint32_t mask = mask_of(sz);
+ if (sz == 32)
+ eng->r[d] = value;
+ else
+ eng->r[d] = (eng->r[d] & ~mask) | (value & mask);
+}
+
+static void set_flags(struct gd_engine *eng, uint32_t result, int sz, int overflow, int carry)
+{
+ uint32_t f = eng->flags & ~0xe00u;
+ if (carry >= 0)
+ f = (f & ~0x100u) | (uint32_t)carry << 8;
+ eng->flags = f | (uint32_t)overflow << 9 |
+ ((result >> (sz - 1)) & 1) << 10 | (result == 0) << 11;
+}
+
+static int condition(uint32_t f, int cc)
+{
+ int c, o, s, z;
+ if (cc < 8)
+ return (f >> cc) & 1;
+ if (cc >= 0x10 && cc < 0x18)
+ return !((f >> (cc - 0x10)) & 1);
+ c = (f >> 8) & 1; o = (f >> 9) & 1; s = (f >> 10) & 1; z = (f >> 11) & 1;
+ switch (cc) {
+ case 8: return c; case 9: return o; case 10: return s; case 11: return z;
+ case 12: return !c && !z; case 13: return c || z; case 14: return 1;
+ case 24: return !c; case 25: return !o; case 26: return !s; case 27: return !z;
+ case 28: return !(o ^ s) && !z; case 29: return (o ^ s) || z;
+ case 30: return o ^ s; case 31: return !(o ^ s); default: return -1;
+ }
+}
+
+
+/* ---- crypto coprocessor and code TLB ---------------------------------- */
+
+static void crypto_gfmul2(const uint8_t in[16], uint8_t out[16])
+{
+ uint8_t v[16];
+ int carry = in[0] >> 7;
+ for (int i = 0; i < 15; i++)
+ v[i] = (uint8_t)((in[i] << 1) | (in[i + 1] >> 7));
+ v[15] = (uint8_t)(in[15] << 1);
+ if (carry)
+ v[15] ^= 0x87;
+ memcpy(out, v, 16);
+}
+
+static void crypto_record(struct gd_engine *eng, unsigned op, unsigned d,
+ unsigned a, unsigned imm)
+{
+ unsigned slot = eng->crypto_record_slot;
+ unsigned n;
+ if (!eng->crypto_recording || slot > 1)
+ return;
+ n = eng->crypto_script_count[slot];
+ if (n >= 64)
+ return;
+ eng->crypto_script_op[slot][n] = (uint8_t)op;
+ eng->crypto_script_d[slot][n] = (uint8_t)d;
+ eng->crypto_script_a[slot][n] = (uint8_t)a;
+ eng->crypto_script_imm[slot][n] = (uint8_t)imm;
+ eng->crypto_script_count[slot] = (uint8_t)(n + 1);
+ if (eng->crypto_record_left)
+ eng->crypto_record_left--;
+ if (!eng->crypto_record_left)
+ eng->crypto_recording = false;
+}
+
+static void crypto_exec_script(struct gd_engine *eng, unsigned slot, unsigned count);
+
+static void crypto_apply(struct gd_engine *eng, unsigned op, unsigned d,
+ unsigned a, unsigned imm)
+{
+ uint8_t tmp[16], key[16], round[176];
+ uint64_t lo;
+ bool cs_op = op == 86 || op == 87 || op == 88 || op == 104 ||
+ op == 105 || op == 106 || op == 107;
+
+ if (!cs_op)
+ crypto_record(eng, op, d, a, imm);
+ if (eng->crypto_recording && !cs_op)
+ return;
+
+ switch (op) {
+ case 70: /* csecret */
+ memset(eng->c[d], 0, 16);
+ if (imm < 256 && eng->dev->secret_set[imm])
+ memcpy(eng->c[d], eng->dev->secrets[imm], 16);
+ eng->c_hidden[d] = true;
+ break;
+ case 71: /* cenc */
+ gd_aes128_encrypt_block(eng->c[eng->keyreg & 7], eng->c[a], eng->c[d]);
+ eng->c_hidden[d] = eng->c_hidden[eng->keyreg & 7] || eng->c_hidden[a];
+ break;
+ case 72: /* cxor */
+ for (int i = 0; i < 16; i++)
+ eng->c[d][i] ^= eng->c[a][i];
+ eng->c_hidden[d] = eng->c_hidden[d] || eng->c_hidden[a];
+ break;
+ case 73: /* crnd */
+ eng->crypto_rng = eng->crypto_rng * 6364136223846793005ULL + 1;
+ for (int i = 0; i < 16; i++) {
+ eng->crypto_rng = eng->crypto_rng * 6364136223846793005ULL + 1;
+ eng->c[d][i] = (uint8_t)(eng->crypto_rng >> 56);
+ }
+ eng->c_hidden[d] = false;
+ break;
+ case 74: /* ckeyreg */
+ eng->keyreg = d;
+ break;
+ case 75: /* cmov */
+ memcpy(eng->c[d], eng->c[a], 16);
+ eng->c_hidden[d] = eng->c_hidden[a];
+ break;
+ case 81: /* cxsin */
+ if (eng->crypto_stream_in_count) {
+ eng->crypto_stream_in_count--;
+ memcpy(eng->c[d], eng->crypto_stream_in[eng->crypto_stream_in_count], 16);
+ } else {
+ memset(eng->c[d], 0, 16);
+ }
+ eng->c_hidden[d] = false;
+ break;
+ case 82: /* cxsout */
+ if (eng->crypto_stream_out_count < 8) {
+ memcpy(eng->crypto_stream_out[eng->crypto_stream_out_count], eng->c[d], 16);
+ eng->crypto_stream_out_count++;
+ }
+ break;
+ case 86: /* cs0begin */
+ eng->crypto_script_count[0] = 0;
+ eng->crypto_stream_in_count = 0;
+ eng->crypto_stream_out_count = 0;
+ eng->seq_pending = 0;
+ eng->crypto_recording = imm != 0;
+ eng->crypto_record_slot = 0;
+ eng->crypto_record_left = (uint8_t)imm;
+ break;
+ case 87: /* cs0exec: arm the sequencer; data flow drives it */
+ eng->seq_pending = imm;
+ break;
+ case 88: /* csigclr */
+ break;
+ case 96: /* cadd: 64-bit big-endian counter in bytes 8..15 */
+ lo = 0;
+ for (int i = 8; i < 16; i++)
+ lo = (lo << 8) | eng->c[d][i];
+ lo += imm;
+ for (int i = 15; i >= 8; i--) {
+ eng->c[d][i] = (uint8_t)lo;
+ lo >>= 8;
+ }
+ break;
+ case 97: /* cand */
+ for (int i = 0; i < 16; i++)
+ eng->c[d][i] &= eng->c[a][i];
+ eng->c_hidden[d] = eng->c_hidden[d] || eng->c_hidden[a];
+ break;
+ case 98: /* crev */
+ for (int i = 0; i < 16; i++)
+ tmp[i] = eng->c[a][15 - i];
+ memcpy(eng->c[d], tmp, 16);
+ eng->c_hidden[d] = eng->c_hidden[a];
+ break;
+ case 99: /* cgfmul */
+ crypto_gfmul2(eng->c[a], eng->c[d]);
+ eng->c_hidden[d] = eng->c_hidden[a];
+ break;
+ case 100: /* cchmod */
+ break;
+ case 101: /* ckexp: last round key */
+ gd_aes128_expand_key(eng->c[a], round);
+ memcpy(eng->c[d], round + 160, 16);
+ eng->c_hidden[d] = eng->c_hidden[a];
+ break;
+ case 102: /* ckrexp */
+ gd_aes128_reverse_key(eng->c[a], key);
+ memcpy(eng->c[d], key, 16);
+ eng->c_hidden[d] = eng->c_hidden[a];
+ break;
+ case 103: /* cdec */
+ gd_aes128_reverse_key(eng->c[eng->keyreg & 7], key);
+ gd_aes128_decrypt_block(key, eng->c[a], eng->c[d]);
+ eng->c_hidden[d] = eng->c_hidden[eng->keyreg & 7] || eng->c_hidden[a];
+ break;
+ case 104: /* csigcmp: no signature model */
+ break;
+ case 105: /* csigenc: no code signature, zeros */
+ memset(eng->c[d], 0, 16);
+ eng->c_hidden[d] = true;
+ break;
+ case 106: /* cs1begin */
+ eng->crypto_script_count[1] = 0;
+ eng->crypto_recording = imm != 0;
+ eng->crypto_record_slot = 1;
+ eng->crypto_record_left = (uint8_t)imm;
+ break;
+ case 107: /* cs1exec */
+ crypto_exec_script(eng, 1, imm);
+ break;
+ case 120: case 121: case 122: case 123: case 124: case 125:
+ case 126: case 127: case 128: case 129: case 130: case 131:
+ case 132: case 133: case 134: case 135: case 136: case 137:
+ case 138: case 139: case 140: case 141: case 142: {
+ static const uint8_t map[] = {
+ 75, 81, 82, 73, 86, 87, 106, 107, 100, 72, 96, 97,
+ 98, 99, 70, 74, 101, 102, 71, 103, 104, 105, 88
+ };
+ uint32_t v = eng->r[a];
+ crypto_apply(eng, map[op - 120], (v >> 8) & 7, v & 7, v & 0x3f);
+ break;
+ }
+ default:
+ break;
+ }
+ (void)tmp;
+}
+
+static void crypto_exec_script(struct gd_engine *eng, unsigned slot, unsigned count)
+{
+ for (unsigned n = 0; n < count; n++)
+ for (unsigned i = 0; i < eng->crypto_script_count[slot]; i++)
+ crypto_apply(eng, eng->crypto_script_op[slot][i],
+ eng->crypto_script_d[slot][i],
+ eng->crypto_script_a[slot][i],
+ eng->crypto_script_imm[slot][i]);
+}
+
+static void tlb_set(struct gd_engine *eng, unsigned phys, unsigned virt, unsigned flags)
+{
+ if (phys >= 256)
+ return;
+ eng->tlb_virt[phys] = (uint8_t)(virt & 0xff);
+ eng->tlb_flags[phys] = (uint8_t)(flags & 7);
+}
+
+static void tlb_itlb(struct gd_engine *eng, unsigned phys)
+{
+ if (phys < 256 && !(eng->tlb_flags[phys] & 4))
+ tlb_set(eng, phys, 0, 0);
+}
+
+static uint32_t tlb_ptlb(struct gd_engine *eng, unsigned phys)
+{
+ if (phys >= 256)
+ return 0;
+ return (uint32_t)eng->tlb_flags[phys] << 24 | (uint32_t)eng->tlb_virt[phys] << 8;
+}
+
+static uint32_t tlb_vtlb(struct gd_engine *eng, unsigned virt)
+{
+ unsigned page = (virt >> 8) & 0xff;
+ unsigned flags = 0, phys = 0, count = 0;
+ for (unsigned i = 0; i < 256; i++)
+ if (eng->tlb_flags[i] && eng->tlb_virt[i] == page) {
+ flags |= eng->tlb_flags[i];
+ phys = i;
+ count++;
+ }
+ uint32_t res = phys | (flags << 24);
+ if (!count)
+ res |= 0x80000000u;
+ if (count > 1)
+ res |= 0x40000000u;
+ return res;
+}
+
+static uint32_t dmem_read(struct gd_engine *eng, uint32_t address, int sz)
+{
+ uint32_t value = 0;
+ int width = sz / 8, i;
+ if (address >= GD_DEVMAP_BASE && address < GD_DEVMAP_BASE + GD_DEVMAP_SIZE) {
+ uint32_t at = address - GD_DEVMAP_BASE;
+ for (i = 0; i < width && at + i < GD_DEVMAP_SIZE; i++)
+ value |= (uint32_t)eng->devmap[at + i] << (8 * i);
+ return value;
+ }
+ address &= 0xffffu;
+ address &= ~(uint32_t)(width - 1);
+ for (i = 0; i < width; i++)
+ value |= (uint32_t)eng->dmem[(address + i) & 0xffffu] << (8 * i);
+ return value;
+}
+
+static void dmem_write(struct gd_engine *eng, uint32_t address, int sz, uint32_t value)
+{
+ int width = sz / 8, i;
+ if (eng->index == 0 && address == 0xff50)
+ GD_DIAG( "diag: w-ff50 val=%#x sz=%d pc=%#x\n", value, sz, eng->pc);
+ if (eng->index == 0 && address >= 0xff40 && address <= 0xffe0 && value == 0xd44)
+ GD_DIAG( "diag: write-d44 [%#x] pc=%#x sized=%d sz=%d\n", address, eng->pc, 0, sz);
+ if (eng->index == 0 && address >= 0xff40 && address <= 0xffe0 && value == 0x1fce)
+ GD_DIAG( "diag: push1fce [%#x] pc=%#x\n", address, eng->pc);
+ if (eng->index == 0 && address >= 0xffa0 && address <= 0xffc8)
+ GD_DIAG( "diag: dmem-write [%#x]=%#x sz=%d pc=%#x\n", address, value, sz, eng->pc);
+ if (eng->index == 0 && (address == 0x1650 || address == 0x1654 ||
+ address == 0x1658 || address == 0x165c))
+ GD_DIAG( "diag: dmem1650[%#x] = %#x pc=%#x\n", address, value, eng->pc);
+ if (address >= GD_DEVMAP_BASE && address < GD_DEVMAP_BASE + GD_DEVMAP_SIZE) {
+ uint32_t at = address - GD_DEVMAP_BASE;
+ for (i = 0; i < width && at + i < GD_DEVMAP_SIZE; i++)
+ eng->devmap[at + i] = value >> (8 * i);
+ return;
+ }
+ address &= 0xffffu;
+ address &= ~(uint32_t)(width - 1);
+ for (i = 0; i < width; i++)
+ eng->dmem[(address + i) & 0xffffu] = value >> (8 * i);
+}
+
+static uint32_t rotr32(uint32_t x, int n)
+{
+ return (x >> n) | (x << (32 - n));
+}
+
+static void sha256_block(uint32_t h[8], const uint8_t block[64])
+{
+ static const uint32_t k[64] = {
+ 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+ 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+ 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+ 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+ 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+ 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+ 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+ 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+ 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+ 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+ 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+ 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+ 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+ 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+ 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+ 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2,
+ };
+ uint32_t w[64], a, b, c, d, e, f, g, hh;
+ int i;
+
+ for (i = 0; i < 16; i++)
+ w[i] = (uint32_t)block[i * 4] << 24 |
+ (uint32_t)block[i * 4 + 1] << 16 |
+ (uint32_t)block[i * 4 + 2] << 8 |
+ block[i * 4 + 3];
+ for (i = 16; i < 64; i++)
+ w[i] = w[i - 16] + (rotr32(w[i - 15], 7) ^
+ rotr32(w[i - 15], 18) ^ (w[i - 15] >> 3)) +
+ w[i - 7] + (rotr32(w[i - 2], 17) ^
+ rotr32(w[i - 2], 19) ^ (w[i - 2] >> 10));
+ a = h[0]; b = h[1]; c = h[2]; d = h[3];
+ e = h[4]; f = h[5]; g = h[6]; hh = h[7];
+ for (i = 0; i < 64; i++) {
+ uint32_t t1 = hh + (rotr32(e, 6) ^ rotr32(e, 11) ^ rotr32(e, 25)) +
+ ((e & f) ^ (~e & g)) + k[i] + w[i];
+ uint32_t t2 = (rotr32(a, 2) ^ rotr32(a, 13) ^ rotr32(a, 22)) +
+ ((a & b) ^ (a & c) ^ (b & c));
+ hh = g; g = f; f = e; e = d + t1;
+ d = c; c = b; b = a; a = t1 + t2;
+ }
+ h[0] += a; h[1] += b; h[2] += c; h[3] += d;
+ h[4] += e; h[5] += f; h[6] += g; h[7] += hh;
+}
+
+static void sha256_init(uint32_t h[8])
+{
+ h[0] = 0x6a09e667; h[1] = 0xbb67ae85;
+ h[2] = 0x3c6ef372; h[3] = 0xa54ff53a;
+ h[4] = 0x510e527f; h[5] = 0x9b05688c;
+ h[6] = 0x1f83d9ab; h[7] = 0x5be0cd19;
+}
+
+static void sha256_update(struct gd_engine *eng, const uint8_t *data, size_t length)
+{
+ size_t at = 0;
+ while (at < length) {
+ size_t n = 64 - eng->sha_buf_len;
+ if (n > length - at)
+ n = length - at;
+ memcpy(eng->sha_buf + eng->sha_buf_len, data + at, n);
+ eng->sha_buf_len += n;
+ at += n;
+ if (eng->sha_buf_len == 64) {
+ sha256_block(eng->sha_h, eng->sha_buf);
+ eng->sha_buf_len = 0;
+ }
+ }
+}
+
+static void sha256_final(struct gd_engine *eng, uint8_t out[32])
+{
+ uint8_t tail[128] = { 0 };
+ size_t rem = eng->sha_buf_len;
+ uint64_t bits = eng->sha_total_bits;
+
+ memcpy(tail, eng->sha_buf, rem);
+ tail[rem] = 0x80;
+ if (rem >= 56) {
+ for (int i = 0; i < 8; i++)
+ tail[127 - i] = (uint8_t)(bits >> (8 * i));
+ sha256_block(eng->sha_h, tail);
+ sha256_block(eng->sha_h, tail + 64);
+ } else {
+ for (int i = 0; i < 8; i++)
+ tail[63 - i] = (uint8_t)(bits >> (8 * i));
+ sha256_block(eng->sha_h, tail);
+ }
+ for (int i = 0; i < 8; i++) {
+ out[i * 4] = (uint8_t)(eng->sha_h[i] >> 24);
+ out[i * 4 + 1] = (uint8_t)(eng->sha_h[i] >> 16);
+ out[i * 4 + 2] = (uint8_t)(eng->sha_h[i] >> 8);
+ out[i * 4 + 3] = (uint8_t)eng->sha_h[i];
+ }
+}
+
+static void sha_feed(struct gd_engine *eng)
+{
+ size_t length = eng->sha_in_hi & 0xffffffu;
+
+ if (!eng->sha_update_bits)
+ return;
+ if (eng->sha_update_bits && length > eng->sha_update_bits / 8)
+ length = eng->sha_update_bits / 8;
+ if (!eng->sha_started) {
+ sha256_init(eng->sha_h);
+ eng->sha_buf_len = 0;
+ eng->sha_started = true;
+ }
+ if (eng->dev->config.verbose)
+ gd_log(eng->dev, GD_LOG_DEBUG, "device: sha-feed pc=%#x addr=%#x hi=%#x len=%#x total=%#x\n",
+ eng->pc, eng->sha_in_addr, eng->sha_in_hi, eng->sha_update_bits, eng->sha_msg_len);
+ if (length && eng->sha_in_addr + length <= sizeof(eng->dmem)) {
+ if (eng->dev->config.verbose)
+ gd_log(eng->dev, GD_LOG_DEBUG, "device: sha-feed-data addr=%#x len=%#zx bytes=%02x%02x%02x%02x %02x%02x%02x%02x\n",
+ eng->sha_in_addr, length, eng->dmem[eng->sha_in_addr],
+ eng->dmem[eng->sha_in_addr+1], eng->dmem[eng->sha_in_addr+2], eng->dmem[eng->sha_in_addr+3],
+ eng->dmem[eng->sha_in_addr+4], eng->dmem[eng->sha_in_addr+5], eng->dmem[eng->sha_in_addr+6], eng->dmem[eng->sha_in_addr+7]);
+ sha256_update(eng, eng->dmem + eng->sha_in_addr, length);
+ }
+ else if (length) {
+ gd_log(eng->dev, GD_LOG_ERROR,
+ "device: sha update outside DMEM addr=%#x len=%#x\n",
+ eng->sha_in_addr, eng->sha_update_bits);
+ eng->core_fault = true;
+ }
+}
+
+static void sha_publish(struct gd_engine *eng, bool final)
+{
+ uint8_t digest[32];
+ uint32_t hsave[8];
+ uint32_t blen = eng->sha_buf_len;
+ memcpy(hsave, eng->sha_h, sizeof(hsave));
+ sha256_final(eng, digest);
+ if (!final) {
+ memcpy(eng->sha_h, hsave, sizeof(hsave));
+ eng->sha_buf_len = blen;
+ }
+ if (eng->dev->config.verbose) {
+ gd_log(eng->dev, GD_LOG_DEBUG,
+ "device: sha-final pc=%#x total=%#llx stream=%u words=%08x %08x %08x %08x %08x %08x %08x %08x\n",
+ eng->pc, (unsigned long long)eng->sha_total_bits, eng->sha_stream,
+ eng->sha_h[0], eng->sha_h[1], eng->sha_h[2], eng->sha_h[3],
+ eng->sha_h[4], eng->sha_h[5], eng->sha_h[6], eng->sha_h[7]);
+ }
+ for (int i = 0; i < 8; i++)
+ eng->sha_result[i] = (uint32_t)digest[i * 4] << 24 |
+ (uint32_t)digest[i * 4 + 1] << 16 |
+ (uint32_t)digest[i * 4 + 2] << 8 |
+ (uint32_t)digest[i * 4 + 3];
+ eng->sha_status = 0;
+ eng->sha_mutex = 0;
+ if (final) {
+ eng->sha_started = false;
+ eng->sha_buf_len = 0;
+ }
+}
+
+static void sha_run(struct gd_engine *eng)
+{
+ uint32_t op = eng->sha_op;
+
+ if ((op & 0x40u) && !(op & 0x01u)) {
+ if (!eng->sha_started) {
+ sha256_init(eng->sha_h);
+ eng->sha_buf_len = 0;
+ eng->sha_total_bits = eng->sha_msg_len;
+ }
+ sha_publish(eng, true);
+ } else if (op & 0x01u) {
+ if (!eng->sha_started) {
+ sha256_init(eng->sha_h);
+ eng->sha_buf_len = 0;
+ }
+ {
+ uint64_t chunk_bits = (uint64_t)(eng->sha_in_hi & 0xffffffu) * 8u;
+ sha_publish(eng, eng->sha_update_bits <= chunk_bits);
+ }
+ }
+ eng->sha_op = op & ~(0x20u | 0x40u | 0x01u);
+}
+
+static uint32_t se_read(struct gd_engine *eng, uint32_t address)
+{
+ return gd_trng_read(&eng->trng, address);
+}
+
+static void pka_modexp(struct gd_engine *eng, uint32_t mod_addr, uint32_t exp_addr,
+ uint32_t base_addr, uint32_t out_addr);
+
+static void se_write(struct gd_engine *eng, uint32_t address, uint32_t value)
+{
+ if (address >= sizeof(eng->se_regs))
+ return;
+ eng->se_regs[address / 4] = value;
+ if (address == 0x4000 && (value & 0x80000000u)) {
+ if (eng->se_regs[0x4004 / 4] == 0x14)
+ pka_modexp(eng, 0x5000, 0x5400, 0x4400, 0x4400);
+ eng->se_regs[0x4008 / 4] = 0;
+ eng->se_regs[0x4020 / 4] = 0x40000000u;
+ }
+}
+
+
+#define PKA_LIMBS 96
+
+static int pka_cmp(const uint32_t *a, const uint32_t *b)
+{
+ for (int i = PKA_LIMBS - 1; i >= 0; i--) {
+ if (a[i] != b[i])
+ return a[i] > b[i] ? 1 : -1;
+ }
+ return 0;
+}
+
+static void pka_mulmod(const uint32_t a[PKA_LIMBS], const uint32_t b[PKA_LIMBS],
+ const uint32_t mod[PKA_LIMBS], uint32_t out[PKA_LIMBS])
+{
+ uint32_t prod[PKA_LIMBS * 2] = { 0 };
+ uint32_t rem[PKA_LIMBS + 1] = { 0 };
+
+ for (int i = 0; i < PKA_LIMBS; i++) {
+ uint64_t carry = 0;
+ for (int j = 0; j < PKA_LIMBS; j++) {
+ uint64_t cur = (uint64_t)a[i] * b[j] + prod[i + j] + carry;
+ prod[i + j] = (uint32_t)cur;
+ carry = cur >> 32;
+ }
+ int k = i + PKA_LIMBS;
+ while (carry && k < PKA_LIMBS * 2) {
+ uint64_t cur = (uint64_t)prod[k] + carry;
+ prod[k] = (uint32_t)cur;
+ carry = cur >> 32;
+ k++;
+ }
+ }
+
+ for (int bit = PKA_LIMBS * 64 - 1; bit >= 0; bit--) {
+ uint32_t carry = 0;
+ for (int i = 0; i <= PKA_LIMBS; i++) {
+ uint64_t v = ((uint64_t)rem[i] << 1) | carry;
+ rem[i] = (uint32_t)v;
+ carry = (uint32_t)(v >> 32);
+ }
+ rem[0] |= (prod[bit >> 5] >> (bit & 31)) & 1u;
+ if (rem[PKA_LIMBS] || pka_cmp(rem, mod) >= 0) {
+ uint32_t borrow = 0;
+ for (int i = 0; i < PKA_LIMBS; i++) {
+ uint64_t sub = (uint64_t)mod[i] + borrow;
+ uint32_t old = rem[i];
+ rem[i] = old - (uint32_t)sub;
+ borrow = (uint64_t)old < sub;
+ }
+ rem[PKA_LIMBS] -= borrow;
+ }
+ }
+ memcpy(out, rem, PKA_LIMBS * sizeof(uint32_t));
+}
+
+static void pka_modexp(struct gd_engine *eng, uint32_t mod_addr, uint32_t exp_addr,
+ uint32_t base_addr, uint32_t out_addr)
+{
+ if (eng->dev->config.verbose)
+ gd_log(eng->dev, GD_LOG_DEBUG,
+ "device: pka-modexp pc=%#x mod=%#x exp=%#x base=%#x out=%#x\n",
+ eng->pc, mod_addr, exp_addr, base_addr, out_addr);
+ uint32_t mod[PKA_LIMBS], exp[PKA_LIMBS], base[PKA_LIMBS], result[PKA_LIMBS] = { 0 };
+ int high = -1;
+
+ for (int i = 0; i < PKA_LIMBS; i++) {
+ mod[i] = eng->se_regs[(mod_addr / 4) + i];
+ exp[i] = eng->se_regs[(exp_addr / 4) + i];
+ base[i] = eng->se_regs[(base_addr / 4) + i];
+ }
+ for (int i = PKA_LIMBS - 1; i >= 0 && high < 0; i--)
+ for (int bit = 31; bit >= 0; bit--)
+ if (exp[i] & (1u << bit)) {
+ high = i * 32 + bit;
+ break;
+ }
+ result[0] = 1;
+ for (int bit = high; bit >= 0; bit--) {
+ pka_mulmod(result, result, mod, result);
+ if (exp[bit >> 5] & (1u << (bit & 31)))
+ pka_mulmod(result, base, mod, result);
+ }
+ for (int i = 0; i < PKA_LIMBS; i++)
+ eng->se_regs[(out_addr / 4) + i] = result[i];
+ if (eng->dev->config.verbose)
+ gd_log(eng->dev, GD_LOG_DEBUG, "device: pka-result pc=%#x first=%08x %08x %08x %08x\n",
+ eng->pc, result[0], result[1], result[2], result[3]);
+}
+
+static bool plm_mask_register(uint32_t address)
+{
+ switch (address) {
+ case 0x823804: case 0x1fa7c4: case 0x1fa7cc: case 0x1fa7c8:
+ case 0x9a0148: case 0x88ff4: case 0x88ab4: case 0x88ff8:
+ case 0x8200fc: case 0x8200d8: case 0xc840: case 0xc848:
+ return true;
+ default:
+ return false;
+ }
+}
+
+static uint32_t plm_mask_for(uint32_t address)
+{
+ switch (address) {
+ case 0x823818: case 0x82381c: case 0x823820:
+ return 0x823804;
+ /* NV_PFB_PRI_MMU_WPR2_ADDR_LO/HI__PRIV_LEVEL_MASK */
+ case 0x1fa824: case 0x1fa828:
+ return 0x1fa7cc;
+ /* WPR1 and WPR_ALLOW are also gated by WPR_CFG_PLM. */
+ case 0x1fa814: case 0x1fa818: case 0x1fa81c: case 0x1fa820:
+ return 0x1fa7cc;
+ /* NV_PFB_PRI_MMU_LOCK_ADDR_LO/HI__PRIV_LEVEL_MASK */
+ case 0x1fa82c: case 0x1fa830:
+ return 0x1fa7c8;
+ /* NV_PFB_PRI_MMU_LOCAL_MEMORY_RANGE__PRIV_LEVEL_MASK */
+ case 0x100ce0:
+ return 0x1fa7c4;
+ default:
+ return 0;
+ }
+}
+
+static void plm_latch_write(struct gd_device *dev, uint32_t address, uint32_t value)
+{
+ if (address == 0x823804) {
+ dev->plm_feat_latched = value;
+ if (value == 0xffffffffu)
+ dev->target_write_seen = true;
+ if ((value & 0x11u) == 0x11u)
+ dev->plm_opened_seen = true;
+ } else if (address == 0x1fa7cc) {
+ dev->plm_wpr_cfg_latched = value;
+ if ((value & 0xfffff0ffu) == 0xfffff0ffu)
+ dev->target_write_seen = true;
+ if (value & 0x10u)
+ dev->plm_opened_seen = true;
+ }
+}
+
+static bool plm_allowed(struct gd_device *dev, uint32_t address, int privilege, bool write)
+{
+ uint32_t mask_address, mask;
+ int bit;
+
+ if (plm_mask_register(address))
+ return !write || privilege >= 3;
+ mask_address = plm_mask_for(address);
+ if (!mask_address)
+ return true;
+ mask = __atomic_load_n(&dev->bar[mask_address / 4], __ATOMIC_RELAXED);
+ /* SOURCE_ENABLE[31:12] must name at least one source. The individual
+ * source bits are not documented in the available GA102 headers. */
+ if (!(mask >> 12))
+ return false;
+ bit = privilege + (write ? 4 : 0);
+ return (mask >> bit) & 1;
+}
+
+/* Writable mask from the generated GA102 fixture. Registers with a
+ * partial mask (e.g. SS1's low nibble) must not store the reserved bits. */
+static uint32_t bar_write_mask(uint32_t address)
+{
+ size_t i;
+ for (i = 0; i < sizeof(gd_bar0_defaults) / sizeof(gd_bar0_defaults[0]); i++)
+ if (gd_bar0_defaults[i].addr == address)
+ return gd_bar0_defaults[i].mask;
+ return 0xffffffffu;
+}
+
+static uint32_t bar0_raw_read(struct gd_device *dev, uint32_t address)
+{
+ uint32_t value;
+ if (address >= PROM_OFFSET && address < PROM_OFFSET + PROM_SIZE) {
+ const uint8_t *p = dev->prom + (address - PROM_OFFSET);
+ return (uint32_t)p[0] | (uint32_t)p[1] << 8 |
+ (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24;
+ }
+ pthread_mutex_lock(&dev->bar_lock);
+ value = (address + 4 <= GD_BAR_SIZE && dev->bar_valid[address / 4]) ?
+ dev->bar[address / 4] : 0;
+ pthread_mutex_unlock(&dev->bar_lock);
+ if (!(address + 4 <= GD_BAR_SIZE) || !dev->bar_valid[address / 4])
+ gd_log(dev, GD_LOG_DEBUG, "device: unmodelled secure-bus read %#x -> 0\n", address);
+ return value;
+}
+
+static void bar0_raw_write(struct gd_device *dev, uint32_t address, uint32_t value)
+{
+ value &= bar_write_mask(address);
+ pthread_mutex_lock(&dev->bar_lock);
+ if (address + 4 <= GD_BAR_SIZE) {
+ dev->bar[address / 4] = value;
+ dev->bar_valid[address / 4] = 1;
+ }
+ pthread_mutex_unlock(&dev->bar_lock);
+}
+
+static void csb_init(struct gd_device *dev)
+{
+ unsigned g;
+ for (g = 0; g < 3; g++) {
+ dev->csb.plm[g] = g == 2 ? 4u : 0u;
+ /* Hardware-assigned 8-bit token; deliberately not 1. */
+ dev->csb.id[g] = (uint8_t)(0x5a + g);
+ memset(dev->csb.mutex[g], 0, sizeof(dev->csb.mutex[g]));
+ dev->csb.released[g] = false;
+ }
+}
+
+/* Clear every held SE common mutex. Called by the Booter's release
+ * primitive when it sets BAR0 0x1180f8 bit 24. */
+static void csb_release_all(struct gd_device *dev)
+{
+ unsigned g, j;
+ for (g = 0; g < 3; g++) {
+ for (j = 0; j < 16; j++)
+ dev->csb.mutex[g][j] = 0;
+ dev->csb.released[g] = true;
+ }
+}
+
+static bool csb_mutex_slot(uint32_t addr, unsigned *group, unsigned *slot)
+{
+ uint32_t rem;
+ if (addr < 0x8e00 || addr >= 0x8f00)
+ return false;
+ rem = addr - 0x8e00;
+ *group = rem / 0x50;
+ rem %= 0x50;
+ if (rem < 0x0c || rem >= 0x0c + 16 * 4 || ((rem - 0x0c) & 3))
+ return false;
+ *slot = (rem - 0x0c) / 4;
+ return *group < 3;
+}
+
+static bool csb_allowed(struct gd_device *dev, unsigned g, int priv, bool write)
+{
+ uint32_t plm = dev->csb.plm[g];
+ if (priv >= 3)
+ return true;
+ return (plm >> ((priv & 3) + (write ? 4 : 0))) & 1;
+}
+
+static uint32_t csb_read(struct gd_device *dev, uint32_t addr, int priv)
+{
+ unsigned g, slot;
+ uint32_t rem;
+ if (addr >= 0x8e00 && addr < 0x8f00) {
+ rem = (addr - 0x8e00) % 0x50;
+ g = (addr - 0x8e00) / 0x50;
+ if (!csb_allowed(dev, g, priv, false)) {
+ dev->denied_writes++;
+ return 0xbadf0000u;
+ }
+ if (rem == 0x00)
+ return dev->csb.plm[g];
+ if (rem == 0x04)
+ return dev->csb.id[g];
+ if (rem == 0x08)
+ return 0;
+ if (csb_mutex_slot(addr, &g, &slot))
+ return dev->csb.mutex[g][slot];
+ return 0;
+ }
+ return bar0_raw_read(dev, addr);
+}
+
+static void csb_write(struct gd_device *dev, uint32_t addr, uint32_t value, int priv)
+{
+ unsigned g, slot, j;
+ uint32_t rem;
+ if (addr >= 0x8e00 && addr < 0x8f00) {
+ rem = (addr - 0x8e00) % 0x50;
+ g = (addr - 0x8e00) / 0x50;
+ if (!csb_allowed(dev, g, priv, true)) {
+ dev->denied_writes++;
+ dev->plm_violations++;
+ return;
+ }
+ if (rem == 0x00) {
+ dev->csb.plm[g] = value;
+ return;
+ }
+ if (rem == 0x04)
+ return; /* read-only ID */
+ if (rem == 0x08) {
+ /* ID_RELEASE: returning a token releases it and any
+ * slot still carrying it. */
+ if ((value & 0xffu) == dev->csb.id[g]) {
+ for (j = 0; j < 16; j++)
+ if (dev->csb.mutex[g][j] == (value & 0xffu))
+ dev->csb.mutex[g][j] = 0;
+ dev->csb.released[g] = true;
+ }
+ return;
+ }
+ if (csb_mutex_slot(addr, &g, &slot)) {
+ dev->csb.mutex[g][slot] = value & 0xffu;
+ return;
+ }
+ return;
+ }
+ bar0_raw_write(dev, addr, value);
+}
+
+static uint8_t *fb_window(struct gd_device *dev, uint64_t addr, size_t size)
+{
+ if (!dev->fb_base || addr < dev->fb_base || addr - dev->fb_base > dev->fb_size ||
+ size > dev->fb_size - (addr - dev->fb_base))
+ return NULL;
+ return dev->fb + (addr - dev->fb_base);
+}
+
+static uint32_t dio_read(struct gd_device *dev, struct gd_engine *eng, uint32_t addr, int priv)
+{
+ addr &= 0xffffu;
+ if (addr >= 0x8e00 && addr < 0x8f00)
+ return csb_read(dev, addr, priv);
+ return se_read(eng, addr);
+}
+
+static void dio_write(struct gd_device *dev, struct gd_engine *eng, uint32_t addr, uint32_t value, int priv)
+{
+ addr &= 0xffffu;
+ if (addr >= 0x8e00 && addr < 0x8f00)
+ csb_write(dev, addr, value, priv);
+ else
+ se_write(eng, addr, value);
+}
+
+/* Falcon per-register privilege masks (NV_PFALCON_FALCON_*__PRIV_LEVEL_MASK).
+ * Offsets are identical for every Falcon engine; the mask register itself is
+ * L3-write-only and its low bits gate the protected register. */
+static uint32_t engine_plm_off(uint32_t off)
+{
+ switch (off) {
+ case 0x3c0: return 0x3c4; /* ENGINE -> RESET_PRIV_LEVEL_MASK */
+ case 0x100: case 0x130: return 0x288; /* CPUCTL/ALIAS */
+ case 0x104: return 0x194; /* BOOTVEC */
+ case 0x10c: return 0x28c; /* DMACTL */
+ case 0x110: case 0x114: case 0x118: case 0x11c: case 0x128:
+ return 0x190; /* DMATRF* */
+ case 0x180: case 0x184: case 0x188: return 0x18c; /* PMB IMEM */
+ case 0x1c0: case 0x1c4: case 0x1c8: case 0x1cc: return 0x2b0; /* PMB DMEM */
+ default: return 0;
+ }
+}
+
+static bool engine_plm_allowed(struct gd_engine *eng, uint32_t off, int priv, bool write)
+{
+ uint32_t mask_off = engine_plm_off(off);
+ uint32_t mask;
+
+ if (!mask_off)
+ return true;
+ mask = eng->io[mask_off / 4];
+ if (off == mask_off) {
+ /* A PLM register cannot be opened by the level it gates. */
+ if (write)
+ return priv >= 3;
+ return (mask >> (priv & 3)) & 1;
+ }
+ if (!(mask >> 12))
+ return false;
+ return (mask >> ((priv & 3) + (write ? 4 : 0))) & 1;
+}
+
+static uint32_t engine_read_locked(struct gd_device *dev, struct gd_engine *eng, uint32_t off, int priv);
+static bool engine_write_locked(struct gd_device *dev, struct gd_engine *eng, uint32_t off, uint32_t value, int priv);
+
+/* The firmware reaches the same Falcon registers through the secure-bus
+ * (CSEC CSB) view, whose offsets differ from the host PFALCON view. Translate
+ * the CSB aliases onto the one engine state so firmware writes and host reads
+ * observe the same registers. dev_sec_csb.h carries the offsets; the PFALCON
+ * view is defined in dev_falcon_v4.h. */
+static uint32_t csb_alias(uint32_t offset)
+{
+ switch (offset) {
+ case 0x1000: return OFF_MAILBOX0;
+ case 0x1100: return OFF_MAILBOX1;
+ case 0x2100: return 0x084; /* FALCON_RM */
+ case 0x3d00: return OFF_HWCFG2;
+ case 0x4000: return OFF_CPUCTL;
+ case 0x4100: return OFF_BOOTVEC;
+ case 0x4300: return OFF_DMACTL;
+ case 0x4200: return OFF_HWCFG;
+ case 0x4400: return OFF_DMABASE;
+ case 0x4500: return OFF_DMAMOFFS;
+ case 0x4600: return OFF_DMACMD;
+ case 0x4700: return OFF_DMAFBOFFS;
+ case 0x4a00: return OFF_DMABASE1;
+ case 0xf000: return OFF_ENGINE;
+ case 0xf100: return OFF_RESET_PLM;
+ default: return offset;
+ }
+}
+
+static uint32_t local_io(struct gd_device *dev, struct gd_engine *eng, uint32_t offset, uint32_t value, int write)
+{
+ uint32_t result;
+ uint32_t alias = csb_alias(offset & 0xffffffu);
+ if (dev->config.verbose)
+ gd_log(dev, GD_LOG_DEBUG, "device: io %s pc=%#x off=%#x\n",
+ write ? "w" : "r", eng->pc, offset);
+ pthread_mutex_lock(&eng->lock);
+ if (write) {
+ engine_write_locked(dev, eng, alias & (GD_IO_SIZE - 1), value, 3);
+ result = value;
+ } else {
+ result = engine_read_locked(dev, eng, alias & (GD_IO_SIZE - 1), 3);
+ }
+ pthread_mutex_unlock(&eng->lock);
+ return result;
+}
+
+/* Keep the last 64 executed PCs per engine so a halt or fault can be
+ * reconstructed without a per-instruction trace. The ring is written from
+ * the executing thread only. */
+static void engine_pc_history(struct gd_device *dev, struct gd_engine *eng,
+ uint32_t reason, uint32_t pc)
+{
+ if (eng->index == 1 && reason == 1) {
+ uint32_t o;
+ GD_DIAG( "diag: fwsec-final dmem:");
+ for (o = 0; o < 0x20; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->dmem+o));
+ GD_DIAG( "\n hdr2:");
+ for (o = 0x4e0; o < 0x520; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->dmem+o));
+ GD_DIAG( "\n digest:");
+ for (o = 0x35f0; o < 0x3610; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->dmem+o));
+ GD_DIAG( "\n mac:");
+ for (o = 0x540; o < 0x560; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->dmem+o));
+ GD_DIAG( "\n out:");
+ for (o = 0x8df0; o < 0x8e10; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->dmem+o));
+ GD_DIAG( "\n seed:");
+ for (o = 0xd710; o < 0xd720; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->imem+o));
+ GD_DIAG( " fifo in=%u out=%u seen=%d/%d kdf:", eng->crypto_stream_in_count,
+ eng->crypto_stream_out_count, eng->fwsec_kdf_seen, eng->fwsec_digest_seen);
+ for (o = 0; o < 16; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->fwsec_kdf_out+o));
+ GD_DIAG( " expect:");
+ for (o = 0; o < 32; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->fwsec_expect+o));
+ GD_DIAG( " digest:");
+ for (o = 0; o < 16; o += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->fwsec_digest+o));
+ GD_DIAG( "\n");
+ }
+ if (eng->index == 0 && reason == 3) {
+ uint32_t i;
+ GD_DIAG( "diag: retring:");
+ for (i = 0; i < 32; i++) {
+ uint32_t at = (eng->ret_at - 32 + i) & 63;
+ (void)at;
+ GD_DIAG( " [%#x->%#x sp=%#x]", eng->ret_pc[at], eng->ret_val[at], eng->ret_sp[at]);
+ }
+ GD_DIAG( "\n");
+ GD_DIAG( "diag: brring:");
+ for (i = 0; i < 32; i++) {
+ uint32_t at = (eng->br_at - 32 + i) & 63;
+ (void)at;
+ GD_DIAG( " [%#x->%#x sp=%#x]", eng->br_pc[at], eng->br_tgt[at], eng->br_sp[at]);
+ }
+ GD_DIAG( "\n");
+ }
+ uint32_t n = eng->pc_ring_at < 64 ? eng->pc_ring_at : 64;
+ uint32_t first = eng->pc_ring_at - n;
+ uint32_t i;
+
+ eng->halt_pc = pc;
+ eng->halt_reason = reason;
+ trace_add(dev, GD_TRACE_PC, eng->index + 1, 0x10000u | reason, pc, pc);
+ for (i = 0; i < n; i++)
+ trace_add(dev, GD_TRACE_PC, eng->index + 1, i,
+ eng->pc_ring[(first + i) & 63], 0);
+}
+
+static int engine_execute(struct gd_device *dev, struct gd_engine *eng, unsigned budget)
+{
+ unsigned step;
+ uint32_t pc = eng->bootvec & 0xffffu;
+
+ for (step = 0; step < budget; step++) {
+ const struct gd_code_rec *rec;
+ uint32_t d, a, b, result = 0, m, target, address;
+ int sz, overflow = 0, carry = 0, taken = 0;
+ int32_t index;
+
+ eng->pc_ring[eng->pc_ring_at++ & 63] = pc;
+ if (eng->index == 0 && (pc == 0x1fd4 || pc == 0x1fd7 || pc == 0x1fdb)) {
+ uint32_t i;
+ GD_DIAG( "diag: at1fd pc=%#x prev:", pc);
+ for (i = 1; i <= 10; i++)
+ GD_DIAG( " %#x", eng->pc_ring[(eng->pc_ring_at - 1 - i) & 63]);
+ GD_DIAG( " sp=%#x depth=%u r15=%#x\n", eng->sp, eng->call_depth, eng->r[15]);
+ }
+ if (eng->index == 1 && (pc == 0x988 || pc == 0xa69 || pc == 0xa6c || pc == 0xa72)) {
+ GD_DIAG( "diag: seq pc=%#x r10=%#x r11=%#x r14=%#x r9=%#x trap=%d pend=%u in=%u out=%u\n",
+ pc, eng->r[10], eng->r[11], eng->r[14], eng->r[9],
+ eng->xfer_trap, eng->seq_pending,
+ eng->crypto_stream_in_count, eng->crypto_stream_out_count);
+ if (pc == 0xa6c) {
+ uint32_t off = eng->r[14] & 0xffff;
+ (void)off;
+ GD_DIAG( "diag: seq imem@%#x:", off);
+ for (uint32_t k = 0; k < 16; k += 4) GD_DIAG( " %08x", *(uint32_t *)(eng->imem+off+k));
+ GD_DIAG( "\n");
+ }
+ }
+ if (eng->index == 0 && pc == 0x576b) {
+ uint32_t i;
+ GD_DIAG( "diag: at576b prev:");
+ for (i = 1; i <= 8; i++)
+ GD_DIAG( " %#x", eng->pc_ring[(eng->pc_ring_at - 1 - i) & 63]);
+ GD_DIAG( " sp=%#x depth=%u\n", eng->sp, eng->call_depth);
+ }
+ if (eng->index == 0 && pc == 0x1fa3) {
+ uint32_t i;
+ GD_DIAG( "diag: at1fa3 prev:");
+ for (i = 1; i <= 10; i++)
+ GD_DIAG( " %#x", eng->pc_ring[(eng->pc_ring_at - 1 - i) & 63]);
+ GD_DIAG( " sp=%#x depth=%u\n", eng->sp, eng->call_depth);
+ }
+ if (eng->index == 0 && (pc == 0x330f || pc == 0x576b || pc == 0x57b2))
+ GD_DIAG( "diag: callframe pc=%#x sp=%#x w0=%#x w4=%#x w8=%#x depth=%u\n",
+ pc, eng->sp, *(uint32_t *)(eng->dmem+((eng->sp)&0xffff)),
+ *(uint32_t *)(eng->dmem+((eng->sp+4)&0xffff)),
+ *(uint32_t *)(eng->dmem+((eng->sp+8)&0xffff)), eng->call_depth);
+ if (eng->index == 0 && (pc == 0x15cd || pc == 0x193b || pc == 0x1940))
+ GD_DIAG( "diag: retpath pc=%#x sp=%#x w0=%#x w4=%#x depth=%u\n",
+ pc, eng->sp, *(uint32_t *)(eng->dmem+((eng->sp)&0xffff)),
+ *(uint32_t *)(eng->dmem+((eng->sp+4)&0xffff)), eng->call_depth);
+ if (eng->index == 0 && (pc == 0x29d3 || pc == 0x28b1 || pc == 0x28b8 || pc == 0x28aa)) {
+ uint32_t i;
+ GD_DIAG( "diag: errsite %#x r0=%#x r4=%#x r1=%#x r14=%#x r15=%#x r12=%#x r13=%#x stack:",
+ pc, eng->r[0], eng->r[4], eng->r[1], eng->r[14], eng->r[15], eng->r[12], eng->r[13]);
+ for (i = 0; i < eng->call_depth && i < 12; i++)
+ GD_DIAG( " %#x", eng->call_pc[i]);
+ GD_DIAG( " last:");
+ for (i = 0; i < 64; i++)
+ GD_DIAG( " %#x", eng->pc_ring[(eng->pc_ring_at - 64 + i) & 63]);
+ GD_DIAG( "\n wpr2lo=%08x wpr2hi=%08x wprplm=%08x mmucfg=%08x g03=%08x g6a=%08x g6b=%08x\n",
+ dev->bar[0x1fa824/4], dev->bar[0x1fa828/4], dev->bar[0x1fa7c4/4],
+ dev->bar[0x1fa7c8/4], dev->bar[0x11821c/4], dev->bar[0x118244/4],
+ dev->bar[0x118248/4]);
+ GD_DIAG( " d400=%08x %08x d500=%08x %08x d504=%08x d1650=%08x %08x\n",
+ *(uint32_t *)(eng->dmem+0x400), *(uint32_t *)(eng->dmem+0x404),
+ *(uint32_t *)(eng->dmem+0x500), *(uint32_t *)(eng->dmem+0x504),
+ *(uint32_t *)(eng->dmem+0x508),
+ *(uint32_t *)(eng->dmem+0x1650), *(uint32_t *)(eng->dmem+0x1658));
+ }
+ if (eng->index == 0 && (pc == 0x18f9 || pc == 0x1900 || pc == 0x1907 ||
+ pc == 0x190e || pc == 0x1915 || pc == 0x191c)) {
+ uint32_t b4 = eng->r[4];
+ (void)b4;
+ GD_DIAG( "diag: errsite %#x r4=%#x d34=%#x d30=%#x d3c=%#x d38=%#x d44=%#x d40=%#x stack:",
+ pc, b4,
+ (b4+0x34 < 0x10000) ? *(uint32_t *)(eng->dmem+b4+0x34) : 0,
+ (b4+0x30 < 0x10000) ? *(uint32_t *)(eng->dmem+b4+0x30) : 0,
+ (b4+0x3c < 0x10000) ? *(uint32_t *)(eng->dmem+b4+0x3c) : 0,
+ (b4+0x38 < 0x10000) ? *(uint32_t *)(eng->dmem+b4+0x38) : 0,
+ (b4+0x44 < 0x10000) ? *(uint32_t *)(eng->dmem+b4+0x44) : 0,
+ (b4+0x40 < 0x10000) ? *(uint32_t *)(eng->dmem+b4+0x40) : 0);
+ for (uint32_t di = 0; di < eng->call_depth && di < 12; di++)
+ GD_DIAG( " %#x", eng->call_pc[di]);
+ GD_DIAG( "\n");
+ }
+ if (!eng->code_index || pc + 1 > 0x10000) {
+ dev->fault_pc = pc;
+ device_error(dev, "code fetch outside decoded table", pc, 0);
+ eng->core_fault = true;
+ engine_pc_history(dev, eng, 3, pc);
+ return -EFAULT;
+ }
+ index = eng->code_index[pc];
+ if (index < 0) {
+ uint32_t opcode = (uint32_t)eng->imem[pc] | (uint32_t)eng->imem[(pc + 1) & 0xffff] << 8;
+ dev->fault_pc = pc;
+ dev->fault_opcode = opcode;
+ device_error(dev, "no decoded instruction", pc, opcode);
+ eng->core_fault = true;
+ engine_pc_history(dev, eng, 3, pc);
+ return -EINVAL;
+ }
+ rec = &eng->code[index];
+ if (dev->config.verbose && pc == 0x8fc) {
+ gd_log(dev, GD_LOG_DEBUG, "device: frts head %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x\n",
+ eng->dmem[0], eng->dmem[1], eng->dmem[2], eng->dmem[3], eng->dmem[4],
+ eng->dmem[5], eng->dmem[6], eng->dmem[7], eng->dmem[8], eng->dmem[9]);
+ }
+ if (dev->config.verbose && (pc == 0x1382 || pc == 0x13a5 || pc == 0x13c9 || pc == 0x22b || pc == 0x235 || pc == 0x23f || pc == 0x241 || pc == 0x271)) {
+ gd_log(dev, GD_LOG_DEBUG, "device: status pc=%#x r10=%#x r0=%#x\n", pc, eng->r[10], eng->r[0]);
+ }
+ if (dev->config.verbose && (pc == 0x1c74 || pc == 0x1ed4)) {
+ uint32_t v[8];
+ for (int i = 0; i < 8; i++)
+ memcpy(&v[i], eng->dmem + 0x3000 + i * 4, 4);
+ gd_log(dev, GD_LOG_DEBUG,
+ "device: page-state pc=%#x r10=%#x r11=%#x r12=%#x r13=%#x d3000=%08x d3004=%08x d3010=%08x d3014=%08x d301c=%08x d3020=%08x d3024=%08x\n",
+ pc, eng->r[10], eng->r[11], eng->r[12], eng->r[13],
+ v[0], v[1], v[4], v[5], v[7], *(uint32_t *)(eng->dmem+0x3020), *(uint32_t *)(eng->dmem+0x3024));
+ for (unsigned di = 0; di < dev->dma_count; di++)
+ if (dev->dma[di].live)
+ gd_log(dev, GD_LOG_DEBUG, "device: dma-span %u addr=%#llx size=%#zx\n",
+ di, (unsigned long long)dev->dma[di].address, dev->dma[di].size);
+ }
+ if (dev->config.verbose && (pc == 0xa8dd || pc == 0xa95b || pc == 0xa9d2 ||
+ pc == 0xac3b || pc == 0xa6d1)) {
+ gd_log(dev, GD_LOG_DEBUG,
+ "device: sha-call pc=%#x r0=%#x r1=%#x r2=%#x r3=%#x r10=%#x r11=%#x r12=%#x r13=%#x r14=%#x in=%#x len=%#x\n",
+ pc, eng->r[0], eng->r[1], eng->r[2], eng->r[3], eng->r[10], eng->r[11],
+ eng->r[12], eng->r[13], eng->r[14], eng->sha_in_addr, eng->sha_msg_len);
+ }
+ if (dev->config.verbose && pc == 0xd0a1) {
+ uint32_t a = 0, b = 0;
+ if (0x8e04 + 8 < sizeof(eng->dmem)) {
+ memcpy(&a, eng->dmem + 0x8e04, 4);
+ memcpy(&b, eng->dmem + 0x8e08, 4);
+ }
+ gd_log(dev, GD_LOG_DEBUG, "device: stage-buffers d8e04=%08x d8e08=%08x d3600=%08x\n",
+ a, b, *(uint32_t *)(eng->dmem + 0x3600));
+ }
+ if (dev->config.verbose && (pc == 0xcfec || pc == 0xd003 || pc == 0xd08a)) {
+ uint32_t desc = eng->r[pc == 0xcfec ? 10 : 6];
+ uint32_t off = 0, len = 0;
+ if (desc + 12 < sizeof(eng->dmem)) {
+ memcpy(&off, eng->dmem + desc + 4, 4);
+ memcpy(&len, eng->dmem + desc + 8, 4);
+ }
+ gd_log(dev, GD_LOG_DEBUG,
+ "device: cert-desc pc=%#x r10=%#x r11=%#x r6=%#x desc=%#x off=%#x len=%#x\n",
+ pc, eng->r[10], eng->r[11], eng->r[6], desc, off, len);
+ }
+ if (dev->config.verbose && (pc == 0xd08a || pc == 0xd0a1 || pc == 0x1b3c ||
+ pc == 0xa215 || pc == 0xd445 || pc == 0xd49a || pc == 0xd4f6 ||
+ pc == 0xd50f || pc == 0xd51a)) {
+ uint32_t h = 0;
+ if (eng->r[10] + 4 <= sizeof(eng->dmem))
+ memcpy(&h, eng->dmem + eng->r[10], 4);
+ gd_log(dev, GD_LOG_DEBUG,
+ "device: stage pc=%#x r0=%#x r1=%#x r2=%#x r3=%#x r4=%#x r5=%#x r10=%#x r11=%#x r12=%#x hash=%08x\n",
+ pc, eng->r[0], eng->r[1], eng->r[2], eng->r[3], eng->r[4], eng->r[5],
+ eng->r[10], eng->r[11], eng->r[12], h);
+ }
+ if (dev->config.verbose && (pc == 0xa09c || pc == 0xa0a0 || pc == 0xa0a4 || pc == 0xe33)) {
+ unsigned i;
+ gd_log(dev, GD_LOG_DEBUG, "device: sig-check pc=%#x r10=%#x r11=%#x r12=%#x\n",
+ pc, eng->r[10], eng->r[11], eng->r[12]);
+ for (i = 0; i < 32 && i < eng->r[12] && eng->r[10] + i < sizeof(eng->dmem); i += 4) {
+ uint32_t av = 0, bv = 0;
+ memcpy(&av, eng->dmem + eng->r[10] + i, 4);
+ if (eng->r[11] + i < sizeof(eng->dmem))
+ memcpy(&bv, eng->dmem + eng->r[11] + i, 4);
+ gd_log(dev, GD_LOG_DEBUG, "device: sig-data %02x %08x %08x\n", i, av, bv);
+ }
+ }
+ if (dev->config.verbose && (pc == 0xb6ef || pc == 0xb75c || pc == 0xa9b2 ||
+ pc == 0xaa3b || pc == 0xaa4c)) {
+ gd_log(dev, GD_LOG_DEBUG, "device: master-check pc=%#x r1=%#x r2=%#x d0=%#x d1=%#x d2=%#x d3=%#x\n",
+ pc, eng->r[1], eng->r[2],
+ eng->r[1] + 0 < sizeof(eng->dmem) ? *(uint32_t *)(eng->dmem + eng->r[1]) : 0,
+ eng->r[1] + 4 < sizeof(eng->dmem) ? *(uint32_t *)(eng->dmem + eng->r[1] + 4) : 0,
+ eng->r[1] + 8 < sizeof(eng->dmem) ? *(uint32_t *)(eng->dmem + eng->r[1] + 8) : 0,
+ eng->r[1] + 12 < sizeof(eng->dmem) ? *(uint32_t *)(eng->dmem + eng->r[1] + 12) : 0);
+ }
+ if (dev->config.verbose && (pc == 0xd15d || pc == 0xd168 || pc == 0xd16c)) {
+ unsigned off;
+ gd_log(dev, GD_LOG_DEBUG, "device: cert-check pc=%#x r0=%#x r2=%#x r10=%#x r11=%#x r12=%#x\n",
+ pc, eng->r[0], eng->r[2], eng->r[10], eng->r[11], eng->r[12]);
+ for (off = 0; off < 0x20; off += 4) {
+ uint32_t a = eng->r[0] + off, b = eng->r[2] + 0x14 + off;
+ gd_log(dev, GD_LOG_DEBUG, "device: cert hash %02x %08x %08x\n", off,
+ a < sizeof(eng->dmem) ? *(uint32_t *)(eng->dmem + a) : 0,
+ b < sizeof(eng->dmem) ? *(uint32_t *)(eng->dmem + b) : 0);
+ }
+ }
+ if (dev->config.verbose && (pc == 0x2e6 || pc == 0x2ea || pc == 0x2f4 ||
+ pc == 0x74c || pc == 0x2e2)) {
+ gd_log(dev, GD_LOG_DEBUG,
+ "device: fwsec-check pc=%#x r0=%#x r10=%#x r11=%#x r15=%#x\n",
+ pc, eng->r[0], eng->r[10], eng->r[11], eng->r[15]);
+ }
+ if (dev->config.verbose && pc == 0x959) {
+ int i;
+ gd_log(dev, GD_LOG_DEBUG, "device: sigcmp r1=%#x r10=%#x r11=%#x r12=%#x\n",
+ eng->r[1], eng->r[10], eng->r[11], eng->r[12]);
+ gd_log(dev, GD_LOG_DEBUG, "device: sig expected:");
+ for (i = 0; i < 16; i++)
+ gd_log(dev, GD_LOG_DEBUG, " %02x", eng->dmem[0x540 + i]);
+ gd_log(dev, GD_LOG_DEBUG, "\n");
+ gd_log(dev, GD_LOG_DEBUG, "device: sig computed:");
+ for (i = 0; i < 16; i++)
+ gd_log(dev, GD_LOG_DEBUG, " %02x", eng->dmem[0x8df0 + i]);
+ gd_log(dev, GD_LOG_DEBUG, "\n");
+ }
+ sz = rec->sz ? rec->sz : 32;
+ m = mask_of(sz);
+ d = rec->d; a = rec->a; b = rec->b;
+ target = rec->target;
+ switch (rec->op) {
+ case 1: /* mov */
+ if (rec->sized) set_r(eng, d, eng->r[a], sz);
+ else eng->r[d] = a;
+ break;
+ case 2: /* sethi */
+ eng->r[d] = (eng->r[d] & 0xffffu) | (a << 16);
+ break;
+ case 3: case 4: case 5: case 6: { /* add adc sub sbb */
+ uint32_t x = eng->r[a] & m;
+ uint32_t y = (rec->bimm ? b : eng->r[b]) & m;
+ uint32_t cin = (rec->op == 4 || rec->op == 6) ? (eng->flags >> 8) & 1 : 0;
+ uint64_t wide;
+ if (rec->op == 5 || rec->op == 6) {
+ wide = (uint64_t)x - y - cin;
+ carry = (uint64_t)x < (uint64_t)y + cin;
+ result = (uint32_t)wide & m;
+ overflow = (((x ^ y) & (x ^ result)) >> (sz - 1)) & 1;
+ } else {
+ wide = (uint64_t)x + y + cin;
+ carry = (wide >> sz) & 1;
+ result = (uint32_t)wide & m;
+ overflow = (((x ^ result) & (y ^ result)) >> (sz - 1)) & 1;
+ }
+ set_r(eng, d, result, sz);
+ set_flags(eng, result, sz, overflow, carry);
+ break;
+ }
+ case 7: case 8: case 9: /* cmpu cmps cmp */
+ a = eng->r[a] & m;
+ b = (rec->bimm ? b : eng->r[b]) & m;
+ result = (a - b) & m;
+ if (rec->op == 8) {
+ int64_t sa = (int64_t)a - ((uint64_t)(a >> (sz - 1)) << sz);
+ int64_t sb = (int64_t)b - ((uint64_t)(b >> (sz - 1)) << sz);
+ carry = sa < sb;
+ } else carry = a < b;
+ if (rec->op == 9) {
+ overflow = (((a ^ b) & (a ^ result)) >> (sz - 1)) & 1;
+ set_flags(eng, result, sz, overflow, carry);
+ } else eng->flags = (eng->flags & ~0x900u) | (uint32_t)carry << 8 | (result == 0) << 11;
+ break;
+ case 10: case 11: case 12: case 13: case 14: { /* shifts */
+ uint32_t x = eng->r[a] & m;
+ uint32_t n = (rec->bimm ? b : eng->r[b]) & (sz - 1);
+ if (rec->op == 10 || rec->op == 13) {
+ result = (x << n) & m;
+ if (rec->op == 13 && n)
+ result |= ((eng->flags >> 8) & 1) << (n - 1);
+ carry = n ? (x >> (sz - n)) & 1 : 0;
+ } else {
+ result = x >> n;
+ if (rec->op == 14 && n)
+ result |= ((eng->flags >> 8) & 1) << (sz - n);
+ if (rec->op == 12 && (x >> (sz - 1)))
+ result |= (uint32_t)((uint64_t)m << (sz - n)) & m;
+ carry = n ? (x >> (n - 1)) & 1 : 0;
+ }
+ set_r(eng, d, result, sz);
+ set_flags(eng, result, sz, 0, carry);
+ break;
+ }
+ case 15: case 16: case 17: case 18: case 19:
+ a = eng->r[a] & m;
+ if (rec->op == 18) { set_r(eng, d, 0, sz); break; }
+ if (rec->op == 15) result = ~a & m;
+ else if (rec->op == 16) result = -a & m;
+ else if (rec->op == 17) result = ((a >> (sz / 2)) | (a << (sz / 2))) & m;
+ else result = a;
+ if (rec->op != 19) set_r(eng, d, result, sz);
+ set_flags(eng, result, sz, rec->op == 16 && result == (1u << (sz - 1)), -1);
+ break;
+ case 20: case 21: { /* ld st */
+ uint32_t base = rec->base == 16 ? eng->sp : eng->r[rec->base];
+ address = base + (rec->idx >= 0 ? eng->r[rec->idx] * (uint32_t)rec->scale : (uint32_t)rec->off);
+ if (rec->space) {
+ if (rec->op == 20) {
+ result = local_io(dev, eng, address, 0, 0);
+ set_r(eng, d, result, sz);
+ } else {
+ local_io(dev, eng, address, eng->r[a], 1);
+ }
+ } else if (rec->op == 20) {
+ result = dmem_read(eng, address, sz);
+ set_r(eng, d, result, sz);
+ } else {
+ dmem_write(eng, address, sz, eng->r[a]);
+ }
+ break;
+ }
+ case 22: case 23:
+ a = eng->r[a]; b = rec->bimm ? b : eng->r[b];
+ eng->r[d] = rec->op == 22 ? (a & 0xffffu) * (b & 0xffffu) :
+ (uint32_t)((int32_t)(int16_t)a * (int32_t)(int16_t)b);
+ break;
+ case 24:
+ a = eng->r[a]; b = (rec->bimm ? b : eng->r[b]) & 31;
+ result = a & ((1u << b) - 1);
+ if ((a >> b) & 1) result |= 0xffffffffu << b;
+ eng->r[d] = result;
+ eng->flags = (eng->flags & ~0xc00u) | (result >> 31) << 10 | (result == 0) << 11;
+ break;
+ case 25: case 26: {
+ uint32_t low, sm1, bf, fill;
+ a = eng->r[a]; b = rec->bimm ? b : eng->r[b];
+ low = b & 31; sm1 = (b >> 5) & 31;
+ bf = (a >> low) & ((2u << sm1) - 1);
+ fill = rec->op == 26 ? (a >> ((low + sm1) & 31)) & 1 : 0;
+ if (fill) bf |= (uint32_t)(-(2u << sm1));
+ eng->r[d] = bf;
+ eng->flags = (eng->flags & ~0xc00u) | fill << 10 | (bf == 0) << 11;
+ break;
+ }
+ case 27: {
+ uint32_t low, size, fm;
+ b = rec->bimm ? b : eng->r[b];
+ low = b & 31; size = ((b >> 5) & 31) + 1;
+ if (low + size <= 32) {
+ fm = ((1u << size) - 1) << low;
+ eng->r[d] = (eng->r[d] & ~fm) | ((eng->r[a] << low) & fm);
+ }
+ break;
+ }
+ case 28: case 29: case 30:
+ a = eng->r[a]; b = rec->bimm ? b : eng->r[b];
+ result = rec->op == 28 ? a & b : rec->op == 29 ? a | b : a ^ b;
+ eng->r[d] = result;
+ set_flags(eng, result, 32, 0, 0);
+ break;
+ case 31: case 32:
+ b = (rec->bimm ? b : eng->r[b]) & 31;
+ result = ((rec->op == 31 ? eng->r[a] : eng->flags) >> b) & 1;
+ eng->r[d] = result;
+ eng->flags = (eng->flags & ~0xc00u) | (result == 0) << 11;
+ break;
+ case 33: case 34: case 35:
+ b = 1u << ((rec->bimm ? b : eng->r[b]) & 31);
+ if (rec->op == 33) eng->r[d] |= b;
+ else if (rec->op == 34) eng->r[d] &= ~b;
+ else eng->r[d] ^= b;
+ break;
+ case 36: case 37:
+ a = eng->r[a]; b = rec->bimm ? b : eng->r[b];
+ eng->r[d] = rec->op == 36 ? (b ? a / b : 0xffffffffu) : (b ? a % b : a);
+ break;
+ case 38: case 39: case 40:
+ if (rec->op == 39)
+ taken = ((eng->r[a] & m) == (uint32_t)b) != !!d;
+ else if (rec->op == 38)
+ taken = condition(eng->flags, d);
+ else
+ taken = 1;
+ if (taken) {
+ if (rec->op == 40) {
+ eng->br_pc[eng->br_at & 63] = pc;
+ eng->br_tgt[eng->br_at & 63] = rec->bimm ? (uint32_t)a : eng->r[a];
+ eng->br_sp[eng->br_at & 63] = eng->sp;
+ eng->br_at++;
+ if (eng->index == 0 && (eng->br_tgt[(eng->br_at - 1) & 63] == 0x576b ||
+ eng->br_tgt[(eng->br_at - 1) & 63] == 0x1fa3))
+ GD_DIAG( "diag: enter-jmp pc=%#x tgt=%#x sp=%#x\n",
+ pc, eng->br_tgt[(eng->br_at - 1) & 63], eng->sp);
+ pc = (rec->bimm ? (uint32_t)a : eng->r[a]) & 0xffff;
+ }
+ else
+ pc = target & 0xffff;
+ } else {
+ pc = (pc + rec->len) & 0xffff;
+ }
+ break;
+ case 41:
+ eng->sp = (eng->sp + (rec->bimm ? a : eng->r[a])) & ~3u;
+ break;
+ case 42: break;
+ case 43: /* movsr */
+ if (d == 4) eng->sp = eng->r[a] & ~3u;
+ else if (d == 5) eng->pc = rec->pc;
+ else eng->sr[d] = eng->r[a];
+ break;
+ case 44: /* movrs */
+ if (a == 4) eng->r[d] = eng->sp;
+ else if (a == 5) eng->r[d] = rec->pc;
+ else if (a == 8) eng->r[d] = eng->flags;
+ else eng->r[d] = eng->sr[a];
+ break;
+ case 45: case 65: case 66: /* call lbra lcall */
+ eng->br_pc[eng->br_at & 63] = pc;
+ eng->br_tgt[eng->br_at & 63] = (rec->bimm ? (uint32_t)a : eng->r[a]);
+ eng->br_sp[eng->br_at & 63] = eng->sp;
+ eng->br_at++;
+ if (eng->index == 0 && eng->br_tgt[(eng->br_at - 1) & 63] >= 0x5760 &&
+ eng->br_tgt[(eng->br_at - 1) & 63] <= 0x57b5)
+ GD_DIAG( "diag: call-into-57 pc=%#x tgt=%#x op=%d sp=%#x\n",
+ pc, eng->br_tgt[(eng->br_at - 1) & 63], rec->op, eng->sp);
+ if (eng->index == 0 && (eng->br_tgt[(eng->br_at - 1) & 63] == 0x576b ||
+ eng->br_tgt[(eng->br_at - 1) & 63] == 0x1fa3))
+ GD_DIAG( "diag: enter pc=%#x tgt=%#x op=%d sp=%#x bimm=%d a=%d\n",
+ pc, eng->br_tgt[(eng->br_at - 1) & 63], rec->op, eng->sp,
+ rec->bimm, rec->a);
+ if (rec->op == 45 || rec->op == 66) {
+ uint32_t ret = (pc + rec->len) & 0xffff;
+
+ eng->sp -= 4;
+ dmem_write(eng, eng->sp, 32, ret);
+ if (eng->call_depth < 64)
+ eng->call_pc[eng->call_depth] = pc;
+ eng->call_depth++;
+ if (eng->sh_at < 128) {
+ eng->sh_ret[eng->sh_at] = ret;
+ eng->sh_sp[eng->sh_at] = eng->sp;
+ eng->sh_at++;
+ }
+ }
+ pc = (rec->op == 45 ? (rec->bimm ? a : eng->r[a]) : target) & 0xffff;
+ break;
+ case 46: /* ret */
+ {
+ uint32_t rv = dmem_read(eng, eng->sp, 32) & 0xffff;
+ if (eng->sh_at && eng->index == 0) {
+ eng->sh_at--;
+ if ((rv != eng->sh_ret[eng->sh_at] || eng->sp != eng->sh_sp[eng->sh_at]) &&
+ eng->sh_bad++ < 20)
+ GD_DIAG( "diag: SHADOW ret pc=%#x rv=%#x exp=%#x sp=%#x expsp=%#x\n",
+ pc, rv, eng->sh_ret[eng->sh_at], eng->sp, eng->sh_sp[eng->sh_at]);
+ }
+ eng->ret_pc[eng->ret_at & 63] = pc;
+ eng->ret_val[eng->ret_at & 63] = rv;
+ eng->ret_sp[eng->ret_at & 63] = eng->sp;
+ eng->ret_at++;
+ if (eng->index == 0 && pc == 0x1fdb) {
+ uint32_t di;
+ GD_DIAG( "diag: ret1fdb rv=%#x sp=%#x depth=%u calls:", rv, eng->sp, eng->call_depth);
+ for (di = 0; di < eng->call_depth && di < 12; di++)
+ GD_DIAG( " %#x", eng->call_pc[di]);
+ GD_DIAG( "\n");
+ }
+ pc = rv;
+ }
+ eng->sp += 4;
+ if (eng->call_depth) eng->call_depth--;
+ break;
+ case 47: case 48: /* push pop */
+ if (rec->op == 47) {
+ eng->sp -= 4;
+ dmem_write(eng, eng->sp, 32, eng->r[a]);
+ } else {
+ eng->r[d] = dmem_read(eng, eng->sp, 32);
+ eng->sp += 4;
+ }
+ break;
+ case 49: case 50: case 51: case 52: case 53: {
+ int i, count = a + 1;
+ if (rec->op == 49) {
+ for (i = 0; i < count; i++) {
+ eng->sp -= 4;
+ dmem_write(eng, eng->sp, 32, eng->r[i]);
+ }
+ } else {
+ for (i = a; i >= 0; i--) {
+ eng->r[i] = dmem_read(eng, eng->sp, 32);
+ eng->sp += 4;
+ }
+ if (rec->op == 52 || rec->op == 53)
+ eng->sp = (eng->sp + (uint32_t)b) & ~3u;
+ if (rec->op == 51 || rec->op == 53) {
+ uint32_t rv = dmem_read(eng, eng->sp, 32) & 0xffff;
+ if (eng->sh_at && eng->index == 0) {
+ eng->sh_at--;
+ if ((rv != eng->sh_ret[eng->sh_at] || eng->sp != eng->sh_sp[eng->sh_at]) &&
+ eng->sh_bad++ < 20)
+ GD_DIAG( "diag: SHADOW mpopret pc=%#x rv=%#x exp=%#x sp=%#x expsp=%#x\n",
+ pc, rv, eng->sh_ret[eng->sh_at], eng->sp, eng->sh_sp[eng->sh_at]);
+ }
+ eng->ret_pc[eng->ret_at & 63] = pc;
+ eng->ret_val[eng->ret_at & 63] = rv;
+ eng->ret_sp[eng->ret_at & 63] = eng->sp;
+ eng->ret_at++;
+ if (eng->index == 0 && pc == 0x1fdb) {
+ uint32_t di;
+ GD_DIAG( "diag: ret1fdb mpop op=%d rv=%#x sp=%#x depth=%u calls:", rec->op, rv, eng->sp, eng->call_depth);
+ for (di = 0; di < eng->call_depth && di < 12; di++)
+ GD_DIAG( " %#x", eng->call_pc[di]);
+ GD_DIAG( "\n");
+ }
+ pc = rv;
+ eng->sp += 4;
+ if (eng->call_depth) eng->call_depth--;
+ }
+ }
+ break;
+ }
+ case 60: case 61: /* iord/iords */
+ address = eng->r[rec->base] + (rec->idx >= 0 ? eng->r[rec->idx] * (uint32_t)rec->scale : (uint32_t)rec->off);
+ result = local_io(dev, eng, address, 0, 0);
+ set_r(eng, d, result, 32);
+ break;
+ case 62: /* iowr/iowrs */
+ address = eng->r[rec->base] + (rec->idx >= 0 ? eng->r[rec->idx] * (uint32_t)rec->scale : (uint32_t)rec->off);
+ if (eng->index == 0 && (address & 0xffffu) == 0x1000u) {
+ uint32_t di, dn = eng->call_depth < 12 ? eng->call_depth : 12;
+ GD_DIAG( "diag: status %#x pc=%#x stack:", eng->r[a], pc);
+ for (di = 0; di < dn; di++)
+ GD_DIAG( " %#x", eng->call_pc[di]);
+ GD_DIAG( "\n");
+ }
+ local_io(dev, eng, address, eng->r[a], 1);
+ break;
+ case 63: /* exit */
+ gd_log(dev, GD_LOG_DEBUG, "device: engine %d exit at %#x after %llu instructions\n",
+ eng->index, pc, (unsigned long long)eng->instructions);
+ eng->pc = pc;
+ eng->halted = true;
+ eng->cpuctl = CPUCTL_HALTED;
+ trace_add(dev, GD_TRACE_HALT, eng->index + 1, eng->base, pc, pc);
+ engine_pc_history(dev, eng, 1, pc);
+ __atomic_add_fetch(&eng->instructions, 1, __ATOMIC_RELAXED);
+ return 0;
+ case 64: /* trap */
+ dev->fault_pc = pc;
+ device_error(dev, "software trap", pc, a);
+ eng->core_fault = true;
+ engine_pc_history(dev, eng, 4, pc);
+ return -EINVAL;
+ case 67: /* sleep */
+ if ((eng->flags >> a) & 1u) {
+ eng->pc = pc;
+ eng->halted = true;
+ eng->cpuctl = CPUCTL_HALTED;
+ trace_add(dev, GD_TRACE_HALT, eng->index + 1, eng->base, pc, pc);
+ engine_pc_history(dev, eng, 2, pc);
+ __atomic_add_fetch(&eng->instructions, 1, __ATOMIC_RELAXED);
+ return 0;
+ }
+ break;
+ case 70: case 71: case 72: case 73: case 74: case 75:
+ crypto_apply(eng, rec->op, d, a, b);
+ break;
+ case 76: case 77: /* xdst / xdld */
+ {
+ uint32_t bv = eng->r[b];
+ uint32_t av = eng->r[a];
+ if (eng->xfer_trap) {
+ uint32_t local = bv & 0xffffu;
+ uint32_t size = 4u << ((bv >> 16) & 7);
+
+ if (local + size > sizeof(eng->imem)) {
+ device_error(dev, "SCP transfer overrun", local, size);
+ eng->core_fault = true;
+ return -EINVAL;
+ }
+ if (rec->op == 76) {
+ uint32_t off;
+ for (off = 0; off + 16 <= size && eng->crypto_stream_in_count < 64; off += 16) {
+ memcpy(eng->crypto_stream_in[eng->crypto_stream_in_count],
+ eng->imem + local + off, 16);
+ eng->crypto_stream_in_count++;
+ }
+ eng->crypto_stream_out_count = 0;
+ while (eng->seq_pending && eng->crypto_stream_in_count) {
+ crypto_exec_script(eng, 0, 1);
+ eng->seq_pending--;
+ }
+ } else {
+ uint32_t off;
+ uint32_t at = 0;
+ for (off = 0; off + 16 <= size && at < eng->crypto_stream_out_count; off += 16) {
+ if (eng->index == 0 && ((local + off) & 0xffff) == 0xff50)
+ GD_DIAG( "diag: trap-xdld-ff50 pc=%#x block=%08x\n", pc,
+ *(uint32_t *)eng->crypto_stream_out[at]);
+ memcpy(eng->dmem + ((local + off) & 0xffff),
+ eng->crypto_stream_out[at], 16);
+ at++;
+ }
+ eng->crypto_stream_out_count = 0;
+ }
+ if (eng->xfer_count != UINT32_MAX && eng->xfer_count)
+ eng->xfer_count--;
+ break;
+ }
+ if (eng->xfer_count) {
+ uint32_t creg = (bv >> 16) & 7;
+ uint32_t local = bv & 0xffff;
+ gd_log(dev, GD_LOG_DEBUG, "device: crypto %s pc=%#x b=%#x count=%u\n",
+ rec->op == 76 ? "xdst" : "xdld", pc, b, eng->xfer_count);
+ if (rec->op == 76) {
+ for (int i = 0; i < 16; i++)
+ eng->c[creg][i] = eng->dmem[(local + i) & 0xffff];
+ eng->c_hidden[creg] = false;
+ } else {
+ for (int i = 0; i < 16; i++)
+ eng->dmem[(local + i) & 0xffff] = eng->c[creg][i];
+ if (eng->index == 1 && local == 0x8df0) {
+ memcpy(eng->fwsec_kdf_out, eng->c[creg], 16);
+ memcpy(eng->fwsec_expect, eng->dmem + 0x540, 32);
+ eng->fwsec_kdf_seen = true;
+ }
+ if (eng->index == 1 && local == 0x35f0) {
+ memcpy(eng->fwsec_digest, eng->c[creg], 16);
+ eng->fwsec_digest_seen = true;
+ }
+ }
+ eng->xfer_count--;
+ } else {
+ uint64_t base = ((uint64_t)eng->sr[15] << 32) | eng->sr[7];
+ uint64_t ext = (base << 8) + av;
+ uint32_t port = (rec->op == 77) ? (eng->sr[11] >> 8) & 7 :
+ (eng->sr[11] >> 12) & 7;
+ uint32_t local = bv & 0xffff;
+ uint32_t size = 4u << ((bv >> 16) & 7);
+ uint32_t i;
+ struct gd_dma_span *span = NULL;
+ if (dev->config.verbose)
+ gd_log(dev, GD_LOG_DEBUG,
+ "device: xfer %s pc=%#x port=%u ext=%#llx local=%#x size=%#x\n",
+ rec->op == 76 ? "xdst" : "xdld", pc, port,
+ (unsigned long long)ext, local, size);
+ if (local + size > sizeof(eng->dmem)) {
+ device_error(dev, "XDMA DMEM overrun", local, size);
+ eng->core_fault = true;
+ return -EINVAL;
+ }
+ uint8_t *fb = fb_window(dev, ext, size);
+ if (fb) {
+ if (eng->index == 0 && local <= 0xff50 && local + size > 0xff50)
+ GD_DIAG( "diag: ext-xdld-fb-ff50 pc=%#x size=%#x\n", pc, size);
+ if (rec->op == 77)
+ memcpy(eng->dmem + local, fb, size);
+ else
+ memcpy(fb, eng->dmem + local, size);
+ } else {
+ pthread_mutex_lock(&dev->dma_lock);
+ span = dma_find(dev, ext, size);
+ if (span) {
+ if (eng->index == 0 && local <= 0xff50 && local + size > 0xff50)
+ GD_DIAG( "diag: ext-xdld-span-ff50 pc=%#x size=%#x\n", pc, size);
+ if (rec->op == 77)
+ memcpy(eng->dmem + local, span->cpu + (ext - span->address), size);
+ else
+ memcpy(span->cpu + (ext - span->address), eng->dmem + local, size);
+ }
+ pthread_mutex_unlock(&dev->dma_lock);
+ if (!span) {
+ for (i = 0; i < size; i += 4) {
+ if (rec->op == 77) {
+ uint32_t v = bar0_raw_read(dev, (uint32_t)ext + i);
+ if (eng->index == 0 && local + i <= 0xff50 && local + i + 4 > 0xff50)
+ GD_DIAG( "diag: ext-xdld-bar-ff50 pc=%#x v=%#x\n", pc, v);
+ memcpy(eng->dmem + local + i, &v, 4);
+ } else {
+ uint32_t v;
+ memcpy(&v, eng->dmem + local + i, 4);
+ bar0_raw_write(dev, (uint32_t)ext + i, v);
+ }
+ }
+ }
+ }
+ }
+ }
+ break;
+ case 78: case 79: case 80:
+ break; /* xdwait/xdfence/xcwait */
+ case 81: case 82: case 86: case 87: case 88:
+ crypto_apply(eng, rec->op, d, a, b);
+ break;
+ case 83: /* vtlb */
+ eng->r[d] = tlb_vtlb(eng, eng->r[a] & 0xffffffu);
+ break;
+ case 84: /* itlb */
+ tlb_itlb(eng, eng->r[a] & 0xffffffu);
+ break;
+ case 85: /* ptlb */
+ eng->r[d] = tlb_ptlb(eng, eng->r[a] & 0xffffffu);
+ break;
+ case 94: { /* iret */
+ eng->pc = pc = dmem_read(eng, eng->sp, 32) & 0xffffu;
+ eng->sp += 4;
+ uint32_t is = (eng->flags >> 20) & 7u;
+ eng->flags = (eng->flags & ~(7u << 16)) | (is << 16);
+ break;
+ }
+ case 95: { /* xcld: code-load XDMA */
+ uint32_t av = eng->r[a], bv = eng->r[b];
+ uint64_t base = ((uint64_t)eng->sr[14] << 32) | eng->sr[6];
+ uint64_t ext = (base << 8) + av;
+ uint32_t local = bv & 0xffffu;
+ if (local + 0x100u > sizeof(eng->imem)) {
+ device_error(dev, "XCLD IMEM overrun", local, 0x100);
+ eng->core_fault = true;
+ return -EINVAL;
+ }
+ for (uint32_t i = 0; i < 0x100; i += 4) {
+ uint8_t *fbw = fb_window(dev, ext + i, 4);
+ if (fbw) memcpy(eng->imem + local + i, fbw, 4);
+ else {
+ struct gd_dma_span *sp = dma_find(dev, ext + i, 4);
+ if (sp) memcpy(eng->imem + local + i, sp->cpu + (ext + i - sp->address), 4);
+ else {
+ uint32_t v = bar0_raw_read(dev, (uint32_t)(ext + i));
+ memcpy(eng->imem + local + i, &v, 4);
+ }
+ }
+ }
+ tlb_set(eng, local >> 8, local >> 8, (eng->sr[10] >> 16 & 1) ? 4 : 1);
+ break;
+ }
+ case 96: case 97: case 98: case 99: case 100: case 101:
+ case 102: case 103: case 104: case 105: case 106: case 107:
+ case 120: case 121: case 122: case 123: case 124: case 125:
+ case 126: case 127: case 128: case 129: case 130: case 131:
+ case 132: case 133: case 134: case 135: case 136: case 137:
+ case 138: case 139: case 140: case 141: case 142:
+ crypto_apply(eng, rec->op, d, a, b);
+ break;
+ case 89: case 90: case 91: {
+ uint32_t bit = 1u << ((rec->bimm ? b : eng->r[b]) & 31);
+ if (rec->op == 89) eng->flags |= bit;
+ else if (rec->op == 90) eng->flags &= ~bit;
+ else eng->flags ^= bit;
+ break;
+ }
+ case 93: /* cxset: bit7 trap, bit5 shortcut, low 5 bits count */
+ {
+ uint32_t count = a & 0x1f;
+
+ eng->xfer_trap = !!(a & 0x20); /* 0xa1 and 0x22 */
+ eng->xfer_shortcut = !!(a & 0x80); /* 0xa1 */
+ if (count == 0x1f)
+ eng->xfer_count = UINT32_MAX;
+ else if (!count)
+ eng->xfer_count = 0;
+ else
+ eng->xfer_count = count; /* hardware counts transfers */
+ }
+ break;
+ case 92: {
+ uint32_t bit = (rec->bimm ? b : eng->r[b]) & 31;
+ eng->flags = (eng->flags & ~(1u << bit)) | ((eng->r[a] & 1) << bit);
+ break;
+ }
+ default:
+ dev->fault_pc = pc;
+ device_error(dev, "unimplemented opcode", pc, rec->op);
+ eng->core_fault = true;
+ return -EINVAL;
+ }
+ if (rec->op != 38 && rec->op != 39 && rec->op != 40 &&
+ rec->op != 45 && rec->op != 46 && rec->op != 51 && rec->op != 53 &&
+ rec->op != 65 && rec->op != 66)
+ pc = (pc + rec->len) & 0xffff;
+ eng->pc = pc;
+ __atomic_add_fetch(&eng->instructions, 1, __ATOMIC_RELAXED);
+ }
+ dev->fault_pc = pc;
+ device_error(dev, "Falcon instruction budget exhausted", pc, budget);
+ eng->core_fault = true;
+ engine_pc_history(dev, eng, 5, pc);
+ return -ETIMEDOUT;
+}
+
+static void load_code_table(struct gd_engine *eng, const char *path)
+{
+ FILE *file;
+ uint32_t header[3];
+ size_t i;
+ struct gd_device *dev = eng->dev;
+ if (!path)
+ return;
+ file = fopen(path, "rb");
+ if (!file) {
+ device_error(dev, "cannot open decoded table", 0, 0);
+ return;
+ }
+ if (fread(header, sizeof(header), 1, file) != 1 || header[0] != 0x31434447u) {
+ fclose(file);
+ device_error(dev, "bad decoded table", 0, 0);
+ return;
+ }
+ eng->code_count = header[1];
+ eng->code = calloc(eng->code_count, sizeof(*eng->code));
+ eng->code_index = malloc(0x10000 * sizeof(*eng->code_index));
+ if (!eng->code || !eng->code_index) {
+ fclose(file);
+ return;
+ }
+ for (i = 0; i < 0x10000; i++)
+ eng->code_index[i] = -1;
+ for (i = 0; i < eng->code_count; i++) {
+ if (fread(&eng->code[i], sizeof(eng->code[i]), 1, file) != 1) {
+ eng->code_count = i;
+ break;
+ }
+ if (eng->code[i].pc < 0x10000)
+ eng->code_index[eng->code[i].pc] = (int32_t)i;
+ }
+ fclose(file);
+}
+
+static bool decrypt_pio_code(struct gd_device *dev, struct gd_engine *eng)
+{
+ uint32_t lo = eng->pio_secure_lo, hi = eng->pio_secure_hi;
+
+ if (hi <= lo)
+ return true;
+ if (!dev->config.key_set || (lo & 15) || ((hi - lo) & 15)) {
+ device_error(dev, "secure PIO code range is not AES-block aligned", lo, hi - lo);
+ return false;
+ }
+ if (gd_aes128_decrypt(dev->config.key, eng->imem + lo, hi - lo) != 0) {
+ device_error(dev, "secure PIO code decrypt failed", lo, hi - lo);
+ return false;
+ }
+ eng->pio_secure_lo = eng->pio_secure_hi = 0;
+ return true;
+}
+
+static void engine_kick(struct gd_device *dev, struct gd_engine *eng)
+{
+ {
+ if (eng->index == 0) {
+ struct gd_dma_span *sp = dma_find_size(dev, 0x100);
+ if (sp) {
+ uint32_t i;
+ GD_DIAG( "diag: wprmeta:");
+ for (i = 0; i < 200; i += 8)
+ GD_DIAG( " %016llx", (unsigned long long)*(uint64_t *)(sp->cpu+i));
+ GD_DIAG( "\n");
+ }
+ }
+ if (eng->index == 0 && (dev->bar[0x11821c / 4] & (1u << 17))) {
+ uint32_t lo = dev->bar[0x1fa824 / 4], hi = dev->bar[0x1fa828 / 4];
+ uint32_t want;
+
+ /* The debug FWSEC image publishes an inclusive 4 KiB FRTS
+ * end; the 610 Booter decodes WPR2_HI with the 128 KiB
+ * alignment term from booter_wpr2_tu10x.c. Convert the
+ * encoding and record that it happened. */
+ if (!gd_wpr2_fwsec_hi(lo, &want) && want != hi) {
+ dev->bar[0x1fa828 / 4] = want;
+ dev->wpr2_normalized = true;
+ }
+ }
+ if (eng->index == 0) {
+ unsigned o;
+ GD_DIAG( "diag: scratch:");
+ for (o = 0x1400; o <= 0x1440; o += 4)
+ GD_DIAG( " %#x=%#x", o, dev->bar[o / 4]);
+ GD_DIAG( "\n");
+ }
+
+ if (eng->index == 1)
+ dev->fwsec_launch_reached = true;
+ else
+ dev->booter_launch_reached = true;
+ if (!dev->config.key_set) {
+ device_error(dev, "debug firmware key missing; execution refused", 0, 0);
+ eng->core_fault = true;
+ eng->started = true;
+ eng->halted = true;
+ eng->cpuctl = CPUCTL_HALTED;
+ return;
+ }
+ /* The real FWSEC needs more than the old 1M-instruction smoke
+ * budget after the SE PKA path; keep the run bounded. */
+ if (!decrypt_pio_code(dev, eng) ||
+ engine_execute(dev, eng, 100000000) != 0) {
+ eng->core_fault = true;
+ eng->halted = true;
+ eng->cpuctl = CPUCTL_HALTED;
+ return;
+ }
+ }
+ eng->started = true;
+ eng->halted = true;
+ eng->cpuctl = CPUCTL_HALTED;
+ __atomic_add_fetch(&dev->launches[eng->index], 1, __ATOMIC_RELAXED);
+ __atomic_add_fetch(&eng->instructions, 1, __ATOMIC_RELAXED);
+ trace_add(dev, GD_TRACE_HALT, eng->index + 1, eng->base, eng->reset_plm, 0);
+}
+
+/* Resolve a DMA address against framebuffer or coherent sysmem. The same
+ * address can only belong to one; a miss is a modelling gap, never a no-op. */
+static uint8_t *dma_target(struct gd_device *dev, uint64_t address, size_t size,
+ struct gd_dma_span **span_out)
+{
+ struct gd_dma_span *span;
+ uint8_t *fb = fb_window(dev, address, size);
+
+ if (fb) {
+ *span_out = NULL;
+ return fb;
+ }
+ span = dma_find(dev, address, size);
+ *span_out = span;
+ if (!span)
+ return NULL;
+ if ((uint64_t)(address - span->address) + size > span->size)
+ return NULL;
+ return span->cpu + (address - span->address);
+}
+
+/* Falcon DMATRFCMD: FINAL/IDLE status, SEC (2), IMEM (4), WRITE (5),
+ * SIZE (10:8, byte count 4<<n), CTXDMA (14:12). */
+static void dma_execute(struct gd_device *dev, struct gd_engine *eng, uint32_t cmd)
+{
+ uint64_t address;
+ size_t size;
+ bool imem = !!(cmd & (1u << 4));
+ bool store = !!(cmd & (1u << 5));
+ uint32_t base = eng->dma_base, base1 = eng->dma_base1;
+ uint32_t moffs = eng->dma_moffs, fboffs = eng->dma_fboffs;
+ struct gd_dma_span *span = NULL;
+ uint8_t decoded[512];
+ uint8_t *target;
+ size_t limit = imem ? sizeof(eng->imem) : sizeof(eng->dmem);
+
+ size = 4u << ((cmd >> 8) & 7);
+ if (size > sizeof(decoded)) {
+ device_error(dev, "DMA size", moffs, cmd);
+ return;
+ }
+ address = ((uint64_t)(base1 & 0x1ffu) << 32 | base) << 8;
+ address += fboffs;
+ if (size == 0x100 && moffs >= 0x4e00)
+ GD_DIAG( "diag: dmabounce eng=%d src=%#llx local=%#x cmd=%#x wr=%d base=%#x base1=%#x fboffs=%#x pc=%#x\n",
+ eng->index, (unsigned long long)address, moffs, cmd, store, base, base1, fboffs, eng->pc);
+ if (size == 0x100 && moffs == 0x400)
+ GD_DIAG( "diag: dma100 src=%#llx store=%d eng=%d pc=%#x\n",
+ (unsigned long long)address, store, eng->index, eng->pc);
+ if ((size_t)moffs + size > limit) {
+ device_error(dev, imem ? "DMA IMEM overrun" : "DMA DMEM overrun", moffs, cmd);
+ return;
+ }
+ pthread_mutex_lock(&dev->dma_lock);
+ target = dma_target(dev, address, size, &span);
+ if (!target) {
+ pthread_mutex_unlock(&dev->dma_lock);
+ device_error(dev, store ? "DMA destination unmapped" : "DMA source unmapped",
+ (uint32_t)address, cmd);
+ return;
+ }
+ if (store) {
+ memcpy(target, (imem ? eng->imem : eng->dmem) + moffs, size);
+ } else {
+ memcpy(decoded, target, size);
+ if (imem && (cmd & (1u << 2))) {
+ if (!dev->config.key_set ||
+ gd_aes128_decrypt(dev->config.key, decoded, size) != 0) {
+ pthread_mutex_unlock(&dev->dma_lock);
+ device_error(dev, "secure code decrypt failed", moffs, cmd);
+ return;
+ }
+ }
+ if (!imem && moffs <= 0xffe0 && moffs + size > 0xff40)
+ GD_DIAG( "diag: dma-stack moffs=%#x size=%zu src=%#llx cmd=%#x pc=%#x\n",
+ moffs, size, (unsigned long long)address, cmd, eng->pc);
+ memcpy((imem ? eng->imem : eng->dmem) + moffs, decoded, size);
+ if (imem) {
+ eng->imem_tag = fboffs >> 8;
+ tlb_set(eng, moffs >> 8, fboffs >> 8,
+ (cmd & (1u << 2)) ? 4 : 1);
+ }
+ }
+ pthread_mutex_unlock(&dev->dma_lock);
+ dev->engine_writes++;
+ trace_add(dev, GD_TRACE_DMA, eng->index + 1, (uint32_t)address, cmd, 0);
+ trace_add(dev, GD_TRACE_HALT, eng->index + 1, eng->base, eng->reset_plm, 0);
+}
+
+static void *engine_thread(void *arg)
+{
+ struct gd_engine *eng = arg;
+ struct gd_device *dev = eng->dev;
+
+ pthread_mutex_lock(&eng->lock);
+ while (!eng->stop) {
+ int action = 0;
+ uint32_t cmd = 0;
+
+ if (eng->qhead != eng->qtail) {
+ cmd = eng->dma_q[eng->qhead++];
+ eng->qhead %= 64;
+ action = 1;
+ } else if (eng->started && !eng->halted && !eng->faulted) {
+ action = 2;
+ }
+ if (!action) {
+ pthread_cond_wait(&eng->wake, &eng->lock);
+ continue;
+ }
+ eng->busy = true;
+ pthread_mutex_unlock(&eng->lock);
+ if (action == 1)
+ dma_execute(dev, eng, cmd);
+ else
+ engine_kick(dev, eng);
+ pthread_mutex_lock(&eng->lock);
+ if (action == 1)
+ eng->dma_stat = 2;
+ eng->busy = false;
+ pthread_cond_broadcast(&eng->idle);
+ }
+ pthread_mutex_unlock(&eng->lock);
+ return NULL;
+}
+
+static void engine_start_locked(struct gd_device *dev, struct gd_engine *eng)
+{
+ eng->started = true;
+ eng->halted = false;
+ if (eng->thread_up) {
+ eng->pending_kick = false;
+ pthread_cond_signal(&eng->wake);
+ return;
+ }
+ eng->pending_kick = true;
+}
+
+static void engine_reset_locked(struct gd_device *dev, struct gd_engine *eng, uint32_t value)
+{
+ while (eng->busy)
+ pthread_cond_wait(&eng->idle, &eng->lock);
+ eng->pending_kick = false;
+ eng->engine_reg = value;
+ if (value & 1) {
+ engine_reset_state(dev, eng);
+ eng->engine_reg = value | 1;
+ eng->scrub_pending = true;
+ eng->scrub_done_us = dev->time_us + 2000;
+ eng->hwcfg2 |= HWCFG2_SCRUB;
+ gd_log(dev, GD_LOG_DEBUG, "device: engine %d reset asserted\n", eng->index);
+ } else {
+ eng->engine_reg = value & ~1u;
+ settle_scrub(dev, eng);
+ eng->hwcfg2 &= ~HWCFG2_SCRUB;
+ gd_log(dev, GD_LOG_DEBUG, "device: engine %d reset released\n", eng->index);
+ }
+}
+
+static uint32_t engine_read_locked(struct gd_device *dev, struct gd_engine *eng, uint32_t off, int priv)
+{
+ settle_scrub(dev, eng);
+ if (!engine_plm_allowed(eng, off, priv, false))
+ return 0xbadf0000u;
+ switch (off) {
+ case OFF_MAILBOX0: return eng->mailbox[0];
+ case OFF_MAILBOX1: return eng->mailbox[1];
+ case OFF_HWCFG2: return eng->hwcfg2 | (eng->scrub_pending ? HWCFG2_SCRUB : 0);
+ case OFF_CPUCTL:
+ case OFF_CPUCTL_ALT:
+ return eng->halted ? CPUCTL_HALTED : 0;
+ case OFF_BOOTVEC: return eng->bootvec;
+ case OFF_HWCFG: return eng->hwcfg;
+ case OFF_DMACTL: return eng->dmactl;
+ case OFF_DMABASE: return eng->dma_base;
+ case OFF_DMABASE1: return eng->dma_base1;
+ case OFF_DMAMOFFS: return eng->dma_moffs;
+ case OFF_DMAFBOFFS: return eng->dma_fboffs;
+ case OFF_DMACMD: return eng->dma_stat;
+ case OFF_ENGINE: return eng->engine_reg;
+ case OFF_RESET_PLM: return eng->reset_plm;
+ case OFF_IMEMC: return eng->imemc;
+ case 0x1100: return 0x40000000u; /* FWSEC boot-progress gate */
+ case 0x9000: return 0x2u; /* crypto engine available */
+ case 0x1000: return eng->io[0x1000 / 4]; /* NV_CSEC_FALCON_MAILBOX0 */
+ case 0x4f00: return gd_falcon_engid();
+ case 0x1070: return gd_pbus_emulation_rev(0);
+ case 0x1074: return gd_pbus_emulation_rev(1);
+ case 0x10200: return dev->config.key_set ? 0x100000u : 0; /* SCP_CTL_STAT.DEBUG_MODE */
+ case 0x9100: return gd_csberrstat(eng->csb_enable, eng->csb_valid, eng->csb_pc);
+ case 0x1c000: case 0x1c100: case 0x1c200:
+ return gd_csec_read(&eng->csec, off);
+ case 0x1c300: return 0x0063bb0bu;
+ case 0x40300: return eng->sha_in_addr;
+ case 0x40400: return eng->sha_in_hi;
+ case 0x40700: return eng->sha_msg_len;
+ case 0x41f00: return eng->sha_op;
+ case 0x42000: return eng->sha_status;
+ case 0x44200: return eng->sha_mutex;
+ case 0x47200: return 2; /* SHA hash selection/version: SHA-256 */
+ case 0xd000: case 0xd100: case 0xd200: case 0xd300:
+ case 0xd400: case 0xd500: case 0xd600: case 0xd700:
+ return gd_doc_read(&eng->doc, off, priv);
+ case 0x344: return eng->io[0x344 / 4];
+ case 0x348: return eng->io[0x348 / 4];
+ case 0x350: return eng->response_ready;
+ case 0x354: return eng->response;
+ case 0x700: return eng->io[0x700 / 4];
+ case 0x704: return eng->io[0x704 / 4];
+ case 0x708: return eng->io[0x708 / 4];
+ case OFF_DMEMC: return eng->dmemc;
+ case 0x404: case 0x10100: return eng->io[0x404 / 4];
+ case 0x008: return 0;
+ case 0x00c: return 0;
+ case 0x018: return 0;
+ case 0x01c: return 0;
+ case 0x04c: return eng->started && !eng->halted;
+ default:
+ if (off >= 0x40f00 && off < 0x40f00 + 8 * 0x100 &&
+ ((off - 0x40f00) & 0xff) == 0)
+ return eng->sha_result[(off - 0x40f00) / 0x100];
+ if (off < GD_IO_SIZE) {
+ if (eng->io_valid[off / 4])
+ return eng->io[off / 4];
+ gd_log(dev, GD_LOG_DEBUG, "device: unmodelled engine read %#x -> 0\n", eng->base + off);
+ trace_add(dev, GD_TRACE_BAR_READ, eng->index + 1, eng->base + off, 0, 0);
+ return 0;
+ }
+ record_unknown(dev, false, eng->base + off, 0, 0);
+ return 0xffffffffu;
+ }
+}
+
+static bool engine_write_locked(struct gd_device *dev, struct gd_engine *eng, uint32_t off, uint32_t value, int priv)
+{
+ settle_scrub(dev, eng);
+ if (!engine_plm_allowed(eng, off, priv, true)) {
+ dev->denied_writes++;
+ dev->plm_violations++;
+ gd_log(dev, GD_LOG_DEBUG, "device: PLM denied engine write base=%#x off=%#x priv=%d\n",
+ eng->base, off, priv);
+ return false;
+ }
+ switch (off) {
+ case OFF_MAILBOX0: eng->mailbox[0] = value; break;
+ case OFF_MAILBOX1: eng->mailbox[1] = value; break;
+ case OFF_HWCFG2: eng->hwcfg2 = value; break;
+ case OFF_CPUCTL:
+ case OFF_CPUCTL_ALT:
+ eng->cpuctl = value;
+ if (value & CPUCTL_START)
+ engine_start_locked(dev, eng);
+ break;
+ case OFF_BOOTVEC: eng->bootvec = value; break;
+ case OFF_DMACTL: eng->dmactl = value; break;
+ case OFF_DMABASE:
+ eng->dma_base = value;
+ GD_DIAG( "diag: dma_base=%#x pc=%#x eng=%d\n", value, eng->pc, eng->index);
+ break;
+ case OFF_DMABASE1:
+ eng->dma_base1 = value;
+ GD_DIAG( "diag: dma_base1=%#x pc=%#x eng=%d\n", value, eng->pc, eng->index);
+ break;
+ case OFF_DMAMOFFS: eng->dma_moffs = value; break;
+ case OFF_DMAFBOFFS: eng->dma_fboffs = value; break;
+ case OFF_DMACMD:
+ eng->dma_cmd = value;
+ GD_DIAG( "diag: dmacmd=%#x pc=%#x eng=%d base=%#x fboffs=%#x\n",
+ value, eng->pc, eng->index, eng->dma_base, eng->dma_fboffs);
+ if (eng->thread_up) {
+ eng->dma_stat = 1;
+ eng->dma_q[eng->qtail++] = value;
+ eng->qtail %= 64;
+ pthread_cond_signal(&eng->wake);
+ } else {
+ pthread_mutex_unlock(&eng->lock);
+ dma_execute(dev, eng, value);
+ pthread_mutex_lock(&eng->lock);
+ eng->dma_stat = 2;
+ }
+ break;
+ case OFF_ENGINE: engine_reset_locked(dev, eng, value); break;
+ case OFF_RESET_PLM: eng->reset_plm = value; break;
+ case OFF_IMEMC:
+ eng->imemc = value;
+ eng->pio_imem = value & 0xffffu;
+ break;
+ case OFF_IMEMD:
+ if (eng->pio_imem + 4 <= sizeof(eng->imem))
+ memcpy(eng->imem + eng->pio_imem, &value, 4);
+ else
+ device_error(dev, "IMEM PIO overrun", eng->pio_imem, value);
+ if (eng->imemc & (1u << 28)) {
+ uint32_t at = eng->pio_imem;
+ if (!eng->pio_secure_hi) {
+ eng->pio_secure_lo = at;
+ eng->pio_secure_hi = at + 4;
+ } else {
+ if (at < eng->pio_secure_lo)
+ eng->pio_secure_lo = at;
+ if (at + 4 > eng->pio_secure_hi)
+ eng->pio_secure_hi = at + 4;
+ }
+ }
+ if (eng->imemc & (1u << 24))
+ eng->pio_imem += 4;
+ break;
+ case OFF_IMEMT: eng->imem_tag = value; break;
+ case 0x1000: eng->io[0x1000 / 4] = value; break;
+ case 0x9100:
+ if (value & 0x40000000u)
+ eng->csb_enable = true;
+ if (value & 0x80000000u)
+ eng->csb_valid = true;
+ eng->csb_pc = value & 0xffffffu;
+ break;
+ case 0x344: {
+ uint32_t address = value & 0xffffu;
+ eng->io[0x344 / 4] = value;
+ if (value & 0x10000u) {
+ eng->response = (address >= 0x8e00 && address < 0x8f00) ?
+ csb_read(dev, address, priv) : bar0_raw_read(dev, address);
+ eng->response_ready = 1;
+ } else {
+ uint32_t data = eng->io[0x348 / 4];
+ if (address >= 0x8e00 && address < 0x8f00)
+ csb_write(dev, address, data, priv);
+ else
+ bar0_raw_write(dev, address, data);
+ }
+ break;
+ }
+ case 0x348: eng->io[0x348 / 4] = value; break;
+ case 0x350:
+ eng->io[0x350 / 4] = value;
+ if (value & 0x100000u)
+ eng->response_ready = 0;
+ break;
+ case 0x700: {
+ eng->io[0x700 / 4] = value;
+ if (value & 0x80000000u) {
+ uint32_t address = eng->io[0x704 / 4];
+ if ((value & 3u) == 1u) {
+ eng->io[0x708 / 4] = bar0_raw_read(dev, address);
+ eng->io[0x700 / 4] = 0x400u;
+ } else {
+ bar0_raw_write(dev, address, eng->io[0x708 / 4]);
+ eng->io[0x700 / 4] = 0x400u;
+ }
+ }
+ break;
+ }
+ case 0x704: eng->io[0x704 / 4] = value; break;
+ case 0x708: eng->io[0x708 / 4] = value; break;
+ case 0x1c000: case 0x1c100: case 0x1c200:
+ gd_csec_write(&eng->csec, off, value);
+ break;
+ case 0x41f00:
+ eng->sha_op = value;
+ if (dev->config.verbose) gd_log(dev, GD_LOG_DEBUG, "device: sha-op pc=%#x value=%#x\n", eng->pc, value);
+ sha_run(eng);
+ break;
+ case 0x42000: eng->sha_status = value; break;
+ case 0x44000: eng->sha_mutex = value; break;
+ case 0x40300: eng->sha_in_addr = value; break;
+ case 0x40400: eng->sha_in_hi = value; break;
+ case 0x40700:
+ eng->sha_msg_len = value;
+ eng->sha_total_bits = value;
+ if (dev->config.verbose) gd_log(dev, GD_LOG_DEBUG, "device: sha-total pc=%#x value=%#x\n", eng->pc, value);
+ break;
+ case 0x40b00:
+ eng->sha_update_bits = value;
+ if (dev->config.verbose) gd_log(dev, GD_LOG_DEBUG, "device: sha-update pc=%#x value=%#x\n", eng->pc, value);
+ sha_feed(eng);
+ break;
+ case 0xd000: case 0xd100: case 0xd200: case 0xd300:
+ case 0xd400: case 0xd500: case 0xd600: case 0xd700:
+ gd_doc_write(&eng->doc, off, value, priv);
+ break;
+ case 0x404: case 0x10100:
+ eng->io[0x404 / 4] = value & ~0x100u;
+ break;
+ case OFF_DMEMC:
+ eng->dmemc = value;
+ eng->pio_dmem = value & 0xffffu;
+ break;
+ case OFF_DMEMD:
+ if (eng->pio_dmem + 4 <= sizeof(eng->dmem))
+ memcpy(eng->dmem + eng->pio_dmem, &value, 4);
+ else
+ device_error(dev, "DMEM PIO overrun", eng->pio_dmem, value);
+ if (eng->dmemc & (1u << 24))
+ eng->pio_dmem += 4;
+ break;
+ default:
+ if (off < GD_IO_SIZE) {
+ eng->io[off / 4] = value;
+ eng->io_valid[off / 4] = 1;
+ } else {
+ record_unknown(dev, true, eng->base + off, value, 0);
+ }
+ break;
+ }
+ dev->engine_writes++;
+ return true;
+}
+
+uint32_t gd_bar0_read(struct gd_device *dev, uint32_t address)
+{
+ struct gd_engine *eng;
+ uint32_t off, value;
+
+ dev->host_reads++;
+ dev->time_us = now_us();
+ trace_add(dev, GD_TRACE_BAR_READ, 0, address, 0, 0);
+ if (address >= PROM_OFFSET && address < PROM_OFFSET + PROM_SIZE)
+ return 0xffffffffu;
+ eng = engine_for(dev, address, &off);
+ if (eng) {
+ pthread_mutex_lock(&eng->lock);
+ value = engine_read_locked(dev, eng, off, 0);
+ pthread_mutex_unlock(&eng->lock);
+ return value;
+ }
+ if (!plm_allowed(dev, address, 0, false))
+ return 0;
+ pthread_mutex_lock(&dev->bar_lock);
+ bool valid = address + 4 <= GD_BAR_SIZE && dev->bar_valid[address / 4];
+ value = valid ? dev->bar[address / 4] : 0xffffffffu;
+ pthread_mutex_unlock(&dev->bar_lock);
+ if (!valid)
+ record_unknown(dev, false, address, 0, 0);
+ return value;
+}
+
+void gd_bar0_write(struct gd_device *dev, uint32_t address, uint32_t value)
+{
+ struct gd_engine *eng;
+ uint32_t off;
+
+ dev->host_writes++;
+ dev->time_us = now_us();
+ trace_add(dev, GD_TRACE_BAR_WRITE, 0, address, value, 0);
+ if (address >= PROM_OFFSET && address < PROM_OFFSET + PROM_SIZE) {
+ record_unknown(dev, true, address, value, 0);
+ return;
+ }
+ eng = engine_for(dev, address, &off);
+ if (eng) {
+ pthread_mutex_lock(&eng->lock);
+ bool allowed = engine_write_locked(dev, eng, off, value, 0);
+ bool kick = eng->pending_kick;
+ eng->pending_kick = false;
+ pthread_mutex_unlock(&eng->lock);
+ if (!allowed) {
+ dev->denied_writes++;
+ gd_log(dev, GD_LOG_DEBUG, "device: PLM denied host engine write %#x=%#x\n",
+ address, value);
+ return;
+ }
+ if (kick) {
+ gd_log(dev, GD_LOG_DEBUG, "device: host kick engine %d\n", eng->index);
+ engine_kick(dev, eng);
+ }
+ return;
+ }
+ if (!plm_allowed(dev, address, 0, true)) {
+ uint32_t ma = plm_mask_register(address) ? address : plm_mask_for(address);
+ dev->denied_writes++;
+ if (ma && ((dev->bar[ma / 4] >> 9) & 1))
+ dev->plm_violations++;
+ gd_log(dev, GD_LOG_DEBUG, "device: PLM denied host write %#x=%#x\n", address, value);
+ return;
+ }
+ value &= bar_write_mask(address);
+ if (address == 0x82381c || address == 0x823820)
+ dev->plm_opened_seen = true;
+ pthread_mutex_lock(&dev->bar_lock);
+ if (address + 4 <= GD_BAR_SIZE) {
+ dev->bar[address / 4] = value;
+ dev->bar_valid[address / 4] = 1;
+ if (address == 0x1180f8 && (value & (1u << 24)))
+ csb_release_all(dev);
+ plm_latch_write(dev, address, value);
+ if (address == 0x823804 && dev->feat_count < 8)
+ dev->feats[dev->feat_count++] = value;
+ } else {
+ record_unknown(dev, true, address, value, 0);
+ }
+ pthread_mutex_unlock(&dev->bar_lock);
+}
+
+int gd_prom_copy(struct gd_device *dev, uint32_t offset, void *dest, size_t size)
+{
+ if (offset < PROM_OFFSET || offset >= PROM_OFFSET + PROM_SIZE)
+ return -EINVAL;
+ offset -= PROM_OFFSET;
+ if (offset + size > PROM_SIZE)
+ size = PROM_SIZE - offset;
+ memcpy(dest, dev->prom + offset, size);
+ return (int)size;
+}
+
+int gd_pci_read(struct gd_device *dev, int space, int offset, int width, uint32_t *value)
+{
+ uint32_t v = 0xffffffffu;
+ (void)width;
+
+ if (space == 0) {
+ switch (offset) {
+ case 0x00: v = 0x10deu | (uint32_t)dev->board->device << 16; break;
+ case 0x04: v = dev->pci_command; break;
+ case 0x08: v = 0x030000a1u; break;
+ case 0x10: case 0x14: case 0x18: case 0x1c: case 0x20: case 0x24:
+ v = 0xf0000000u + (uint32_t)(offset - 0x10) * 4; break;
+ default:
+ if (offset >= 0x40 && offset < 0x60)
+ v = 0;
+ else
+ v = 0;
+ break;
+ }
+ } else if (space == 1) {
+ switch (offset) {
+ case 4: v = 1u << 28; break;
+ case 10: v = 0; break;
+ default: v = 0; break;
+ }
+ } else {
+ return -EINVAL;
+ }
+ *value = v;
+ return 0;
+}
+
+int gd_pci_write(struct gd_device *dev, int space, int offset, int width, uint32_t value)
+{
+ (void)width;
+ if (space != 0)
+ return -EINVAL;
+ if (offset == 0x04) {
+ dev->pci_command = value;
+ dev->bar[0x200 / 4] = value;
+ dev->bar_valid[0x200 / 4] = 1;
+ }
+ return 0;
+}
+
+int gd_dma_map(struct gd_device *dev, uint64_t address, void *cpu, size_t size)
+{
+ uint32_t i;
+ if (dev->dma_count >= GD_MAX_DMA)
+ return -ENOMEM;
+ for (i = 0; i < dev->dma_count; i++) {
+ struct gd_dma_span *s = &dev->dma[i];
+ if (s->live && address < s->address + s->size && s->address < address + size)
+ return -EINVAL;
+ }
+ struct gd_dma_span *s = &dev->dma[dev->dma_count++];
+ s->address = address;
+ s->cpu = cpu;
+ s->size = size;
+ s->live = true;
+ return 0;
+}
+
+void gd_dma_unmap(struct gd_device *dev, uint64_t address)
+{
+ uint32_t i;
+ for (i = 0; i < dev->dma_count; i++)
+ if (dev->dma[i].live && dev->dma[i].address == address) {
+ dev->dma[i].live = false;
+ return;
+ }
+}
+
+void gd_dma_barrier(struct gd_device *dev)
+{
+ __sync_synchronize();
+ (void)dev;
+}
+
+static void load_bar_defaults(struct gd_device *dev)
+{
+ size_t i;
+ uint32_t chip = dev->board->chip;
+
+ for (i = 0; i < sizeof(gd_bar0_defaults) / sizeof(gd_bar0_defaults[0]); i++) {
+ uint32_t addr = gd_bar0_defaults[i].addr;
+ dev->bar[addr / 4] = gd_bar0_defaults[i].reset;
+ dev->bar_valid[addr / 4] = 1;
+ }
+ /* The register table is a GA102 fixture; override identity per board. */
+ set_bar_locked(dev, 0x0, (chip << 20) | 0xa1u);
+ dev->wpr2_lo = 0x1ffffe00u;
+ dev->wpr2_hi = 0;
+ set_bar_locked(dev, 0x1fa824, dev->wpr2_lo);
+ set_bar_locked(dev, 0x1fa828, dev->wpr2_hi);
+ set_bar_locked(dev, 0x110100, CPUCTL_HALTED);
+ set_bar_locked(dev, 0x1100f4, HWCFG2_RESET);
+ set_bar_locked(dev, 0x118234, 0xff);
+ set_bar_locked(dev, 0x1183a4, 0x6000);
+ set_bar_locked(dev, 0x2240c, 0x2);
+ set_bar_locked(dev, 0xa00, chip << 20);
+ set_bar_locked(dev, 0x1180fc, 1u << 20);
+ set_bar_locked(dev, 0x1180f8, 0u); /* BSI scratch 14: no level/error */
+ set_bar_locked(dev, 0x118220, 0u);
+ set_bar_locked(dev, 0x118244, 0u); /* declared pre-FWSEC boundary: FRTS not cached */
+ set_bar_locked(dev, 0x118248, 0u);
+ set_bar_locked(dev, 0x118c6c, 0u);
+
+ /* GC6 AON secure-scratch bootstrap latches (groups 40/41). The real
+ * early boot/SCP flow writes these before FWSEC; we do not emulate that
+ * flow, so seed the post-bootstrap values here. This is a test fixture,
+ * not evidence of the hardware sequence.
+ */
+ set_bar_locked(dev, 0x11839c, 0xff);
+ set_bar_locked(dev, 0x1183a0, 0xff);
+ dev->ss0 = dev->bar[0x82381c / 4];
+ dev->ss1 = dev->bar[0x823820 / 4];
+}
+
+static uint32_t gd_doc_io_read(void *ctx, uint32_t address, int priv)
+{
+ struct gd_engine *eng = ctx;
+ return dio_read(eng->dev, eng, address, priv);
+}
+
+static void gd_doc_io_write(void *ctx, uint32_t address, uint32_t value, int priv)
+{
+ struct gd_engine *eng = ctx;
+ dio_write(eng->dev, eng, address, value, priv);
+}
+
+static bool csec_bar_read(void *ctx, uint32_t address, uint32_t *value)
+{
+ struct gd_device *dev = ctx;
+ if (address + 4 > GD_BAR_SIZE || !plm_allowed(dev, address, 3, false))
+ return false;
+ *value = bar0_raw_read(dev, address);
+ if (address == 0x118244 || address == 0x118248)
+ GD_DIAG( "diag: group6 read [%#x] -> %#x booterpc=%#x\n",
+ address, *value, dev->engine[0].pc);
+ return true;
+}
+
+static bool csec_bar_write(void *ctx, uint32_t address, uint32_t value)
+{
+ struct gd_device *dev = ctx;
+ if (address + 4 > GD_BAR_SIZE || !plm_allowed(dev, address, 3, true)) {
+ dev->denied_writes++;
+ dev->plm_violations++;
+ return false;
+ }
+ if (address == 0x118244 || address == 0x118248)
+ GD_DIAG( "diag: group6 write [%#x]=%#x fwsec=%d pc=%#x\n",
+ address, value, !!dev->engine[1].pc, dev->engine[1].pc);
+ if (address == 0x1fa824 || address == 0x1fa828) {
+ uint32_t di;
+ GD_DIAG( "diag: wpr2 write [%#x]=%#x pc=%#x r0=%#x r1=%#x r10=%#x r11=%#x stack:",
+ address, value, dev->engine[1].pc, dev->engine[1].r[0],
+ dev->engine[1].r[1], dev->engine[1].r[10], dev->engine[1].r[11]);
+ for (di = 0; di < dev->engine[1].call_depth && di < 12; di++)
+ GD_DIAG( " %#x", dev->engine[1].call_pc[di]);
+ GD_DIAG( "\n");
+ }
+ /* The payload writes through the secure-bus bridge, not the host
+ * register path; record the same target-write evidence. */
+ /* The Booter's release primitive pulses BAR0 0x1180f8 bit 24;
+ * that is the SE common-mutex release doorbell. */
+ value &= bar_write_mask(address);
+ if (address == 0x1180f8 && (value & (1u << 24)))
+ csb_release_all(dev);
+ plm_latch_write(dev, address, value);
+ bar0_raw_write(dev, address, value);
+ return true;
+}
+
+static void load_engine_defaults(struct gd_device *dev, struct gd_engine *eng, uint32_t base, int index)
+{
+ size_t i;
+ memset(eng, 0, sizeof(*eng));
+ eng->dev = dev;
+ gd_doc_init(&eng->doc, gd_doc_io_read, gd_doc_io_write, eng);
+ gd_trng_init(&eng->trng, eng->se_regs, sizeof(eng->se_regs) / 4);
+ gd_csec_init(&eng->csec, csec_bar_read, csec_bar_write, dev);
+ eng->index = index;
+ eng->base = base;
+ eng->hwcfg = 0x20100u;
+ eng->hwcfg2 = HWCFG2_RESET;
+ eng->dma_stat = 2;
+ eng->csb_enable = true;
+ eng->csb_valid = false;
+ eng->cpuctl = 0;
+ pthread_mutex_init(&eng->lock, NULL);
+ pthread_cond_init(&eng->wake, NULL);
+ pthread_cond_init(&eng->idle, NULL);
+ for (i = 0; i < sizeof(gd_io_defaults) / sizeof(gd_io_defaults[0]); i++) {
+ uint32_t addr = gd_io_defaults[i].addr;
+ if (addr < 0x1000) {
+ eng->io[addr / 4] = gd_io_defaults[i].reset;
+ eng->io_valid[addr / 4] = 1;
+ }
+ }
+ /* PLM reset values are all-levels until the Booter locks them. */
+ eng->reset_plm = 0xffffffffu;
+ eng->io[0x288 / 4] = 0xffffffffu;
+ eng->io[0x194 / 4] = 0xffffffffu;
+ eng->io[0x28c / 4] = 0xffffffffu;
+ eng->io[0x190 / 4] = 0xffffffffu;
+ eng->io[0x18c / 4] = 0xffffffffu;
+ eng->io[0x2b0 / 4] = 0xffffffffu;
+ eng->io[0x3c4 / 4] = eng->reset_plm;
+ if (index == 1) {
+ eng->cpuctl = CPUCTL_HALTED;
+ eng->halted = true;
+ }
+}
+
+void *gd_bar_pointer(struct gd_device *dev)
+{
+ return dev ? dev->bar : NULL;
+}
+
+const char *gd_asset_root(struct gd_device *dev)
+{
+ return dev && dev->config.asset_root ? dev->config.asset_root : ".";
+}
+
+struct gd_device *gd_create(const struct gd_board *board, const struct gd_run_config *config)
+{
+ struct gd_device *dev = calloc(1, sizeof(*dev));
+ FILE *file;
+ long size;
+
+ if (!dev)
+ return NULL;
+ csb_init(dev);
+ dev->fb_size = GD_FB_SIZE;
+ dev->fb = calloc(dev->fb_size, 1);
+ if (!dev->fb)
+ goto fail;
+ dev->board = board;
+ dev->config = *config;
+ if (dev->config.key_set) {
+ /* Debug-fused SCP slots: 37 decrypts the HS image, 0 authenticates
+ * the debug DMEM MAC. The debug secret 0 is the HS key with its
+ * first byte cleared; production slot 0x28 stays unavailable. */
+ memcpy(dev->secrets[37], dev->config.key, 16);
+ memcpy(dev->secrets[0], dev->config.key, 16);
+ dev->secrets[0][0] = 0;
+ dev->secret_set[37] = true;
+ dev->secret_set[0] = true;
+ }
+ dev->bar = calloc(GD_BAR_WORDS, sizeof(uint32_t));
+ dev->bar_valid = calloc(GD_BAR_WORDS, 1);
+ dev->prom = calloc(PROM_SIZE, 1);
+ if (!dev->bar || !dev->bar_valid || !dev->prom)
+ goto fail;
+ pthread_mutex_init(&dev->lock, NULL);
+ pthread_mutex_init(&dev->bar_lock, NULL);
+ pthread_mutex_init(&dev->dma_lock, NULL);
+ pthread_mutex_init(&dev->trace_lock, NULL);
+ if (!gd_aes128_selftest())
+ gd_log(dev, GD_LOG_ERROR, "device: AES-128 self-test failed\n");
+ load_bar_defaults(dev);
+ /* The layout constrains every reservation to the top 256 MiB of usable
+ * framebuffer, so one window derived from the strap covers all of them. */
+ dev->fb_base = (uint64_t)dev->bar[0x1183a4 / 4] << 20;
+ dev->fb_base = dev->fb_base > GD_FB_SIZE ? dev->fb_base - GD_FB_SIZE : 0;
+ load_engine_defaults(dev, &dev->engine[0], SEC2_BASE, 0);
+ load_engine_defaults(dev, &dev->engine[1], GSP_BASE, 1);
+ load_code_table(&dev->engine[1], config->code_path);
+ load_code_table(&dev->engine[0], config->booter_code_path);
+ dev->pci_command = 0x2;
+ set_bar_locked(dev, 0x200, 0x2);
+ dev->time_us = now_us();
+
+ memset(dev->prom, 0xff, PROM_SIZE);
+ file = fopen(config->rom_path, "rb");
+ if (!file) {
+ gd_log(dev, GD_LOG_ERROR, "device: cannot open ROM %s\n", config->rom_path);
+ goto fail;
+ }
+ fseek(file, 0, SEEK_END);
+ size = ftell(file);
+ fseek(file, 0, SEEK_SET);
+ if (size > 0) {
+ if (size > (long)PROM_SIZE)
+ size = PROM_SIZE;
+ if (fread(dev->prom, 1, (size_t)size, file) != (size_t)size) {
+ fclose(file);
+ goto fail;
+ }
+ }
+ fclose(file);
+ return dev;
+fail:
+ gd_destroy(dev);
+ return NULL;
+}
+
+void gd_destroy(struct gd_device *dev)
+{
+ int i;
+ if (!dev)
+ return;
+ for (i = 0; i < 2; i++) {
+ struct gd_engine *eng = &dev->engine[i];
+ if (eng->thread_up) {
+ pthread_mutex_lock(&eng->lock);
+ eng->stop = true;
+ pthread_cond_signal(&eng->wake);
+ pthread_mutex_unlock(&eng->lock);
+ pthread_join(eng->thread, NULL);
+ eng->thread_up = false;
+ }
+ pthread_mutex_destroy(&eng->lock);
+ pthread_cond_destroy(&eng->wake);
+ pthread_cond_destroy(&eng->idle);
+ }
+ pthread_mutex_destroy(&dev->trace_lock);
+ for (i = 0; i < 2; i++) {
+ free(dev->engine[i].code);
+ free(dev->engine[i].code_index);
+ }
+ free(dev->fb);
+ free(dev->bar);
+ free(dev->bar_valid);
+ free(dev->prom);
+ free(dev);
+}
+
+void gd_set_engines(struct gd_device *dev, bool enabled)
+{
+ int i;
+ for (i = 0; i < 2; i++) {
+ struct gd_engine *eng = &dev->engine[i];
+ eng->enabled = enabled;
+ if (enabled && !eng->thread_up) {
+ eng->stop = false;
+ if (pthread_create(&eng->thread, NULL, engine_thread, eng) == 0) {
+ eng->thread_up = true;
+ pthread_mutex_lock(&eng->lock);
+ pthread_cond_signal(&eng->wake);
+ pthread_mutex_unlock(&eng->lock);
+ } else {
+ eng->enabled = false;
+ }
+ }
+ }
+}
+
+bool gd_engine_progress(struct gd_device *dev, int engine, uint64_t *instructions)
+{
+ struct gd_engine *eng;
+ if (engine < 0 || engine > 1)
+ return false;
+ eng = &dev->engine[engine];
+ pthread_mutex_lock(&eng->lock);
+ if (instructions)
+ *instructions = eng->instructions;
+ bool progress = eng->instructions != 0;
+ pthread_mutex_unlock(&eng->lock);
+ return progress;
+}
+
+void gd_advance_us(struct gd_device *dev, uint64_t usec)
+{
+ int i;
+ dev->time_us += usec;
+ for (i = 0; i < 2; i++) {
+ struct gd_engine *eng = &dev->engine[i];
+ pthread_mutex_lock(&eng->lock);
+ settle_scrub(dev, eng);
+ pthread_mutex_unlock(&eng->lock);
+ }
+}
+
+int gd_reset(struct gd_device *dev)
+{
+ int i;
+ dev->resets++;
+ dev->flr_seen = true;
+ for (i = 0; i < 2; i++) {
+ struct gd_engine *eng = &dev->engine[i];
+ pthread_mutex_lock(&eng->lock);
+ while (eng->busy)
+ pthread_cond_wait(&eng->idle, &eng->lock);
+ engine_reset_state(dev, eng);
+ /* FLR resets the function; MAILBOX0/1 DATA_INIT is 0. */
+ eng->mailbox[0] = eng->mailbox[1] = 0;
+ eng->io[0x0f4 / 4] = HWCFG2_RESET;
+ eng->io_valid[0x0f4 / 4] = 1;
+ pthread_mutex_unlock(&eng->lock);
+ }
+ pthread_mutex_lock(&dev->bar_lock);
+ uint32_t ss0 = dev->bar[0x82381c / 4];
+ uint32_t ss1 = dev->bar[0x823820 / 4];
+ load_bar_defaults(dev);
+ set_bar_locked(dev, 0x2240c, 0x2);
+ set_bar_locked(dev, 0xa00, dev->board->chip << 20);
+ set_bar_locked(dev, 0x1180fc, 1u << 20);
+ set_bar_locked(dev, 0x1180f8, 0u); /* BSI scratch 14: no level/error */
+ set_bar_locked(dev, 0x118220, 0u);
+ set_bar_locked(dev, 0x118244, 0u); /* declared pre-FWSEC boundary: FRTS not cached */
+ set_bar_locked(dev, 0x118248, 0u);
+ set_bar_locked(dev, 0x118c6c, 0u);
+
+ /* GC6 AON secure-scratch bootstrap latches (groups 40/41). The real
+ * early boot/SCP flow writes these before FWSEC; we do not emulate that
+ * flow, so seed the post-bootstrap values here. This is a test fixture,
+ * not evidence of the hardware sequence.
+ */
+ set_bar_locked(dev, 0x11839c, 0xff);
+ set_bar_locked(dev, 0x1183a0, 0xff);
+ set_bar_locked(dev, 0x82381c, ss0);
+ set_bar_locked(dev, 0x823820, ss1);
+ dev->ss0 = ss0;
+ dev->ss1 = ss1;
+ dev->pci_command &= ~0x4u;
+ dev->bar[0x200 / 4] = dev->pci_command;
+ dev->pci_saved_command = dev->pci_command;
+ pthread_mutex_unlock(&dev->bar_lock);
+ gd_log(dev, GD_LOG_INFO, "device: FLR\n");
+ trace_add(dev, GD_TRACE_FLR, 0, 0, 0, 0);
+ return 0;
+}
+
+uint32_t gd_trace_count(struct gd_device *dev)
+{
+ return dev ? dev->trace_count : 0;
+}
+
+int gd_trace_get(struct gd_device *dev, uint32_t index, struct gd_trace_record *record)
+{
+ uint32_t first;
+ if (!dev || !record || index >= dev->trace_count)
+ return -EINVAL;
+ first = dev->trace_head >= GD_MAX_TRACE ? dev->trace_head - GD_MAX_TRACE : 0;
+ *record = dev->trace[(first + index) % GD_MAX_TRACE];
+ return 0;
+}
+
+int gd_trace_dump(struct gd_device *dev, const char *path)
+{
+ FILE *file;
+ uint32_t i;
+ if (!path)
+ return 0;
+ file = fopen(path, "wb");
+ if (!file)
+ return -EIO;
+ fwrite(&dev->trace_count, sizeof(dev->trace_count), 1, file);
+ for (i = 0; i < dev->trace_count; i++) {
+ struct gd_trace_record record;
+ gd_trace_get(dev, i, &record);
+ fwrite(&record, sizeof(record), 1, file);
+ }
+ fclose(file);
+ return 0;
+}
+
+void gd_collect(struct gd_device *dev, struct gd_result *out)
+{
+ uint32_t i;
+ memset(out, 0, sizeof(*out));
+ out->driver_result = 0;
+ out->device_error = dev->device_error;
+ out->fwsec_launch_reached = dev->fwsec_launch_reached;
+ out->booter_launch_reached = dev->booter_launch_reached;
+ out->fwsec_kdf_ok = dev->fwsec_kdf_ok;
+ out->flr_seen = dev->flr_seen;
+ out->launches[0] = dev->launches[0];
+ out->launches[1] = dev->launches[1];
+ out->resets = dev->resets;
+ out->host_writes = dev->host_writes;
+ out->host_reads = dev->host_reads;
+ out->engine_writes = dev->engine_writes;
+ out->engine_reads = dev->engine_reads;
+ out->instructions[0] = dev->engine[0].instructions;
+ out->instructions[1] = dev->engine[1].instructions;
+ out->unknown_count = dev->unknown_count;
+ for (i = 0; i < dev->unknown_count && i < GD_MAX_UNKNOWN; i++)
+ out->unknown[i] = dev->unknown[i];
+ out->feat_count = dev->feat_count;
+ for (i = 0; i < dev->feat_count && i < 8; i++)
+ out->feats[i] = dev->feats[i];
+ out->wpr2_lo = dev->bar[0x1fa824 / 4];
+ out->wpr2_hi = dev->bar[0x1fa828 / 4];
+ out->ss0 = dev->bar[0x82381c / 4];
+ out->ss1 = dev->bar[0x823820 / 4];
+ out->pci_command = dev->pci_command;
+ out->dma_live = dev->dma_count;
+ out->fault_pc = dev->fault_pc;
+ out->fault_opcode = dev->fault_opcode;
+ out->plm_feat = dev->plm_feat_latched;
+ out->plm_wpr_cfg = dev->plm_wpr_cfg_latched;
+ out->plm_wpr = dev->bar[0x1fa7c4 / 4];
+ out->denied_writes = dev->denied_writes;
+ out->booter_error = dev->engine[0].io[0x1000 / 4];
+ out->fwsec_executed = dev->engine[1].instructions > 0 &&
+ !dev->engine[1].core_fault;
+ out->wpr2_normalized = dev->wpr2_normalized;
+ out->fwsec_mac_ok = dev->engine[1].fwsec_kdf_seen &&
+ (!memcmp(dev->engine[1].fwsec_kdf_out, dev->engine[1].fwsec_expect, 16) ||
+ !memcmp(dev->engine[1].fwsec_kdf_out, dev->engine[1].fwsec_expect + 16, 16));
+ out->fwsec_kdf_ok = out->fwsec_mac_ok && !(dev->bar[0x1438 / 4] >> 16);
+ out->booter_executed = dev->engine[0].instructions > 0 &&
+ !dev->engine[0].core_fault;
+ out->booter_halted = dev->engine[0].instructions > 0 &&
+ dev->engine[0].halted;
+ out->plm_opened = dev->plm_opened_seen;
+ out->target_written = dev->target_write_seen;
+ out->unlock_verified = out->fwsec_executed &&
+ out->booter_executed && out->plm_opened && out->target_written &&
+ out->ss0 == 0x88888888u && out->ss1 == 0x8u &&
+ dev->engine[0].reset_plm == 0xffffffffu;
+ out->fwsec_halt_pc = dev->engine[1].halt_pc;
+ out->booter_halt_pc = dev->engine[0].halt_pc;
+ out->fwsec_status = dev->bar[0x1438 / 4];
+ out->booter_status = dev->engine[0].mailbox[0];
+ out->booter_mailbox0 = dev->engine[0].mailbox[0];
+ out->booter_mailbox1 = dev->engine[0].mailbox[1];
+ out->booter_reset_plm = dev->engine[0].reset_plm;
+ out->group03 = dev->bar[0x11821c / 4];
+ if (!dev->fwsec_launch_reached)
+ out->stage = GD_STAGE_PRE_FWSEC;
+ else if (!dev->engine[1].halted || dev->engine[1].core_fault)
+ out->stage = GD_STAGE_FWSEC_RUNNING;
+ else if (out->wpr2_hi >> 4 && (out->group03 & (3u << 16)) == (3u << 16))
+ out->stage = dev->booter_launch_reached ? GD_STAGE_BOOTER_RUNNING
+ : GD_STAGE_WPR2_READY;
+ else
+ out->stage = GD_STAGE_FWSEC_DONE;
+ if (dev->booter_launch_reached && dev->engine[0].halted) {
+ if (dev->engine[0].core_fault || out->booter_status)
+ out->stage = GD_STAGE_BOOTER_FAILED;
+ else if (out->stage == GD_STAGE_BOOTER_RUNNING)
+ out->stage = GD_STAGE_BOOTER_HALTED;
+ }
+}
diff --git a/tests/integration/device/gd.h b/tests/integration/device/gd.h
new file mode 100644
index 0000000..e103bfa
--- /dev/null
+++ b/tests/integration/device/gd.h
@@ -0,0 +1,160 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Native GA102 device library used by the driver integration harness.
+ *
+ * The device owns the BAR0 shadow, two Falcon execution contexts, DMA
+ * mappings and the reset/interrupt state. The host driver reaches it through
+ * the kernel shim in tests/integration/shim. No Python is involved in an
+ * access.
+ */
+#ifndef GD_DEVICE_H
+#define GD_DEVICE_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define GD_BAR_SIZE 0x01000000u
+#define GD_BAR_WORDS (GD_BAR_SIZE / 4u)
+#define GD_MAX_DMA 32768
+#define GD_MAX_UNKNOWN 16
+#define GD_MAX_TRACE 65536
+
+enum gd_trace_kind { GD_TRACE_BAR_READ, GD_TRACE_BAR_WRITE, GD_TRACE_DMA, GD_TRACE_START, GD_TRACE_HALT, GD_TRACE_FLR, GD_TRACE_FAULT, GD_TRACE_PC };
+
+struct gd_trace_record {
+ uint64_t time_us;
+ uint32_t pc, address, value;
+ uint16_t kind, source;
+ uint16_t size;
+};
+
+enum gd_log_level { GD_LOG_DEBUG, GD_LOG_INFO, GD_LOG_ERROR };
+
+struct gd_device;
+
+struct gd_board {
+ uint16_t device; /* PCI device id, selects the driver recipe */
+ uint32_t chip; /* PMC_BOOT_0 architecture+implementation */
+};
+
+struct gd_run_config {
+ const char *asset_root; /* directory holding the driver's firmware names */
+ const char *rom_path;
+ bool verbose;
+ bool key_set; /* debug key present; execution permitted */
+ uint8_t key[16]; /* AES-128 debug key for FWSEC IMEM code */
+ const char *code_path; /* pre-decoded FWSEC instruction table, optional */
+ const char *booter_code_path; /* pre-decoded SEC2 Booter instruction table */
+};
+
+struct gd_dma_span {
+ uint64_t address;
+ uint8_t *cpu;
+ size_t size;
+ bool live;
+};
+
+struct gd_unknown {
+ bool write;
+ uint32_t address;
+ uint32_t value;
+ uint32_t pc;
+};
+
+struct gd_result {
+ int driver_result; /* the driver's return value */
+ int device_error; /* first device-side failure */
+ bool fwsec_launch_reached;
+ bool booter_launch_reached;
+ bool fwsec_executed;
+ bool fwsec_kdf_ok;
+ bool booter_executed;
+ bool booter_halted;
+ bool plm_opened;
+ bool target_written;
+ bool unlock_verified;
+ bool flr_seen;
+ uint32_t launches[2];
+ uint32_t resets;
+ uint64_t host_writes;
+ uint64_t host_reads;
+ uint64_t engine_writes;
+ uint64_t engine_reads;
+ uint64_t instructions[2];
+ uint32_t unknown_count;
+ struct gd_unknown unknown[GD_MAX_UNKNOWN];
+ uint32_t feats[8];
+ uint32_t feat_count;
+ uint32_t wpr2_lo, wpr2_hi;
+ uint32_t ss0, ss1;
+ uint32_t pci_command;
+ uint32_t dma_live;
+ uint32_t fault_pc, fault_opcode;
+ uint32_t plm_feat, plm_wpr_cfg, plm_wpr;
+ uint32_t denied_writes;
+ uint32_t booter_error; /* NV_CSEC_FALCON_MAILBOX0 at 0x1000 */
+ /* Boot-chain evidence. The stage is derived from the same registers
+ * and scratch the driver observes. */
+ uint32_t stage; /* enum gd_stage */
+ uint32_t fwsec_halt_pc;
+ uint32_t booter_halt_pc;
+ uint32_t fwsec_status; /* SW_SCRATCH(14) value left by FWSEC */
+ uint32_t booter_status; /* SEC2 CSB MAILBOX0 status */
+ uint32_t booter_mailbox0;
+ uint32_t booter_mailbox1;
+ uint32_t booter_reset_plm;
+ uint32_t group03; /* AON secure scratch group 03(2) */
+ uint32_t fwsec_mac_ok; /* debug/prod MAC matched the image */
+ uint32_t wpr2_normalized; /* debug FWSEC end encoding converted */
+};
+
+enum gd_stage {
+ GD_STAGE_PRE_FWSEC = 0,
+ GD_STAGE_FWSEC_RUNNING,
+ GD_STAGE_FWSEC_DONE,
+ GD_STAGE_WPR2_READY,
+ GD_STAGE_BOOTER_RUNNING,
+ GD_STAGE_BOOTER_HALTED,
+ GD_STAGE_BOOTER_FAILED,
+};
+
+struct gd_device *gd_create(const struct gd_board *board, const struct gd_run_config *config);
+void gd_destroy(struct gd_device *dev);
+
+/* Host BAR0 access. Unknown registers are recorded and fail closed. */
+uint32_t gd_bar0_read(struct gd_device *dev, uint32_t address);
+void gd_bar0_write(struct gd_device *dev, uint32_t address, uint32_t value);
+
+/* PROM window copy for memcpy_fromio(). */
+int gd_prom_copy(struct gd_device *dev, uint32_t offset, void *dest, size_t size);
+
+/* PCI configuration space. space 0 is config, 1 is the PCIe capability. */
+int gd_pci_read(struct gd_device *dev, int space, int offset, int width, uint32_t *value);
+int gd_pci_write(struct gd_device *dev, int space, int offset, int width, uint32_t value);
+
+/* DMA mappings are identified by their device address. */
+int gd_dma_map(struct gd_device *dev, uint64_t address, void *cpu, size_t size);
+void gd_dma_unmap(struct gd_device *dev, uint64_t address);
+void gd_dma_barrier(struct gd_device *dev);
+
+/* FLR / device reset. */
+int gd_reset(struct gd_device *dev);
+
+void gd_advance_us(struct gd_device *dev, uint64_t usec);
+void gd_log(struct gd_device *dev, enum gd_log_level level, const char *format, ...)
+ __attribute__((format(printf, 3, 4)));
+
+/* Engine threads start when enabled and are joined by gd_destroy(). */
+void gd_set_engines(struct gd_device *dev, bool enabled);
+bool gd_engine_progress(struct gd_device *dev, int engine, uint64_t *instructions);
+
+void gd_collect(struct gd_device *dev, struct gd_result *out);
+uint32_t gd_trace_count(struct gd_device *dev);
+int gd_trace_get(struct gd_device *dev, uint32_t index, struct gd_trace_record *record);
+int gd_trace_dump(struct gd_device *dev, const char *path);
+
+/* BAR0 shadow used by the shim to turn a kernel pointer into an offset. */
+void *gd_bar_pointer(struct gd_device *dev);
+const char *gd_asset_root(struct gd_device *dev);
+
+#endif
diff --git a/tests/integration/device/services.c b/tests/integration/device/services.c
new file mode 100644
index 0000000..353195e
--- /dev/null
+++ b/tests/integration/device/services.c
@@ -0,0 +1,215 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <string.h>
+
+#include "services.h"
+
+/* ---- CSEC BAR0 bridge ------------------------------------------------- */
+
+void gd_csec_init(struct gd_csec *csec,
+ bool (*bar_read)(void *, uint32_t, uint32_t *),
+ bool (*bar_write)(void *, uint32_t, uint32_t), void *ctx)
+{
+ memset(csec, 0, sizeof(*csec));
+ csec->bar_read = bar_read;
+ csec->bar_write = bar_write;
+ csec->ctx = ctx;
+ csec->csr = 0x00000400u; /* XACT_FINISHED, STATUS=IDLE */
+}
+
+uint32_t gd_csec_read(struct gd_csec *csec, uint32_t offset)
+{
+ switch (offset) {
+ case 0x1c000:
+ /* CMD, byte enables and the completed/idle status. */
+ return (csec->csr & 0x800070f3u) | 0x00000400u;
+ case 0x1c100:
+ return csec->addr;
+ case 0x1c200:
+ return csec->data;
+ default:
+ return 0;
+ }
+}
+
+void gd_csec_write(struct gd_csec *csec, uint32_t offset, uint32_t value)
+{
+ uint32_t address, word, out, i;
+
+ switch (offset) {
+ case 0x1c100:
+ csec->addr = value;
+ return;
+ case 0x1c200:
+ csec->data = value;
+ return;
+ case 0x1c000:
+ break;
+ default:
+ return;
+ }
+
+ csec->csr = value & ~0x80000000u;
+ if (!(value & 0x80000000u))
+ return;
+
+ csec->cmd = value & 3u;
+ csec->byte_enable = (value >> 4) & 0xfu;
+ address = csec->addr;
+ word = 0;
+ if (!csec->byte_enable)
+ csec->byte_enable = 0xfu;
+
+ if (csec->cmd == 1u) { /* read */
+ if (csec->bar_read && csec->bar_read(csec->ctx, address & ~3u, &word)) {
+ out = 0;
+ for (i = 0; i < 4; i++)
+ if (csec->byte_enable & (1u << i))
+ out |= ((word >> (8 * ((address + i) & 3))) & 0xffu) << (8 * i);
+ csec->data = out;
+ }
+ } else if (csec->cmd == 2u || csec->cmd == 3u) { /* write */
+ if (csec->bar_write && csec->bar_read &&
+ csec->bar_read(csec->ctx, address & ~3u, &word)) {
+ for (i = 0; i < 4; i++)
+ if (csec->byte_enable & (1u << i))
+ word = (word & ~(0xffu << (8 * ((address + i) & 3)))) |
+ (((csec->data >> (8 * i)) & 0xffu) <<
+ (8 * ((address + i) & 3)));
+ csec->bar_write(csec->ctx, address & ~3u, word);
+ }
+ }
+ csec->finished = true;
+ csec->csr = 0x00000400u;
+}
+
+/* ---- DOC / DIC output and input queues -------------------------------- */
+
+void gd_doc_reset(struct gd_doc *doc)
+{
+ memset(doc, 0, sizeof(*doc));
+ /*
+ * The debug FWSEC/Booter expect the pre-existing DIO idle count
+ * (0x1c0004). CSEC's hardware reset is 0x001c0000; COUNT is a queue
+ * property and is firmware-visible, so keep the observed value here.
+ */
+ doc->ctrl = 0x001c0004u;
+}
+
+void gd_doc_init(struct gd_doc *doc,
+ uint32_t (*dio_read)(void *, uint32_t, int),
+ void (*dio_write)(void *, uint32_t, uint32_t, int), void *ctx)
+{
+ gd_doc_reset(doc);
+ doc->dio_read = dio_read;
+ doc->dio_write = dio_write;
+ doc->ctx = ctx;
+}
+
+uint32_t gd_doc_read(struct gd_doc *doc, uint32_t offset, int priv)
+{
+ (void)priv;
+ switch (offset) {
+ case 0xd000:
+ return doc->ctrl;
+ case 0xd100:
+ return doc->addr;
+ case 0xd200:
+ case 0xd300:
+ return doc->d[(offset - 0xd200) / 0x100];
+ case 0xd400:
+ return (doc->dic_ctrl & 0xffu) |
+ ((doc->dic_ctrl & (1u << 19)) ? (1u << 19) : 0);
+ case 0xd500:
+ case 0xd600:
+ case 0xd700:
+ return doc->dic_d[(offset - 0xd500) / 0x100];
+ default:
+ return 0;
+ }
+}
+
+void gd_doc_write(struct gd_doc *doc, uint32_t offset, uint32_t value, int priv)
+{
+ switch (offset) {
+ case 0xd000:
+ /* The debug firmware treats DOC_CTRL writes as error clears. The
+ * queue status is read-only from its point of view. */
+ return;
+ case 0xd100:
+ doc->addr = value & 0xffffu;
+ if (value & 0x10000u) {
+ if (doc->dio_read)
+ doc->dic_d[0] = doc->dio_read(doc->ctx, doc->addr, priv);
+ doc->dic_ctrl = (doc->dic_ctrl & ~0xffu) | 1u;
+ doc->dic_ctrl |= 1u << 19;
+ } else if (doc->dio_write) {
+ doc->dio_write(doc->ctx, doc->addr, doc->d[0], priv);
+ }
+ return;
+ case 0xd200:
+ case 0xd300:
+ doc->d[(offset - 0xd200) / 0x100] = value;
+ return;
+ case 0xd400:
+ doc->dic_ctrl = value;
+ return;
+ case 0xd500:
+ case 0xd600:
+ case 0xd700:
+ doc->dic_d[(offset - 0xd500) / 0x100] = value;
+ return;
+ default:
+ return;
+ }
+}
+
+/* ---- SE TRNG ---------------------------------------------------------- */
+
+void gd_trng_init(struct gd_trng *trng, uint32_t *regs, size_t reg_count)
+{
+ memset(trng, 0, sizeof(*trng));
+ trng->regs = regs;
+ trng->reg_count = reg_count;
+}
+
+uint32_t gd_trng_read(struct gd_trng *trng, uint32_t address)
+{
+ if (!trng->regs || address / 4 >= trng->reg_count)
+ return 0;
+ /* The debug image only requires the post-reseed ready state. Full
+ * entropy generation remains in emulator/falcon_emu/se_trng.py. */
+ if (address == 0x3f04)
+ return 0x300u;
+ if (address == 0x104)
+ return trng->regs[0x104 / 4] | 1u;
+ return trng->regs[address / 4];
+}
+
+void gd_trng_write(struct gd_trng *trng, uint32_t address, uint32_t value)
+{
+ if (!trng->regs || address / 4 >= trng->reg_count)
+ return;
+ trng->regs[address / 4] = value;
+}
+
+/* ---- Source-defined identification registers -------------------------- */
+
+uint32_t gd_falcon_engid(void)
+{
+ /* NV_CSEC_FALCON_ENGID: FAMILYID_SEC (1), instance 0. */
+ return 0x00000100u;
+}
+
+uint32_t gd_csberrstat(bool enable, bool valid, uint32_t pc)
+{
+ return (valid ? 0x80000000u : 0) |
+ (enable ? 0x40000000u : 0) |
+ (pc & 0x00ffffffu);
+}
+
+uint32_t gd_pbus_emulation_rev(unsigned index)
+{
+ (void)index;
+ /* NV_PBUS_EMULATION_REV0/1_VALUE_HARDWARE. */
+ return 0;
+}
diff --git a/tests/integration/device/services.h b/tests/integration/device/services.h
new file mode 100644
index 0000000..79ca006
--- /dev/null
+++ b/tests/integration/device/services.h
@@ -0,0 +1,62 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Address-services shared by the Falcon engine models.
+ *
+ * The Falcon core only performs an MMIO access. CSEC BAR0, DOC/DIC and the
+ * SE TRNG are separate hardware blocks; keeping their state here avoids
+ * teaching the instruction interpreter about individual addresses.
+ */
+#ifndef GD_SERVICES_H
+#define GD_SERVICES_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+struct gd_csec {
+ bool (*bar_read)(void *ctx, uint32_t address, uint32_t *value);
+ bool (*bar_write)(void *ctx, uint32_t address, uint32_t value);
+ void *ctx;
+ uint32_t csr, addr, data;
+ uint32_t cmd, byte_enable;
+ bool finished;
+};
+
+void gd_csec_init(struct gd_csec *csec,
+ bool (*bar_read)(void *, uint32_t, uint32_t *),
+ bool (*bar_write)(void *, uint32_t, uint32_t), void *ctx);
+uint32_t gd_csec_read(struct gd_csec *csec, uint32_t offset);
+void gd_csec_write(struct gd_csec *csec, uint32_t offset, uint32_t value);
+
+struct gd_doc {
+ uint32_t ctrl;
+ uint32_t d[3];
+ uint32_t dic_ctrl;
+ uint32_t dic_d[3];
+ uint32_t addr;
+ uint32_t (*dio_read)(void *ctx, uint32_t addr, int priv);
+ void (*dio_write)(void *ctx, uint32_t addr, uint32_t value, int priv);
+ void *ctx;
+};
+
+void gd_doc_reset(struct gd_doc *doc);
+void gd_doc_init(struct gd_doc *doc,
+ uint32_t (*dio_read)(void *, uint32_t, int),
+ void (*dio_write)(void *, uint32_t, uint32_t, int), void *ctx);
+uint32_t gd_doc_read(struct gd_doc *doc, uint32_t offset, int priv);
+void gd_doc_write(struct gd_doc *doc, uint32_t offset, uint32_t value, int priv);
+
+struct gd_trng {
+ uint32_t *regs; /* SE register file, owned by the engine */
+ size_t reg_count;
+};
+
+void gd_trng_init(struct gd_trng *trng, uint32_t *regs, size_t reg_count);
+uint32_t gd_trng_read(struct gd_trng *trng, uint32_t address);
+void gd_trng_write(struct gd_trng *trng, uint32_t address, uint32_t value);
+
+/* Source-defined identity values. */
+uint32_t gd_falcon_engid(void);
+uint32_t gd_csberrstat(bool enable, bool valid, uint32_t pc);
+uint32_t gd_pbus_emulation_rev(unsigned index);
+
+#endif
diff --git a/tests/integration/device/wpr2.c b/tests/integration/device/wpr2.c
new file mode 100644
index 0000000..0268311
--- /dev/null
+++ b/tests/integration/device/wpr2.c
@@ -0,0 +1,130 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <errno.h>
+#include <stddef.h>
+
+#include "wpr2.h"
+
+void gd_wpr2_decode(uint32_t lo, uint32_t hi, struct gd_wpr2_range *out)
+{
+ out->start = (uint64_t)(lo & 0xfffffff0u) << 8;
+ out->end = ((uint64_t)(hi & 0xfffffff0u) << 8) + GD_WPR2_ALIGN;
+}
+
+int gd_wpr2_encode(uint64_t start, uint64_t end, uint32_t *lo, uint32_t *hi)
+{
+ if (!lo || !hi || start >= end || (start & (GD_WPR2_ALIGN - 1)) ||
+ (end & (GD_WPR2_ALIGN - 1)) ||
+ (end < GD_WPR2_ALIGN) || ((end - GD_WPR2_ALIGN) >> 12) > 0x0fffffffu ||
+ (start >> 12) > 0x0fffffffu)
+ return -EINVAL;
+ *lo = (uint32_t)((start >> 12) << 4);
+ *hi = (uint32_t)(((end - GD_WPR2_ALIGN) >> 12) << 4);
+ return 0;
+}
+
+int gd_wpr2_fwsec_hi(uint32_t lo, uint32_t *hi)
+{
+ uint64_t start = (uint64_t)(lo & 0xfffffff0u) << 8;
+ uint32_t dummy;
+
+ if (!hi)
+ return -EINVAL;
+ /* The debug FWSEC image writes the inclusive 4 KiB end. The Booter's
+ * decode adds the 128 KiB alignment term, so the conforming HI is the
+ * last aligned block of a 1 MiB FRTS window. */
+ return gd_wpr2_encode(start, start + GD_WPR2_FRTS_SIZE, &dummy, hi);
+}
+
+int gd_wpr2_validate(const struct gd_wpr2_meta *m)
+{
+ uint64_t gsp_end, boot_end;
+
+ if (!m || m->wpr_start >= m->wpr_end ||
+ (m->wpr_start & (GD_WPR2_ALIGN - 1)) ||
+ (m->wpr_end & (GD_WPR2_ALIGN - 1)))
+ return -EINVAL;
+ if (!m->frts_size || m->frts_offset >= m->wpr_end ||
+ (m->frts_offset & 0xfff))
+ return -EINVAL;
+ if (m->frts_offset != m->wpr_end - m->frts_size)
+ return -EINVAL;
+ if (m->gsp_offset <= m->wpr_start || m->gsp_offset >= m->wpr_end ||
+ m->gsp_size == 0)
+ return -EINVAL;
+ gsp_end = m->gsp_offset + m->gsp_size;
+ if (gsp_end < m->gsp_offset || gsp_end > m->wpr_end)
+ return -EINVAL;
+ if (m->boot_offset <= m->wpr_start || m->boot_offset >= m->wpr_end ||
+ m->boot_size == 0)
+ return -EINVAL;
+ boot_end = m->boot_offset + m->boot_size;
+ if (boot_end < m->boot_offset || boot_end > m->wpr_end)
+ return -EINVAL;
+ if (m->gsp_offset < m->boot_offset) {
+ if (m->gsp_offset - m->wpr_start < m->meta_size)
+ return -EINVAL;
+ if (boot_end > m->frts_offset ||
+ m->boot_offset - m->gsp_offset < m->gsp_size)
+ return -EINVAL;
+ } else {
+ if (m->boot_offset - m->wpr_start < m->meta_size)
+ return -EINVAL;
+ if (gsp_end > m->frts_offset ||
+ m->gsp_offset - m->boot_offset < m->boot_size)
+ return -EINVAL;
+ }
+ return 0;
+}
+
+int gd_wpr2_selftest(void)
+{
+ struct gd_wpr2_range r;
+ struct gd_wpr2_meta m;
+ const uint64_t start = 0x5f5400000ull;
+ const uint64_t end = 0x5fff00000ull;
+ uint32_t lo, hi, norm;
+ int ret = 0;
+
+ if (gd_wpr2_encode(start, end, &lo, &hi) || lo != 0x5f54000u ||
+ hi != 0x5ffee00u)
+ ret |= 1;
+ gd_wpr2_decode(lo, hi, &r);
+ if (r.start != start || r.end != end)
+ ret |= 2;
+ if (gd_wpr2_encode(0x5ffe00000ull, 0x5ffe00000ull + GD_WPR2_FRTS_SIZE,
+ &lo, &hi) || hi != 0x5ffee00u)
+ ret |= 4;
+ if (gd_wpr2_fwsec_hi(lo, &norm) || norm != hi)
+ ret |= 8;
+ /* Unaligned ranges are refused. */
+ if (!gd_wpr2_encode(start + 0x1000, end, &lo, &hi))
+ ret |= 16;
+ if (!gd_wpr2_encode(start, end - 0x1000, &lo, &hi))
+ ret |= 32;
+
+ m = (struct gd_wpr2_meta){
+ .wpr_start = 0x5f5400000ull, .wpr_end = 0x5fff00000ull,
+ .frts_offset = 0x5ffe00000ull, .frts_size = 0x100000ull,
+ .gsp_offset = 0x5f5400000ull + 0x100000ull,
+ .gsp_size = 0x4000000ull,
+ .boot_offset = 0x5fda00000ull, .boot_size = 0x6000ull,
+ .meta_size = 0x100ull,
+ };
+ if (gd_wpr2_validate(&m))
+ ret |= 64;
+ m.frts_offset -= 0x1000; /* FRTS no longer at the end of WPR2 */
+ if (!gd_wpr2_validate(&m))
+ ret |= 128;
+ m = (struct gd_wpr2_meta){
+ .wpr_start = 0x5f5400000ull, .wpr_end = 0x5fff00000ull,
+ .frts_offset = 0x5ffe00000ull, .frts_size = 0x100000ull,
+ .gsp_offset = 0x5f5400000ull + 0x100000ull,
+ .gsp_size = 0x4000000ull,
+ .boot_offset = 0x5fda00000ull, .boot_size = 0x6000ull,
+ .meta_size = 0x100ull,
+ };
+ m.boot_offset = m.gsp_offset + 0x100; /* GSP image overlaps the BL */
+ if (!gd_wpr2_validate(&m))
+ ret |= 256;
+ return ret;
+}
diff --git a/tests/integration/device/wpr2.h b/tests/integration/device/wpr2.h
new file mode 100644
index 0000000..e565715
--- /dev/null
+++ b/tests/integration/device/wpr2.h
@@ -0,0 +1,49 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* WPR2 register encoding and range rules.
+ *
+ * The values and tests follow booter_wpr2_tu10x.c and
+ * booter_load_tu10x.c: the LO/HI fields are address >> 8, the end carries a
+ * 128 KiB (0x20000) alignment term, and the FRTS window must be exactly the
+ * 1 MiB the FWSEC command requests. The debug FWSEC image publishes an
+ * inclusive 4 KiB end; gd_wpr2_fwsec_hi() converts that convention for the
+ * Booter, and the caller reports the conversion as evidence.
+ */
+#ifndef GD_WPR2_H
+#define GD_WPR2_H
+
+#include <stdbool.h>
+#include <stdint.h>
+
+#define GD_WPR2_ALIGN 0x20000u
+#define GD_WPR2_FRTS_SIZE 0x100000u
+
+struct gd_wpr2_range {
+ uint64_t start;
+ uint64_t end; /* exclusive */
+};
+
+/* Booter view: start = LO << 12; end = (HI << 12) + 0x20000. */
+void gd_wpr2_decode(uint32_t lo, uint32_t hi, struct gd_wpr2_range *out);
+
+/* Booter view encoder. end must be 128 KiB aligned. */
+int gd_wpr2_encode(uint64_t start, uint64_t end, uint32_t *lo, uint32_t *hi);
+
+/* FWSEC publishes the inclusive 4 KiB end as ((end - 1) >> 12) << 4.
+ * Return the HI value the Booter expects for a 1 MiB FRTS window. */
+int gd_wpr2_fwsec_hi(uint32_t lo, uint32_t *hi);
+
+struct gd_wpr2_meta {
+ uint64_t wpr_start, wpr_end;
+ uint64_t frts_offset, frts_size;
+ uint64_t gsp_offset, gsp_size;
+ uint64_t boot_offset, boot_size;
+ uint64_t meta_size;
+};
+
+/* Rules the Booter enforces on a WprMeta-proposed range. */
+int gd_wpr2_validate(const struct gd_wpr2_meta *meta);
+
+/* Unit-test entry: returns 0 when every vector passes. */
+int gd_wpr2_selftest(void);
+
+#endif
diff --git a/tests/integration/hooks.py b/tests/integration/hooks.py
new file mode 100644
index 0000000..cd98977
--- /dev/null
+++ b/tests/integration/hooks.py
@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Hook parity for the device model.
+
+booter.toml is the source of the register inventory. The generated header in
+device/booter_regs.h must match its resets and masks, and every non-storage
+operation must have a compiled counterpart in gd.c. Unsupported operations
+fail the parity check instead of silently drifting.
+"""
+from __future__ import annotations
+
+import re
+import sys
+import tomllib
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[1]
+TOML = ROOT / "emulator" / "hw" / "booter.toml"
+HEADER = HERE / "device" / "booter_regs.h"
+DEVICE = HERE / "device" / "gd.c"
+
+# These are the only non-storage operations in booter.toml.
+COMPILED_HOOKS = {0x404, 0x700}
+
+
+def inventory():
+ with TOML.open("rb") as file:
+ data = tomllib.load(file)
+ rows = {"bar0": [], "io": []}
+ special = []
+ for name, reg in data.get("reg", {}).items():
+ space = reg.get("space", "io")
+ rows[space].append((reg["addr"], reg.get("reset", 0), reg.get("mask", 0xFFFFFFFF)))
+ known = {"space", "addr", "reset", "mask", "w1c", "plm", "read_levels",
+ "write_levels", "flr_reset", "denied_read", "host", "io"}
+ if set(reg) - known:
+ special.append((name, reg))
+ for space in rows:
+ rows[space].sort()
+ return rows, special
+
+
+def header_rows(space: str):
+ text = HEADER.read_text()
+ start = text.index("gd_%s_defaults[]" % space)
+ body = text[start:text.index("};", start)]
+ return [(int(a, 16), int(v, 16), int(m, 16))
+ for a, v, m in re.findall(r"\{ 0x([0-9a-f]+)u, 0x([0-9a-f]+)u, 0x([0-9a-f]+)u \}", body)]
+
+
+def check() -> list[str]:
+ failures = []
+ rows, special = inventory()
+ for space in ("bar0", "io"):
+ if header_rows(space) != rows[space]:
+ failures.append(f"{space} reset/mask inventory drifted from booter.toml")
+ addresses = {reg["addr"] for _, reg in special}
+ if addresses != COMPILED_HOOKS:
+ failures.append(f"TOML hooks {sorted(addresses)} != compiled {sorted(COMPILED_HOOKS)}")
+ device = DEVICE.read_text()
+ for address in sorted(COMPILED_HOOKS):
+ if "0x%x" % address not in device:
+ failures.append(f"compiled hook 0x{address:x} missing from gd.c")
+ return failures
+
+
+if __name__ == "__main__":
+ problems = check()
+ for problem in problems:
+ print(problem, file=sys.stderr)
+ raise SystemExit(1 if problems else 0)
diff --git a/tests/integration/inputs.py b/tests/integration/inputs.py
new file mode 100644
index 0000000..f0a754e
--- /dev/null
+++ b/tests/integration/inputs.py
@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Run-directory inputs: prepared ROM, assets and the per-run manifest."""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from . import assets
+
+
+def prepare(family: str, run_dir: Path) -> dict:
+ manifest = assets.prepare_run(family, run_dir)
+ manifest["rom_sha256"] = __import__("hashlib").sha256(
+ (Path(run_dir) / "rom.bin").read_bytes()).hexdigest()
+ (Path(run_dir) / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n")
+ return manifest
diff --git a/tests/integration/run.py b/tests/integration/run.py
new file mode 100644
index 0000000..7a78156
--- /dev/null
+++ b/tests/integration/run.py
@@ -0,0 +1,228 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Run one transaction through the native runner and write evidence."""
+from __future__ import annotations
+
+import ctypes as C
+import json
+import os
+import time
+from pathlib import Path
+
+from . import assets, build, codetable, inputs
+
+MAX_UNKNOWN = 16
+
+
+class Unknown(C.Structure):
+ _fields_ = [
+ ("write", C.c_bool),
+ ("address", C.c_uint32),
+ ("value", C.c_uint32),
+ ("pc", C.c_uint32),
+ ]
+
+
+class Result(C.Structure):
+ _fields_ = [
+ ("driver_result", C.c_int),
+ ("device_error", C.c_int),
+ ("fwsec_launch_reached", C.c_bool),
+ ("booter_launch_reached", C.c_bool),
+ ("fwsec_executed", C.c_bool),
+ ("fwsec_kdf_ok", C.c_bool),
+ ("booter_executed", C.c_bool),
+ ("booter_halted", C.c_bool),
+ ("plm_opened", C.c_bool),
+ ("target_written", C.c_bool),
+ ("unlock_verified", C.c_bool),
+ ("flr_seen", C.c_bool),
+ ("launches", C.c_uint32 * 2),
+ ("resets", C.c_uint32),
+ ("host_writes", C.c_uint64),
+ ("host_reads", C.c_uint64),
+ ("engine_writes", C.c_uint64),
+ ("engine_reads", C.c_uint64),
+ ("instructions", C.c_uint64 * 2),
+ ("unknown_count", C.c_uint32),
+ ("unknown", Unknown * MAX_UNKNOWN),
+ ("feats", C.c_uint32 * 8),
+ ("feat_count", C.c_uint32),
+ ("wpr2_lo", C.c_uint32),
+ ("wpr2_hi", C.c_uint32),
+ ("ss0", C.c_uint32),
+ ("ss1", C.c_uint32),
+ ("pci_command", C.c_uint32),
+ ("dma_live", C.c_uint32),
+ ("fault_pc", C.c_uint32),
+ ("fault_opcode", C.c_uint32),
+ ("plm_feat", C.c_uint32),
+ ("plm_wpr_cfg", C.c_uint32),
+ ("plm_wpr", C.c_uint32),
+ ("denied_writes", C.c_uint32),
+ ("booter_error", C.c_uint32),
+ ("stage", C.c_uint32),
+ ("fwsec_halt_pc", C.c_uint32),
+ ("booter_halt_pc", C.c_uint32),
+ ("fwsec_status", C.c_uint32),
+ ("booter_status", C.c_uint32),
+ ("booter_mailbox0", C.c_uint32),
+ ("booter_mailbox1", C.c_uint32),
+ ("booter_reset_plm", C.c_uint32),
+ ("group03", C.c_uint32),
+ ("fwsec_mac_ok", C.c_uint32),
+ ("wpr2_normalized", C.c_uint32),
+ ]
+
+
+def load_runner(library: Path):
+ lib = C.CDLL(str(library))
+ lib.gd_run_transaction.argtypes = [
+ C.c_uint16, C.c_uint32, C.c_char_p, C.c_char_p, C.c_char_p, C.c_char_p,
+ C.c_char_p, C.c_char_p,
+ C.c_int, C.c_int, C.POINTER(Result),
+ ]
+ lib.gd_run_transaction.restype = C.c_int
+ return lib
+
+
+def as_dict(family: str, manifest: dict, result: Result, elapsed: float) -> dict:
+ return {
+ "family": family,
+ "run_dir": manifest.get("run_dir"),
+ "device": manifest["device"],
+ "chip": manifest["chip"],
+ "fwsec_executed": bool(result.fwsec_executed),
+ "fwsec_kdf_ok": bool(result.fwsec_kdf_ok),
+ "booter_executed": bool(result.booter_executed),
+ "booter_halted": bool(result.booter_halted),
+ "plm_opened": bool(result.plm_opened),
+ "target_written": bool(result.target_written),
+ "unlock_verified": bool(result.unlock_verified),
+ "driver_result": result.driver_result,
+ "device_error": result.device_error,
+ "fwsec_launch_reached": bool(result.fwsec_launch_reached),
+ "booter_launch_reached": bool(result.booter_launch_reached),
+ "flr_seen": bool(result.flr_seen),
+ "launches": list(result.launches),
+ "resets": result.resets,
+ "host_writes": result.host_writes,
+ "host_reads": result.host_reads,
+ "engine_writes": result.engine_writes,
+ "engine_reads": result.engine_reads,
+ "instructions": list(result.instructions),
+ "unknown": [
+ {
+ "write": bool(result.unknown[i].write),
+ "address": result.unknown[i].address,
+ "value": result.unknown[i].value,
+ "pc": result.unknown[i].pc,
+ }
+ for i in range(min(result.unknown_count, MAX_UNKNOWN))
+ ],
+ "feats": list(result.feats)[: result.feat_count],
+ "wpr2": [result.wpr2_lo, result.wpr2_hi],
+ "selectors": [result.ss0, result.ss1],
+ "pci_command": result.pci_command,
+ "dma_live": result.dma_live,
+ "fault_pc": result.fault_pc,
+ "fault_opcode": result.fault_opcode,
+ "plm_feat": result.plm_feat,
+ "plm_wpr_cfg": result.plm_wpr_cfg,
+ "plm_wpr": result.plm_wpr,
+ "denied_writes": result.denied_writes,
+ "booter_error": result.booter_error,
+ "stage": STAGES.get(result.stage, result.stage),
+ "fwsec_halt_pc": result.fwsec_halt_pc,
+ "booter_halt_pc": result.booter_halt_pc,
+ "fwsec_status": result.fwsec_status,
+ "booter_status": result.booter_status,
+ "booter_mailbox0": result.booter_mailbox0,
+ "booter_mailbox1": result.booter_mailbox1,
+ "booter_reset_plm": result.booter_reset_plm,
+ "group03": result.group03,
+ "fwsec_mac_ok": bool(result.fwsec_mac_ok),
+ "wpr2_normalized": bool(result.wpr2_normalized),
+ "elapsed_s": round(elapsed, 3),
+ }
+
+
+def run_transaction(family: str, run_dir: Path, *,
+ threads: bool = False, verbose: bool = False,
+ library: Path | None = None) -> dict:
+ run_dir = Path(run_dir)
+ manifest = inputs.prepare(family, run_dir)
+ library = Path(library) if library else build.build()
+ lib = load_runner(library)
+ key_hex = os.environ.get("GPU_UNLOCK_DEBUG_KEY")
+ key = bytes.fromhex(key_hex) if key_hex else None
+ if key is not None and len(key) != 16:
+ raise SystemExit("GPU_UNLOCK_DEBUG_KEY must be 16 bytes of hex")
+ code_path = ""
+ booter_code_path = ""
+ if key is not None:
+ codetable.build(library, run_dir / "rom.bin", key, run_dir / "code.bin")
+ code_path = str(run_dir / "code.bin")
+ booter = run_dir / "assets" / assets.FAMILIES[family]["booter"]
+ codetable.build_booter(library, booter, key, run_dir / "booter_code.bin")
+ booter_code_path = str(run_dir / "booter_code.bin")
+ result = Result()
+ started = time.monotonic()
+ ret = lib.gd_run_transaction(
+ manifest["device"], manifest["chip"],
+ str(run_dir / "rom.bin").encode(),
+ str(run_dir / "assets").encode(),
+ str(run_dir / "trace.bin").encode(),
+ code_path.encode(), booter_code_path.encode(), key,
+ 1 if threads else 0, 1 if verbose else 0, C.byref(result),
+ )
+ elapsed = time.monotonic() - started
+ record = as_dict(family, manifest, result, elapsed)
+ record["return"] = ret
+ (run_dir / "result.json").write_text(json.dumps(record, indent=2) + "\n")
+ decode_trace(run_dir / "trace.bin", run_dir / "trace.jsonl")
+ return record
+
+
+TRACE_KINDS = {0: "bar-read", 1: "bar-write", 2: "dma", 3: "start", 4: "halt", 5: "flr", 6: "fault", 7: "pc"}
+
+STAGES = {
+ 0: "pre-fwsec",
+ 1: "fwsec-running",
+ 2: "fwsec-done",
+ 3: "wpr2-ready",
+ 4: "booter-running",
+ 5: "booter-halted",
+ 6: "booter-failed",
+}
+
+
+def decode_trace(binary: Path, text: Path):
+ data = Path(binary).read_bytes()
+ if len(data) < 4:
+ return
+ count = int.from_bytes(data[:4], "little")
+ lines = []
+ offset = 4
+ for _ in range(count):
+ if offset + 32 > len(data):
+ break
+ time_us, pc, address, value, kind, source, size = __import__("struct").unpack_from("<QIIIHHH", data, offset)
+ offset += 32
+ lines.append({"time_us": time_us, "pc": pc, "address": address,
+ "value": value, "kind": TRACE_KINDS.get(kind, kind),
+ "source": source, "size": size})
+ Path(text).write_text("".join(json.dumps(line) + "\n" for line in lines))
+
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("family", choices=sorted(assets.FAMILIES))
+ parser.add_argument("--run-dir", default=None)
+ parser.add_argument("--threads", action="store_true")
+ parser.add_argument("--verbose", action="store_true")
+ args = parser.parse_args()
+ directory = Path(args.run_dir or (Path("build") / "runs" / args.family))
+ print(json.dumps(run_transaction(
+ args.family, directory, threads=args.threads, verbose=args.verbose), indent=2))
diff --git a/tests/integration/runner.c b/tests/integration/runner.c
new file mode 100644
index 0000000..626dc6b
--- /dev/null
+++ b/tests/integration/runner.c
@@ -0,0 +1,153 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* One exported entry for the integration harness. The driver, shim and
+ * device are linked into one shared object; this function is the only
+ * boundary Python crosses. */
+#include <errno.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "device/gd.h"
+#include "shim/compat.h"
+#include "../../kernel/gpu_unlock/firmware_610.h"
+#include "../../kernel/gpu_unlock/recipes.h"
+#include "../../kernel/gpu_unlock/unlock_610.h"
+#include "../../kernel/gpu_unlock/vbios_610.h"
+#include "device/aes.h"
+#include "device/wpr2.h"
+
+#if defined(__GNUC__)
+#define GD_EXPORT __attribute__((visibility("default")))
+#else
+#define GD_EXPORT
+#endif
+
+struct gd_fwsec_info {
+ uint32_t version, image_off, image_size, data_off, imem_size, dmem_size, interface_off;
+};
+
+GD_EXPORT int gd_fwsec_extract(const char *rom_path, struct gd_fwsec_info *info)
+{
+ FILE *file;
+ long size;
+ uint8_t *rom;
+ struct gpu_unlock_fwsec fw;
+ int ret;
+ if (!rom_path || !info)
+ return -EINVAL;
+ file = fopen(rom_path, "rb");
+ if (!file)
+ return -ENOENT;
+ fseek(file, 0, SEEK_END);
+ size = ftell(file);
+ fseek(file, 0, SEEK_SET);
+ rom = malloc((size_t)size);
+ if (!rom) {
+ fclose(file);
+ return -ENOMEM;
+ }
+ if (fread(rom, 1, (size_t)size, file) != (size_t)size) {
+ free(rom);
+ fclose(file);
+ return -EIO;
+ }
+ fclose(file);
+ ret = gpu_unlock_vbios_fwsec(rom, (size_t)size, &fw);
+ free(rom);
+ if (ret)
+ return ret;
+ info->version = fw.version;
+ info->image_off = fw.image_off;
+ info->image_size = fw.image_size;
+ info->data_off = fw.data_off;
+ info->imem_size = fw.imem_size;
+ info->dmem_size = fw.dmem_size;
+ info->interface_off = fw.interface_off;
+ return 0;
+}
+
+GD_EXPORT int gd_aes_decrypt_buffer(const uint8_t *key, uint8_t *data, size_t size)
+{
+ if (!key || !data)
+ return -EINVAL;
+ return gd_aes128_decrypt(key, data, size);
+}
+
+GD_EXPORT int gd_test_aes(void)
+{
+ return gd_aes128_selftest() ? 1 : 0;
+}
+
+GD_EXPORT int gd_test_wpr2(void)
+{
+ return gd_wpr2_selftest();
+}
+
+GD_EXPORT int gd_run_transaction(uint16_t device, uint32_t chip,
+ const char *rom_path, const char *asset_root, const char *trace_path, const char *code_path,
+ const char *booter_code_path, const uint8_t *key,
+ int threads, int verbose,
+ struct gd_result *out)
+{
+ struct gd_board board = {
+ .device = device,
+ .chip = chip,
+ };
+ struct gd_run_config config = {
+ .asset_root = asset_root,
+ .rom_path = rom_path,
+ .verbose = verbose != 0,
+ .code_path = code_path,
+ .booter_code_path = booter_code_path,
+ };
+ if (key) {
+ memcpy(config.key, key, sizeof(config.key));
+ config.key_set = true;
+ }
+ struct gd_device *dev;
+ struct pci_dev pdev;
+ struct gu_device gpu;
+ struct gpu_unlock_610_shared shared;
+ uint64_t mask = 0xffffffffull;
+ int ret;
+
+ if (!out)
+ return -EINVAL;
+ memset(out, 0, sizeof(*out));
+ memset(&shared, 0, sizeof(shared));
+ dev = gd_create(&board, &config);
+ if (!dev) {
+ out->driver_result = -ENODEV;
+ out->device_error = -ENODEV;
+ return -ENODEV;
+ }
+ memset(&pdev, 0, sizeof(pdev));
+ pdev.dev.gd = dev;
+ pdev.dev.dma_mask = &mask;
+ pdev.dev.coherent_dma_mask = 0xffffffffull;
+ pdev.vendor = 0x10de;
+ pdev.device = device;
+ pdev.current_state = 0; /* PCI_D0 */
+ pdev.pm_cap = 0;
+ pdev.refs = 1;
+ gpu.pdev = &pdev;
+ gpu.recipe = gu_card_find(device);
+ if (!gpu.recipe) {
+ out->driver_result = -ENODEV;
+ out->device_error = -ENODEV;
+ gd_destroy(dev);
+ return -ENODEV;
+ }
+ /* Stage 1 runs the completion path synchronously; stage 2 turns the
+ * free-running engine threads on through the same device. */
+ gd_set_engines(dev, threads != 0);
+
+ ret = gpu_unlock_610_load_shared(&gpu, &shared);
+ if (!ret)
+ ret = gpu_unlock_610_unlock(&gpu, &shared, false);
+ gpu_unlock_610_release_shared(&shared);
+ gd_collect(dev, out);
+ gd_trace_dump(dev, trace_path);
+ out->driver_result = ret;
+ gd_destroy(dev);
+ return ret;
+}
diff --git a/tests/integration/shim/compat.h b/tests/integration/shim/compat.h
new file mode 100644
index 0000000..165ffc7
--- /dev/null
+++ b/tests/integration/shim/compat.h
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Userspace kernel API shim for the gpu_unlock driver. Every call below is a
+ * plain C call into the device library; no Python and no mock driver. */
+#ifndef GD_SHIM_COMPAT_H
+#define GD_SHIM_COMPAT_H
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "../device/gd.h"
+
+typedef uint8_t u8;
+typedef uint16_t u16;
+typedef uint32_t u32;
+typedef uint64_t u64;
+typedef int32_t s32;
+typedef uint64_t dma_addr_t;
+typedef uint64_t resource_size_t;
+typedef int64_t ktime_t;
+
+#define __iomem
+#define __init
+#define __exit
+#define GFP_KERNEL 0
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
+#define min_t(t, a, b) ((t)(a) < (t)(b) ? (t)(a) : (t)(b))
+#define max_t(t, a, b) ((t)(a) > (t)(b) ? (t)(a) : (t)(b))
+#define lower_32_bits(v) ((u32)(v))
+#define upper_32_bits(v) ((u32)((u64)(v) >> 32))
+#define DMA_BIT_MASK(n) ((1ull << (n)) - 1)
+#define MODULE_FIRMWARE(x)
+#define MODULE_LICENSE(x)
+#define MODULE_DESCRIPTION(x)
+#define MODULE_VERSION(x)
+#define MODULE_PARM_DESC(x, y)
+#define THIS_MODULE NULL
+
+/* dev_* receives a struct device *, which always belongs to a pci_dev. */
+#define dev_err(dev, ...) gd_log((dev)->gd, GD_LOG_ERROR, __VA_ARGS__)
+#define dev_info(dev, ...) gd_log((dev)->gd, GD_LOG_INFO, __VA_ARGS__)
+#define dev_dbg(dev, ...) gd_log((dev)->gd, GD_LOG_DEBUG, __VA_ARGS__)
+/* The firmware/driver diagnostics are useful when investigating a failured
+ * run but make the acceptance output unreadable. They are opt-in. */
+static inline bool gd_diag_on(void)
+{
+ const char *on = getenv("GPU_UNLOCK_DIAG");
+ return on && *on && strcmp(on, "0");
+}
+#define pr_info(...) do { if (gd_diag_on()) fprintf(stderr, __VA_ARGS__); } while (0)
+#define pr_err(...) do { if (gd_diag_on()) fprintf(stderr, __VA_ARGS__); } while (0)
+#define pr_debug(...) do { if (gd_diag_on()) fprintf(stderr, __VA_ARGS__); } while (0)
+
+#define PCI_D0 0
+#define PCI_UNKNOWN 5
+#define PCI_DEV_FLAGS_NO_FLR_RESET (1u << 10)
+#define PCI_VENDOR_ID 0x00
+#define PCI_COMMAND 0x04
+#define PCI_COMMAND_MEMORY 0x02
+#define PCI_COMMAND_MASTER 0x04
+#define PCI_PM_CTRL 0x04
+#define PCI_PM_CTRL_STATE_MASK 0x03
+#define PCI_EXP_DEVCAP 0x04
+#define PCI_EXP_DEVCAP_FLR (1u << 28)
+#define PCI_EXP_DEVSTA 0x0a
+#define PCI_EXP_DEVSTA_TRPND 0x20
+#define PCI_BASE_ADDRESS_0 0x10
+#define IORESOURCE_MEM 0x200
+
+struct list_head { unsigned int count; };
+
+static inline bool list_empty(const struct list_head *h)
+{
+ return !h->count;
+}
+
+#define spin_lock_irqsave(lock, flags) do { (void)(lock); (flags) = 0; } while (0)
+#define spin_unlock_irqrestore(lock, flags) do { (void)(lock); (void)(flags); } while (0)
+
+struct device {
+ struct gd_device *gd;
+ u64 *dma_mask;
+ u64 coherent_dma_mask;
+ struct list_head devres_head;
+ int devres_lock;
+};
+
+struct pci_dev {
+ struct device dev;
+ unsigned short vendor, device;
+ int current_state, pm_cap, refs;
+ unsigned int dev_flags;
+ unsigned int saved_command;
+ unsigned int bars[6];
+};
+
+struct firmware {
+ size_t size;
+ const u8 *data;
+};
+
+void *devres_open_group(struct device *, void *, int);
+static inline void __module_get(void *module) { (void)module; }
+static inline void module_put(void *module) { (void)module; }
+int devres_release_group(struct device *, void *);
+
+void usleep_range(unsigned long min, unsigned long max);
+ktime_t ktime_get(void);
+static inline ktime_t ktime_add_us(ktime_t t, unsigned long us) { return t + (ktime_t)us; }
+static inline int ktime_compare(ktime_t a, ktime_t b) { return a > b ? 1 : a < b ? -1 : 0; }
+
+u32 readl(const void __iomem *address);
+void writel(u32 value, void __iomem *address);
+int memcpy_fromio(void *dest, const void __iomem *source, size_t size);
+
+void *dma_alloc_coherent(struct device *, size_t, dma_addr_t *, int);
+void dma_free_coherent(struct device *, size_t, void *, dma_addr_t);
+int dma_set_mask(struct device *, u64);
+int dma_set_coherent_mask(struct device *, u64);
+void dma_wmb(void);
+
+void *kzalloc(size_t, int);
+static inline void *kvcalloc(size_t n, size_t s, int flags) { (void)flags; return calloc(n, s); }
+static inline void *kvmalloc(size_t n, int flags) { (void)flags; return malloc(n); }
+#define kfree free
+#define kvfree free
+
+int request_firmware(const struct firmware **, const char *, struct device *);
+void release_firmware(const struct firmware *);
+
+int pci_read_config_word(struct pci_dev *, int, u16 *);
+int pci_read_config_dword(struct pci_dev *, int, u32 *);
+int pci_write_config_word(struct pci_dev *, int, u16);
+int pcie_capability_read_word(struct pci_dev *, int, u16 *);
+int pcie_capability_read_dword(struct pci_dev *, int, u32 *);
+int pci_is_pcie(struct pci_dev *);
+int pci_select_bars(struct pci_dev *, int);
+int pci_request_region(struct pci_dev *, int, const char *);
+void pci_release_region(struct pci_dev *, int);
+int aperture_remove_conflicting_pci_devices(struct pci_dev *, const char *);
+int pci_enable_device_mem(struct pci_dev *);
+void pci_disable_device(struct pci_dev *);
+resource_size_t pci_resource_len(struct pci_dev *, int);
+void *pci_iomap(struct pci_dev *, int, unsigned long);
+void pci_iounmap(struct pci_dev *, void *);
+void *pci_map_rom(struct pci_dev *, size_t *);
+void pci_unmap_rom(struct pci_dev *, void *);
+int pci_save_state(struct pci_dev *);
+void pci_restore_state(struct pci_dev *);
+void pci_set_master(struct pci_dev *);
+void pci_clear_master(struct pci_dev *);
+int pcie_flr(struct pci_dev *);
+struct pci_dev *pci_dev_get(struct pci_dev *);
+void pci_dev_put(struct pci_dev *);
+
+#endif
diff --git a/tests/unlock_shim/linux/aperture.h b/tests/integration/shim/linux/aperture.h
similarity index 100%
rename from tests/unlock_shim/linux/aperture.h
rename to tests/integration/shim/linux/aperture.h
diff --git a/tests/unlock_shim/linux/delay.h b/tests/integration/shim/linux/delay.h
similarity index 100%
rename from tests/unlock_shim/linux/delay.h
rename to tests/integration/shim/linux/delay.h
diff --git a/tests/main_shim/linux/async.h b/tests/integration/shim/linux/device.h
similarity index 100%
rename from tests/main_shim/linux/async.h
rename to tests/integration/shim/linux/device.h
diff --git a/tests/unlock_shim/linux/dma-mapping.h b/tests/integration/shim/linux/dma-mapping.h
similarity index 100%
rename from tests/unlock_shim/linux/dma-mapping.h
rename to tests/integration/shim/linux/dma-mapping.h
diff --git a/tests/unlock_shim/linux/firmware.h b/tests/integration/shim/linux/firmware.h
similarity index 100%
rename from tests/unlock_shim/linux/firmware.h
rename to tests/integration/shim/linux/firmware.h
diff --git a/tests/unlock_shim/linux/io.h b/tests/integration/shim/linux/io.h
similarity index 100%
rename from tests/unlock_shim/linux/io.h
rename to tests/integration/shim/linux/io.h
diff --git a/tests/main_shim/linux/list.h b/tests/integration/shim/linux/list.h
similarity index 100%
rename from tests/main_shim/linux/list.h
rename to tests/integration/shim/linux/list.h
diff --git a/tests/main_shim/linux/module.h b/tests/integration/shim/linux/module.h
similarity index 100%
rename from tests/main_shim/linux/module.h
rename to tests/integration/shim/linux/module.h
diff --git a/tests/main_shim/linux/pci.h b/tests/integration/shim/linux/pci.h
similarity index 100%
rename from tests/main_shim/linux/pci.h
rename to tests/integration/shim/linux/pci.h
diff --git a/tests/main_shim/linux/string.h b/tests/integration/shim/linux/printk.h
similarity index 77%
rename from tests/main_shim/linux/string.h
rename to tests/integration/shim/linux/printk.h
index 4715430..b5a2194 100644
--- a/tests/main_shim/linux/string.h
+++ b/tests/integration/shim/linux/printk.h
@@ -1,2 +1,3 @@
/* SPDX-License-Identifier: GPL-2.0-only */
+#include <stdio.h>
#include "../compat.h"
diff --git a/tests/main_shim/linux/slab.h b/tests/integration/shim/linux/slab.h
similarity index 100%
rename from tests/main_shim/linux/slab.h
rename to tests/integration/shim/linux/slab.h
diff --git a/tests/integration/shim/linux/string.h b/tests/integration/shim/linux/string.h
new file mode 100644
index 0000000..3b2f590
--- /dev/null
+++ b/tests/integration/shim/linux/string.h
@@ -0,0 +1 @@
+#include <string.h>
diff --git a/tests/unlock_shim/linux/timekeeping.h b/tests/integration/shim/linux/timekeeping.h
similarity index 100%
rename from tests/unlock_shim/linux/timekeeping.h
rename to tests/integration/shim/linux/timekeeping.h
diff --git a/tests/unlock_shim/linux/types.h b/tests/integration/shim/linux/types.h
similarity index 100%
rename from tests/unlock_shim/linux/types.h
rename to tests/integration/shim/linux/types.h
diff --git a/tests/integration/shim/shim.c b/tests/integration/shim/shim.c
new file mode 100644
index 0000000..d14f417
--- /dev/null
+++ b/tests/integration/shim/shim.c
@@ -0,0 +1,383 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Userspace kernel shim: every operation is a plain C call into the device. */
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+#include "compat.h"
+
+struct bar_window {
+ void *base;
+ struct gd_device *dev;
+};
+
+static struct bar_window windows[8];
+static pthread_mutex_t window_lock = PTHREAD_MUTEX_INITIALIZER;
+
+static struct bar_window *window_for(const void *address)
+{
+ unsigned int i;
+ for (i = 0; i < ARRAY_SIZE(windows); i++) {
+ struct bar_window *w = &windows[i];
+ if (w->base && (const char *)address >= (const char *)w->base &&
+ (const char *)address < (const char *)w->base + GD_BAR_SIZE)
+ return w;
+ }
+ return NULL;
+}
+
+static u32 pci_command(struct pci_dev *pdev)
+{
+ u32 command = 0;
+ gd_pci_read(pdev->dev.gd, 0, PCI_COMMAND, 2, &command);
+ return command;
+}
+
+static void pci_set_command(struct pci_dev *pdev, u32 command)
+{
+ gd_pci_write(pdev->dev.gd, 0, PCI_COMMAND, 2, command);
+}
+
+void *devres_open_group(struct device *dev, void *id, int flags)
+{
+ (void)flags;
+ unsigned int *group;
+ if (id)
+ return NULL;
+ group = malloc(sizeof(*group));
+ if (!group)
+ return NULL;
+ *group = dev->devres_head.count;
+ dev->devres_head.count++;
+ return group;
+}
+
+int devres_release_group(struct device *dev, void *id)
+{
+ unsigned int *group = id;
+ int released;
+ if (!group)
+ return 0;
+ released = (int)dev->devres_head.count - (int)*group - 1;
+ if (released < 0)
+ released = 0;
+ dev->devres_head.count = *group;
+ free(group);
+ return released;
+}
+
+ktime_t ktime_get(void)
+{
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (ktime_t)ts.tv_sec * 1000000 + ts.tv_nsec / 1000;
+}
+
+void usleep_range(unsigned long min, unsigned long max)
+{
+ struct timespec ts;
+ (void)max;
+ ts.tv_sec = min / 1000000;
+ ts.tv_nsec = (min % 1000000) * 1000;
+ while (nanosleep(&ts, &ts) == -1 && errno == EINTR)
+ ;
+}
+
+u32 readl(const void __iomem *address)
+{
+ struct bar_window *w = window_for(address);
+ if (!w)
+ return 0xffffffffu;
+ return gd_bar0_read(w->dev, (u32)((const char *)address - (const char *)w->base));
+}
+
+void writel(u32 value, void __iomem *address)
+{
+ struct bar_window *w = window_for(address);
+ if (!w)
+ return;
+ gd_bar0_write(w->dev, (u32)((char *)address - (char *)w->base), value);
+}
+
+int memcpy_fromio(void *dest, const void __iomem *source, size_t size)
+{
+ struct bar_window *w = window_for(source);
+ if (!w)
+ return -EINVAL;
+ return gd_prom_copy(w->dev, (u32)((const char *)source - (const char *)w->base), dest, size);
+}
+
+void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *address, int flags)
+{
+ static uint64_t next = 0x10000000;
+ void *cpu;
+ (void)flags;
+ size = (size + 255) & ~(size_t)255;
+ if (!size)
+ size = 256;
+ cpu = calloc(1, size);
+ if (!cpu)
+ return NULL;
+ *address = next;
+ next += (size + 0xfff) & ~0xfffull;
+ if (gd_dma_map(dev->gd, *address, cpu, size)) {
+ free(cpu);
+ return NULL;
+ }
+ return cpu;
+}
+
+void dma_free_coherent(struct device *dev, size_t size, void *cpu, dma_addr_t address)
+{
+ (void)size;
+ gd_dma_unmap(dev->gd, address);
+ free(cpu);
+}
+
+int dma_set_mask(struct device *dev, u64 mask)
+{
+ if (mask < 0xffffffffull)
+ return -EIO;
+ *dev->dma_mask = mask;
+ return 0;
+}
+
+int dma_set_coherent_mask(struct device *dev, u64 mask)
+{
+ if (mask < 0xffffffffull)
+ return -EIO;
+ dev->coherent_dma_mask = mask;
+ return 0;
+}
+
+void dma_wmb(void)
+{
+ __sync_synchronize();
+}
+
+void *kzalloc(size_t size, int flags)
+{
+ (void)flags;
+ return calloc(1, size ? size : 1);
+}
+
+int request_firmware(const struct firmware **out, const char *name, struct device *dev)
+{
+ char path[4096];
+ const char *base = strrchr(name, '/');
+ struct firmware *fw;
+ FILE *file;
+ long size;
+
+ base = base ? base + 1 : name;
+ snprintf(path, sizeof(path), "%s/%s", gd_asset_root(dev->gd), base);
+ file = fopen(path, "rb");
+ if (!file)
+ return -ENOENT;
+ if (fseek(file, 0, SEEK_END) || (size = ftell(file)) <= 0 || fseek(file, 0, SEEK_SET)) {
+ fclose(file);
+ return -EIO;
+ }
+ fw = malloc(sizeof(*fw));
+ if (!fw) {
+ fclose(file);
+ return -ENOMEM;
+ }
+ fw->size = (size_t)size;
+ fw->data = malloc(fw->size);
+ if (!fw->data) {
+ free(fw);
+ fclose(file);
+ return -ENOMEM;
+ }
+ if (fread((void *)fw->data, 1, fw->size, file) != fw->size) {
+ free((void *)fw->data);
+ free(fw);
+ fclose(file);
+ return -EIO;
+ }
+ fclose(file);
+ *out = fw;
+ return 0;
+}
+
+void release_firmware(const struct firmware *fw)
+{
+ if (!fw)
+ return;
+ free((void *)fw->data);
+ free((void *)fw);
+}
+
+int pci_read_config_word(struct pci_dev *pdev, int offset, u16 *value)
+{
+ u32 v;
+ int ret = gd_pci_read(pdev->dev.gd, 0, offset, 2, &v);
+ *value = (u16)v;
+ return ret;
+}
+
+int pci_read_config_dword(struct pci_dev *pdev, int offset, u32 *value)
+{
+ return gd_pci_read(pdev->dev.gd, 0, offset, 4, value);
+}
+
+int pci_write_config_word(struct pci_dev *pdev, int offset, u16 value)
+{
+ return gd_pci_write(pdev->dev.gd, 0, offset, 2, value);
+}
+
+int pcie_capability_read_word(struct pci_dev *pdev, int offset, u16 *value)
+{
+ u32 v;
+ int ret = gd_pci_read(pdev->dev.gd, 1, offset, 2, &v);
+ *value = (u16)v;
+ return ret;
+}
+
+int pcie_capability_read_dword(struct pci_dev *pdev, int offset, u32 *value)
+{
+ return gd_pci_read(pdev->dev.gd, 1, offset, 4, value);
+}
+
+int pci_is_pcie(struct pci_dev *pdev)
+{
+ (void)pdev;
+ return 1;
+}
+
+int pci_select_bars(struct pci_dev *pdev, int flags)
+{
+ (void)pdev;
+ (void)flags;
+ return 1;
+}
+
+int pci_request_region(struct pci_dev *pdev, int bar, const char *name)
+{
+ (void)pdev;
+ (void)bar;
+ (void)name;
+ return 0;
+}
+
+void pci_release_region(struct pci_dev *pdev, int bar)
+{
+ (void)pdev;
+ (void)bar;
+}
+
+int aperture_remove_conflicting_pci_devices(struct pci_dev *pdev, const char *name)
+{
+ (void)pdev;
+ (void)name;
+ return 0;
+}
+
+int pci_enable_device_mem(struct pci_dev *pdev)
+{
+ pci_set_command(pdev, pci_command(pdev) | PCI_COMMAND_MEMORY);
+ return 0;
+}
+
+void pci_disable_device(struct pci_dev *pdev)
+{
+ pci_set_command(pdev, pci_command(pdev) & ~PCI_COMMAND_MEMORY);
+}
+
+resource_size_t pci_resource_len(struct pci_dev *pdev, int bar)
+{
+ (void)pdev;
+ (void)bar;
+ return GD_BAR_SIZE;
+}
+
+void *pci_iomap(struct pci_dev *pdev, int bar, unsigned long maxlen)
+{
+ unsigned int i;
+ void *base;
+ (void)bar;
+ (void)maxlen;
+ base = gd_bar_pointer(pdev->dev.gd);
+ pthread_mutex_lock(&window_lock);
+ for (i = 0; i < ARRAY_SIZE(windows); i++) {
+ if (!windows[i].base) {
+ windows[i].base = base;
+ windows[i].dev = pdev->dev.gd;
+ pthread_mutex_unlock(&window_lock);
+ return base;
+ }
+ }
+ pthread_mutex_unlock(&window_lock);
+ return NULL;
+}
+
+void pci_iounmap(struct pci_dev *pdev, void *address)
+{
+ unsigned int i;
+ (void)pdev;
+ pthread_mutex_lock(&window_lock);
+ for (i = 0; i < ARRAY_SIZE(windows); i++) {
+ if (windows[i].base == address) {
+ windows[i].base = NULL;
+ windows[i].dev = NULL;
+ }
+ }
+ pthread_mutex_unlock(&window_lock);
+}
+
+void *pci_map_rom(struct pci_dev *pdev, size_t *size)
+{
+ (void)pdev;
+ if (size)
+ *size = 0;
+ return NULL;
+}
+
+void pci_unmap_rom(struct pci_dev *pdev, void *rom)
+{
+ (void)pdev;
+ (void)rom;
+}
+
+int pci_save_state(struct pci_dev *pdev)
+{
+ pdev->saved_command = pci_command(pdev);
+ return 0;
+}
+
+void pci_restore_state(struct pci_dev *pdev)
+{
+ pci_set_command(pdev, pdev->saved_command);
+}
+
+void pci_set_master(struct pci_dev *pdev)
+{
+ pci_set_command(pdev, pci_command(pdev) | PCI_COMMAND_MASTER);
+}
+
+void pci_clear_master(struct pci_dev *pdev)
+{
+ pci_set_command(pdev, pci_command(pdev) & ~PCI_COMMAND_MASTER);
+}
+
+int pcie_flr(struct pci_dev *pdev)
+{
+ return gd_reset(pdev->dev.gd);
+}
+
+struct pci_dev *pci_dev_get(struct pci_dev *pdev)
+{
+ if (pdev)
+ __atomic_add_fetch(&pdev->refs, 1, __ATOMIC_SEQ_CST);
+ return pdev;
+}
+
+void pci_dev_put(struct pci_dev *pdev)
+{
+ if (pdev)
+ __atomic_sub_fetch(&pdev->refs, 1, __ATOMIC_SEQ_CST);
+}
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
new file mode 100644
index 0000000..654417f
--- /dev/null
+++ b/tests/integration/test_integration.py
@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: GPL-2.0-only
+"""Behavioral gates for the native integration runner.
+
+The keyed GA10x run executes the real FWSEC and SEC2 Booter and injects the
+real V67 payload. The payload does not yet reach a clean Booter halt, so the
+final unlock checks fail honestly while the FWSEC and Booter execution evidence
+is still asserted.
+"""
+from __future__ import annotations
+
+import os
+from contextlib import contextmanager
+from pathlib import Path
+
+from . import build, hooks
+from .run import load_runner, run_transaction
+
+ROOT = Path(__file__).resolve().parents[2]
+RUNS = ROOT / "build" / "runs" / "tests"
+FAILURES: list[str] = []
+CHECKS = 0
+
+
+def check(condition, message, detail=None, fail=None):
+ global CHECKS
+
+ CHECKS += 1
+ if not condition:
+ text = fail or f"not: {message}"
+ FAILURES.append(text)
+ print("FAIL:", text)
+ if detail:
+ print(" ", detail)
+ else:
+ print("ok:", message)
+
+
+@contextmanager
+def section(title):
+ print(f"\n== {title} ==")
+ checks = CHECKS
+ failures = len(FAILURES)
+ yield
+ count = CHECKS - checks
+ print(f"-- {title}: {count} check{'s' if count != 1 else ''}, "
+ f"{len(FAILURES) - failures} failures")
+
+
+def report(record, label):
+ print(f"[{label}] driver={record['driver_result']} stage={record['stage']} "
+ f"launches={record['launches']} unlocked={record['unlock_verified']}")
+ print(f" fwsec={record['fwsec_executed']} mac={record['fwsec_mac_ok']} "
+ f"booter_insns={record['instructions'][0]} "
+ f"halt={record['booter_halt_pc']:#x} fault={record['fault_pc']:#x} "
+ f"status={record['booter_status']} reset_plm={record['booter_reset_plm']:#x}")
+
+
+def main():
+ print("gpu-unlock integration tests")
+ print("driver diagnostics: off (set GPU_UNLOCK_DIAG=1 to show them)")
+ RUNS.mkdir(parents=True, exist_ok=True)
+
+ with section("build"):
+ lib = build.build()
+ print(f"library: {lib}")
+
+ with section("unit self-tests"):
+ runner = load_runner(lib)
+ check(runner.gd_test_aes() == 1, "AES-128 FIPS-197 vector round-trips")
+ check(runner.gd_test_wpr2() == 0, "WPR2 encoding and range vectors pass")
+ check(hooks.check() == [], "TOML inventory and compiled hooks agree")
+
+ key = os.environ.get("GPU_UNLOCK_DEBUG_KEY")
+ if not key:
+ check(False, "real path: GPU_UNLOCK_DEBUG_KEY is required")
+ return 1
+
+ with section("real path: FWSEC + Booter + V67 payload"):
+ record = run_transaction("GA10x", RUNS / "GA10x", threads=True, library=lib)
+ report(record, "GA10x real")
+ check(record["fwsec_launch_reached"], "FWSEC launch observed")
+ check(record["instructions"][1] > 1000, "real FWSEC executed")
+ check(record["fwsec_executed"], "real FWSEC completed")
+ check(record["fwsec_mac_ok"], "real FWSEC DMEM MAC matched")
+ check(record["fwsec_kdf_ok"], "SCP op4 KDF produced the debug MAC")
+ check(record["fwsec_status"] == 0, "real FWSEC left a clean status")
+ check(record["booter_launch_reached"], "real Booter launch observed")
+ check(record["instructions"][0] > 1000, "real Booter executed")
+ check(record["booter_halt_pc"] != 0 or record["fault_pc"] != 0,
+ "real Booter stop PC recorded")
+ check(Path(record["run_dir"], "trace.jsonl").exists(), "trace decoded")
+ # Final acceptance. Report each failed precondition with the observed
+ # state so the verdict is readable without decoding raw booleans.
+ check(record["booter_halted"] and record["booter_status"] == 0 and
+ (record["booter_reset_plm"] & 0xff) == 0xff,
+ "real payload: Booter halted cleanly",
+ f"halted={record['booter_halted']} status={record['booter_status']:#x} "
+ f"reset_plm={record['booter_reset_plm']:#x} "
+ f"halt_pc={record['booter_halt_pc']:#x} fault_pc={record['fault_pc']:#x}",
+ fail="real payload: Booter did not halt cleanly")
+ check(record["target_written"], "real payload: V67 wrote the target",
+ f"target_written={record['target_written']} "
+ f"plm_opened={record['plm_opened']}",
+ fail="real payload: V67 did not write the target")
+ check(record["unlock_verified"], "real payload: card reached the unlocked state",
+ "requires a clean Booter halt, PLM open, target write, "
+ f"selectors 0x88888888/0x8 and RESET_PLM=0xffffffff; observed "
+ f"plm_opened={record['plm_opened']} "
+ f"target_written={record['target_written']} "
+ f"selectors={record['selectors'][0]:#010x}/{record['selectors'][1]:#x} "
+ f"reset_plm={record['booter_reset_plm']:#x}",
+ fail="real payload: card is not unlocked")
+
+ print(f"\n{len(FAILURES)} failure(s)")
+ return 1 if FAILURES else 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/main_shim/compat.h b/tests/main_shim/compat.h
deleted file mode 100644
index bc22ad2..0000000
--- a/tests/main_shim/compat.h
+++ /dev/null
@@ -1,105 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Just enough kernel for main.c. Async work runs on real threads, so the
- * test sees true concurrency and main.c's own synchronization. */
-#ifndef MAIN_SHIM_H
-#define MAIN_SHIM_H
-#include <pthread.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-
-#define __init
-#define __exit
-#define module_param(name, type, perm)
-#define MODULE_PARM_DESC(name, desc)
-#define MODULE_LICENSE(x)
-#define MODULE_DESCRIPTION(x)
-#define MODULE_VERSION(x)
-/* Keep the entry points reachable so the test calls what the kernel would. */
-#define module_init(fn) int (*test_module_init)(void) = fn;
-#define module_exit(fn) void (*test_module_exit)(void) = fn;
-
-#define PCI_VENDOR_ID_NVIDIA 0x10de
-#define PCI_ANY_ID (~0u)
-#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
-struct pci_driver {
- const char *name;
-};
-struct pci_dev {
- const char *name;
- unsigned short vendor, device;
- struct pci_driver *driver;
- int refs;
- pthread_mutex_t lock;
- pthread_t owner;
- bool locked;
-};
-struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, struct pci_dev *from);
-struct pci_dev *pci_dev_get(struct pci_dev *);
-void pci_dev_put(struct pci_dev *);
-void pci_dev_lock(struct pci_dev *);
-void pci_dev_unlock(struct pci_dev *);
-static inline const char *pci_name(const struct pci_dev *pdev) { return pdev->name; }
-
-void test_log(const struct pci_dev *, const char *, ...) __attribute__((format(printf, 2, 3)));
-#define pci_err(pdev, ...) test_log(pdev, __VA_ARGS__)
-#define pci_info(pdev, ...) test_log(pdev, __VA_ARGS__)
-#define pr_err(...) test_log(NULL, __VA_ARGS__)
-
-typedef uint64_t async_cookie_t;
-typedef void (*async_func_t)(void *data, async_cookie_t cookie);
-struct async_domain {
- pthread_mutex_t lock;
- pthread_t threads[16];
- unsigned int count;
-};
-#define ASYNC_DOMAIN_EXCLUSIVE(name) struct async_domain name = { .lock = PTHREAD_MUTEX_INITIALIZER }
-async_cookie_t async_schedule_domain(async_func_t, void *, struct async_domain *);
-void async_synchronize_full_domain(struct async_domain *);
-#define GFP_KERNEL 0
-static inline void *kzalloc(size_t size, int flags) { return calloc(1, size); }
-#define kfree free
-
-/* The list operations main.c uses. */
-struct list_head {
- struct list_head *next, *prev;
-};
-#define LIST_HEAD(name) struct list_head name = { &(name), &(name) }
-static inline bool list_empty(const struct list_head *head) { return head->next == head; }
-static inline void list_add_tail(struct list_head *node, struct list_head *head)
-{
- node->prev = head->prev;
- node->next = head;
- head->prev->next = node;
- head->prev = node;
-}
-static inline void list_del(struct list_head *node)
-{
- node->prev->next = node->next;
- node->next->prev = node->prev;
- node->next = node->prev = NULL;
-}
-#define list_entry(ptr, type, member) ((type *)((char *)(ptr) - offsetof(type, member)))
-#define list_for_each_entry(pos, head, member) \
- for (pos = list_entry((head)->next, __typeof__(*pos), member); \
- &pos->member != (head); \
- pos = list_entry(pos->member.next, __typeof__(*pos), member))
-#define list_for_each_entry_safe(pos, n, head, member) \
- for (pos = list_entry((head)->next, __typeof__(*pos), member), \
- n = list_entry(pos->member.next, __typeof__(*pos), member); \
- &pos->member != (head); \
- pos = n, n = list_entry(n->member.next, __typeof__(*n), member))
-
-#define PM_SUSPEND_PREPARE 0x0003
-#define PM_POST_SUSPEND 0x0004
-#define NOTIFY_DONE 0x0000
-struct notifier_block {
- int (*notifier_call)(struct notifier_block *, unsigned long, void *);
- int priority;
-};
-int register_pm_notifier(struct notifier_block *);
-int unregister_pm_notifier(struct notifier_block *);
-#endif
diff --git a/tests/main_shim/linux/init.h b/tests/main_shim/linux/init.h
deleted file mode 100644
index 4715430..0000000
--- a/tests/main_shim/linux/init.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include "../compat.h"
diff --git a/tests/main_shim/linux/suspend.h b/tests/main_shim/linux/suspend.h
deleted file mode 100644
index 4715430..0000000
--- a/tests/main_shim/linux/suspend.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include "../compat.h"
diff --git a/tests/test_gfw_eval.c b/tests/test_gfw_eval.c
deleted file mode 100644
index 4596e75..0000000
--- a/tests/test_gfw_eval.c
+++ /dev/null
@@ -1,48 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-
-#include "gfw_eval.h"
-
-int main(void)
-{
- struct gpu_unlock_gfw_sample sample = {
- .config_ok = true,
- .plm = GU_GFW_PLM_LEVEL0,
- .boot = GU_GFW_BOOT_COMPLETED,
- .cpuctl = GU_GFW_CPUCTL_HALTED,
- };
-
- assert(gpu_unlock_gfw_assess(&sample) == 0);
- sample.cpuctl = 0;
- assert(gpu_unlock_gfw_assess(&sample) == -EAGAIN);
- sample.cpuctl = GU_GFW_CPUCTL_HALTED;
- sample.plm = 0;
- assert(gpu_unlock_gfw_assess(&sample) == -EAGAIN);
- sample.plm = GU_GFW_PLM_LEVEL0;
- sample.boot = 0xfe;
- assert(gpu_unlock_gfw_assess(&sample) == -EAGAIN);
- sample.boot = (u32)~0U;
- sample.plm = (u32)~0U;
- sample.cpuctl = (u32)~0U;
- assert(gpu_unlock_gfw_assess(&sample) == -ENODEV);
- sample.boot = GU_GFW_BOOT_COMPLETED;
- sample.plm = GU_GFW_PLM_LEVEL0;
- sample.cpuctl = GU_GFW_CPUCTL_HALTED;
- sample.config_ok = false;
- assert(gpu_unlock_gfw_assess(&sample) == -ENODEV);
-
- assert(gpu_unlock_command_restore(0, GU_PCI_COMMAND_MEMORY |
- GU_PCI_COMMAND_MASTER) == 0);
- assert(gpu_unlock_command_restore(GU_PCI_COMMAND_MEMORY,
- GU_PCI_COMMAND_MEMORY |
- GU_PCI_COMMAND_MASTER) ==
- GU_PCI_COMMAND_MEMORY);
- assert(gpu_unlock_command_restore(GU_PCI_COMMAND_IO,
- GU_PCI_COMMAND_IO | GU_PCI_COMMAND_MEMORY) ==
- GU_PCI_COMMAND_IO);
-
- puts("gfw predicate and command restore checks passed");
- return 0;
-}
diff --git a/tests/test_layout_610.c b/tests/test_layout_610.c
deleted file mode 100644
index c59e589..0000000
--- a/tests/test_layout_610.c
+++ /dev/null
@@ -1,35 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <string.h>
-
-#include "layout_610.h"
-
-int main(void)
-{
- struct gpu_unlock_layout_in in = {
- .usable_fb = 24ull << 30,
- .vga_base = (24ull << 30) - (1024ull * 1024),
- .frts_size = 1024ull * 1024,
- .boot_size = 4096,
- .image_size = 64ull * 1024,
- .nonwpr_size = 1,
- .heap_request = 1024ull * 1024,
- };
- struct gpu_unlock_layout_out out;
-
- assert(gpu_unlock_layout(NULL, &out) == -EINVAL);
- assert(!gpu_unlock_layout(&in, &out));
- assert(out.wpr_end > out.frts_offset && out.frts_offset > out.boot_offset);
- assert(out.boot_offset > out.gsp_offset && out.gsp_offset > out.heap_offset);
- assert(out.heap_offset > out.wpr_start && out.wpr_start > out.nonwpr_start);
- assert((out.wpr_end & ((128ull * 1024) - 1)) == 0);
- assert(out.heap_size >= in.heap_request);
- /* Reservations that do not fit below the VGA workspace fail cleanly. */
- in.frts_size = in.vga_base;
- assert(gpu_unlock_layout(&in, &out) == -EINVAL ||
- gpu_unlock_layout(&in, &out) == -ERANGE);
- puts("layout checks passed");
- return 0;
-}
diff --git a/tests/test_main.c b/tests/test_main.c
deleted file mode 100644
index d1c2a92..0000000
--- a/tests/test_main.c
+++ /dev/null
@@ -1,422 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* main.c against mocked PCI and a mocked unlock: which cards are unlocked,
- * that they run concurrently, that the shared firmware outlives them, and
- * that cards unlocked at boot are unlocked again after resume, before the
- * notifier chain reaches nvidia. */
-#define _GNU_SOURCE
-#include <assert.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <time.h>
-#include <unistd.h>
-#include "../kernel/gpu_unlock/main.c"
-
-#define MAX_DEVICES 8
-#define RTX_3090 0x2204
-#define RTX_3070 0x2484
-#define RTX_A2000 0x2531 /* GA106 professional: not supported */
-
-static struct pci_dev devices[MAX_DEVICES];
-static unsigned int device_count;
-static char gsp_marker;
-#define fake_gsp ((const struct firmware *)&gsp_marker)
-static pthread_mutex_t fake_lock = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t fake_cond = PTHREAD_COND_INITIALIZER;
-static struct pci_driver nvidia_driver = { "nvidia" }, vfio_driver = { "vfio-pci" };
-static struct {
- int loads, releases, fail_load;
- bool loaded;
- struct pci_dev *load_dev;
- int started, finished, in_flight, max_in_flight;
- int expected_concurrent; /* unlocks that must overlap */
- int unlocks[MAX_DEVICES], fail_unlock[MAX_DEVICES];
- int resumes[MAX_DEVICES], fail_resume[MAX_DEVICES];
- struct notifier_block *nb;
- int registrations;
- char log[16384];
-} fake;
-
-void test_log(const struct pci_dev *pdev, const char *format, ...)
-{
- va_list args;
- size_t at;
-
- pthread_mutex_lock(&fake_lock);
- at = strlen(fake.log);
- if (pdev)
- at += snprintf(fake.log + at, sizeof(fake.log) - at, "%s: ", pdev->name);
- va_start(args, format);
- vsnprintf(fake.log + at, sizeof(fake.log) - at, format, args);
- va_end(args);
- pthread_mutex_unlock(&fake_lock);
-}
-
-struct pci_dev *pci_dev_get(struct pci_dev *pdev)
-{
- __atomic_add_fetch(&pdev->refs, 1, __ATOMIC_SEQ_CST);
- return pdev;
-}
-
-void pci_dev_put(struct pci_dev *pdev)
-{
- if (pdev)
- assert(__atomic_sub_fetch(&pdev->refs, 1, __ATOMIC_SEQ_CST) >= 1);
-}
-
-/* Like the kernel: returns the next match with a reference, drops from's. */
-struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, struct pci_dev *from)
-{
- unsigned int i = from ? (unsigned int)(from - devices) + 1 : 0;
-
- pci_dev_put(from);
- for (; i < device_count; i++)
- if (devices[i].vendor == vendor && (device == PCI_ANY_ID || devices[i].device == device))
- return pci_dev_get(&devices[i]);
- return NULL;
-}
-
-void pci_dev_lock(struct pci_dev *pdev)
-{
- pthread_mutex_lock(&pdev->lock);
- pdev->owner = pthread_self();
- pdev->locked = true;
-}
-
-void pci_dev_unlock(struct pci_dev *pdev)
-{
- assert(pdev->locked && pthread_equal(pdev->owner, pthread_self()));
- pdev->locked = false;
- pthread_mutex_unlock(&pdev->lock);
-}
-
-struct async_entry {
- async_func_t fn;
- void *data;
- async_cookie_t cookie;
-};
-
-static void *async_thread(void *arg)
-{
- struct async_entry *entry = arg;
-
- entry->fn(entry->data, entry->cookie);
- free(entry);
- return NULL;
-}
-
-async_cookie_t async_schedule_domain(async_func_t fn, void *data, struct async_domain *domain)
-{
- struct async_entry *entry = malloc(sizeof(*entry));
- async_cookie_t cookie;
-
- assert(entry);
- pthread_mutex_lock(&domain->lock);
- assert(domain->count < sizeof(domain->threads) / sizeof(domain->threads[0]));
- cookie = domain->count + 1;
- *entry = (struct async_entry){ fn, data, cookie };
- assert(!pthread_create(&domain->threads[domain->count++], NULL, async_thread, entry));
- pthread_mutex_unlock(&domain->lock);
- return cookie;
-}
-
-void async_synchronize_full_domain(struct async_domain *domain)
-{
- pthread_mutex_lock(&domain->lock);
- for (unsigned int i = 0; i < domain->count; i++)
- assert(!pthread_join(domain->threads[i], NULL));
- domain->count = 0;
- pthread_mutex_unlock(&domain->lock);
-}
-
-int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared)
-{
- pthread_mutex_lock(&fake_lock);
- /* Loaded once per round, before any unlock can use it. */
- assert(!fake.in_flight && !shared->gsp);
- fake.loads++;
- fake.load_dev = pdev;
- pthread_mutex_unlock(&fake_lock);
- if (fake.fail_load)
- return fake.fail_load;
- shared->gsp = fake_gsp;
- fake.loaded = true;
- return 0;
-}
-
-void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
-{
- pthread_mutex_lock(&fake_lock);
- /* Only after every unlock has returned. */
- assert(!fake.in_flight && fake.finished == fake.started);
- assert(shared->gsp == (fake.loaded ? fake_gsp : NULL));
- fake.releases++;
- fake.loaded = false;
- shared->gsp = NULL;
- pthread_mutex_unlock(&fake_lock);
-}
-
-int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared,
- bool resume)
-{
- unsigned int index = pdev - devices;
- struct timespec deadline;
- int ret;
-
- assert(shared->gsp == fake_gsp);
- assert(pdev->locked && pthread_equal(pdev->owner, pthread_self()));
- /* Unbound at boot; after resume only a card nvidia suspended. */
- assert(resume ? pdev->driver == &nvidia_driver : !pdev->driver);
- /* The initial reference, the card list's, and nothing dropped early. */
- assert(__atomic_load_n(&pdev->refs, __ATOMIC_SEQ_CST) >= 2);
- pthread_mutex_lock(&fake_lock);
- if (resume)
- fake.resumes[index]++;
- else
- fake.unlocks[index]++;
- fake.started++;
- if (++fake.in_flight > fake.max_in_flight)
- fake.max_in_flight = fake.in_flight;
- pthread_cond_broadcast(&fake_cond);
- /* Wait until every expected card is inside an unlock at once. Serial
- * unlocking never gets there and falls through on the timeout. */
- clock_gettime(CLOCK_REALTIME, &deadline);
- deadline.tv_sec += 2;
- while (fake.max_in_flight < fake.expected_concurrent &&
- pthread_cond_timedwait(&fake_cond, &fake_lock, &deadline) != ETIMEDOUT)
- ;
- pthread_mutex_unlock(&fake_lock);
- /* Still busy after the rendezvous, so an unsynchronized init would
- * return, and release the firmware, while this card is mid-unlock. */
- usleep(20000);
- pthread_mutex_lock(&fake_lock);
- fake.in_flight--;
- fake.finished++;
- ret = resume ? fake.fail_resume[index] : fake.fail_unlock[index];
- pthread_mutex_unlock(&fake_lock);
- return ret;
-}
-
-int register_pm_notifier(struct notifier_block *nb)
-{
- assert(!fake.nb);
- fake.nb = nb;
- fake.registrations++;
- return 0;
-}
-
-int unregister_pm_notifier(struct notifier_block *nb)
-{
- assert(fake.nb == nb);
- fake.nb = NULL;
- return 0;
-}
-
-static void add(const char *name, unsigned short device, bool bound)
-{
- struct pci_dev *pdev = &devices[device_count++];
-
- *pdev = (struct pci_dev){ .name = name, .vendor = PCI_VENDOR_ID_NVIDIA, .device = device,
- .driver = bound ? &nvidia_driver : NULL, .refs = 1 };
- pthread_mutex_init(&pdev->lock, NULL);
-}
-
-static void reset(char *list)
-{
- memset(&fake, 0, sizeof(fake));
- device_count = 0;
- targets = list;
- resume = true;
-}
-
-static void boot(void)
-{
- assert(!test_module_init());
- assert(!fake.in_flight && fake.finished == fake.started);
- for (unsigned int i = 0; i < device_count; i++)
- assert(!devices[i].locked && fake.unlocks[i] <= 1);
-}
-
-/* Calls the notifier as the PM core does; returns once it has. */
-static void notify(unsigned long event)
-{
- assert(fake.nb && fake.nb->notifier_call(fake.nb, event, NULL) == NOTIFY_DONE);
- /* nvidia's notifier runs next: no unlock may still be running. */
- assert(!fake.in_flight && fake.finished == fake.started);
-}
-
-/* Unloads the module and checks every reference came back. */
-static void shutdown(void)
-{
- test_module_exit();
- assert(!fake.nb);
- for (unsigned int i = 0; i < device_count; i++)
- assert(devices[i].refs == 1 && !devices[i].locked);
-}
-
-/* Runs init, then checks what holds for every case. */
-static void run(void)
-{
- boot();
- shutdown();
- assert(fake.loads <= 1 && fake.releases <= 1);
-}
-
-static bool logged(const char *text)
-{
- return strstr(fake.log, text);
-}
-
-int main(void)
-{
- /* Every unbound supported card at once; other chips and bound cards left alone. */
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", RTX_3070, false);
- add("0000:31:00.0", RTX_A2000, false);
- add("0000:41:00.0", RTX_3090, true);
- add("0000:61:00.0", 0x2208, false);
- fake.expected_concurrent = 3;
- run();
- assert(fake.max_in_flight == 3 && fake.started == 3);
- assert(fake.unlocks[0] && fake.unlocks[1] && !fake.unlocks[2] && !fake.unlocks[3] && fake.unlocks[4]);
- assert(fake.loads == 1 && fake.load_dev == &devices[0] && fake.releases == 1);
- assert(logged("0000:01:00.0: gpu_unlock: unlocked") && logged("0000:61:00.0: gpu_unlock: unlocked"));
- assert(logged("0000:41:00.0: gpu_unlock: unlock failed (-16)"));
- puts("all: three cards unlocked concurrently, bound card skipped PASS");
-
- /* A list picks cards in any case; the rest are not touched. */
- reset("0000:61:00.0,0000:0A:00.0");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:0a:00.0", RTX_3090, false);
- add("0000:61:00.0", RTX_3090, false);
- fake.expected_concurrent = 2;
- run();
- assert(fake.max_in_flight == 2 && !fake.unlocks[0] && fake.unlocks[1] && fake.unlocks[2]);
- assert(fake.loads == 1 && fake.load_dev == &devices[1] && fake.releases == 1);
- assert(!logged("no supported card"));
- puts("list: only listed cards, concurrently PASS");
-
- /* One card failing leaves the others unlocked. */
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", RTX_3090, false);
- fake.fail_unlock[0] = -EIO;
- fake.expected_concurrent = 2;
- run();
- assert(fake.max_in_flight == 2);
- assert(logged("0000:01:00.0: gpu_unlock: unlock failed (-5)"));
- assert(logged("0000:21:00.0: gpu_unlock: unlocked"));
- puts("failure: other cards unaffected PASS");
-
- /* Without the shared firmware no card is attempted. */
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", RTX_3090, false);
- fake.fail_load = -ENOENT;
- run();
- assert(fake.loads == 1 && !fake.started && fake.releases == 1 && !fake.registrations);
- assert(logged("0000:01:00.0: gpu_unlock: firmware load failed (-2)"));
- puts("firmware load failure: nothing attempted PASS");
-
- /* Unmatched entries, including unsupported chips, are reported; nothing to load for no cards. */
- reset("0000:99:00.0,0000:31:00.0");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:31:00.0", RTX_A2000, false);
- run();
- assert(!fake.loads && !fake.started && fake.releases == 1 && !fake.registrations);
- assert(logged("gpu_unlock: no supported card at \"0000:99:00.0\""));
- assert(logged("gpu_unlock: no supported card at \"0000:31:00.0\""));
- puts("unmatched target: reported, nothing loaded PASS");
-
- /* No opt-in on the command line: the module does nothing. */
- for (int empty = 0; empty < 2; empty++) {
- reset(empty ? "" : NULL);
- add("0000:01:00.0", RTX_3090, false);
- run();
- assert(!fake.loads && !fake.started && !fake.releases && !fake.log[0]);
- assert(!fake.registrations);
- }
- puts("no targets: idle PASS");
-
- /* After resume, the cards unlocked at boot, concurrently, and all of them
- * finished before the notifier returns. A card that failed at boot is not
- * tried again; suspending does nothing. */
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", RTX_3070, false);
- add("0000:41:00.0", RTX_3090, false);
- fake.fail_unlock[2] = -EIO;
- boot();
- assert(fake.registrations == 1 && fake.nb->priority > 0);
- for (unsigned int i = 0; i < device_count; i++)
- devices[i].driver = &nvidia_driver;
- notify(PM_SUSPEND_PREPARE);
- assert(fake.loads == 1 && fake.started == 3);
- fake.expected_concurrent = 2;
- fake.max_in_flight = 0;
- notify(PM_POST_SUSPEND);
- assert(fake.max_in_flight == 2 && fake.started == 5);
- assert(fake.resumes[0] == 1 && fake.resumes[1] == 1 && !fake.resumes[2]);
- assert(fake.loads == 2 && fake.load_dev == &devices[0] && fake.releases == 2);
- assert(logged("0000:01:00.0: gpu_unlock: unlocked again after resume"));
- assert(logged("0000:21:00.0: gpu_unlock: unlocked again after resume"));
- assert(!logged("0000:41:00.0: gpu_unlock: unlocked again"));
- shutdown();
- assert(fake.unlocks[0] == 1 && fake.unlocks[1] == 1 && fake.unlocks[2] == 1);
- puts("resume: boot-unlocked cards unlocked again before nvidia PASS");
-
- /* Only cards nvidia suspended are touched. A card that kept power, or one
- * that fails, stays listed for the next resume. */
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", RTX_3090, false);
- add("0000:31:00.0", RTX_3090, false);
- add("0000:61:00.0", RTX_3090, false);
- boot();
- devices[0].driver = &nvidia_driver;
- devices[1].driver = &nvidia_driver;
- devices[2].driver = &vfio_driver;
- fake.fail_resume[0] = -EALREADY;
- fake.fail_resume[1] = -EBUSY;
- notify(PM_POST_SUSPEND);
- assert(fake.resumes[0] == 1 && fake.resumes[1] == 1 && !fake.resumes[2] && !fake.resumes[3]);
- assert(logged("0000:01:00.0: gpu_unlock: still unlocked after resume"));
- assert(logged("0000:21:00.0: gpu_unlock: unlock after resume failed (-16)"));
- assert(logged("0000:31:00.0: gpu_unlock: not bound to nvidia, left as is after resume"));
- assert(logged("0000:61:00.0: gpu_unlock: not bound to nvidia, left as is after resume"));
- notify(PM_POST_SUSPEND);
- assert(fake.resumes[0] == 2 && fake.resumes[1] == 2 && !fake.resumes[2] && !fake.resumes[3]);
- shutdown();
- puts("resume: only nvidia-suspended cards, kept for the next resume PASS");
-
- /* gpu_unlock.resume=0, and a firmware load failure, leave the cards to nvidia. */
- reset("0000:01:00.0");
- add("0000:01:00.0", RTX_3090, false);
- boot();
- devices[0].driver = &nvidia_driver;
- resume = false;
- notify(PM_POST_SUSPEND);
- assert(fake.loads == 1 && !fake.resumes[0]);
- resume = true;
- fake.fail_load = -ENOENT;
- notify(PM_POST_SUSPEND);
- assert(fake.loads == 2 && fake.releases == 2 && !fake.resumes[0]);
- assert(logged("0000:01:00.0: gpu_unlock: firmware load failed (-2)"));
- fake.fail_load = 0;
- notify(PM_POST_SUSPEND);
- assert(fake.resumes[0] == 1);
- shutdown();
- puts("resume: switch off and firmware failure leave cards locked PASS");
-
- /* Nothing unlocked at boot: no notifier at all. */
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- fake.fail_unlock[0] = -EIO;
- boot();
- assert(!fake.registrations && !fake.nb);
- shutdown();
- puts("resume: no notifier without an unlocked card PASS");
- return 0;
-}
diff --git a/tests/test_packaging.py b/tests/test_packaging.py
deleted file mode 100644
index 4b161c4..0000000
--- a/tests/test_packaging.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-"""Check the generated payload and the shared install layout."""
-import hashlib
-import json
-from pathlib import Path
-import subprocess
-import sys
-import tempfile
-import unittest
-
-ROOT = Path(__file__).resolve().parents[1]
-sys.path.insert(0, str(ROOT / "firmware"))
-import generate_v67 # noqa: E402
-
-
-class PackageTests(unittest.TestCase):
- def test_v67_payload_matches_pin(self):
- digest = hashlib.sha256(generate_v67.build_payload()).hexdigest()
- self.assertEqual(digest, generate_v67.PAYLOAD_SHA256)
- self.assertEqual(digest, self.manifest()["generated_payload"]["sha256"])
-
- def test_install_uses_standard_paths(self):
- version = (ROOT / "VERSION").read_text().strip()
- with tempfile.TemporaryDirectory() as directory:
- subprocess.run(["make", "-s", f"DESTDIR={directory}", "install-dkms",
- "install-mkinitcpio", "install-initramfs-tools",
- "install-dracut"],
- cwd=ROOT, check=True)
- root = Path(directory)
- paths = {path.relative_to(root).as_posix() for path in root.rglob("*")
- if path.is_file()}
- for expected in (f"usr/src/gpu-unlock-{version}/dkms.conf",
- f"usr/src/gpu-unlock-{version}/main.c",
- "etc/mkinitcpio.conf.d/zz-gpu-unlock.conf",
- "usr/share/initramfs-tools/modules.d/gpu-unlock.conf",
- "usr/lib/dracut/dracut.conf.d/gpu-unlock.conf",
- "usr/share/doc/gpu-unlock/README.md"):
- self.assertIn(expected, paths)
- self.assertFalse(any(path.endswith(".mod.c") for path in paths))
- # Another package's "softdep nvidia" line would silently win over
- # ours (kmod honours only the first), so the order comes from the
- # initramfs instead: gpu_unlock goes right before nvidia.
- self.assertFalse(any("modprobe.d" in path for path in paths))
- dropin = root / "etc/mkinitcpio.conf.d/zz-gpu-unlock.conf"
- for before, after in (("zfs nvidia nvidia_drm", "zfs gpu_unlock? nvidia nvidia_drm"),
- ("zfs", "zfs gpu_unlock?")):
- result = subprocess.run(["bash", "-c", f"MODULES=({before}); . {dropin}; echo ${{MODULES[*]}}"],
- capture_output=True, text=True, check=True)
- self.assertEqual(result.stdout.strip(), after)
-
- def test_pkgbuild_pins_match_manifest(self):
- manifest = json.loads((ROOT / "firmware/artifacts-610.43.03.json").read_text())
- pkgbuild = (ROOT / "packaging/arch/PKGBUILD").read_text()
- for index in (0, 2):
- artifact = manifest["source_artifacts"][index]
- self.assertIn(artifact["sha256"], pkgbuild)
- self.assertIn(artifact["url"].rsplit("/", 1)[0], pkgbuild)
- self.assertIn(f"pkgver={(ROOT / 'VERSION').read_text().strip()}\n", pkgbuild)
-
- def test_native_versions_match(self):
- version = (ROOT / "VERSION").read_text().strip()
- spec = (ROOT / "packaging/rpm/gpu-unlock.spec").read_text()
- self.assertIn(f"Version: {version}\n", spec)
- for index in (0, 2):
- self.assertIn(self.manifest()["source_artifacts"][index]["sha256"], spec)
- changelog = (ROOT / "debian/changelog").read_text()
- self.assertTrue(changelog.startswith(f"gpu-unlock ({version}) "))
-
- @staticmethod
- def manifest():
- return json.loads((ROOT / "firmware/artifacts-610.43.03.json").read_text())
diff --git a/tests/test_prep_610.c b/tests/test_prep_610.c
deleted file mode 100644
index 49abe19..0000000
--- a/tests/test_prep_610.c
+++ /dev/null
@@ -1,146 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <string.h>
-
-#include "layout_610.h"
-#include "prep_610.h"
-
-static void put16(unsigned char *p, unsigned int v)
-{
- p[0] = v & 0xff;
- p[1] = (v >> 8) & 0xff;
-}
-
-static void put32(unsigned char *p, unsigned int v)
-{
- p[0] = v & 0xff;
- p[1] = (v >> 8) & 0xff;
- p[2] = (v >> 16) & 0xff;
- p[3] = (v >> 24) & 0xff;
-}
-
-static void put64(unsigned char *p, unsigned long long v)
-{
- unsigned int i;
-
- for (i = 0; i < 8; i++)
- p[i] = (unsigned char)(v >> (8 * i));
-}
-
-int main(void)
-{
- unsigned char elf[256];
- unsigned char image[64];
- unsigned char sigs[8] = { 1, 2, 3, 4, 5, 6, 7, 8 };
- unsigned char meta[GU_WPR_META_SIZE];
- struct gpu_unlock_span span;
- struct gpu_unlock_radix_plan radix;
- struct gpu_unlock_booter_hdr hdr = {
- .num_apps = 1,
- .patch_loc = 16,
- .num_sigs = 2,
- .sig_bytes = 8,
- };
- struct gpu_unlock_layout_in in = {
- .usable_fb = 24ull << 30,
- .vga_base = (24ull << 30) - (1024ull * 1024),
- .frts_size = 1024ull * 1024,
- .boot_size = 4096,
- .image_size = 64ull * 1024,
- .nonwpr_size = 1,
- .heap_request = 1024ull * 1024,
- };
- struct gpu_unlock_layout_out layout;
- struct gpu_unlock_wpr_args args = {
- .fb_size = in.usable_fb, .vga_offset = in.vga_base,
- .frts_size = in.frts_size, .signature_dma = 0x1000,
- .signature_size = 0xfa00, .radix_dma = 0x2000,
- .image_size = in.image_size, .boot_dma = 0x3000,
- .boot = { .code_offset = 256, .data_offset = 512, .image_size = 4096 },
- };
-
- assert(gpu_unlock_fuse_version(0) == 0);
- assert(gpu_unlock_fuse_version(1) == 1);
- assert(gpu_unlock_fuse_version(0x8) == 4);
-
- memset(image, 0, sizeof(image));
- assert(!gpu_unlock_booter_patch(image, sizeof(image), &hdr, sigs,
- sizeof(sigs), 0));
- assert(!memcmp(image + 16, sigs + 4, 4));
- assert(gpu_unlock_booter_patch(image, sizeof(image), &hdr, sigs,
- sizeof(sigs), 2) == -ERANGE);
-
- memset(elf, 0, sizeof(elf));
- elf[0] = 0x7f;
- elf[1] = 'E';
- elf[2] = 'L';
- elf[3] = 'F';
- elf[4] = 2;
- elf[5] = 1;
- elf[6] = 1;
- put32(elf + 20, 1);
- put16(elf + 52, 64);
- put64(elf + 40, 64);
- put16(elf + 58, 64);
- put16(elf + 60, 2);
- put16(elf + 62, 1);
- put32(elf + 64, 0);
- put64(elf + 64 + 24, 220);
- put64(elf + 64 + 32, 4);
- put32(elf + 128, 9);
- put64(elf + 128 + 24, 192);
- put64(elf + 128 + 32, 16);
- memcpy(elf + 192, ".fwimage", 8);
- elf[200] = 0;
- memcpy(elf + 220, "name", 4);
- assert(!gpu_unlock_elf_section(elf, sizeof(elf), ".fwimage", &span));
- assert(span.offset == 220 && span.size == 4);
- assert(gpu_unlock_elf_section(elf, sizeof(elf), ".missing", &span) == -ENOENT);
-
- assert(!gpu_unlock_radix_plan(4096, &radix));
- assert(radix.pages[0] == 1 && radix.pages[3] == 1);
- assert(!gpu_unlock_radix_plan(4097, &radix));
- assert(radix.pages[3] == 2);
- assert(gpu_unlock_radix_plan(~(u64)0, &radix) == -EOVERFLOW);
- /* A name that ends at the string-table boundary must include its NUL. */
- put64(elf + 128 + 32, 8);
- assert(gpu_unlock_elf_section(elf, sizeof(elf), ".fwimage", &span) == -EINVAL);
- put64(elf + 128 + 32, 16);
- put32(elf + 128, 0);
- assert(gpu_unlock_elf_section(elf, sizeof(elf), ".fwimage", &span) == -EINVAL);
- assert(gpu_unlock_dma_cmd(true) == ((6u << 8) | (1u << 4) | (1u << 2)));
- assert(gpu_unlock_dma_cmd(false) == (6u << 8));
-
- assert(!gpu_unlock_layout(&in, &layout));
- assert(!gpu_unlock_wpr_meta_pack(meta, &layout, &args));
- assert(meta[0] == 0xb3 && meta[7] == 0xdc);
- assert(meta[72] == 0x00 && meta[73] == 0x10);
- assert(meta[49] == 1 && meta[57] == 2);
- assert(!memcmp(meta + 184, (unsigned char[]){0,0,0xf0,0xff,5,0,0,0}, 8));
- args.image_size = ~0ull;
- assert(gpu_unlock_wpr_meta_pack(meta, &layout, &args) == -EINVAL);
- {
- unsigned char desc[84] = {0}, page[GU_RADIX_PAGE];
- u64 addresses[] = {0x123456000ull, 0xdeadbe000ull};
- struct gpu_unlock_boot_desc boot;
- const u32 words[] = {5,20480,2176,22656,16,0,0,0,0,2048,2048,4096,6144,10496,1,0,0,0,0,24576,0};
-
- for (unsigned int i=0; i<21; i++) put32(desc+i*4,words[i]);
- assert(!gpu_unlock_boot_desc_parse(desc, sizeof(desc), 24576, &boot));
- assert(boot.code_offset == 6144 && boot.data_offset == 2048 && !boot.manifest_offset);
- assert(gpu_unlock_boot_desc_parse(desc, 83, 24576, &boot) == -EINVAL);
- put32(desc+48, ~0u);
- assert(gpu_unlock_boot_desc_parse(desc, 84, 24576, &boot) == -ERANGE);
- assert(!gpu_unlock_radix_page(page, addresses, 2));
- assert(page[3] == 0x23 && page[4] == 1 && page[12] == 0xd);
- for (unsigned int i=16; i<sizeof(page); i++) assert(page[i] == 0);
- addresses[0]++;
- assert(gpu_unlock_radix_page(page, addresses, 2) == -EINVAL);
- assert(gpu_unlock_radix_page(page, addresses, 513) == -EINVAL);
- }
-
- puts("prep checks passed");
- return 0;
-}
diff --git a/tests/test_release.py b/tests/test_release.py
deleted file mode 100644
index f5586a7..0000000
--- a/tests/test_release.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0-only
-"""Host checks for the retained module components."""
-from pathlib import Path
-import os
-import subprocess
-import tempfile
-import unittest
-
-
-ROOT = Path(__file__).resolve().parents[1]
-KERNEL = ROOT / "kernel/gpu_unlock"
-# Generated or installed firmware lets the adapter run the whole unlock
-# against mocked hardware.
-ASSETS = next((path for path in (Path(os.environ.get("GPU_UNLOCK_ASSETS", "")),
- Path("/usr/lib/firmware/gpu-unlock/610.43.03"))
- if (path / "gsp_ga10x.bin").is_file()), None)
-
-CASES = {
- "gfw_eval": ["gfw_eval.c"],
- "layout_610": ["layout_610.c"],
- "prep_610": ["layout_610.c", "prep_610.c"],
- "vbios_610": ["vbios_610.c"],
- "unlock_adapter": ["gfw_eval.c", "prep_610.c", "layout_610.c", "vbios_610.c"],
- "main": [],
-}
-# Tests that include a kernel source directly build it against a shim.
-SHIMS = {
- "unlock_adapter": ["-I", str(ROOT / "tests/unlock_shim")],
- # The kernel build does not warn on sign-compare; match it for main.c.
- "main": ["-I", str(ROOT / "tests/main_shim"), "-pthread", "-Wno-sign-compare"],
-}
-
-
-class ComponentTests(unittest.TestCase):
- def test_components(self):
- with tempfile.TemporaryDirectory() as directory:
- for name, sources in CASES.items():
- with self.subTest(name=name):
- output = Path(directory) / name
- shim = SHIMS.get(name, [])
- command = [os.environ.get("CC", "cc"), "-std=c11", "-Wall",
- "-Wextra", "-Werror", "-Wno-unused-parameter",
- "-I", str(KERNEL), *shim,
- *(str(KERNEL / source) for source in sources),
- str(ROOT / "tests" / f"test_{name}.c"),
- "-o", str(output)]
- subprocess.run(command, check=True)
- args = [str(output)]
- if name == "unlock_adapter" and ASSETS:
- args += ["-", str(ASSETS)]
- result = subprocess.run(args, capture_output=True, text=True)
- self.assertEqual(result.returncode, 0,
- (result.stdout + result.stderr)[-4000:])
diff --git a/tests/test_unlock_adapter.c b/tests/test_unlock_adapter.c
deleted file mode 100644
index 778b251..0000000
--- a/tests/test_unlock_adapter.c
+++ /dev/null
@@ -1,719 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <assert.h>
-#include <stdarg.h>
-#include <sys/stat.h>
-#define __KERNEL__
-#include "../kernel/gpu_unlock/unlock_610.c"
-#include "vbios_fixture.h"
-static u32 regs[0x900000/4];
-static const char *asset_root;
-static struct unlock_transaction_work *last_work;
-static bool full_active;
-static void check_published_image(void);
-static void check_restored_signature(void);
-static struct {
- u16 cmd,saved;
- unsigned long now;
- int allocs,live,frees,fail_alloc,fail_mask,flrs,fail_reset,starts,boot_mode,bar_bad,barriers,writes;
- int gsp_starts,fwsec_mode,master_fail,master_stuck,selector_writes,ss1_writes,fw_live;
- int pending,lost_identity,corrupt_selector,alien_wpr;
- int fw_requests,fail_fw,fail_claim,fail_map,fail_save,fail_context,pins,selector_bad;
- int restore_mask_fail,late_devres;
- int group_requests,fail_group,group_live,group_before,cache_records,cache_released;
- int switch_stuck,dma_commands,dma_stall_after;
- int reset_delay,reset_reads[2],reset_pending[2],reset_post_fault;
- int reset_locked_until_flr,hold_writes,bar_ready_after,bar_ready_reads,wpr_after;
- int post_gfw,post_gfw_reads,gsp_post_reset_writes;
- int fwsec_wpr;u32 fwsec_wpr_lo,fwsec_wpr_hi; /* WPR2 that FWSEC-FRTS enables */
- int apertures,regions;
- int bound,gfw_late,gfw_late_reads; /* resume: nvidia bound; GFW done after N reads */
- int busmaster; /* the PCI core's is_busmaster */
- char log[32768];
-} fake;
-void *kzalloc(size_t n,int flags) {
- if(fake.fail_context)return NULL;
- last_work=calloc(1,n);return last_work;
-}
-void test_device_log(const char *format, ...) {
- va_list args;
- size_t at=strlen(fake.log);
- va_start(args,format);
- vsnprintf(fake.log+at,sizeof(fake.log)-at,format,args);
- va_end(args);
-}
-static u64 mask;
-static struct pci_dev dev;
-static u8 fw_image[4097];
-static struct firmware fw={sizeof(fw_image),fw_image};
-static void setup(struct unlock_transaction_work *w) {
- memset(&fake,0,sizeof(fake)); memset(regs,0,sizeof(regs)); memset(w,0,sizeof(*w));
- mask=0xffffffffffffull; dev=(struct pci_dev){.dev={.dma_mask=&mask,.coherent_dma_mask=0xfffffffffull},.vendor=0x10de,.device=0x2204,.refs=1};
- w->pdev=&dev; w->bar=regs; w->bar_len=sizeof(regs); w->mapped=true;
- fake.cmd=2;fake.saved=2;regs[(SEC2_BASE+0xf4)/4]=1u<<31;regs[(0x110000+0xf4)/4]=1u<<31;
- regs[(SEC2_BASE+0x118)/4]=2;regs[(0x110000+0x118)/4]=2;
- regs[GU_FEAT_SS0_OFFSET/4]=0x88888888;regs[GU_FEAT_SS1_OFFSET/4]=0;
- regs[0x1fa824/4]=0xfffffff0;regs[0x1fa828/4]=0;
- regs[0]=0x172000a1;
-}
-void *dma_alloc_coherent(struct device *d,size_t n,dma_addr_t *a,int flags) {
- fake.allocs++; if(fake.fail_alloc==fake.allocs)return NULL;
- void *p=calloc(1,n); assert(p);
- *a=(d->coherent_dma_mask<=0xffffffffull?0x100000ull:0x100000000ull)+fake.allocs*0x10000ull;
- assert(*a+n-1<=d->coherent_dma_mask);fake.live++;return p;
-}
-void dma_free_coherent(struct device *d,size_t n,void *p,dma_addr_t a) {
- assert(p && a>=0x100000ull);free(p);fake.live--;fake.frees++;
-}
-int dma_set_mask(struct device *d,u64 v) {
- *d->dma_mask=v;return fake.fail_mask==1 || (fake.restore_mask_fail && v>0xffffffffull)?-EIO:0;
-}
-int dma_set_coherent_mask(struct device *d,u64 v) { d->coherent_dma_mask=v;return fake.fail_mask==2?-EIO:0; }
-void dma_wmb(void) { fake.barriers++; }
-void usleep_range(unsigned long a,unsigned long b) {fake.now+=a;}
-ktime_t ktime_get(void) {return fake.now;}
-u32 readl(const void *p) {
- size_t n=((const u8 *)p-(const u8 *)regs)/4;assert(n<ARRAY_SIZE(regs));
- /* Power-on GFW still running: the FB size appears only once it is done. */
- if(!fake.flrs && n==GU_GFW_BOOT_OFFSET/4 && fake.gfw_late && ++fake.gfw_late_reads==fake.gfw_late) {
- regs[n]=0xff;regs[0x1183a4/4]=0x6000;
- }
- if(fake.flrs && !n && fake.bar_ready_after) {
- fake.bar_ready_reads++;
- if(fake.bar_ready_after<0 || fake.bar_ready_reads<fake.bar_ready_after)
- return fake.bar_ready_after==-2?0x12345678:0xbadf0000;
- }
- if(fake.flrs && n==GU_GFW_CPUCTL_OFFSET/4 && fake.post_gfw==1 && ++fake.post_gfw_reads==5) {
- regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
- }
- int eng=n==(SEC2_BASE+0x3c0)/4?0:n==0x1103c0/4?1:-1;
- if(eng>=0 && fake.reset_pending[eng] && ++fake.reset_reads[eng]>=fake.reset_delay) {
- regs[n]=1;fake.reset_pending[eng]=0;
- }
- return regs[n];
-}
-void writel(u32 v,void *p) {
- fake.writes++;
- size_t off=(u8 *)p-(u8 *)regs;assert(off<sizeof(regs)); regs[off/4]=v;
- if(fake.flrs && off==0x1103c0) {
- fake.gsp_post_reset_writes++;
- regs[GU_GFW_CPUCTL_OFFSET/4]=0; /* Holding GSP reset breaks RM's entry wait. */
- }
- if(off==GU_FEAT_SS0_OFFSET)fake.selector_writes++;
- if(off==GU_FEAT_SS0_OFFSET && fake.selector_bad==1)regs[off/4]=0;
- if(off==GU_FEAT_SS1_OFFSET){fake.ss1_writes++;regs[off/4]=v&0xf;} /* 4-bit register, as on the card */
- if(off==GU_FEAT_SS1_OFFSET && fake.selector_bad==2)regs[off/4]=0;
- if(off==SEC2_BASE+0x118 || off==0x110118) {
- fake.dma_commands++;
- regs[off/4]=fake.dma_stall_after && fake.dma_commands>=fake.dma_stall_after?1:2;
- }
- if(off==SEC2_FALCON2+0x668 || off==0x111668)regs[off/4]=fake.switch_stuck?0:1;
- if(off==0x110100 || off==0x110130) {
- fake.gsp_starts++;
- if(fake.fwsec_wpr){regs[0x1fa824/4]=fake.fwsec_wpr_lo;regs[0x1fa828/4]=fake.fwsec_wpr_hi;}
- regs[0x110100/4]=fake.fwsec_mode==1?2:0x10;
- }
- if(off==SEC2_BASE+0x100 || off==SEC2_BASE+0x130) {
- if(full_active)check_published_image();
- fake.starts++;
- if(fake.boot_mode==0)regs[GU_FEAT_PLM_OFFSET/4]=~0u;
- if(fake.boot_mode==2)regs[(SEC2_BASE+0x100)/4]=~0u;
- if(fake.boot_mode==3)regs[(SEC2_BASE+0x100)/4]=0x10;
- }
- if(fake.fail_reset==2 && (off==SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0;
- if(fake.fail_reset==8 && (off==SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0xbadf0001;
- if((off==SEC2_BASE+0x3c0 || off==0x1103c0) && (v&1) && !(fake.cmd&4)) {
- fake.hold_writes++;
- int eng=off==SEC2_BASE+0x3c0?0:1;
- if(fake.reset_delay) {
- regs[off/4]=0;fake.reset_reads[eng]=0;fake.reset_pending[eng]=1;
- }
- if(fake.flrs && fake.reset_post_fault) {
- regs[off/4]=fake.reset_post_fault==1?0:0xbadf0001;
- fake.reset_pending[eng]=0;
- }
- if(fake.reset_locked_until_flr && !fake.flrs) {
- regs[off/4]=0;fake.reset_pending[eng]=0;
- }
- if(fake.flrs && fake.post_gfw==4)regs[GU_GFW_CPUCTL_OFFSET/4]=0;
- }
-}
-void *devres_open_group(struct device *d,void *id,int flags) {
- if(++fake.group_requests==fake.fail_group)return NULL;
- assert(!fake.group_live && !id);
- fake.group_live=1;fake.group_before=d->devres_head.count;
- d->devres_head.count++; /* open group marker */
- return &fake.group_live;
-}
-int devres_release_group(struct device *d,void *id) {
- assert(fake.group_live && id==&fake.group_live);
- int released=d->devres_head.count-fake.group_before-1;
- fake.cache_released+=released;d->devres_head.count=fake.group_before;
- fake.group_live=0;return released;
-}
-int request_firmware(const struct firmware **f,const char *n,struct device *d) {
- if(++fake.fw_requests==fake.fail_fw)return -ENOENT;
- if(!asset_root)return -ENOENT;
- char path[4096];
- int length=snprintf(path,sizeof(path),"%s/%s",asset_root,strrchr(n,'/')+1);
- assert(length>0 && (size_t)length<sizeof(path));
- FILE *file=fopen(path,"rb");assert(file);
- assert(!fseek(file,0,SEEK_END));long size=ftell(file);assert(size>0);assert(!fseek(file,0,SEEK_SET));
- struct firmware *loaded=malloc(sizeof(*loaded));u8 *data=malloc(size);assert(loaded && data);
- assert(fread(data,1,size,file)==(size_t)size);fclose(file);
- *loaded=(struct firmware){size,data};*f=loaded;fake.fw_live++;
- /* Linux CONFIG_FW_CACHE: successful request adds a device resource;
- * release_firmware frees the image, not this suspend-cache record. */
- d->devres_head.count++;fake.cache_records++;return 0;
-}
-void release_firmware(const struct firmware *f) {if(f){free((void *)f->data);free((void *)f);fake.fw_live--;}}
-void __module_get(void *p) {fake.pins++;}
-int pci_read_config_word(struct pci_dev *p,int r,u16 *v) {*v=r==PCI_COMMAND?fake.cmd:r==PCI_VENDOR_ID?0x10de:0;return 0;}
-int pci_read_config_dword(struct pci_dev *p,int r,u32 *v) {
- *v=r==PCI_VENDOR_ID?(fake.flrs && fake.lost_identity?~0u:0x220410de):(u32)fake.bar_bad;return 0;
-}
-int pci_write_config_word(struct pci_dev *p,int r,u16 v) {assert(r==PCI_COMMAND);fake.cmd=v|(fake.master_stuck?4:0);return 0;}
-int pcie_capability_read_word(struct pci_dev *p,int r,u16 *v) {*v=r==PCI_EXP_DEVSTA?fake.pending:0;return 0;}
-int pcie_capability_read_dword(struct pci_dev *p,int r,u32 *v) {*v=PCI_EXP_DEVCAP_FLR;return 0;}
-int pci_is_pcie(struct pci_dev *p) {return 1;}
-int pci_select_bars(struct pci_dev *p,int r) {return 7;}
-int pci_request_region(struct pci_dev *p,int bar,const char *n) {
- assert(!bar); /* only BAR0: the display card's framebuffer keeps BAR1 */
- if(fake.fail_claim || fake.bound)return -EBUSY; /* nvidia holds it after resume */
- fake.regions++;return 0;
-}
-void pci_release_region(struct pci_dev *p,int bar) {assert(!bar && fake.regions==1);fake.regions--;}
-int aperture_remove_conflicting_pci_devices(struct pci_dev *p,const char *n) {
- /* The framebuffer stays through FWSEC and Booter; it goes right before FLR.
- * After resume nvidia-drm already replaced it. */
- assert(!fake.flrs && !(fake.cmd&4) && !fake.bound);
- if(full_active)assert(fake.starts==1);
- fake.apertures++;return 0;
-}
-int pci_enable_device_mem(struct pci_dev *p) {fake.cmd|=2;return 0;}
-void pci_disable_device(struct pci_dev *p) {}
-resource_size_t pci_resource_len(struct pci_dev *p,int r) {return sizeof(regs);}
-void *pci_iomap(struct pci_dev *p,int r,unsigned long n) {return fake.fail_map?NULL:regs;}
-void pci_iounmap(struct pci_dev *p,void *v) {}
-void *pci_map_rom(struct pci_dev *p,size_t *n) {return NULL;}
-void pci_unmap_rom(struct pci_dev *p,void *v) {}
-int pci_save_state(struct pci_dev *p) {fake.saved=fake.cmd;return fake.fail_save?-EIO:0;}
-void pci_restore_state(struct pci_dev *p) {assert(!(fake.saved&4));fake.cmd=fake.saved;}
-void pci_set_master(struct pci_dev *p) {if(!fake.master_fail){fake.cmd|=4;fake.busmaster=1;}}
-void pci_clear_master(struct pci_dev *p) {fake.cmd&=~4;fake.busmaster=0;}
-int pcie_flr(struct pci_dev *p) {
- if(full_active)check_restored_signature();
- if(full_active)assert(fake.apertures==!fake.bound);
- fake.flrs++;
- if(fake.late_devres)p->dev.devres_head.count++;
- /* FLR releases engine reset; post-FLR assertion must work independently. */
- regs[(SEC2_BASE+0x3c0)/4]=regs[0x1103c0/4]=0;
- regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;
- regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
- if(fake.post_gfw==1 || fake.post_gfw==2) {
- regs[GU_GFW_CPUCTL_OFFSET/4]=0;regs[GU_GFW_BOOT_OFFSET/4]=0;
- }
- if(fake.post_gfw==3)regs[GU_GFW_PLM_OFFSET/4]=0xbadf0001;
- if(fake.wpr_after!=4)regs[0x1fa824/4]=regs[0x1fa828/4]=0;
- if(fake.corrupt_selector==1)regs[GU_FEAT_SS0_OFFSET/4]=0;
- if(fake.corrupt_selector==2)regs[GU_FEAT_SS1_OFFSET/4]=0;
- if(fake.alien_wpr){regs[0x1fa824/4]=0x1000;regs[0x1fa828/4]=0x2000;}
- if(fake.wpr_after==1)regs[0x1fa824/4]=regs[0x1fa828/4]=0;
- if(fake.wpr_after==2){regs[0x1fa824/4]=0x2000;regs[0x1fa828/4]=0x1000;}
- if(fake.wpr_after==3)regs[0x1fa824/4]=0xbadf0000;
- return fake.fail_reset==1?-EIO:0;
-}
-struct pci_dev *pci_dev_get(struct pci_dev *p) {p->refs++;return p;}
-void pci_dev_put(struct pci_dev *p) {p->refs--;}
-static u64 get64(const void *p) {const u8 *s=p;u64 v=0;for(int i=7;i>=0;i--)v=(v<<8)|s[i];return v;}
-static const u8 *dma_page(u64 address) {
- struct unlock_transaction_work *w=last_work;
- assert(address>=w->pages[0].dma && !((address-w->pages[0].dma)%0x10000));
- u64 index=(address-w->pages[0].dma)/0x10000;
- assert(index<w->pages_allocated && w->pages[index].dma==address);
- return w->pages[index].cpu;
-}
-static void check_published_image(void) {
- struct unlock_transaction_work *w=last_work;const u8 *meta=w->meta_cpu;
- assert(w->published && !fake.pins && (fake.cmd&4));
- assert(get64(meta)==GU_WPR_META_MAGIC && get64(meta+8)==1);
- assert(get64(meta+72)==w->v67_dma && get64(meta+80)==GU_V67_SIZE);
- assert(!memcmp(w->v67_cpu,w->v67_fw->data,GU_V67_SIZE));
- assert(get64(meta+112)==0x5f5400000ull && get64(meta+136)==0x5fada0000ull);
- assert(get64(meta+144)==0x5ffdfa000ull && get64(meta+152)==0x5ffe00000ull);
- assert(get64(meta+168)==0x5fff00000ull && get64(meta+176)==0x600000000ull);
- /* Independently walk the radix the way a firmware consumer does. Compare
- * every published image byte, including transitions across 512-entry tables. */
- u64 image_bytes=get64(meta+24);assert(image_bytes==84226048);
- for(u64 page=0,off=0;off<image_bytes;page++,off+=4096) {
- const u8 *level0=dma_page(get64(meta+16));
- const u8 *level1=dma_page(get64(level0+(page>>18)*8));
- const u8 *level2=dma_page(get64(level1+((page>>9)&511)*8));
- const u8 *leaf=dma_page(get64(level2+(page&511)*8));
- size_t bytes=image_bytes-off<4096?image_bytes-off:4096;
- assert(!memcmp(leaf,w->gsp_fw->data+w->image_span.offset+off,bytes));
- for(size_t i=bytes;i<4096;i++)assert(!leaf[i]);
- }
- assert(!memcmp(w->boot_cpu,w->boot_fw->data,w->boot_desc.image_size));
- /* SEC2 fuse v1 selects slot 0 of the two-signature Booter asset. */
- assert(!memcmp((u8 *)w->booter_cpu+35344,w->booter_fw->data+72+60416,384));
-}
-static void check_restored_signature(void) {
- struct unlock_transaction_work *w=last_work;const u8 *meta=w->meta_cpu;
- assert(w->signature_calls==2 && get64(meta+72)==w->stock_dma && get64(meta+80)==4096);
- assert(!memcmp(w->stock_cpu,w->gsp_fw->data+w->sig_span.offset,4096));
- assert(!(fake.cmd&4));
- /* All published buffers remain alive; FLR must precede engine holds. */
- assert(fake.live==fake.allocs && !fake.frees && !fake.hold_writes);
-}
-static void synthetic_rom(u8 *rom) {
- vbios_fixture(rom,6);
- rom[292]=0;rom[293]=4;rom[294]=9; /* GSP engine and FWSEC ucode */
- u8 *dmem=rom+684+256;
- dmem[0]=1;dmem[1]=4;dmem[2]=8;dmem[3]=2;
- vbios_w32(dmem+4,4);vbios_w32(dmem+8,32);
- dmem[36]=3;dmem[38]=64;
- vbios_w32(dmem+40,128);vbios_w32(dmem+44,44);
-}
-static void load_rom(const char *path) {
- u8 *rom=(u8 *)regs+0x300000;
- if(!path || !strcmp(path,"-")) { synthetic_rom(rom); return; }
- FILE *prom=fopen(path,"rb");
- assert(prom && fread(rom,1,1024*1024,prom)==1024*1024);
- assert(fgetc(prom)==EOF);fclose(prom);
-}
-int main(int argc, char **argv) {
- struct unlock_transaction_work w;
- setup(&w);
- regs[0x110118/4]=0x1230;
- assert(poll_us(&w,0x110118,2,2,100,true)==-ETIMEDOUT);
- assert(strstr(fake.log,"poll timeout: reg=0x110118 last=0x1230 mask=0x2 expected=0x2 budget_us=100"));
- assert(strstr(fake.log,"poll_us error=-110"));
- assert(!fake.writes);
- setup(&w);
- regs[0x110118/4]=~0u;
- assert(poll_us(&w,0x110118,2,2,100,true)==-EIO);
- assert(strstr(fake.log,"poll inaccessible: reg=0x110118 last=0xffffffff"));
- /* Exercise the dual-core reset path, including RISC-V -> Falcon switching
- * and the documented tolerated pre-reset timeout. */
- for(int mode=0;mode<8;mode++) {
- setup(&w);
- const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
- regs[(SEC2_BASE+0xf4)/4]|=1u<<10;
- regs[(SEC2_FALCON2+0x668)/4]=mode==0?1:0x10;
- if(mode==2)regs[(SEC2_FALCON2+0x668)/4]=0xbadf0001;
- if(mode==3)fake.switch_stuck=1;
- if(mode==4)regs[(SEC2_BASE+0xf4)/4]|=1u<<12;
- if(mode==5)regs[(SEC2_BASE+0xf4)/4]&=~(1u<<31);
- if(mode==6)regs[0]=~0u;
- if(mode==7)regs[(SEC2_BASE+0x3c0)/4]=0xbadf0001;
- int ret=falcon_reset(&w,&eng);
- assert((!ret)==(mode==0 || mode==1 || mode==5));
- if(mode==1)assert(regs[(SEC2_FALCON2+0x668)/4]==1);
- if(mode==5){
- assert(strstr(fake.log,"pre-reset timeout tolerated"));
- assert(strstr(fake.log,"poll timeout tolerated:"));
- assert(!strstr(fake.log,"poll timeout:"));
- assert(!strstr(fake.log,"poll_us error=-110"));
- assert(!strstr(fake.log,"failure unlock_610.c"));
- }
- assert(!fake.gsp_starts && !fake.starts && !fake.dma_commands);
- }
- for(int mode=0;mode<3;mode++) {
- setup(&w);
- if(mode==0)regs[(SEC2_BASE+0x118)/4]=1;
- else fake.dma_stall_after=1;
- assert(falcon_dma(&w,SEC2_BASE,0,0,0x100000,mode==2?256:512,0x614)==-ETIMEDOUT);
- assert(fake.dma_commands==(mode?1:0) && fake.now<=1000100);
- if(mode==1)assert(strstr(fake.log,"Falcon DMA progress: done=256 total=512"));
- }
- for(int mode=0;mode<4;mode++) {
- setup(&w);
- const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
- w.plm_entry=0x8f;
- if(mode==0)regs[(SEC2_FBIF+0x24)/4]=0xbadf0001;
- if(mode==1)regs[SEC2_FBIF/4]=~0u;
- if(mode==2)regs[(SEC2_BASE+0x100)/4]=~0u;
- if(mode==3)regs[(SEC2_BASE+0x100)/4]=1u<<6; /* CPUCTL alias path */
- int ret=falcon_hs(&w,&eng,0,256,0,0,256,256,0,0,1,3,0x100000,0,0,true);
- assert((!ret)==(mode==3));
- assert(fake.starts==(mode==3?1:0));
- if(mode<2)assert(!fake.dma_commands);
- }
- /* Reject DMA address truncation/wrap before any reset or transfer. */
- for(int bad=0;bad<5;bad++) {
- setup(&w);
- u32 dest=bad==0?0xffff00:0, off=bad==1?0xffffff00:0;
- dma_addr_t src=bad==2?(1ull<<49)-256:0x100000;
- u32 size=bad==3?257:512;
- if(bad==4)src++;
- assert(falcon_dma(&w,SEC2_BASE,dest,off,src,size,0x614)==-EINVAL);
- assert(!fake.writes);
- }
- {
- const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
- setup(&w);
- assert(falcon_hs(&w,&eng,0,256,0x200000,0,256,256,0,0,1,3,
- 0x100000,0,0,false)==-ERANGE);
- assert(!fake.writes);
- /* Bad DMEM geometry must not leave an already-submitted IMEM DMA. */
- assert(falcon_hs(&w,&eng,0,256,0,0,256,257,0,0,1,3,
- 0x100000,0,0,false)==-EINVAL);
- assert(!fake.writes);
- }
- /* Actual PROM -> descriptor -> patch -> GSP DMA/start -> WPR validation.
- * Faults after firmware access retain DMA, without starting SEC2. */
- for(int mode=0;mode<10;mode++) {
- setup(&w);
- u8 *rom=(u8 *)regs+0x300000;
- load_rom(argc>=2?argv[1]:NULL);
- struct gpu_unlock_fwsec parsed;
- assert(!gpu_unlock_vbios_fwsec(rom,1024*1024,&parsed));
- w.layout.frts_offset=0x5ffe00000ull;w.layout.wpr_end=0x5fff00000ull;
- regs[0x1fa824/4]=(u32)(w.layout.frts_offset>>12)<<4;
- regs[0x1fa828/4]=(u32)((w.layout.wpr_end-4096)>>12)<<4;
- fake.fwsec_mode=mode;
- if(mode==2)regs[(0x1400+0x0e*4)/4]=1u<<16;
- if(mode==3)regs[0x1fa828/4]=0; /* disabled, low alone matched */
- if(mode==4)regs[0x1fa828/4]+=16; /* outside reservation */
- if(mode==5)regs[0x1fa824/4]-=16;
- if(mode==6)regs[(0x8241c0+8*4)/4]=0xffff; /* unavailable fuse version */
- if(mode==7)fake.fail_alloc=1;
- if(mode==8)fake.master_fail=1;
- if(mode==9)rom[parsed.sig_off-44+37]=1; /* wrong engine mask */
- int ret=unlock_transaction_fwsec(&w);
- assert(!fake.starts && !fake.flrs);
- if(mode==0) {
- assert(!ret && fake.gsp_starts==1);
- assert(strstr(fake.log,"FWSEC WPR2:"));
- } else {
- assert(ret);
- if(mode>=6)assert(!fake.gsp_starts && !w.firmware_started);
- else assert(fake.gsp_starts==1 && w.firmware_started);
- }
- if(w.firmware_started) {
- assert(unlock_transaction_unwind(&w)==-EBUSY && w.retained && fake.live==1);
- free_coherent(&w); /* harness disposal only */
- } else assert(!unlock_transaction_unwind(&w) && !fake.live);
- }
- /* Exercise the register-to-layout adapter with the pinned image geometry.
- * A range error is pre-DMA/pre-firmware; never "fix" it by skipping bounds. */
- for(int scenario=0;scenario<8;scenario++) {
- setup(&w);
- regs[0x1183a4/4]=24*1024;
- regs[0x1fa7c8/4]=1;
- regs[0x1fa82c/4]=0xfffffff0; /* disabled MMU lock: hi < lo */
- regs[0x1fa830/4]=0;
- w.image_span.size=0x5053000;
- w.boot_desc.image_size=24576;
- if(scenario==1)regs[0x625f04/4]=(u32)(((24ull<<30)-(1ull<<20))>>16)<<8 | 8;
- if(scenario==2)regs[0x625f04/4]=8; /* valid old low workspace -> VBIOS fallback */
- if(scenario==3 || scenario==4) {
- u64 low=(24ull<<30)-((scenario==3?2ull:200ull)<<20);
- regs[0x1fa82c/4]=(u32)(low>>12)<<4;
- regs[0x1fa830/4]=(u32)(((24ull<<30)-4096)>>12)<<4;
- }
- if(scenario==5)regs[0x1183a4/4]=128; /* insufficient usable FB */
- if(scenario==6)regs[0x625f04/4]=(u32)((24ull<<30)>>16)<<8 | 8; /* beyond FB */
- /* Captured cold GA102 lock: lo=256 MiB, hi exclusive is not FB top. */
- if(scenario==7) {
- regs[0x1183a4/4]=0x6000; regs[0x625f04/4]=1; regs[0x1fa7c8/4]=0x45cb8f;
- regs[0x1fa82c/4]=0x100000; regs[0x1fa830/4]=0x5cffff0;
- w.image_span.size=84226048;
- }
- int ret=unlock_transaction_layout(&w);
- assert(ret==(scenario>=4 && scenario<=6?-ERANGE:0));
- assert(strstr(fake.log,"layout inputs: fb_mb="));
- if(scenario==4) assert(strstr(fake.log,"reservations below pre-scrubbed"));
- if(scenario==5) assert(strstr(fake.log,"usable FB below 256 MiB"));
- if(scenario==6) assert(strstr(fake.log,"VGA base at or above usable FB"));
- if(scenario==7) assert(strstr(fake.log,"mmu lock ignored: high address is not the FB top"));
- if(scenario<4 || scenario==7) {
- assert(w.layout.nonwpr_start>=w.layout_in.usable_fb-(256ull<<20));
- assert(w.layout.wpr_end<=w.layout_in.vga_base);
- assert(w.layout.heap_size>=88ull<<20);
- }
- assert(!fake.writes && !fake.allocs && !fake.starts && !fake.flrs);
- }
- /* Cold entry: completed GFW, closed PLM and WPR2 down. A used card is refused. */
- for(int mode=0;mode<3;mode++) {
- setup(&w);
- regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;
- regs[GU_FEAT_PLM_OFFSET/4]=mode==1?~0u:0x8f;
- if(mode==2)regs[0x1fa828/4]=0x5ffef0;
- int ret=unlock_transaction_gfw(&w);
- assert(ret==(mode==0?0:mode==1?-EALREADY:-EBUSY));
- assert(!fake.writes && !fake.gsp_starts);
- if(mode==2)assert(strstr(fake.log,"card already initialised since power on"));
- }
- setup(&w); dev.dev_flags=PCI_DEV_FLAGS_NO_FLR_RESET;
- assert(unlock_transaction_claim(&w)==-EOPNOTSUPP && !fake.flrs && !fake.allocs);
- /* A setter may mutate a mask before returning failure; restore both. */
- for(int fail=1;fail<=2;fail++) {
- setup(&w); fake.fail_mask=fail;
- assert(unlock_transaction_dma(&w)==-EIO);
- fake.fail_mask=0;
- assert(!unlock_transaction_unwind(&w));
- assert(mask==0xffffffffffffull && dev.dev.coherent_dma_mask==0xfffffffffull);
- }
- /* Stock restoration must preserve layout and must not republish V67. */
- {
- setup(&w);
- struct unlock_transaction_page page={.dma=0x200000};
- u8 metadata[GU_WPR_META_SIZE], stock[4096], before[GU_WPR_META_SIZE];
- w.meta_cpu=metadata;w.stock_cpu=stock;w.stock_dma=0x400000;w.stock_size=4096;
- w.v67_dma=0x300000;w.boot_dma=0x500000;w.pages=&page;w.pages_allocated=1;
- w.layout_in=(struct gpu_unlock_layout_in){.usable_fb=24ull<<30,
- .vga_base=(24ull<<30)-(1ull<<20),.frts_size=1ull<<20,
- .image_size=4097,.boot_size=24576,.nonwpr_size=1ull<<20,.heap_request=88ull<<20};
- assert(!gpu_unlock_layout(&w.layout_in,&w.layout));w.image_span.size=4097;
- w.boot_desc=(struct gpu_unlock_boot_desc){.code_offset=6144,.data_offset=2048,.image_size=24576};
- assert(!unlock_transaction_signature(&w) && w.published && w.signature_calls==1);
- assert(get64(metadata+72)==w.v67_dma && get64(metadata+80)==GU_V67_SIZE);
- memcpy(before,metadata,sizeof(before));
- assert(!unlock_transaction_signature(&w) && w.signature_calls==2);
- assert(get64(metadata+72)==w.stock_dma && get64(metadata+80)==4096);
- assert(!memcmp(before,metadata,72) && !memcmp(before+88,metadata+88,sizeof(before)-88));
- assert(unlock_transaction_signature(&w)==-EINVAL && fake.barriers==2);
- }
- /* Exercise every coherent-allocation failure in a real multi-page radix. */
- for(int fail=0;fail<=5;fail++) {
- setup(&w);w.gsp_fw=&fw;w.image_span.size=sizeof(fw_image);
- assert(!gpu_unlock_radix_plan(w.image_span.size,&w.radix));fake.fail_alloc=fail;
- int ret=unlock_transaction_radix(&w);
- if(fail) {
- assert(ret==-ENOMEM);
- assert(strstr(fake.log,"DMA allocation failed: asset=radix-page bytes=4096"));
- } else {
- assert(ret==0 && w.pages_allocated==5);
- assert(get64(w.pages[0].cpu)==w.pages[1].dma);
- assert(get64(w.pages[1].cpu)==w.pages[2].dma);
- assert(get64(w.pages[2].cpu)==w.pages[3].dma);
- assert(get64((u8 *)w.pages[2].cpu+8)==w.pages[4].dma);
- assert(get64((u8 *)w.pages[2].cpu+16)==0);
- }
- free_coherent(&w);assert(fake.live==0);
- }
- /* Early PLM polling must not wait for HALT or accept inaccessible engines. */
- for(int mode=0;mode<4;mode++) {
- setup(&w);fake.boot_mode=mode;w.master=w.firmware_started=w.published=true;
- w.booter_hdr=(struct gpu_unlock_booter_hdr){.app_code_off=256,.app_code_size=256,
- .os_data_off=512,.os_data_size=256,.patch_loc=512,.engine_id=1,.ucode_id=3};
- w.booter_dma=0x100000;w.meta_dma=0x200000;w.plm_entry=0x8f;
- int ret=unlock_transaction_boot(&w);
- assert(fake.starts==1);
- if(mode==0)assert(!ret);
- if(mode==1)assert(ret==-ETIMEDOUT && fake.now<=5000100);
- if(mode==1)assert(strstr(fake.log,"early PLM failed: pci=0x220410de"));
- if(mode==2)assert(ret==-ENODEV);
- if(mode==3)assert(ret==-EIO);
- }
- /* Failed FLR/reset/BAR checks retain allocations; success excludes fetches. */
- for(int failure=0;failure<10;failure++) {
- setup(&w);w.firmware_started=true;w.original_cmd=2;w.cmd_valid=true;
- w.v67_cpu=dma_alloc_coherent(&dev.dev,GU_V67_SIZE,&w.v67_dma,0);
- fake.fail_reset=(failure<3 || failure==8)?failure:0;fake.bar_bad=failure==3;
- fake.pending=failure==4?PCI_EXP_DEVSTA_TRPND:0;
- fake.lost_identity=failure==5;fake.corrupt_selector=failure==6?1:failure==9?2:0;fake.alien_wpr=failure==7;
- w.signature_calls=2;regs[GU_FEAT_SS1_OFFSET/4]=0x8;
- int ret=unlock_transaction_reset(&w);
- if(failure==6 || failure==9) {
- /* Lost selector state fails the unlock, but proven reset permits cleanup. */
- assert(ret==-EIO && w.quiesced);
- assert(!unlock_transaction_unwind(&w) && !fake.live && !w.retained);
- } else if(failure) {
- assert(ret && !w.quiesced);
- assert(unlock_transaction_unwind(&w)==-EBUSY && w.retained && fake.live==1);
- assert(strstr(fake.log,"failure unlock_610.c:"));
- if(failure==1)assert(strstr(fake.log,"PCIe FLR returned: result=-5"));
- if(failure==3)assert(strstr(fake.log,"post-FLR BAR0: actual=0x1 expected=0"));
- /* Host harness disposal only; the kernel intentionally retains. */
- free_coherent(&w);
- } else {
- assert(!ret && w.quiesced && fake.flrs==1);
- assert(!(regs[(SEC2_BASE+0x3c0)/4]&1) && !regs[0x1103c0/4]);
- assert(!unlock_transaction_unwind(&w) && fake.live==0);
- }
- }
- /* Entire production transaction with pinned assets and a board ROM (a real
- * dump, or "-" for the synthetic fixture). Only PCI/MMIO/DMA are mocked;
- * parsers, patching and ordering are real. */
- if(argc==3) {
- asset_root=argv[2];
- full_active=true;
- static const char *cases[]={"success","already-full","context-oom","asset-1","asset-2",
- "asset-3","asset-4","asset-5","claim-busy","map-failed","save-failed",
- "dma-1","dma-6","dma-7","dma-8","dma-48","dma-last",
- "gfw-timeout","preexisting-plm","fwsec-timeout","fwsec-error","fwsec-wpr",
- "booter-timeout","booter-inaccessible","booter-halted","selector-readback",
- "flr-failed","reset-inaccessible","pci-lost","pending","wpr-alien","selector-lost","mask-restore","decode-off",
- "reset-delay10","reset-delay12","reset-never","reset-post-never","reset-post-inaccessible",
- "reset-locked-before-flr","bar-ready-late","bar-inaccessible","bar-wrong-chip",
- "wpr-disabled-zero","wpr-inverted","wpr-inaccessible",
- "post-gfw-late","post-gfw-timeout","post-gfw-inaccessible","post-gsp-resumed",
- "wpr-frts-retained","firmware-group-oom-first","firmware-group-oom-later","devres-preexisting","devres-at-handoff",
- "warm-wpr2","ss1-readback","ss1-lost","power-unknown","power-unknown-no-pm",
- "firmware-bme","firmware-bme-stuck",
- "resume-success","resume-gfw-late","resume-kept-power","resume-gsp-running",
- "resume-gfw-timeout","resume-oom"};
- int total_allocations=0;
- for(unsigned scenario=0;scenario<ARRAY_SIZE(cases);scenario++) {
- setup(&w);
- load_rom(argv[1]);
- regs[0x1183a4/4]=0x6000;regs[0x625f04/4]=1;regs[0x1fa7c8/4]=0x45cb8f;
- regs[0x1fa82c/4]=0x100000;regs[0x1fa830/4]=0x5cffff0;
- regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
- regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;
- /* Cold values captured from the Palit card at 0000:21:00.0. */
- regs[GU_FEAT_PLM_OFFSET/4]=0xffffff8f;regs[GU_FEAT_SS0_OFFSET/4]=0x40171022;
- regs[GU_FEAT_SS1_OFFSET/4]=0x2;
- fake.fwsec_wpr=1;
- fake.fwsec_wpr_lo=(u32)(0x5ffe00000ull>>12)<<4;
- fake.fwsec_wpr_hi=(u32)(0x5ffeff000ull>>12)<<4;
- regs[(GU_FUSE_SEC2_UCODE1+2*4)/4]=1;
- /* FWSEC fuse: the real board ROM needs version 1, the fixture 0. */
- if(strcmp(argv[1],"-"))regs[(0x8241c0+8*4)/4]=1;
- if(scenario==1){regs[GU_FEAT_SS0_OFFSET/4]=0x88888888;regs[GU_FEAT_SS1_OFFSET/4]=0x8;}
- if(scenario==2)fake.fail_context=1;
- if(scenario>=3 && scenario<=7)fake.fail_fw=scenario-2;
- if(scenario==8)fake.fail_claim=1;
- if(scenario==9)fake.fail_map=1;
- if(scenario==10)fake.fail_save=1;
- if(scenario>=11 && scenario<=16) {
- const int failures[]={1,6,7,8,48,total_allocations};
- fake.fail_alloc=failures[scenario-11];
- }
- if(scenario==17)regs[GU_GFW_BOOT_OFFSET/4]=0;
- if(scenario==18)regs[GU_FEAT_PLM_OFFSET/4]=~0u;
- if(scenario==19)fake.fwsec_mode=1;
- if(scenario==20)regs[(0x1400+0x0e*4)/4]=1u<<16;
- if(scenario==21)fake.fwsec_wpr_hi=0;
- if(scenario>=22 && scenario<=24)fake.boot_mode=scenario-21;
- if(scenario==25)fake.selector_bad=1;
- if(scenario==26)fake.fail_reset=1;
- if(scenario==27)fake.fail_reset=8;
- if(scenario==28)fake.lost_identity=1;
- if(scenario==29)fake.pending=PCI_EXP_DEVSTA_TRPND;
- if(scenario==30)fake.alien_wpr=1;
- if(scenario==31)fake.corrupt_selector=1;
- if(scenario==32)fake.restore_mask_fail=1;
- if(scenario==33)fake.cmd=0;
- if(scenario==34)fake.reset_delay=10;
- if(scenario==35)fake.reset_delay=12;
- if(scenario==36)fake.fail_reset=2;
- if(scenario==37)fake.reset_post_fault=1;
- if(scenario==38)fake.reset_post_fault=2;
- if(scenario==39)fake.reset_locked_until_flr=1;
- if(scenario==40)fake.bar_ready_after=5;
- if(scenario==41)fake.bar_ready_after=-1;
- if(scenario==42)fake.bar_ready_after=-2;
- if(scenario>=43 && scenario<=45)fake.wpr_after=scenario-42;
- if(scenario>=46 && scenario<=49)fake.post_gfw=scenario-45;
- if(scenario==50)fake.wpr_after=4;
- if(scenario==51)fake.fail_group=1;
- if(scenario==52)fake.fail_group=3;
- if(scenario==53)dev.dev.devres_head.count=1;
- if(scenario==54)fake.late_devres=1;
- if(scenario==55)regs[0x1fa828/4]=fake.fwsec_wpr_hi; /* a driver already ran FWSEC */
- if(scenario==56)fake.selector_bad=2;
- if(scenario==57)fake.corrupt_selector=2;
- if(scenario==58 || scenario==59)dev.current_state=PCI_UNKNOWN;
- if(scenario==58)dev.pm_cap=0x60;
- if(scenario==60 || scenario==61)fake.cmd=6; /* firmware left BME set */
- if(scenario==61)fake.master_stuck=1;
- /* After resume: nvidia bound with its BME and device resources,
- * the card powered off and on again. */
- bool bound=!strncmp(cases[scenario],"resume-",7);
- if(bound){fake.bound=1;fake.cmd=6;fake.busmaster=1;dev.dev.devres_head.count=1;}
- if(!strcmp(cases[scenario],"resume-gfw-late")) {
- fake.gfw_late=5;regs[GU_GFW_BOOT_OFFSET/4]=0;regs[0x1183a4/4]=0;
- }
- if(!strcmp(cases[scenario],"resume-kept-power")) {
- regs[GU_FEAT_SS0_OFFSET/4]=0x88888888;regs[GU_FEAT_SS1_OFFSET/4]=0x8;
- }
- if(!strcmp(cases[scenario],"resume-gsp-running")) { /* GSP-RM up: not halted, WPR2 enabled */
- regs[0x1fa828/4]=fake.fwsec_wpr_hi;regs[GU_GFW_CPUCTL_OFFSET/4]=0;
- }
- if(!strcmp(cases[scenario],"resume-gfw-timeout"))regs[GU_GFW_BOOT_OFFSET/4]=0;
- if(!strcmp(cases[scenario],"resume-oom"))fake.fail_alloc=1;
- /* As main.c: the GSP file is loaded once, ahead of every card. */
- struct gpu_unlock_610_shared shared={0};last_work=NULL;
- int ret=gpu_unlock_610_load_shared(&dev,&shared);
- if(!ret)ret=gpu_unlock_610_unlock(&dev,&shared,bound);
- gpu_unlock_610_release_shared(&shared);
- bool expected=scenario<2 || (scenario>=33 && scenario<=35) || scenario==39 || scenario==40 || scenario==43 || scenario==46 ||
- scenario==58 || scenario==60 ||
- !strcmp(cases[scenario],"resume-success") || !strcmp(cases[scenario],"resume-gfw-late");
- bool retained=fake.pins;
- if((!ret)!=expected)fprintf(stderr,"%s\n%s: error=%d\n",fake.log,cases[scenario],ret);
- assert((!ret)==expected);
- if(expected) {
- assert(fake.gsp_starts==1 && fake.starts==1 && fake.flrs==1 && !fake.pins);
- assert(fake.apertures==!bound && !fake.regions);
- assert(fake.selector_writes==(scenario==1?0:1) && fake.ss1_writes==(scenario==1?0:1));
- assert(regs[GU_FEAT_SS0_OFFSET/4]==0x88888888 && regs[GU_FEAT_SS1_OFFSET/4]==0x8);
- assert(mask==0xffffffffffffull && dev.dev.coherent_dma_mask==0xfffffffffull);
- /* Mirror Linux really_probe: driver entry rejects nonempty devres.
- * A bound driver keeps its own. */
- assert(dev.dev.devres_head.count==(bound?1u:0u));
- assert(fake.cache_records==5 && fake.cache_released==5);
- /* Pinned RM checks GSP halt + readable completed GFW BEFORE
- * its own reset/bootstrap. Preserve that entry contract. */
- assert((regs[GU_GFW_CPUCTL_OFFSET/4]&0x10) && (regs[GU_GFW_PLM_OFFSET/4]&1));
- assert((regs[GU_GFW_BOOT_OFFSET/4]&0xff)==0xff && !regs[0x1103c0/4]);
- assert(!(regs[0x1fa828/4]>>4)); /* Stock RM rejects any active WPR2. */
- if(!scenario)total_allocations=fake.allocs;
- }
- if(scenario>=34 && scenario<=50) {
- assert(fake.starts==1 && fake.flrs==1);
- if(!expected)assert(retained && !last_work->quiesced);
- if(scenario==41 || scenario==42)assert(!fake.hold_writes && ret==-ETIMEDOUT);
- if(scenario==39)assert(fake.hold_writes==1);
- if(scenario==47 || scenario==48)assert(!fake.hold_writes && ret==-ETIMEDOUT);
- }
- assert(!fake.gsp_post_reset_writes);
- assert(fake.gsp_starts<=1 && fake.starts<=1 && fake.flrs<=1 && fake.pins<=1);
- assert(fake.selector_writes<=1 && fake.ss1_writes<=1);
- if(retained) {
- assert(last_work->retained && fake.live>0 && dev.refs==2);
- /* Harness-only disposal of intentionally retained production state. */
- last_work->firmware_started=false;fake.restore_mask_fail=0;
- assert(!unlock_transaction_unwind(last_work));
- pci_dev_put(last_work->pdev);free(last_work);
- }
- /* A card whose BME will not clear is left as found; nvidia gets its own back. */
- assert(!fake.live && !fake.fw_live && dev.refs==1 && !!(fake.cmd&4)==(scenario==61 || bound));
- if(bound)assert(fake.busmaster);
- assert(!fake.group_live && fake.cache_records==fake.cache_released);
- assert(dev.dev.devres_head.count==(scenario==53 || scenario==54 || bound?1u:0u));
- assert(dev.dev.dma_mask==&mask && mask==0xffffffffffffull && dev.dev.coherent_dma_mask==0xfffffffffull);
- if(!strcmp(cases[scenario],"resume-gfw-late"))assert(fake.gfw_late_reads>=fake.gfw_late);
- /* A card that kept power or runs GSP-RM is only read. */
- if(!strcmp(cases[scenario],"resume-kept-power") || !strcmp(cases[scenario],"resume-gsp-running")) {
- assert(ret==(cases[scenario][7]=='k'?-EALREADY:-EBUSY));
- assert(!fake.writes && !fake.allocs && fake.fw_requests==1 && fake.cmd==6 && !fake.now);
- }
- if(!strcmp(cases[scenario],"resume-gfw-timeout"))
- assert(ret==-ETIMEDOUT && !fake.writes && !fake.allocs && fake.cmd==6);
- if(!strcmp(cases[scenario],"resume-oom"))
- assert(ret==-ENOMEM && !fake.gsp_starts && !fake.flrs && fake.cmd==6);
- if(scenario>=51 && scenario<=53)assert(!fake.writes && !fake.allocs && !fake.flrs);
- if(scenario==53)assert(fake.fw_requests==1 && ret==-EBUSY); /* only the shared load */
- if(scenario==3 || scenario==51)assert(!last_work && fake.fw_requests==(scenario==3));
- if(scenario==54)assert(fake.flrs==1 && ret==-EBUSY);
- if(scenario==53 || scenario==54)assert(strstr(fake.log,"driver probe resources: phase=handoff empty=0"));
- if(scenario==33)assert(fake.cmd==0);
- if(scenario==56)assert(ret==-EIO && retained && fake.ss1_writes==1 && !fake.flrs);
- if(scenario==57)assert(ret==-EIO && !retained && fake.flrs==1);
- if(scenario==59)assert(ret==-EOPNOTSUPP && !fake.allocs && !fake.flrs);
- if(scenario==60)assert(strstr(fake.log,"claim: cleared firmware BME"));
- if(scenario==61)assert(ret==-EBUSY && !fake.allocs && !fake.flrs);
- if(scenario==55) {
- assert(ret==-EBUSY && !retained && !fake.gsp_starts && !fake.starts && !fake.flrs);
- assert(strstr(fake.log,"card already initialised since power on"));
- }
- printf("full transaction %s: error=%d retained=%u PASS\n",cases[scenario],ret,retained);
- }
- puts("pinned firmware full BF16 transactions passed under mocked hardware");
- }
- puts("actual unlock_transaction radix, early-PLM and reset/retention checks passed");
-}
diff --git a/tests/test_vbios_610.c b/tests/test_vbios_610.c
deleted file mode 100644
index 2276b7e..0000000
--- a/tests/test_vbios_610.c
+++ /dev/null
@@ -1,125 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <assert.h>
-#include <errno.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include "vbios_610.h"
-
-static void w32(u8 *p, u32 v)
-{
- for (int i = 0; i < 4; i++)
- p[i] = v >> (8 * i);
-}
-
-#include "vbios_fixture.h"
-
-int main(int argc, char **argv)
-{
- u8 rom[4096], image[1280] = {0};
- struct gpu_unlock_fwsec fw;
- u32 offset;
-
- for (int stride = 6; stride <= 8; stride += 2) {
- vbios_fixture(rom, stride);
- assert(!gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw));
- assert(fw.image_off == 684 && fw.image_size == 1280);
- }
- /* Allocate the actual short length: a larger backing array hides overreads. */
- for (size_t len = 0; len < 4096; len++) {
- u8 *short_rom = malloc(len ? len : 1);
- assert(short_rom);
- memcpy(short_rom, rom, len);
- assert(gpu_unlock_vbios_fwsec(short_rom, len, &fw));
- free(short_rom);
- }
- /* Legacy + EFI with NPDE continuation + VN/NPDS extension. The BIT
- * pointers address the extension as if it followed the legacy image. */
- for (int ext_len = 10; ext_len <= 11; ext_len++) {
- u8 chain[8192] = {0};
- vbios_fixture(rom, 6);
- memcpy(chain, rom, 4096);
- chain[53] = 0;
- chain[4096] = 0x55; chain[4097] = 0xaa; chain[4096+0x18] = 32;
- memcpy(chain+4096+32, "PCIR", 4);
- chain[4096+42] = 24; chain[4096+48] = 8;
- chain[4096+52] = 3; chain[4096+53] = 0x80;
- memcpy(chain+4096+64, "NPDE", 4);
- chain[4096+69] = 1; chain[4096+70] = ext_len;
- chain[4096+72] = 1; /* one 512-byte subimage */
- chain[4608] = 'V'; chain[4609] = 'N'; chain[4608+0x18] = 32;
- memcpy(chain+4608+32, "NPDS", 4);
- chain[4608+42] = 24; chain[4608+48] = 7;
- chain[4608+52] = 0xe0; chain[4608+53] = 0x80;
- w32(chain+100, 4096+128);
- memcpy(chain+4608+128, rom+128, 18);
- w32(chain+4608+142, 4096+256);
- memcpy(chain+4608+256, rom+256, 428+1280);
- assert(!gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw));
- assert(fw.image_off == 4608+684);
- assert(gpu_unlock_vbios_fwsec(chain, 4608, &fw) == -EINVAL);
- /* RGIS/BB77 is also supported, malformed data signatures are not. */
- chain[4608] = 0x77; chain[4609] = 0xbb;
- memcpy(chain+4608+32, "RGIS", 4);
- assert(!gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw));
- chain[4096+70] = 9;
- assert(gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw) == -EINVAL);
- chain[4096+70] = ext_len; chain[4096+72] = 0;
- assert(gpu_unlock_vbios_fwsec(chain, sizeof(chain), &fw) == -EINVAL);
- }
- vbios_fixture(rom, 8);
- w32(rom + 264, 0xffffffff);
- assert(gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw) == -EINVAL);
- vbios_fixture(rom, 8);
- w32(rom + 288, 0xffffffff);
- assert(gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw) == -EINVAL);
- vbios_fixture(rom, 8);
- assert(!gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw));
- image[256] = 1; image[257] = 4; image[258] = 8; image[259] = 2;
- w32(image + 260, 4); w32(image + 264, 32);
- image[292] = 3; image[294] = 64;
- w32(image + 296, 128); w32(image + 300, 44);
- assert(!gpu_unlock_fwsec_patch_frts(image, &fw, 0x100000));
- assert(image[332] == 0x15 && image[384] == 1);
- fw.interface_off = fw.dmem_size - 1;
- assert(gpu_unlock_fwsec_patch_frts(image, &fw, 0x100000) == -EINVAL);
- fw.interface_off = 0;
- w32(image + 264, 0xfffffff0);
- assert(gpu_unlock_fwsec_patch_frts(image, &fw, 0x100000) == -EINVAL);
- assert(!gpu_unlock_fwsec_sig_offset(7, 0, 384, &offset) && offset == 0);
- assert(!gpu_unlock_fwsec_sig_offset(5, 2, 384, &offset) && offset == 384);
- assert(gpu_unlock_fwsec_sig_offset(1, 1, 384, &offset) == -ERANGE);
- /* Optional integration check; normal tests contain no board-owned blobs. */
- if (argc == 2) {
- FILE *prom = fopen(argv[1], "rb");
- u8 *image;
- size_t got;
-
- assert(prom);
- assert(!fseek(prom, 0, SEEK_END));
- got = ftell(prom);
- assert(got == 1024 * 1024);
- rewind(prom);
- image = malloc(got);
- assert(image && fread(image, 1, got, prom) == got);
- fclose(prom);
- assert(!gpu_unlock_vbios_fwsec(image, got, &fw));
- assert(fw.ucode_id == 9 && fw.sig_count == 3 && fw.sig_versions == 7);
- assert(fw.engine_mask == 0x400 && fw.imem_size == 0xda00 && fw.dmem_size == 0x800);
- assert(fw.image_off == 0x4289c && fw.image_size == 0xe200);
- assert(fw.pkc_off == 0x5a4 && fw.interface_off == 0x1c);
- /* Revision 05 recorded Booter fuse version 1. That selects signature slot 1. */
- assert(!gpu_unlock_fwsec_sig_offset(fw.sig_versions, 1, 384, &offset) && offset == 384);
- assert(!gpu_unlock_fwsec_patch_frts(image + fw.image_off, &fw, 0x5ffe00000ull));
- {
- const u8 *cmd = image + fw.image_off + fw.data_off + 0x7c0;
- u32 frts4k = (u32)cmd[32] | ((u32)cmd[33] << 8) | ((u32)cmd[34] << 16) | ((u32)cmd[35] << 24);
-
- assert(frts4k == (0x5ffe00000ull >> 12));
- assert(cmd[36] == 0 && cmd[37] == 1 && cmd[40] == 2);
- }
- free(image);
- }
- puts("vbios valid-chain, malformed range, patch and signature checks passed");
- return 0;
-}
diff --git a/tests/unlock_shim/compat.h b/tests/unlock_shim/compat.h
deleted file mode 100644
index 2e575f4..0000000
--- a/tests/unlock_shim/compat.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef UNLOCK_TRANSACTION_SHIM_H
-#define UNLOCK_TRANSACTION_SHIM_H
-#include <stdint.h>
-#include <stdbool.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <string.h>
-#include <stdio.h>
-#include <asm-generic/errno.h>
-typedef uint8_t u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-typedef int32_t s32;
-typedef uint64_t dma_addr_t;
-typedef uint64_t resource_size_t;
-typedef uint64_t ktime_t;
-#define __iomem
-#define GFP_KERNEL 0
-#define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0]))
-#define min_t(t,a,b) ((t)(a)<(t)(b)?(t)(a):(t)(b))
-#define lower_32_bits(v) ((u32)(v))
-#define upper_32_bits(v) ((u32)((u64)(v)>>32))
-#define DMA_BIT_MASK(n) ((1ull<<(n))-1)
-#define MODULE_FIRMWARE(x)
-#define THIS_MODULE NULL
-void test_device_log(const char *, ...) __attribute__((format(printf, 1, 2)));
-#define dev_err(dev, ...) test_device_log(__VA_ARGS__)
-#define dev_info(dev, ...) test_device_log(__VA_ARGS__)
-#define PCI_D0 0
-#define PCI_UNKNOWN 5
-#define PCI_DEV_FLAGS_NO_FLR_RESET (1u << 10)
-#define PCI_VENDOR_ID 0
-#define PCI_COMMAND 4
-#define PCI_COMMAND_MEMORY 2
-#define PCI_COMMAND_MASTER 4
-#define PCI_PM_CTRL 4
-#define PCI_PM_CTRL_STATE_MASK 3
-#define PCI_EXP_DEVCAP 4
-#define PCI_EXP_DEVCAP_FLR (1u<<28)
-#define PCI_EXP_DEVSTA 10
-#define PCI_EXP_DEVSTA_TRPND 32
-#define PCI_BASE_ADDRESS_0 16
-#define IORESOURCE_MEM 0x200
-struct list_head { unsigned int count; };
-static inline bool list_empty(const struct list_head *h) { return !h->count; }
-#define spin_lock_irqsave(lock, flags) do { (void)(lock); (flags)=0; } while (0)
-#define spin_unlock_irqrestore(lock, flags) do { (void)(lock); (void)(flags); } while (0)
-struct device { u64 *dma_mask; u64 coherent_dma_mask; struct list_head devres_head; int devres_lock; };
-void *devres_open_group(struct device *,void *,int);
-int devres_release_group(struct device *,void *);
-struct pci_dev { struct device dev; unsigned short vendor, device; int current_state, pm_cap, refs; unsigned int dev_flags; };
-struct firmware { size_t size; const u8 *data; };
-void usleep_range(unsigned long, unsigned long);
-ktime_t ktime_get(void);
-static inline ktime_t ktime_add_us(ktime_t t, unsigned long u) { return t+u; }
-static inline int ktime_compare(ktime_t a,ktime_t b) { return a>b?1:a<b?-1:0; }
-u32 readl(const void *);
-void writel(u32,void *);
-#define memcpy_fromio memcpy
-void *dma_alloc_coherent(struct device *,size_t,dma_addr_t *,int);
-void dma_free_coherent(struct device *,size_t,void *,dma_addr_t);
-int dma_set_mask(struct device *,u64);
-int dma_set_coherent_mask(struct device *,u64);
-void dma_wmb(void);
-void *kzalloc(size_t n,int flags);
-static inline void *kvcalloc(size_t n,size_t s,int flags) { return calloc(n,s); }
-static inline void *kvmalloc(size_t n,int flags) { return malloc(n); }
-#define kfree free
-#define kvfree free
-int request_firmware(const struct firmware **,const char *,struct device *);
-void release_firmware(const struct firmware *);
-void __module_get(void *);
-int pci_read_config_word(struct pci_dev *,int,u16 *);
-int pci_read_config_dword(struct pci_dev *,int,u32 *);
-int pci_write_config_word(struct pci_dev *,int,u16);
-int pcie_capability_read_word(struct pci_dev *,int,u16 *);
-int pcie_capability_read_dword(struct pci_dev *,int,u32 *);
-int pci_is_pcie(struct pci_dev *);
-int pci_select_bars(struct pci_dev *,int);
-int pci_request_region(struct pci_dev *,int,const char *);
-void pci_release_region(struct pci_dev *,int);
-int aperture_remove_conflicting_pci_devices(struct pci_dev *,const char *);
-int pci_enable_device_mem(struct pci_dev *);
-void pci_disable_device(struct pci_dev *);
-resource_size_t pci_resource_len(struct pci_dev *,int);
-void *pci_iomap(struct pci_dev *,int,unsigned long);
-void pci_iounmap(struct pci_dev *,void *);
-void *pci_map_rom(struct pci_dev *,size_t *);
-void pci_unmap_rom(struct pci_dev *,void *);
-int pci_save_state(struct pci_dev *);
-void pci_restore_state(struct pci_dev *);
-void pci_set_master(struct pci_dev *);
-void pci_clear_master(struct pci_dev *);
-int pcie_flr(struct pci_dev *);
-struct pci_dev *pci_dev_get(struct pci_dev *);
-void pci_dev_put(struct pci_dev *);
-#endif
diff --git a/tests/unlock_shim/linux/module.h b/tests/unlock_shim/linux/module.h
deleted file mode 100644
index 4715430..0000000
--- a/tests/unlock_shim/linux/module.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include "../compat.h"
diff --git a/tests/unlock_shim/linux/pci.h b/tests/unlock_shim/linux/pci.h
deleted file mode 100644
index 4715430..0000000
--- a/tests/unlock_shim/linux/pci.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include "../compat.h"
diff --git a/tests/unlock_shim/linux/printk.h b/tests/unlock_shim/linux/printk.h
deleted file mode 100644
index 4ac5987..0000000
--- a/tests/unlock_shim/linux/printk.h
+++ /dev/null
@@ -1,3 +0,0 @@
-#include <stdio.h>
-#define pr_info(...) fprintf(stderr, __VA_ARGS__)
-#define pr_err(...) fprintf(stderr, __VA_ARGS__)
diff --git a/tests/unlock_shim/linux/slab.h b/tests/unlock_shim/linux/slab.h
deleted file mode 100644
index 4715430..0000000
--- a/tests/unlock_shim/linux/slab.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include "../compat.h"
diff --git a/tests/vbios_fixture.h b/tests/vbios_fixture.h
deleted file mode 100644
index 7951214..0000000
--- a/tests/vbios_fixture.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_TEST_VBIOS_FIXTURE_H
-#define GPU_UNLOCK_TEST_VBIOS_FIXTURE_H
-/* Synthetic, redistributable ROM; no board firmware contents. */
-static void vbios_w32(u8 *p, u32 v)
-{
- for (int i = 0; i < 4; i++) p[i] = v >> (8 * i);
-}
-static void vbios_fixture(u8 *rom, int stride)
-{
- unsigned int sum = 0;
-
- memset(rom, 0, 4096);
- rom[0] = 0x55; rom[1] = 0xaa; rom[0x18] = 32;
- memcpy(rom + 32, "PCIR", 4);
- rom[42] = 24; rom[48] = 8; rom[53] = 0x80;
- rom[64] = 0xff; rom[65] = 0xb8; memcpy(rom + 66, "BIT", 3);
- rom[71] = 1; rom[72] = 12; rom[73] = stride; rom[74] = 1;
- for (int i = 64; i < 75; i++)
- sum += rom[i];
- rom[75] = -sum;
- rom[76] = 0x70; rom[77] = 2; rom[78] = 4; vbios_w32(rom + 80, 100);
- vbios_w32(rom + 100, 128);
- rom[128] = 1; rom[129] = 6; rom[130] = 6; rom[131] = 2;
- /* FWSEC need not be the first entry. */
- rom[134] = 0x45; rom[140] = 0x85; vbios_w32(rom + 142, 256);
- vbios_w32(rom + 256, (428u << 16) | 0x301);
- vbios_w32(rom + 260, 1280); vbios_w32(rom + 264, 512); vbios_w32(rom + 268, 0);
- vbios_w32(rom + 276, 256); vbios_w32(rom + 288, 1024);
- rom[292] = 1; rom[294] = 1; rom[295] = 1; rom[296] = 1;
-}
-
-#endif