Skip to content

Your GeForce runs AI at half speed. This fixes it for 20/30/40 series cards. Up to 2× FP16 / BF16 / TF32 tensor throughput

C

Revert main to working state at c274e74

Zagitta4189b17711bd
diff --git a/.gitattributes b/.gitattributes
index 307c347..7438baf 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -11,5 +11,3 @@
*.bin export-ignore
*.cu export-ignore
/debian -export-ignore
-/docs -export-ignore
-/docs/* export-ignore
diff --git a/Makefile b/Makefile
index c4d4491..d6ac3a0 100644
--- a/Makefile
+++ b/Makefile
@@ -14,9 +14,8 @@ PKG ?= gpu-unlock
MODULE_FILES := $(filter-out %.mod.c,$(sort $(wildcard kernel/gpu_unlock/*.[ch]))) \
kernel/gpu_unlock/Kbuild kernel/gpu_unlock/Makefile kernel/gpu_unlock/dkms.conf
-FIRMWARE_FILES := booter_load_ga100.bin falcon_loader.bin gsp_ga10x.bin gsp_tu10x.bin \
- gsp_boot_ga102.bin gsp_boot_desc_ga102.bin \
- gsp_boot_ga100.bin gsp_boot_desc_ga100.bin
+FIRMWARE_FILES := booter_load_ga102.bin gsp_ga10x.bin gsp_boot_ga102.bin \
+ gsp_boot_desc_ga102.bin v67-ga102-$(NV).bin
LICENSES := licenses/GPL-2.0.txt licenses/CMP40HX-MIT.txt licenses/NVIDIA-MIT.txt
NV_SOURCE := NVIDIA-kernel-module-source-$(NV)
NV_UTILS := nvidia-utils-$(NV)-1-x86_64_v3.pkg.tar.zst
diff --git a/debian/control b/debian/control
index 99e9348..5eae7f7 100644
--- a/debian/control
+++ b/debian/control
@@ -2,7 +2,7 @@ Source: gpu-unlock
Section: kernel
Priority: optional
Maintainer: Zagitta <[email protected]>
-Build-Depends: debhelper-compat (= 13), dh-sequence-dkms, python3, zstd, libssl-dev
+Build-Depends: debhelper-compat (= 13), dh-sequence-dkms, python3, zstd
Standards-Version: 4.7.0
Rules-Requires-Root: no
@@ -10,7 +10,7 @@ Package: gpu-unlock-dkms
Architecture: all
Depends: ${misc:Depends}, dkms,
gpu-unlock-firmware (= ${source:Version})
-Description: Ampere pre-driver hardware unlock
+Description: RTX 3090 BF16 unlock before the NVIDIA driver loads
DKMS source for gpu_unlock, loaded ahead of nvidia on every boot. It does
nothing unless gpu_unlock.targets= is on the kernel command line.
diff --git a/firmware/NOTICE.md b/firmware/NOTICE.md
index 6b067ec..eb14cc3 100644
--- a/firmware/NOTICE.md
+++ b/firmware/NOTICE.md
@@ -1,6 +1,6 @@
# Firmware provenance and distribution scope
-`gsp_ga10x.bin` and `gsp_tu10x.bin` are extracted byte-for-byte from the checksummed NVIDIA
+`gsp_ga10x.bin` is extracted byte-for-byte from the checksummed NVIDIA
610.43.03 package in `artifacts-610.43.03.json`. The Booter and GSP boot images
are decoded unchanged from the pinned 610.43.03 source archive. The extraction
also preserves that package's complete `usr/share/licenses/nvidia-utils/LICENSE`
@@ -14,19 +14,17 @@ apply, including NVIDIA-platform use. Personal local packaging is the initial
delivery target. Review the full pinned agreement and the separately generated
payload before any public distribution; redistribution approval is not claimed.
-The payload constants and ordered diagnostic PLM group are adapted from
-amoghmunikote/cmpunlocker at commit
-`2d10f891a58824f0591c3332c4eccf37ff36b086`, patch
-`driver/patches/sec2-postbl-plm-ss-cfg.patch`, function
-`_kgspSec2PostblTimingFillPayload`. Its repository license is GPL version 2,
-preserved in `licenses/GPL-2.0.txt`. The C runtime and offline generator
-reproduce its built-in template, including the full exit tail. The generated
-payload is separate from the unchanged NVIDIA firmware.
+The V67 constants are adapted from WildFlash1st/cmp90hx-unlock at commit
+`76940ae583281daa2b883518a2e89b56c11d3a46`, patch
+`driver/patches/cmp90-rejoin15/0014-6104303-cmp90hx-stockflow-rejoin14-multigpu-state.patch`,
+function `_kgspCmp90FillV67Signature`. The repository's actual `LICENSE` is
+GPL version 2; the earlier manifest's MIT label was incorrect. That license is
+preserved in `licenses/GPL-2.0.txt`. The generated image is a separate file;
+the extracted NVIDIA images are never rewritten.
NVIDIA source files inspected during the loader audit carry their own MIT
copyright notices. The minimum helper's metadata layout and Falcon operations
follow those source contracts; `licenses/NVIDIA-MIT.txt` preserves the notice.
The inspected EFI reference's notice is in `licenses/CMP40HX-MIT.txt`.
-The payload reference targets GA100 CMP 170HX. It does not establish GA10x
-firmware acceptance, standalone multiwrite, final FLR retention or stock-driver
-handover. Those require the Stage 1 hardware experiments.
+The reference changes a CMP 90HX, not an RTX 3090, and is not evidence of
+standalone boot or persistence through NVIDIA 615.71.09.
diff --git a/firmware/README.md b/firmware/README.md
index 30ff529..32348bb 100644
--- a/firmware/README.md
+++ b/firmware/README.md
@@ -1,24 +1,19 @@
# Pinned 610.43.03 firmware
-`make inputs` downloads the two NVIDIA inputs from
-`artifacts-610.43.03.json`, checking SHA-256 before accepting cached files.
-`make firmware` extracts the two GSP ELFs, signed GA100 SEC2 Booter bundle,
-GA100 generic Falcon loader, and per-family GSP boot images/descriptors.
-Every derived file has a pinned SHA-256. Extracted NVIDIA bytes are unchanged.
+`make inputs` runs `fetch_artifacts.py`, which downloads the NVIDIA open kernel
+module source archive and the exact CachyOS `nvidia-utils` package pinned in
+`artifacts-610.43.03.json`. Downloads use HTTPS and are checked against their
+SHA-256 before they enter the cache; cached files are rechecked on every run.
+Nothing downloaded is executed or installed.
-`generate_v67.py --payload` additionally emits `cmp-signature.bin` as an offline
-reference image; normal firmware builds omit it. It is not installed or requested
-by the module. Runtime builds the same
-63,488-byte payload in C and changes only the requested address/value.
-`--header PATH` reproduces the committed `payload_words.h` from the manifest.
-There is no optional external `dmem.bin` override.
+`make firmware` then builds the five files the kernel module loads from
+`gpu-unlock/610.43.03/`:
-The SEC2 Booter executes on Falcon and loads the separate GSP boot image.
-GA100 uses `gsp_tu10x.bin` with `.fwsignature_ga100`; GA10x uses
-`gsp_ga10x.bin` with `.fwsignature_ga10x`. The selected card determines its
-GSP assets once, separately from register tables. All new hardware contexts
-remain experimental; see [Stage 1 evidence](../docs/STAGE1.md).
+- `generate_v67.py` extracts `gsp_ga10x.bin` and the NVIDIA license from the
+ package and generates the 64,000-byte `v67-ga102-610.43.03.bin`.
+- `extract_boot_assets.py` builds `booter_load_ga102.bin`,
+ `gsp_boot_ga102.bin` and `gsp_boot_desc_ga102.bin` from the source archive.
-Python 3, zstd, and the pinned inputs are sufficient for extraction. The host
-C tests additionally require a compiler and OpenSSL development headers.
-See [NOTICE.md](NOTICE.md) for provenance and licensing.
+Both scripts check every input and output against its pinned SHA-256 before
+writing anything. Python 3.14+ is required for zstd tar support. See
+[NOTICE.md](NOTICE.md) for provenance and licensing.
diff --git a/firmware/artifacts-610.43.03.json b/firmware/artifacts-610.43.03.json
index 6f2b34a..d5405f1 100644
--- a/firmware/artifacts-610.43.03.json
+++ b/firmware/artifacts-610.43.03.json
@@ -1,6 +1,6 @@
{
"version": 1,
- "target": "Stage 1 cmpunlocker multiwrite; hardware verification pending",
+ "target": "GA102 / 610.43.03 rejoin15 reference path",
"source_artifacts": [
{
"name": "NVIDIA open kernel modules source archive",
@@ -12,22 +12,22 @@
"filename": "NVIDIA-kernel-module-source-610.43.03.tar.xz"
},
{
- "name": "cmpunlocker built-in multiwrite reference",
- "url": "https://github.com/amoghmunikote/cmpunlocker",
- "revision": "2d10f891a58824f0591c3332c4eccf37ff36b086",
- "license": "GPL-2.0-only",
+ "name": "GA102 CMP90 reference and rejoin15 patch set",
+ "url": "https://github.com/WildFlash1st/cmp90hx-unlock",
+ "revision": "76940ae583281daa2b883518a2e89b56c11d3a46",
+ "license": "GPL-2.0-only (repository LICENSE; previous MIT label was incorrect)",
"files": [
{
- "path": "driver/patches/sec2-postbl-plm-ss-cfg.patch",
- "sha256": "1a9ecd419dc5e732951c8f7093be2a2314233c17e9126297459025a9ff08bc78"
+ "path": "driver/patches/cmp90-rejoin15/0014-6104303-cmp90hx-stockflow-rejoin14-multigpu-state.patch",
+ "sha256": "c51805dc332c3b6be2e6dd34964a47ce6b1ea0ce73017e0351c216d319094bb1"
},
{
- "path": "common/constants.yaml",
- "sha256": "de3900c14eab61428fb36be7776df0ea8555db72d01dfdba9a7b3d3bb2fffceb"
+ "path": "driver/patches/cmp90-rejoin15/0015-6104303-cmp90hx-stockflow-rejoin15-serialized-start.patch",
+ "sha256": "3dd3883479d10a822d54292db87a1552f15d6b144b8c42bfc88a519674836cb4"
},
{
- "path": "driver/build.sh",
- "sha256": "ac3a3f0b6f374e8593fd640fb3eb9f3b9bc25ae36f3df192d16de7f796d37c64"
+ "path": "LICENSE",
+ "sha256": "8177f97513213526df2cf6184d8ff986c675afb514d4e68a404010521b880643"
}
]
},
@@ -42,11 +42,6 @@
"path": "usr/lib/firmware/nvidia/610.43.03/gsp_ga10x.bin",
"size": 84277400,
"sha256": "572373620a37418f24dc16b5031c39338778c3257e48e8408de9a57291b24f3a"
- },
- {
- "path": "usr/lib/firmware/nvidia/610.43.03/gsp_tu10x.bin",
- "size": 29352832,
- "sha256": "73065619db9ec921d19fc4e519dd04d91a9199b525eaca9b257b89fb8c5ec52c"
}
],
"filename": "nvidia-utils-610.43.03-1-x86_64_v3.pkg.tar.zst",
@@ -58,121 +53,109 @@
}
],
"generated_payload": {
- "name": "cmpunlocker built-in signature template (FEAT_PLM inputs)",
- "size": 63488,
- "fill_dword_le": "0x000004a7",
+ "name": "rejoin15 GA102 610.43.03 V67 signature image",
+ "size": 64000,
+ "size_hex": "0x0000fa00",
+ "fill_dword_le": "0x0000019c",
+ "stack_map": {
+ "active_canary_dmem": "0x0000ff4c",
+ "initial_return_slot_dmem": "0x0000ff50"
+ },
"overrides": [
{
- "offset": "0x1100",
- "value": "0x7"
- },
- {
- "offset": "0x5b40",
- "value": "0xc0deca7e"
- },
- {
- "offset": "0xf758",
- "value": "0xc0deca7e"
- },
- {
- "offset": "0xf75c",
- "value": "0xcbd"
- },
- {
- "offset": "0xf774",
- "value": "0x1fbd"
- },
- {
- "offset": "0xf780",
- "value": "0x0"
+ "offset": "0x00001100",
+ "value": "0x00000007"
},
{
- "offset": "0xf788",
- "value": "0x10aa"
+ "offset": "0x0000f948",
+ "value": "0xffffffff",
+ "role": "write value"
},
{
- "offset": "0xf78c",
- "value": "0x815a"
+ "offset": "0x0000f950",
+ "value": "0x00000d44"
},
{
- "offset": "0xf790",
- "value": "0x8e18"
+ "offset": "0x0000f960",
+ "value": "0x00823804",
+ "role": "FEAT_OVR_PLM address"
},
{
- "offset": "0xf794",
- "value": "0xc0deca7e"
+ "offset": "0x0000f968",
+ "value": "0x00001fce"
},
{
- "offset": "0xf798",
- "value": "0x815a"
+ "offset": "0x0000f974",
+ "value": "0x00000000"
},
{
- "offset": "0xf79c",
- "value": "0x0"
+ "offset": "0x0000f97c",
+ "value": "0x00001101"
},
{
- "offset": "0xf7a0",
- "value": "0xc0deca7e"
+ "offset": "0x0000f980",
+ "value": "0x000084c8"
},
{
- "offset": "0xf7a4",
- "value": "0x1fbd"
+ "offset": "0x0000f984",
+ "value": "0x00008e18"
},
{
- "offset": "0xf7b0",
- "value": "0xffbc"
+ "offset": "0x0000f98c",
+ "value": "0x000084c8"
},
{
- "offset": "0xf7b8",
- "value": "0x582d"
+ "offset": "0x0000f990",
+ "value": "0x00000000"
},
{
- "offset": "0xf7c4",
- "value": "0xc0deca7e"
+ "offset": "0x0000f998",
+ "value": "0x00001fce"
},
{
- "offset": "0xf7c8",
- "value": "0xcbd"
+ "offset": "0x0000f9a4",
+ "value": "0x0000ffbc"
},
{
- "offset": "0xf7d8",
- "value": "0x3"
+ "offset": "0x0000f9ac",
+ "value": "0x00005789"
},
{
- "offset": "0xf7e0",
- "value": "0x1fbd"
+ "offset": "0x0000f9bc",
+ "value": "0x00000d44"
},
{
- "offset": "0xf7f4",
- "value": "0xccb"
+ "offset": "0x0000f9cc",
+ "value": "0x00000003"
},
{
- "offset": "0xf7f8",
- "value": "0x7f2f"
+ "offset": "0x0000f9d4",
+ "value": "0x00001fce"
},
{
- "offset": "0xf754",
- "value": "0xffffffff"
+ "offset": "0x0000f9e8",
+ "value": "0x00000d52"
},
{
- "offset": "0xf76c",
- "value": "0x00823804"
+ "offset": "0x0000f9ec",
+ "value": "0x000081ee",
+ "role": "terminator"
}
],
- "write_value_offset": "0xf754",
- "write_address_offset": "0xf76c",
- "terminator_offset": "0xf7f8",
- "terminator": "0x7f2f",
- "sha256": "00de1fab4e97b88216d0aac2abfbc8ff75f90fa3342ecc3216f9fed656bc1abd",
- "generator": "firmware/generate_v67.py"
+ "write_value_offset": "0x0000f948",
+ "write_value": "0xffffffff",
+ "write_address_offset": "0x0000f960",
+ "write_address": "0x00823804",
+ "terminator_offset": "0x0000f9ec",
+ "terminator": "0x000081ee",
+ "sha256": "2a0aa1f86d0b6afff5214d641d87d46dc98ef06ed908ff05d0ddbad5ffb7e197",
+ "generator": "firmware/generate_v67.py",
+ "reference": "rejoin15 patch 0014, _kgspCmp90FillV67Signature"
},
"derived_assets": {
+ "booter_load_ga102.bin": "056cbc064a888f80717726c54d7f64fcd68044d123870c95a74f37f4975dcaa4",
"gsp_boot_ga102.bin": "3df651acb174fd5f5561a1996965b192f50c5d763168446ed0d4ead19d895768",
- "gsp_boot_desc_ga102.bin": "cc5a3fc1f4c47f2fe107b1b68f0176d8a448e6dec84b575e5560ccef5262bf15",
- "booter_load_ga100.bin": "b6e70d37fdbd23f3dee008165d9b3510e8faeb32a60a1689fb098e18d0c721bf",
- "gsp_boot_ga100.bin": "b80c6623f09df2f155ab335095575c637aef5e66cba6542e047422a6bab094e4",
- "gsp_boot_desc_ga100.bin": "9e7cca2cfdc5ce601b0adba39967850046108df6c2d23fff95578ca3f0ccbe01",
- "falcon_loader.bin": "f21f1cfbb8fffa8067685314367fc4c50f3f84dd99eda7c19db4292d9269b9d9"
+ "gsp_boot_desc_ga102.bin": "cc5a3fc1f4c47f2fe107b1b68f0176d8a448e6dec84b575e5560ccef5262bf15"
},
- "limitation": "Source and synthetic host verification only. GA100 direct launch and GA10x HS launch with the GA100 Booter are experimental until the hardware gates in docs/CMP-PORT-PLAN.md pass."
+ "limitation": "The manifest and generator pin/extract firmware artifacts and reproduce the reference V67 image. They do not implement an independent GSP boot/BooterLoad path or establish applicability to RTX 3090 / driver 615.71.09."
}
diff --git a/firmware/extract_boot_assets.py b/firmware/extract_boot_assets.py
index dddce7a..eb277a5 100644
--- a/firmware/extract_boot_assets.py
+++ b/firmware/extract_boot_assets.py
@@ -16,7 +16,14 @@ MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
MAGIC = 0x314C5442
-
+LABELS = (
+ "IMAGE_PROD",
+ "HEADER_PROD",
+ "SIG_PROD",
+ "PATCH_LOC",
+ "PATCH_META",
+ "NUM_SIGS",
+)
def _array_bytes(text: str, label: str) -> tuple[bytes, bool, int]:
@@ -89,9 +96,7 @@ def build_bundle(source_text: str) -> bytes:
PINNED = {
- 'g_bindata_ksec2GetBinArchiveBlUcode_TU102.c': 'fc0bc277eeee754ea3071ae0fe8360a52193565bba7c07dcea60b00e8d56d9c8',
- 'g_bindata_kgspGetBinArchiveBooterLoadUcode_GA100.c': '49fb8759c238ba32308ed604d5932d5cb2e0f496650063fbc064bbbdd4fdfaf2',
- 'g_bindata_kgspGetBinArchiveGspRmBoot_GA100.c': '9546d42c42181ec9b7ab7f9759c22180a5c31e22c592d5925a16fa85079e2700',
+ 'g_bindata_kgspGetBinArchiveBooterLoadUcode_GA102.c': '04cd3db7f670adc4efd79d2e76217bb809fffddfbd6f1af884cde2af26dd39af',
'g_bindata_kgspGetBinArchiveGspRmBoot_GA102.c': '35e5aa9075ad3898b6022ffd5820c953d809caa7dabd530cfc1a83dc2a67c5bd',
}
@@ -103,7 +108,7 @@ def extract_assets(source_root: Path, output: Path) -> None:
if hashlib.sha256(data).hexdigest() != digest:
raise ValueError('pinned generated source mismatch: ' + name)
inputs[name] = data.decode('ascii')
- booter = build_bundle(inputs['g_bindata_kgspGetBinArchiveBooterLoadUcode_GA100.c'])
+ booter = build_bundle(inputs['g_bindata_kgspGetBinArchiveBooterLoadUcode_GA102.c'])
boot = inputs['g_bindata_kgspGetBinArchiveGspRmBoot_GA102.c']
image = _array_bytes(boot, 'UCODE_IMAGE_PROD')[0]
desc = _array_bytes(boot, 'UCODE_DESC_PROD')[0]
@@ -115,14 +120,7 @@ def extract_assets(source_root: Path, output: Path) -> None:
for index in (1,3,8,10,12):
if not fields[index+1] or fields[index]+fields[index+1] > len(image):
raise ValueError('bootloader descriptor range outside image')
- blobs = {'booter_load_ga100.bin': booter, 'gsp_boot_ga102.bin': image, 'gsp_boot_desc_ga102.bin': desc}
- ga100_boot = inputs['g_bindata_kgspGetBinArchiveGspRmBoot_GA100.c']
- blobs['gsp_boot_ga100.bin'] = _array_bytes(ga100_boot, 'UCODE_IMAGE')[0]
- blobs['gsp_boot_desc_ga100.bin'] = _array_bytes(ga100_boot, 'UCODE_DESC')[0]
- loader = inputs['g_bindata_ksec2GetBinArchiveBlUcode_TU102.c']
- if not re.search(r'UCODE_DESC_data\s*=\s*\{\s*0xfd,\s*0,\s*\{\s*0x0,\s*0x200,\s*0x200,\s*0x100\s*}', loader):
- raise ValueError('unsupported generic loader descriptor')
- blobs['falcon_loader.bin'] = _array_bytes(loader, 'UCODE_IMAGE')[0]
+ blobs = {'booter_load_ga102.bin': booter, 'gsp_boot_ga102.bin': image, 'gsp_boot_desc_ga102.bin': desc}
pinned = json.loads(MANIFEST.read_text())['derived_assets']
for name, data in blobs.items():
if hashlib.sha256(data).hexdigest() != pinned[name]:
diff --git a/firmware/generate_v67.py b/firmware/generate_v67.py
index 4bc738f..3275a6f 100644
--- a/firmware/generate_v67.py
+++ b/firmware/generate_v67.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0-only
-"""Extract pinned firmware and generate the cmpunlocker built-in signature."""
+"""Extract pinned 610.43.03 GA10x firmware and generate the rejoin15 V67 image."""
from __future__ import annotations
@@ -13,14 +13,34 @@ from pathlib import Path
PACKAGE_SHA256 = "e17e7c92eeffa178b0fe1f40eb17a47095f8fc3ce4a77bf115cf5bd5a4df86e9"
-PAYLOAD_SHA256 = "00de1fab4e97b88216d0aac2abfbc8ff75f90fa3342ecc3216f9fed656bc1abd"
+GSP_SHA256 = "572373620a37418f24dc16b5031c39338778c3257e48e8408de9a57291b24f3a"
+PAYLOAD_SHA256 = "2a0aa1f86d0b6afff5214d641d87d46dc98ef06ed908ff05d0ddbad5ffb7e197"
PACKAGE_VERSION = "610.43.03-1"
+FIRMWARE_PREFIX = "usr/lib/firmware/nvidia/610.43.03/"
MANIFEST = Path(__file__).with_name("artifacts-610.43.03.json")
-PAYLOAD_SIZE = 0xF800
-FILL_DWORD = 0x000004A7
+
+PAYLOAD_SIZE = 0xFA00
+FILL_DWORD = 0x0000019C
PATCHED_WORDS = {
- int(item['offset'], 16): int(item['value'], 16)
- for item in json.loads(MANIFEST.read_text())['generated_payload']['overrides']
+ 0x1100: 0x00000007,
+ 0xF948: 0xFFFFFFFF,
+ 0xF950: 0x00000D44,
+ 0xF960: 0x00823804,
+ 0xF968: 0x00001FCE,
+ 0xF974: 0x00000000,
+ 0xF97C: 0x00001101,
+ 0xF980: 0x000084C8,
+ 0xF984: 0x00008E18,
+ 0xF98C: 0x000084C8,
+ 0xF990: 0x00000000,
+ 0xF998: 0x00001FCE,
+ 0xF9A4: 0x0000FFBC,
+ 0xF9AC: 0x00005789,
+ 0xF9BC: 0x00000D44,
+ 0xF9CC: 0x00000003,
+ 0xF9D4: 0x00001FCE,
+ 0xF9E8: 0x00000D52,
+ 0xF9EC: 0x000081EE,
}
@@ -65,30 +85,18 @@ def read_member(archive: tarfile.TarFile, name: str) -> bytes:
return result
-def build_payload(address: int = 0x00823804, value: int = 0xffffffff) -> bytes:
+def build_payload() -> bytes:
word = FILL_DWORD.to_bytes(4, "little")
payload = bytearray(word * (PAYLOAD_SIZE // len(word)))
- for offset, word_value in PATCHED_WORDS.items():
- payload[offset : offset + 4] = word_value.to_bytes(4, "little")
- payload[0xf754:0xf758] = value.to_bytes(4, "little")
- payload[0xf76c:0xf770] = address.to_bytes(4, "little")
+ for offset, value in PATCHED_WORDS.items():
+ payload[offset : offset + 4] = value.to_bytes(4, "little")
return bytes(payload)
-def payload_header() -> str:
- return ("/* SPDX-License-Identifier: GPL-2.0-only */\n"
- "/* Generated by firmware/generate_v67.py from the pinned cmpunlocker template. */\n"
- + "".join(f"GU_PAYLOAD_WORD(0x{offset:04x}, 0x{value:08x})\n"
- for offset, value in PATCHED_WORDS.items()
- if offset not in (0xf754, 0xf76c)))
-
-
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("package", type=Path, help="pinned nvidia-utils-610.43.03 package")
parser.add_argument("output", type=Path, help="directory for extracted firmware and payload")
- parser.add_argument("--header", type=Path, help="also reproduce the committed C template header")
- parser.add_argument("--payload", action="store_true", help="also emit the offline cmp-signature.bin reference image")
args = parser.parse_args()
package_bytes_hash = hashlib.sha256()
@@ -102,31 +110,22 @@ def main() -> int:
if f"pkgver = {PACKAGE_VERSION}" not in pkginfo.splitlines():
raise SystemExit("package metadata version does not match 610.43.03-1")
- firmware = {}
- for item in json.loads(MANIFEST.read_text())["source_artifacts"][2]["firmware"]:
- blob = extract_member(args.package, item["path"])
- if len(blob) != item["size"] or sha256(blob) != item["sha256"]:
- raise SystemExit("GSP firmware does not match the pinned artifact")
- firmware[Path(item["path"]).name] = blob
-
- payload = None
- if args.payload or args.header:
- payload = build_payload()
- if sha256(payload) != PAYLOAD_SHA256:
- raise SystemExit("generated payload differs from its pinned SHA-256")
+ gsp = extract_member(args.package, FIRMWARE_PREFIX + "gsp_ga10x.bin")
+ if sha256(gsp) != GSP_SHA256:
+ raise SystemExit("GA10x GSP firmware SHA-256 does not match the pinned artifact")
+
+ payload = build_payload()
+ if sha256(payload) != PAYLOAD_SHA256:
+ raise SystemExit("generated V67 image differs from its pinned SHA-256")
manifest = json.loads(MANIFEST.read_text())
notice = manifest["source_artifacts"][2]["notice"]
license_text = extract_member(args.package, notice["path"])
if sha256(license_text) != notice["sha256"]:
raise SystemExit("NVIDIA license notice differs from pinned package")
args.output.mkdir(parents=True, exist_ok=True)
- for name, blob in firmware.items():
- (args.output / name).write_bytes(blob)
- if args.payload:
- (args.output / "cmp-signature.bin").write_bytes(payload)
+ (args.output / "gsp_ga10x.bin").write_bytes(gsp)
+ (args.output / "v67-ga102-610.43.03.bin").write_bytes(payload)
(args.output / "NVIDIA-LICENSE").write_bytes(license_text)
- if args.header:
- args.header.write_text(payload_header())
return 0
diff --git a/kernel/gpu_unlock/Kbuild b/kernel/gpu_unlock/Kbuild
index eb493b8..1687c9f 100644
--- a/kernel/gpu_unlock/Kbuild
+++ b/kernel/gpu_unlock/Kbuild
@@ -1,4 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-m += gpu_unlock.o
gpu_unlock-y := main.o gfw_eval.o prep_610.o layout_610.o \
- vbios_610.o recipes.o payload_610.o firmware_610.o falcon.o unlock_610.o
+ vbios_610.o unlock_610.o
diff --git a/kernel/gpu_unlock/device.h b/kernel/gpu_unlock/device.h
deleted file mode 100644
index 07bf04c..0000000
--- a/kernel/gpu_unlock/device.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_DEVICE_H
-#define GPU_UNLOCK_DEVICE_H
-
-struct pci_dev;
-struct gu_card;
-struct gpu_unlock_610_fw_desc;
-
-/* Resolved once during enumeration; descriptors outlive boot/resume rounds.
- * The coordinator owns the PCI reference and keeps recipe and firmware
- * selection separate. Round-scoped firmware assets are never stored here. */
-struct gu_device {
- struct pci_dev *pdev;
- const struct gu_card *recipe;
- const struct gpu_unlock_610_fw_desc *firmware;
-};
-
-#endif
diff --git a/kernel/gpu_unlock/diagnostic.h b/kernel/gpu_unlock/diagnostic.h
index 06a7080..57aa7de 100644
--- a/kernel/gpu_unlock/diagnostic.h
+++ b/kernel/gpu_unlock/diagnostic.h
@@ -3,10 +3,12 @@
#define GPU_UNLOCK_DIAGNOSTIC_H
#ifdef __KERNEL__
#include <linux/printk.h>
-#define GU_NOTE(fmt, ...) pr_debug("gpu_unlock: " fmt, ##__VA_ARGS__)
+#define GU_NOTE(fmt, ...) pr_info("gpu_unlock: " fmt, ##__VA_ARGS__)
+#define GU_ERROR(fmt, ...) pr_err("gpu_unlock: " fmt, ##__VA_ARGS__)
#else
#include <stdio.h>
#define GU_NOTE(fmt, ...) fprintf(stderr, "gpu_unlock: " fmt, ##__VA_ARGS__)
+#define GU_ERROR(fmt, ...) fprintf(stderr, "gpu_unlock: " fmt, ##__VA_ARGS__)
#endif
/* No extra device reads. Preserve the errno and identify the exact guard in
@@ -15,7 +17,7 @@ static inline int gu_diagnostic_result(int error, const char *file,
const char *function, unsigned int line)
{
if (error)
- GU_NOTE("failure %s:%u %s error=%d\n", file, line, function, error);
+ GU_ERROR("failure %s:%u %s error=%d\n", file, line, function, error);
return error;
}
#define GU_DIAG(result) gu_diagnostic_result((result), __FILE__, __func__, __LINE__)
diff --git a/kernel/gpu_unlock/falcon.c b/kernel/gpu_unlock/falcon.c
deleted file mode 100644
index 53a9c23..0000000
--- a/kernel/gpu_unlock/falcon.c
+++ /dev/null
@@ -1,399 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Falcon register transport. Firmware policy and ownership belong to the caller. */
-#include <linux/delay.h>
-#include <linux/dma-mapping.h>
-#include <linux/device.h>
-#include <linux/string.h>
-#include <linux/timekeeping.h>
-
-#include "falcon.h"
-#include "prep_610.h"
-#include "registers.h"
-
-static int falcon_read(const struct gu_falcon_io *io, u32 offset, u32 *value)
-{
- return io->read(io->context, offset, value);
-}
-static void falcon_write(const struct gu_falcon_io *io, u32 offset, u32 value)
-{
- io->write(io->context, offset, value);
-}
-static bool inaccessible(u32 value)
-{
- return value == ~0u || (value & 0xffff0000u) == 0xbadf0000u;
-}
-
-int gu_falcon_poll(const struct gu_falcon_io *io, u32 offset, u32 mask, u32 expect,
- unsigned int budget_us, bool timeout_is_failure, u32 *last)
-{
- ktime_t deadline = ktime_add_us(ktime_get(), budget_us);
- u32 value = 0;
-
- for (;;) {
- int ret = falcon_read(io, offset, &value);
-
- if (last)
- *last = value;
- if (ret || value == ~0u ||
- (value & 0xffff0000u) == 0xbadf0000u) {
- dev_err(io->dev, "poll inaccessible: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
- offset, value, mask, expect, budget_us);
- return -EIO;
- }
- if ((value & mask) == expect)
- return 0;
- if (ktime_compare(ktime_get(), deadline) >= 0) {
- /* Console loglevel hides info. A tolerated timeout must not
- * look like the failure that stopped the boot. */
- if (!timeout_is_failure) {
- dev_dbg(io->dev, "poll timeout tolerated: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
- offset, value, mask, expect, budget_us);
- return -ETIMEDOUT;
- }
- dev_err(io->dev, "poll timeout: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
- offset, value, mask, expect, budget_us);
- return -ETIMEDOUT;
- }
- usleep_range(50, 100);
- }
-}
-
-int gu_falcon_engine_reset(const struct gu_falcon_io *io, u32 base,
- const char *phase, u32 *hwcfg_out,
- struct gu_falcon_evidence *evidence)
-{
- unsigned int j;
- u32 engine, before, first = 0, hwcfg, last = 0;
- int ret;
-
- memset(evidence, 0, sizeof(*evidence));
- if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
- return -ENODEV;
- before = engine;
- dev_dbg(io->dev, "engine reset assert: phase=%s base=%#x before=%#x write=%#x\n",
- phase, base, before, before | 1);
- falcon_write(io, base + GU_FALCON_ENGINE_OFFSET, engine | 1);
- /* 610 ksec2ResetHw_TU102 requires ten register cycles for signal
- * propagation. A stale zero during those cycles is not evidence that
- * the assertion failed. No CPU start and no reset retry.
- */
- for (j = 0; j < 10; j++) {
- if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) ||
- inaccessible(engine)) {
- dev_err(io->dev, "engine reset inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
- phase, base, j + 1, engine);
- return -EIO;
- }
- if (!j)
- first = engine;
- }
- dev_dbg(io->dev, "engine reset propagation: phase=%s base=%#x first=%#x tenth=%#x\n",
- phase, base, first, engine);
- ret = gu_falcon_poll(io, base + GU_FALCON_ENGINE_OFFSET, 1, 1, 1000, true, &last);
- dev_dbg(io->dev, "engine reset asserted: phase=%s base=%#x error=%d last=%#x\n",
- phase, base, ret, last);
- if (ret)
- return ret;
- evidence->reset_asserted = true;
- if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
- return -EIO;
- dev_dbg(io->dev, "engine reset release: phase=%s base=%#x before=%#x write=%#x\n",
- phase, base, engine, engine & ~1u);
- falcon_write(io, base + GU_FALCON_ENGINE_OFFSET, engine & ~1u);
- for (j = 0; j < 10; j++) {
- if (falcon_read(io, base + GU_FALCON_ENGINE_OFFSET, &engine) ||
- inaccessible(engine)) {
- dev_err(io->dev, "engine reset release inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
- phase, base, j + 1, engine);
- return -EIO;
- }
- }
- ret = gu_falcon_poll(io, base + GU_FALCON_ENGINE_OFFSET, 1, 0, 1000, true, NULL);
- if (ret)
- return ret;
- evidence->reset_released = true;
- /* NV_PFALCON_FALCON_HWCFG2 bit 12 clears when memory scrubbing finishes.
- * A PRI error fails this wait instead of being handed to RM.
- */
- ret = gu_falcon_poll(io, base + GU_FALCON_HWCFG2_OFFSET, 1u << 12, 0, 1000000, true, NULL);
- if (ret)
- return ret;
- if (falcon_read(io, base + GU_FALCON_HWCFG2_OFFSET, &hwcfg) || inaccessible(hwcfg))
- return -EIO;
- dev_dbg(io->dev, "engine reset released: phase=%s base=%#x engine=%#x hwcfg2=%#x\n",
- phase, base, engine, hwcfg);
- if (hwcfg_out)
- *hwcfg_out = hwcfg;
- return 0;
-}
-
-int gu_falcon_reset(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- struct gu_falcon_evidence *evidence)
-{
- u32 hwcfg, bcr, chip;
- int ret;
-
- memset(evidence, 0, sizeof(*evidence));
- /* 610 kflcnPreResetWait_GA102 permits this timeout (HW bug 3419321). */
- dev_dbg(io->dev, "Falcon reset begin: base=%#x\n", eng->base);
- ret = gu_falcon_poll(io, eng->base + GU_FALCON_HWCFG2_OFFSET,
- 1u << 31, 1u << 31, 1000, false, NULL);
- if (ret && ret != -ETIMEDOUT)
- return ret;
- if (ret)
- dev_dbg(io->dev, "Falcon pre-reset timeout tolerated per HW bug 3419321\n");
- ret = gu_falcon_engine_reset(io, eng->base, "pre-launch", &hwcfg, evidence);
- if (ret)
- return ret;
- if (hwcfg & (1u << 10)) {
- if (falcon_read(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET, &bcr) || inaccessible(bcr))
- return -ENODEV;
- if (bcr & (1u << 4)) {
- falcon_write(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET, 0);
- ret = gu_falcon_poll(io, eng->falcon2 + GU_RISCV_BCR_CTRL_OFFSET,
- 0x11, 1, 1000000, true, NULL);
- if (ret)
- return ret;
- }
- }
- if (falcon_read(io, GU_PMC_BOOT_0_OFFSET, &chip) || inaccessible(chip))
- return -ENODEV;
- falcon_write(io, eng->base + GU_FALCON_RM_OFFSET, chip);
- dev_dbg(io->dev, "Falcon reset done: base=%#x hwcfg=%#x chip=%#x\n", eng->base, hwcfg, chip);
- return 0;
-}
-
-/* DMATRFMOFFS is 24 bits, FBOFFS is 32 bits, and BASE is 49 bits.
- * Reject truncation/wrap before submitting any part of a transfer. */
-static bool falcon_dma_valid(u32 dest, u32 mem_off, dma_addr_t src, u32 size)
-{
- return size && !((size | dest | mem_off | src) & 255) &&
- (u64)dest + size <= (1ull << 24) &&
- (u64)mem_off + size <= (1ull << 32) &&
- src < (1ull << 49) &&
- (u64)mem_off + size <= (1ull << 49) - src;
-}
-
-int gu_falcon_dma(const struct gu_falcon_io *io, u32 base, u32 dest, u32 mem_off,
- dma_addr_t src, u32 size, u32 cmd)
-{
- u32 done = 0;
- int ret;
-
- dev_dbg(io->dev, "Falcon DMA: base=%#x dest=%#x offset=%#x src=%#llx size=%#x cmd=%#x\n",
- base, dest, mem_off, (unsigned long long)src, size, cmd);
- if (!falcon_dma_valid(dest, mem_off, src, size))
- return -EINVAL;
- ret = gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 1, 0, 1000000, true, NULL);
- if (ret)
- return ret;
- falcon_write(io, base + GU_FALCON_DMATRFBASE_OFFSET, lower_32_bits(src >> 8));
- falcon_write(io, base + GU_FALCON_DMATRFBASE1_OFFSET, upper_32_bits(src >> 8) & 0x1ff);
- while (done < size) {
- ret = gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 1, 0, 1000000, true, NULL);
- if (ret) {
- dev_err(io->dev, "Falcon DMA progress: done=%u total=%u\n", done, size);
- return ret;
- }
- falcon_write(io, base + GU_FALCON_DMATRFMOFFS_OFFSET, dest);
- falcon_write(io, base + GU_FALCON_DMATRFFBOFFS_OFFSET, mem_off);
- falcon_write(io, base + GU_FALCON_DMATRFCMD_OFFSET, cmd);
- done += 256;
- dest += 256;
- mem_off += 256;
- }
- return gu_falcon_poll(io, base + GU_FALCON_DMATRFCMD_OFFSET, 2, 2, 1000000, true, NULL);
-}
-
-int gu_falcon_hs(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- const struct gu_falcon_hs_image *program,
- struct gu_falcon_evidence *evidence)
-{
- u32 trans, cpuctl;
- dma_addr_t code_src;
- int ret;
-
- memset(evidence, 0, sizeof(*evidence));
- if (program->dma > ~0ull - program->code_off ||
- program->dma + program->code_off < program->code_va ||
- program->dma > ~0ull - program->data_off)
- return -ERANGE;
- code_src = program->dma + program->code_off - program->code_va;
- if (!falcon_dma_valid(program->code_pa, program->code_va, code_src, program->code_size) ||
- (program->data_size && !falcon_dma_valid(program->data_pa, 0,
- program->dma + program->data_off, program->data_size)))
- return -EINVAL;
- ret = gu_falcon_reset(io, eng, evidence);
- if (ret)
- return ret;
- ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 0, 1000, true, NULL);
- if (ret)
- return ret;
- if (falcon_read(io, eng->fbif + GU_FBIF_CTL_OFFSET, &trans) || inaccessible(trans))
- return -EIO;
- trans |= 1u << 7;
- falcon_write(io, eng->fbif + GU_FBIF_CTL_OFFSET, trans);
- falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, 0);
- if (falcon_read(io, eng->fbif + GU_FBIF_TRANSCFG(0), &trans) || inaccessible(trans))
- return -ENODEV;
- trans = (trans & ~7u) | 5u;
- falcon_write(io, eng->fbif + GU_FBIF_TRANSCFG(0), trans);
- dma_wmb();
- ret = gu_falcon_dma(io, eng->base, program->code_pa, program->code_va, code_src,
- program->code_size, gpu_unlock_dma_cmd(true));
- if (ret)
- return ret;
- if (program->data_size) {
- ret = gu_falcon_dma(io, eng->base, program->data_pa, 0,
- program->dma + program->data_off, program->data_size,
- gpu_unlock_dma_cmd(false));
- if (ret)
- return ret;
- }
- evidence->reloaded = true;
- falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_PARAADDR_OFFSET, program->sig_dmem);
- falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_ENGIDMASK_OFFSET, program->engine_mask);
- falcon_write(io, eng->falcon2 + GU_FALCON2_BROM_CURR_UCODE_ID_OFFSET, program->ucode_id & 0xff);
- falcon_write(io, eng->falcon2 + GU_FALCON2_MOD_SEL_OFFSET, 1);
- falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, program->code_va);
- falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, program->mailbox0);
- falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, program->mailbox1);
- if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &cpuctl) || inaccessible(cpuctl))
- return -EIO;
- dev_dbg(io->dev, "Falcon start: base=%#x cpuctl=%#x code_va=%#x sig_dmem=%#x engine_mask=%#x ucode=%u mailbox=%#x:%#x\n",
- eng->base, cpuctl, program->code_va, program->sig_dmem, program->engine_mask,
- program->ucode_id, program->mailbox1, program->mailbox0);
- evidence->started = true;
- if (cpuctl & (1u << 6))
- falcon_write(io, eng->base + GU_FALCON_CPUCTL_ALIAS_OFFSET, 1u << 1);
- else
- falcon_write(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 1u << 1);
-
- ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET,
- 1u << 4, 1u << 4, 5000000, true, NULL);
- if (ret)
- return ret;
- evidence->halted = true;
- return 0;
-}
-
-/* TU102/GA100 generic loader for board FWSEC v2. Descriptor offsets are
- * RM_FLCN_BL_DMEM_DESC (84 bytes); loader identity/geometry is pinned by the
- * extractor. The loader uses DMA index 4, coherent physical system memory.
- */
-int gu_falcon_loader(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- const struct gu_falcon_loader_image *program,
- struct gu_falcon_evidence *evidence)
-{
- u32 desc[21] = { 0 }, hwcfg, trans, ctl, i, dst;
- const u32 *code;
- int ret;
-
- memset(evidence, 0, sizeof(*evidence));
- if (!program->loader || program->loader_size != 768 ||
- (program->dma & 255) || program->dma > ~0ull - program->data_off ||
- (program->data_off & 255))
- return -EINVAL;
- code = program->loader;
- ret = gu_falcon_reset(io, eng, evidence);
- if (ret)
- return ret;
- if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl) || (ctl & 16) ||
- falcon_read(io, eng->base + GU_FALCON_HWCFG_OFFSET, &hwcfg) || inaccessible(hwcfg) ||
- falcon_read(io, eng->base + GU_FALCON_DMACTL_OFFSET, &trans) || inaccessible(trans))
- return -ENODEV;
- if ((hwcfg & 0x1ff) < 2)
- return -ERANGE;
- dst = ((hwcfg & 0x1ff) - 2) << 8;
- falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, trans & ~1u);
- if (falcon_read(io, eng->fbif + GU_FBIF_TRANSCFG(4), &trans) || inaccessible(trans))
- return -ENODEV;
- falcon_write(io, eng->fbif + GU_FBIF_TRANSCFG(4), (trans & ~7u) | 5u);
- desc[8] = 4;
- desc[9] = lower_32_bits(program->dma);
- desc[10] = upper_32_bits(program->dma);
- desc[11] = program->imem_pa;
- desc[12] = program->imem_ns_size;
- desc[13] = program->imem_sec_pa;
- desc[14] = program->imem_sec_size;
- desc[16] = lower_32_bits(program->dma + program->data_off);
- desc[17] = upper_32_bits(program->dma + program->data_off);
- desc[18] = program->dmem_size;
- falcon_write(io, eng->base + GU_FALCON_DMEMC_OFFSET, 1u << 24);
- for (i = 0; i < 21; i++)
- falcon_write(io, eng->base + GU_FALCON_DMEMD_OFFSET, desc[i]);
- falcon_write(io, eng->base + GU_FALCON_IMEMC_OFFSET, dst | (1u << 24));
- for (i = 0; i < 512; i += 4) {
- if (!(i & 255))
- falcon_write(io, eng->base + GU_FALCON_IMEMT_OFFSET, 0xfd + (i >> 8));
- falcon_write(io, eng->base + GU_FALCON_IMEMD_OFFSET, code[i / 4]);
- }
- evidence->reloaded = true;
- falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, 0xfd00);
- falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, 0);
- falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, 0);
- dma_wmb();
- evidence->started = true;
- falcon_write(io, eng->base + ((ctl & 64) ? GU_FALCON_CPUCTL_ALIAS_OFFSET :
- GU_FALCON_CPUCTL_OFFSET), 2);
- ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 16, 5000000, true, NULL);
- evidence->halted = !ret;
- return ret;
-}
-
-/* GA100's direct loader: nonsecure OS stub, tagged secure app, DMEM.
- * GA10x uses the HS DMA/BROM loader with the same signed GA100 image. Its
- * cross-family acceptance is an explicit hardware gate, not an assumption.
- */
-static void falcon_pio_imem(const struct gu_falcon_io *io, u32 base, const u32 *image,
- u32 offset, u32 size, bool secure)
-{
- u32 i;
-
- falcon_write(io, base + GU_FALCON_IMEMC_OFFSET, offset | (1u << 24) |
- (secure ? 1u << 28 : 0));
- for (i = 0; i < size; i += 4) {
- if (!(i & 255))
- falcon_write(io, base + GU_FALCON_IMEMT_OFFSET, (offset + i) >> 8);
- falcon_write(io, base + GU_FALCON_IMEMD_OFFSET, image[(offset + i) / 4]);
- }
-}
-
-int gu_falcon_direct(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- const struct gu_falcon_direct_image *program,
- struct gu_falcon_evidence *evidence)
-{
- const u32 *image = program->words;
- u32 i, ctl;
- int ret;
-
- memset(evidence, 0, sizeof(*evidence));
- ret = gu_falcon_reset(io, eng, evidence);
- if (ret)
- return ret;
- ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 0, 1000, true, NULL);
- if (ret)
- return ret;
- /* REQUIRE_CTX=false, matching kflcnDisableCtxReq. */
- if (falcon_read(io, eng->base + GU_FALCON_DMACTL_OFFSET, &ctl) || inaccessible(ctl))
- return -ENODEV;
- falcon_write(io, eng->base + GU_FALCON_DMACTL_OFFSET, ctl & ~1u);
- falcon_pio_imem(io, eng->base, image, program->os_code_off, program->os_code_size, false);
- falcon_pio_imem(io, eng->base, image, program->app_code_off, program->app_code_size, true);
- falcon_write(io, eng->base + GU_FALCON_DMEMC_OFFSET, 1u << 24);
- for (i = 0; i < program->os_data_size; i += 4)
- falcon_write(io, eng->base + GU_FALCON_DMEMD_OFFSET, image[(program->os_data_off + i) / 4]);
- evidence->reloaded = true;
- falcon_write(io, eng->base + GU_FALCON_BOOTVEC_OFFSET, 0);
- falcon_write(io, eng->base + GU_FALCON_MAILBOX0_OFFSET, program->mailbox0);
- falcon_write(io, eng->base + GU_FALCON_MAILBOX1_OFFSET, program->mailbox1);
- if (falcon_read(io, eng->base + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl))
- return -ENODEV;
- dma_wmb();
- evidence->started = true;
- falcon_write(io, eng->base + ((ctl & 64) ? GU_FALCON_CPUCTL_ALIAS_OFFSET :
- GU_FALCON_CPUCTL_OFFSET), 2);
- ret = gu_falcon_poll(io, eng->base + GU_FALCON_CPUCTL_OFFSET, 16, 16, 5000000, true, NULL);
- evidence->halted = !ret;
- return ret;
-}
diff --git a/kernel/gpu_unlock/falcon.h b/kernel/gpu_unlock/falcon.h
deleted file mode 100644
index d69df1a..0000000
--- a/kernel/gpu_unlock/falcon.h
+++ /dev/null
@@ -1,62 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_FALCON_H
-#define GPU_UNLOCK_FALCON_H
-
-#include <linux/types.h>
-
-struct device;
-
-/* Borrowed hardware access only: no PCI, allocation or transaction ownership. */
-struct gu_falcon_io {
- struct device *dev;
- void *context;
- int (*read)(void *context, u32 offset, u32 *value);
- void (*write)(void *context, u32 offset, u32 value);
-};
-struct gu_falcon_engine { u32 base, fbif, falcon2; };
-
-/* Reset for every operation, including validation failures. Available on error.
- * started records a submitted start; halted records a successful halt wait.
- * Neither proves firmware-specific success, cleanup or safe DMA disposal. */
-struct gu_falcon_evidence {
- bool reset_asserted, reset_released, reloaded, started, halted;
-};
-
-/* Image spans/firmware geometry are validated by preparation before calling.
- * Transport additionally rejects unrepresentable HS/DMA addresses before reset. */
-struct gu_falcon_hs_image {
- dma_addr_t dma;
- u32 code_off, code_size, code_va, code_pa;
- u32 data_off, data_size, data_pa;
- u32 sig_dmem, engine_mask, ucode_id;
- u32 mailbox0, mailbox1;
-};
-struct gu_falcon_direct_image {
- const u32 *words;
- u32 os_code_off, os_code_size, app_code_off, app_code_size;
- u32 os_data_off, os_data_size;
- u32 mailbox0, mailbox1;
-};
-struct gu_falcon_loader_image {
- const u32 *loader;
- size_t loader_size;
- dma_addr_t dma;
- u32 data_off, imem_pa, imem_ns_size, imem_sec_pa, imem_sec_size, dmem_size;
-};
-
-int gu_falcon_poll(const struct gu_falcon_io *io, u32 offset, u32 mask, u32 expect,
- unsigned int budget_us, bool timeout_is_failure, u32 *last);
-int gu_falcon_dma(const struct gu_falcon_io *io, u32 base, u32 dest, u32 mem_off,
- dma_addr_t src, u32 size, u32 cmd);
-int gu_falcon_engine_reset(const struct gu_falcon_io *io, u32 base, const char *phase,
- u32 *hwcfg, struct gu_falcon_evidence *evidence);
-int gu_falcon_reset(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- struct gu_falcon_evidence *evidence);
-int gu_falcon_hs(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- const struct gu_falcon_hs_image *image, struct gu_falcon_evidence *evidence);
-int gu_falcon_direct(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- const struct gu_falcon_direct_image *image, struct gu_falcon_evidence *evidence);
-int gu_falcon_loader(const struct gu_falcon_io *io, const struct gu_falcon_engine *eng,
- const struct gu_falcon_loader_image *image, struct gu_falcon_evidence *evidence);
-
-#endif
diff --git a/kernel/gpu_unlock/firmware_610.c b/kernel/gpu_unlock/firmware_610.c
deleted file mode 100644
index ee50593..0000000
--- a/kernel/gpu_unlock/firmware_610.c
+++ /dev/null
@@ -1,218 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Firmware format validation and immutable assets for one coordination round. */
-#include <linux/firmware.h>
-#include <linux/module.h>
-#include <linux/pci.h>
-#include <linux/slab.h>
-#include <crypto/sha2.h>
-
-#include "diagnostic.h"
-#include "firmware_610.h"
-#include "recipes.h"
-
-#define BOOTER_MAGIC 0x314c5442u
-#define SEC2_BOOTER_FILE "gpu-unlock/610.43.03/booter_load_ga100.bin"
-#define GSP_FILE "gpu-unlock/610.43.03/gsp_ga10x.bin"
-#define GSP_BOOT_FILE "gpu-unlock/610.43.03/gsp_boot_ga102.bin"
-#define GSP_BOOT_DESC_FILE "gpu-unlock/610.43.03/gsp_boot_desc_ga102.bin"
-
-#define GSP_GA100_FILE "gpu-unlock/610.43.03/gsp_tu10x.bin"
-#define BOOT_GA100_FILE "gpu-unlock/610.43.03/gsp_boot_ga100.bin"
-#define BOOT_DESC_GA100_FILE "gpu-unlock/610.43.03/gsp_boot_desc_ga100.bin"
-#define FALCON_LOADER_FILE "gpu-unlock/610.43.03/falcon_loader.bin"
-static const struct gpu_unlock_610_fw_desc ga100_load = {
- .gsp = GSP_GA100_FILE, .boot = BOOT_GA100_FILE, .desc = BOOT_DESC_GA100_FILE,
- .signature_section = ".fwsignature_ga100", .boot_size = 4096, .direct = true,
-};
-static const struct gpu_unlock_610_fw_desc ga10x_load = {
- .gsp = GSP_FILE, .boot = GSP_BOOT_FILE, .desc = GSP_BOOT_DESC_FILE,
- .signature_section = ".fwsignature_ga10x", .boot_size = 24576, .direct = false,
-};
-
-const struct gpu_unlock_610_fw_desc *
-gpu_unlock_610_select_firmware(const struct gu_card *recipe)
-{
- if (!recipe)
- return NULL;
- return recipe->direct_boot ? &ga100_load : &ga10x_load;
-}
-
-/* CONFIG_FW_CACHE attaches name records to devres even though this is not
- * devm_request_firmware(). release_firmware() does not remove them, and
- * really_probe() refuses a device with any existing devres. Scope each
- * synchronous request so only its cache records are released; the returned
- * firmware remains owned by the caller.
- */
-static int request_scoped(struct device *dev, const struct firmware **fw, const char *name)
-{
- void *group;
- int ret, released;
-
- group = devres_open_group(dev, NULL, GFP_KERNEL);
- if (!group) {
- dev_err(dev, "asset request failed: %s error=%d\n", name, -ENOMEM);
- return -ENOMEM;
- }
- dev_dbg(dev, "asset request: %s\n", name);
- ret = request_firmware(fw, name, dev);
- released = devres_release_group(dev, group);
- if (ret)
- dev_err(dev, "asset request failed: %s error=%d\n", name, ret);
- dev_dbg(dev, "asset result: %s error=%d bytes=%zu\n",
- name, ret, ret ? (size_t)0 : (*fw)->size);
- if (!ret) {
- u8 digest[SHA256_DIGEST_SIZE];
- sha256((*fw)->data, (*fw)->size, digest);
- dev_dbg(dev, "asset identity: %s sha256=%*phN\n",
- name, SHA256_DIGEST_SIZE, digest);
- }
- dev_dbg(dev, "asset cache cleanup: %s devres_released=%d\n", name, released);
- return ret;
-}
-
-static int validate_booter(const struct firmware *fw, struct gpu_unlock_booter_hdr *out)
-{
- const u8 *data;
- u32 magic, version, image_size, sig_bytes;
- struct gpu_unlock_booter_hdr hdr;
-
- data = fw->data;
- if (fw->size < 72)
- return GU_DIAG(-EINVAL);
- memcpy(&magic, data, sizeof(magic));
- memcpy(&version, data + 4, sizeof(version));
- if (magic != BOOTER_MAGIC || version != 1)
- return GU_DIAG(-EINVAL);
- memcpy(&hdr, data + 8, sizeof(hdr));
- memcpy(&image_size, data + 8 + sizeof(hdr), sizeof(image_size));
- memcpy(&sig_bytes, data + 12 + sizeof(hdr), sizeof(sig_bytes));
- if (!hdr.ucode_id || hdr.num_apps != 1 || sig_bytes != hdr.sig_bytes ||
- image_size != GU_BOOTER_IMAGE_SIZE ||
- fw->size != 72u + image_size + sig_bytes)
- return GU_DIAG(-EINVAL);
- /* Pinned signed GA100 Booter geometry, used by both launch paths. */
- if (hdr.ucode_id != 3 || hdr.engine_id != 1 || hdr.num_sigs != 1 ||
- sig_bytes != GU_BOOTER_SIG_TOTAL || hdr.os_code_off != 0 ||
- hdr.os_code_size != 256 || hdr.os_data_off != 34560 ||
- hdr.os_data_size != 25600 || hdr.app_code_off != 256 ||
- hdr.app_code_size != 34304 || hdr.app_data_off != 256 ||
- hdr.app_data_size != 0 || hdr.patch_loc != 35072)
- return GU_DIAG(-EINVAL);
- *out = hdr;
- return 0;
-}
-
-static void release_family(struct gpu_unlock_610_firmware *assets)
-{
- release_firmware(assets->boot);
- release_firmware(assets->gsp);
- memset(assets, 0, sizeof(*assets));
-}
-
-static int prepare_family(struct pci_dev *pdev, struct gpu_unlock_610_firmware *assets,
- const struct gpu_unlock_610_fw_desc *load)
-{
- const struct firmware *desc = NULL;
- int ret;
-
- ret = request_scoped(&pdev->dev, &assets->gsp,
- load->gsp);
- if (ret)
- return ret;
- ret = gpu_unlock_elf_section(assets->gsp->data, assets->gsp->size,
- ".fwimage", &assets->image_span);
- if (ret)
- return ret;
- ret = gpu_unlock_elf_section(assets->gsp->data, assets->gsp->size,
- load->signature_section, &assets->sig_span);
- if (ret)
- return ret;
- if (!assets->image_span.size || assets->image_span.size > 128ull << 20 ||
- assets->sig_span.size != 4096)
- return -EINVAL;
- ret = gpu_unlock_radix_plan(assets->image_span.size, &assets->radix);
- if (ret)
- return ret;
- ret = request_scoped(&pdev->dev, &assets->boot, load->boot);
- if (ret)
- return ret;
- if (assets->boot->size != load->boot_size)
- return -EINVAL;
- ret = request_scoped(&pdev->dev, &desc, load->desc);
- if (ret)
- return ret;
- ret = gpu_unlock_boot_desc_parse(desc->data, desc->size,
- assets->boot->size, &assets->boot_desc);
- dev_dbg(&pdev->dev, "asset geometry: image=%llu signature=%llu boot=%u code=%#x data=%#x manifest=%#x parse=%d\n",
- (unsigned long long)assets->image_span.size, (unsigned long long)assets->sig_span.size,
- assets->boot_desc.image_size, assets->boot_desc.code_offset,
- assets->boot_desc.data_offset, assets->boot_desc.manifest_offset, ret);
- release_firmware(desc);
- return ret;
-}
-
-int gpu_unlock_610_load_shared(const struct gu_device *device, struct gpu_unlock_610_shared *shared)
-{
- struct pci_dev *pdev = device->pdev;
- const struct gpu_unlock_610_fw_desc *load = device->firmware;
- struct gpu_unlock_610_firmware *assets;
- const struct firmware *booter = NULL, *loader = NULL;
- int ret;
-
- if (!device->recipe || !load)
- return -ENODEV;
- assets = load->direct ? &shared->ga100 : &shared->ga10x;
- if (assets->prepared)
- return assets->error;
- ret = prepare_family(pdev, assets, load);
- if (ret)
- goto out;
- if (!shared->booter) {
- ret = request_scoped(&pdev->dev, &booter, SEC2_BOOTER_FILE);
- if (ret)
- goto out;
- ret = validate_booter(booter, &shared->booter_hdr);
- if (ret) {
- dev_err(&pdev->dev, "invalid SEC2 Booter bundle: error=%d\n", ret);
- release_firmware(booter);
- goto out;
- }
- shared->booter = booter;
- }
- if (load->direct && !shared->loader) {
- ret = request_scoped(&pdev->dev, &loader, FALCON_LOADER_FILE);
- if (ret)
- goto out;
- if (loader->size != 768) {
- dev_err(&pdev->dev, "invalid generic Falcon loader: bytes=%zu\n", loader->size);
- release_firmware(loader);
- ret = -EINVAL;
- goto out;
- }
- shared->loader = loader;
- }
-out:
- if (ret)
- release_family(assets);
- assets->error = ret;
- assets->prepared = true;
- return ret;
-}
-
-void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
-{
- release_family(&shared->ga100);
- release_family(&shared->ga10x);
- release_firmware(shared->loader);
- release_firmware(shared->booter);
- memset(shared, 0, sizeof(*shared));
-}
-
-MODULE_FIRMWARE(SEC2_BOOTER_FILE);
-MODULE_FIRMWARE(FALCON_LOADER_FILE);
-MODULE_FIRMWARE(GSP_FILE);
-MODULE_FIRMWARE(GSP_BOOT_FILE);
-MODULE_FIRMWARE(GSP_BOOT_DESC_FILE);
-MODULE_FIRMWARE(GSP_GA100_FILE);
-MODULE_FIRMWARE(BOOT_GA100_FILE);
-MODULE_FIRMWARE(BOOT_DESC_GA100_FILE);
diff --git a/kernel/gpu_unlock/firmware_610.h b/kernel/gpu_unlock/firmware_610.h
deleted file mode 100644
index 4550936..0000000
--- a/kernel/gpu_unlock/firmware_610.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_FIRMWARE_610_H
-#define GPU_UNLOCK_FIRMWARE_610_H
-
-#include "prep_610.h"
-#include "device.h"
-
-struct firmware;
-/* Immutable family format/launch descriptor, selected only for a known recipe. */
-struct gpu_unlock_610_fw_desc {
- const char *gsp, *boot, *desc, *signature_section;
- u32 boot_size;
- bool direct;
-};
-
-const struct gpu_unlock_610_fw_desc *
-gpu_unlock_610_select_firmware(const struct gu_card *recipe);
-
-/* Immutable after preparation, including decoded metadata. Owns gsp and boot. */
-struct gpu_unlock_610_firmware {
- const struct firmware *gsp, *boot;
- struct gpu_unlock_span image_span, sig_span;
- struct gpu_unlock_boot_desc boot_desc;
- struct gpu_unlock_radix_plan radix;
- bool prepared;
- int error;
-};
-
-/* One coordination round. Common images are shared by both firmware families. */
-struct gpu_unlock_610_shared {
- struct gpu_unlock_610_firmware ga100, ga10x;
- const struct firmware *booter, *loader;
- struct gpu_unlock_booter_hdr booter_hdr;
-};
-
-/* Serial preparation only; no workers may run until all calls have returned.
- * Cache failures for this round without invalidating another family's assets.
- * A fresh round starts after release_shared(), including on resume. */
-int gpu_unlock_610_load_shared(const struct gu_device *device, struct gpu_unlock_610_shared *shared);
-void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared);
-
-#endif
diff --git a/kernel/gpu_unlock/gfw_eval.h b/kernel/gpu_unlock/gfw_eval.h
index 81d853c..f721971 100644
--- a/kernel/gpu_unlock/gfw_eval.h
+++ b/kernel/gpu_unlock/gfw_eval.h
@@ -2,8 +2,6 @@
#ifndef GPU_UNLOCK_GFW_EVAL_H
#define GPU_UNLOCK_GFW_EVAL_H
-#include "registers.h"
-
#ifdef __KERNEL__
#include <linux/types.h>
#else
@@ -18,7 +16,13 @@ typedef uint32_t u32;
* dev_gc6_island.h, dev_gc6_island_addendum.h, dev_gsp.h, dev_falcon_v4.h.
* GFW completion uses kflcnWaitForHalt_TU102 (Falcon CPUCTL), not RISC-V CPUCTL.
*/
+#define GU_GFW_PLM_OFFSET 0x00118128u
+#define GU_GFW_BOOT_OFFSET 0x00118234u
+#define GU_GFW_CPUCTL_OFFSET 0x00110100u
#define GU_GFW_TIMEOUT_US 2050000u
+#define GU_FEAT_PLM_OFFSET 0x00823804u
+#define GU_FEAT_SS0_OFFSET 0x0082381cu
+#define GU_FEAT_SS1_OFFSET 0x00823820u
#define GU_GFW_PLM_LEVEL0 0x1u
#define GU_GFW_BOOT_COMPLETED 0xffu
#define GU_GFW_CPUCTL_HALTED 0x10u
diff --git a/kernel/gpu_unlock/main.c b/kernel/gpu_unlock/main.c
index df17f04..0a6b950 100644
--- a/kernel/gpu_unlock/main.c
+++ b/kernel/gpu_unlock/main.c
@@ -9,23 +9,41 @@
#include <linux/suspend.h>
#include "unlock_610.h"
-#include "recipes.h"
static char *targets;
module_param(targets, charp, 0444);
-MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of supported Ampere cards to unlock");
+MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of GA102/GA103/GA104/GA106 cards to unlock");
static bool resume = true;
module_param(resume, bool, 0644);
MODULE_PARM_DESC(resume, "Unlock the cards again after resume from suspend, which powers them off (default: on)");
+/* GA102, GA103, GA104 and GA106 GPU functions in pci.ids, excluding audio
+ * functions and the professional (GL) parts, which already run BF16 at full
+ * rate. GA103 and GA106 also exclude mobile parts.
+ */
+static const unsigned short supported_ids[] = {
+ /* GA102 */
+ 0x2200, 0x2203, 0x2204, 0x2205, 0x2206, 0x2207, 0x2208, 0x220a,
+ 0x220d, 0x2216, 0x222b, 0x222f,
+ /* GA104 */
+ 0x2480, 0x2482, 0x2483, 0x2484, 0x2486, 0x2487, 0x2488, 0x2489,
+ 0x248a, 0x248c, 0x248d, 0x248e, 0x249c, 0x249d, 0x249f, 0x24a0,
+ 0x24a4, 0x24ac, 0x24ad, 0x24af, 0x24bf, 0x24c0, 0x24c7, 0x24c8,
+ 0x24c9, 0x24dc, 0x24dd, 0x24df, 0x24e0,
+ /* GA103 (untested) */
+ 0x2414,
+ /* GA106 (validated only on RTX 3060 12GB 0x2504) */
+ 0x2501, 0x2503, 0x2504, 0x2505, 0x2507, 0x2508, 0x2509, 0x252f,
+ 0x2544,
+};
+
/* A target card. Those unlocked at boot stay listed, with a reference, to be
* unlocked again after each resume. */
struct card {
struct list_head node;
- struct gu_device device;
+ struct pci_dev *pdev;
bool unlocked;
- bool prepared;
};
static LIST_HEAD(cards);
@@ -33,13 +51,15 @@ static ASYNC_DOMAIN_EXCLUSIVE(unlock_domain);
static struct gpu_unlock_610_shared shared;
/* The next supported NVIDIA card after from, like pci_get_device(). */
-static struct pci_dev *next_supported(struct pci_dev *from, const struct gu_card **recipe)
+static struct pci_dev *next_supported(struct pci_dev *from)
{
struct pci_dev *pdev = from;
+ unsigned int i;
while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, pdev)))
- if ((*recipe = gu_card_find(pdev->device)))
- return pdev;
+ for (i = 0; i < ARRAY_SIZE(supported_ids); i++)
+ if (pdev->device == supported_ids[i])
+ return pdev;
return NULL;
}
@@ -47,11 +67,11 @@ static struct pci_dev *next_supported(struct pci_dev *from, const struct gu_card
static void unlock_one(void *data, async_cookie_t cookie)
{
struct card *card = data;
- struct pci_dev *pdev = card->device.pdev;
+ struct pci_dev *pdev = card->pdev;
int ret;
pci_dev_lock(pdev);
- ret = pdev->driver ? -EBUSY : gpu_unlock_610_unlock(&card->device, &shared, false);
+ ret = pdev->driver ? -EBUSY : gpu_unlock_610_unlock(pdev, &shared, false);
pci_dev_unlock(pdev);
if (ret)
pci_err(pdev, "gpu_unlock: unlock failed (%d)\n", ret);
@@ -65,14 +85,14 @@ static void unlock_one(void *data, async_cookie_t cookie)
static void unlock_again(void *data, async_cookie_t cookie)
{
struct card *card = data;
- struct pci_dev *pdev = card->device.pdev;
+ struct pci_dev *pdev = card->pdev;
bool nvidia;
int ret = 0;
pci_dev_lock(pdev);
nvidia = pdev->driver && !strcmp(pdev->driver->name, "nvidia");
if (nvidia)
- ret = gpu_unlock_610_unlock(&card->device, &shared, true);
+ ret = gpu_unlock_610_unlock(pdev, &shared, true);
pci_dev_unlock(pdev);
if (!nvidia)
pci_info(pdev, "gpu_unlock: not bound to nvidia, left as is after resume\n");
@@ -84,22 +104,21 @@ static void unlock_again(void *data, async_cookie_t cookie)
pci_info(pdev, "gpu_unlock: unlocked again after resume\n");
}
-/* Prepare all selected firmware families, then run the ready cards concurrently. */
+/* Unlock every listed card concurrently, with the GSP firmware loaded once. */
static void unlock_cards(async_func_t unlock)
{
struct card *card;
int ret;
list_for_each_entry(card, &cards, node) {
- ret = gpu_unlock_610_load_shared(&card->device, &shared);
- card->prepared = !ret;
- if (ret)
- pci_err(card->device.pdev, "gpu_unlock: firmware load failed (%d)\n", ret);
- }
- /* Publish the completed cache only after every selected family is prepared. */
- list_for_each_entry(card, &cards, node) {
- if (card->prepared)
- async_schedule_domain(unlock, card, &unlock_domain);
+ if (!shared.gsp) {
+ ret = gpu_unlock_610_load_shared(card->pdev, &shared);
+ if (ret) {
+ pci_err(card->pdev, "gpu_unlock: firmware load failed (%d)\n", ret);
+ break;
+ }
+ }
+ async_schedule_domain(unlock, card, &unlock_domain);
}
async_synchronize_full_domain(&unlock_domain);
gpu_unlock_610_release_shared(&shared);
@@ -108,7 +127,7 @@ static void unlock_cards(async_func_t unlock)
static void drop_card(struct card *card)
{
list_del(&card->node);
- pci_dev_put(card->device.pdev);
+ pci_dev_put(card->pdev);
kfree(card);
}
@@ -142,21 +161,18 @@ static bool listed(const char *name)
static void report_unmatched(void)
{
const char *entry, *end;
- struct card *card;
- bool matched;
+ struct pci_dev *pdev;
for (entry = targets; *entry; entry = *end ? end + 1 : end) {
end = strchrnul(entry, ',');
- matched = false;
- list_for_each_entry(card, &cards, node) {
- const char *name = pci_name(card->device.pdev);
-
- if (strlen(name) == end - entry && !strncasecmp(name, entry, end - entry)) {
- matched = true;
+ pdev = NULL;
+ while ((pdev = next_supported(pdev)))
+ if (strlen(pci_name(pdev)) == end - entry &&
+ !strncasecmp(pci_name(pdev), entry, end - entry))
break;
- }
- }
- if (!matched)
+ if (pdev)
+ pci_dev_put(pdev);
+ else
pr_err("gpu_unlock: no supported card at \"%.*s\"\n",
(int)(end - entry), entry);
}
@@ -166,30 +182,25 @@ static int __init gpu_unlock_init(void)
{
struct pci_dev *pdev = NULL;
struct card *card, *next;
- const struct gu_card *recipe;
- bool all, enumerated = true;
+ bool all;
/* Loaded ahead of nvidia on every boot; the command line is the opt-in. */
if (!targets || !*targets)
return 0;
all = !strcmp(targets, "all");
- while ((pdev = next_supported(pdev, &recipe))) {
+ if (!all)
+ report_unmatched();
+ while ((pdev = next_supported(pdev))) {
if (!all && !listed(pci_name(pdev)))
continue;
card = kzalloc(sizeof(*card), GFP_KERNEL);
if (!card) {
- pci_err(pdev, "gpu_unlock: card allocation failed\n");
- enumerated = false;
pci_dev_put(pdev);
break;
}
- card->device.pdev = pci_dev_get(pdev);
- card->device.recipe = recipe;
- card->device.firmware = gpu_unlock_610_select_firmware(recipe);
+ card->pdev = pci_dev_get(pdev);
list_add_tail(&card->node, &cards);
}
- if (!all && enumerated)
- report_unmatched();
/* nvidia loads once this returns: every card must be finished by then. */
unlock_cards(unlock_one);
list_for_each_entry_safe(card, next, &cards, node)
@@ -213,5 +224,5 @@ static void __exit gpu_unlock_exit(void)
module_init(gpu_unlock_init);
module_exit(gpu_unlock_exit);
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Ampere pre-driver hardware unlock");
+MODULE_DESCRIPTION("Ampere pre-driver BF16 unlock");
MODULE_VERSION("1.0.0");
diff --git a/kernel/gpu_unlock/payload_610.c b/kernel/gpu_unlock/payload_610.c
deleted file mode 100644
index 527f2a2..0000000
--- a/kernel/gpu_unlock/payload_610.c
+++ /dev/null
@@ -1,19 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include "payload_610.h"
-static void put32(u8 *image, u32 offset, u32 value)
-{
- unsigned int i;
- for (i = 0; i < 4; i++)
- image[offset + i] = value >> (8 * i);
-}
-void gu_payload_fill(u8 *image, u32 address, u32 value)
-{
- u32 offset;
- for (offset = 0; offset < GU_PAYLOAD_SIZE; offset += 4)
- put32(image, offset, 0x4a7);
-#define GU_PAYLOAD_WORD(offset, value) put32(image, offset, value);
-#include "payload_words.h"
-#undef GU_PAYLOAD_WORD
- put32(image, 0xf754, value);
- put32(image, 0xf76c, address);
-}
diff --git a/kernel/gpu_unlock/payload_610.h b/kernel/gpu_unlock/payload_610.h
deleted file mode 100644
index c35365f..0000000
--- a/kernel/gpu_unlock/payload_610.h
+++ /dev/null
@@ -1,6 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_PAYLOAD_610_H
-#define GPU_UNLOCK_PAYLOAD_610_H
-#include "prep_610.h"
-void gu_payload_fill(u8 *image, u32 address, u32 value);
-#endif
diff --git a/kernel/gpu_unlock/payload_words.h b/kernel/gpu_unlock/payload_words.h
deleted file mode 100644
index 5acec74..0000000
--- a/kernel/gpu_unlock/payload_words.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Generated by firmware/generate_v67.py from the pinned cmpunlocker template. */
-GU_PAYLOAD_WORD(0x1100, 0x00000007)
-GU_PAYLOAD_WORD(0x5b40, 0xc0deca7e)
-GU_PAYLOAD_WORD(0xf758, 0xc0deca7e)
-GU_PAYLOAD_WORD(0xf75c, 0x00000cbd)
-GU_PAYLOAD_WORD(0xf774, 0x00001fbd)
-GU_PAYLOAD_WORD(0xf780, 0x00000000)
-GU_PAYLOAD_WORD(0xf788, 0x000010aa)
-GU_PAYLOAD_WORD(0xf78c, 0x0000815a)
-GU_PAYLOAD_WORD(0xf790, 0x00008e18)
-GU_PAYLOAD_WORD(0xf794, 0xc0deca7e)
-GU_PAYLOAD_WORD(0xf798, 0x0000815a)
-GU_PAYLOAD_WORD(0xf79c, 0x00000000)
-GU_PAYLOAD_WORD(0xf7a0, 0xc0deca7e)
-GU_PAYLOAD_WORD(0xf7a4, 0x00001fbd)
-GU_PAYLOAD_WORD(0xf7b0, 0x0000ffbc)
-GU_PAYLOAD_WORD(0xf7b8, 0x0000582d)
-GU_PAYLOAD_WORD(0xf7c4, 0xc0deca7e)
-GU_PAYLOAD_WORD(0xf7c8, 0x00000cbd)
-GU_PAYLOAD_WORD(0xf7d8, 0x00000003)
-GU_PAYLOAD_WORD(0xf7e0, 0x00001fbd)
-GU_PAYLOAD_WORD(0xf7f4, 0x00000ccb)
-GU_PAYLOAD_WORD(0xf7f8, 0x00007f2f)
diff --git a/kernel/gpu_unlock/prep_610.c b/kernel/gpu_unlock/prep_610.c
index f0e673e..6013f57 100644
--- a/kernel/gpu_unlock/prep_610.c
+++ b/kernel/gpu_unlock/prep_610.c
@@ -164,22 +164,6 @@ int gpu_unlock_boot_desc_parse(const u8 *data, size_t size, u32 image_size,
static const unsigned int ranges[] = { 1, 3, 8, 10, 12 };
unsigned int i;
- /* GA100's unchanged v4 monitor descriptor has no manifest or image-size
- * fields. Keep this exact pinned geometry distinct from the v5 parser. */
- if (data && out && size == 76 && image_size == 4096 && rd32(data) == 4) {
- unsigned int j;
- if (rd32(data + 4) || rd32(data + 8) != 1160 ||
- rd32(data + 12) != 1160 || rd32(data + 16) != 16)
- return GU_DIAG(-EINVAL);
- for (j = 5; j < 19; j++)
- if (rd32(data + j * 4))
- return GU_DIAG(-EINVAL);
- *out = (struct gpu_unlock_boot_desc) {
- .code_offset = 0, .data_offset = 1160,
- .manifest_offset = 0, .image_size = image_size,
- };
- return 0;
- }
if (!data || !out || size != 84 || !image_size ||
rd32(data) != 5 || rd32(data + 56) != 1 ||
rd32(data + 76) != image_size || rd32(data + 80))
diff --git a/kernel/gpu_unlock/prep_610.h b/kernel/gpu_unlock/prep_610.h
index 086b9f2..273f345 100644
--- a/kernel/gpu_unlock/prep_610.h
+++ b/kernel/gpu_unlock/prep_610.h
@@ -17,12 +17,13 @@ typedef uint64_t u64;
#include "layout_610.h"
-#define GU_BOOTER_IMAGE_SIZE 60160u
-#define GU_BOOTER_SIG_TOTAL 384u
-#define GU_PAYLOAD_SIZE 0xf800u
+#define GU_BOOTER_IMAGE_SIZE 60416u
+#define GU_BOOTER_SIG_TOTAL 768u
+#define GU_V67_SIZE 0xfa00u
#define GU_WPR_META_SIZE 256u
#define GU_WPR_META_MAGIC 0xdc3aae21371a60b3ull
#define GU_RADIX_PAGE 4096u
+#define GU_FUSE_SEC2_UCODE1 0x00824140u
struct gpu_unlock_booter_hdr {
u32 os_code_off;
diff --git a/kernel/gpu_unlock/recipes.c b/kernel/gpu_unlock/recipes.c
deleted file mode 100644
index 721fc64..0000000
--- a/kernel/gpu_unlock/recipes.c
+++ /dev/null
@@ -1,68 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include "recipes.h"
-#include "registers.h"
-
-/* cmpunlocker 2d10f891, sec2-postbl-plm-ss-cfg.patch, plmTable.
- * Order and WPR_CFG's source-control bits are significant. No geometry,
- * link retraining, ECC or GPC/TPC configuration is performed in Stage 1.
- */
-/* Only the masks needed by the GA10x compute recipe. */
-static const struct gu_reg_write ga10x_plm[] = {
- { GU_WPR_CFG_PLM_OFFSET, 0xfffff0ff },
- { GU_FEAT_PLM_OFFSET, 0xffffffff },
-};
-static const struct gu_reg_write ga100_plm[] = {
- { GU_WPR_CFG_PLM_OFFSET, 0xfffff0ff },
- { GU_FBPA_PLM_OFFSET, 0xffffffff },
- { GU_WPR_PLM_OFFSET, 0xffffffff },
- { GU_FEAT_PLM_OFFSET, 0xffffffff },
- { GU_XVE_PLM_OFFSET, 0xffffffff },
- { GU_XVE_B_PLM_OFFSET, 0xffffffff },
- { GU_XVE_C_PLM_OFFSET, 0xffffffff },
- { GU_FEAT2_PLM_OFFSET, 0xffffffff },
- { GU_OPT_PLM_OFFSET, 0xffffffff },
- { GU_RECONFIG_PLM_OFFSET, 0xffffffff },
- { GU_PJTAG_PLM_OFFSET, 0xffffffff },
- { GU_PJTAG_SEC_PLM_OFFSET, 0xffffffff },
-};
-static const struct gu_reg_write selectors[] = {
- { GU_FEAT_SS0_OFFSET, GU_SS0_UNLOCK }, { GU_FEAT_SS1_OFFSET, GU_SS1_UNLOCK },
-};
-static const struct gu_family_writes ga100 = {
- .name = "GA100",
- .privileged = GU_WRITE_TABLE(ga100_plm), .host = GU_WRITE_TABLE(selectors),
-};
-static const struct gu_family_writes ga10x = {
- .name = "GA102/GA103/GA104/GA106",
- .privileged = GU_WRITE_TABLE(ga10x_plm), .host = GU_WRITE_TABLE(selectors),
-};
-#define GA100(id) { id, 0x170, true, &ga100 }
-#define GA102(id) { id, 0x172, false, &ga10x }
-#define GA103(id) { id, 0x173, false, &ga10x }
-#define GA104(id) { id, 0x174, false, &ga10x }
-#define GA106(id) { id, 0x176, false, &ga10x }
-static const struct gu_card cards[] = {
- GA100(0x20c2), GA100(0x2082),
- GA102(0x2200), GA102(0x2203), GA102(0x2204), GA102(0x2205),
- GA102(0x2206), GA102(0x2207), GA102(0x2208), GA102(0x220a),
- GA102(0x220d), GA102(0x2216), GA102(0x222b), GA102(0x222f),
- GA104(0x2480), GA104(0x2482), GA104(0x2483), GA104(0x2484),
- GA104(0x2486), GA104(0x2487), GA104(0x2488), GA104(0x2489),
- GA104(0x248a), GA104(0x248c), GA104(0x248d), GA104(0x248e),
- GA104(0x249c), GA104(0x249d), GA104(0x249f), GA104(0x24a0),
- GA104(0x24a4), GA104(0x24ac), GA104(0x24ad), GA104(0x24af),
- GA104(0x24bf), GA104(0x24c0), GA104(0x24c7), GA104(0x24c8),
- GA104(0x24c9), GA104(0x24dc), GA104(0x24dd), GA104(0x24df),
- GA104(0x24e0), GA103(0x2414),
- GA106(0x2501), GA106(0x2503), GA106(0x2504), GA106(0x2505),
- GA106(0x2507), GA106(0x2508), GA106(0x2509), GA106(0x252f),
- GA106(0x2544),
-};
-const struct gu_card *gu_card_find(u16 device)
-{
- size_t i;
- for (i = 0; i < sizeof(cards) / sizeof(cards[0]); i++)
- if (cards[i].device == device)
- return &cards[i];
- return NULL;
-}
diff --git a/kernel/gpu_unlock/recipes.h b/kernel/gpu_unlock/recipes.h
deleted file mode 100644
index 6bb197b..0000000
--- a/kernel/gpu_unlock/recipes.h
+++ /dev/null
@@ -1,21 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_RECIPES_H
-#define GPU_UNLOCK_RECIPES_H
-#include "prep_610.h"
-
-struct gu_reg_write { u32 addr, value; };
-struct gu_write_table { const struct gu_reg_write *entries; size_t count; };
-struct gu_family_writes {
- const char *name;
- struct gu_write_table privileged, host;
-};
-struct gu_card {
- u16 device;
- u32 chip; /* PMC_BOOT_0 architecture + implementation */
- bool direct_boot;
- const struct gu_family_writes *writes;
-};
-#define GU_WRITE_TABLE(t) { .entries = (t), .count = sizeof(t) / sizeof((t)[0]) }
-
-const struct gu_card *gu_card_find(u16 device);
-#endif
diff --git a/kernel/gpu_unlock/registers.h b/kernel/gpu_unlock/registers.h
deleted file mode 100644
index 578c447..0000000
--- a/kernel/gpu_unlock/registers.h
+++ /dev/null
@@ -1,88 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_REGISTERS_H
-#define GPU_UNLOCK_REGISTERS_H
-
-/* BAR0 registers used by the Ampere recipes and firmware transaction.
- * Names follow the pinned NVIDIA 610.43.03 register headers and cmpunlocker
- * PLM table. Falcon/FBIF/Falcon2 offsets are relative to their engine bases.
- */
-#define GU_PMC_BOOT_0_OFFSET 0x00000000u
-#define GU_PROM_DATA_OFFSET 0x00300000u
-#define GU_BUS_SW_SCRATCH(index) (0x00001400u + (index) * 4u)
-#define GU_ACR_SCRATCH_OFFSET GU_BUS_SW_SCRATCH(14)
-
-#define GU_GSP_BASE 0x00110000u
-#define GU_GSP_FBIF_BASE 0x00110600u
-#define GU_GSP_FALCON2_BASE 0x00111000u
-#define GU_SEC2_BASE 0x00840000u
-#define GU_SEC2_FBIF_BASE 0x00840600u
-#define GU_SEC2_FALCON2_BASE 0x00841000u
-#define GU_FALCON2_WINDOW_SIZE 0x00001000u
-
-#define GU_FALCON_MAILBOX0_OFFSET 0x00000040u
-#define GU_FALCON_MAILBOX1_OFFSET 0x00000044u
-#define GU_FALCON_RM_OFFSET 0x00000084u
-#define GU_FALCON_HWCFG2_OFFSET 0x000000f4u
-#define GU_FALCON_CPUCTL_OFFSET 0x00000100u
-#define GU_FALCON_BOOTVEC_OFFSET 0x00000104u
-#define GU_FALCON_HWCFG_OFFSET 0x00000108u
-#define GU_FALCON_DMACTL_OFFSET 0x0000010cu
-#define GU_FALCON_DMATRFBASE_OFFSET 0x00000110u
-#define GU_FALCON_DMATRFMOFFS_OFFSET 0x00000114u
-#define GU_FALCON_DMATRFCMD_OFFSET 0x00000118u
-#define GU_FALCON_DMATRFFBOFFS_OFFSET 0x0000011cu
-#define GU_FALCON_DMATRFBASE1_OFFSET 0x00000128u
-#define GU_FALCON_CPUCTL_ALIAS_OFFSET 0x00000130u
-/* Host IMEM/DMEM access port 0. */
-#define GU_FALCON_IMEMC_OFFSET 0x00000180u
-#define GU_FALCON_IMEMD_OFFSET 0x00000184u
-#define GU_FALCON_IMEMT_OFFSET 0x00000188u
-#define GU_FALCON_DMEMC_OFFSET 0x000001c0u
-#define GU_FALCON_DMEMD_OFFSET 0x000001c4u
-#define GU_FALCON_ENGINE_OFFSET 0x000003c0u
-#define GU_FALCON_RESET_PLM_OFFSET 0x000003c4u
-#define GU_SEC2_RESET_PLM_OFFSET (GU_SEC2_BASE + GU_FALCON_RESET_PLM_OFFSET)
-
-#define GU_FBIF_TRANSCFG(index) (0x00000000u + (index) * 4u)
-#define GU_FBIF_CTL_OFFSET 0x00000024u
-#define GU_FALCON2_MOD_SEL_OFFSET 0x00000180u
-#define GU_FALCON2_BROM_CURR_UCODE_ID_OFFSET 0x00000198u
-#define GU_FALCON2_BROM_ENGIDMASK_OFFSET 0x0000019cu
-/* BROM parameter-address register 0 and RISC-V boot control. */
-#define GU_FALCON2_BROM_PARAADDR_OFFSET 0x00000210u
-#define GU_RISCV_BCR_CTRL_OFFSET 0x00000668u
-
-#define GU_FUSE_SEC2_UCODE1 0x00824140u
-#define GU_FUSE_GSP_UCODE1 0x008241c0u
-
-#define GU_GFW_PLM_OFFSET 0x00118128u
-#define GU_GFW_BOOT_OFFSET 0x00118234u
-#define GU_GFW_CPUCTL_OFFSET (GU_GSP_BASE + GU_FALCON_CPUCTL_OFFSET)
-/* PGC6 AON_SECURE_SCRATCH_GROUP_42 reports framebuffer size in MiB. */
-#define GU_GFW_FB_SIZE_OFFSET 0x001183a4u
-#define GU_VGA_WORKSPACE_BASE_OFFSET 0x00625f04u
-#define GU_MMU_LOCK_CFG_PLM_OFFSET 0x001fa7c8u
-#define GU_MMU_LOCK_ADDR_LO_OFFSET 0x001fa82cu
-#define GU_MMU_LOCK_ADDR_HI_OFFSET 0x001fa830u
-#define GU_WPR2_ADDR_LO_OFFSET 0x001fa824u
-#define GU_WPR2_ADDR_HI_OFFSET 0x001fa828u
-
-#define GU_WPR_CFG_PLM_OFFSET 0x001fa7ccu
-#define GU_FBPA_PLM_OFFSET 0x009a0148u
-#define GU_WPR_PLM_OFFSET 0x001fa7c4u
-#define GU_FEAT_PLM_OFFSET 0x00823804u
-#define GU_XVE_PLM_OFFSET 0x00088ff4u
-#define GU_XVE_B_PLM_OFFSET 0x00088ab4u
-#define GU_XVE_C_PLM_OFFSET 0x00088ff8u
-#define GU_FEAT2_PLM_OFFSET 0x00823b00u
-#define GU_OPT_PLM_OFFSET 0x008200fcu
-#define GU_RECONFIG_PLM_OFFSET 0x008200d8u
-#define GU_PJTAG_PLM_OFFSET 0x0000c840u
-#define GU_PJTAG_SEC_PLM_OFFSET 0x0000c848u
-#define GU_FEAT_SS0_OFFSET 0x0082381cu
-#define GU_FEAT_SS1_OFFSET 0x00823820u
-/* SS0 has eight 4-bit fields; SS1 has one. Value 8 selects full rate. */
-#define GU_SS0_UNLOCK 0x88888888u
-#define GU_SS1_UNLOCK 0x8u
-
-#endif
diff --git a/kernel/gpu_unlock/unlock_610.c b/kernel/gpu_unlock/unlock_610.c
index 83c960e..2d45133 100644
--- a/kernel/gpu_unlock/unlock_610.c
+++ b/kernel/gpu_unlock/unlock_610.c
@@ -16,16 +16,25 @@
#include <linux/timekeeping.h>
#include "diagnostic.h"
-#include "falcon.h"
#include "gfw_eval.h"
#include "prep_610.h"
-#include "recipes.h"
-#include "registers.h"
-#include "payload_610.h"
#include "unlock_610.h"
#include "vbios_610.h"
-enum gu_lifecycle { GU_ALLOCATED, GU_PUBLISHED, GU_ACTIVE, GU_STOCK, GU_QUIESCED };
+#define SEC2_BASE 0x00840000u
+#define SEC2_FBIF 0x00840600u
+#define SEC2_FALCON2 0x00841000u
+/* SS0 holds eight 4-bit fields; SS1 stores only its low 4 bits (0x2 when
+ * cold). Both get the same per-field value 8. */
+#define GU_SS0_UNLOCK 0x88888888u
+#define GU_SS1_UNLOCK 0x8u
+#define BOOTER_MAGIC 0x314c5442u
+#define BOOTER_FILE "gpu-unlock/610.43.03/booter_load_ga102.bin"
+#define V67_FILE "gpu-unlock/610.43.03/v67-ga102-610.43.03.bin"
+#define GSP_FILE "gpu-unlock/610.43.03/gsp_ga10x.bin"
+#define BOOT_FILE "gpu-unlock/610.43.03/gsp_boot_ga102.bin"
+#define BOOT_DESC_FILE "gpu-unlock/610.43.03/gsp_boot_desc_ga102.bin"
+
struct unlock_transaction_page {
void *cpu;
dma_addr_t dma;
@@ -42,8 +51,8 @@ struct unlock_transaction_work {
bool region;
bool mapped;
bool master;
- const struct firmware *booter_fw; /* borrowed immutable shared assets */
- const struct firmware *loader_fw;
+ const struct firmware *booter_fw;
+ const struct firmware *v67_fw;
const struct firmware *gsp_fw; /* borrowed from gpu_unlock_610_shared */
const struct firmware *boot_fw;
struct gpu_unlock_span image_span, sig_span;
@@ -59,8 +68,8 @@ struct unlock_transaction_work {
dma_addr_t booter_dma;
u32 booter_image_size;
struct gpu_unlock_booter_hdr booter_hdr;
- void *payload_cpu;
- dma_addr_t payload_dma;
+ void *v67_cpu;
+ dma_addr_t v67_dma;
void *stock_cpu;
dma_addr_t stock_dma;
u64 stock_size;
@@ -71,20 +80,12 @@ struct unlock_transaction_work {
u64 coherent_before;
bool streaming_set;
bool coherent_set;
- enum gu_lifecycle state;
- const struct gu_card *card;
- const struct gpu_unlock_610_fw_desc *load;
- const struct gu_family_writes *family;
- size_t write_index;
- unsigned int attempt;
- u32 frts_lo, frts_hi;
- bool frts_saved, invocation_halted, fresh, cleanup_complete;
- struct gu_falcon_evidence launch, fwsec_launch;
- u32 target_before, target_after, mailbox0, mailbox1;
+ unsigned int signature_calls;
+ bool published;
bool masks_saved;
bool firmware_started;
+ bool quiesced;
bool retained;
- bool error_reported;
u32 plm_entry;
u32 bars[6];
void *fwsec_cpu;
@@ -100,13 +101,13 @@ struct unlock_transaction_work {
static int unlock_transaction_result(struct unlock_transaction_work *work, int error,
const char *function, unsigned int line)
{
- if (error && !work->error_reported) {
- work->error_reported = true;
+ if (error)
+ {
dev_err(&work->pdev->dev,
"failure unlock_610.c:%u %s error=%d last_mmio=%#x value=%#x valid=%u mapped=%u master=%u firmware_started=%u quiesced=%u pages=%u/%u\n",
line, function, error, work->last_offset, work->last_value,
work->last_valid, work->mapped, work->master, work->firmware_started,
- (work->state == GU_QUIESCED), work->pages_allocated, work->page_count);
+ work->quiesced, work->pages_allocated, work->page_count);
dev_err(&work->pdev->dev, "last PCI sample: capability=%u offset=%#x value=%#x result=%d\n",
work->last_pci_cap, work->last_pci_offset, work->last_pci_value, work->last_pci_error);
}
@@ -156,6 +157,35 @@ static void *diag_dma_alloc(struct unlock_transaction_work *work, const char *na
return memory;
}
+/* CONFIG_FW_CACHE attaches name records to devres even though this is not
+ * devm_request_firmware(). release_firmware() does not remove them, and
+ * really_probe() refuses a device with any existing devres. Scope each
+ * synchronous request so only its cache records are released; the returned
+ * firmware remains owned by the caller.
+ */
+static int request_scoped(struct device *dev, const struct firmware **fw, const char *name)
+{
+ void *group;
+ int ret, released;
+
+ group = devres_open_group(dev, NULL, GFP_KERNEL);
+ if (!group)
+ return -ENOMEM;
+ dev_info(dev, "asset request: %s\n", name);
+ ret = request_firmware(fw, name, dev);
+ released = devres_release_group(dev, group);
+ dev_info(dev, "asset result: %s error=%d bytes=%zu\n",
+ name, ret, ret ? (size_t)0 : (*fw)->size);
+ dev_info(dev, "asset cache cleanup: %s devres_released=%d\n", name, released);
+ return ret;
+}
+
+static int request_asset(struct unlock_transaction_work *work, const struct firmware **fw,
+ const char *name)
+{
+ return GU_RESULT(request_scoped(&work->pdev->dev, fw, name));
+}
+
static int check_devres_empty(struct unlock_transaction_work *work, const char *phase)
{
struct device *dev = &work->pdev->dev;
@@ -166,7 +196,7 @@ static int check_devres_empty(struct unlock_transaction_work *work, const char *
spin_lock_irqsave(&dev->devres_lock, flags);
empty = list_empty(&dev->devres_head);
spin_unlock_irqrestore(&dev->devres_lock, flags);
- dev_dbg(dev, "driver probe resources: phase=%s empty=%u\n", phase, empty);
+ dev_info(dev, "driver probe resources: phase=%s empty=%u\n", phase, empty);
return GU_RESULT(empty ? 0 : -EBUSY);
}
@@ -187,25 +217,256 @@ static void write_bar(struct unlock_transaction_work *work, u32 offset, u32 valu
writel(value, work->bar + offset);
}
-static int transport_read(void *context, u32 offset, u32 *value)
+static int poll_us(struct unlock_transaction_work *work, u32 offset, u32 mask, u32 expect,
+ unsigned int budget_us, bool timeout_is_failure)
{
- return read_bar(context, offset, value);
+ ktime_t deadline = ktime_add_us(ktime_get(), budget_us);
+ u32 value = 0;
+
+ for (;;) {
+ if (read_bar(work, offset, &value) || value == ~0u ||
+ (value & 0xffff0000u) == 0xbadf0000u) {
+ dev_err(&work->pdev->dev, "poll inaccessible: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return GU_RESULT(-EIO);
+ }
+ if ((value & mask) == expect)
+ return 0;
+ if (ktime_compare(ktime_get(), deadline) >= 0) {
+ /* Console loglevel hides info. A tolerated timeout must not
+ * look like the failure that stopped the boot. */
+ if (!timeout_is_failure) {
+ dev_info(&work->pdev->dev, "poll timeout tolerated: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return -ETIMEDOUT;
+ }
+ dev_err(&work->pdev->dev, "poll timeout: reg=%#x last=%#x mask=%#x expected=%#x budget_us=%u\n",
+ offset, value, mask, expect, budget_us);
+ return GU_RESULT(-ETIMEDOUT);
+ }
+ usleep_range(50, 100);
+ }
}
-static void transport_write(void *context, u32 offset, u32 value)
+
+struct falcon_eng {
+ u32 base;
+ u32 fbif;
+ u32 falcon2;
+};
+
+static bool inaccessible(u32 value)
{
- write_bar(context, offset, value);
+ return value == ~0u || (value & 0xffff0000u) == 0xbadf0000u;
}
-static struct gu_falcon_io falcon_io(struct unlock_transaction_work *work)
+
+static int falcon_reset(struct unlock_transaction_work *work, const struct falcon_eng *eng)
{
- return (struct gu_falcon_io) {
- .dev = &work->pdev->dev, .context = work,
- .read = transport_read, .write = transport_write,
- };
+ u32 engine, hwcfg, bcr, chip;
+ unsigned int i;
+ int ret;
+
+ /* 610 kflcnPreResetWait_GA102 permits this timeout (HW bug 3419321). */
+ dev_info(&work->pdev->dev, "Falcon reset begin: base=%#x\n", eng->base);
+ ret = poll_us(work, eng->base + 0xf4, 1u << 31, 1u << 31, 1000, false);
+ if (ret && ret != -ETIMEDOUT)
+ return GU_RESULT(ret);
+ if (ret)
+ dev_info(&work->pdev->dev, "Falcon pre-reset timeout tolerated per HW bug 3419321\n");
+ if (read_bar(work, eng->base + 0x3c0, &engine) || inaccessible(engine))
+ return GU_RESULT(-ENODEV);
+ write_bar(work, eng->base + 0x3c0, engine | 1);
+ for (i = 0; i < 10; i++)
+ readl(work->bar + eng->base + 0x3c0);
+ write_bar(work, eng->base + 0x3c0, engine & ~1u);
+ for (i = 0; i < 10; i++)
+ readl(work->bar + eng->base + 0x3c0);
+ ret = poll_us(work, eng->base + 0xf4, 1u << 12, 0, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, eng->base + 0xf4, &hwcfg) || inaccessible(hwcfg))
+ return GU_RESULT(-EIO);
+ if (hwcfg & (1u << 10)) {
+ if (read_bar(work, eng->falcon2 + 0x668, &bcr) || inaccessible(bcr))
+ return GU_RESULT(-ENODEV);
+ if (bcr & (1u << 4)) {
+ write_bar(work, eng->falcon2 + 0x668, 0);
+ ret = poll_us(work, eng->falcon2 + 0x668, 0x11, 1, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ }
+ }
+ if (read_bar(work, 0, &chip) || inaccessible(chip))
+ return GU_RESULT(-ENODEV);
+ write_bar(work, eng->base + 0x84, chip);
+ dev_info(&work->pdev->dev, "Falcon reset done: base=%#x hwcfg=%#x chip=%#x\n", eng->base, hwcfg, chip);
+ return 0;
}
-static bool inaccessible(u32 value)
+/* DMATRFMOFFS is 24 bits, FBOFFS is 32 bits, and BASE is 49 bits.
+ * Reject truncation/wrap before submitting any part of a transfer. */
+static bool falcon_dma_valid(u32 dest, u32 mem_off, dma_addr_t src, u32 size)
{
- return value == ~0u || (value & 0xffff0000u) == 0xbadf0000u;
+ return size && !((size | dest | mem_off | src) & 255) &&
+ (u64)dest + size <= (1ull << 24) &&
+ (u64)mem_off + size <= (1ull << 32) &&
+ src < (1ull << 49) &&
+ (u64)mem_off + size <= (1ull << 49) - src;
+}
+
+static int falcon_dma(struct unlock_transaction_work *work, u32 base, u32 dest, u32 mem_off,
+ dma_addr_t src, u32 size, u32 cmd)
+{
+ u32 done = 0;
+ int ret;
+
+ dev_info(&work->pdev->dev, "Falcon DMA: base=%#x dest=%#x offset=%#x src=%#llx size=%#x cmd=%#x\n",
+ base, dest, mem_off, (unsigned long long)src, size, cmd);
+ if (!falcon_dma_valid(dest, mem_off, src, size))
+ return GU_RESULT(-EINVAL);
+ ret = poll_us(work, base + 0x118, 1, 0, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ write_bar(work, base + 0x110, lower_32_bits(src >> 8));
+ write_bar(work, base + 0x128, upper_32_bits(src >> 8) & 0x1ff);
+ while (done < size) {
+ ret = poll_us(work, base + 0x118, 1, 0, 1000000, true);
+ if (ret) {
+ dev_err(&work->pdev->dev, "Falcon DMA progress: done=%u total=%u\n", done, size);
+ return GU_RESULT(ret);
+ }
+ write_bar(work, base + 0x114, dest);
+ write_bar(work, base + 0x11c, mem_off);
+ write_bar(work, base + 0x118, cmd);
+ done += 256;
+ dest += 256;
+ mem_off += 256;
+ }
+ return GU_RESULT(poll_us(work, base + 0x118, 2, 2, 1000000, true));
+}
+
+static int falcon_hs(struct unlock_transaction_work *work, const struct falcon_eng *eng,
+ u32 code_off, u32 code_size, u32 code_va, u32 code_pa,
+ u32 data_off, u32 data_size, u32 data_pa,
+ u32 sig_dmem, u32 engine_mask, u32 ucode_id,
+ dma_addr_t image, u32 mailbox0, u32 mailbox1,
+ bool wait_plm)
+{
+ u32 trans, cpuctl;
+ dma_addr_t code_src;
+ int ret;
+
+ if (image > ~0ull - code_off || image + code_off < code_va ||
+ image > ~0ull - data_off)
+ return GU_RESULT(-ERANGE);
+ code_src = image + code_off - code_va;
+ if (!falcon_dma_valid(code_pa, code_va, code_src, code_size) ||
+ (data_size && !falcon_dma_valid(data_pa, 0, image + data_off, data_size)))
+ return GU_RESULT(-EINVAL);
+ ret = falcon_reset(work, eng);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, eng->fbif + 0x24, &trans) || inaccessible(trans))
+ return GU_RESULT(-EIO);
+ trans |= 1u << 7;
+ write_bar(work, eng->fbif + 0x24, trans);
+ write_bar(work, eng->base + 0x10c, 0);
+ if (read_bar(work, eng->fbif, &trans) || inaccessible(trans))
+ return GU_RESULT(-ENODEV);
+ trans = (trans & ~7u) | 5u;
+ write_bar(work, eng->fbif, trans);
+ dma_wmb();
+ ret = falcon_dma(work, eng->base, code_pa, code_va, code_src,
+ code_size, gpu_unlock_dma_cmd(true));
+ if (ret)
+ return GU_RESULT(ret);
+ if (data_size) {
+ ret = falcon_dma(work, eng->base, data_pa, 0, image + data_off, data_size,
+ gpu_unlock_dma_cmd(false));
+ if (ret)
+ return GU_RESULT(ret);
+ }
+ write_bar(work, eng->falcon2 + 0x210, sig_dmem);
+ write_bar(work, eng->falcon2 + 0x19c, engine_mask);
+ write_bar(work, eng->falcon2 + 0x198, ucode_id & 0xff);
+ write_bar(work, eng->falcon2 + 0x180, 1);
+ write_bar(work, eng->base + 0x104, code_va);
+ write_bar(work, eng->base + 0x40, mailbox0);
+ write_bar(work, eng->base + 0x44, mailbox1);
+ if (read_bar(work, eng->base + 0x100, &cpuctl) || inaccessible(cpuctl))
+ return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev, "Falcon start: base=%#x cpuctl=%#x code_va=%#x sig_dmem=%#x engine_mask=%#x ucode=%u mailbox=%#x:%#x\n",
+ eng->base, cpuctl, code_va, sig_dmem, engine_mask, ucode_id, mailbox1, mailbox0);
+ if (cpuctl & (1u << 6))
+ write_bar(work, eng->base + 0x130, 1u << 1);
+ else
+ write_bar(work, eng->base + 0x100, 1u << 1);
+ if (wait_plm) {
+ ktime_t deadline = ktime_add_us(ktime_get(), 5000000);
+
+ for (;;) {
+ u32 plm, engine, identity;
+
+ if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
+ identity != pci_identity(work->pdev) ||
+ read_bar(work, eng->base + 0x100, &engine) || engine == ~0u ||
+ (engine & 0xffff0000u) == 0xbadf0000u ||
+ read_bar(work, GU_FEAT_PLM_OFFSET, &plm))
+ return GU_RESULT(-ENODEV);
+ /* Cold entry was closed: this transition proves payload execution. */
+ if (plm == ~0u && work->plm_entry != ~0u)
+ return 0;
+ if ((engine & (1u << 4)) || ktime_compare(ktime_get(), deadline) >= 0)
+ dev_err(&work->pdev->dev, "early PLM failed: pci=%#x cpuctl=%#x plm=%#x entry=%#x budget_us=5000000\n",
+ identity, engine, plm, work->plm_entry);
+ if (engine & (1u << 4))
+ return GU_RESULT(-EIO);
+ if (ktime_compare(ktime_get(), deadline) >= 0)
+ return GU_RESULT(-ETIMEDOUT);
+ usleep_range(50, 100);
+ }
+ }
+ ret = poll_us(work, eng->base + 0x100, 1u << 4, 1u << 4, 5000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ return 0;
+}
+
+static int load_booter(struct unlock_transaction_work *work)
+{
+ const u8 *data;
+ u32 magic, version, image_size, sig_bytes;
+ struct gpu_unlock_booter_hdr hdr;
+ int ret;
+
+ ret = request_asset(work, &work->booter_fw, BOOTER_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ data = work->booter_fw->data;
+ if (work->booter_fw->size < 72)
+ return GU_RESULT(-EINVAL);
+ memcpy(&magic, data, sizeof(magic));
+ memcpy(&version, data + 4, sizeof(version));
+ if (magic != BOOTER_MAGIC || version != 1)
+ return GU_RESULT(-EINVAL);
+ memcpy(&hdr, data + 8, sizeof(hdr));
+ memcpy(&image_size, data + 8 + sizeof(hdr), sizeof(image_size));
+ memcpy(&sig_bytes, data + 12 + sizeof(hdr), sizeof(sig_bytes));
+ if (!hdr.ucode_id || hdr.num_apps != 1 || sig_bytes != hdr.sig_bytes ||
+ image_size != GU_BOOTER_IMAGE_SIZE ||
+ work->booter_fw->size != 72u + image_size + sig_bytes)
+ return GU_RESULT(-EINVAL);
+ /* Only this pinned production BOOT_FROM_HS geometry is implemented. */
+ if (hdr.ucode_id != 3 || hdr.engine_id != 1 || hdr.num_sigs != 2 ||
+ sig_bytes != GU_BOOTER_SIG_TOTAL || hdr.os_code_off != 0 ||
+ hdr.os_code_size != 256 || hdr.os_data_off != 35328 ||
+ hdr.os_data_size != 25088 || hdr.app_code_off != 256 ||
+ hdr.app_code_size != 35072 || hdr.app_data_off != 256 ||
+ hdr.app_data_size != 0 || hdr.patch_loc != 35344)
+ return GU_RESULT(-EINVAL);
+ work->booter_hdr = hdr;
+ work->booter_image_size = image_size;
+ /* Allocation follows the ownership and DMA-mask snapshot, not asset lookup. */
+ return 0;
}
static int patch_booter(struct unlock_transaction_work *work)
@@ -217,7 +478,7 @@ static int patch_booter(struct unlock_transaction_work *work)
if (read_bar(work, GU_FUSE_SEC2_UCODE1 + index * 4, &fuse_raw))
return GU_RESULT(-EIO);
fuse_ver = gpu_unlock_fuse_version(fuse_raw);
- dev_dbg(&work->pdev->dev, "Booter signature: fuse_raw=%#x version=%u signatures=%u size=%u\n",
+ dev_info(&work->pdev->dev, "Booter signature: fuse_raw=%#x version=%u signatures=%u size=%u\n",
fuse_raw, fuse_ver, work->booter_hdr.num_sigs, work->booter_hdr.sig_bytes);
return GU_RESULT(gpu_unlock_booter_patch(work->booter_cpu, work->booter_image_size,
&work->booter_hdr,
@@ -247,9 +508,9 @@ static void free_coherent(struct unlock_transaction_work *work)
if (work->booter_cpu)
dma_free_coherent(&work->pdev->dev, GU_BOOTER_IMAGE_SIZE,
work->booter_cpu, work->booter_dma);
- if (work->payload_cpu)
- dma_free_coherent(&work->pdev->dev, GU_PAYLOAD_SIZE, work->payload_cpu,
- work->payload_dma);
+ if (work->v67_cpu)
+ dma_free_coherent(&work->pdev->dev, GU_V67_SIZE, work->v67_cpu,
+ work->v67_dma);
if (work->stock_cpu)
dma_free_coherent(&work->pdev->dev, work->stock_size,
work->stock_cpu, work->stock_dma);
@@ -257,7 +518,7 @@ static void free_coherent(struct unlock_transaction_work *work)
dma_free_coherent(&work->pdev->dev, GU_WPR_META_SIZE,
work->meta_cpu, work->meta_dma);
work->booter_cpu = NULL;
- work->payload_cpu = NULL;
+ work->v67_cpu = NULL;
work->stock_cpu = NULL;
work->meta_cpu = NULL;
}
@@ -268,13 +529,8 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
int ret = 0;
/* A timeout never authorizes freeing firmware-visible memory. */
- if (work->firmware_started && work->state != GU_QUIESCED) {
- pci_clear_master(work->pdev);
- work->master = false;
+ if (work->firmware_started && !work->quiesced) {
work->retained = true;
- ret = diag_pci_word(work, false, PCI_COMMAND, &now);
- dev_err(&work->pdev->dev, "handover refused: incomplete transaction; BME_off=%u command=%#x config_result=%d; BAR ownership and DMA buffers retained until power off\n",
- !ret && now != 0xffff && !(now & PCI_COMMAND_MASTER), now, ret);
return GU_RESULT(-EBUSY);
}
if (work->master) {
@@ -295,6 +551,18 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
work->enabled = false;
}
free_coherent(work);
+ if (work->booter_fw) {
+ release_firmware(work->booter_fw);
+ work->booter_fw = NULL;
+ }
+ if (work->v67_fw) {
+ release_firmware(work->v67_fw);
+ work->v67_fw = NULL;
+ }
+ if (work->boot_fw) {
+ release_firmware(work->boot_fw);
+ work->boot_fw = NULL;
+ }
/* Match the driver core's entry condition, not just BAR/mask cleanup.
* Never release unrelated resources to make this check pass. A bound
* driver keeps its own resources and is not probed again.
@@ -307,14 +575,14 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
if (dma_set_mask(&work->pdev->dev, work->streaming_before) ||
*work->streaming_ptr != work->streaming_before)
ret = -EIO;
- dev_dbg(&work->pdev->dev, "restore streaming mask: actual=%#llx expected=%#llx\n",
+ dev_info(&work->pdev->dev, "restore streaming mask: actual=%#llx expected=%#llx\n",
(unsigned long long)*work->streaming_ptr, (unsigned long long)work->streaming_before);
}
if (work->coherent_set) {
if (dma_set_coherent_mask(&work->pdev->dev, work->coherent_before) ||
work->pdev->dev.coherent_dma_mask != work->coherent_before)
ret = -EIO;
- dev_dbg(&work->pdev->dev, "restore coherent mask: actual=%#llx expected=%#llx\n",
+ dev_info(&work->pdev->dev, "restore coherent mask: actual=%#llx expected=%#llx\n",
(unsigned long long)work->pdev->dev.coherent_dma_mask,
(unsigned long long)work->coherent_before);
}
@@ -331,7 +599,7 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
pci_set_master(work->pdev);
if (diag_pci_word(work, false, PCI_COMMAND, &now))
return GU_RESULT(-EIO);
- dev_dbg(&work->pdev->dev, "restore PCI command: actual=%#x original=%#x mask=%#x\n",
+ dev_info(&work->pdev->dev, "restore PCI command: actual=%#x original=%#x mask=%#x\n",
now, work->original_cmd, GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER);
if ((now & (GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER)) !=
(work->original_cmd & (GU_PCI_COMMAND_MEMORY | GU_PCI_COMMAND_MASTER)))
@@ -339,21 +607,68 @@ static int unlock_transaction_unwind(struct unlock_transaction_work *work)
return GU_RESULT(ret);
}
+static int unlock_transaction_assets(struct unlock_transaction_work *work)
+{
+ const struct firmware *desc;
+ int ret;
+
+ if (!work->resume) {
+ ret = check_devres_empty(work, "entry");
+ if (ret)
+ return ret;
+ }
+ ret = request_asset(work, &work->v67_fw, V67_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ if (work->v67_fw->size != GU_V67_SIZE)
+ return GU_RESULT(-EBADMSG);
+ ret = gpu_unlock_elf_section(work->gsp_fw->data, work->gsp_fw->size,
+ ".fwimage", &work->image_span);
+ if (ret)
+ return GU_RESULT(ret);
+ ret = gpu_unlock_elf_section(work->gsp_fw->data, work->gsp_fw->size,
+ ".fwsignature_ga10x", &work->sig_span);
+ if (ret)
+ return GU_RESULT(ret);
+ if (!work->image_span.size || work->image_span.size > 128ull << 20 ||
+ work->sig_span.size != 4096)
+ return GU_RESULT(-EINVAL);
+ ret = gpu_unlock_radix_plan(work->image_span.size, &work->radix);
+ if (ret)
+ return GU_RESULT(ret);
+ ret = request_asset(work, &work->boot_fw, BOOT_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ if (work->boot_fw->size != 24576)
+ return GU_RESULT(-EINVAL);
+ ret = request_asset(work, &desc, BOOT_DESC_FILE);
+ if (ret)
+ return GU_RESULT(ret);
+ ret = gpu_unlock_boot_desc_parse(desc->data, desc->size,
+ work->boot_fw->size, &work->boot_desc);
+ dev_info(&work->pdev->dev, "asset geometry: image=%llu signature=%llu boot=%u code=%#x data=%#x manifest=%#x parse=%d\n",
+ (unsigned long long)work->image_span.size, (unsigned long long)work->sig_span.size,
+ work->boot_desc.image_size, work->boot_desc.code_offset, work->boot_desc.data_offset,
+ work->boot_desc.manifest_offset, ret);
+ release_firmware(desc);
+ return GU_RESULT(ret ? ret : load_booter(work));
+}
+
static int unlock_transaction_layout(struct unlock_transaction_work *work)
{
struct gpu_unlock_layout_in *in = &work->layout_in;
u32 mb, vga, lo, hi, plm;
int ret;
- if (read_bar(work, GU_GFW_FB_SIZE_OFFSET, &mb) || !mb || mb == ~0u ||
- read_bar(work, GU_VGA_WORKSPACE_BASE_OFFSET, &vga) || vga == ~0u ||
- read_bar(work, GU_MMU_LOCK_CFG_PLM_OFFSET, &plm) || plm == ~0u || !(plm & 1) ||
- read_bar(work, GU_MMU_LOCK_ADDR_LO_OFFSET, &lo) || lo == ~0u ||
- read_bar(work, GU_MMU_LOCK_ADDR_HI_OFFSET, &hi) || hi == ~0u)
+ if (read_bar(work, 0x001183a4, &mb) || !mb || mb == ~0u ||
+ read_bar(work, 0x00625f04, &vga) || vga == ~0u ||
+ read_bar(work, 0x001fa7c8, &plm) || plm == ~0u || !(plm & 1) ||
+ read_bar(work, 0x001fa82c, &lo) || lo == ~0u ||
+ read_bar(work, 0x001fa830, &hi) || hi == ~0u)
return GU_RESULT(-ENODEV);
/* Keep the cold board inputs in the USB kernel log even on layout failure.
* Register values after NVIDIA binds cannot reconstruct this state. */
- dev_dbg(&work->pdev->dev,
+ dev_info(&work->pdev->dev,
"layout inputs: fb_mb=%#x vga=%#x mmu_plm=%#x mmu_lo=%#x mmu_hi=%#x image=%llu boot=%u\n",
mb, vga, plm, lo, hi, (unsigned long long)work->image_span.size,
work->boot_desc.image_size);
@@ -387,12 +702,12 @@ static int unlock_transaction_layout(struct unlock_transaction_work *work)
in->mmu_lock_valid = mmu_hi >= in->mmu_lock_low;
top_lock = in->mmu_lock_valid && (mmu_top >> 20) == (in->usable_fb >> 20);
- dev_dbg(&work->pdev->dev,
+ dev_info(&work->pdev->dev,
"layout mmu: lo=%#llx hi=%#llx hi_exclusive=%#llx constrain=%u\n",
(unsigned long long)in->mmu_lock_low, (unsigned long long)mmu_hi,
(unsigned long long)mmu_top, top_lock);
if (!top_lock) {
- dev_dbg(&work->pdev->dev,
+ dev_info(&work->pdev->dev,
"mmu lock ignored: high address is not the FB top\n");
in->mmu_lock_valid = false;
}
@@ -404,7 +719,7 @@ static int unlock_transaction_layout(struct unlock_transaction_work *work)
/* GA102's supported minimum heap; no GSP-RM clients are started here. */
in->heap_request = 88ull << 20;
ret = gpu_unlock_layout(in, &work->layout);
- dev_dbg(&work->pdev->dev,
+ dev_info(&work->pdev->dev,
"layout decoded: fb=%#llx vga=%#llx mmu_valid=%u mmu_low=%#llx frts=%#llx heap=%#llx result=%d\n",
(unsigned long long)in->usable_fb, (unsigned long long)in->vga_base,
in->mmu_lock_valid, (unsigned long long)in->mmu_lock_low,
@@ -413,7 +728,7 @@ static int unlock_transaction_layout(struct unlock_transaction_work *work)
dev_err(&work->pdev->dev, "layout rejected: reservation calculation (%d)\n", ret);
return GU_RESULT(ret);
}
- dev_dbg(&work->pdev->dev,
+ dev_info(&work->pdev->dev,
"layout ranges: nonwpr=%#llx wpr=%#llx heap=%#llx gsp=%#llx boot=%#llx frts=%#llx end=%#llx scrub_floor=%#llx\n",
(unsigned long long)work->layout.nonwpr_start,
(unsigned long long)work->layout.wpr_start,
@@ -490,7 +805,7 @@ static int unlock_transaction_claim(struct unlock_transaction_work *work)
unsigned int i;
int ret;
- dev_dbg(&work->pdev->dev, "claim: power=%d pm_cap=%#x flags=%#x\n",
+ dev_info(&work->pdev->dev, "claim: power=%d pm_cap=%#x flags=%#x\n",
work->pdev->current_state, work->pdev->pm_cap, work->pdev->dev_flags);
/* Older kernels (seen on 6.8 and 6.14) leave a device no driver has
* enabled at PCI_UNKNOWN. Accept that only when PMCSR confirms D0. */
@@ -518,13 +833,13 @@ static int unlock_transaction_claim(struct unlock_transaction_work *work)
diag_pci_word(work, false, PCI_COMMAND, &command) ||
command == 0xffff || (command & PCI_COMMAND_MASTER))
return GU_RESULT(-EBUSY);
- dev_dbg(&work->pdev->dev, "claim: cleared %s BME\n",
+ dev_info(&work->pdev->dev, "claim: cleared %s BME\n",
work->resume ? "driver" : "firmware");
/* Firmware's BME stays off; the suspended driver gets its own back. */
if (!work->resume)
work->original_cmd = command;
}
- dev_dbg(&work->pdev->dev, "claim: command=%#x capability=%#x BAR_mask=%#x\n",
+ dev_info(&work->pdev->dev, "claim: command=%#x capability=%#x BAR_mask=%#x\n",
command, cap, pci_select_bars(work->pdev, IORESOURCE_MEM));
/* Only BAR0 is mapped. On the display card the firmware framebuffer keeps
* BAR1 until just before the FLR, as it does while the stock driver boots.
@@ -543,7 +858,7 @@ static int unlock_transaction_claim(struct unlock_transaction_work *work)
work->enabled = true;
}
work->bar_len = pci_resource_len(work->pdev, 0);
- if (work->bar_len < GU_SEC2_FALCON2_BASE + GU_FALCON2_WINDOW_SIZE)
+ if (work->bar_len < SEC2_FALCON2 + 0x1000)
return GU_RESULT(-EINVAL);
for (i = 0; i < ARRAY_SIZE(work->bars); i++)
if (diag_pci_dword(work, false, PCI_BASE_ADDRESS_0 + i * 4, &work->bars[i]))
@@ -570,7 +885,7 @@ static int unlock_transaction_dma(struct unlock_transaction_work *work)
work->streaming_ptr = dev->dma_mask;
work->streaming_before = *dev->dma_mask;
work->coherent_before = dev->coherent_dma_mask;
- dev_dbg(dev, "DMA masks: streaming=%#llx coherent=%#llx requested=%#llx\n",
+ dev_info(dev, "DMA masks: streaming=%#llx coherent=%#llx requested=%#llx\n",
(unsigned long long)work->streaming_before, (unsigned long long)work->coherent_before,
(unsigned long long)DMA_BIT_MASK(32));
work->streaming_set = true;
@@ -584,11 +899,11 @@ static int unlock_transaction_dma(struct unlock_transaction_work *work)
if (!work->booter_cpu)
return GU_RESULT(-ENOMEM);
memcpy(work->booter_cpu, work->booter_fw->data + 72, work->booter_image_size);
- work->payload_cpu = diag_dma_alloc(work, "payload", GU_PAYLOAD_SIZE, &work->payload_dma, GFP_KERNEL);
+ work->v67_cpu = diag_dma_alloc(work, "v67", GU_V67_SIZE, &work->v67_dma, GFP_KERNEL);
work->meta_cpu = diag_dma_alloc(work, "metadata", GU_WPR_META_SIZE, &work->meta_dma, GFP_KERNEL);
- if (!work->payload_cpu || !work->meta_cpu)
+ if (!work->v67_cpu || !work->meta_cpu)
return GU_RESULT(-ENOMEM);
- gu_payload_fill(work->payload_cpu, GU_FEAT_PLM_OFFSET, ~0u);
+ memcpy(work->v67_cpu, work->v67_fw->data, GU_V67_SIZE);
work->stock_size = work->sig_span.size;
work->stock_cpu = diag_dma_alloc(work, "stock-signature", work->stock_size, &work->stock_dma, GFP_KERNEL);
work->boot_cpu = diag_dma_alloc(work, "boot-image", work->boot_desc.image_size, &work->boot_dma, GFP_KERNEL);
@@ -605,10 +920,9 @@ static int unlock_transaction_dma(struct unlock_transaction_work *work)
static int unlock_transaction_fwsec(struct unlock_transaction_work *work)
{
- static const struct gu_falcon_engine gsp = {
- .base = GU_GSP_BASE, .fbif = GU_GSP_FBIF_BASE, .falcon2 = GU_GSP_FALCON2_BASE,
+ static const struct falcon_eng gsp = {
+ .base = 0x00110000, .fbif = 0x00110600, .falcon2 = 0x00111000,
};
- struct gu_falcon_io io = falcon_io(work);
size_t mapped = 0;
u8 *rom = NULL;
struct gpu_unlock_fwsec fw;
@@ -621,23 +935,22 @@ static int unlock_transaction_fwsec(struct unlock_transaction_work *work)
/* kgspExtractVbiosFromRom_TU102 reads NV_PROM_DATA, up to 1 MiB.
* pci_map_rom stops at the PCI last-image flag and hides the VN extension. */
- dev_dbg(&work->pdev->dev, "FWSEC: reading PROM at %#x\n", GU_PROM_DATA_OFFSET);
- if (work->bar_len < GU_PROM_DATA_OFFSET + 0x1a)
+ dev_info(&work->pdev->dev, "FWSEC: reading PROM at %#x\n", 0x300000);
+ if (work->bar_len < 0x300000 + 0x1a)
return GU_RESULT(-ENODEV);
- mapped = min_t(size_t, 1024 * 1024, work->bar_len - GU_PROM_DATA_OFFSET);
+ mapped = min_t(size_t, 1024 * 1024, work->bar_len - 0x300000);
rom = kvmalloc(mapped, GFP_KERNEL);
if (!rom)
return GU_RESULT(-ENOMEM);
- memcpy_fromio(rom, work->bar + GU_PROM_DATA_OFFSET, mapped);
- dev_dbg(&work->pdev->dev, "FWSEC: PROM bytes=%zu signature=%#x\n",
+ memcpy_fromio(rom, work->bar + 0x300000, mapped);
+ dev_info(&work->pdev->dev, "FWSEC: PROM bytes=%zu signature=%#x\n",
mapped, (unsigned int)(rom[0] | (rom[1] << 8)));
ret = gpu_unlock_vbios_fwsec(rom, mapped, &fw);
if (ret) {
dev_err(&work->pdev->dev, "FWSEC-FRTS not found in board ROM (%d)\n", ret);
goto out;
}
- if ((fw.version == 2) != work->load->direct ||
- (fw.version == 3 && fw.engine_mask != 0x400)) {
+ if (fw.engine_mask != 0x400) {
dev_err(&work->pdev->dev, "FWSEC: unsupported GSP engine mask=%#x\n", fw.engine_mask);
ret = -EOPNOTSUPP;
goto out;
@@ -654,15 +967,16 @@ static int unlock_transaction_fwsec(struct unlock_transaction_work *work)
ret = gpu_unlock_fwsec_patch_frts(image, &fw, frts);
if (ret)
goto out;
- if (fw.version == 3) {
+ if (!fw.ucode_id ||
+ read_bar(work, 0x008241c0 + (fw.ucode_id - 1) * 4, &fuse_raw)) {
+ ret = -EIO;
+ goto out;
+ }
+ version = gpu_unlock_fuse_version(fuse_raw);
+ dev_info(&work->pdev->dev, "FWSEC fuse: raw=%#x version=%u\n", fuse_raw, version);
+ {
u32 sig_at = 0;
- if (!fw.ucode_id ||
- read_bar(work, GU_FUSE_GSP_UCODE1 + (fw.ucode_id - 1) * 4, &fuse_raw)) {
- ret = -EIO;
- goto out;
- }
- version = gpu_unlock_fuse_version(fuse_raw);
- dev_dbg(&work->pdev->dev, "FWSEC fuse: raw=%#x version=%u\n", fuse_raw, version);
+
ret = gpu_unlock_fwsec_sig_offset(fw.sig_versions, version, 384, &sig_at);
if (ret)
goto out;
@@ -675,60 +989,36 @@ static int unlock_transaction_fwsec(struct unlock_transaction_work *work)
ret = -EIO;
goto out;
}
- /* Preserve conservative DMA retention from the moment FWSEC may access it,
- * including reset/load failures. Launch evidence is observed separately. */
work->firmware_started = true;
- if (fw.version == 2) {
- const struct gu_falcon_loader_image program = {
- .loader = work->loader_fw ? (const u32 *)work->loader_fw->data : NULL,
- .loader_size = work->loader_fw ? work->loader_fw->size : 0,
- .dma = image_dma, .data_off = fw.data_off,
- .imem_pa = fw.imem_pa, .imem_ns_size = fw.imem_ns_size,
- .imem_sec_pa = fw.imem_sec_pa, .imem_sec_size = fw.imem_sec_size,
- .dmem_size = fw.dmem_size,
- };
- ret = gu_falcon_loader(&io, &gsp, &program, &work->fwsec_launch);
- } else {
- const struct gu_falcon_hs_image program = {
- .dma = image_dma, .code_off = 0, .code_size = fw.imem_size,
- .code_va = fw.imem_va, .code_pa = fw.imem_pa,
- .data_off = fw.data_off, .data_size = fw.dmem_size,
- .data_pa = fw.dmem_pa, .sig_dmem = fw.pkc_off,
- .engine_mask = fw.engine_mask, .ucode_id = fw.ucode_id,
- };
- ret = gu_falcon_hs(&io, &gsp, &program, &work->fwsec_launch);
- }
+ ret = falcon_hs(work, &gsp, 0, fw.imem_size, fw.imem_va, fw.imem_pa,
+ fw.data_off, fw.dmem_size, fw.dmem_pa, fw.pkc_off,
+ fw.engine_mask, fw.ucode_id,
+ image_dma, 0, 0, false);
if (ret)
goto out;
- if (read_bar(work, GU_ACR_SCRATCH_OFFSET, &scratch)) {
+ if (read_bar(work, 0x00001400 + 0x0e * 4, &scratch)) {
ret = -EIO;
goto out;
}
- dev_dbg(&work->pdev->dev, "FWSEC result: scratch=%#x expected_hi16=0\n", scratch);
+ dev_info(&work->pdev->dev, "FWSEC result: scratch=%#x expected_hi16=0\n", scratch);
if (scratch >> 16) {
ret = -EIO;
goto out;
}
- if (read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &wpr_lo) || wpr_lo == ~0u ||
- read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &wpr_hi) || wpr_hi == ~0u) {
+ if (read_bar(work, 0x001fa824, &wpr_lo) || wpr_lo == ~0u ||
+ read_bar(work, 0x001fa828, &wpr_hi) || wpr_hi == ~0u) {
ret = -EIO;
goto out;
}
- dev_dbg(&work->pdev->dev, "FWSEC WPR2: low=%#x high=%#x expected_page=%#llx end=%#llx\n",
+ dev_info(&work->pdev->dev, "FWSEC WPR2: low=%#x high=%#x expected_page=%#llx end=%#llx\n",
wpr_lo, wpr_hi, (unsigned long long)(frts >> 12),
(unsigned long long)work->layout.wpr_end);
/* Match NVIDIA's initialized-WPR2 check as well as the expected low
* address. Never publish Booter metadata for disabled/alien WPR2. */
if (!(wpr_hi >> 4) || (wpr_hi >> 4) < (wpr_lo >> 4) ||
((u64)(wpr_hi >> 4) << 12) >= work->layout.wpr_end ||
- (wpr_lo >> 4) != (u32)(frts >> 12) ||
- (((u64)(wpr_hi >> 4) + 1) << 12) != work->layout.wpr_end)
+ (wpr_lo >> 4) != (u32)(frts >> 12))
ret = -EIO;
- if (!ret) {
- work->frts_lo = wpr_lo;
- work->frts_hi = wpr_hi;
- work->frts_saved = true;
- }
out:
/* Ownership transfers to work before publication, including failure paths. */
kvfree(rom);
@@ -758,9 +1048,9 @@ static int unlock_transaction_gfw(struct unlock_transaction_work *work)
if (read_bar(work, GU_FEAT_PLM_OFFSET, &work->plm_entry) ||
read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || ss0 == ~0u ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || ss1 == ~0u ||
- read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &wpr_hi) || inaccessible(wpr_hi))
+ read_bar(work, 0x001fa828, &wpr_hi) || inaccessible(wpr_hi))
return GU_RESULT(-ENODEV);
- dev_dbg(&work->pdev->dev, "entry: plm=%#x ss0=%#x ss1=%#x wpr2_hi=%#x\n",
+ dev_info(&work->pdev->dev, "entry: plm=%#x ss0=%#x ss1=%#x wpr2_hi=%#x\n",
work->plm_entry, ss0, ss1, wpr_hi);
/* An open PLM means the card was already unlocked or not cold. */
if (work->plm_entry == ~0u)
@@ -797,21 +1087,21 @@ static int unlock_transaction_suspended(struct unlock_transaction_work *work)
!(command & PCI_COMMAND_MEMORY))
return GU_RESULT(-ENODEV);
work->bar_len = pci_resource_len(work->pdev, 0);
- if (work->bar_len < GU_SEC2_FALCON2_BASE + GU_FALCON2_WINDOW_SIZE)
+ if (work->bar_len < SEC2_FALCON2 + 0x1000)
return GU_RESULT(-EINVAL);
work->bar = pci_iomap(work->pdev, 0, 0);
if (!work->bar)
return GU_RESULT(-ENOMEM);
work->mapped = true;
/* Read before GFW completes, so an unreadable value proves nothing yet. */
- if (read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &wpr_hi) ||
+ if (read_bar(work, 0x001fa828, &wpr_hi) ||
read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1))
return GU_RESULT(-EIO);
- dev_dbg(&work->pdev->dev, "resume entry: wpr2_hi=%#x ss0=%#x ss1=%#x\n",
+ dev_info(&work->pdev->dev, "resume entry: wpr2_hi=%#x ss0=%#x ss1=%#x\n",
wpr_hi, ss0, ss1);
if (!inaccessible(wpr_hi) && (wpr_hi >> 4)) {
- dev_dbg(&work->pdev->dev, "resume: GSP-RM already running (WPR2 enabled); card left as is\n");
+ dev_info(&work->pdev->dev, "resume: GSP-RM already running (WPR2 enabled); card left as is\n");
return -EBUSY;
}
/* Sleep that kept the card powered (suspend to idle, an aborted suspend). */
@@ -822,74 +1112,18 @@ static int unlock_transaction_suspended(struct unlock_transaction_work *work)
static int unlock_transaction_boot(struct unlock_transaction_work *work)
{
- static const struct gu_falcon_engine sec2 = {
- .base = GU_SEC2_BASE, .fbif = GU_SEC2_FBIF_BASE, .falcon2 = GU_SEC2_FALCON2_BASE,
+ static const struct falcon_eng sec2 = {
+ .base = SEC2_BASE, .fbif = SEC2_FBIF, .falcon2 = SEC2_FALCON2,
};
const struct gpu_unlock_booter_hdr *h = &work->booter_hdr;
- struct gu_falcon_io io = falcon_io(work);
- int ret;
- u32 ctl, reset_plm, engine, scratch, identity;
- if ((work->state != GU_PUBLISHED && work->state != GU_ACTIVE) || !work->master || !work->firmware_started)
+
+ if (!work->published || !work->master || !work->firmware_started)
return GU_RESULT(-EINVAL);
- if (work->load->direct) {
- const struct gu_falcon_direct_image program = {
- .words = work->booter_cpu,
- .os_code_off = h->os_code_off, .os_code_size = h->os_code_size,
- .app_code_off = h->app_code_off, .app_code_size = h->app_code_size,
- .os_data_off = h->os_data_off, .os_data_size = h->os_data_size,
- .mailbox0 = lower_32_bits(work->meta_dma),
- .mailbox1 = upper_32_bits(work->meta_dma),
- };
- ret = gu_falcon_direct(&io, &sec2, &program, &work->launch);
- } else {
- const struct gu_falcon_hs_image program = {
- .dma = work->booter_dma,
- .code_off = h->app_code_off, .code_size = h->app_code_size,
- .code_va = h->app_code_off, .code_pa = 0,
- .data_off = h->os_data_off, .data_size = h->os_data_size, .data_pa = 0,
- .sig_dmem = h->patch_loc - h->os_data_off,
- .engine_mask = h->engine_id, .ucode_id = h->ucode_id,
- .mailbox0 = lower_32_bits(work->meta_dma),
- .mailbox1 = upper_32_bits(work->meta_dma),
- };
- ret = gu_falcon_hs(&io, &sec2, &program, &work->launch);
- }
- /* Always capture the first failure, including successful write/no halt. */
- if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
- identity != pci_identity(work->pdev) ||
- read_bar(work, GU_SEC2_BASE + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl) ||
- read_bar(work, GU_SEC2_BASE + GU_FALCON_MAILBOX0_OFFSET, &work->mailbox0) ||
- read_bar(work, GU_SEC2_BASE + GU_FALCON_MAILBOX1_OFFSET, &work->mailbox1) ||
- read_bar(work, GU_SEC2_RESET_PLM_OFFSET, &reset_plm) ||
- read_bar(work, GU_SEC2_BASE + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine) ||
- read_bar(work, GU_ACR_SCRATCH_OFFSET, &scratch))
- return GU_RESULT(-ENODEV);
- dev_dbg(&work->pdev->dev,
- "completion diagnostics: engine=%#x acr_scratch@%#x=%#x secure_bus_owner=unavailable token_return=unobserved dmem_cleanup=unobserved\n",
- engine, GU_ACR_SCRATCH_OFFSET, scratch);
- work->invocation_halted = !!(ctl & 16);
- work->fresh = work->launch.reset_asserted && work->launch.reset_released &&
- work->launch.reloaded && work->launch.started && work->invocation_halted;
- /* The reference exit does not write the mailboxes. Its nonzero input
- * metadata pointer is therefore a reference-specific return, not the
- * stock Booter zero-success convention. Hardware confirmation pending. */
- work->cleanup_complete = work->fresh && (reset_plm & 0xff) == 0xff &&
- work->mailbox0 == lower_32_bits(work->meta_dma) &&
- work->mailbox1 == upper_32_bits(work->meta_dma);
- dev_dbg(&work->pdev->dev,
- "invocation: index=%zu attempt=%u reset=%u/%u reload=%u start=%u cpuctl=%#x reset_plm@%#x=%#x mailbox=%#x:%#x fresh=%u cleanup_observed=%u result=%d\n",
- work->write_index, work->attempt, work->launch.reset_asserted,
- work->launch.reset_released, work->launch.reloaded, work->launch.started, ctl,
- GU_SEC2_RESET_PLM_OFFSET, reset_plm, work->mailbox1, work->mailbox0,
- work->fresh, work->cleanup_complete, ret);
- if (ret || !work->cleanup_complete)
- dev_err(&work->pdev->dev,
- "invocation failed: index=%zu attempt=%u cpuctl=%#x engine=%#x reset_plm=%#x mailbox=%#x:%#x reset=%u/%u reload=%u start=%u fresh=%u cleanup=%u error=%d\n",
- work->write_index, work->attempt, ctl, engine, reset_plm,
- work->mailbox1, work->mailbox0, work->launch.reset_asserted,
- work->launch.reset_released, work->launch.reloaded, work->launch.started,
- work->fresh, work->cleanup_complete, ret ? ret : -EIO);
- return GU_RESULT(ret ? ret : work->cleanup_complete ? 0 : -EIO);
+ return GU_RESULT(falcon_hs(work, &sec2, h->app_code_off, h->app_code_size,
+ h->app_code_off, 0, h->os_data_off, h->os_data_size, 0,
+ h->patch_loc - h->os_data_off, h->engine_id, h->ucode_id,
+ work->booter_dma, lower_32_bits(work->meta_dma),
+ upper_32_bits(work->meta_dma), true));
}
static void put_le64(u8 *dest, u64 value)
@@ -900,137 +1134,36 @@ static void put_le64(u8 *dest, u64 value)
dest[i] = (u8)(value >> (8 * i));
}
-static int publish_signature(struct unlock_transaction_work *work)
+static int unlock_transaction_signature(struct unlock_transaction_work *work)
{
- struct gpu_unlock_wpr_args args = {
- .fb_size = work->layout_in.usable_fb,
- .vga_offset = work->layout_in.vga_base,
- .frts_size = work->layout_in.frts_size,
- .signature_dma = work->payload_dma, .signature_size = GU_PAYLOAD_SIZE,
- .image_size = work->image_span.size,
- .boot_dma = work->boot_dma, .boot = work->boot_desc,
- };
- int ret;
+ u8 *meta = work->meta_cpu;
- if (!work->meta_cpu || !work->stock_cpu || !work->pages_allocated ||
- ((work->payload_dma | work->stock_dma | work->meta_dma) & 255) ||
- work->state != GU_ALLOCATED)
+ if (!meta || !work->stock_cpu || !work->pages_allocated ||
+ work->signature_calls >= 2)
return GU_RESULT(-EINVAL);
- args.radix_dma = work->pages[0].dma;
- ret = gpu_unlock_wpr_meta_pack(work->meta_cpu, &work->layout, &args);
- if (ret)
- return GU_RESULT(ret);
- dma_wmb();
- work->state = GU_PUBLISHED;
- return 0;
-}
-
-static int restore_stock_signature(struct unlock_transaction_work *work)
-{
- if (!work->meta_cpu || !work->stock_cpu ||
- (work->state != GU_PUBLISHED && work->state != GU_ACTIVE))
- return GU_RESULT(-EINVAL);
- put_le64((u8 *)work->meta_cpu + 72, work->stock_dma);
- put_le64((u8 *)work->meta_cpu + 80, work->stock_size);
- dma_wmb();
- work->state = GU_STOCK;
- return 0;
-}
+ if (work->signature_calls == 0) {
+ struct gpu_unlock_wpr_args args = {
+ .fb_size = work->layout_in.usable_fb,
+ .vga_offset = work->layout_in.vga_base,
+ .frts_size = work->layout_in.frts_size,
+ .signature_dma = work->v67_dma, .signature_size = GU_V67_SIZE,
+ .radix_dma = work->pages[0].dma,
+ .image_size = work->image_span.size,
+ .boot_dma = work->boot_dma, .boot = work->boot_desc,
+ };
+ int ret = gpu_unlock_wpr_meta_pack(meta, &work->layout, &args);
-static int restore_frts(struct unlock_transaction_work *work)
-{
- u32 lo = 0, hi = 0;
- if (!work->frts_saved)
- return GU_RESULT(-EPROTO);
- read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo);
- read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi);
- dev_dbg(&work->pdev->dev, "WPR2 before restore: index=%zu attempt=%u lo=%#x hi=%#x\n",
- work->write_index, work->attempt, lo, hi);
- write_bar(work, GU_WPR2_ADDR_LO_OFFSET, work->frts_lo);
- write_bar(work, GU_WPR2_ADDR_HI_OFFSET, work->frts_hi);
- if (read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo) || read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi))
- return GU_RESULT(-EIO);
- dev_dbg(&work->pdev->dev, "WPR2 restored: lo=%#x hi=%#x expected=%#x:%#x\n",
- lo, hi, work->frts_lo, work->frts_hi);
- return GU_RESULT(lo == work->frts_lo && hi == work->frts_hi ? 0 : -EIO);
-}
-static int table_failure(struct unlock_transaction_work *work, const char *phase,
- const struct gu_reg_write *entry, int ret)
-{
- dev_err(&work->pdev->dev,
- "table failure: family=%s phase=%s index=%zu attempt=%u address=%#x expected=%#x observed=%#x error=%d\n",
- work->family->name, phase, work->write_index, work->attempt,
- entry->addr, entry->value, work->target_after, ret);
- return GU_RESULT(ret);
-}
-static int run_privileged_writes(struct unlock_transaction_work *work,
- const struct gu_write_table *table)
-{
- size_t i;
- int ret;
- if (work->state != GU_PUBLISHED && work->state != GU_ACTIVE)
- return GU_RESULT(-EPROTO);
- work->state = GU_ACTIVE;
- for (i = 0; i < table->count; i++) {
- const struct gu_reg_write *entry = &table->entries[i];
- work->write_index = i;
- for (work->attempt = 1; work->attempt <= 2; work->attempt++) {
- u32 lo = 0, hi = 0, ctl = 0, engine = 0;
- work->error_reported = false;
- work->fresh = work->cleanup_complete = false;
- memset(&work->launch, 0, sizeof(work->launch));
- work->target_after = 0;
- /* FWSEC completed before the first attempt. Every subsequent
- * refill requires the prior invocation to have stopped. */
- if ((i || work->attempt > 1) && !work->invocation_halted)
- return table_failure(work, "prior-halt", entry, -EBUSY);
- ret = restore_frts(work);
- if (ret)
- return table_failure(work, "restore", entry, ret);
- if (read_bar(work, entry->addr, &work->target_before) ||
- (work->target_before & 0xffff0000u) == 0xbadf0000u ||
- read_bar(work, GU_SEC2_BASE + GU_FALCON_CPUCTL_OFFSET, &ctl) || inaccessible(ctl) ||
- read_bar(work, GU_SEC2_BASE + GU_FALCON_ENGINE_OFFSET, &engine) || inaccessible(engine))
- return table_failure(work, "prelaunch", entry, -ENODEV);
- dev_dbg(&work->pdev->dev,
- "prelaunch: index=%zu attempt=%u target=%#x cpuctl=%#x engine=%#x\n",
- i, work->attempt, work->target_before, ctl, engine);
- gu_payload_fill(work->payload_cpu, entry->addr, entry->value);
- put_le64((u8 *)work->meta_cpu + 72, work->payload_dma);
- put_le64((u8 *)work->meta_cpu + 80, GU_PAYLOAD_SIZE);
- dma_wmb();
- dev_dbg(&work->pdev->dev,
- "payload: reference=2d10f891 size=%#x dma=%#llx index=%zu attempt=%u address=%#x value=%#x\n",
- GU_PAYLOAD_SIZE, (unsigned long long)work->payload_dma, i,
- work->attempt, entry->addr, entry->value);
- ret = unlock_transaction_boot(work);
- if (read_bar(work, entry->addr, &work->target_after) ||
- (work->target_after & 0xffff0000u) == 0xbadf0000u ||
- read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo) || read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi))
- return table_failure(work, "result-inaccessible", entry, -ENODEV);
- dev_dbg(&work->pdev->dev,
- "write result: index=%zu attempt=%u address=%#x expected=%#x observed=%#x wpr2=%#x:%#x write=%u fresh=%u cleanup_observed=%u\n",
- i, work->attempt, entry->addr, entry->value,
- work->target_after, lo, hi, work->target_after == entry->value,
- work->fresh, work->cleanup_complete);
- if (ret) {
- dev_err(&work->pdev->dev, "failed invocation WPR2: actual=%#x:%#x saved_FRTS=%#x:%#x\n",
- lo, hi, work->frts_lo, work->frts_hi);
- return table_failure(work, "completion", entry, ret);
- }
- if (work->target_after == entry->value)
- break;
- /* Only literal idempotent PLMs belong here. A mismatching
- * readback after proven halt/cleanup may be retried once.
- * Errors/timeouts never replay an uncertain side effect. */
- table_failure(work, "readback", entry, -EIO);
- }
- if (work->target_after != entry->value)
- return table_failure(work, "readback-exhausted", entry, -EIO);
+ if (ret)
+ return GU_RESULT(ret);
+ dma_wmb();
+ work->published = true;
+ } else {
+ put_le64(meta + 72, work->stock_dma);
+ put_le64(meta + 80, work->stock_size);
+ dma_wmb();
}
- ret = restore_frts(work);
- return ret && table->count ?
- table_failure(work, "final-FRTS", &table->entries[table->count - 1], ret) : ret;
+ work->signature_calls++;
+ return 0;
}
/* Write the unlock value to one speed selector and read it back. */
@@ -1041,29 +1174,90 @@ static int write_selector(struct unlock_transaction_work *work, const char *name
if (read_bar(work, offset, &value) || inaccessible(value))
return GU_RESULT(-ENODEV);
- dev_dbg(&work->pdev->dev, "selector read: %s=%#x expected_unlock=%#x\n",
+ dev_info(&work->pdev->dev, "selector read: %s=%#x expected_unlock=%#x\n",
name, value, unlock);
- dev_dbg(&work->pdev->dev, "selector write: %s=%#x\n", name, unlock);
+ if (value == unlock)
+ return 0;
+ dev_info(&work->pdev->dev, "selector write: %s=%#x\n", name, unlock);
write_bar(work, offset, unlock);
if (read_bar(work, offset, &value) || value == ~0u)
return GU_RESULT(-ENODEV);
return GU_RESULT(value == unlock ? 0 : -EIO);
}
-static int run_host_writes(struct unlock_transaction_work *work,
- const struct gu_write_table *table)
+/* SS0 selects the full BF16 rate; SS1 gets the same field value. FLR keeps both. */
+static int unlock_transaction_selector(struct unlock_transaction_work *work)
{
- size_t i;
- for (i = 0; i < table->count; i++) {
- const struct gu_reg_write *entry = &table->entries[i];
- int ret;
- work->write_index = i;
- ret = write_selector(work, "host", entry->addr, entry->value);
- if (ret) {
- read_bar(work, entry->addr, &work->target_after);
- return table_failure(work, "host", entry, ret);
+ int ret = write_selector(work, "SS0", GU_FEAT_SS0_OFFSET, GU_SS0_UNLOCK);
+
+ return ret ? ret : write_selector(work, "SS1", GU_FEAT_SS1_OFFSET, GU_SS1_UNLOCK);
+}
+
+/* Reset SEC2 after FLR and leave it halted with reset released. Do not reset
+ * GSP here: stock RM first requires completed GFW and a halted GSP Falcon.
+ * ksec2ResetHw_TU102 ends with RESET_FALSE. Leaving RESET_TRUE makes later
+ * host reads return FECS_PRI_RESET (0xbadf12 in bits 31:8). RM's scrub wait
+ * treats that value as scrubbing-pending and does not finish.
+ */
+static int hold_sec2_reset(struct unlock_transaction_work *work, const char *phase)
+{
+ unsigned int j;
+ u32 engine, before, first = 0, hwcfg;
+ int ret;
+
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) || inaccessible(engine))
+ return GU_RESULT(-ENODEV);
+ before = engine;
+ dev_info(&work->pdev->dev, "engine reset assert: phase=%s base=%#x before=%#x write=%#x\n",
+ phase, SEC2_BASE, before, before | 1);
+ write_bar(work, SEC2_BASE + 0x3c0, engine | 1);
+ /* 610 ksec2ResetHw_TU102 requires ten register cycles for signal
+ * propagation. A stale zero during those cycles is not evidence that
+ * the assertion failed. No CPU start and no reset retry.
+ */
+ for (j = 0; j < 10; j++) {
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) ||
+ inaccessible(engine)) {
+ dev_err(&work->pdev->dev, "engine reset inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
+ phase, SEC2_BASE, j + 1, engine);
+ return GU_RESULT(-EIO);
}
+ if (!j)
+ first = engine;
}
+ dev_info(&work->pdev->dev, "engine reset propagation: phase=%s base=%#x first=%#x tenth=%#x\n",
+ phase, SEC2_BASE, first, engine);
+ ret = poll_us(work, SEC2_BASE + 0x3c0, 1, 1, 1000, true);
+ dev_info(&work->pdev->dev, "engine reset asserted: phase=%s base=%#x error=%d last=%#x\n",
+ phase, SEC2_BASE, ret, work->last_value);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) || inaccessible(engine))
+ return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev, "engine reset release: phase=%s base=%#x before=%#x write=%#x\n",
+ phase, SEC2_BASE, engine, engine & ~1u);
+ write_bar(work, SEC2_BASE + 0x3c0, engine & ~1u);
+ for (j = 0; j < 10; j++) {
+ if (read_bar(work, SEC2_BASE + 0x3c0, &engine) ||
+ inaccessible(engine)) {
+ dev_err(&work->pdev->dev, "engine reset release inaccessible: phase=%s base=%#x cycle=%u value=%#x\n",
+ phase, SEC2_BASE, j + 1, engine);
+ return GU_RESULT(-EIO);
+ }
+ }
+ ret = poll_us(work, SEC2_BASE + 0x3c0, 1, 0, 1000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ /* NV_PFALCON_FALCON_HWCFG2 bit 12 clears when memory scrubbing finishes.
+ * A PRI error fails this wait instead of being handed to RM.
+ */
+ ret = poll_us(work, SEC2_BASE + 0xf4, 1u << 12, 0, 1000000, true);
+ if (ret)
+ return GU_RESULT(ret);
+ if (read_bar(work, SEC2_BASE + 0xf4, &hwcfg) || inaccessible(hwcfg))
+ return GU_RESULT(-EIO);
+ dev_info(&work->pdev->dev, "engine reset released: phase=%s base=%#x engine=%#x hwcfg2=%#x\n",
+ phase, SEC2_BASE, engine, hwcfg);
return 0;
}
@@ -1077,10 +1271,10 @@ static int wait_post_flr_bar(struct unlock_transaction_work *work, u32 chip)
u32 value = 0;
for (;;) {
- if (read_bar(work, GU_PMC_BOOT_0_OFFSET, &value))
+ if (read_bar(work, 0, &value))
return GU_RESULT(-EIO);
if (value == chip) {
- dev_dbg(&work->pdev->dev, "post-FLR BAR ready: chip=%#x\n", value);
+ dev_info(&work->pdev->dev, "post-FLR BAR ready: chip=%#x\n", value);
return 0;
}
if (ktime_compare(ktime_get(), deadline) >= 0) {
@@ -1131,7 +1325,7 @@ static int wait_post_flr_gfw(struct unlock_transaction_work *work)
}
usleep_range(1000, 2000);
}
- dev_dbg(&work->pdev->dev, "post-FLR GFW result: error=%d pci=%#x plm=%#x boot=%#x cpuctl=%#x budget_us=%u\n",
+ dev_info(&work->pdev->dev, "post-FLR GFW result: error=%d pci=%#x plm=%#x boot=%#x cpuctl=%#x budget_us=%u\n",
ret, identity, sample.plm, sample.boot, sample.cpuctl, GU_GFW_TIMEOUT_US);
return GU_RESULT(ret);
}
@@ -1143,12 +1337,12 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
unsigned int i;
int ret;
- if (work->state != GU_STOCK)
+ if (work->signature_calls != 2)
return GU_RESULT(-EPROTO);
if (read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || ss0 == ~0u ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || ss1 == ~0u)
return GU_RESULT(-ENODEV);
- dev_dbg(&work->pdev->dev, "pre-FLR selectors: ss0=%#x expected=%#x ss1=%#x expected=%#x\n",
+ dev_info(&work->pdev->dev, "pre-FLR selectors: ss0=%#x expected=%#x ss1=%#x expected=%#x\n",
ss0, GU_SS0_UNLOCK, ss1, GU_SS1_UNLOCK);
if (ss0 != GU_SS0_UNLOCK || ss1 != GU_SS1_UNLOCK)
return GU_RESULT(-EIO);
@@ -1157,7 +1351,7 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
if (diag_pci_word(work, false, PCI_COMMAND, &command) ||
command == 0xffff || (command & PCI_COMMAND_MASTER))
return GU_RESULT(-EIO);
- if (read_bar(work, GU_PMC_BOOT_0_OFFSET, &chip) || !chip || inaccessible(chip))
+ if (read_bar(work, 0, &chip) || !chip || inaccessible(chip))
return GU_RESULT(-ENODEV);
/* Ampere PF-FLR requires no engine-reset preconditioning in this cold,
* unbound path (no initialized NVLink/P2P). The pinned GA102 rejoin14
@@ -1174,9 +1368,9 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
if (ret)
return GU_RESULT(ret);
}
- dev_dbg(&work->pdev->dev, "PCIe FLR begin: BME=0 DMA retained, chip=%#x\n", chip);
+ dev_info(&work->pdev->dev, "PCIe FLR begin: BME=0 DMA retained, chip=%#x\n", chip);
ret = pcie_flr(work->pdev);
- dev_dbg(&work->pdev->dev, "PCIe FLR returned: result=%d\n", ret);
+ dev_info(&work->pdev->dev, "PCIe FLR returned: result=%d\n", ret);
if (ret)
return GU_RESULT(ret);
/* The saved image predates BME. No transient restore with BME set. */
@@ -1203,11 +1397,7 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
ret = wait_post_flr_gfw(work);
if (ret)
return GU_RESULT(ret);
- {
- struct gu_falcon_io io = falcon_io(work);
- struct gu_falcon_evidence reset;
- ret = gu_falcon_engine_reset(&io, GU_SEC2_BASE, "post-FLR", NULL, &reset);
- }
+ ret = hold_sec2_reset(work, "post-FLR");
if (ret)
return GU_RESULT(ret);
if (diag_pci_word(work, true, PCI_EXP_DEVSTA, &pending) ||
@@ -1215,14 +1405,14 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
return GU_RESULT(-EBUSY);
if (read_bar(work, GU_GFW_CPUCTL_OFFSET, &cpuctl) || inaccessible(cpuctl) ||
!(cpuctl & GU_GFW_CPUCTL_HALTED) ||
- read_bar(work, GU_WPR2_ADDR_LO_OFFSET, &lo) || inaccessible(lo) ||
- read_bar(work, GU_WPR2_ADDR_HI_OFFSET, &hi) || inaccessible(hi) ||
+ read_bar(work, 0x001fa824, &lo) || inaccessible(lo) ||
+ read_bar(work, 0x001fa828, &hi) || inaccessible(hi) ||
read_bar(work, GU_FEAT_SS0_OFFSET, &ss0) || inaccessible(ss0) ||
read_bar(work, GU_FEAT_SS1_OFFSET, &ss1) || inaccessible(ss1) ||
read_bar(work, GU_FEAT_PLM_OFFSET, &plm) ||
(plm & 0xffff0000u) == 0xbadf0000u)
return GU_RESULT(-ENODEV);
- dev_dbg(&work->pdev->dev,
+ dev_info(&work->pdev->dev,
"post-FLR SEC2 reset released, GSP halted, BME=0 pending=0 WPR2=%08x:%08x SS0=%08x SS1=%08x\n",
lo, hi, ss0, ss1);
/* Both 610 and installed 615 _kgspBootGspRm reject preexisting WPR2
@@ -1233,69 +1423,42 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
dev_err(&work->pdev->dev, "post-FLR WPR2 remains enabled: lo=%#x hi=%#x; stock driver requires HI.VAL=0\n", lo, hi);
return GU_RESULT(-EBUSY);
}
- work->state = GU_QUIESCED;
+ work->quiesced = true;
if (ss0 != GU_SS0_UNLOCK || ss1 != GU_SS1_UNLOCK)
return GU_RESULT(-EIO);
return 0;
}
-int gpu_unlock_610_unlock(const struct gu_device *device, const struct gpu_unlock_610_shared *shared,
+int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared)
+{
+ return request_scoped(&pdev->dev, &shared->gsp, GSP_FILE);
+}
+
+void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
+{
+ release_firmware(shared->gsp);
+ shared->gsp = NULL;
+}
+
+int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared,
bool resume)
{
- struct pci_dev *pdev = device->pdev;
struct unlock_transaction_work *work;
- const struct gpu_unlock_610_firmware *assets;
int ret, cleanup;
- if (!device->recipe || !device->firmware)
- return -ENODEV;
work = kzalloc(sizeof(*work), GFP_KERNEL);
if (!work)
return -ENOMEM;
work->pdev = pci_dev_get(pdev);
- work->card = device->recipe;
- work->load = device->firmware;
- work->family = work->card->writes;
- assets = work->load->direct ? &shared->ga100 : &shared->ga10x;
- if (!assets->prepared || assets->error) {
- ret = assets->prepared ? assets->error : -EINVAL;
- pci_dev_put(work->pdev);
- kfree(work);
- return ret;
- }
- work->gsp_fw = assets->gsp;
- work->boot_fw = assets->boot;
- work->image_span = assets->image_span;
- work->sig_span = assets->sig_span;
- work->boot_desc = assets->boot_desc;
- work->radix = assets->radix;
- work->booter_fw = shared->booter;
- work->loader_fw = work->load->direct ? shared->loader : NULL;
- work->booter_hdr = shared->booter_hdr;
- work->booter_image_size = GU_BOOTER_IMAGE_SIZE;
- dev_dbg(&pdev->dev, "recipe selection: device=%#x family=%s privileged=%zu host=%zu direct=%u\n",
- pdev->device, work->family->name, work->family->privileged.count,
- work->family->host.count, work->load->direct);
+ work->gsp_fw = shared->gsp;
work->resume = resume;
/* After resume GFW may still be running: check and wait before reading
- * the layout. Firmware preparation has already completed for the round. */
+ * the layout, and before loading assets for a card that kept power. */
ret = resume ? unlock_transaction_suspended(work) : 0;
- if (!ret && !resume)
- ret = check_devres_empty(work, "entry");
+ if (!ret)
+ ret = unlock_transaction_assets(work);
if (!ret)
ret = unlock_transaction_claim(work);
- if (!ret) {
- u32 chip;
- if (read_bar(work, GU_PMC_BOOT_0_OFFSET, &chip) || inaccessible(chip) ||
- ((chip >> 20) & 0x1ff) != work->card->chip)
- ret = GU_RESULT(-ENODEV);
- }
- if (!ret) {
- size_t i;
- for (i = 0; i < work->family->privileged.count; i++)
- if ((u64)work->family->privileged.entries[i].addr + 4 > work->bar_len)
- ret = GU_RESULT(-ERANGE);
- }
if (!ret)
ret = unlock_transaction_layout(work);
if (!ret)
@@ -1305,19 +1468,18 @@ int gpu_unlock_610_unlock(const struct gu_device *device, const struct gpu_unloc
if (!ret)
ret = unlock_transaction_fwsec(work);
if (!ret)
- ret = publish_signature(work);
+ ret = unlock_transaction_signature(work); /* publish V67 */
if (!ret)
- ret = run_privileged_writes(work, &work->family->privileged);
+ ret = unlock_transaction_boot(work);
if (!ret)
- ret = restore_stock_signature(work);
+ ret = unlock_transaction_signature(work); /* restore stock */
if (!ret)
- ret = run_host_writes(work, &work->family->host);
+ ret = unlock_transaction_selector(work);
if (!ret)
ret = unlock_transaction_reset(work);
/* Never leave the V67 signature published for a later stock boot. */
- if (ret && (work->state == GU_PUBLISHED || work->state == GU_ACTIVE))
- restore_stock_signature(work);
- work->error_reported = false;
+ if (ret && work->signature_calls == 1)
+ unlock_transaction_signature(work);
cleanup = unlock_transaction_unwind(work);
if (!ret)
ret = cleanup;
@@ -1332,3 +1494,9 @@ int gpu_unlock_610_unlock(const struct gu_device *device, const struct gpu_unloc
kfree(work);
return ret;
}
+
+MODULE_FIRMWARE(BOOTER_FILE);
+MODULE_FIRMWARE(V67_FILE);
+MODULE_FIRMWARE(GSP_FILE);
+MODULE_FIRMWARE(BOOT_FILE);
+MODULE_FIRMWARE(BOOT_DESC_FILE);
diff --git a/kernel/gpu_unlock/unlock_610.h b/kernel/gpu_unlock/unlock_610.h
index f94a146..ff70a76 100644
--- a/kernel/gpu_unlock/unlock_610.h
+++ b/kernel/gpu_unlock/unlock_610.h
@@ -2,22 +2,35 @@
#ifndef GPU_UNLOCK_UNLOCK_610_H
#define GPU_UNLOCK_UNLOCK_610_H
-#include "firmware_610.h"
+#include <linux/types.h>
+
+struct firmware;
+struct pci_dev;
+
+/* Read-only firmware every unlock uses: loaded once, shared by concurrent unlocks. */
+struct gpu_unlock_610_shared {
+ const struct firmware *gsp;
+};
+
+/*
+ * Load the shared firmware. pdev is any target card; the request leaves no
+ * device resources behind on it. Returns 0 or a negative errno.
+ */
+int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared);
+
+/* Release the shared firmware once every unlock using it has returned. */
+void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared);
/*
- * Run the selected family recipe on a cold, unbound supported card, or one nvidia suspended
+ * Unlock one cold, unbound RTX 3090, or with resume one that nvidia suspended
* before a suspend to RAM and resumes after this returns. The caller holds
- * pci_dev_lock and supplies the context resolved during enumeration. Returns
- * 0 when the card is unlocked and handed back as found;
- * otherwise a negative errno. A failed transaction may have applied a prefix
- * of the table: it must not be reported as ready. If quiescence cannot be
- * established, DMA buffers and BAR ownership are retained until power off.
- * After resume,
+ * pci_dev_lock. Returns 0 when the card is unlocked and handed back as found;
+ * otherwise a negative errno, with the card left locked. After resume,
* -EALREADY means the card kept power and is still unlocked, and nothing was
* written. Unlocks of different cards may run concurrently with the same
* shared firmware.
*/
-int gpu_unlock_610_unlock(const struct gu_device *device, const struct gpu_unlock_610_shared *shared,
+int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared,
bool resume);
#endif
diff --git a/kernel/gpu_unlock/vbios_610.c b/kernel/gpu_unlock/vbios_610.c
index 6eec788..87c173b 100644
--- a/kernel/gpu_unlock/vbios_610.c
+++ b/kernel/gpu_unlock/vbios_610.c
@@ -119,49 +119,6 @@ static int descriptor(const u8 *rom, size_t length, u64 off,
size = hdr >> 16;
GU_NOTE("FWSEC descriptor: offset=%#llx ROM_bytes=%zu header=%#x\n",
(unsigned long long)off, length, hdr);
- fw.version = (hdr >> 8) & 0xff;
- /* GA100 uses the v2 generic-loader descriptor, with embedded signatures.
- * Its image is kept contiguous here; the loader receives separate code
- * and data DMA bases within that allocation. */
- if ((hdr & 1) && fw.version == 2) {
- u32 sec_base, sec_size;
- if (size < 60 || range(length, off, size))
- return GU_DIAG(-EINVAL);
- fw.image_off = off + size;
- fw.image_size = r32(rom + off + 4);
- fw.interface_off = r32(rom + off + 16);
- fw.imem_pa = r32(rom + off + 20);
- fw.imem_size = r32(rom + off + 24);
- fw.imem_va = r32(rom + off + 28);
- sec_base = r32(rom + off + 32);
- sec_size = r32(rom + off + 36);
- fw.data_off = r32(rom + off + 40);
- fw.dmem_pa = r32(rom + off + 44);
- fw.dmem_size = r32(rom + off + 48);
- /* Restrict to the layout this contiguous adapter implements; refuse
- * alternate images, relocation, compression and nonzero entry. */
- if (!fw.image_size || !fw.imem_size || !fw.dmem_size ||
- r32(rom + off + 8) != fw.image_size || r32(rom + off + 12) ||
- r32(rom + off + 52) || r32(rom + off + 56) ||
- fw.imem_pa || fw.dmem_pa || sec_size > fw.imem_size ||
- sec_base < fw.imem_va || sec_size > ~(u32)0 - 255 ||
- (fw.data_off & 255) || (fw.imem_size & 255) ||
- (fw.dmem_size & 255) || (fw.image_size & 255) ||
- range(length, fw.image_off, fw.image_size) ||
- range(fw.image_size, 0, fw.imem_size) ||
- fw.data_off < fw.imem_size ||
- range(fw.image_size, fw.data_off, fw.dmem_size) ||
- range(fw.dmem_size, fw.interface_off, 4))
- return GU_DIAG(-EINVAL);
- fw.imem_ns_size = fw.imem_size - sec_size;
- fw.imem_sec_pa = sec_base - fw.imem_va;
- fw.imem_sec_size = (sec_size + 255) & ~255u;
- if ((fw.imem_ns_size & 255) || (fw.imem_sec_pa & 255) ||
- range(fw.imem_size, fw.imem_sec_pa, fw.imem_sec_size))
- return GU_DIAG(-EINVAL);
- *out = fw;
- return 0;
- }
/* vDesc: bit 0 set, version in bits 15:8, size in bits 31:16. */
if (!(hdr & 1) || ((hdr >> 8) & 0xff) != 3 || size < 44)
return GU_DIAG(-EOPNOTSUPP);
diff --git a/kernel/gpu_unlock/vbios_610.h b/kernel/gpu_unlock/vbios_610.h
index 05b4f33..89413d6 100644
--- a/kernel/gpu_unlock/vbios_610.h
+++ b/kernel/gpu_unlock/vbios_610.h
@@ -15,13 +15,11 @@ typedef uint64_t u64;
/* Production FWSEC descriptor extracted from a VBIOS image. */
struct gpu_unlock_fwsec {
- u32 version;
u32 image_off;
u32 image_size;
u32 imem_pa;
u32 imem_size;
u32 imem_va;
- u32 imem_ns_size, imem_sec_pa, imem_sec_size;
u32 dmem_pa;
u32 data_off;
u32 dmem_size;
diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD
index b67ced7..ca3b2de 100644
--- a/packaging/arch/PKGBUILD
+++ b/packaging/arch/PKGBUILD
@@ -8,11 +8,11 @@ pkgver=1.0.0
pkgrel=1
_nv=610.43.03
_utils=nvidia-utils-$_nv-1-x86_64_v3.pkg.tar.zst
-pkgdesc='Ampere pre-driver hardware unlock'
+pkgdesc='RTX 3090 BF16 unlock before the NVIDIA driver loads'
arch=('x86_64')
url=''
license=('GPL-2.0-only')
-makedepends=('git' 'python' 'zstd' 'openssl')
+makedepends=('git' 'python' 'zstd')
options=('!strip' '!debug')
source=("$pkgbase::git+file://${startdir%/packaging/arch}"
"https://download.nvidia.com/XFree86/NVIDIA-kernel-module-source/NVIDIA-kernel-module-source-$_nv.tar.xz"
diff --git a/packaging/rpm/gpu-unlock.spec b/packaging/rpm/gpu-unlock.spec
index d459089..108fad1 100644
--- a/packaging/rpm/gpu-unlock.spec
+++ b/packaging/rpm/gpu-unlock.spec
@@ -3,7 +3,7 @@
Name: gpu-unlock
Version: 1.0.0
Release: 1%{?dist}
-Summary: Ampere pre-driver hardware unlock
+Summary: RTX 3090 BF16 unlock before the NVIDIA driver loads
License: GPL-2.0-only
BuildArch: noarch
Source0: gpu-unlock-%{version}.tar.xz
@@ -16,14 +16,13 @@ BuildRequires: make
BuildRequires: gcc
BuildRequires: python3
BuildRequires: zstd
-BuildRequires: openssl-devel
%description
Kernel module that unlocks RTX 3090 BF16 throughput before the unmodified
NVIDIA driver binds, opted in on the kernel command line.
%package dkms
-Summary: Ampere hardware unlock kernel source
+Summary: RTX 3090 BF16 unlock kernel source
Requires: dkms
Requires(post): dkms
Requires(preun): dkms
diff --git a/tests/cmp_payload_reference.h b/tests/cmp_payload_reference.h
deleted file mode 100644
index 0fd4225..0000000
--- a/tests/cmp_payload_reference.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/* Verbatim functions from cmpunlocker 2d10f891, sec2-postbl-plm-ss-cfg.patch. */
-static void
-_kgspSec2PostblTimingPutU32(NvU8 *pBuffer, NvU32 offset, NvU32 value)
-{
- pBuffer[offset + 0] = (NvU8)(value >> 0);
- pBuffer[offset + 1] = (NvU8)(value >> 8);
- pBuffer[offset + 2] = (NvU8)(value >> 16);
- pBuffer[offset + 3] = (NvU8)(value >> 24);
-}
-
-static void
-_kgspSec2PostblTimingFillPayload(NvU8 *pSignatureVa, NvU64 signatureSize,
- NvU32 writeAddr, NvU32 writeValue)
-{
- NvU64 i;
-
- for (i = 0; i + sizeof(NvU32) <= signatureSize; i += sizeof(NvU32))
- _kgspSec2PostblTimingPutU32(pSignatureVa, (NvU32)i, SEC2_POSTBL_TIMING_FILL_DWORD);
-
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0x1100, 0x00000007U);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0x5b40, 0xc0deca7eU);
-
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf754, writeValue);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf758, 0xc0deca7eU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf75c, 0x00000cbdU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf76c, writeAddr);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf774, 0x00001fbdU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf780, 0x00000000U);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf788, 0x000010aaU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf78c, 0x0000815aU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf790, 0x00008e18U);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf794, 0xc0deca7eU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf798, 0x0000815aU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf79c, 0x00000000U);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7a0, 0xc0deca7eU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7a4, 0x00001fbdU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7b0, 0x0000ffbcU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7b8, 0x0000582dU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7c4, 0xc0deca7eU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7c8, 0x00000cbdU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7d8, 0x00000003U);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7e0, 0x00001fbdU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7f4, 0x00000ccbU);
- _kgspSec2PostblTimingPutU32(pSignatureVa, 0xf7f8, 0x00007f2fU);
-}
-
diff --git a/tests/stage1_tables.h b/tests/stage1_tables.h
deleted file mode 100644
index f90a400..0000000
--- a/tests/stage1_tables.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef GPU_UNLOCK_TEST_STAGE1_TABLES_H
-#define GPU_UNLOCK_TEST_STAGE1_TABLES_H
-#include "recipes.h"
-
-/* Intermediate bring-up fixture. Production selects recipes by device ID. */
-static const struct gu_reg_write test_three_plm[] = {
- { 0x001fa7cc, 0xfffff0ff }, { 0x00823804, 0xffffffff },
- { 0x009a0148, 0xffffffff },
-};
-static const struct gu_write_table test_three_table = GU_WRITE_TABLE(test_three_plm);
-
-static const struct gu_write_table *test_stage1_table(unsigned int count)
-{
- switch (count) {
- case 2: return &gu_card_find(0x2204)->writes->privileged;
- case 3: return &test_three_table;
- case 12: return &gu_card_find(0x20c2)->writes->privileged;
- default: return NULL;
- }
-}
-#endif
diff --git a/tests/test_main.c b/tests/test_main.c
index 649a859..d1c2a92 100644
--- a/tests/test_main.c
+++ b/tests/test_main.c
@@ -11,11 +11,7 @@
#include <stdlib.h>
#include <time.h>
#include <unistd.h>
-#include "../kernel/gpu_unlock/recipes.h"
-static const struct gu_card *counted_card_find(u16 device);
-#define gu_card_find counted_card_find
#include "../kernel/gpu_unlock/main.c"
-#undef gu_card_find
#define MAX_DEVICES 8
#define RTX_3090 0x2204
@@ -31,12 +27,7 @@ static pthread_cond_t fake_cond = PTHREAD_COND_INITIALIZER;
static struct pci_driver nvidia_driver = { "nvidia" }, vfio_driver = { "vfio-pci" };
static struct {
int loads, releases, fail_load;
- bool loaded, dispatching;
- int fail_family, family_loads[2];
- int lookups, selections;
- const struct gu_device *contexts[MAX_DEVICES];
- const struct gu_card *recipes[MAX_DEVICES];
- const struct gpu_unlock_610_fw_desc *firmware[MAX_DEVICES];
+ bool loaded;
struct pci_dev *load_dev;
int started, finished, in_flight, max_in_flight;
int expected_concurrent; /* unlocks that must overlap */
@@ -47,42 +38,6 @@ static struct {
char log[16384];
} fake;
-static const struct gu_card *counted_card_find(u16 device)
-{
- fake.lookups++;
- return gu_card_find(device);
-}
-
-const struct gpu_unlock_610_fw_desc *
-gpu_unlock_610_select_firmware(const struct gu_card *recipe)
-{
- static const struct gpu_unlock_610_fw_desc direct = { .direct = true };
- static const struct gpu_unlock_610_fw_desc hs = { .direct = false };
-
- assert(recipe);
- fake.selections++;
- return recipe->direct_boot ? &direct : &hs;
-}
-
-/* Preparation, boot and every resume must receive the original context. */
-static void check_context(const struct gu_device *device)
-{
- unsigned int index = device->pdev - devices;
-
- assert(index < device_count && device->recipe && device->firmware);
- assert(device->recipe->device == device->pdev->device);
- assert(device->firmware->direct == device->recipe->direct_boot);
- if (fake.contexts[index]) {
- assert(fake.contexts[index] == device);
- assert(fake.recipes[index] == device->recipe);
- assert(fake.firmware[index] == device->firmware);
- } else {
- fake.contexts[index] = device;
- fake.recipes[index] = device->recipe;
- fake.firmware[index] = device->firmware;
- }
-}
-
void test_log(const struct pci_dev *pdev, const char *format, ...)
{
va_list args;
@@ -153,7 +108,6 @@ static void *async_thread(void *arg)
async_cookie_t async_schedule_domain(async_func_t fn, void *data, struct async_domain *domain)
{
- fake.dispatching = true;
struct async_entry *entry = malloc(sizeof(*entry));
async_cookie_t cookie;
@@ -176,27 +130,17 @@ void async_synchronize_full_domain(struct async_domain *domain)
pthread_mutex_unlock(&domain->lock);
}
-int gpu_unlock_610_load_shared(const struct gu_device *device, struct gpu_unlock_610_shared *shared)
+int gpu_unlock_610_load_shared(struct pci_dev *pdev, struct gpu_unlock_610_shared *shared)
{
- struct pci_dev *pdev = device->pdev;
- bool direct = device->firmware->direct;
- check_context(device);
- struct gpu_unlock_610_firmware *assets = direct ? &shared->ga100 : &shared->ga10x;
-
- /* Even a cache hit must finish before the first worker is dispatched. */
- assert(!fake.dispatching && !fake.in_flight);
- if (assets->prepared)
- return assets->error;
+ pthread_mutex_lock(&fake_lock);
+ /* Loaded once per round, before any unlock can use it. */
+ assert(!fake.in_flight && !shared->gsp);
fake.loads++;
- fake.family_loads[direct]++;
fake.load_dev = pdev;
- assets->prepared = true;
- assets->error = fake.fail_load ? fake.fail_load :
- (fake.fail_family == (direct ? 2 : 1) ? -ENOENT : 0);
- if (assets->error)
- return assets->error;
- assets->gsp = fake_gsp;
- shared->booter = fake_gsp;
+ pthread_mutex_unlock(&fake_lock);
+ if (fake.fail_load)
+ return fake.fail_load;
+ shared->gsp = fake_gsp;
fake.loaded = true;
return 0;
}
@@ -206,27 +150,21 @@ void gpu_unlock_610_release_shared(struct gpu_unlock_610_shared *shared)
pthread_mutex_lock(&fake_lock);
/* Only after every unlock has returned. */
assert(!fake.in_flight && fake.finished == fake.started);
- assert(shared->booter == (fake.loaded ? fake_gsp : NULL));
+ assert(shared->gsp == (fake.loaded ? fake_gsp : NULL));
fake.releases++;
fake.loaded = false;
- fake.dispatching = false;
- memset(shared, 0, sizeof(*shared));
+ shared->gsp = NULL;
pthread_mutex_unlock(&fake_lock);
}
-int gpu_unlock_610_unlock(const struct gu_device *device, const struct gpu_unlock_610_shared *shared,
+int gpu_unlock_610_unlock(struct pci_dev *pdev, const struct gpu_unlock_610_shared *shared,
bool resume)
{
- struct pci_dev *pdev = device->pdev;
unsigned int index = pdev - devices;
struct timespec deadline;
int ret;
- check_context(device);
- const struct gpu_unlock_610_firmware *assets = device->firmware->direct ?
- &shared->ga100 : &shared->ga10x;
- assert(assets->prepared && !assets->error && assets->gsp == fake_gsp);
- assert(shared->booter == fake_gsp);
+ assert(shared->gsp == fake_gsp);
assert(pdev->locked && pthread_equal(pdev->owner, pthread_self()));
/* Unbound at boot; after resume only a card nvidia suspended. */
assert(resume ? pdev->driver == &nvidia_driver : !pdev->driver);
@@ -295,11 +233,6 @@ static void reset(char *list)
static void boot(void)
{
assert(!test_module_init());
- assert(fake.lookups == (targets && *targets ? (int)device_count : 0));
- int selected = 0;
- for (unsigned int i = 0; i < device_count; i++)
- selected += !!fake.contexts[i];
- assert(fake.selections == selected);
assert(!fake.in_flight && fake.finished == fake.started);
for (unsigned int i = 0; i < device_count; i++)
assert(!devices[i].locked && fake.unlocks[i] <= 1);
@@ -308,9 +241,7 @@ static void boot(void)
/* Calls the notifier as the PM core does; returns once it has. */
static void notify(unsigned long event)
{
- int lookups = fake.lookups, selections = fake.selections;
assert(fake.nb && fake.nb->notifier_call(fake.nb, event, NULL) == NOTIFY_DONE);
- assert(fake.lookups == lookups && fake.selections == selections);
/* nvidia's notifier runs next: no unlock may still be running. */
assert(!fake.in_flight && fake.finished == fake.started);
}
@@ -329,7 +260,7 @@ static void run(void)
{
boot();
shutdown();
- assert(fake.loads <= (int)device_count && fake.releases <= 1);
+ assert(fake.loads <= 1 && fake.releases <= 1);
}
static bool logged(const char *text)
@@ -355,37 +286,6 @@ int main(void)
assert(logged("0000:41:00.0: gpu_unlock: unlock failed (-16)"));
puts("all: three cards unlocked concurrently, bound card skipped PASS");
- /* Both firmware families are complete before workers start, at boot and resume. */
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", 0x20c2, false);
- add("0000:31:00.0", 0x2082, false);
- fake.expected_concurrent = 3;
- boot();
- assert(fake.family_loads[0] == 1 && fake.family_loads[1] == 1);
- for (unsigned int i = 0; i < device_count; i++)
- devices[i].driver = &nvidia_driver;
- notify(PM_POST_SUSPEND);
- assert(fake.family_loads[0] == 2 && fake.family_loads[1] == 2);
- assert(fake.max_in_flight == 3 && fake.started == 6);
- shutdown();
- puts("mixed families: prepare both before dispatch at boot and resume PASS");
-
- /* A failed family never prevents the other prepared family from running. */
- for (int failed = 1; failed <= 2; failed++) {
- reset("all");
- add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", 0x20c2, false);
- add("0000:31:00.0", 0x2082, false);
- fake.fail_family = failed;
- fake.expected_concurrent = failed == 1 ? 2 : 1;
- run();
- assert(fake.loads == 2 && fake.family_loads[0] == 1 && fake.family_loads[1] == 1);
- assert(fake.unlocks[0] == (failed == 2));
- assert(fake.unlocks[1] == (failed == 1) && fake.unlocks[2] == (failed == 1));
- }
- puts("mixed families: preparation failure isolated in either order PASS");
-
/* A list picks cards in any case; the rest are not touched. */
reset("0000:61:00.0,0000:0A:00.0");
add("0000:01:00.0", RTX_3090, false);
diff --git a/tests/test_packaging.py b/tests/test_packaging.py
index 2365cd2..4b161c4 100644
--- a/tests/test_packaging.py
+++ b/tests/test_packaging.py
@@ -2,13 +2,11 @@
"""Check the generated payload and the shared install layout."""
import hashlib
import json
-import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
-from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "firmware"))
@@ -21,68 +19,6 @@ class PackageTests(unittest.TestCase):
self.assertEqual(digest, generate_v67.PAYLOAD_SHA256)
self.assertEqual(digest, self.manifest()["generated_payload"]["sha256"])
- def test_compiled_payload_header_matches_generator(self):
- self.assertEqual((ROOT / "kernel/gpu_unlock/payload_words.h").read_text(),
- generate_v67.payload_header())
-
- def test_payload_file_is_opt_in(self):
- manifest = self.manifest()
- members = {".PKGINFO": f"pkgver = {generate_v67.PACKAGE_VERSION}\n".encode()}
- for index, item in enumerate(manifest["source_artifacts"][2]["firmware"]):
- blob = f"GSP fixture {index}".encode()
- members[item["path"]] = blob
- item["size"] = len(blob)
- item["sha256"] = hashlib.sha256(blob).hexdigest()
- notice = manifest["source_artifacts"][2]["notice"]
- members[notice["path"]] = b"license fixture"
- notice["sha256"] = hashlib.sha256(members[notice["path"]]).hexdigest()
- with tempfile.TemporaryDirectory() as directory:
- root = Path(directory)
- package = root / "package.tar"
- package.write_bytes(b"package fixture")
- manifest_path = root / "manifest.json"
- manifest_path.write_text(json.dumps(manifest))
- for mode in ("default", "payload", "header"):
- with self.subTest(mode=mode):
- output = root / mode
- header = root / "payload_words.h"
- argv = ["generate_v67.py", str(package), str(output)]
- if mode == "payload":
- argv += ["--payload"]
- if mode == "header":
- argv += ["--header", str(header)]
- with mock.patch.object(generate_v67, "MANIFEST", manifest_path), \
- mock.patch.object(generate_v67, "PACKAGE_SHA256", hashlib.sha256(package.read_bytes()).hexdigest()), \
- mock.patch.object(generate_v67, "extract_member", side_effect=lambda _, name: members[name]), \
- mock.patch.object(sys, "argv", argv):
- self.assertEqual(generate_v67.main(), 0)
- payload = output / "cmp-signature.bin"
- self.assertEqual(payload.exists(), mode == "payload")
- if payload.exists():
- self.assertEqual(payload.read_bytes(), generate_v67.build_payload())
- if mode == "header":
- self.assertEqual(header.read_text(), generate_v67.payload_header())
-
- def test_firmware_install_inventory(self):
- assets = Path(os.environ.get("GPU_UNLOCK_ASSETS", ""))
- manifest = self.manifest()
- names = set(manifest["derived_assets"]) | {"gsp_ga10x.bin", "gsp_tu10x.bin"}
- if not all((assets / name).is_file() for name in names):
- self.skipTest("Stage 1 pinned assets are not available")
- for item in manifest["source_artifacts"][2]["firmware"]:
- data = (assets / Path(item["path"]).name).read_bytes()
- self.assertEqual(len(data), item["size"])
- self.assertEqual(hashlib.sha256(data).hexdigest(), item["sha256"])
- for name, digest in manifest["derived_assets"].items():
- self.assertEqual(hashlib.sha256((assets / name).read_bytes()).hexdigest(), digest)
- with tempfile.TemporaryDirectory() as directory:
- subprocess.run(["make", "-s", f"DESTDIR={directory}", f"ASSETS={assets}",
- "install-firmware"], cwd=ROOT, check=True)
- installed = Path(directory) / "usr/lib/firmware/gpu-unlock/610.43.03"
- self.assertEqual({p.name for p in installed.iterdir()}, names)
- for name in names:
- self.assertEqual((installed / name).read_bytes(), (assets / name).read_bytes())
-
def test_install_uses_standard_paths(self):
version = (ROOT / "VERSION").read_text().strip()
with tempfile.TemporaryDirectory() as directory:
diff --git a/tests/test_payload_610.c b/tests/test_payload_610.c
deleted file mode 100644
index 1ae6de9..0000000
--- a/tests/test_payload_610.c
+++ /dev/null
@@ -1,47 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-#include <assert.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include "payload_610.h"
-#include "recipes.h"
-#include "stage1_tables.h"
-typedef u8 NvU8;
-typedef u32 NvU32;
-typedef u64 NvU64;
-#define SEC2_POSTBL_TIMING_FILL_DWORD 0x4a7
-#include "cmp_payload_reference.h"
-
-int main(int argc, char **argv)
-{
- u8 actual[GU_PAYLOAD_SIZE], expected[GU_PAYLOAD_SIZE];
- if (argc == 3) {
- gu_payload_fill(actual, strtoul(argv[1], NULL, 0), strtoul(argv[2], NULL, 0));
- return fwrite(actual, 1, sizeof(actual), stdout) == sizeof(actual) ? 0 : 1;
- }
- const unsigned int counts[] = { 2, 3, 12 };
- size_t i, j;
- for (i = 0; i < sizeof(counts) / sizeof(counts[0]); i++) {
- const struct gu_write_table *table = test_stage1_table(counts[i]);
- assert(table && table->count == counts[i]);
- for (j = 0; j < table->count; j++) {
- const struct gu_reg_write *e = &table->entries[j];
- gu_payload_fill(actual, e->addr, e->value);
- _kgspSec2PostblTimingFillPayload(expected, sizeof(expected), e->addr, e->value);
- assert(!memcmp(actual, expected, sizeof(actual)));
- }
- }
- assert(!gu_card_find(0xffff) && !gu_card_find(0x2531));
- assert(!test_stage1_table(0) && !test_stage1_table(7));
- const u16 ids[] = { 0x20c2, 0x2082, 0x2204, 0x2414, 0x2484, 0x2504 };
- const u32 chips[] = { 0x170, 0x170, 0x172, 0x173, 0x174, 0x176 };
- for (i = 0; i < sizeof(ids) / sizeof(ids[0]); i++) {
- const struct gu_card *card = gu_card_find(ids[i]);
- assert(card && card->chip == chips[i]);
- assert(card->direct_boot == (i < 2));
- assert(card->writes->privileged.count == (i < 2 ? 12u : 2u) && card->writes->host.count == 2);
- assert(card->writes->privileged.entries[0].addr == 0x1fa7cc);
- assert(card->writes->privileged.entries[0].value == 0xfffff0ff);
- }
- puts("complete payload byte parity against pinned reference, tables and family mappings PASS");
-}
diff --git a/tests/test_prep_610.c b/tests/test_prep_610.c
index 30a2b8a..49abe19 100644
--- a/tests/test_prep_610.c
+++ b/tests/test_prep_610.c
@@ -56,7 +56,7 @@ int main(void)
struct gpu_unlock_wpr_args args = {
.fb_size = in.usable_fb, .vga_offset = in.vga_base,
.frts_size = in.frts_size, .signature_dma = 0x1000,
- .signature_size = GU_PAYLOAD_SIZE, .radix_dma = 0x2000,
+ .signature_size = 0xfa00, .radix_dma = 0x2000,
.image_size = in.image_size, .boot_dma = 0x3000,
.boot = { .code_offset = 256, .data_offset = 512, .image_size = 4096 },
};
diff --git a/tests/test_release.py b/tests/test_release.py
index edb43df..f5586a7 100644
--- a/tests/test_release.py
+++ b/tests/test_release.py
@@ -2,9 +2,6 @@
"""Host checks for the retained module components."""
from pathlib import Path
import os
-import sys
-sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "firmware"))
-import generate_v67
import subprocess
import tempfile
import unittest
@@ -16,19 +13,15 @@ KERNEL = ROOT / "kernel/gpu_unlock"
# against mocked hardware.
ASSETS = next((path for path in (Path(os.environ.get("GPU_UNLOCK_ASSETS", "")),
Path("/usr/lib/firmware/gpu-unlock/610.43.03"))
- if all((path / name).is_file() for name in (
- "gsp_ga10x.bin", "gsp_tu10x.bin", "booter_load_ga100.bin",
- "falcon_loader.bin", "gsp_boot_ga100.bin", "gsp_boot_desc_ga100.bin",
- "gsp_boot_ga102.bin", "gsp_boot_desc_ga102.bin"))), None)
+ if (path / "gsp_ga10x.bin").is_file()), None)
CASES = {
"gfw_eval": ["gfw_eval.c"],
"layout_610": ["layout_610.c"],
"prep_610": ["layout_610.c", "prep_610.c"],
"vbios_610": ["vbios_610.c"],
- "unlock_adapter": ["gfw_eval.c", "prep_610.c", "layout_610.c", "vbios_610.c", "recipes.c", "payload_610.c", "falcon.c"],
- "main": ["recipes.c"],
- "payload_610": ["recipes.c", "payload_610.c"],
+ "unlock_adapter": ["gfw_eval.c", "prep_610.c", "layout_610.c", "vbios_610.c"],
+ "main": [],
}
# Tests that include a kernel source directly build it against a shim.
SHIMS = {
@@ -50,15 +43,11 @@ class ComponentTests(unittest.TestCase):
"-I", str(KERNEL), *shim,
*(str(KERNEL / source) for source in sources),
str(ROOT / "tests" / f"test_{name}.c"),
- "-o", str(output), *(["-lcrypto"] if name == "unlock_adapter" else [])]
+ "-o", str(output)]
subprocess.run(command, check=True)
args = [str(output)]
if name == "unlock_adapter" and ASSETS:
args += ["-", str(ASSETS)]
- if name == "payload_610":
- for address, value in ((0x1fa7cc, 0xfffff0ff), (0x823804, 0xffffffff), (0x9a0148, 0xffffffff)):
- data = subprocess.check_output([str(output), hex(address), hex(value)])
- self.assertEqual(data, generate_v67.build_payload(address, value))
result = subprocess.run(args, capture_output=True, text=True)
self.assertEqual(result.returncode, 0,
(result.stdout + result.stderr)[-4000:])
diff --git a/tests/test_unlock_adapter.c b/tests/test_unlock_adapter.c
index afd436d..778b251 100644
--- a/tests/test_unlock_adapter.c
+++ b/tests/test_unlock_adapter.c
@@ -3,17 +3,14 @@
#include <stdarg.h>
#include <sys/stat.h>
#define __KERNEL__
-#include "../kernel/gpu_unlock/firmware_610.c"
#include "../kernel/gpu_unlock/unlock_610.c"
#include "vbios_fixture.h"
-#include "stage1_tables.h"
-static u32 regs[0x1000000/4];
+static u32 regs[0x900000/4];
static const char *asset_root;
static struct unlock_transaction_work *last_work;
static bool full_active;
static void check_published_image(void);
static void check_restored_signature(void);
-static u32 payload_word(const u8 *p) { return p[0] | (u32)p[1]<<8 | (u32)p[2]<<16 | (u32)p[3]<<24; }
static struct {
u16 cmd,saved;
unsigned long now;
@@ -25,16 +22,11 @@ static struct {
int group_requests,fail_group,group_live,group_before,cache_records,cache_released;
int switch_stuck,dma_commands,dma_stall_after;
int reset_delay,reset_reads[2],reset_pending[2],reset_post_fault;
- int reset_release_stuck,start_read_bad,cpuctl_reads;
int reset_locked_until_flr,hold_writes,bar_ready_after,bar_ready_reads,wpr_after;
int post_gfw,post_gfw_reads,gsp_post_reset_writes;
int fwsec_wpr;u32 fwsec_wpr_lo,fwsec_wpr_hi; /* WPR2 that FWSEC-FRTS enables */
int apertures,regions;
int bound,gfw_late,gfw_late_reads; /* resume: nvidia bound; GFW done after N reads */
- int deny_wpr, deny_reset, fail_write, fail_attempt, write_nohalt, mailbox_error, stale_halt;
- u32 fail_address;
- int corrupt_boot_bundle;
- const char *corrupt_asset;
int busmaster; /* the PCI core's is_busmaster */
char log[32768];
} fake;
@@ -56,33 +48,13 @@ static struct firmware fw={sizeof(fw_image),fw_image};
static void setup(struct unlock_transaction_work *w) {
memset(&fake,0,sizeof(fake)); memset(regs,0,sizeof(regs)); memset(w,0,sizeof(*w));
mask=0xffffffffffffull; dev=(struct pci_dev){.dev={.dma_mask=&mask,.coherent_dma_mask=0xfffffffffull},.vendor=0x10de,.device=0x2204,.refs=1};
- last_work=w; w->card=gu_card_find(dev.device);w->family=w->card->writes;w->load=gpu_unlock_610_select_firmware(w->card);
w->pdev=&dev; w->bar=regs; w->bar_len=sizeof(regs); w->mapped=true;
- fake.cmd=2;fake.saved=2;regs[(GU_SEC2_BASE+0xf4)/4]=1u<<31;regs[(0x110000+0xf4)/4]=1u<<31;
- regs[(GU_SEC2_BASE+0x118)/4]=2;regs[(0x110000+0x118)/4]=2;
+ fake.cmd=2;fake.saved=2;regs[(SEC2_BASE+0xf4)/4]=1u<<31;regs[(0x110000+0xf4)/4]=1u<<31;
+ regs[(SEC2_BASE+0x118)/4]=2;regs[(0x110000+0x118)/4]=2;
regs[GU_FEAT_SS0_OFFSET/4]=0x88888888;regs[GU_FEAT_SS1_OFFSET/4]=0;
regs[0x1fa824/4]=0xfffffff0;regs[0x1fa828/4]=0;
regs[0]=0x172000a1;
}
-/* Standalone Falcon transport fixture: no transaction object in its context. */
-static int standalone_read(void *context,u32 offset,u32 *value) {
- assert(offset<=sizeof(regs)-4);
- *value=readl((u8 *)context+offset);
- if(fake.start_read_bad && offset==GU_SEC2_BASE+GU_FALCON_CPUCTL_OFFSET && ++fake.cpuctl_reads==2)
- *value=~0u;
- return 0;
-}
-static void standalone_write(void *context,u32 offset,u32 value) {
- assert(offset<=sizeof(regs)-4);
- writel(value,(u8 *)context+offset);
-}
-static void evidence_empty(const struct gu_falcon_evidence *e) {
- assert(!e->reset_asserted && !e->reset_released && !e->reloaded && !e->started && !e->halted);
-}
-static struct gu_device resolve_device(struct pci_dev *pdev) {
- const struct gu_card *recipe=gu_card_find(pdev->device);
- return (struct gu_device){pdev,recipe,gpu_unlock_610_select_firmware(recipe)};
-}
void *dma_alloc_coherent(struct device *d,size_t n,dma_addr_t *a,int flags) {
fake.allocs++; if(fake.fail_alloc==fake.allocs)return NULL;
void *p=calloc(1,n); assert(p);
@@ -113,7 +85,7 @@ u32 readl(const void *p) {
if(fake.flrs && n==GU_GFW_CPUCTL_OFFSET/4 && fake.post_gfw==1 && ++fake.post_gfw_reads==5) {
regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
}
- int eng=n==(GU_SEC2_BASE+0x3c0)/4?0:n==0x1103c0/4?1:-1;
+ int eng=n==(SEC2_BASE+0x3c0)/4?0:n==0x1103c0/4?1:-1;
if(eng>=0 && fake.reset_pending[eng] && ++fake.reset_reads[eng]>=fake.reset_delay) {
regs[n]=1;fake.reset_pending[eng]=0;
}
@@ -130,47 +102,28 @@ void writel(u32 v,void *p) {
if(off==GU_FEAT_SS0_OFFSET && fake.selector_bad==1)regs[off/4]=0;
if(off==GU_FEAT_SS1_OFFSET){fake.ss1_writes++;regs[off/4]=v&0xf;} /* 4-bit register, as on the card */
if(off==GU_FEAT_SS1_OFFSET && fake.selector_bad==2)regs[off/4]=0;
- if(off==GU_SEC2_BASE+0x118 || off==0x110118) {
+ if(off==SEC2_BASE+0x118 || off==0x110118) {
fake.dma_commands++;
regs[off/4]=fake.dma_stall_after && fake.dma_commands>=fake.dma_stall_after?1:2;
}
- if(off==GU_SEC2_FALCON2_BASE+0x668 || off==0x111668)regs[off/4]=fake.switch_stuck?0:1;
+ if(off==SEC2_FALCON2+0x668 || off==0x111668)regs[off/4]=fake.switch_stuck?0:1;
if(off==0x110100 || off==0x110130) {
fake.gsp_starts++;
if(fake.fwsec_wpr){regs[0x1fa824/4]=fake.fwsec_wpr_lo;regs[0x1fa828/4]=fake.fwsec_wpr_hi;}
regs[0x110100/4]=fake.fwsec_mode==1?2:0x10;
}
- if(off==GU_SEC2_BASE+0x100 || off==GU_SEC2_BASE+0x130) {
+ if(off==SEC2_BASE+0x100 || off==SEC2_BASE+0x130) {
if(full_active)check_published_image();
fake.starts++;
- struct unlock_transaction_work *w=last_work;
- if(fake.boot_mode==0) {
- u32 addr=w && w->payload_cpu?payload_word((u8 *)w->payload_cpu+0xf76c):GU_FEAT_PLM_OFFSET;
- u32 value=w && w->payload_cpu?payload_word((u8 *)w->payload_cpu+0xf754):~0u;
- if(addr!=fake.fail_address && fake.starts!=fake.fail_write && (!fake.fail_attempt || fake.starts!=fake.fail_attempt))
- regs[addr/4]=value;
- regs[(GU_SEC2_BASE+0x100)/4]=fake.write_nohalt?2:0x10;
- regs[GU_SEC2_RESET_PLM_OFFSET/4]=~0u;
- if(fake.mailbox_error)regs[(GU_SEC2_BASE+0x40)/4]=0x91;
- /* Booter extends WPR2. The next invocation must restore FRTS. */
- regs[GU_WPR2_ADDR_LO_OFFSET/4]-=0x100;
- }
- if(fake.boot_mode==2)regs[(GU_SEC2_BASE+0x100)/4]=~0u;
- if(fake.boot_mode==3){regs[(GU_SEC2_BASE+0x100)/4]=0x10;regs[(GU_SEC2_BASE+0x40)/4]=0x91;}
- }
- if(off==0x1103c0 && (v&1) && !fake.flrs)regs[0x110100/4]=0;
- if(off==GU_SEC2_BASE+0x3c0 && (v&1)) {
- if(regs[(GU_SEC2_BASE+0x100)/4]!=~0u)
- regs[(GU_SEC2_BASE+0x100)/4]=fake.stale_halt?0x10:regs[(GU_SEC2_BASE+0x100)/4]&64;
- if(fake.deny_reset)regs[off/4]=0;
+ if(fake.boot_mode==0)regs[GU_FEAT_PLM_OFFSET/4]=~0u;
+ if(fake.boot_mode==2)regs[(SEC2_BASE+0x100)/4]=~0u;
+ if(fake.boot_mode==3)regs[(SEC2_BASE+0x100)/4]=0x10;
}
- if(fake.reset_release_stuck && (off==GU_SEC2_BASE+0x3c0 || off==0x1103c0) && !(v&1))regs[off/4]|=1;
- if(fake.deny_wpr && (off==GU_WPR2_ADDR_LO_OFFSET || off==GU_WPR2_ADDR_HI_OFFSET))regs[off/4]=0;
- if(fake.fail_reset==2 && (off==GU_SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0;
- if(fake.fail_reset==8 && (off==GU_SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0xbadf0001;
- if((off==GU_SEC2_BASE+0x3c0 || off==0x1103c0) && (v&1) && !(fake.cmd&4)) {
+ if(fake.fail_reset==2 && (off==SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0;
+ if(fake.fail_reset==8 && (off==SEC2_BASE+0x3c0 || off==0x1103c0))regs[off/4]=0xbadf0001;
+ if((off==SEC2_BASE+0x3c0 || off==0x1103c0) && (v&1) && !(fake.cmd&4)) {
fake.hold_writes++;
- int eng=off==GU_SEC2_BASE+0x3c0?0:1;
+ int eng=off==SEC2_BASE+0x3c0?0:1;
if(fake.reset_delay) {
regs[off/4]=0;fake.reset_reads[eng]=0;fake.reset_pending[eng]=1;
}
@@ -207,11 +160,6 @@ int request_firmware(const struct firmware **f,const char *n,struct device *d) {
assert(!fseek(file,0,SEEK_END));long size=ftell(file);assert(size>0);assert(!fseek(file,0,SEEK_SET));
struct firmware *loaded=malloc(sizeof(*loaded));u8 *data=malloc(size);assert(loaded && data);
assert(fread(data,1,size,file)==(size_t)size);fclose(file);
- if(fake.corrupt_boot_bundle && strstr(n,"booter_load_ga100.bin"))data[0]^=1;
- if(fake.corrupt_asset && strstr(n,fake.corrupt_asset)) {
- if(strstr(n,"falcon_loader") || (strstr(n,"gsp_boot_") && !strstr(n,"desc")))size--;
- else data[0]^=1;
- }
*loaded=(struct firmware){size,data};*f=loaded;fake.fw_live++;
/* Linux CONFIG_FW_CACHE: successful request adds a device resource;
* release_firmware frees the image, not this suspend-cache record. */
@@ -221,7 +169,7 @@ void release_firmware(const struct firmware *f) {if(f){free((void *)f->data);fre
void __module_get(void *p) {fake.pins++;}
int pci_read_config_word(struct pci_dev *p,int r,u16 *v) {*v=r==PCI_COMMAND?fake.cmd:r==PCI_VENDOR_ID?0x10de:0;return 0;}
int pci_read_config_dword(struct pci_dev *p,int r,u32 *v) {
- *v=r==PCI_VENDOR_ID?(fake.flrs && fake.lost_identity?~0u:pci_identity(p)):(u32)fake.bar_bad;return 0;
+ *v=r==PCI_VENDOR_ID?(fake.flrs && fake.lost_identity?~0u:0x220410de):(u32)fake.bar_bad;return 0;
}
int pci_write_config_word(struct pci_dev *p,int r,u16 v) {assert(r==PCI_COMMAND);fake.cmd=v|(fake.master_stuck?4:0);return 0;}
int pcie_capability_read_word(struct pci_dev *p,int r,u16 *v) {*v=r==PCI_EXP_DEVSTA?fake.pending:0;return 0;}
@@ -238,7 +186,7 @@ int aperture_remove_conflicting_pci_devices(struct pci_dev *p,const char *n) {
/* The framebuffer stays through FWSEC and Booter; it goes right before FLR.
* After resume nvidia-drm already replaced it. */
assert(!fake.flrs && !(fake.cmd&4) && !fake.bound);
- if(full_active)assert(fake.starts==2);
+ if(full_active)assert(fake.starts==1);
fake.apertures++;return 0;
}
int pci_enable_device_mem(struct pci_dev *p) {fake.cmd|=2;return 0;}
@@ -258,7 +206,7 @@ int pcie_flr(struct pci_dev *p) {
fake.flrs++;
if(fake.late_devres)p->dev.devres_head.count++;
/* FLR releases engine reset; post-FLR assertion must work independently. */
- regs[(GU_SEC2_BASE+0x3c0)/4]=regs[0x1103c0/4]=0;
+ regs[(SEC2_BASE+0x3c0)/4]=regs[0x1103c0/4]=0;
regs[GU_GFW_CPUCTL_OFFSET/4]=0x10;
regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
if(fake.post_gfw==1 || fake.post_gfw==2) {
@@ -286,15 +234,10 @@ static const u8 *dma_page(u64 address) {
}
static void check_published_image(void) {
struct unlock_transaction_work *w=last_work;const u8 *meta=w->meta_cpu;
- assert((w->state == GU_PUBLISHED || w->state == GU_ACTIVE) && !fake.pins && (fake.cmd&4));
+ assert(w->published && !fake.pins && (fake.cmd&4));
assert(get64(meta)==GU_WPR_META_MAGIC && get64(meta+8)==1);
- assert(get64(meta+72)==w->payload_dma && get64(meta+80)==GU_PAYLOAD_SIZE);
- u8 expected[GU_PAYLOAD_SIZE];
- const struct gu_reg_write *entry=&w->family->privileged.entries[w->write_index];
- gu_payload_fill(expected,entry->addr,entry->value);
- assert(!memcmp(w->payload_cpu,expected,GU_PAYLOAD_SIZE));
- assert(regs[GU_WPR2_ADDR_LO_OFFSET/4]==w->frts_lo && regs[GU_WPR2_ADDR_HI_OFFSET/4]==w->frts_hi);
- assert(!fake.flrs && w->payload_dma==get64(meta+72));
+ assert(get64(meta+72)==w->v67_dma && get64(meta+80)==GU_V67_SIZE);
+ assert(!memcmp(w->v67_cpu,w->v67_fw->data,GU_V67_SIZE));
assert(get64(meta+112)==0x5f5400000ull && get64(meta+136)==0x5fada0000ull);
assert(get64(meta+144)==0x5ffdfa000ull && get64(meta+152)==0x5ffe00000ull);
assert(get64(meta+168)==0x5fff00000ull && get64(meta+176)==0x600000000ull);
@@ -311,12 +254,12 @@ static void check_published_image(void) {
for(size_t i=bytes;i<4096;i++)assert(!leaf[i]);
}
assert(!memcmp(w->boot_cpu,w->boot_fw->data,w->boot_desc.image_size));
- /* The shared Booter has one signature; each device patches its own image. */
- assert(!memcmp((u8 *)w->booter_cpu+35072,w->booter_fw->data+72+60160,384));
+ /* SEC2 fuse v1 selects slot 0 of the two-signature Booter asset. */
+ assert(!memcmp((u8 *)w->booter_cpu+35344,w->booter_fw->data+72+60416,384));
}
static void check_restored_signature(void) {
struct unlock_transaction_work *w=last_work;const u8 *meta=w->meta_cpu;
- assert(w->state==GU_STOCK && get64(meta+72)==w->stock_dma && get64(meta+80)==4096);
+ assert(w->signature_calls==2 && get64(meta+72)==w->stock_dma && get64(meta+80)==4096);
assert(!memcmp(w->stock_cpu,w->gsp_fw->data+w->sig_span.offset,4096));
assert(!(fake.cmd&4));
/* All published buffers remain alive; FLR must precede engine holds. */
@@ -341,59 +284,31 @@ static void load_rom(const char *path) {
int main(int argc, char **argv) {
struct unlock_transaction_work w;
setup(&w);
- struct gu_falcon_io io={.dev=&dev.dev,.context=regs,.read=standalone_read,.write=standalone_write};
- struct gu_falcon_evidence evidence;
- /* Unknown devices have no recipe or firmware fallback, even with an empty
- * shared cache. No request, PCI ownership or transaction allocation occurs. */
- dev.device=0xffff;
- struct gu_device unknown=resolve_device(&dev);
- struct gpu_unlock_610_shared empty={0};
- assert(!unknown.recipe && !unknown.firmware);
- assert(gpu_unlock_610_load_shared(&unknown,&empty)==-ENODEV);
- assert(gpu_unlock_610_unlock(&unknown,&empty,false)==-ENODEV);
- assert(!fake.fw_requests && !fake.allocs && !fake.writes && dev.refs==1);
- const u16 ids[]={0x20c2,0x2082,0x2204,0x2414,0x2484,0x2504};
- for(unsigned i=0;i<ARRAY_SIZE(ids);i++) {
- dev.device=ids[i];
- struct gu_device resolved=resolve_device(&dev);
- assert(resolved.recipe && resolved.firmware);
- assert(resolved.firmware->direct==(i<2));
- assert(!strcmp(resolved.firmware->signature_section,i<2?".fwsignature_ga100":".fwsignature_ga10x"));
- assert(resolved.firmware->boot_size==(i<2?4096:24576));
- if(i>2)assert(resolved.firmware==gpu_unlock_610_select_firmware(gu_card_find(0x2204)));
- }
- setup(&w);
regs[0x110118/4]=0x1230;
- assert(gu_falcon_poll(&io,0x110118,2,2,100,true,NULL)==-ETIMEDOUT);
+ assert(poll_us(&w,0x110118,2,2,100,true)==-ETIMEDOUT);
assert(strstr(fake.log,"poll timeout: reg=0x110118 last=0x1230 mask=0x2 expected=0x2 budget_us=100"));
- assert(!strstr(fake.log,"failure unlock_610.c"));
+ assert(strstr(fake.log,"poll_us error=-110"));
assert(!fake.writes);
setup(&w);
regs[0x110118/4]=~0u;
- assert(gu_falcon_poll(&io,0x110118,2,2,100,true,NULL)==-EIO);
+ assert(poll_us(&w,0x110118,2,2,100,true)==-EIO);
assert(strstr(fake.log,"poll inaccessible: reg=0x110118 last=0xffffffff"));
/* Exercise the dual-core reset path, including RISC-V -> Falcon switching
* and the documented tolerated pre-reset timeout. */
- for(int mode=0;mode<9;mode++) {
+ for(int mode=0;mode<8;mode++) {
setup(&w);
- const struct gu_falcon_engine eng={GU_SEC2_BASE,GU_SEC2_FBIF_BASE,GU_SEC2_FALCON2_BASE};
- regs[(GU_SEC2_BASE+0xf4)/4]|=1u<<10;
- regs[(GU_SEC2_FALCON2_BASE+0x668)/4]=mode==0?1:0x10;
- if(mode==2)regs[(GU_SEC2_FALCON2_BASE+0x668)/4]=0xbadf0001;
+ const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
+ regs[(SEC2_BASE+0xf4)/4]|=1u<<10;
+ regs[(SEC2_FALCON2+0x668)/4]=mode==0?1:0x10;
+ if(mode==2)regs[(SEC2_FALCON2+0x668)/4]=0xbadf0001;
if(mode==3)fake.switch_stuck=1;
- if(mode==4)regs[(GU_SEC2_BASE+0xf4)/4]|=1u<<12;
- if(mode==5)regs[(GU_SEC2_BASE+0xf4)/4]&=~(1u<<31);
+ if(mode==4)regs[(SEC2_BASE+0xf4)/4]|=1u<<12;
+ if(mode==5)regs[(SEC2_BASE+0xf4)/4]&=~(1u<<31);
if(mode==6)regs[0]=~0u;
- if(mode==7)regs[(GU_SEC2_BASE+0x3c0)/4]=0xbadf0001;
- if(mode==8)fake.reset_release_stuck=1;
- struct unlock_transaction_work unchanged=w;
- int ret=gu_falcon_reset(&io,&eng,&evidence);
- assert(!memcmp(&w,&unchanged,sizeof(w)));
- assert(!evidence.started && !evidence.reloaded && !evidence.halted);
- assert(evidence.reset_asserted==(mode!=7));
- assert(evidence.reset_released==(mode!=7 && mode!=8));
+ if(mode==7)regs[(SEC2_BASE+0x3c0)/4]=0xbadf0001;
+ int ret=falcon_reset(&w,&eng);
assert((!ret)==(mode==0 || mode==1 || mode==5));
- if(mode==1)assert(regs[(GU_SEC2_FALCON2_BASE+0x668)/4]==1);
+ if(mode==1)assert(regs[(SEC2_FALCON2+0x668)/4]==1);
if(mode==5){
assert(strstr(fake.log,"pre-reset timeout tolerated"));
assert(strstr(fake.log,"poll timeout tolerated:"));
@@ -405,27 +320,21 @@ int main(int argc, char **argv) {
}
for(int mode=0;mode<3;mode++) {
setup(&w);
- if(mode==0)regs[(GU_SEC2_BASE+0x118)/4]=1;
+ if(mode==0)regs[(SEC2_BASE+0x118)/4]=1;
else fake.dma_stall_after=1;
- assert(gu_falcon_dma(&io,GU_SEC2_BASE,0,0,0x100000,mode==2?256:512,0x614)==-ETIMEDOUT);
+ assert(falcon_dma(&w,SEC2_BASE,0,0,0x100000,mode==2?256:512,0x614)==-ETIMEDOUT);
assert(fake.dma_commands==(mode?1:0) && fake.now<=1000100);
if(mode==1)assert(strstr(fake.log,"Falcon DMA progress: done=256 total=512"));
}
for(int mode=0;mode<4;mode++) {
setup(&w);
- const struct gu_falcon_engine eng={GU_SEC2_BASE,GU_SEC2_FBIF_BASE,GU_SEC2_FALCON2_BASE};
+ const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
w.plm_entry=0x8f;
- if(mode==0)regs[(GU_SEC2_FBIF_BASE+0x24)/4]=0xbadf0001;
- if(mode==1)regs[GU_SEC2_FBIF_BASE/4]=~0u;
- if(mode==2)regs[(GU_SEC2_BASE+0x100)/4]=~0u;
- if(mode==3)regs[(GU_SEC2_BASE+0x100)/4]=1u<<6; /* CPUCTL alias path */
- const struct gu_falcon_hs_image program={.dma=0x100000,.code_size=256,
- .data_off=256,.data_size=256,.engine_mask=1,.ucode_id=3};
- struct unlock_transaction_work unchanged=w;
- int ret=gu_falcon_hs(&io,&eng,&program,&evidence);
- assert(!memcmp(&w,&unchanged,sizeof(w)));
- assert(evidence.reset_asserted && evidence.reset_released);
- assert(evidence.reloaded==(mode==3) && evidence.started==(mode==3) && evidence.halted==(mode==3));
+ if(mode==0)regs[(SEC2_FBIF+0x24)/4]=0xbadf0001;
+ if(mode==1)regs[SEC2_FBIF/4]=~0u;
+ if(mode==2)regs[(SEC2_BASE+0x100)/4]=~0u;
+ if(mode==3)regs[(SEC2_BASE+0x100)/4]=1u<<6; /* CPUCTL alias path */
+ int ret=falcon_hs(&w,&eng,0,256,0,0,256,256,0,0,1,3,0x100000,0,0,true);
assert((!ret)==(mode==3));
assert(fake.starts==(mode==3?1:0));
if(mode<2)assert(!fake.dma_commands);
@@ -437,78 +346,20 @@ int main(int argc, char **argv) {
dma_addr_t src=bad==2?(1ull<<49)-256:0x100000;
u32 size=bad==3?257:512;
if(bad==4)src++;
- assert(gu_falcon_dma(&io,GU_SEC2_BASE,dest,off,src,size,0x614)==-EINVAL);
+ assert(falcon_dma(&w,SEC2_BASE,dest,off,src,size,0x614)==-EINVAL);
assert(!fake.writes);
}
{
- const struct gu_falcon_engine eng={GU_SEC2_BASE,GU_SEC2_FBIF_BASE,GU_SEC2_FALCON2_BASE};
+ const struct falcon_eng eng={SEC2_BASE,SEC2_FBIF,SEC2_FALCON2};
setup(&w);
- struct gu_falcon_hs_image program={.dma=0x100000,.code_size=256,.code_va=0x200000,
- .data_off=256,.data_size=256,.engine_mask=1,.ucode_id=3};
- evidence=(struct gu_falcon_evidence){true,true,true,true,true};
- assert(gu_falcon_hs(&io,&eng,&program,&evidence)==-ERANGE);
- evidence_empty(&evidence);
+ assert(falcon_hs(&w,&eng,0,256,0x200000,0,256,256,0,0,1,3,
+ 0x100000,0,0,false)==-ERANGE);
assert(!fake.writes);
/* Bad DMEM geometry must not leave an already-submitted IMEM DMA. */
- program.code_va=0;program.data_size=257;
- evidence=(struct gu_falcon_evidence){true,true,true,true,true};
- assert(gu_falcon_hs(&io,&eng,&program,&evidence)==-EINVAL);
- evidence_empty(&evidence);
+ assert(falcon_hs(&w,&eng,0,256,0,0,256,257,0,0,1,3,
+ 0x100000,0,0,false)==-EINVAL);
assert(!fake.writes);
}
- /* All launch variants return partial evidence without changing transaction
- * policy/ownership, including submitted starts that never produce HALT. */
- for(int variant=0;variant<3;variant++)
- for(int mode=0;mode<5;mode++) {
- if(variant==2 && mode==4)continue;
- setup(&w);
- const struct gu_falcon_engine eng=variant==2?
- (struct gu_falcon_engine){GU_GSP_BASE,GU_GSP_FBIF_BASE,GU_GSP_FALCON2_BASE}:
- (struct gu_falcon_engine){GU_SEC2_BASE,GU_SEC2_FBIF_BASE,GU_SEC2_FALCON2_BASE};
- u32 words[192]={0};
- const struct gu_falcon_hs_image hs={.dma=0x100000,.code_size=256,
- .data_off=256,.data_size=256,.engine_mask=1,.ucode_id=3};
- const struct gu_falcon_direct_image direct={.words=words,.os_code_size=256,
- .app_code_off=256,.app_code_size=256,.os_data_off=512,.os_data_size=256};
- const struct gu_falcon_loader_image loader={.loader=words,.loader_size=sizeof(words),
- .dma=0x100000,.data_off=512,.imem_ns_size=256,.imem_sec_pa=256,
- .imem_sec_size=256,.dmem_size=256};
- regs[(GU_GSP_BASE+GU_FALCON_HWCFG_OFFSET)/4]=256;
- if(mode==1){fake.write_nohalt=1;fake.fwsec_mode=1;}
- if(mode==2)fake.fail_reset=2;
- if(mode==3) {
- u32 offset=variant==0?eng.fbif+GU_FBIF_CTL_OFFSET:
- variant==1?eng.base+GU_FALCON_DMACTL_OFFSET:eng.fbif+GU_FBIF_TRANSCFG(4);
- regs[offset/4]=~0u;
- }
- if(mode==4)fake.start_read_bad=1;
- evidence=(struct gu_falcon_evidence){true,true,true,true,true};
- struct unlock_transaction_work unchanged=w;
- int ret=variant==0?gu_falcon_hs(&io,&eng,&hs,&evidence):
- variant==1?gu_falcon_direct(&io,&eng,&direct,&evidence):
- gu_falcon_loader(&io,&eng,&loader,&evidence);
- assert((!ret)==(mode==0) && !memcmp(&w,&unchanged,sizeof(w)));
- assert(evidence.reset_asserted==(mode!=2) && evidence.reset_released==(mode!=2));
- assert(evidence.reloaded==(mode<2 || mode==4));
- assert(evidence.started==(mode<2) && evidence.halted==(mode==0));
- assert(fake.starts+fake.gsp_starts==(mode<2));
- if(mode==1 || mode==2)assert(ret==-ETIMEDOUT);
- assert(!fake.allocs && !fake.live && !fake.regions && !fake.flrs && !fake.pins);
- }
- /* Generic loader validation failures must clear prior evidence before reset. */
- for(int bad=0;bad<4;bad++) {
- setup(&w);
- u32 code[192]={0};
- const struct gu_falcon_engine eng={GU_GSP_BASE,GU_GSP_FBIF_BASE,GU_GSP_FALCON2_BASE};
- struct gu_falcon_loader_image program={.loader=code,.loader_size=sizeof(code),.dma=0x100000};
- if(bad==0)program.loader=NULL;
- if(bad==1)program.loader_size--;
- if(bad==2)program.dma++;
- if(bad==3)program.data_off=1;
- evidence=(struct gu_falcon_evidence){true,true,true,true,true};
- assert(gu_falcon_loader(&io,&eng,&program,&evidence)==-EINVAL && !fake.writes);
- evidence_empty(&evidence);
- }
/* Actual PROM -> descriptor -> patch -> GSP DMA/start -> WPR validation.
* Faults after firmware access retain DMA, without starting SEC2. */
for(int mode=0;mode<10;mode++) {
@@ -610,20 +461,19 @@ int main(int argc, char **argv) {
struct unlock_transaction_page page={.dma=0x200000};
u8 metadata[GU_WPR_META_SIZE], stock[4096], before[GU_WPR_META_SIZE];
w.meta_cpu=metadata;w.stock_cpu=stock;w.stock_dma=0x400000;w.stock_size=4096;
- w.payload_dma=0x300000;w.boot_dma=0x500000;w.pages=&page;w.pages_allocated=1;
+ w.v67_dma=0x300000;w.boot_dma=0x500000;w.pages=&page;w.pages_allocated=1;
w.layout_in=(struct gpu_unlock_layout_in){.usable_fb=24ull<<30,
.vga_base=(24ull<<30)-(1ull<<20),.frts_size=1ull<<20,
.image_size=4097,.boot_size=24576,.nonwpr_size=1ull<<20,.heap_request=88ull<<20};
assert(!gpu_unlock_layout(&w.layout_in,&w.layout));w.image_span.size=4097;
w.boot_desc=(struct gpu_unlock_boot_desc){.code_offset=6144,.data_offset=2048,.image_size=24576};
- assert(!publish_signature(&w) && w.state==GU_PUBLISHED);
- assert(get64(metadata+72)==w.payload_dma && get64(metadata+80)==GU_PAYLOAD_SIZE);
+ assert(!unlock_transaction_signature(&w) && w.published && w.signature_calls==1);
+ assert(get64(metadata+72)==w.v67_dma && get64(metadata+80)==GU_V67_SIZE);
memcpy(before,metadata,sizeof(before));
- assert(!restore_stock_signature(&w) && w.state==GU_STOCK);
+ assert(!unlock_transaction_signature(&w) && w.signature_calls==2);
assert(get64(metadata+72)==w.stock_dma && get64(metadata+80)==4096);
assert(!memcmp(before,metadata,72) && !memcmp(before+88,metadata+88,sizeof(before)-88));
- assert(restore_stock_signature(&w)==-EINVAL && fake.barriers==2);
- assert(publish_signature(&w)==-EINVAL && fake.barriers==2);
+ assert(unlock_transaction_signature(&w)==-EINVAL && fake.barriers==2);
}
/* Exercise every coherent-allocation failure in a real multi-page radix. */
for(int fail=0;fail<=5;fail++) {
@@ -643,96 +493,35 @@ int main(int argc, char **argv) {
}
free_coherent(&w);assert(fake.live==0);
}
- /* The real shared table runner, using small synthetic firmware buffers.
- * Each mode runs through restore/refill/reset/reload/start/completion.
- * No second FWSEC or FLR can appear inside this loop. */
- for(int mode=0;mode<16;mode++) {
- setup(&w);
- u8 payload[GU_PAYLOAD_SIZE],metadata[GU_WPR_META_SIZE]={0};
- w.payload_cpu=payload;w.meta_cpu=metadata;w.payload_dma=0x100100000ull;
- w.meta_dma=0x100200000ull;
- w.master=w.firmware_started=true;w.state=GU_PUBLISHED;
- w.frts_saved=true;w.frts_lo=0x2000;w.frts_hi=0x2ff0;
- w.booter_hdr=(struct gpu_unlock_booter_hdr){.app_code_off=256,.app_code_size=256,
- .os_data_off=512,.os_data_size=256,.patch_loc=512,.engine_id=1,.ucode_id=3};
- w.booter_dma=0x100000;
- const struct gu_reg_write custom[]={
- {0x1fa7cc,0xfffff0ff},{0x823804,~0u},{0x823804,~0u},{0x9a0148,~0u}};
- const struct gu_write_table table=GU_WRITE_TABLE(custom);
- if(mode==1)fake.deny_reset=1;
- if(mode==2)fake.deny_wpr=1;
- if(mode==3)fake.write_nohalt=1;
- if(mode==4)fake.boot_mode=3;
- if(mode==5)fake.fail_write=2; /* one retry, then succeeds */
- if(mode==6)fake.boot_mode=1;
- if(mode==7)fake.boot_mode=2;
- if(mode==8)fake.mailbox_error=1;
- if(mode==9){fake.stale_halt=1;regs[(GU_SEC2_BASE+0x100)/4]=16;}
- if(mode==10)fake.fail_attempt=1;
- if(mode==11)w.load=gpu_unlock_610_select_firmware(gu_card_find(0x20c2));
- if(mode==12)fake.fail_address=0x823804;
- if(mode==13)regs[0x1fa7cc/4]=0xfffff0ff;
- if(mode==14)fake.selector_bad=1;
- if(mode==15)w.load=gpu_unlock_610_select_firmware(gu_card_find(0x20c2));
- u8 image[1024]={0};w.booter_cpu=image;
- int ret=run_privileged_writes(&w,&table);
- bool success=mode==0 || mode==5 || mode==10 || mode==11 || mode==13 || mode==14 || mode==15;
- if((!ret)!=success)fprintf(stderr,"runner mode %d: %s",mode,fake.log);
- assert((!ret)==success);
- assert(!fake.flrs && !fake.gsp_starts && !fake.allocs);
- if(mode==1) {
- const char *failure=strstr(fake.log,"failure unlock_610.c:");
- assert(failure && !strstr(failure+1,"failure unlock_610.c:"));
- }
- if(success) {
- assert(fake.starts==(mode==5 || mode==10?5:4));
- assert(w.fresh && w.cleanup_complete && w.state==GU_ACTIVE);
- assert(regs[GU_WPR2_ADDR_LO_OFFSET/4]==w.frts_lo && regs[GU_WPR2_ADDR_HI_OFFSET/4]==w.frts_hi);
- assert(get64(metadata+72)==w.payload_dma && get64(metadata+80)==GU_PAYLOAD_SIZE);
- int starts=fake.starts;
- const struct gu_write_table empty={0};
- assert(!run_host_writes(&w,&empty) && fake.starts==starts);
- const struct gu_reg_write host[]={ {0x82381c,0x88888888},{0x823820,8} };
- const struct gu_write_table host_table=GU_WRITE_TABLE(host);
- int host_ret=run_host_writes(&w,&host_table);
- assert(host_ret==(mode==14?-EIO:0) && fake.starts==starts);
- if(mode==14){assert(!fake.ss1_writes);continue;}
- assert(regs[GU_FEAT_SS0_OFFSET/4]==0x88888888 && regs[GU_FEAT_SS1_OFFSET/4]==8);
- } else {
- assert(fake.starts<(mode==12?4:2) && w.state==GU_ACTIVE);
- if(mode==12)assert(fake.starts==3 && !regs[0x9a0148/4]);
- if(mode==1 || mode==2 || mode==9)assert(!fake.starts);
- if(mode==3)assert(w.target_after==0xfffff0ff && !w.invocation_halted);
- if(mode==4 || mode==8)assert(w.invocation_halted && !w.cleanup_complete);
- }
- }
- for(unsigned int count=2;count<=12;count=count==2?3:count==3?12:13) {
- setup(&w);
- u8 payload[GU_PAYLOAD_SIZE], metadata[GU_WPR_META_SIZE]={0};
- w.payload_cpu=payload;w.meta_cpu=metadata;w.payload_dma=0x300000;w.meta_dma=0x400000;
- w.master=w.firmware_started=true;w.state=GU_PUBLISHED;
- w.frts_saved=true;w.frts_lo=0x2000;w.frts_hi=0x2ff0;
+ /* Early PLM polling must not wait for HALT or accept inaccessible engines. */
+ for(int mode=0;mode<4;mode++) {
+ setup(&w);fake.boot_mode=mode;w.master=w.firmware_started=w.published=true;
w.booter_hdr=(struct gpu_unlock_booter_hdr){.app_code_off=256,.app_code_size=256,
.os_data_off=512,.os_data_size=256,.patch_loc=512,.engine_id=1,.ucode_id=3};
- w.booter_dma=0x100000;
- assert(!run_privileged_writes(&w,test_stage1_table(count)) && fake.starts==(int)count);
- assert(!fake.flrs && !fake.gsp_starts && !fake.allocs);
+ w.booter_dma=0x100000;w.meta_dma=0x200000;w.plm_entry=0x8f;
+ int ret=unlock_transaction_boot(&w);
+ assert(fake.starts==1);
+ if(mode==0)assert(!ret);
+ if(mode==1)assert(ret==-ETIMEDOUT && fake.now<=5000100);
+ if(mode==1)assert(strstr(fake.log,"early PLM failed: pci=0x220410de"));
+ if(mode==2)assert(ret==-ENODEV);
+ if(mode==3)assert(ret==-EIO);
}
/* Failed FLR/reset/BAR checks retain allocations; success excludes fetches. */
for(int failure=0;failure<10;failure++) {
setup(&w);w.firmware_started=true;w.original_cmd=2;w.cmd_valid=true;
- w.payload_cpu=dma_alloc_coherent(&dev.dev,GU_PAYLOAD_SIZE,&w.payload_dma,0);
+ w.v67_cpu=dma_alloc_coherent(&dev.dev,GU_V67_SIZE,&w.v67_dma,0);
fake.fail_reset=(failure<3 || failure==8)?failure:0;fake.bar_bad=failure==3;
fake.pending=failure==4?PCI_EXP_DEVSTA_TRPND:0;
fake.lost_identity=failure==5;fake.corrupt_selector=failure==6?1:failure==9?2:0;fake.alien_wpr=failure==7;
- w.state=GU_STOCK;regs[GU_FEAT_SS1_OFFSET/4]=0x8;
+ w.signature_calls=2;regs[GU_FEAT_SS1_OFFSET/4]=0x8;
int ret=unlock_transaction_reset(&w);
if(failure==6 || failure==9) {
/* Lost selector state fails the unlock, but proven reset permits cleanup. */
- assert(ret==-EIO && (w.state == GU_QUIESCED));
+ assert(ret==-EIO && w.quiesced);
assert(!unlock_transaction_unwind(&w) && !fake.live && !w.retained);
} else if(failure) {
- assert(ret && !(w.state == GU_QUIESCED));
+ assert(ret && !w.quiesced);
assert(unlock_transaction_unwind(&w)==-EBUSY && w.retained && fake.live==1);
assert(strstr(fake.log,"failure unlock_610.c:"));
if(failure==1)assert(strstr(fake.log,"PCIe FLR returned: result=-5"));
@@ -740,8 +529,8 @@ int main(int argc, char **argv) {
/* Host harness disposal only; the kernel intentionally retains. */
free_coherent(&w);
} else {
- assert(!ret && (w.state == GU_QUIESCED) && fake.flrs==1);
- assert(!(regs[(GU_SEC2_BASE+0x3c0)/4]&1) && !regs[0x1103c0/4]);
+ assert(!ret && w.quiesced && fake.flrs==1);
+ assert(!(regs[(SEC2_BASE+0x3c0)/4]&1) && !regs[0x1103c0/4]);
assert(!unlock_transaction_unwind(&w) && fake.live==0);
}
}
@@ -765,7 +554,7 @@ int main(int argc, char **argv) {
"warm-wpr2","ss1-readback","ss1-lost","power-unknown","power-unknown-no-pm",
"firmware-bme","firmware-bme-stuck",
"resume-success","resume-gfw-late","resume-kept-power","resume-gsp-running",
- "resume-gfw-timeout","resume-oom","chip-mismatch","resume-chip-mismatch"};
+ "resume-gfw-timeout","resume-oom"};
int total_allocations=0;
for(unsigned scenario=0;scenario<ARRAY_SIZE(cases);scenario++) {
setup(&w);
@@ -785,8 +574,7 @@ int main(int argc, char **argv) {
if(strcmp(argv[1],"-"))regs[(0x8241c0+8*4)/4]=1;
if(scenario==1){regs[GU_FEAT_SS0_OFFSET/4]=0x88888888;regs[GU_FEAT_SS1_OFFSET/4]=0x8;}
if(scenario==2)fake.fail_context=1;
- if(scenario>=3 && scenario<=6)fake.fail_fw=scenario-2;
- if(scenario==7)fake.corrupt_asset="gsp_boot_desc_ga102.bin";
+ if(scenario>=3 && scenario<=7)fake.fail_fw=scenario-2;
if(scenario==8)fake.fail_claim=1;
if(scenario==9)fake.fail_map=1;
if(scenario==10)fake.fail_save=1;
@@ -847,24 +635,10 @@ int main(int argc, char **argv) {
}
if(!strcmp(cases[scenario],"resume-gfw-timeout"))regs[GU_GFW_BOOT_OFFSET/4]=0;
if(!strcmp(cases[scenario],"resume-oom"))fake.fail_alloc=1;
- if(strstr(cases[scenario],"chip-mismatch"))regs[0]=0x174000a1;
/* As main.c: the GSP file is loaded once, ahead of every card. */
struct gpu_unlock_610_shared shared={0};last_work=NULL;
- struct gu_device device=resolve_device(&dev);
- int ret=gpu_unlock_610_load_shared(&device,&shared);
- int prepared_requests=fake.fw_requests;
- struct gpu_unlock_610_shared snapshot=shared;
- u8 before[3][SHA256_DIGEST_SIZE],after[SHA256_DIGEST_SIZE];
- const struct firmware *images[]={shared.ga10x.gsp,shared.ga10x.boot,shared.booter};
- if(!scenario)
- for(unsigned i=0;i<ARRAY_SIZE(images);i++)sha256(images[i]->data,images[i]->size,before[i]);
- if(!ret)ret=gpu_unlock_610_unlock(&device,&shared,bound);
- assert(fake.fw_requests==prepared_requests && !memcmp(&shared,&snapshot,sizeof(shared)));
- if(!scenario)
- for(unsigned i=0;i<ARRAY_SIZE(images);i++) {
- sha256(images[i]->data,images[i]->size,after);
- assert(!memcmp(before[i],after,sizeof(after)));
- }
+ int ret=gpu_unlock_610_load_shared(&dev,&shared);
+ if(!ret)ret=gpu_unlock_610_unlock(&dev,&shared,bound);
gpu_unlock_610_release_shared(&shared);
bool expected=scenario<2 || (scenario>=33 && scenario<=35) || scenario==39 || scenario==40 || scenario==43 || scenario==46 ||
scenario==58 || scenario==60 ||
@@ -873,15 +647,15 @@ int main(int argc, char **argv) {
if((!ret)!=expected)fprintf(stderr,"%s\n%s: error=%d\n",fake.log,cases[scenario],ret);
assert((!ret)==expected);
if(expected) {
- assert(fake.gsp_starts==1 && fake.starts==2 && fake.flrs==1 && !fake.pins);
+ assert(fake.gsp_starts==1 && fake.starts==1 && fake.flrs==1 && !fake.pins);
assert(fake.apertures==!bound && !fake.regions);
- assert(fake.selector_writes==1 && fake.ss1_writes==1);
+ assert(fake.selector_writes==(scenario==1?0:1) && fake.ss1_writes==(scenario==1?0:1));
assert(regs[GU_FEAT_SS0_OFFSET/4]==0x88888888 && regs[GU_FEAT_SS1_OFFSET/4]==0x8);
assert(mask==0xffffffffffffull && dev.dev.coherent_dma_mask==0xfffffffffull);
/* Mirror Linux really_probe: driver entry rejects nonempty devres.
* A bound driver keeps its own. */
assert(dev.dev.devres_head.count==(bound?1u:0u));
- assert(fake.cache_records==4 && fake.cache_released==4);
+ assert(fake.cache_records==5 && fake.cache_released==5);
/* Pinned RM checks GSP halt + readable completed GFW BEFORE
* its own reset/bootstrap. Preserve that entry contract. */
assert((regs[GU_GFW_CPUCTL_OFFSET/4]&0x10) && (regs[GU_GFW_PLM_OFFSET/4]&1));
@@ -889,15 +663,15 @@ int main(int argc, char **argv) {
assert(!(regs[0x1fa828/4]>>4)); /* Stock RM rejects any active WPR2. */
if(!scenario)total_allocations=fake.allocs;
}
- if(scenario>=34 && scenario<=50 && scenario!=36) {
- assert(fake.starts==2 && fake.flrs==1);
- if(!expected)assert(retained && !(last_work->state == GU_QUIESCED));
+ if(scenario>=34 && scenario<=50) {
+ assert(fake.starts==1 && fake.flrs==1);
+ if(!expected)assert(retained && !last_work->quiesced);
if(scenario==41 || scenario==42)assert(!fake.hold_writes && ret==-ETIMEDOUT);
if(scenario==39)assert(fake.hold_writes==1);
if(scenario==47 || scenario==48)assert(!fake.hold_writes && ret==-ETIMEDOUT);
}
assert(!fake.gsp_post_reset_writes);
- assert(fake.gsp_starts<=1 && fake.starts<=2 && fake.flrs<=1 && fake.pins<=1);
+ assert(fake.gsp_starts<=1 && fake.starts<=1 && fake.flrs<=1 && fake.pins<=1);
assert(fake.selector_writes<=1 && fake.ss1_writes<=1);
if(retained) {
assert(last_work->retained && fake.live>0 && dev.refs==2);
@@ -916,14 +690,14 @@ int main(int argc, char **argv) {
/* A card that kept power or runs GSP-RM is only read. */
if(!strcmp(cases[scenario],"resume-kept-power") || !strcmp(cases[scenario],"resume-gsp-running")) {
assert(ret==(cases[scenario][7]=='k'?-EALREADY:-EBUSY));
- assert(!fake.writes && !fake.allocs && fake.fw_requests==4 && fake.cmd==6 && !fake.now);
+ assert(!fake.writes && !fake.allocs && fake.fw_requests==1 && fake.cmd==6 && !fake.now);
}
if(!strcmp(cases[scenario],"resume-gfw-timeout"))
assert(ret==-ETIMEDOUT && !fake.writes && !fake.allocs && fake.cmd==6);
if(!strcmp(cases[scenario],"resume-oom"))
assert(ret==-ENOMEM && !fake.gsp_starts && !fake.flrs && fake.cmd==6);
if(scenario>=51 && scenario<=53)assert(!fake.writes && !fake.allocs && !fake.flrs);
- if(scenario==53)assert(fake.fw_requests==4 && ret==-EBUSY); /* only the shared load */
+ if(scenario==53)assert(fake.fw_requests==1 && ret==-EBUSY); /* only the shared load */
if(scenario==3 || scenario==51)assert(!last_work && fake.fw_requests==(scenario==3));
if(scenario==54)assert(fake.flrs==1 && ret==-EBUSY);
if(scenario==53 || scenario==54)assert(strstr(fake.log,"driver probe resources: phase=handoff empty=0"));
@@ -933,123 +707,12 @@ int main(int argc, char **argv) {
if(scenario==59)assert(ret==-EOPNOTSUPP && !fake.allocs && !fake.flrs);
if(scenario==60)assert(strstr(fake.log,"claim: cleared firmware BME"));
if(scenario==61)assert(ret==-EBUSY && !fake.allocs && !fake.flrs);
- if(strstr(cases[scenario],"chip-mismatch"))
- assert(ret==-ENODEV && !fake.writes && !fake.allocs && !fake.gsp_starts && !fake.flrs);
if(scenario==55) {
assert(ret==-EBUSY && !retained && !fake.gsp_starts && !fake.starts && !fake.flrs);
assert(strstr(fake.log,"card already initialised since power on"));
}
printf("full transaction %s: error=%d retained=%u PASS\n",cases[scenario],ret,retained);
}
- setup(&w);
- dev.device=0x20c2;regs[0]=0x170000a1;
- regs[0x110108/4]=256; /* generic loader's IMEM size in blocks */
- vbios_fixture_v2((u8 *)regs+0x300000);
- regs[0x1183a4/4]=8192;regs[0x625f04/4]=1;regs[0x1fa7c8/4]=1;
- regs[0x1fa82c/4]=0xfffffff0;regs[0x1fa830/4]=0;
- regs[GU_GFW_PLM_OFFSET/4]=1;regs[GU_GFW_BOOT_OFFSET/4]=0xff;
- regs[GU_GFW_CPUCTL_OFFSET/4]=16;
- regs[GU_FEAT_PLM_OFFSET/4]=0xffffff8f;regs[GU_FEAT_SS0_OFFSET/4]=0;
- fake.fwsec_wpr=1;fake.fwsec_wpr_lo=0x1ffe000;
- fake.fwsec_wpr_hi=0x1ffeff0;
- struct gpu_unlock_610_shared ga100_shared={0};
- struct gu_device ga100_device=resolve_device(&dev);
- assert(!gpu_unlock_610_load_shared(&ga100_device,&ga100_shared));
- full_active=false; /* GA100 has different image/layout; runner assertions still apply. */
- int ga100_ret=gpu_unlock_610_unlock(&ga100_device,&ga100_shared,false);
- if(ga100_ret)fprintf(stderr,"GA100 full: %s",fake.log);
- assert(!ga100_ret && fake.gsp_starts==1 && fake.starts==12 && fake.flrs==1);
- assert(!fake.live && fake.fw_live==4); /* shared firmware is still alive */
- gpu_unlock_610_release_shared(&ga100_shared);
- assert(!fake.fw_live && !fake.pins && !fake.regions && dev.refs==1);
- puts("GA100 v2 FWSEC + direct SEC2 full family transaction PASS");
-
- /* Common immutable firmware is requested once across families.
- * Each transaction still patches a separate device DMA allocation. */
- setup(&w);
- struct gpu_unlock_610_shared cached={0};
- struct gu_device device=resolve_device(&dev);
- assert(!gpu_unlock_610_load_shared(&device,&cached) && fake.fw_requests==4);
- const struct firmware *booter=cached.booter;
- assert(booter && cached.booter_hdr.patch_loc==35072);
- struct pci_dev other=dev;
- other.device=0x2484;
- struct gu_device other_device=resolve_device(&other);
- assert(!gpu_unlock_610_load_shared(&other_device,&cached) && fake.fw_requests==4);
- other.device=0x20c2;
- other_device=resolve_device(&other);
- assert(!gpu_unlock_610_load_shared(&other_device,&cached) && fake.fw_requests==8);
- const struct firmware *loader=cached.loader;
- assert(loader && cached.booter==booter);
- assert(!gpu_unlock_610_load_shared(&other_device,&cached) && fake.fw_requests==8);
- struct unlock_transaction_work separate[2]={0};
- for(int i=0;i<2;i++) {
- separate[i].pdev=i?&other:&dev;
- separate[i].bar=regs;separate[i].bar_len=sizeof(regs);separate[i].mapped=true;
- separate[i].booter_fw=booter;separate[i].loader_fw=loader;
- separate[i].booter_hdr=cached.booter_hdr;
- separate[i].booter_image_size=GU_BOOTER_IMAGE_SIZE;
- separate[i].booter_cpu=dma_alloc_coherent(&separate[i].pdev->dev,GU_BOOTER_IMAGE_SIZE,&separate[i].booter_dma,0);
- memcpy(separate[i].booter_cpu,booter->data+72,GU_BOOTER_IMAGE_SIZE);
- assert(!patch_booter(&separate[i]));
- }
- assert(separate[0].booter_dma!=separate[1].booter_dma);
- ((u8 *)separate[0].booter_cpu)[0]^=1;
- assert(((u8 *)separate[1].booter_cpu)[0]==booter->data[72]);
- assert(!unlock_transaction_unwind(&separate[0]) && fake.fw_live==6);
- assert(!unlock_transaction_unwind(&separate[1]) && fake.fw_live==6 && !fake.live);
- gpu_unlock_610_release_shared(&cached);
- assert(!fake.fw_live && !cached.booter && !cached.loader && !cached.ga10x.gsp && !cached.ga100.gsp);
- puts("shared Booter/loader ownership and private DMA buffers PASS");
-
- /* Failure at every family request leaves the other completed set usable.
- * Repeat both selection orders, and reject malformed formats before dispatch. */
- for(int healthy_direct=0;healthy_direct<2;healthy_direct++) {
- static const char *bad_ga100[]={"gsp_tu10x.bin","gsp_boot_ga100.bin",
- "gsp_boot_desc_ga100.bin","falcon_loader.bin"};
- static const char *bad_ga10x[]={"gsp_ga10x.bin","gsp_boot_ga102.bin",
- "gsp_boot_desc_ga102.bin"};
- int requests=healthy_direct?3:4;
- for(int malformed=0;malformed<2;malformed++)
- for(int fail=1;fail<=requests;fail++) {
- setup(&w);
- dev.device=healthy_direct?0x20c2:0x2204;
- struct gpu_unlock_610_shared mixed={0};
- struct gu_device healthy_device=resolve_device(&dev);
- assert(!gpu_unlock_610_load_shared(&healthy_device,&mixed));
- struct gpu_unlock_610_firmware *healthy=healthy_direct?&mixed.ga100:&mixed.ga10x;
- struct gpu_unlock_610_firmware *failed=healthy_direct?&mixed.ga10x:&mixed.ga100;
- struct gpu_unlock_610_firmware saved=*healthy;
- int live_before=fake.fw_live;
- other=dev;other.device=healthy_direct?0x2204:0x20c2;
- struct gu_device failed_device=resolve_device(&other);
- if(malformed)fake.corrupt_asset=healthy_direct?bad_ga10x[fail-1]:bad_ga100[fail-1];
- else fake.fail_fw=fake.fw_requests+fail;
- assert(gpu_unlock_610_load_shared(&failed_device,&mixed)<0);
- assert(failed->prepared && failed->error && !failed->gsp && !failed->boot);
- assert(!memcmp(healthy,&saved,sizeof(saved)) && fake.fw_live==live_before);
- int requested=fake.fw_requests;
- assert(gpu_unlock_610_load_shared(&failed_device,&mixed)==failed->error);
- assert(!gpu_unlock_610_load_shared(&healthy_device,&mixed) && fake.fw_requests==requested);
- gpu_unlock_610_release_shared(&mixed);
- assert(!fake.fw_live && !fake.live && !fake.group_live);
- assert(fake.cache_records==fake.cache_released);
- }
- }
- puts("family request/format failures preserve prepared sibling assets PASS");
-
- setup(&w);
- device=resolve_device(&dev);
- fake.corrupt_boot_bundle=1;
- assert(gpu_unlock_610_load_shared(&device,&cached)==-EINVAL);
- assert(!cached.booter && !cached.ga10x.gsp && fake.fw_live==0);
- fake.corrupt_boot_bundle=0;
- /* Failed preparation stays failed until the next coordination round. */
- assert(gpu_unlock_610_load_shared(&device,&cached)==-EINVAL && fake.fw_requests==4);
- gpu_unlock_610_release_shared(&cached);
- assert(!gpu_unlock_610_load_shared(&device,&cached) && fake.fw_requests==8);
- gpu_unlock_610_release_shared(&cached);
- assert(!fake.fw_live && fake.cache_records==fake.cache_released);
puts("pinned firmware full BF16 transactions passed under mocked hardware");
}
puts("actual unlock_transaction radix, early-PLM and reset/retention checks passed");
diff --git a/tests/test_vbios_610.c b/tests/test_vbios_610.c
index 14316ca..2276b7e 100644
--- a/tests/test_vbios_610.c
+++ b/tests/test_vbios_610.c
@@ -19,17 +19,6 @@ int main(int argc, char **argv)
u8 rom[4096], image[1280] = {0};
struct gpu_unlock_fwsec fw;
u32 offset;
- vbios_fixture_v2(rom);
- assert(!gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw));
- assert(fw.version == 2 && fw.image_off == 316 && fw.data_off == 256);
- assert(fw.imem_ns_size == 0 && fw.imem_sec_size == 256);
- assert(!gpu_unlock_fwsec_patch_frts(rom + fw.image_off, &fw, 0x100000));
- for (int field = 0; field < 6; field++) {
- const u32 offsets[] = { 264, 268, 276, 292, 296, 304 };
- vbios_fixture_v2(rom);
- w32(rom + offsets[field], 0xffffffff);
- assert(gpu_unlock_vbios_fwsec(rom, sizeof(rom), &fw));
- }
for (int stride = 6; stride <= 8; stride += 2) {
vbios_fixture(rom, stride);
diff --git a/tests/unlock_shim/compat.h b/tests/unlock_shim/compat.h
index f069665..2e575f4 100644
--- a/tests/unlock_shim/compat.h
+++ b/tests/unlock_shim/compat.h
@@ -28,7 +28,6 @@ typedef uint64_t ktime_t;
void test_device_log(const char *, ...) __attribute__((format(printf, 1, 2)));
#define dev_err(dev, ...) test_device_log(__VA_ARGS__)
#define dev_info(dev, ...) test_device_log(__VA_ARGS__)
-#define dev_dbg(dev, ...) test_device_log(__VA_ARGS__)
#define PCI_D0 0
#define PCI_UNKNOWN 5
#define PCI_DEV_FLAGS_NO_FLR_RESET (1u << 10)
diff --git a/tests/unlock_shim/crypto/sha2.h b/tests/unlock_shim/crypto/sha2.h
deleted file mode 100644
index 932e066..0000000
--- a/tests/unlock_shim/crypto/sha2.h
+++ /dev/null
@@ -1,4 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include <openssl/sha.h>
-#define SHA256_DIGEST_SIZE SHA256_DIGEST_LENGTH
-static inline void sha256(const u8 *data, size_t len, u8 *out) { SHA256(data, len, out); }
diff --git a/tests/unlock_shim/linux/device.h b/tests/unlock_shim/linux/device.h
deleted file mode 100644
index 4715430..0000000
--- a/tests/unlock_shim/linux/device.h
+++ /dev/null
@@ -1,2 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#include "../compat.h"
diff --git a/tests/unlock_shim/linux/printk.h b/tests/unlock_shim/linux/printk.h
index 692085b..4ac5987 100644
--- a/tests/unlock_shim/linux/printk.h
+++ b/tests/unlock_shim/linux/printk.h
@@ -1,5 +1,3 @@
#include <stdio.h>
#define pr_info(...) fprintf(stderr, __VA_ARGS__)
#define pr_err(...) fprintf(stderr, __VA_ARGS__)
-
-#define pr_debug(...) fprintf(stderr, __VA_ARGS__)
diff --git a/tests/vbios_fixture.h b/tests/vbios_fixture.h
index 4d29047..7951214 100644
--- a/tests/vbios_fixture.h
+++ b/tests/vbios_fixture.h
@@ -30,24 +30,4 @@ static void vbios_fixture(u8 *rom, int stride)
rom[292] = 1; rom[294] = 1; rom[295] = 1; rom[296] = 1;
}
-/* GA100 v2 descriptor; signed code/data contents are synthetic. */
-static void vbios_fixture_v2(u8 *rom)
-{
- vbios_fixture(rom, 6);
- memset(rom + 256, 0, 1536);
- vbios_w32(rom + 256, (60u << 16) | 0x201);
- vbios_w32(rom + 260, 1280);
- vbios_w32(rom + 264, 1280);
- vbios_w32(rom + 280, 256);
- vbios_w32(rom + 288, 0);
- vbios_w32(rom + 292, 256);
- vbios_w32(rom + 296, 256);
- vbios_w32(rom + 304, 1024);
- u8 *dmem = rom + 316 + 256;
- dmem[0] = 1; dmem[1] = 4; dmem[2] = 8; dmem[3] = 2;
- vbios_w32(dmem + 4, 4); vbios_w32(dmem + 8, 32);
- dmem[36] = 3; dmem[38] = 64;
- vbios_w32(dmem + 40, 128); vbios_w32(dmem + 44, 44);
-}
-
#endif