Skip to content

Your GeForce runs AI at half speed. This fixes it for 20/30/40 series cards. Up to 2× FP16 / BF16 / TF32 tensor throughput

C

Add all GA102 and GA104 device ids based on user reports

Zagittabb1088b6b5b8
diff --git a/README.md b/README.md
index 5854b4e..60c6eff 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,10 @@ It is neither a UEFI application nor a patched NVIDIA driver.
The observed result is GPU1, an RTX 3090 at `0000:21:00.0`, on the main
CachyOS `7.2.8-1-cachyos` kernel with NVIDIA `615.71.09`. GPU0 at
`0000:01:00.0` was an unmodified control. Other RTX 3090 boards and
-environments have not been verified. Other GA102 PCI device IDs are rejected.
+environments have not been verified. GeForce and CMP GA102/GA104 PCI device
+IDs are accepted; professional RTX A-series and data-centre parts are not.
+No GA104 card and no GA102 card other than the RTX 3090 has been tested; the
+firmware and V67 image are the GA102 ones. Other chips are rejected.
## Local build
@@ -58,14 +61,14 @@ There is no `softdep nvidia pre:` rule: kmod honours only the first
The module does nothing unless the kernel command line opts in:
```
-gpu_unlock.targets=all # every unbound 10de:2204 card
+gpu_unlock.targets=all # every unbound GA102/GA104 card
gpu_unlock.targets=0000:21:00.0 # only these cards
gpu_unlock.targets=0000:21:00.0,0000:22:00.0 # comma-separated
```
Addresses are full PCI names as `lspci -D` prints them (hex digits in either
case), independent of GPU numbering and display role. An entry that is not
-an RTX 3090, including `all` mixed with addresses, is logged and ignored.
+a GA102/GA104 card, including `all` mixed with addresses, is logged and ignored.
The cards are unlocked concurrently, about a second in all however many
there are, and each on its own: a card that is already bound, powered down
or bus mastering is skipped. Then `nvidia` loads and binds every card as
diff --git a/kernel/gpu_unlock/main.c b/kernel/gpu_unlock/main.c
index c84abbc..d6c2a23 100644
--- a/kernel/gpu_unlock/main.c
+++ b/kernel/gpu_unlock/main.c
@@ -9,13 +9,38 @@
static char *targets;
module_param(targets, charp, 0444);
-MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of RTX 3090 cards to unlock");
+MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of GA102/GA104 cards to unlock");
-#define RTX_3090 0x2204
+/* GA102 and GA104 GPU functions in pci.ids, excluding audio functions and the
+ * professional (GL) parts, which already run BF16 at full rate.
+ */
+static const unsigned short supported_ids[] = {
+ /* GA102 */
+ 0x2200, 0x2203, 0x2204, 0x2205, 0x2206, 0x2207, 0x2208, 0x220a,
+ 0x220d, 0x2216, 0x222b, 0x222f,
+ /* GA104 */
+ 0x2480, 0x2482, 0x2483, 0x2484, 0x2486, 0x2487, 0x2488, 0x2489,
+ 0x248a, 0x248c, 0x248d, 0x248e, 0x249c, 0x249d, 0x249f, 0x24a0,
+ 0x24a4, 0x24ac, 0x24ad, 0x24af, 0x24bf, 0x24c0, 0x24c7, 0x24c8,
+ 0x24c9, 0x24dc, 0x24dd, 0x24df, 0x24e0,
+};
static ASYNC_DOMAIN_EXCLUSIVE(unlock_domain);
static struct gpu_unlock_610_shared shared;
+/* The next supported NVIDIA card after from, like pci_get_device(). */
+static struct pci_dev *next_supported(struct pci_dev *from)
+{
+ struct pci_dev *pdev = from;
+ unsigned int i;
+
+ while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, pdev)))
+ for (i = 0; i < ARRAY_SIZE(supported_ids); i++)
+ if (pdev->device == supported_ids[i])
+ return pdev;
+ return NULL;
+}
+
/* Runs concurrently for each card; consumes the reference it is given. */
static void unlock_one(void *data, async_cookie_t cookie)
{
@@ -53,14 +78,14 @@ static void report_unmatched(void)
for (entry = targets; *entry; entry = *end ? end + 1 : end) {
end = strchrnul(entry, ',');
pdev = NULL;
- while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, RTX_3090, pdev)))
+ while ((pdev = next_supported(pdev)))
if (strlen(pci_name(pdev)) == end - entry &&
!strncasecmp(pci_name(pdev), entry, end - entry))
break;
if (pdev)
pci_dev_put(pdev);
else
- pr_err("gpu_unlock: no RTX 3090 at \"%.*s\"\n",
+ pr_err("gpu_unlock: no GA102/GA104 card at \"%.*s\"\n",
(int)(end - entry), entry);
}
}
@@ -77,7 +102,7 @@ static int __init gpu_unlock_init(void)
all = !strcmp(targets, "all");
if (!all)
report_unmatched();
- while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, RTX_3090, pdev))) {
+ while ((pdev = next_supported(pdev))) {
if (!all && !listed(pci_name(pdev)))
continue;
if (!shared.gsp) {
@@ -103,5 +128,5 @@ static void __exit gpu_unlock_exit(void)
module_init(gpu_unlock_init);
module_exit(gpu_unlock_exit);
MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("RTX 3090 pre-driver BF16 unlock");
+MODULE_DESCRIPTION("GA102/GA104 pre-driver BF16 unlock");
MODULE_VERSION("1.0.0");
diff --git a/kernel/gpu_unlock/unlock_610.c b/kernel/gpu_unlock/unlock_610.c
index 79c39cc..82a3396 100644
--- a/kernel/gpu_unlock/unlock_610.c
+++ b/kernel/gpu_unlock/unlock_610.c
@@ -139,6 +139,12 @@ static int diag_pci_dword(struct unlock_transaction_work *work, bool cap, int of
return ret;
}
+/* The config-space vendor/device dword this card answered at enumeration. */
+static u32 pci_identity(const struct pci_dev *pdev)
+{
+ return (u32)pdev->device << 16 | pdev->vendor;
+}
+
static void *diag_dma_alloc(struct unlock_transaction_work *work, const char *name,
size_t size, dma_addr_t *dma, unsigned int flags)
{
@@ -399,7 +405,7 @@ static int falcon_hs(struct unlock_transaction_work *work, const struct falcon_e
u32 plm, engine, identity;
if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
- identity != 0x220410de ||
+ identity != pci_identity(work->pdev) ||
read_bar(work, eng->base + 0x100, &engine) || engine == ~0u ||
(engine & 0xffff0000u) == 0xbadf0000u ||
read_bar(work, GU_FEAT_PLM_OFFSET, &plm))
@@ -1226,7 +1232,7 @@ static int wait_post_flr_gfw(struct unlock_transaction_work *work)
for (;;) {
sample = (struct gpu_unlock_gfw_sample){ };
ret = diag_pci_dword(work, false, PCI_VENDOR_ID, &identity);
- sample.config_ok = !ret && identity == 0x220410de;
+ sample.config_ok = !ret && identity == pci_identity(work->pdev);
if (!sample.config_ok) {
ret = -ENODEV;
break;
@@ -1301,7 +1307,7 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)
/* The saved image predates BME. No transient restore with BME set. */
pci_restore_state(work->pdev);
if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||
- identity != 0x220410de ||
+ identity != pci_identity(work->pdev) ||
diag_pci_word(work, false, PCI_COMMAND, &command) ||
command == 0xffff || (command & PCI_COMMAND_MASTER) ||
!(command & PCI_COMMAND_MEMORY))
diff --git a/tests/main_shim/compat.h b/tests/main_shim/compat.h
index 0e78a16..8ae0e13 100644
--- a/tests/main_shim/compat.h
+++ b/tests/main_shim/compat.h
@@ -21,6 +21,8 @@
#define module_exit(fn) void (*test_module_exit)(void) = fn;
#define PCI_VENDOR_ID_NVIDIA 0x10de
+#define PCI_ANY_ID (~0u)
+#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))
struct pci_dev {
const char *name;
unsigned short vendor, device;
diff --git a/tests/test_main.c b/tests/test_main.c
index 52f75e6..38e2f5b 100644
--- a/tests/test_main.c
+++ b/tests/test_main.c
@@ -12,6 +12,9 @@
#include "../kernel/gpu_unlock/main.c"
#define MAX_DEVICES 8
+#define RTX_3090 0x2204
+#define RTX_3070 0x2484
+#define RTX_3060 0x2503 /* GA106: not supported */
static struct pci_dev devices[MAX_DEVICES];
static unsigned int device_count;
@@ -62,7 +65,7 @@ struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, struct
pci_dev_put(from);
for (; i < device_count; i++)
- if (devices[i].vendor == vendor && devices[i].device == device)
+ if (devices[i].vendor == vendor && (device == PCI_ANY_ID || devices[i].device == device))
return pci_dev_get(&devices[i]);
return NULL;
}
@@ -218,13 +221,13 @@ static bool logged(const char *text)
int main(void)
{
- /* Every unbound RTX 3090 at once; other models and bound cards left alone. */
+ /* Every unbound GA102/GA104 card at once; other chips and bound cards left alone. */
reset("all");
add("0000:01:00.0", RTX_3090, false);
- add("0000:21:00.0", RTX_3090, false);
- add("0000:31:00.0", 0x2208, false);
+ add("0000:21:00.0", RTX_3070, false);
+ add("0000:31:00.0", RTX_3060, false);
add("0000:41:00.0", RTX_3090, true);
- add("0000:61:00.0", RTX_3090, false);
+ add("0000:61:00.0", 0x2208, false);
fake.expected_concurrent = 3;
run();
assert(fake.max_in_flight == 3 && fake.started == 3);
@@ -243,7 +246,7 @@ int main(void)
run();
assert(fake.max_in_flight == 2 && !fake.unlocks[0] && fake.unlocks[1] && fake.unlocks[2]);
assert(fake.loads == 1 && fake.load_dev == &devices[1] && fake.releases == 1);
- assert(!logged("no RTX 3090"));
+ assert(!logged("no GA102/GA104 card"));
puts("list: only listed cards, concurrently PASS");
/* One card failing leaves the others unlocked. */
@@ -268,12 +271,14 @@ int main(void)
assert(logged("0000:01:00.0: gpu_unlock: firmware load failed (-2)"));
puts("firmware load failure: nothing attempted PASS");
- /* Unmatched entries are reported; nothing to load for no cards. */
- reset("0000:99:00.0");
+ /* Unmatched entries, including unsupported chips, are reported; nothing to load for no cards. */
+ reset("0000:99:00.0,0000:31:00.0");
add("0000:01:00.0", RTX_3090, false);
+ add("0000:31:00.0", RTX_3060, false);
run();
assert(!fake.loads && !fake.started && fake.releases == 1);
- assert(logged("gpu_unlock: no RTX 3090 at \"0000:99:00.0\""));
+ assert(logged("gpu_unlock: no GA102/GA104 card at \"0000:99:00.0\""));
+ assert(logged("gpu_unlock: no GA102/GA104 card at \"0000:31:00.0\""));
puts("unmatched target: reported, nothing loaded PASS");
/* No opt-in on the command line: the module does nothing. */
diff --git a/tests/test_unlock_adapter.c b/tests/test_unlock_adapter.c
index 59ca342..bc9e55b 100644
--- a/tests/test_unlock_adapter.c
+++ b/tests/test_unlock_adapter.c
@@ -45,7 +45,7 @@ static u8 fw_image[4097];
static struct firmware fw={sizeof(fw_image),fw_image};
static void setup(struct unlock_transaction_work *w) {
memset(&fake,0,sizeof(fake)); memset(regs,0,sizeof(regs)); memset(w,0,sizeof(*w));
- mask=0xffffffffffffull; dev=(struct pci_dev){.dev={.dma_mask=&mask,.coherent_dma_mask=0xfffffffffull},.refs=1};
+ mask=0xffffffffffffull; dev=(struct pci_dev){.dev={.dma_mask=&mask,.coherent_dma_mask=0xfffffffffull},.vendor=0x10de,.device=0x2204,.refs=1};
w->pdev=&dev; w->bar=regs; w->bar_len=sizeof(regs); w->mapped=true;
fake.cmd=2;fake.saved=2;regs[(SEC2_BASE+0xf4)/4]=1u<<31;regs[(0x110000+0xf4)/4]=1u<<31;
regs[(SEC2_BASE+0x118)/4]=2;regs[(0x110000+0x118)/4]=2;
diff --git a/tests/unlock_shim/compat.h b/tests/unlock_shim/compat.h
index cb2f2a3..9d930de 100644
--- a/tests/unlock_shim/compat.h
+++ b/tests/unlock_shim/compat.h
@@ -49,7 +49,7 @@ static inline bool list_empty(const struct list_head *h) { return !h->count; }
struct device { u64 *dma_mask; u64 coherent_dma_mask; struct list_head devres_head; int devres_lock; };
void *devres_open_group(struct device *,void *,int);
int devres_release_group(struct device *,void *);
-struct pci_dev { struct device dev; int current_state, pm_cap, refs; unsigned int dev_flags; };
+struct pci_dev { struct device dev; unsigned short vendor, device; int current_state, pm_cap, refs; unsigned int dev_flags; };
struct firmware { size_t size; const u8 *data; };
void usleep_range(unsigned long, unsigned long);
ktime_t ktime_get(void);