Your GeForce runs AI at half speed. This fixes it for 20/30/40 series cards. Up to 2× FP16 / BF16 / TF32 tensor throughput
C
Add all GA102 and GA104 device ids based on user reports
Zagittabb1088b6b5b8
diff --git a/README.md b/README.mdindex 5854b4e..60c6eff 100644--- a/README.md+++ b/README.md@@ -10,7 +10,10 @@ It is neither a UEFI application nor a patched NVIDIA driver.The observed result is GPU1, an RTX 3090 at `0000:21:00.0`, on the mainCachyOS `7.2.8-1-cachyos` kernel with NVIDIA `615.71.09`. GPU0 at`0000:01:00.0` was an unmodified control. Other RTX 3090 boards and-environments have not been verified. Other GA102 PCI device IDs are rejected.+environments have not been verified. GeForce and CMP GA102/GA104 PCI device+IDs are accepted; professional RTX A-series and data-centre parts are not.+No GA104 card and no GA102 card other than the RTX 3090 has been tested; the+firmware and V67 image are the GA102 ones. Other chips are rejected.## Local build@@ -58,14 +61,14 @@ There is no `softdep nvidia pre:` rule: kmod honours only the firstThe module does nothing unless the kernel command line opts in:```-gpu_unlock.targets=all # every unbound 10de:2204 card+gpu_unlock.targets=all # every unbound GA102/GA104 cardgpu_unlock.targets=0000:21:00.0 # only these cardsgpu_unlock.targets=0000:21:00.0,0000:22:00.0 # comma-separated```Addresses are full PCI names as `lspci -D` prints them (hex digits in eithercase), independent of GPU numbering and display role. An entry that is not-an RTX 3090, including `all` mixed with addresses, is logged and ignored.+a GA102/GA104 card, including `all` mixed with addresses, is logged and ignored.The cards are unlocked concurrently, about a second in all however manythere are, and each on its own: a card that is already bound, powered downor bus mastering is skipped. Then `nvidia` loads and binds every card asdiff --git a/kernel/gpu_unlock/main.c b/kernel/gpu_unlock/main.cindex c84abbc..d6c2a23 100644--- a/kernel/gpu_unlock/main.c+++ b/kernel/gpu_unlock/main.c@@ -9,13 +9,38 @@static char *targets;module_param(targets, charp, 0444);-MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of RTX 3090 cards to unlock");+MODULE_PARM_DESC(targets, "\"all\" or comma-separated PCI addresses of GA102/GA104 cards to unlock");-#define RTX_3090 0x2204+/* GA102 and GA104 GPU functions in pci.ids, excluding audio functions and the+ * professional (GL) parts, which already run BF16 at full rate.+ */+static const unsigned short supported_ids[] = {+ /* GA102 */+ 0x2200, 0x2203, 0x2204, 0x2205, 0x2206, 0x2207, 0x2208, 0x220a,+ 0x220d, 0x2216, 0x222b, 0x222f,+ /* GA104 */+ 0x2480, 0x2482, 0x2483, 0x2484, 0x2486, 0x2487, 0x2488, 0x2489,+ 0x248a, 0x248c, 0x248d, 0x248e, 0x249c, 0x249d, 0x249f, 0x24a0,+ 0x24a4, 0x24ac, 0x24ad, 0x24af, 0x24bf, 0x24c0, 0x24c7, 0x24c8,+ 0x24c9, 0x24dc, 0x24dd, 0x24df, 0x24e0,+};static ASYNC_DOMAIN_EXCLUSIVE(unlock_domain);static struct gpu_unlock_610_shared shared;+/* The next supported NVIDIA card after from, like pci_get_device(). */+static struct pci_dev *next_supported(struct pci_dev *from)+{+ struct pci_dev *pdev = from;+ unsigned int i;++ while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID, pdev)))+ for (i = 0; i < ARRAY_SIZE(supported_ids); i++)+ if (pdev->device == supported_ids[i])+ return pdev;+ return NULL;+}+/* Runs concurrently for each card; consumes the reference it is given. */static void unlock_one(void *data, async_cookie_t cookie){@@ -53,14 +78,14 @@ static void report_unmatched(void)for (entry = targets; *entry; entry = *end ? end + 1 : end) {end = strchrnul(entry, ',');pdev = NULL;- while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, RTX_3090, pdev)))+ while ((pdev = next_supported(pdev)))if (strlen(pci_name(pdev)) == end - entry &&!strncasecmp(pci_name(pdev), entry, end - entry))break;if (pdev)pci_dev_put(pdev);else- pr_err("gpu_unlock: no RTX 3090 at \"%.*s\"\n",+ pr_err("gpu_unlock: no GA102/GA104 card at \"%.*s\"\n",(int)(end - entry), entry);}}@@ -77,7 +102,7 @@ static int __init gpu_unlock_init(void)all = !strcmp(targets, "all");if (!all)report_unmatched();- while ((pdev = pci_get_device(PCI_VENDOR_ID_NVIDIA, RTX_3090, pdev))) {+ while ((pdev = next_supported(pdev))) {if (!all && !listed(pci_name(pdev)))continue;if (!shared.gsp) {@@ -103,5 +128,5 @@ static void __exit gpu_unlock_exit(void)module_init(gpu_unlock_init);module_exit(gpu_unlock_exit);MODULE_LICENSE("GPL");-MODULE_DESCRIPTION("RTX 3090 pre-driver BF16 unlock");+MODULE_DESCRIPTION("GA102/GA104 pre-driver BF16 unlock");MODULE_VERSION("1.0.0");diff --git a/kernel/gpu_unlock/unlock_610.c b/kernel/gpu_unlock/unlock_610.cindex 79c39cc..82a3396 100644--- a/kernel/gpu_unlock/unlock_610.c+++ b/kernel/gpu_unlock/unlock_610.c@@ -139,6 +139,12 @@ static int diag_pci_dword(struct unlock_transaction_work *work, bool cap, int ofreturn ret;}+/* The config-space vendor/device dword this card answered at enumeration. */+static u32 pci_identity(const struct pci_dev *pdev)+{+ return (u32)pdev->device << 16 | pdev->vendor;+}+static void *diag_dma_alloc(struct unlock_transaction_work *work, const char *name,size_t size, dma_addr_t *dma, unsigned int flags){@@ -399,7 +405,7 @@ static int falcon_hs(struct unlock_transaction_work *work, const struct falcon_eu32 plm, engine, identity;if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||- identity != 0x220410de ||+ identity != pci_identity(work->pdev) ||read_bar(work, eng->base + 0x100, &engine) || engine == ~0u ||(engine & 0xffff0000u) == 0xbadf0000u ||read_bar(work, GU_FEAT_PLM_OFFSET, &plm))@@ -1226,7 +1232,7 @@ static int wait_post_flr_gfw(struct unlock_transaction_work *work)for (;;) {sample = (struct gpu_unlock_gfw_sample){ };ret = diag_pci_dword(work, false, PCI_VENDOR_ID, &identity);- sample.config_ok = !ret && identity == 0x220410de;+ sample.config_ok = !ret && identity == pci_identity(work->pdev);if (!sample.config_ok) {ret = -ENODEV;break;@@ -1301,7 +1307,7 @@ static int unlock_transaction_reset(struct unlock_transaction_work *work)/* The saved image predates BME. No transient restore with BME set. */pci_restore_state(work->pdev);if (diag_pci_dword(work, false, PCI_VENDOR_ID, &identity) ||- identity != 0x220410de ||+ identity != pci_identity(work->pdev) ||diag_pci_word(work, false, PCI_COMMAND, &command) ||command == 0xffff || (command & PCI_COMMAND_MASTER) ||!(command & PCI_COMMAND_MEMORY))diff --git a/tests/main_shim/compat.h b/tests/main_shim/compat.hindex 0e78a16..8ae0e13 100644--- a/tests/main_shim/compat.h+++ b/tests/main_shim/compat.h@@ -21,6 +21,8 @@#define module_exit(fn) void (*test_module_exit)(void) = fn;#define PCI_VENDOR_ID_NVIDIA 0x10de+#define PCI_ANY_ID (~0u)+#define ARRAY_SIZE(a) (sizeof(a) / sizeof((a)[0]))struct pci_dev {const char *name;unsigned short vendor, device;diff --git a/tests/test_main.c b/tests/test_main.cindex 52f75e6..38e2f5b 100644--- a/tests/test_main.c+++ b/tests/test_main.c@@ -12,6 +12,9 @@#include "../kernel/gpu_unlock/main.c"#define MAX_DEVICES 8+#define RTX_3090 0x2204+#define RTX_3070 0x2484+#define RTX_3060 0x2503 /* GA106: not supported */static struct pci_dev devices[MAX_DEVICES];static unsigned int device_count;@@ -62,7 +65,7 @@ struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, structpci_dev_put(from);for (; i < device_count; i++)- if (devices[i].vendor == vendor && devices[i].device == device)+ if (devices[i].vendor == vendor && (device == PCI_ANY_ID || devices[i].device == device))return pci_dev_get(&devices[i]);return NULL;}@@ -218,13 +221,13 @@ static bool logged(const char *text)int main(void){- /* Every unbound RTX 3090 at once; other models and bound cards left alone. */+ /* Every unbound GA102/GA104 card at once; other chips and bound cards left alone. */reset("all");add("0000:01:00.0", RTX_3090, false);- add("0000:21:00.0", RTX_3090, false);- add("0000:31:00.0", 0x2208, false);+ add("0000:21:00.0", RTX_3070, false);+ add("0000:31:00.0", RTX_3060, false);add("0000:41:00.0", RTX_3090, true);- add("0000:61:00.0", RTX_3090, false);+ add("0000:61:00.0", 0x2208, false);fake.expected_concurrent = 3;run();assert(fake.max_in_flight == 3 && fake.started == 3);@@ -243,7 +246,7 @@ int main(void)run();assert(fake.max_in_flight == 2 && !fake.unlocks[0] && fake.unlocks[1] && fake.unlocks[2]);assert(fake.loads == 1 && fake.load_dev == &devices[1] && fake.releases == 1);- assert(!logged("no RTX 3090"));+ assert(!logged("no GA102/GA104 card"));puts("list: only listed cards, concurrently PASS");/* One card failing leaves the others unlocked. */@@ -268,12 +271,14 @@ int main(void)assert(logged("0000:01:00.0: gpu_unlock: firmware load failed (-2)"));puts("firmware load failure: nothing attempted PASS");- /* Unmatched entries are reported; nothing to load for no cards. */- reset("0000:99:00.0");+ /* Unmatched entries, including unsupported chips, are reported; nothing to load for no cards. */+ reset("0000:99:00.0,0000:31:00.0");add("0000:01:00.0", RTX_3090, false);+ add("0000:31:00.0", RTX_3060, false);run();assert(!fake.loads && !fake.started && fake.releases == 1);- assert(logged("gpu_unlock: no RTX 3090 at \"0000:99:00.0\""));+ assert(logged("gpu_unlock: no GA102/GA104 card at \"0000:99:00.0\""));+ assert(logged("gpu_unlock: no GA102/GA104 card at \"0000:31:00.0\""));puts("unmatched target: reported, nothing loaded PASS");/* No opt-in on the command line: the module does nothing. */diff --git a/tests/test_unlock_adapter.c b/tests/test_unlock_adapter.cindex 59ca342..bc9e55b 100644--- a/tests/test_unlock_adapter.c+++ b/tests/test_unlock_adapter.c@@ -45,7 +45,7 @@ static u8 fw_image[4097];static struct firmware fw={sizeof(fw_image),fw_image};static void setup(struct unlock_transaction_work *w) {memset(&fake,0,sizeof(fake)); memset(regs,0,sizeof(regs)); memset(w,0,sizeof(*w));- mask=0xffffffffffffull; dev=(struct pci_dev){.dev={.dma_mask=&mask,.coherent_dma_mask=0xfffffffffull},.refs=1};+ mask=0xffffffffffffull; dev=(struct pci_dev){.dev={.dma_mask=&mask,.coherent_dma_mask=0xfffffffffull},.vendor=0x10de,.device=0x2204,.refs=1};w->pdev=&dev; w->bar=regs; w->bar_len=sizeof(regs); w->mapped=true;fake.cmd=2;fake.saved=2;regs[(SEC2_BASE+0xf4)/4]=1u<<31;regs[(0x110000+0xf4)/4]=1u<<31;regs[(SEC2_BASE+0x118)/4]=2;regs[(0x110000+0x118)/4]=2;diff --git a/tests/unlock_shim/compat.h b/tests/unlock_shim/compat.hindex cb2f2a3..9d930de 100644--- a/tests/unlock_shim/compat.h+++ b/tests/unlock_shim/compat.h@@ -49,7 +49,7 @@ static inline bool list_empty(const struct list_head *h) { return !h->count; }struct device { u64 *dma_mask; u64 coherent_dma_mask; struct list_head devres_head; int devres_lock; };void *devres_open_group(struct device *,void *,int);int devres_release_group(struct device *,void *);-struct pci_dev { struct device dev; int current_state, pm_cap, refs; unsigned int dev_flags; };+struct pci_dev { struct device dev; unsigned short vendor, device; int current_state, pm_cap, refs; unsigned int dev_flags; };struct firmware { size_t size; const u8 *data; };void usleep_range(unsigned long, unsigned long);ktime_t ktime_get(void);