A high-performance inference engine for AI models
add qmm mxu
eugened32e99756973
diff --git a/.gitignore b/.gitignoreindex d3bf51e56..1cedb0c35 100644--- a/.gitignore+++ b/.gitignore@@ -34,3 +34,4 @@ tools/gpu_trace/*.json.build/.swiftpm/Package.resolved+.dinghy.tomldiff --git a/Cargo.lock b/Cargo.lockindex 624222622..e18d21558 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -364,9 +364,9 @@ dependencies = [[[package]]name = "autocfg"-version = "1.5.0"+version = "1.5.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"[[package]]name = "autocxx"@@ -395,7 +395,7 @@ dependencies = ["quote","regex","rustc-hash 2.1.2",- "shlex",+ "shlex 1.3.0","syn 2.0.117",]@@ -517,9 +517,9 @@ dependencies = [[[package]]name = "aws-lc-rs"-version = "1.16.3"+version = "1.17.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"+checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"dependencies = ["aws-lc-sys","zeroize",@@ -527,9 +527,9 @@ dependencies = [[[package]]name = "aws-lc-sys"-version = "0.40.0"+version = "0.41.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"+checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"dependencies = ["cc","cmake",@@ -544,7 +544,7 @@ dependencies = ["async-openai","futures","indexmap 2.14.0",- "reqwest 0.13.3",+ "reqwest 0.13.4","serde","serde_json","shoji",@@ -688,7 +688,7 @@ dependencies = ["quote","regex","rustc-hash 2.1.2",- "shlex",+ "shlex 1.3.0","syn 2.0.117",]@@ -838,15 +838,15 @@ dependencies = [[[package]]name = "built"-version = "0.8.0"+version = "0.8.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64"+checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9"[[package]]name = "bumpalo"-version = "3.20.2"+version = "3.20.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"[[package]]name = "bytecount"@@ -947,9 +947,9 @@ dependencies = [[[package]]name = "cbindgen"-version = "0.29.2"+version = "0.29.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799"+checksum = "c95537b45400390270fae69ac098d057c8f5399001cde9d04f700c105ddfff2d"dependencies = ["clap","heck",@@ -966,14 +966,14 @@ dependencies = [[[package]]name = "cc"-version = "1.2.62"+version = "1.2.63"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"+checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"dependencies = ["find-msvc-tools","jobserver","libc",- "shlex",+ "shlex 2.0.1",][[package]]@@ -1230,9 +1230,9 @@ dependencies = [[[package]]name = "compact_str"-version = "0.9.0"+version = "0.9.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"+checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab"dependencies = ["castaway","cfg-if",@@ -1582,9 +1582,9 @@ dependencies = [[[package]]name = "ctor"-version = "0.11.1"+version = "1.0.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "400a21f1014a968ec518c7ccdf9b4a4ed0cac8c56ccb6d604f8b91f00110501e"+checksum = "01334b89b69ff726750c5ce5073fc8bd860e99aa9a8fc5ca11b04730e3aee97a"[[package]]name = "cxx"@@ -1903,9 +1903,9 @@ dependencies = [[[package]]name = "displaydoc"-version = "0.2.5"+version = "0.2.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"+checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"dependencies = ["proc-macro2","quote",@@ -1942,7 +1942,7 @@ dependencies = ["objc-sys","objc2","objc2-foundation",- "reqwest 0.13.3",+ "reqwest 0.13.4","rstest","serde","serde_json",@@ -1969,9 +1969,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"[[package]]name = "either"-version = "1.15.0"+version = "1.16.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"[[package]]name = "email_address"@@ -2372,9 +2372,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"[[package]]name = "futures-timer"-version = "3.0.3"+version = "3.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"[[package]]name = "futures-util"@@ -2673,9 +2673,9 @@ checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"[[package]]name = "http"-version = "1.4.0"+version = "1.4.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"+checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"dependencies = ["bytes","itoa",@@ -2724,9 +2724,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"[[package]]name = "hyper"-version = "1.9.0"+version = "1.10.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"dependencies = ["atomic-waker","bytes",@@ -3092,9 +3092,9 @@ dependencies = [[[package]]name = "iocraft"-version = "0.8.2"+version = "0.8.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "58c8c25269f0eae8c9791eed1519d04ca1ebcd7e11c8a10fc6c15091a4ed0019"+checksum = "ec09d74eb53878be6a7bd49f0ecb0e68d7322963d0a4fbbade67465083f38dbd"dependencies = ["bitflags 2.11.1","crossterm",@@ -3287,9 +3287,9 @@ dependencies = [[[package]]name = "js-sys"-version = "0.3.98"+version = "0.3.99"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"+checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"dependencies = ["cfg-if","futures-util",@@ -3315,9 +3315,9 @@ dependencies = [[[package]]name = "jsonschema"-version = "0.46.4"+version = "0.46.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fc59d2432e047d6090ba1d83c782d0128bd6203857978218f5614dbd3287281f"+checksum = "6a5fe5206f06e589caf25e79fc05ccdf91fca745685fe9fe1a13bbdfb479a631"dependencies = ["ahash","bytecount",@@ -3334,7 +3334,7 @@ dependencies = ["referencing","regex","regex-syntax 0.8.10",- "reqwest 0.13.3",+ "reqwest 0.13.4","rustls","serde","serde_json",@@ -3436,9 +3436,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"[[package]]name = "libredox"-version = "0.1.16"+version = "0.1.17"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"+checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3"dependencies = ["libc",]@@ -3501,9 +3501,9 @@ dependencies = [[[package]]name = "log"-version = "0.4.29"+version = "0.4.30"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"+checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"[[package]]name = "logos"@@ -3635,9 +3635,9 @@ dependencies = [[[package]]name = "memchr"-version = "2.8.0"+version = "2.8.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"+checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"[[package]]name = "memmap2"@@ -3716,9 +3716,9 @@ dependencies = [[[package]]name = "minijinja"-version = "2.19.0"+version = "2.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "805bfd7352166bae857ee569628b52bcd85a1cecf7810861ebceb1686b72b75d"+checksum = "2929e494b2280e1e18959bb2e121da03347ae896896fdfaceaab43c88a02803f"dependencies = ["indexmap 2.14.0","memo-map",@@ -3728,9 +3728,9 @@ dependencies = [[[package]]name = "minijinja-contrib"-version = "2.19.0"+version = "2.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "45092d80391870622fcf3bd82f5d2af18f99533ea60debb4bc9db0c76f0e809a"+checksum = "99df5123c54391e2a228014c1dbbd85a3dab08a25e776c810526f2f47542b3de"dependencies = ["minijinja","serde",@@ -3754,9 +3754,9 @@ dependencies = [[[package]]name = "mio"-version = "1.2.0"+version = "1.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"+checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"dependencies = ["libc","log",@@ -3856,7 +3856,7 @@ dependencies = ["pyo3-async-runtimes","pyo3-build-config","pyo3-stub-gen",- "reqwest 0.13.3",+ "reqwest 0.13.4","serde","serde-wasm-bindgen","serde_json",@@ -3873,9 +3873,9 @@ dependencies = [[[package]]name = "napi"-version = "3.8.6"+version = "3.9.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8e55037284865448ecf329baa86a4d05401f647ebde99f5747b640d32c2c5226"+checksum = "f1d395473824516f38dd1071a1a37bc57daa7be65b293ebba4ead5f7abb017a2"dependencies = ["bitflags 2.11.1","ctor",@@ -3890,15 +3890,15 @@ dependencies = [[[package]]name = "napi-build"-version = "2.3.1"+version = "2.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d376940fd5b723c6893cd1ee3f33abbfd86acb1cd1ec079f3ab04a2a3bc4d3b1"+checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"[[package]]name = "napi-derive"-version = "3.5.5"+version = "3.5.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a4ba740fe4c9524d86fd90798fd8ccdb23402b3eef7e7c30897a8a369b529fcf"+checksum = "89b3f766e04667e6da0e181e2da4f85475d5a6513b7cf6a80bea184e224a5b42"dependencies = ["convert_case 0.11.0","ctor",@@ -4109,9 +4109,9 @@ dependencies = [[[package]]name = "num-conv"-version = "0.2.1"+version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"+checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"[[package]]name = "num-derive"@@ -4422,9 +4422,9 @@ dependencies = [[[package]]name = "openssl"-version = "0.10.79"+version = "0.10.80"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542"+checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"dependencies = ["bitflags 2.11.1","cfg-if",@@ -4453,9 +4453,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"[[package]]name = "openssl-sys"-version = "0.9.115"+version = "0.9.116"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781"+checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"dependencies = ["cc","libc",@@ -4932,9 +4932,9 @@ dependencies = [[[package]]name = "pyo3-stub-gen"-version = "0.22.2"+version = "0.22.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b695173a33bec37b6acd288efe74f902f123f5891e5ff5ad53ff429e18833c14"+checksum = "1242a6080f45edcfaf3be3b9968914dd47a408c5590aea22770b978f7c36ffa7"dependencies = ["anyhow","chrono",@@ -4958,9 +4958,9 @@ dependencies = [[[package]]name = "pyo3-stub-gen-derive"-version = "0.22.2"+version = "0.22.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c2f901834d55c74f36be3353994062e3c3b3d47fda25572f5535e99434612ac8"+checksum = "fd9a73abe6f57890e657c265b86805637138f83203e1a56bc156078abf215e36"dependencies = ["heck","indexmap 2.14.0",@@ -5383,9 +5383,9 @@ dependencies = [[[package]]name = "referencing"-version = "0.46.4"+version = "0.46.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "cb674900ca31acd75c4aaf63f48e43e719631c0539ea5a9e64163d1296bcb730"+checksum = "69e4e17ef386c5383591d07623d3de49cbc601156e7582973e6db98d66a57de2"dependencies = ["ahash","fluent-uri",@@ -5525,9 +5525,9 @@ dependencies = [[[package]]name = "reqwest"-version = "0.13.3"+version = "0.13.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"dependencies = ["base64 0.22.1","bytes",@@ -5592,9 +5592,9 @@ checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"[[package]]name = "rhai"-version = "1.24.0"+version = "1.25.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1f9ef5dabe4c0b43d8f1187dc6beb67b53fe607fff7e30c5eb7f71b814b8c2c1"+checksum = "dd4dd0f8c36625202a4ba553c416c19b719947cd2a31d1bda06126e4a5727daf"dependencies = ["ahash","bitflags 2.11.1",@@ -5609,9 +5609,9 @@ dependencies = [[[package]]name = "rhai_codegen"-version = "3.1.0"+version = "3.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d4322a2a4e8cf30771dd9f27f7f37ca9ac8fe812dddd811096a98483080dabe6"+checksum = "3cd3a7535e50bf36857e7be7bec276d334e8c2dfa469c2201226fd01638ea5ca"dependencies = ["proc-macro2","quote",@@ -6050,9 +6050,9 @@ dependencies = [[[package]]name = "serde_json"-version = "1.0.149"+version = "1.0.150"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"dependencies = ["indexmap 2.14.0","itoa",@@ -6158,6 +6158,12 @@ version = "1.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"+[[package]]+name = "shlex"+version = "2.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"+[[package]]name = "shoji"version = "0.5.3"@@ -6294,9 +6300,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"[[package]]name = "socket2"-version = "0.6.3"+version = "0.6.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"+checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"dependencies = ["libc","windows-sys 0.61.2",@@ -6947,7 +6953,7 @@ dependencies = ["toml_datetime 1.1.1+spec-1.1.0","toml_parser","toml_writer",- "winnow 1.0.2",+ "winnow 1.0.3",][[package]]@@ -6970,15 +6976,15 @@ dependencies = [[[package]]name = "toml_edit"-version = "0.25.11+spec-1.1.0"+version = "0.25.12+spec-1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b"+checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7"dependencies = ["indexmap 2.14.0","toml_datetime 1.1.1+spec-1.1.0","toml_parser","toml_writer",- "winnow 1.0.2",+ "winnow 1.0.3",][[package]]@@ -6987,7 +6993,7 @@ version = "1.1.2+spec-1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"dependencies = [- "winnow 1.0.2",+ "winnow 1.0.3",][[package]]@@ -7013,9 +7019,9 @@ dependencies = [[[package]]name = "tower-http"-version = "0.6.10"+version = "0.6.11"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"+checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"dependencies = ["bitflags 2.11.1","bytes",@@ -7161,9 +7167,9 @@ dependencies = [[[package]]name = "typenum"-version = "1.20.0"+version = "1.20.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"[[package]]name = "ucd-trie"@@ -7519,9 +7525,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"[[package]]name = "uuid"-version = "1.23.1"+version = "1.23.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"+checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"dependencies = ["atomic","getrandom 0.4.2",@@ -7578,7 +7584,7 @@ dependencies = ["pyo3-async-runtimes","pyo3-build-config","pyo3-stub-gen",- "reqwest 0.13.3",+ "reqwest 0.13.4","rodio","rstest","schemars 1.2.1",@@ -7701,9 +7707,9 @@ dependencies = [[[package]]name = "wasm-bindgen"-version = "0.2.121"+version = "0.2.122"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"+checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"dependencies = ["cfg-if","once_cell",@@ -7716,9 +7722,9 @@ dependencies = [[[package]]name = "wasm-bindgen-futures"-version = "0.4.71"+version = "0.4.72"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"+checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"dependencies = ["js-sys","wasm-bindgen",@@ -7726,9 +7732,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro"-version = "0.2.121"+version = "0.2.122"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"+checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"dependencies = ["quote","wasm-bindgen-macro-support",@@ -7736,9 +7742,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro-support"-version = "0.2.121"+version = "0.2.122"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"+checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"dependencies = ["bumpalo","proc-macro2",@@ -7749,9 +7755,9 @@ dependencies = [[[package]]name = "wasm-bindgen-shared"-version = "0.2.121"+version = "0.2.122"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"+checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"dependencies = ["unicode-ident",]@@ -7818,9 +7824,9 @@ dependencies = [[[package]]name = "web-sys"-version = "0.3.98"+version = "0.3.99"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"+checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"dependencies = ["js-sys","wasm-bindgen",@@ -8335,9 +8341,9 @@ dependencies = [[[package]]name = "winnow"-version = "1.0.2"+version = "1.0.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0"+checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"dependencies = ["memchr",]@@ -8518,18 +8524,18 @@ dependencies = [[[package]]name = "zerocopy"-version = "0.8.48"+version = "0.8.50"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"+checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"dependencies = ["zerocopy-derive",][[package]]name = "zerocopy-derive"-version = "0.8.48"+version = "0.8.50"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"+checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"dependencies = ["proc-macro2","quote",@@ -8538,9 +8544,9 @@ dependencies = [[[package]]name = "zerofrom"-version = "0.1.7"+version = "0.1.8"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"dependencies = ["zerofrom-derive",]diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rsindex f71a9c7e8..57d6b6007 100644--- a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs@@ -3,7 +3,7 @@ use derive_more::Display;#[repr(C)]#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]-pub enum GemmWeightPrologueKind {+pub enum GemmBPrologueKind {FullPrecision,ScaleBiasDequant,ScaleZeroPointDequant,@@ -23,80 +23,117 @@ bitflags! {#[repr(C)]#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]+#[allow(non_camel_case_types)]pub enum GemmTiling {- T8x32x32_1x1,- T64x32x32_2x2,- T64x64x16_2x2,- T64x64x32_2x2,- T32x32x32_2x2,- T32x64x32_2x2,- T64x32x32_4x1,- T128x128x32_4x4,+ Tile8x32x32_Simdgroups1x1,+ Tile64x32x32_Simdgroups2x2,+ Tile64x64x16_Simdgroups2x2,+ Tile64x64x32_Simdgroups2x2,+ Tile32x32x32_Simdgroups2x2,+ Tile32x64x256_Simdgroups2x2,+ Tile64x32x256_Simdgroups4x1,+ Tile64x64x256_Simdgroups2x2,+ Tile128x128x256_Simdgroups4x4,}+const MXU_SIMDGROUP_BLOCK_K: u32 = 32;+impl GemmTiling {pub const fn block_m(self) -> u32 {match self {- Self::T8x32x32_1x1 => 8,- Self::T64x32x32_2x2 => 64,- Self::T64x64x16_2x2 => 64,- Self::T64x64x32_2x2 => 64,- Self::T32x32x32_2x2 => 32,- Self::T32x64x32_2x2 => 32,- Self::T64x32x32_4x1 => 64,- Self::T128x128x32_4x4 => 128,+ Self::Tile8x32x32_Simdgroups1x1 => 8,+ Self::Tile64x32x32_Simdgroups2x2 => 64,+ Self::Tile64x64x16_Simdgroups2x2 => 64,+ Self::Tile64x64x32_Simdgroups2x2 => 64,+ Self::Tile32x32x32_Simdgroups2x2 => 32,+ Self::Tile32x64x256_Simdgroups2x2 => 32,+ Self::Tile64x32x256_Simdgroups4x1 => 64,+ Self::Tile64x64x256_Simdgroups2x2 => 64,+ Self::Tile128x128x256_Simdgroups4x4 => 128,}}pub const fn block_n(self) -> u32 {match self {- Self::T8x32x32_1x1 => 32,- Self::T64x32x32_2x2 => 32,- Self::T64x64x16_2x2 => 64,- Self::T64x64x32_2x2 => 64,- Self::T32x32x32_2x2 => 32,- Self::T32x64x32_2x2 => 64,- Self::T64x32x32_4x1 => 32,- Self::T128x128x32_4x4 => 128,+ Self::Tile8x32x32_Simdgroups1x1 => 32,+ Self::Tile64x32x32_Simdgroups2x2 => 32,+ Self::Tile64x64x16_Simdgroups2x2 => 64,+ Self::Tile64x64x32_Simdgroups2x2 => 64,+ Self::Tile32x32x32_Simdgroups2x2 => 32,+ Self::Tile32x64x256_Simdgroups2x2 => 64,+ Self::Tile64x32x256_Simdgroups4x1 => 32,+ Self::Tile64x64x256_Simdgroups2x2 => 64,+ Self::Tile128x128x256_Simdgroups4x4 => 128,}}pub const fn block_k(self) -> u32 {match self {- Self::T8x32x32_1x1 => 32,- Self::T64x32x32_2x2 => 32,- Self::T64x64x16_2x2 => 16,- Self::T64x64x32_2x2 => 32,- Self::T32x32x32_2x2 => 32,- Self::T32x64x32_2x2 => 32,- Self::T64x32x32_4x1 => 32,- Self::T128x128x32_4x4 => 32,+ Self::Tile8x32x32_Simdgroups1x1 => 32,+ Self::Tile64x32x32_Simdgroups2x2 => 32,+ Self::Tile64x64x16_Simdgroups2x2 => 16,+ Self::Tile64x64x32_Simdgroups2x2 => 32,+ Self::Tile32x32x32_Simdgroups2x2 => 32,+ Self::Tile32x64x256_Simdgroups2x2 => 256,+ Self::Tile64x32x256_Simdgroups4x1 => 256,+ Self::Tile64x64x256_Simdgroups2x2 => 256,+ Self::Tile128x128x256_Simdgroups4x4 => 256,}}pub const fn simdgroups_m(self) -> u32 {match self {- Self::T8x32x32_1x1 => 1,- Self::T64x32x32_2x2 => 2,- Self::T64x64x16_2x2 => 2,- Self::T64x64x32_2x2 => 2,- Self::T32x32x32_2x2 => 2,- Self::T32x64x32_2x2 => 2,- Self::T64x32x32_4x1 => 4,- Self::T128x128x32_4x4 => 4,+ Self::Tile8x32x32_Simdgroups1x1 => 1,+ Self::Tile64x32x32_Simdgroups2x2 => 2,+ Self::Tile64x64x16_Simdgroups2x2 => 2,+ Self::Tile64x64x32_Simdgroups2x2 => 2,+ Self::Tile32x32x32_Simdgroups2x2 => 2,+ Self::Tile32x64x256_Simdgroups2x2 => 2,+ Self::Tile64x32x256_Simdgroups4x1 => 4,+ Self::Tile64x64x256_Simdgroups2x2 => 2,+ Self::Tile128x128x256_Simdgroups4x4 => 4,}}pub const fn simdgroups_n(self) -> u32 {match self {- Self::T8x32x32_1x1 => 1,- Self::T64x32x32_2x2 => 2,- Self::T64x64x16_2x2 => 2,- Self::T64x64x32_2x2 => 2,- Self::T32x32x32_2x2 => 2,- Self::T32x64x32_2x2 => 2,- Self::T64x32x32_4x1 => 1,- Self::T128x128x32_4x4 => 4,+ Self::Tile8x32x32_Simdgroups1x1 => 1,+ Self::Tile64x32x32_Simdgroups2x2 => 2,+ Self::Tile64x64x16_Simdgroups2x2 => 2,+ Self::Tile64x64x32_Simdgroups2x2 => 2,+ Self::Tile32x32x32_Simdgroups2x2 => 2,+ Self::Tile32x64x256_Simdgroups2x2 => 2,+ Self::Tile64x32x256_Simdgroups4x1 => 1,+ Self::Tile64x64x256_Simdgroups2x2 => 2,+ Self::Tile128x128x256_Simdgroups4x4 => 4,+ }+ }++ pub const fn is_mxu_variant(self) -> bool {+ matches!(+ self,+ Self::Tile32x64x256_Simdgroups2x2+ | Self::Tile64x32x256_Simdgroups4x1+ | Self::Tile64x64x256_Simdgroups2x2+ | Self::Tile128x128x256_Simdgroups4x4+ )+ }++ pub const fn simdgroup_block_k(self) -> u32 {+ if self.is_mxu_variant() {+ MXU_SIMDGROUP_BLOCK_K+ } else {+ self.block_k()+ }+ }++ pub const fn fits_quant_group_size(+ self,+ group_size: u32,+ ) -> bool {+ match self {+ Self::Tile128x128x256_Simdgroups4x4 => group_size <= 64,+ _ => true,}}}diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rsindex fa61fb877..6e9173281 100644--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs@@ -1,7 +1,7 @@use crate::{backends::common::{Allocation, AsBufferRangeRef, Backend,- gpu_types::{QuantizationMode, gemm::GemmWeightPrologueKind},+ gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},},data_type::DataType,};@@ -33,24 +33,24 @@ pub enum MatmulB<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {}impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {- pub fn weight_prologue(&self) -> GemmWeightPrologueKind {+ pub fn b_prologue(&self) -> GemmBPrologueKind {match self {Self::FullPrecision {..- } => GemmWeightPrologueKind::FullPrecision,+ } => GemmBPrologueKind::FullPrecision,Self::ScaleBiasDequant {..- } => GemmWeightPrologueKind::ScaleBiasDequant,+ } => GemmBPrologueKind::ScaleBiasDequant,Self::ScaleZeroPointDequant {..- } => GemmWeightPrologueKind::ScaleZeroPointDequant,+ } => GemmBPrologueKind::ScaleZeroPointDequant,Self::ScaleSymmetricDequant {..- } => GemmWeightPrologueKind::ScaleSymmetricDequant,+ } => GemmBPrologueKind::ScaleSymmetricDequant,}}- pub fn bits_per_weight(&self) -> Option<u32> {+ pub fn bits_per_b(&self) -> Option<u32> {match self {Self::FullPrecision {..diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rsindex 0dd4b332b..87213c086 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs@@ -1,19 +1,66 @@-use half::{bf16, f16};-use num_traits::Float;+use half::bf16;+use num_traits::{Float, NumCast};use proc_macros::kernel;use crate::{array::ArrayElement, backends::common::gpu_types::HadamardTransformOrder};+const SIMD_SIZE: usize = 32;++fn hadamard_transform(values: &mut [f32; SIMD_SIZE]) {+ let mut stride = 1;+ while stride < SIMD_SIZE {+ for lane in 0..SIMD_SIZE {+ if lane & stride == 0 {+ let a = values[lane];+ let b = values[lane | stride];+ values[lane] = a + b;+ values[lane | stride] = a - b;+ }+ }+ stride <<= 1;+ }+ let scale = 1.0 / (SIMD_SIZE as f32).sqrt();+ for v in values.iter_mut() {+ *v *= scale;+ }+}+#[kernel(HadamardTransform)]-#[variants(T, f32, f16, bf16)]+#[variants(T, f32, bf16)]pub fn hadamard_transform_mul<T: ArrayElement + Float>(- #[allow(unused)] data: *mut T,- #[allow(unused)] factors: *const i32,- #[allow(unused)] hidden_dim: u32,- #[allow(unused)] batch_size: u32,- #[allow(unused)]- #[specialize]- transform_order: HadamardTransformOrder,+ data: *mut T,+ factors: *const i32,+ hidden_dim: u32,+ batch_size: u32,+ #[specialize] transform_order: HadamardTransformOrder,) {- todo!()+ let hidden_dim = hidden_dim as usize;+ let batch_size = batch_size as usize;+ assert_eq!(hidden_dim % SIMD_SIZE, 0, "hidden_dim must be a multiple of {SIMD_SIZE}");++ for batch in 0..batch_size {+ let row_offset = batch * hidden_dim;+ for stripe_start in (0..hidden_dim).step_by(SIMD_SIZE) {+ let mut stripe = [0.0f32; SIMD_SIZE];+ for lane in 0..SIMD_SIZE {+ let v: f32 = NumCast::from(unsafe { *data.add(row_offset + stripe_start + lane) }).unwrap();+ let f = unsafe { *factors.add(stripe_start + lane) } as f32;+ stripe[lane] = match transform_order {+ HadamardTransformOrder::Input => v * f,+ HadamardTransformOrder::Output => v,+ };+ }++ hadamard_transform(&mut stripe);++ for lane in 0..SIMD_SIZE {+ let f = unsafe { *factors.add(stripe_start + lane) } as f32;+ let result = match transform_order {+ HadamardTransformOrder::Input => stripe[lane],+ HadamardTransformOrder::Output => stripe[lane] * f,+ };+ unsafe { *data.add(row_offset + stripe_start + lane) = <T as NumCast>::from(result).unwrap() };+ }+ }+ }}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rsdeleted file mode 100644index a20050452..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs+++ /dev/null@@ -1,2 +0,0 @@-pub mod qmm_transposed;-pub mod shaders;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rsdeleted file mode 100644index 4741e74a1..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs+++ /dev/null@@ -1,82 +0,0 @@-use num_traits::Float;--use crate::{array::ArrayElement, backends::common::gpu_types::QuantizationMethod};--pub fn qmm_transposed<T: ArrayElement + Float>(- weights: *const u32,- scales: *const T,- zero_points: Option<*const u8>,- biases: Option<*const T>,- input: *const T,- output: *mut T,- in_vec_size: usize,- out_vec_size: usize,- batch_size: usize,- quant_method: QuantizationMethod,- group_size: usize,- bits: usize,-) {- let num_groups_k = in_vec_size.div_ceil(group_size);- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- unsafe {- for i in 0..batch_size {- for j in 0..out_vec_size {- let mut acc = 0.0f32;-- for l in 0..in_vec_size {- let group_idx = l / group_size;-- let weight_linear_idx = j * in_vec_size + l;-- let val_q = if bits == 4 {- let u32_idx = weight_linear_idx / 8;- let bit_offset = (weight_linear_idx % 8) * 4;- ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32- } else {- let u32_idx = weight_linear_idx / 4;- let byte_offset = (weight_linear_idx % 4) * 8;- ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32- };-- let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();- let scale_t = *scales.add(j * num_groups_k + group_idx);-- let w_dequant_f32 = match quant_method {- QuantizationMethod::ScaleBias => {- let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);- scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()- },- QuantizationMethod::ScaleZeroPoint => {- let zp = zero_points.unwrap();- let zp_val = if bits == 4 {- let byte_index = j * zp_stride + (group_idx >> 1);- let byte_val = *zp.add(byte_index);- if (group_idx & 1) == 0 {- (byte_val & 0x0F) as f32- } else {- ((byte_val >> 4) & 0x0F) as f32- }- } else {- *zp.add(j * zp_stride + group_idx) as f32- };- scale_t.to_f32().unwrap() * (val_q - zp_val)- },- QuantizationMethod::ScaleSymmetric => {- let midpoint = (1 << (bits - 1)) as f32;- scale_t.to_f32().unwrap() * (val_q - midpoint)- },- };-- acc += val_a * w_dequant_f32;- }-- *output.add(i * out_vec_size + j) = T::from(acc).unwrap();- }- }- }-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rsdeleted file mode 100644index c55461396..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs+++ /dev/null@@ -1 +0,0 @@-pub mod steel_gemm;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rsdeleted file mode 100644index 472476e88..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs+++ /dev/null@@ -1,100 +0,0 @@-use num_traits::Float;--use crate::{- array::ArrayElement,- utils::pointers::{SendPtr, SendPtrMut},-};--#[inline(always)]-unsafe fn matmul_gemm_compute_row<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(- a: SendPtr<TA>,- b: SendPtr<TB>,- ab_scale: f32,- d: SendPtrMut<TD>,- row: usize,- n: usize,- k: usize,- leading_dimension_a: usize,- leading_dimension_b: usize,- leading_dimension_d: usize,- b_transpose: bool,- is_accumulate: bool,-) {- unsafe {- for col in 0..n {- let mut acc: f32 = 0.0;- for i in 0..k {- let a_val = *a.0.add(row * leading_dimension_a + i);- let b_val = if b_transpose {- *b.0.add(col * leading_dimension_b + i)- } else {- *b.0.add(i * leading_dimension_b + col)- };- acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();- }- let d_element = d.0.add(row * leading_dimension_d + col);- acc *= ab_scale;- if is_accumulate {- acc += (*d_element).to_f32().unwrap();- }- *d_element = TD::from(acc).unwrap();- }- }-}--pub fn matmul_gemm_impl<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(- a: *const TA,- b: *const TB,- ab_scale: f32,- d: *mut TD,- m: usize,- n: usize,- k: usize,- leading_dimension_a: usize,- leading_dimension_b: usize,- leading_dimension_d: usize,- b_transpose: bool,- is_accumulate: bool,-) {- let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);- let a = SendPtr(a);- let b = SendPtr(b);- let d = SendPtrMut(d);-- std::thread::scope(|s| {- let rows_per_thread = m / num_threads;- let rows_per_thread_remainder = m % num_threads;- let mut start = 0;-- for t in 0..num_threads {- let chunk_add = if t < rows_per_thread_remainder {- 1- } else {- 0- };- let chunk = rows_per_thread + chunk_add;- let end = start + chunk;-- s.spawn(move || unsafe {- for row in start..end {- matmul_gemm_compute_row(- a,- b,- ab_scale,- d,- row,- n,- k,- leading_dimension_a,- leading_dimension_b,- leading_dimension_d,- b_transpose,- is_accumulate,- );- }- });-- start = end;- }- });-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rsdeleted file mode 100644index e4e00c1e2..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs+++ /dev/null@@ -1,2 +0,0 @@-pub mod qmv;-pub mod qmv_fast;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rsdeleted file mode 100644index a0fa1d2e8..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs+++ /dev/null@@ -1,141 +0,0 @@-use half::bf16;-use num_traits::Float;-use proc_macros::kernel;--use crate::{array::ArrayElement, backends::common::gpu_types::QuantizationMethod};--/// CPU reference implementation of quantized matrix-vector multiply.-///-/// Weight layout is transposed compared to qmm:-/// weights: [N, K], scales: [N, num_groups_k], zero_points: [N, zp_stride_k]-/// where groups are along the K dimension.-///-/// Computes y[i, j] = sum_l( x[i, l] * dequant(w[j, l]) ) for each batch i and output j.-pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT: ArrayElement + Float>(- weights: *const u32,- scales: *const WeightT,- zero_points: Option<*const u8>,- biases: Option<*const WeightT>,- input: *const InputT,- output: *mut OutputT,- in_vec_size: usize,- out_vec_size: usize,- batch_size: usize,- quant_method: QuantizationMethod,- group_size: usize,- bits: usize,-) {- let num_groups_k = in_vec_size.div_ceil(group_size);- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- // Number of weight elements packed per u32- let pack_factor = if bits == 4 {- 8- } else {- 4- };-- unsafe {- for i in 0..batch_size {- for j in 0..out_vec_size {- let mut acc = 0.0f32;-- for l in 0..in_vec_size {- // Weights are [N, K]: row j, column l- let weight_linear_idx = j * in_vec_size + l;-- let val_q = if bits == 4 {- let u32_idx = weight_linear_idx / pack_factor;- let bit_offset = (weight_linear_idx % pack_factor) * 4;- ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32- } else {- let u32_idx = weight_linear_idx / pack_factor;- let byte_offset = (weight_linear_idx % pack_factor) * 8;- ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32- };-- let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();-- // Scales are [N, num_groups_k]: groups along K- let group_idx = l / group_size;- let scale = (*scales.add(j * num_groups_k + group_idx)).to_f32().unwrap();-- let bias = match quant_method {- QuantizationMethod::ScaleBias => {- // Biases are [N, num_groups_k]- (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()- },- QuantizationMethod::ScaleZeroPoint => {- let zp = zero_points.unwrap();- // Zero points are [N, zp_stride_k]- let zp_val = if bits == 4 {- let byte_index = j * zp_stride + (group_idx >> 1);- let byte_val = *zp.add(byte_index);- if (group_idx & 1) == 0 {- (byte_val & 0x0F) as f32- } else {- ((byte_val >> 4) & 0x0F) as f32- }- } else {- *zp.add(j * zp_stride + group_idx) as f32- };- -scale * zp_val- },- QuantizationMethod::ScaleSymmetric => {- let midpoint = 1 << (bits - 1);- -scale * midpoint as f32- },- };-- acc += val_a * (scale * val_q + bias);- }-- *output.add(i * out_vec_size + j) = OutputT::from(acc).unwrap();- }- }- }-}--#[kernel(QuantizedMatmulQmv)]-#[variants(WeightT, f32, bf16)]-#[variants(InputT, f32, bf16)]-#[variants(OutputT, f32, bf16)]-#[variants(GROUP_SIZE, 16, 32, 64, 128)]-#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmv<- WeightT: ArrayElement + Float,- InputT: ArrayElement + Float,- OutputT: ArrayElement + Float,- const GROUP_SIZE: u32,- const BITS: u32,->(- weights: *const u32,- scales: *const WeightT,- #[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,- input: *const InputT,- output: *mut OutputT,- in_vec_size: u32,- out_vec_size: u32,- batch_size: u32,- #[specialize] quant_method: QuantizationMethod,-) {- qmv::<WeightT, InputT, OutputT>(- weights,- scales,- zero_points,- biases,- input,- output,- in_vec_size as usize,- out_vec_size as usize,- batch_size as usize,- quant_method,- GROUP_SIZE as usize,- BITS as usize,- );-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rsdeleted file mode 100644index 93f004b8b..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs+++ /dev/null@@ -1,55 +0,0 @@-use half::bf16;-use num_traits::Float;-use proc_macros::kernel;--use crate::{- array::ArrayElement,- backends::{common::gpu_types::QuantizationMethod, cpu::kernel::matmul::gemv::qmv::qmv},-};--#[kernel(QuantizedMatmulQmvFast)]-#[variants(WeightT, f32, bf16)]-#[variants(InputT, f32, bf16)]-#[variants(OutputT, f32, bf16)]-#[variants(GROUP_SIZE, 16, 32, 64, 128)]-#[variants(BITS, 4, 8)]-pub fn quantized_matmul_qmv_fast<- WeightT: ArrayElement + Float,- InputT: ArrayElement + Float,- OutputT: ArrayElement + Float,- const GROUP_SIZE: u32,- const BITS: u32,->(- weights: *const u32,- scales: *const WeightT,- #[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,- input: *const InputT,- output: *mut OutputT,- #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,- in_vec_size: u32,- out_vec_size: u32,- batch_size: u32,- #[specialize] quant_method: QuantizationMethod,- #[specialize] use_hadamard: bool,-) {- assert_eq!(hadamard_factors.is_some(), use_hadamard);-- if use_hadamard {- unimplemented!("not supported yet");- }- qmv::<WeightT, InputT, OutputT>(- weights,- scales,- zero_points,- biases,- input,- output,- in_vec_size as usize,- out_vec_size as usize,- batch_size as usize,- quant_method,- GROUP_SIZE as usize,- BITS as usize,- );-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rsnew file mode 100644index 000000000..23a351825--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs@@ -0,0 +1,212 @@+use super::reference::{WeightData, read_f32, write_f32};+use crate::{+ backends::{+ common::{+ AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,+ gpu_types::HadamardTransformOrder,+ kernel::{+ HadamardTransformKernel,+ matmul::{MatmulArguments, MatmulError, MatmulKernel},+ },+ },+ cpu::{Cpu, context::CpuContext, error::CpuError},+ },+ data_type::DataType,+ utils::pointers::{SendPtr, SendPtrMut},+};++pub struct MatmulCpuKernel {+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,+}++impl MatmulKernel for MatmulCpuKernel {+ type Backend = Cpu;++ fn new(+ context: &CpuContext,+ weights_data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,+ ) -> Result<Self, CpuError> {+ for data_type in [weights_data_type, input_data_type, output_data_type] {+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {+ return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());+ }+ }+ let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(+ context,+ output_data_type,+ HadamardTransformOrder::Output,+ )?;+ Ok(Self {+ weights_data_type,+ input_data_type,+ output_data_type,+ hadamard,+ })+ }++ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(+ &mut self,+ arguments: MatmulArguments<Cpu, TB>,+ encoder: &mut Encoder<Cpu>,+ ) -> Result<(), CpuError> {+ let output_scale = arguments.d_transform.ab_scale;+ let accumulate = arguments.d_transform.accumulate;+ let bias_alloc = arguments.d_transform.bias;+ let post_rht = arguments.d_transform.rht_factors;++ let MatmulArguments {+ a,+ a_offset,+ b,+ b_offset,+ b_leading_dimension,+ b_transpose,+ d,+ m,+ n,+ k,+ ..+ } = arguments;++ let m_u = m as usize;+ let n_u = n as usize;+ let k_u = k as usize;+ let weights_data_type = self.weights_data_type;+ let input_data_type = self.input_data_type;+ let output_data_type = self.output_data_type;++ let a_buffer_range = a.as_buffer_range_ref();+ let a_byte_off = a_buffer_range.range().start + a_offset * input_data_type.size_in_bytes();+ let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));+ let bias_ptr = bias_alloc.map(|bias| {+ let r = bias.as_buffer_range_ref();+ SendPtr(unsafe { &*r.buffer().get() }.as_ptr().wrapping_byte_add(r.range().start))+ });+ let d_buffer_range = d.as_buffer_range_mut();+ let d_ptr = SendPtrMut(unsafe {+ (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_buffer_range.range().start) as *mut u8+ });++ let weight_data =+ WeightData::from_b(b, b_offset, b_leading_dimension, b_transpose, weights_data_type, k_u, n_u);++ let command_buffer = encoder.as_command_buffer_mut();+ command_buffer.push_command(move || {+ let quant_layout = match &weight_data {+ WeightData::Quantized {+ bits,+ group_size,+ ..+ } => {+ let num_groups_k = k_u.div_ceil(*group_size);+ let zero_point_stride = if *bits == 4 {+ num_groups_k.div_ceil(2)+ } else {+ num_groups_k+ };+ let pack_factor = if *bits == 4 {+ 8+ } else {+ 4+ };+ Some((num_groups_k, zero_point_stride, pack_factor))+ },+ WeightData::FullPrecision {+ ..+ } => None,+ };++ unsafe {+ for row in 0..m_u {+ for col in 0..n_u {+ let mut accumulator = 0.0f32;+ for inner in 0..k_u {+ let a_value = read_f32(a_ptr.as_ptr(), input_data_type, row * k_u + inner);+ let b_value = match &weight_data {+ WeightData::FullPrecision {+ ptr,+ leading_dimension,+ transpose,+ } => {+ let index = if *transpose {+ col * leading_dimension + inner+ } else {+ inner * leading_dimension + col+ };+ read_f32(ptr.as_ptr(), weights_data_type, index)+ },+ WeightData::Quantized {+ weights,+ scales,+ zero_points,+ biases,+ bits,+ group_size,+ } => {+ let (num_groups_k, zero_point_stride, pack_factor) = quant_layout.unwrap();+ let weight_linear_index = col * k_u + inner;+ let quantized_value = if *bits == 4 {+ let word_index = weight_linear_index / pack_factor;+ let bit_offset = (weight_linear_index % pack_factor) * 4;+ let w = weights.as_ptr() as *const u32;+ ((w.add(word_index).read_unaligned() >> bit_offset) & 0xF) as f32+ } else {+ let word_index = weight_linear_index / pack_factor;+ let bit_offset = (weight_linear_index % pack_factor) * 8;+ let w = weights.as_ptr() as *const u32;+ ((w.add(word_index).read_unaligned() >> bit_offset) & 0xFF) as f32+ };+ let group_index = inner / group_size;+ let scale =+ read_f32(scales.as_ptr(), weights_data_type, col * num_groups_k + group_index);+ let bias_term = if let Some(zp) = zero_points {+ let zero_point = if *bits == 4 {+ let byte_index = col * zero_point_stride + (group_index >> 1);+ let byte_value = *zp.as_ptr().add(byte_index);+ if (group_index & 1) == 0 {+ (byte_value & 0x0F) as f32+ } else {+ ((byte_value >> 4) & 0x0F) as f32+ }+ } else {+ *zp.as_ptr().add(col * zero_point_stride + group_index) as f32+ };+ -scale * zero_point+ } else if let Some(b) = biases {+ read_f32(b.as_ptr(), weights_data_type, col * num_groups_k + group_index)+ } else {+ let midpoint = (1u32 << (bits - 1)) as f32;+ -scale * midpoint+ };+ scale * quantized_value + bias_term+ },+ };+ accumulator += a_value * b_value;+ }++ let output_index = row * n_u + col;+ let mut value = output_scale * accumulator;+ if accumulate {+ value += read_f32(d_ptr.as_ptr(), output_data_type, output_index);+ }+ if let Some(bias) = bias_ptr {+ value += read_f32(bias.as_ptr(), weights_data_type, col);+ }+ write_f32(d_ptr.as_ptr(), output_data_type, output_index, value);+ }+ }+ }+ });++ if let Some(factors) = post_rht {+ self.hadamard.encode(d, factors, n, m, encoder);+ }++ Ok(())+ }+}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rsdeleted file mode 100644index 733c30139..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs+++ /dev/null@@ -1,406 +0,0 @@-use crate::{- backends::{- common::{- Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,- gpu_types::{HadamardTransformOrder, QuantizationMethod, QuantizationMode, gemm::GemmDTransform},- kernel::{- HadamardTransformKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, TensorAddBiasKernel,- matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel},- },- },- cpu::{- Cpu, buffer::BufferDowncastExt, context::CpuContext, error::CpuError,- kernel::matmul::quant::encode_quantized_gemm,- },- },- data_type::DataType,- dispatch_dtype,- utils::pointers::{SendPtr, SendPtrMut},-};--pub struct MatmulCpuKernel {- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,- hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,- bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,-}--impl MatmulKernel for MatmulCpuKernel {- type Backend = Cpu;-- fn new(- context: &CpuContext,- weights_data_type: DataType,- input_data_type: DataType,- output_data_type: DataType,- ) -> Result<Self, CpuError> {- for data_type in [weights_data_type, input_data_type, output_data_type] {- if !matches!(data_type, DataType::BF16 | DataType::F32) {- return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());- }- }- let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(- context,- output_data_type,- HadamardTransformOrder::Output,- )?;- let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(- context,- output_data_type,- weights_data_type,- true,- )?;- Ok(Self {- weights_data_type,- input_data_type,- output_data_type,- hadamard,- bias_add,- })- }-- fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(- &mut self,- arguments: MatmulArguments<Cpu, TB>,- encoder: &mut Encoder<Cpu>,- ) -> Result<(), CpuError> {- match arguments.b {- MatmulB::FullPrecision {- ..- } => Ok(self.encode_fp(arguments, encoder)?),- MatmulB::ScaleBiasDequant {- ..- }- | MatmulB::ScaleZeroPointDequant {- ..- }- | MatmulB::ScaleSymmetricDequant {- ..- } => Ok(self.encode_quant(arguments, encoder)?),- }- }-}--impl MatmulCpuKernel {- fn encode_fp<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(- &mut self,- arguments: MatmulArguments<Cpu, TB>,- encoder: &mut Encoder<Cpu>,- ) -> Result<(), MatmulError<Cpu>> {- if arguments.d_transform.mask().contains(GemmDTransform::RHT) {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::RHT,- path: "MatmulCpuKernel",- });- }-- let ab_scale = arguments.d_transform.ab_scale;- let bias_alloc = arguments.d_transform.bias;- let is_accumulate = arguments.d_transform.accumulate;-- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d,- m,- n,- k,- ..- } = arguments;- let weights = match b {- MatmulB::FullPrecision {- b: w,- } => w,- _ => unreachable!(),- };-- let command_buffer = encoder.as_command_buffer_mut();- let m_u = m as usize;- let n_u = n as usize;- let k_u = k as usize;- let lda = k_u;- let ldb = b_leading_dimension.map(|n| n as usize).unwrap_or(if b_transpose {- k_u- } else {- n_u- });- let ldd = n_u;- let weights_data_type = self.weights_data_type;- let input_data_type = self.input_data_type;- let output_data_type = self.output_data_type;- let a_buffer_range = a.as_buffer_range_ref();- let b_buffer_range = weights.as_buffer_range_ref();- let d_buffer_range = d.as_buffer_range_mut();- let a_byte_off = a_buffer_range.range().start + a_offset;- let b_byte_off = b_buffer_range.range().start + b_offset;- let d_byte_off = d_buffer_range.range().start;-- let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));- let b_ptr =- SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_byte_off));- let d_ptr =- SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });-- let bias_ptr = bias_alloc.map(|bias| {- let bias_buffer_range = bias.as_buffer_range_ref();- let bias_range = bias_buffer_range.range();- SendPtr(unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start))- });-- command_buffer.push_command(move || {- dispatch_dtype!(|(TW: weights_data_type, TA: input_data_type, TD: output_data_type)| {- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<TA, TW, TD>(- a_ptr.as_ptr() as *const TA,- b_ptr.as_ptr() as *const TW,- ab_scale,- d_ptr.as_ptr() as *mut TD,- m_u,- n_u,- k_u,- lda,- ldb,- ldd,- b_transpose,- is_accumulate,- );- if let Some(bias) = bias_ptr {- assert_eq!(- weights_data_type, output_data_type,- "mixed precision matmul with bias is not supported until bias dtype ownership is settled",- );- apply_bias::<TD>(d_ptr.as_ptr() as *mut TD, bias.as_ptr() as *const TD, m_u, n_u);- }- })- });-- Ok(())- }-- fn encode_quant<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(- &mut self,- arguments: MatmulArguments<Cpu, TB>,- encoder: &mut Encoder<Cpu>,- ) -> Result<(), MatmulError<Cpu>> {- let d_mask = arguments.d_transform.mask();- if d_mask.contains(GemmDTransform::SCALE) {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::SCALE,- path: "MatmulCpuKernel/Quant",- });- }- if d_mask.contains(GemmDTransform::ACCUMULATE) {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::ACCUMULATE,- path: "MatmulCpuKernel/Quant",- });- }- if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {- return Err(MatmulError::UnsupportedLayout {- path: "MatmulCpuKernel/Quant",- });- }-- let group_size = match arguments.b {- MatmulB::ScaleBiasDequant {- group_size,- ..- }- | MatmulB::ScaleZeroPointDequant {- group_size,- ..- }- | MatmulB::ScaleSymmetricDequant {- group_size,- ..- } => group_size,- MatmulB::FullPrecision {- ..- } => unreachable!(),- };- if !matches!(group_size, 16 | 32 | 64 | 128) {- return Err(MatmulError::UnsupportedGroupSize(group_size as usize));- }-- let post_rht = arguments.d_transform.rht_factors;- let post_bias = arguments.d_transform.bias;-- if arguments.m >= 5 && arguments.n > 1 {- assert_eq!(- self.input_data_type, self.weights_data_type,- "mixed precision quantized GEMM input dtype is not supported yet",- );- assert_eq!(- self.output_data_type, self.weights_data_type,- "mixed precision quantized GEMM output dtype is not supported yet",- );- assert!(- !(post_bias.is_some() && post_rht.is_some()),- "MatmulCpuKernel/Quant GEMM with both output bias and output RHT is not supported: bias must be applied after RHT",- );- let MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d,- d_transform,- m,- n,- k,- } = arguments;- let d_transform = d_transform.without(GemmDTransform::RHT | GemmDTransform::BIAS);- encode_quantized_gemm(- encoder,- MatmulArguments {- a,- a_offset,- b,- b_offset,- b_leading_dimension,- b_transpose,- d: &mut *d,- d_transform,- m,- n,- k,- },- self.weights_data_type,- );- if let Some(bias) = post_bias {- self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, m * n, encoder);- }- if let Some(factors) = post_rht {- self.hadamard.encode(d, factors, n, m, encoder);- }- return Ok(());- }-- let hadamard_factors = post_rht;-- let MatmulArguments {- a,- a_offset,- b,- d,- m,- n,- k,- ..- } = arguments;- let (weights, scales, zp_or_bias, method, mode, group_size) = match b {- MatmulB::ScaleBiasDequant {- b: w,- scales,- biases,- mode,- group_size,- } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),- MatmulB::ScaleZeroPointDequant {- b: w,- scales,- zero_points,- mode,- group_size,- } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),- MatmulB::ScaleSymmetricDequant {- b: w,- scales,- mode,- group_size,- } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),- MatmulB::FullPrecision {- ..- } => unreachable!(),- };-- let bits = match mode {- QuantizationMode::U4 => 4u32,- QuantizationMode::I8 | QuantizationMode::U8 => 8u32,- };- let use_fast = n % 8 == 0 && k % 512 == 0;- let (zero_points, biases) = match method {- QuantizationMethod::ScaleBias => {- (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))- },- QuantizationMethod::ScaleZeroPoint => {- (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)- },- QuantizationMethod::ScaleSymmetric => (None, None),- };-- let context = encoder.context();- if use_fast {- let kernel =- <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(- context,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- group_size,- bits,- method,- hadamard_factors.is_some(),- )- .map_err(MatmulError::BackendError)?;- kernel.encode(- weights,- scales,- zero_points,- biases,- (a, a_offset),- &mut *d,- hadamard_factors,- k,- n,- m,- encoder,- );- } else {- if hadamard_factors.is_some() {- return Err(MatmulError::UnsupportedDOp {- bit: GemmDTransform::RHT,- path: "MatmulCpuKernel/Quant",- });- }- let kernel =- <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(- context,- self.weights_data_type,- self.input_data_type,- self.output_data_type,- group_size,- bits,- method,- )- .map_err(MatmulError::BackendError)?;- kernel.encode(weights, scales, zero_points, biases, (a, a_offset), &mut *d, k, n, m, encoder);- }- if let Some(bias) = post_bias {- self.bias_add.encode(None::<&Allocation<Cpu>>, bias, d, n, m * n, encoder);- }- Ok(())- }-}--fn apply_bias<T: num_traits::Float + Copy>(- d: *mut T,- bias: *const T,- m: usize,- n: usize,-) {- unsafe {- for row in 0..m {- for col in 0..n {- let idx = row * n + col;- *d.add(idx) = *d.add(idx) + *bias.add(col);- }- }- }-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rsindex 7c21054fd..aa648840e 100644--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs@@ -1,6 +1,4 @@-pub mod gemm;-pub mod gemv;-mod matmul;-pub(crate) mod quant;+mod kernel;+mod reference;-pub use matmul::MatmulCpuKernel;+pub use kernel::MatmulCpuKernel;diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rsdeleted file mode 100644index 76eb56b89..000000000--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs+++ /dev/null@@ -1,156 +0,0 @@-use half::bf16;-use num_traits::Float;--use crate::{- array::ArrayElement,- backends::{- common::{- AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,- gpu_types::{QuantizationMethod, QuantizationMode},- kernel::matmul::{MatmulArguments, MatmulB},- },- cpu::{Cpu, kernel::matmul::gemm::qmm_transposed::qmm_transposed},- },- data_type::DataType,- utils::pointers::{SendPtr, SendPtrMut},-};--pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(- encoder: &mut Encoder<Cpu>,- arguments: MatmulArguments<Cpu, TB>,- data_type: DataType,-) {- let MatmulArguments {- a,- a_offset,- b,- d,- m,- n,- k,- ..- } = arguments;- let (weights, scales, zp_or_bias, method, mode, group_size) = match b {- MatmulB::ScaleBiasDequant {- b: w,- scales,- biases,- mode,- group_size,- } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),- MatmulB::ScaleZeroPointDequant {- b: w,- scales,- zero_points,- mode,- group_size,- } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),- MatmulB::ScaleSymmetricDequant {- b: w,- scales,- mode,- group_size,- } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),- MatmulB::FullPrecision {- ..- } => unreachable!(),- };-- let bits: usize = match mode {- QuantizationMode::U4 => 4,- QuantizationMode::I8 | QuantizationMode::U8 => 8,- };- let in_vec_size = k as usize;- let out_vec_size = n as usize;- let batch_size = m as usize;- let group_size = group_size as usize;-- let a_buf = a.as_buffer_range_ref();- let b_buf = weights.as_buffer_range_ref();- let scales_buf = scales.as_buffer_range_ref();- let zp_buf = zp_or_bias.map(|zp_or_bias| zp_or_bias.as_buffer_range_ref());- let d_buf = d.as_buffer_range_mut();-- let a_byte_off = a_buf.range().start + a_offset * data_type.size_in_bytes();- let b_byte_off = b_buf.range().start;- let scales_byte_off = scales_buf.range().start;- let zp_byte_off = zp_buf.as_ref().map(|zp_buf| zp_buf.range().start);- let d_byte_off = d_buf.range().start;-- let a_ptr = SendPtr(unsafe { &*a_buf.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));- let b_ptr = SendPtr(unsafe { &*b_buf.buffer().get() }.as_ptr().wrapping_byte_add(b_byte_off));- let scales_ptr = SendPtr(unsafe { &*scales_buf.buffer().get() }.as_ptr().wrapping_byte_add(scales_byte_off));- let zp_ptr = zp_buf- .map(|zp_buf| SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off.unwrap())));- let d_ptr = SendPtrMut(unsafe { (&*d_buf.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });-- let command_buffer = encoder.as_command_buffer_mut();- command_buffer.push_command(move || match data_type {- DataType::F32 => run::<f32>(- a_ptr,- b_ptr,- scales_ptr,- zp_ptr,- d_ptr,- in_vec_size,- out_vec_size,- batch_size,- method,- group_size,- bits,- ),- DataType::BF16 => run::<bf16>(- a_ptr,- b_ptr,- scales_ptr,- zp_ptr,- d_ptr,- in_vec_size,- out_vec_size,- batch_size,- method,- group_size,- bits,- ),- _ => unreachable!(),- });-}--fn run<T: ArrayElement + Float>(- a_ptr: SendPtr<u8>,- b_ptr: SendPtr<u8>,- scales_ptr: SendPtr<u8>,- zp_or_bias_ptr: Option<SendPtr<u8>>,- d_ptr: SendPtrMut<u8>,- in_vec_size: usize,- out_vec_size: usize,- batch_size: usize,- quant_method: QuantizationMethod,- group_size: usize,- bits: usize,-) {- let (zp, bias) = match quant_method {- QuantizationMethod::ScaleBias => {- (None, Some(zp_or_bias_ptr.expect("ScaleBias quantized matmul requires biases").as_ptr() as *const T))- },- QuantizationMethod::ScaleZeroPoint => (- Some(zp_or_bias_ptr.expect("ScaleZeroPoint quantized matmul requires zero_points").as_ptr() as *const u8),- None,- ),- QuantizationMethod::ScaleSymmetric => (None, None),- };- qmm_transposed::<T>(- b_ptr.as_ptr() as *const u32,- scales_ptr.as_ptr() as *const T,- zp,- bias,- a_ptr.as_ptr() as *const T,- d_ptr.as_ptr() as *mut T,- in_vec_size,- out_vec_size,- batch_size,- quant_method,- group_size,- bits,- );-}diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rsnew file mode 100644index 000000000..5076857f0--- /dev/null+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs@@ -0,0 +1,139 @@+use half::{bf16, f16};++use crate::{+ backends::{+ common::{AsBufferRangeRef, Buffer, gpu_types::QuantizationMode, kernel::matmul::MatmulB},+ cpu::{Cpu, buffer::BufferDowncastExt},+ },+ data_type::DataType,+ utils::pointers::SendPtr,+};++pub(super) enum WeightData {+ FullPrecision {+ ptr: SendPtr<u8>,+ leading_dimension: usize,+ transpose: bool,+ },+ Quantized {+ weights: SendPtr<u8>,+ scales: SendPtr<u8>,+ zero_points: Option<SendPtr<u8>>,+ biases: Option<SendPtr<u8>>,+ bits: usize,+ group_size: usize,+ },+}++impl WeightData {+ pub(super) fn from_b<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(+ b: MatmulB<'_, Cpu, TB>,+ b_offset: usize,+ b_leading_dimension: Option<u32>,+ b_transpose: bool,+ weights_data_type: DataType,+ k: usize,+ n: usize,+ ) -> Self {+ let alloc_ptr = |a: &crate::backends::common::Allocation<Cpu>| {+ let r = a.as_buffer_range_ref();+ SendPtr(unsafe { &*r.buffer().get() }.as_ptr().wrapping_byte_add(r.range().start))+ };+ let bits_of = |mode| match mode {+ QuantizationMode::U4 => 4usize,+ _ => 8usize,+ };+ match b {+ MatmulB::FullPrecision {+ b: weights,+ } => {+ let leading_dimension = b_leading_dimension.map(|ld| ld as usize).unwrap_or(if b_transpose {+ k+ } else {+ n+ });+ let r = weights.as_buffer_range_ref();+ let byte_off = r.range().start + b_offset * weights_data_type.size_in_bytes();+ WeightData::FullPrecision {+ ptr: SendPtr(unsafe { &*r.buffer().downcast().get() }.as_ptr().wrapping_byte_add(byte_off)),+ leading_dimension,+ transpose: b_transpose,+ }+ },+ MatmulB::ScaleBiasDequant {+ b: weights,+ scales,+ biases,+ mode,+ group_size,+ } => WeightData::Quantized {+ weights: alloc_ptr(weights),+ scales: alloc_ptr(scales),+ zero_points: None,+ biases: Some(alloc_ptr(biases)),+ bits: bits_of(mode),+ group_size: group_size as usize,+ },+ MatmulB::ScaleZeroPointDequant {+ b: weights,+ scales,+ zero_points,+ mode,+ group_size,+ } => WeightData::Quantized {+ weights: alloc_ptr(weights),+ scales: alloc_ptr(scales),+ zero_points: Some(alloc_ptr(zero_points)),+ biases: None,+ bits: bits_of(mode),+ group_size: group_size as usize,+ },+ MatmulB::ScaleSymmetricDequant {+ b: weights,+ scales,+ mode,+ group_size,+ } => WeightData::Quantized {+ weights: alloc_ptr(weights),+ scales: alloc_ptr(scales),+ zero_points: None,+ biases: None,+ bits: bits_of(mode),+ group_size: group_size as usize,+ },+ }+ }+}++#[inline]+pub(super) unsafe fn read_f32(+ base: *const u8,+ data_type: DataType,+ index: usize,+) -> f32 {+ unsafe {+ match data_type {+ DataType::F32 => *(base as *const f32).add(index),+ DataType::F16 => (*(base as *const f16).add(index)).to_f32(),+ DataType::BF16 => (*(base as *const bf16).add(index)).to_f32(),+ _ => unreachable!(),+ }+ }+}++#[inline]+pub(super) unsafe fn write_f32(+ base: *mut u8,+ data_type: DataType,+ index: usize,+ value: f32,+) {+ unsafe {+ match data_type {+ DataType::F32 => *(base as *mut f32).add(index) = value,+ DataType::F16 => *(base as *mut f16).add(index) = f16::from_f32(value),+ DataType::BF16 => *(base as *mut bf16).add(index) = bf16::from_f32(value),+ _ => unreachable!(),+ }+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.hindex f165a6585..a2e90ab48 100644--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h@@ -5,7 +5,7 @@using namespace metal;namespace uzu::gemm {-enum class GemmWeightPrologueKind : uint32_t {+enum class GemmBPrologueKind : uint32_t {FullPrecision = 0,ScaleBiasDequant = 1,ScaleZeroPointDequant = 2,@@ -31,14 +31,15 @@ struct GemmDTransform {};enum class GemmTiling : uint32_t {- T8x32x32_1x1 = 0,- T64x32x32_2x2 = 1,- T64x64x16_2x2 = 2,- T64x64x32_2x2 = 3,- T32x32x32_2x2 = 4,- T32x64x32_2x2 = 5,- T64x32x32_4x1 = 6,- T128x128x32_4x4 = 7,+ Tile8x32x32_Simdgroups1x1 = 0,+ Tile64x32x32_Simdgroups2x2 = 1,+ Tile64x64x16_Simdgroups2x2 = 2,+ Tile64x64x32_Simdgroups2x2 = 3,+ Tile32x32x32_Simdgroups2x2 = 4,+ Tile32x64x256_Simdgroups2x2 = 5,+ Tile64x32x256_Simdgroups4x1 = 6,+ Tile64x64x256_Simdgroups2x2 = 7,+ Tile128x128x256_Simdgroups4x4 = 8,};struct GemmAlignment {diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.hindex 314628029..10bc5ce2f 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h@@ -1,3 +1,5 @@+#pragma once+#include <metal_stdlib>#include "../common/defines.h"#include "../common/thread_context.h"diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metalindex 78b038b02..6678f8060 100644--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal@@ -8,7 +8,7 @@ using namespace metal;using namespace uzu::hadamard_order;template <typename T>-VARIANTS(T, half, bfloat, float)+VARIANTS(T, bfloat, float)PUBLIC KERNEL(HadamardTransform)(device T* data,const device int32_t* factors,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex 9b64d6989..2552c3a0a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -117,8 +117,6 @@ struct ThreadgroupTile {METAL_PRAGMA_UNROLLfor (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH;k_block_index += SIMDGROUP_BLOCK_SIZE) {- simdgroup_barrier(mem_flags::mem_none);-a_fragment.template load<AT,SIMDGROUPS_PER_ROW,@@ -126,8 +124,6 @@ struct ThreadgroupTile {A_STRIDE_ROW,A_STRIDE_INNER>(a_shared);- simdgroup_barrier(mem_flags::mem_none);-b_fragment.template load<BT,1,@@ -135,8 +131,6 @@ struct ThreadgroupTile {B_STRIDE_INNER,B_STRIDE_COL>(b_shared);- simdgroup_barrier(mem_flags::mem_none);-tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);a_shared += TILE_STRIDE_A;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.hnew file mode 100644index 000000000..864a0af1c--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h@@ -0,0 +1,34 @@+#pragma once++#include "../../../common/thread_context.h"+#include "../../../hadamard_transform/hadamard_transform.h"++using namespace metal;++template <typename T>+static METAL_FUNC void apply_output_random_hadamard_transform(+ device T* output_block,+ const device int32_t* rht_factors_block,+ ushort tile_block_rows,+ ushort tile_block_cols,+ uint leading_dimension_d,+ ushort simdgroup_count,+ const thread ThreadContext& thread_context+) {+ const ushort stripes_per_row = tile_block_cols / METAL_SIMD_SIZE;+ const ushort simd_lane = thread_context.simd_lane_id;+ const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);+ for (uint cell = thread_context.simdgroup_index; cell < total_work;+ cell += simdgroup_count) {+ const ushort row_local = ushort(cell / stripes_per_row);+ const ushort stripe = ushort(cell % stripes_per_row);+ const ushort col_local = stripe * ushort(METAL_SIMD_SIZE) + simd_lane;+ const size_t output_index =+ size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);+ output_block[output_index] = simdgroup_output_random_hadamard_transform(+ simd_lane,+ output_block[output_index],+ rht_factors_block[col_local]+ );+ }+}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.hindex 0ee3354d6..7556db8c7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h@@ -6,63 +6,68 @@ namespace uzu {namespace gemm {constexpr uint gemm_tiling_block_m(GemmTiling t) {- return t == GemmTiling::T8x32x32_1x1 ? 8- : t == GemmTiling::T64x32x32_2x2 ? 64- : t == GemmTiling::T64x64x16_2x2 ? 64- : t == GemmTiling::T64x64x32_2x2 ? 64- : t == GemmTiling::T32x32x32_2x2 ? 32- : t == GemmTiling::T32x64x32_2x2 ? 32- : t == GemmTiling::T64x32x32_4x1 ? 64- : t == GemmTiling::T128x128x32_4x4 ? 128- : 0;+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 8+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 32+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 64+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128+ : 0;}constexpr uint gemm_tiling_block_n(GemmTiling t) {- return t == GemmTiling::T8x32x32_1x1 ? 32- : t == GemmTiling::T64x32x32_2x2 ? 32- : t == GemmTiling::T64x64x16_2x2 ? 64- : t == GemmTiling::T64x64x32_2x2 ? 64- : t == GemmTiling::T32x32x32_2x2 ? 32- : t == GemmTiling::T32x64x32_2x2 ? 64- : t == GemmTiling::T64x32x32_4x1 ? 32- : t == GemmTiling::T128x128x32_4x4 ? 128- : 0;+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 32+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128+ : 0;}constexpr uint gemm_tiling_block_k(GemmTiling t) {- return t == GemmTiling::T8x32x32_1x1 ? 32- : t == GemmTiling::T64x32x32_2x2 ? 32- : t == GemmTiling::T64x64x16_2x2 ? 16- : t == GemmTiling::T64x64x32_2x2 ? 32- : t == GemmTiling::T32x32x32_2x2 ? 32- : t == GemmTiling::T32x64x32_2x2 ? 32- : t == GemmTiling::T64x32x32_4x1 ? 32- : t == GemmTiling::T128x128x32_4x4 ? 32- : 0;+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 16+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 32+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 256+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 256+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 256+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 256+ : 0;}constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {- return t == GemmTiling::T8x32x32_1x1 ? 1- : t == GemmTiling::T64x32x32_2x2 ? 2- : t == GemmTiling::T64x64x16_2x2 ? 2- : t == GemmTiling::T64x64x32_2x2 ? 2- : t == GemmTiling::T32x32x32_2x2 ? 2- : t == GemmTiling::T32x64x32_2x2 ? 2- : t == GemmTiling::T64x32x32_4x1 ? 4- : t == GemmTiling::T128x128x32_4x4 ? 4- : 0;+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 4+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4+ : 0;}constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {- return t == GemmTiling::T8x32x32_1x1 ? 1- : t == GemmTiling::T64x32x32_2x2 ? 2- : t == GemmTiling::T64x64x16_2x2 ? 2- : t == GemmTiling::T64x64x32_2x2 ? 2- : t == GemmTiling::T32x32x32_2x2 ? 2- : t == GemmTiling::T32x64x32_2x2 ? 2- : t == GemmTiling::T64x32x32_4x1 ? 1- : t == GemmTiling::T128x128x32_4x4 ? 4- : 0;+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 1+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4+ : 0;}} // namespace gemmdiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.hindex cca0830a2..55b8ad1cb 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h@@ -2,13 +2,18 @@#include "../../../common/integral_constant.h"#include "../../../common/thread_context.h"+#include "../../../hadamard_transform/hadamard_transform.h"#include "../../common/fragment.h"+#include "gemm_rht.h"#include "../../common/mxu_fragment_ops.h"#include "../../common/mxu_gemm_loop.h"#include "../../../generated/matmul.h"#include "../generated/gemm.h"#include "block_geometry.h"#include "gemm_tiling.h"+#include "quant_pack.h"+#include "quant_scale_bias.h"+#include "quant_scale_zero_point.h"using namespace metal;@@ -20,7 +25,10 @@ template <typename BT,typename DT,GemmTiling GEMM_TILING,- bool TRANSPOSE_B>+ bool TRANSPOSE_B,+ GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,+ int BITS = 0,+ int GROUP_SIZE = 0>struct MxuMmaCore {METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);@@ -33,14 +41,134 @@ struct MxuMmaCore {METAL_CONST ushort SIMDGROUP_BLOCK_N =THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;- METAL_CONST ushort THREADGROUP_BLOCK_K = 256;+ METAL_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);+ static_assert(+ THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,+ "FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K"+ );METAL_CONST ushort TILES_M =SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;METAL_CONST ushort TILES_N =SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;+ METAL_CONST ushort TILES_K =+ SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;++ METAL_CONST ushort QUANT_BK =+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;+ METAL_CONST ushort PADDING_B = 16 / sizeof(BT);+ METAL_CONST ushort SHARED_STRIDE_B =+ (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;+ METAL_CONST ushort THREADGROUP_THREADS =+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;+ static_assert(+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||+ QUANT_BK % SIMDGROUP_BLOCK_K == 0,+ "QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K"+ );+ static_assert(+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||+ THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,+ "Tile block_k must be a multiple of QUANT_BK"+ );using AccumulatorType = float;+ using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<+ BT,+ THREADGROUP_BLOCK_N,+ (QUANT_BK > 0) ? QUANT_BK : 1,+ SHARED_STRIDE_B,+ 1,+ THREADGROUP_THREADS,+ (GROUP_SIZE > 0) ? GROUP_SIZE : 1,+ (BITS > 0) ? BITS : 4>;+ using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<+ BT,+ THREADGROUP_BLOCK_N,+ (QUANT_BK > 0) ? QUANT_BK : 1,+ SHARED_STRIDE_B,+ 1,+ THREADGROUP_THREADS,+ (GROUP_SIZE > 0) ? GROUP_SIZE : 1,+ (BITS > 0) ? BITS : 4>;+ using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<+ BT,+ THREADGROUP_BLOCK_N,+ (QUANT_BK > 0) ? QUANT_BK : 1,+ SHARED_STRIDE_B,+ 1,+ THREADGROUP_THREADS,+ (GROUP_SIZE > 0) ? GROUP_SIZE : 1,+ (BITS > 0) ? BITS : 4,+ true>;++ using AccumFragment = uzu::matmul::+ Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;++ template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>+ static METAL_FUNC AccumFragment quant_k_loop(+ const device AT* a_simdgroup,+ threadgroup BT* b_shared,+ const int leading_dimension_a,+ const int aligned_k_iterations,+ const short simdgroup_limit_m,+ const short simdgroup_limit_n,+ const ushort tile_col_offset,+ const ushort tile_block_cols,+ thread Loader& loader_b,+ const thread ThreadContext& thread_context+ ) {+ AccumFragment accumulator(thread_context);+ accumulator.clear();++ threadgroup BT* b_shared_simdgroup =+ b_shared + tile_col_offset * SHARED_STRIDE_B;+ const short2 tile_dimensions_b = short2(QUANT_BK, tile_block_cols);++ METAL_PRAGMA_NO_UNROLL+ for (int outer_k = 0; outer_k < aligned_k_iterations; ++outer_k) {+ threadgroup_barrier(mem_flags::mem_threadgroup);+ if constexpr (ALIGNED_N) {+ loader_b.load_unsafe();+ } else {+ loader_b.load_safe(tile_dimensions_b);+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);++ METAL_PRAGMA_NO_UNROLL+ for (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps>+ left_tile(thread_context);+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps>+ right_tile(thread_context);++ const int left_offset = inner_k;+ if constexpr (ALIGNED_M) {+ left_tile.load(a_simdgroup + left_offset, leading_dimension_a);+ } else {+ left_tile.load_safe(+ a_simdgroup + left_offset,+ leading_dimension_a,+ short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m)+ );+ }++ right_tile.load(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B));++ uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(+ accumulator,+ left_tile,+ right_tile+ );+ }++ a_simdgroup += QUANT_BK;+ loader_b.next();+ }++ return accumulator;+ }+static METAL_FUNC void run(const device AT* a,const device BT* b,@@ -48,10 +176,14 @@ struct MxuMmaCore {const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,GemmDTransform output_transform,+ const device BT* scales,+ const device BT* biases,+ const device uint8_t* zero_points,const device BT* output_bias,+ const device int32_t* rht_factors,+ threadgroup BT* b_shared,const thread ThreadContext& thread_context) {- (void)output_bias;const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);const auto geometry =ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::@@ -64,7 +196,7 @@ struct MxuMmaCore {const size_t block_col = size_t(geometry.block_col_start);const device AT* a_block = a + block_row * params->leading_dimension_a;- const device BT* b_block =+ const device BT* b_block_fp =b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);const ushort tile_row_offset =@@ -97,18 +229,35 @@ struct MxuMmaCore {const device AT* a_simdgroup =a_block + size_t(tile_row_offset) * params->leading_dimension_a;- const device BT* b_simdgroup =- b_block + (TRANSPOSE_B ? size_t(tile_col_offset) *- int(params->leading_dimension_b)- : size_t(tile_col_offset));+ const device BT* b_simdgroup_fp =+ b_block_fp + (TRANSPOSE_B ? size_t(tile_col_offset) *+ int(params->leading_dimension_b)+ : size_t(tile_col_offset));- const int aligned_k_iterations = int(params->K) / int(THREADGROUP_BLOCK_K);+ const ushort tile_block_cols = ushort(+ min(int(THREADGROUP_BLOCK_N),+ int(params->N) - int(geometry.block_col_start))+ );const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);const bool apply_accumulate =output_transform.contains(GemmDTransform::ACCUMULATE);+ const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);- dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {+ const device BT* bias_simdgroup =+ output_bias + size_t(block_col) + size_t(tile_col_offset);+ using FragType = uzu::matmul::+ Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;+ const short2 thread_position = FragType::get_position(thread_context);++ auto dispatch_aligned_k = [&](auto body) {+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {+ dispatch_bool(alignment.contains(GemmAlignment::K), body);+ } else {+ body(true_type{});+ }+ };+ dispatch_aligned_k([&](auto aligned_k) {dispatch_bool(alignment.contains(GemmAlignment::M) ||(simdgroup_limit_m == SIMDGROUP_BLOCK_M),@@ -117,29 +266,114 @@ struct MxuMmaCore {alignment.contains(GemmAlignment::N) ||(simdgroup_limit_n == SIMDGROUP_BLOCK_N),[&](auto aligned_n) {- auto accumulator_tile = uzu::matmul::gemm_loop<- AT,- BT,- SIMDGROUP_BLOCK_M,- SIMDGROUP_BLOCK_N,- SIMDGROUP_BLOCK_K,- THREADGROUP_BLOCK_K,- false,- TRANSPOSE_B,- aligned_m.value,- aligned_n.value,- aligned_k.value,- AccumulatorType>(- a_simdgroup,- b_simdgroup,- int(params->leading_dimension_a),- int(params->leading_dimension_b),- int(params->K),- aligned_k_iterations,- simdgroup_limit_m,- simdgroup_limit_n,- thread_context- );+ auto accumulator_tile = [&]() {+ if constexpr (+ B_PROLOGUE == GemmBPrologueKind::FullPrecision+ ) {+ const int aligned_k_iterations_fp =+ int(params->K) / int(THREADGROUP_BLOCK_K_FP);+ return uzu::matmul::gemm_loop<+ AT,+ BT,+ SIMDGROUP_BLOCK_M,+ SIMDGROUP_BLOCK_N,+ SIMDGROUP_BLOCK_K,+ THREADGROUP_BLOCK_K_FP,+ false,+ TRANSPOSE_B,+ aligned_m.value,+ aligned_n.value,+ aligned_k.value,+ AccumulatorType>(+ a_simdgroup,+ b_simdgroup_fp,+ int(params->leading_dimension_a),+ int(params->leading_dimension_b),+ int(params->K),+ aligned_k_iterations_fp,+ simdgroup_limit_m,+ simdgroup_limit_n,+ thread_context+ );+ } else {+ const int aligned_k_iterations_q =+ int(params->K) / int(QUANT_BK);+ constexpr int pack_factor = get_pack_factor<BITS, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();+ const int k_elements = int(params->K);+ const int weights_row_stride_bytes =+ k_elements * bytes_per_pack / pack_factor;+ const int groups_per_row =+ (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const device uint8_t* weights_block =+ reinterpret_cast<const device uint8_t*>(b) ++ block_col * weights_row_stride_bytes;+ const device BT* scales_offset =+ scales + block_col * groups_per_row;++ auto loader_b = [&]() {+ if constexpr (+ B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant+ ) {+ const device BT* biases_offset =+ biases + block_col * groups_per_row;+ return BLoaderScaleBias(+ weights_block,+ scales_offset,+ biases_offset,+ k_elements,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ } else if constexpr (+ B_PROLOGUE ==+ GemmBPrologueKind::ScaleZeroPointDequant+ ) {+ const int zero_point_stride_per_row =+ (BITS == 4) ? ((groups_per_row + 1) / 2)+ : groups_per_row;+ const device uint8_t* zero_points_row_start =+ zero_points ++ block_col * zero_point_stride_per_row;+ return BLoaderScaleZeroPoint(+ weights_block,+ scales_offset,+ zero_points_row_start,+ k_elements,+ groups_per_row,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ } else {+ return BLoaderScaleSymmetric(+ weights_block,+ scales_offset,+ nullptr,+ k_elements,+ groups_per_row,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ }+ }();++ return quant_k_loop<aligned_m.value, aligned_n.value>(+ a_simdgroup,+ b_shared,+ int(params->leading_dimension_a),+ aligned_k_iterations_q,+ simdgroup_limit_m,+ simdgroup_limit_n,+ tile_col_offset,+ tile_block_cols,+ loader_b,+ thread_context+ );+ }+ }();if (apply_scale) {const AccumulatorType scale =@@ -178,6 +412,49 @@ struct MxuMmaCore {}}+ if (apply_bias) {+ METAL_PRAGMA_UNROLL+ for (ushort tile_row = 0; tile_row < TILES_M; ++tile_row) {+ METAL_PRAGMA_UNROLL+ for (ushort tile_col = 0; tile_col < TILES_N;+ ++tile_col) {+ thread auto& frag =+ accumulator_tile.fragment_at(tile_row, tile_col);+ const short col_base = short(+ tile_col *+ uzu::matmul::MxuFragmentOps::FRAGMENT_COLS ++ thread_position.x+ );+ METAL_PRAGMA_UNROLL+ for (ushort i = 0;+ i <+ uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS;+ ++i) {+ METAL_PRAGMA_UNROLL+ for (ushort j = 0;+ j <+ uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS;+ ++j) {+ const ushort element_index =+ i * uzu::matmul::MxuFragmentOps::+ THREAD_ELEMENT_COLS ++ j;+ const short col_local = short(col_base + j);+ if constexpr (aligned_n.value) {+ frag[element_index] +=+ AccumulatorType(bias_simdgroup[col_local]);+ } else {+ if (col_local < simdgroup_limit_n) {+ frag[element_index] +=+ AccumulatorType(bias_simdgroup[col_local]);+ }+ }+ }+ }+ }+ }+ }+if constexpr (aligned_m.value && aligned_n.value) {accumulator_tile.store(d_simdgroup,@@ -195,6 +472,27 @@ struct MxuMmaCore {});});++ if (output_transform.contains(GemmDTransform::RHT)) {+ threadgroup_barrier(mem_flags::mem_device);+ device DT* d_block =+ d + block_row * params->leading_dimension_d + block_col;+ const ushort tile_block_rows = ushort(+ min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row))+ );+ const ushort tile_block_cols = ushort(+ min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col))+ );+ apply_output_random_hadamard_transform(+ d_block,+ rht_factors + block_col,+ tile_block_rows,+ tile_block_cols,+ params->leading_dimension_d,+ ushort(SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN),+ thread_context+ );+ }}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.hindex 88c5dcbbb..e329b85a8 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h@@ -7,7 +7,9 @@#include "../../common/threadgroup_tile.h"#include "../../../generated/matmul.h"#include "../generated/gemm.h"+#include "../../../hadamard_transform/hadamard_transform.h"#include "block_geometry.h"+#include "gemm_rht.h"#include "gemm_alignment.h"#include "gemm_tiling.h"#include "quant_scale_bias.h"@@ -24,8 +26,7 @@ template <typename DT,GemmTiling GEMM_TILING,bool TRANSPOSE_B,- GemmWeightPrologueKind WEIGHT_PROLOGUE =- GemmWeightPrologueKind::FullPrecision,+ GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,int BITS = 0,int GROUP_SIZE = 0>struct SimdgroupMmaCore {@@ -168,7 +169,10 @@ struct SimdgroupMmaCore {const thread uzu::matmul::TransformScaleAccumulate<float, float>&epilogue,const device BT* bias_block,- const bool needs_bias+ const bool needs_bias,+ const device int32_t* rht_factors_block,+ const bool needs_rht,+ const thread ThreadContext& thread_context) {constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};if constexpr (@@ -204,11 +208,24 @@ struct SimdgroupMmaCore {short2(tile_block_cols, tile_block_rows));}++ if (needs_rht) {+ threadgroup_barrier(mem_flags::mem_device);+ apply_output_random_hadamard_transform(+ d,+ rht_factors_block,+ tile_block_rows,+ tile_block_cols,+ params->leading_dimension_d,+ ushort(SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN),+ thread_context+ );+ }}static METAL_FUNC void run(const device AT* a,- const device uint8_t* b_packed,+ const device BT* b,device DT* d,const constant uzu::matmul::GemmParams* params,GemmAlignment alignment,@@ -217,6 +234,7 @@ struct SimdgroupMmaCore {const device BT* biases,const device uint8_t* zero_points,const device BT* output_bias,+ const device int32_t* rht_factors,threadgroup AT* a_shared,threadgroup BT* b_shared,const thread ThreadContext& thread_context@@ -256,22 +274,88 @@ struct SimdgroupMmaCore {const bool needs_accumulate =output_transform.contains(GemmDTransform::ACCUMULATE);const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);+ const bool needs_rht = output_transform.contains(GemmDTransform::RHT);const bool needs_epilogue = needs_scale || needs_accumulate;const float alpha = needs_scale ? params->ab_scale : 1.0f;const float beta = needs_accumulate ? 1.0f : 0.0f;uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);const device BT* bias_block = output_bias + block_col;+ const device int32_t* rht_factors_block = rht_factors + block_col;++ auto loader_b = [&]() {+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {+ const device BT* b_block_fp =+ b ++ (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);+ return BLoaderFp(+ b_block_fp,+ params->leading_dimension_b,+ b_shared,+ thread_context+ );+ } else {+ constexpr int pack_factor = get_pack_factor<BITS, 8>();+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();+ const int k_elements = int(params->K);+ const int weights_row_stride_bytes =+ k_elements * bytes_per_pack / pack_factor;+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;+ const device uint8_t* weights_block =+ reinterpret_cast<const device uint8_t*>(b) ++ block_col * weights_row_stride_bytes;+ const device BT* scales_offset = scales + block_col * groups_per_row;++ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {+ const device BT* biases_offset = biases + block_col * groups_per_row;+ return BLoaderScaleBias(+ weights_block,+ scales_offset,+ biases_offset,+ k_elements,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ } else if constexpr (+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant+ ) {+ const int zero_point_stride_per_row =+ (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;+ const device uint8_t* zero_points_row_start =+ zero_points + block_col * zero_point_stride_per_row;+ return BLoaderScaleZeroPoint(+ weights_block,+ scales_offset,+ zero_points_row_start,+ k_elements,+ groups_per_row,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ } else {+ return BLoaderScaleSymmetric(+ weights_block,+ scales_offset,+ nullptr,+ k_elements,+ groups_per_row,+ b_shared,+ thread_context.simdgroup_index,+ thread_context.simd_lane_id+ );+ }+ }+ }();const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||(tile_block_rows == THREADGROUP_BLOCK_M)) &&((alignment.contains(GemmAlignment::N)) ||(tile_block_cols == THREADGROUP_BLOCK_N)) &&alignment.contains(GemmAlignment::K);-constexpr uint MASK_ALL = static_cast<uint>(GemmAlignment::M) |static_cast<uint>(GemmAlignment::N) |static_cast<uint>(GemmAlignment::K);-const uint dynamic_alignment_mask =alignment.raw_value |((tile_block_rows == THREADGROUP_BLOCK_M)@@ -281,196 +365,39 @@ struct SimdgroupMmaCore {? static_cast<uint>(GemmAlignment::N): 0u);- auto dispatch_kernel = [&](auto kernel_invoke) {- if (all_aligned) {- kernel_invoke(integral_constant<uint, MASK_ALL>{});- } else {- dispatch_gemm_alignment(dynamic_alignment_mask, kernel_invoke);- }- };-- if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) {- const device BT* b = reinterpret_cast<const device BT*>(b_packed);- b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;- thread BLoaderFp- loader_b(b, params->leading_dimension_b, b_shared, thread_context);- dispatch_kernel([&](auto gemm_alignment_mask) {- constexpr uint mask = gemm_alignment_mask.value;- k_loop<mask, BLoaderFp>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- accumulator,- tile_block_rows,- tile_block_cols,- leftover_block_depth- );- finalize<mask>(- accumulator,- d,- params,- tile_block_rows,- tile_block_cols,- needs_epilogue,- epilogue,- bias_block,- needs_bias- );- });- return;- }-- if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant) {- constexpr int pack_factor = get_pack_factor<BITS, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();- const int k_elements = int(params->K);- const int weights_row_stride_bytes =- k_elements * bytes_per_pack / pack_factor;- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;- const device uint8_t* weights_block =- b_packed + block_col * weights_row_stride_bytes;- const device BT* scales_offset = scales + block_col * groups_per_row;- const device BT* biases_offset = biases + block_col * groups_per_row;- thread BLoaderScaleBias loader_b(- weights_block,- scales_offset,- biases_offset,- k_elements,+ auto kernel_invoke = [&](auto gemm_alignment_mask) {+ constexpr uint gemm_alignment = gemm_alignment_mask.value;+ k_loop<gemm_alignment>(+ a_shared,b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id+ params->aligned_inner_iterations,+ loader_a,+ loader_b,+ accumulator,+ tile_block_rows,+ tile_block_cols,+ leftover_block_depth);- dispatch_kernel([&](auto gemm_alignment_mask) {- constexpr uint mask = gemm_alignment_mask.value;- k_loop<mask, BLoaderScaleBias>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- accumulator,- tile_block_rows,- tile_block_cols,- leftover_block_depth- );- finalize<mask>(- accumulator,- d,- params,- tile_block_rows,- tile_block_cols,- needs_epilogue,- epilogue,- bias_block,- needs_bias- );- });- return;- }-- if constexpr (- WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant- ) {- constexpr int pack_factor = get_pack_factor<BITS, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();- const int k_elements = int(params->K);- const int weights_row_stride_bytes =- k_elements * bytes_per_pack / pack_factor;- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;- const device uint8_t* weights_block =- b_packed + block_col * weights_row_stride_bytes;- const device BT* scales_offset = scales + block_col * groups_per_row;- const int zero_point_stride_per_row =- (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;- const device uint8_t* zero_points_row_start =- zero_points + block_col * zero_point_stride_per_row;- thread BLoaderScaleZeroPoint loader_b(- weights_block,- scales_offset,- zero_points_row_start,- k_elements,- groups_per_row,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id+ finalize<gemm_alignment>(+ accumulator,+ d,+ params,+ tile_block_rows,+ tile_block_cols,+ needs_epilogue,+ epilogue,+ bias_block,+ needs_bias,+ rht_factors_block,+ needs_rht,+ thread_context);- dispatch_kernel([&](auto gemm_alignment_mask) {- constexpr uint mask = gemm_alignment_mask.value;- k_loop<mask, BLoaderScaleZeroPoint>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- accumulator,- tile_block_rows,- tile_block_cols,- leftover_block_depth- );- finalize<mask>(- accumulator,- d,- params,- tile_block_rows,- tile_block_cols,- needs_epilogue,- epilogue,- bias_block,- needs_bias- );- });- return;- }+ };- if constexpr (- WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleSymmetricDequant- ) {- constexpr int pack_factor = get_pack_factor<BITS, 8>();- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();- const int k_elements = int(params->K);- const int weights_row_stride_bytes =- k_elements * bytes_per_pack / pack_factor;- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;- const device uint8_t* weights_block =- b_packed + block_col * weights_row_stride_bytes;- const device BT* scales_offset = scales + block_col * groups_per_row;- thread BLoaderScaleSymmetric loader_b(- weights_block,- scales_offset,- nullptr,- k_elements,- groups_per_row,- b_shared,- thread_context.simdgroup_index,- thread_context.simd_lane_id- );- dispatch_kernel([&](auto gemm_alignment_mask) {- constexpr uint mask = gemm_alignment_mask.value;- k_loop<mask, BLoaderScaleSymmetric>(- a_shared,- b_shared,- params->aligned_inner_iterations,- loader_a,- loader_b,- accumulator,- tile_block_rows,- tile_block_cols,- leftover_block_depth- );- finalize<mask>(- accumulator,- d,- params,- tile_block_rows,- tile_block_cols,- needs_epilogue,- epilogue,- bias_block,- needs_bias- );- });+ if (all_aligned) {+ kernel_invoke(integral_constant<uint, MASK_ALL>{});+ } else {+ dispatch_gemm_alignment(dynamic_alignment_mask, kernel_invoke);}}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rsindex 5663f7268..f80edad41 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs@@ -1,14 +1,26 @@use thiserror::Error;+use crate::backends::common::gpu_types::gemm::GemmTiling;+#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]pub enum GemmSpecializationError {- #[error("threadgroup K={threadgroup_k} exceeds group size {group_size}")]- ThreadgroupKExceedsGroupSize {- threadgroup_k: u32,+ #[error("simdgroup K={simdgroup_k} exceeds group size {group_size}")]+ SimdgroupKExceedsGroupSize {+ simdgroup_k: u32,group_size: u32,},- #[error("quantized B requires the simdgroup path")]- QuantizedRequiresSimdgroup,#[error("quantized B requires transposed layout")]QuantizedRequiresTransposedB,+ #[error("tiling {tiling} does not match use_mxu={use_mxu}")]+ TilingUseMxuMismatch {+ tiling: GemmTiling,+ use_mxu: bool,+ },+ #[error(+ "MXU quantized GEMM with tile {tiling} requires group_size <= 64 (got {group_size}) due to threadgroup memory budget"+ )]+ MxuQuantTileTooLarge {+ tiling: GemmTiling,+ group_size: u32,+ },}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metalindex 3e94d9973..70ac6cb24 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal@@ -10,14 +10,18 @@using namespace metal;using namespace uzu::gemm;+#define GEMM_MXU_QUANT \+ (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision)#define GEMM_TGA_ELEMENTS \- (USE_MXU ? 1 \- : (gemm_tiling_block_m(GEMM_TILING) * \- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))+ ((USE_MXU) ? 1 \+ : (gemm_tiling_block_m(GEMM_TILING) * \+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))#define GEMM_TGB_ELEMENTS \- (USE_MXU ? 1 \- : (gemm_tiling_block_n(GEMM_TILING) * \- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * \+ (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \+ : 1) \+ : (gemm_tiling_block_n(GEMM_TILING) * \+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))template <typename AT,@@ -26,7 +30,7 @@ template <GemmTiling GEMM_TILING,bool TRANSPOSE_B,bool USE_MXU,- GemmWeightPrologueKind WEIGHT_PROLOGUE,+ GemmBPrologueKind B_PROLOGUE,uint BITS,uint GROUP_SIZE>VARIANTS(AT, bfloat, float)@@ -35,69 +39,60 @@ VARIANTS(DT, bfloat, float)CONSTRAINT(BT != "float" || (AT == "float" && DT == "float"))VARIANTS(GEMM_TILING,- GemmTiling::T8x32x32_1x1,- GemmTiling::T64x32x32_2x2,- GemmTiling::T64x64x16_2x2,- GemmTiling::T64x64x32_2x2,- GemmTiling::T32x32x32_2x2,- GemmTiling::T32x64x32_2x2,- GemmTiling::T64x32x32_4x1,- GemmTiling::T128x128x32_4x4)+ GemmTiling::Tile8x32x32_Simdgroups1x1,+ GemmTiling::Tile64x32x32_Simdgroups2x2,+ GemmTiling::Tile64x64x16_Simdgroups2x2,+ GemmTiling::Tile64x64x32_Simdgroups2x2,+ GemmTiling::Tile32x32x32_Simdgroups2x2,+ GemmTiling::Tile32x64x256_Simdgroups2x2,+ GemmTiling::Tile64x32x256_Simdgroups4x1,+ GemmTiling::Tile64x64x256_Simdgroups2x2,+ GemmTiling::Tile128x128x256_Simdgroups4x4)VARIANTS(TRANSPOSE_B, false, true)VARIANTS(USE_MXU, false, true)VARIANTS(- WEIGHT_PROLOGUE,- GemmWeightPrologueKind::FullPrecision,- GemmWeightPrologueKind::ScaleBiasDequant,- GemmWeightPrologueKind::ScaleZeroPointDequant,- GemmWeightPrologueKind::ScaleSymmetricDequant)+ B_PROLOGUE,+ GemmBPrologueKind::FullPrecision,+ GemmBPrologueKind::ScaleBiasDequant,+ GemmBPrologueKind::ScaleZeroPointDequant,+ GemmBPrologueKind::ScaleSymmetricDequant)VARIANTS(BITS, 0, 4, 8)VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)CONSTRAINT(- !USE_MXU || GEMM_TILING == GemmTiling::T64x64x32_2x2 ||- GEMM_TILING == GemmTiling::T32x64x32_2x2 ||- GEMM_TILING == GemmTiling::T64x32x32_4x1 ||- GEMM_TILING == GemmTiling::T128x128x32_4x4)-CONSTRAINT(- WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision || USE_MXU ||- GEMM_TILING == GemmTiling::T64x64x16_2x2 ||- GEMM_TILING == GemmTiling::T64x32x32_2x2)-CONSTRAINT(- TRANSPOSE_B ||- (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision &&- ((!USE_MXU &&- (GEMM_TILING == GemmTiling::T64x64x16_2x2 ||- GEMM_TILING == GemmTiling::T64x32x32_2x2)) ||- (USE_MXU &&- (GEMM_TILING == GemmTiling::T32x64x32_2x2 ||- GEMM_TILING == GemmTiling::T64x64x32_2x2 ||- GEMM_TILING == GemmTiling::T128x128x32_4x4)))))-CONSTRAINT((WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) == (BITS == 0))+ USE_MXU ==+ (GEMM_TILING == GemmTiling::Tile32x64x256_Simdgroups2x2 ||+ GEMM_TILING == GemmTiling::Tile64x32x256_Simdgroups4x1 ||+ GEMM_TILING == GemmTiling::Tile64x64x256_Simdgroups2x2 ||+ GEMM_TILING == GemmTiling::Tile128x128x256_Simdgroups4x4))+CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))-CONSTRAINT(WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision || BT != "float")+CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")+CONSTRAINT(+ GROUP_SIZE != 16 ||+ GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2)CONSTRAINT(- WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||- (!USE_MXU && TRANSPOSE_B &&- (GEMM_TILING == GemmTiling::T8x32x32_1x1 ||- GEMM_TILING == GemmTiling::T64x64x16_2x2 ||- GEMM_TILING == GemmTiling::T32x32x32_2x2 ||- GEMM_TILING == GemmTiling::T64x32x32_2x2 ||- GEMM_TILING == GemmTiling::T64x64x32_2x2)))+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||+ (TRANSPOSE_B &&+ (GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 ||+ GROUP_SIZE == 16)))CONSTRAINT(- WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||- GROUP_SIZE != 16 || GEMM_TILING == GemmTiling::T64x64x16_2x2)+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||+ GEMM_TILING != GemmTiling::Tile128x128x256_Simdgroups4x4 ||+ GROUP_SIZE <= 64)KERNEL(Gemm)(const device AT* a,- const device uint8_t* b_packed,+ const device BT* b,device DT* d,const device BT* scales- OPTIONAL(WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision),+ OPTIONAL(B_PROLOGUE != GemmBPrologueKind::FullPrecision),const device BT* biases- OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant),+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant),const device uint8_t* zero_points- OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant),+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant),const device BT* output_biasOPTIONAL(output_transform.contains(GemmDTransform::BIAS)),+ const device int32_t* rht_factors+ OPTIONAL(output_transform.contains(GemmDTransform::RHT)),const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,@@ -119,20 +114,28 @@ KERNEL(Gemm)((void)thread_z;if constexpr (USE_MXU) {- (void)scales;- (void)biases;- (void)zero_points;- const device BT* b = reinterpret_cast<const device BT*>(b_packed);- MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B>::run(- a,- b,- d,- params,- alignment,- output_transform,- output_bias,- thread_context- );+ MxuMmaCore<+ AT,+ BT,+ DT,+ GEMM_TILING,+ TRANSPOSE_B,+ B_PROLOGUE,+ BITS,+ GROUP_SIZE>::+ run(a,+ b,+ d,+ params,+ alignment,+ output_transform,+ scales,+ biases,+ zero_points,+ output_bias,+ rht_factors,+ b_shared,+ thread_context);} else {SimdgroupMmaCore<AT,@@ -140,11 +143,11 @@ KERNEL(Gemm)(DT,GEMM_TILING,TRANSPOSE_B,- WEIGHT_PROLOGUE,+ B_PROLOGUE,BITS,GROUP_SIZE>::run(a,- b_packed,+ b,d,params,alignment,@@ -153,6 +156,7 @@ KERNEL(Gemm)(biases,zero_points,output_bias,+ rht_factors,a_shared,b_shared,thread_context);diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rsindex 6b52829bd..0a19a29a2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs@@ -1,6 +1,6 @@use std::collections::{HashMap, hash_map::Entry};-use super::{MXU_THREADGROUP_BLOCK_K, specialization::GemmSpecialization};+use super::specialization::GemmSpecialization;use crate::{backends::{common::{@@ -94,8 +94,8 @@ impl GemmKernel {specialization.tiling,specialization.transpose_b,specialization.use_mxu,- specialization.weight_prologue,- specialization.bits_per_weight.unwrap_or(0),+ specialization.b_prologue,+ specialization.bits_per_b.unwrap_or(0),specialization.group_size.unwrap_or(0),specialization.output_transform,specialization.alignment,@@ -110,11 +110,30 @@ impl GemmKernel {arguments: MatmulArguments<'a, Metal, TB>,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {+ let mxu_eligible_for_quant = match &arguments.b {+ MatmulB::FullPrecision {+ ..+ } => true,+ MatmulB::ScaleBiasDequant {+ ..+ }+ | MatmulB::ScaleZeroPointDequant {+ ..+ }+ | MatmulB::ScaleSymmetricDequant {+ ..+ } => {+ arguments.b_transpose+ && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.k)+ && arguments.b_offset == 0+ && arguments.k % select_mxu_tiling(arguments.m, arguments.n).block_k() == 0+ },+ };let path = if encoder.context().device.supports_mxu()&& [self.weights_data_type, self.input_data_type, self.output_data_type].into_iter().all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32))- && matches!(arguments.b, MatmulB::FullPrecision { .. })+ && mxu_eligible_for_quant{GemmDispatchPath::Mxu} else {@@ -129,29 +148,21 @@ impl GemmKernel {path: GemmDispatchPath,encoder: &mut Encoder<Metal>,) -> Result<(), MetalError> {- let use_mxu = match path {- GemmDispatchPath::Mxu => {- assert!(- encoder.context().device.supports_mxu(),- "GemmDispatchPath::Mxu requested on hardware without MXU support",- );- assert!(- [self.weights_data_type, self.input_data_type, self.output_data_type]- .into_iter()- .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32)),- "GemmDispatchPath::Mxu requires BF16 or F32 data types, got weights {:?}, input {:?}, output {:?}",- self.weights_data_type,- self.input_data_type,- self.output_data_type,- );- assert!(- matches!(arguments.b, MatmulB::FullPrecision { .. }),- "GemmDispatchPath::Mxu requires FullPrecision B",- );- true- },- GemmDispatchPath::Simdgroup => false,- };+ if matches!(path, GemmDispatchPath::Mxu) {+ assert!(+ encoder.context().device.supports_mxu(),+ "GemmDispatchPath::Mxu requested on hardware without MXU support",+ );+ assert!(+ [self.weights_data_type, self.input_data_type, self.output_data_type]+ .into_iter()+ .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32)),+ "GemmDispatchPath::Mxu requires BF16 or F32 data types, got weights {:?}, input {:?}, output {:?}",+ self.weights_data_type,+ self.input_data_type,+ self.output_data_type,+ );+ }let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });if is_quant {@@ -176,25 +187,12 @@ impl GemmKernel {}let ab_scale = arguments.d_transform.ab_scale;- let post_bias = if use_mxu {- arguments.d_transform.bias- } else {- None- };- let output_bias = if use_mxu {- None- } else {- arguments.d_transform.bias- };- let post_rht = arguments.d_transform.rht_factors;- let mut output_transform = arguments.d_transform.mask();- output_transform.remove(GemmDTransform::RHT);- if use_mxu {- output_transform.remove(GemmDTransform::BIAS);- }+ let output_bias = arguments.d_transform.bias;+ let rht_factors = arguments.d_transform.rht_factors;+ let output_transform = arguments.d_transform.mask();- let weight_prologue = arguments.b.weight_prologue();- let bits_per_weight = arguments.b.bits_per_weight();+ let b_prologue = arguments.b.b_prologue();+ let bits_per_b = arguments.b.bits_per_b();let group_size = arguments.b.group_size();let MatmulArguments {@@ -211,6 +209,8 @@ impl GemmKernel {..} = arguments;+ let use_mxu = matches!(path, GemmDispatchPath::Mxu);+match b {MatmulB::FullPrecision {b: weights,@@ -220,11 +220,6 @@ impl GemmKernel {} else {select_simdgroup_tiling(m, n, k)};- let k_block = if use_mxu {- MXU_THREADGROUP_BLOCK_K- } else {- tiling.block_k()- };let threadgroups_per_row = n.div_ceil(tiling.block_n());let threadgroups_per_column = m.div_ceil(tiling.block_m());@@ -246,7 +241,7 @@ impl GemmKernel {};let alignment =- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % k_block == 0);+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);let default_ldb = if b_transpose {k@@ -262,7 +257,7 @@ impl GemmKernel {leading_dimension_d: n,threadgroups_per_row,threadgroups_per_column,- aligned_inner_iterations: k / k_block,+ aligned_inner_iterations: k / tiling.block_k(),use_morton,ab_scale,};@@ -274,8 +269,8 @@ impl GemmKernel {output_transform,alignment,transpose_b: b_transpose,- weight_prologue,- bits_per_weight,+ b_prologue,+ bits_per_b,group_size,};specialization.validate()?;@@ -288,6 +283,7 @@ impl GemmKernel {None::<&Allocation<Metal>>,None::<&Allocation<Metal>>,output_bias,+ rht_factors,std::slice::from_ref(¶ms),group_count_x,group_count_y,@@ -324,7 +320,11 @@ impl GemmKernel {_ => unreachable!(),};- let tiling = select_quant_tiling(m, n, group_size.unwrap());+ let tiling = if use_mxu {+ select_mxu_quant_tiling(m, n, group_size.unwrap_or(0))+ } else {+ select_quant_tiling(m, n, group_size.unwrap_or(0))+ };let alignment =GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);let params = quant_params(m, n, k, tiling, ab_scale);@@ -338,8 +338,8 @@ impl GemmKernel {output_transform,alignment,transpose_b: true,- weight_prologue,- bits_per_weight,+ b_prologue,+ bits_per_b,group_size,};specialization.validate()?;@@ -352,6 +352,7 @@ impl GemmKernel {biases,zero_points,output_bias,+ rht_factors,std::slice::from_ref(¶ms),group_count_x,group_count_y,@@ -360,12 +361,6 @@ impl GemmKernel {},}- if let Some(bias) = post_bias {- self.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, &mut *d, n, m * n, encoder);- }- if let Some(factors) = post_rht {- self.hadamard.encode(d, factors, n, m, encoder);- }Ok(())}}@@ -398,9 +393,9 @@ fn select_simdgroup_tiling(k: u32,) -> GemmTiling {if 2 * m.max(n) > k {- GemmTiling::T64x64x16_2x2+ GemmTiling::Tile64x64x16_Simdgroups2x2} else {- GemmTiling::T64x32x32_2x2+ GemmTiling::Tile64x32x32_Simdgroups2x2}}@@ -409,13 +404,29 @@ fn select_mxu_tiling(n: u32,) -> GemmTiling {if m >= 256 && n >= 128 {- GemmTiling::T128x128x32_4x4+ GemmTiling::Tile128x128x256_Simdgroups4x4+ } else if n < 64 {+ GemmTiling::Tile64x32x256_Simdgroups4x1+ } else if m < 64 {+ GemmTiling::Tile32x64x256_Simdgroups2x2+ } else {+ GemmTiling::Tile64x64x256_Simdgroups2x2+ }+}++fn select_mxu_quant_tiling(+ m: u32,+ n: u32,+ group_size: u32,+) -> GemmTiling {+ if m >= 256 && n >= 128 && GemmTiling::Tile128x128x256_Simdgroups4x4.fits_quant_group_size(group_size) {+ GemmTiling::Tile128x128x256_Simdgroups4x4} else if n < 64 {- GemmTiling::T64x32x32_4x1+ GemmTiling::Tile64x32x256_Simdgroups4x1} else if m < 64 {- GemmTiling::T32x64x32_2x2+ GemmTiling::Tile32x64x256_Simdgroups2x2} else {- GemmTiling::T64x64x32_2x2+ GemmTiling::Tile64x64x256_Simdgroups2x2}}@@ -425,14 +436,14 @@ fn select_quant_tiling(group_size: u32,) -> GemmTiling {if group_size < 32 {- GemmTiling::T64x64x16_2x2+ GemmTiling::Tile64x64x16_Simdgroups2x2} else if m < 32 {- GemmTiling::T8x32x32_1x1+ GemmTiling::Tile8x32x32_Simdgroups1x1} else if m >= 64 && n <= 2048 {- GemmTiling::T64x32x32_2x2+ GemmTiling::Tile64x32x32_Simdgroups2x2} else if m >= 64 && n >= 6144 && n % 64 == 0 {- GemmTiling::T64x64x32_2x2+ GemmTiling::Tile64x64x32_Simdgroups2x2} else {- GemmTiling::T32x32x32_2x2+ GemmTiling::Tile32x32x32_Simdgroups2x2}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rsindex 05ecef5a5..3841e73d9 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs@@ -4,6 +4,3 @@ mod specialization;pub use error::GemmSpecializationError;pub use kernel::{GemmDispatchPath, GemmKernel};--// Matches `MxuMmaCore::THREADGROUP_BLOCK_K` in `common/mxu_mma_core.h`.-pub(crate) const MXU_THREADGROUP_BLOCK_K: u32 = 256;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rsindex 4e6d594c1..78cafb8cc 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs@@ -3,7 +3,7 @@ use crate::{backends::common::{gpu_types::{QuantizationMethod,- gemm::{GemmAlignment, GemmDTransform, GemmTiling, GemmWeightPrologueKind},+ gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},},kernel::matmul::MatmulQuantCombo,},@@ -18,29 +18,43 @@ pub(crate) struct GemmSpecialization {pub(crate) output_transform: GemmDTransform,pub(crate) alignment: GemmAlignment,pub(crate) transpose_b: bool,- pub(crate) weight_prologue: GemmWeightPrologueKind,- pub(crate) bits_per_weight: Option<u32>,+ pub(crate) b_prologue: GemmBPrologueKind,+ pub(crate) bits_per_b: Option<u32>,pub(crate) group_size: Option<u32>,}impl GemmSpecialization {pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {- if let Some(group_size) = self.group_size {- if self.tiling.block_k() > group_size {- return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {- threadgroup_k: self.tiling.block_k(),- group_size,- });- }+ if self.use_mxu != self.tiling.is_mxu_variant() {+ return Err(GemmSpecializationError::TilingUseMxuMismatch {+ tiling: self.tiling,+ use_mxu: self.use_mxu,+ });}- if self.weight_prologue != GemmWeightPrologueKind::FullPrecision {- if self.use_mxu {- return Err(GemmSpecializationError::QuantizedRequiresSimdgroup);+ if self.use_mxu && self.b_prologue != GemmBPrologueKind::FullPrecision {+ if let Some(group_size) = self.group_size {+ if !self.tiling.fits_quant_group_size(group_size) {+ return Err(GemmSpecializationError::MxuQuantTileTooLarge {+ tiling: self.tiling,+ group_size,+ });+ }}- if !self.transpose_b {- return Err(GemmSpecializationError::QuantizedRequiresTransposedB);+ }+ if !self.use_mxu {+ if let Some(group_size) = self.group_size {+ let simdgroup_block_k = self.tiling.simdgroup_block_k();+ if simdgroup_block_k > group_size {+ return Err(GemmSpecializationError::SimdgroupKExceedsGroupSize {+ simdgroup_k: simdgroup_block_k,+ group_size,+ });+ }}}+ if self.b_prologue != GemmBPrologueKind::FullPrecision && !self.transpose_b {+ return Err(GemmSpecializationError::QuantizedRequiresTransposedB);+ }Ok(())}@@ -48,7 +62,12 @@ impl GemmSpecialization {let mut out = Vec::new();for &tiling in simdgroup_tiling_set(weights_data_type) {for align_mn in [true, false] {- for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {+ for output_transform in [+ GemmDTransform::empty(),+ GemmDTransform::BIAS,+ GemmDTransform::RHT,+ GemmDTransform::BIAS | GemmDTransform::RHT,+ ] {out.push(Self {weights_data_type,tiling,@@ -56,8 +75,8 @@ impl GemmSpecialization {output_transform,alignment: GemmAlignment::new(align_mn, align_mn, true),transpose_b: true,- weight_prologue: GemmWeightPrologueKind::FullPrecision,- bits_per_weight: None,+ b_prologue: GemmBPrologueKind::FullPrecision,+ bits_per_b: None,group_size: None,});}@@ -73,6 +92,21 @@ impl GemmSpecialization {GemmDTransform::SCALE,GemmDTransform::ACCUMULATE,GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,+ GemmDTransform::BIAS,+ GemmDTransform::BIAS | GemmDTransform::SCALE,+ GemmDTransform::BIAS | GemmDTransform::ACCUMULATE,+ GemmDTransform::BIAS | GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,+ GemmDTransform::RHT,+ GemmDTransform::RHT | GemmDTransform::SCALE,+ GemmDTransform::RHT | GemmDTransform::ACCUMULATE,+ GemmDTransform::RHT | GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,+ GemmDTransform::BIAS | GemmDTransform::RHT,+ GemmDTransform::BIAS | GemmDTransform::RHT | GemmDTransform::SCALE,+ GemmDTransform::BIAS | GemmDTransform::RHT | GemmDTransform::ACCUMULATE,+ GemmDTransform::BIAS+ | GemmDTransform::RHT+ | GemmDTransform::SCALE+ | GemmDTransform::ACCUMULATE,] {out.push(Self {weights_data_type,@@ -81,8 +115,8 @@ impl GemmSpecialization {output_transform,alignment: GemmAlignment::new(align_m, align_n, align_k),transpose_b: true,- weight_prologue: GemmWeightPrologueKind::FullPrecision,- bits_per_weight: None,+ b_prologue: GemmBPrologueKind::FullPrecision,+ bits_per_b: None,group_size: None,});}@@ -99,18 +133,23 @@ impl GemmSpecialization {) -> Vec<Self> {let bits = DataType::from(combo.mode).size_in_bits() as u32;let group_size = combo.group_size;- let weight_prologue = match combo.method {- QuantizationMethod::ScaleBias => GemmWeightPrologueKind::ScaleBiasDequant,- QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,- QuantizationMethod::ScaleSymmetric => GemmWeightPrologueKind::ScaleSymmetricDequant,+ let b_prologue = match combo.method {+ QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,+ QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,+ QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,};let mut out = Vec::new();for &tiling in quant_tiling_set(weights_data_type) {- if tiling.block_k() > group_size {+ if tiling.simdgroup_block_k() > group_size {continue;}for align_n in [true, false] {- for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {+ for output_transform in [+ GemmDTransform::empty(),+ GemmDTransform::BIAS,+ GemmDTransform::RHT,+ GemmDTransform::BIAS | GemmDTransform::RHT,+ ] {out.push(Self {weights_data_type,tiling,@@ -118,20 +157,54 @@ impl GemmSpecialization {output_transform,alignment: GemmAlignment::new(true, align_n, true),transpose_b: true,- weight_prologue,- bits_per_weight: Some(bits),+ b_prologue,+ bits_per_b: Some(bits),group_size: Some(group_size),});}}}+ for &tiling in mxu_tiling_set(weights_data_type) {+ if group_size % 32 != 0 {+ continue;+ }+ if !tiling.fits_quant_group_size(group_size) {+ continue;+ }+ for align_m in [true, false] {+ for align_n in [true, false] {+ for output_transform in [+ GemmDTransform::empty(),+ GemmDTransform::SCALE,+ GemmDTransform::BIAS,+ GemmDTransform::RHT,+ GemmDTransform::BIAS | GemmDTransform::RHT,+ GemmDTransform::BIAS | GemmDTransform::SCALE,+ ] {+ out.push(Self {+ weights_data_type,+ tiling,+ use_mxu: true,+ output_transform,+ alignment: GemmAlignment::new(align_m, align_n, true),+ transpose_b: true,+ b_prologue,+ bits_per_b: Some(bits),+ group_size: Some(group_size),+ });+ }+ }+ }+ }out}}fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {match data_type {- DataType::BF16 | DataType::F32 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],+ DataType::BF16 | DataType::F32 => {+ &[GemmTiling::Tile64x32x32_Simdgroups2x2, GemmTiling::Tile64x64x16_Simdgroups2x2]+ },_ => &[],}}@@ -140,17 +213,21 @@ fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {if !matches!(data_type, DataType::BF16 | DataType::F32) {return &[];}- &[GemmTiling::T64x64x32_2x2, GemmTiling::T32x64x32_2x2, GemmTiling::T64x32x32_4x1, GemmTiling::T128x128x32_4x4]+ &[+ GemmTiling::Tile64x64x256_Simdgroups2x2,+ GemmTiling::Tile32x64x256_Simdgroups2x2,+ GemmTiling::Tile64x32x256_Simdgroups4x1,+ GemmTiling::Tile128x128x256_Simdgroups4x4,+ ]}pub(crate) fn quant_tiling_set(data_type: DataType) -> &'static [GemmTiling] {match data_type {DataType::BF16 => &[- GemmTiling::T64x64x16_2x2,- GemmTiling::T8x32x32_1x1,- GemmTiling::T32x32x32_2x2,- GemmTiling::T64x32x32_2x2,- GemmTiling::T64x64x32_2x2,+ GemmTiling::Tile8x32x32_Simdgroups1x1,+ GemmTiling::Tile32x32x32_Simdgroups2x2,+ GemmTiling::Tile64x32x32_Simdgroups2x2,+ GemmTiling::Tile64x64x32_Simdgroups2x2,],_ => &[],}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rsindex ca75e02b7..d396315d2 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs@@ -4,4 +4,4 @@ mod quant_kernel;mod spec;pub(crate) use kernel::GemvKernel;-pub(crate) use quant_kernel::QuantGemvKernel;+pub use quant_kernel::QuantGemvKernel;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metalindex daa728584..7fbc08019 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal@@ -18,7 +18,7 @@ VARIANTS(InputT, float, bfloat)VARIANTS(OutputT, float, bfloat)VARIANTS(GROUP_SIZE, 16, 32, 64, 128)VARIANTS(BITS, 4, 8)-PUBLIC KERNEL(QuantizedMatmulQmv)(+KERNEL(QuantizedMatmulQmv)(const device uint32_t* weights,const device WeightT* scales,const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metalindex 0ce1b822f..8fe649a8a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal@@ -19,7 +19,7 @@ VARIANTS(InputT, float, bfloat)VARIANTS(OutputT, float, bfloat)VARIANTS(GROUP_SIZE, 16, 32, 64, 128)VARIANTS(BITS, 4, 8)-PUBLIC KERNEL(QuantizedMatmulQmvFast)(+KERNEL(QuantizedMatmulQmvFast)(const device uint32_t* weights,const device WeightT* scales,const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rsindex c1e769f08..9e456cd82 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs@@ -5,12 +5,13 @@ use crate::{common::{AsBufferRangeRef, Buffer, Encoder,gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},- kernel::{- Kernels, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,- matmul::{MatmulArguments, MatmulB, MatmulError},- },+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},+ },+ metal::{+ Metal,+ context::MetalContext,+ kernel::{QuantizedMatmulQmvFastMetalKernel, QuantizedMatmulQmvMetalKernel},},- metal::{Metal, context::MetalContext},},data_type::DataType,};@@ -25,19 +26,16 @@ struct QmvKey {use_hadamard: bool,}-pub(crate) struct QuantGemvKernel {+pub struct QuantGemvKernel {weights_data_type: DataType,input_data_type: DataType,output_data_type: DataType,- qmv: HashMap<QmvKey, <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel>,- qmv_fast: HashMap<- QmvKey,- <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel,- >,+ qmv: HashMap<QmvKey, QuantizedMatmulQmvMetalKernel>,+ qmv_fast: HashMap<QmvKey, QuantizedMatmulQmvFastMetalKernel>,}impl QuantGemvKernel {- pub(crate) fn new(+ pub fn new(_context: &MetalContext,weights_data_type: DataType,input_data_type: DataType,@@ -52,7 +50,7 @@ impl QuantGemvKernel {}}- pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(+ pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(&mut self,encoder: &mut Encoder<Metal>,arguments: MatmulArguments<'a, Metal, TB>,@@ -141,7 +139,7 @@ impl QuantGemvKernel {let kernel = match self.qmv_fast.entry(key) {Entry::Occupied(entry) => entry.into_mut(),Entry::Vacant(entry) => {- let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ let kernel = QuantizedMatmulQmvFastMetalKernel::new(context,self.weights_data_type,self.input_data_type,@@ -173,7 +171,7 @@ impl QuantGemvKernel {let kernel = match self.qmv.entry(key) {Entry::Occupied(entry) => entry.into_mut(),Entry::Vacant(entry) => {- let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(+ let kernel = QuantizedMatmulQmvMetalKernel::new(context,self.weights_data_type,self.input_data_type,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex 562a76ab3..982c80ca7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -3,8 +3,11 @@ pub mod gemv;use std::sync::OnceLock;-pub use self::gemm::{GemmDispatchPath, GemmKernel};-use self::gemv::{GemvKernel, QuantGemvKernel};+use self::gemv::GemvKernel;+pub use self::{+ gemm::{GemmDispatchPath, GemmKernel},+ gemv::QuantGemvKernel,+};use crate::{backends::{common::{@@ -22,7 +25,7 @@ use crate::{pub struct MatmulMetalKernel {gemv: GemvKernel,- quant_gemv: QuantGemvKernel,+ pub quant_gemv: QuantGemvKernel,pub gemm: GemmKernel,}diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rsindex 899bccb05..512debeee 100644--- a/crates/backend-uzu/tests/common/matmul/harness.rs+++ b/crates/backend-uzu/tests/common/matmul/harness.rs@@ -29,6 +29,8 @@ pub struct Case {pub ab_scale: f32,pub accumulate: bool,pub b_transpose: bool,+ pub enable_rht: bool,+ pub enable_bias: bool,}impl Case {@@ -38,6 +40,8 @@ impl Case {ab_scale: 1.0,accumulate: false,b_transpose: true,+ enable_rht: false,+ enable_bias: false,}}@@ -56,12 +60,30 @@ impl Case {self.accumulate = accumulate;self}++ pub const fn with_rht(+ mut self,+ enable_rht: bool,+ ) -> Self {+ self.enable_rht = enable_rht;+ self+ }++ pub const fn with_bias(+ mut self,+ enable_bias: bool,+ ) -> Self {+ self.enable_bias = enable_bias;+ self+ }}pub struct Input<T: ArrayElement + Float> {pub a: Box<[T]>,pub b: Box<[T]>,pub d_prefill: Option<Box<[T]>>,+ pub rht_factors: Option<Box<[i32]>>,+ pub bias: Option<Box<[T]>>,pub case: Case,}@@ -76,10 +98,27 @@ pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {let d_prefill = case.accumulate.then(|| {(0..m * n).map(|i| T::from(((i % 7) as f32) * 0.03 - 0.09).unwrap()).collect::<Vec<_>>().into_boxed_slice()});+ let rht_factors = (case.enable_rht && n % 32 == 0).then(|| {+ (0..n)+ .map(|i| {+ if (i % 2) == 0 {+ 1i32+ } else {+ -1i32+ }+ })+ .collect::<Vec<_>>()+ .into_boxed_slice()+ });+ let bias = case.enable_bias.then(|| {+ (0..n).map(|i| T::from(((i % 11) as f32) * 0.05 - 0.25).unwrap()).collect::<Vec<_>>().into_boxed_slice()+ });Input {a: a.into_boxed_slice(),b: b.into_boxed_slice(),d_prefill,+ rht_factors,+ bias,case,}}@@ -108,12 +147,15 @@ fn run<B: Backend, T: ArrayElement + Float>(.create_allocation(m * n * std::mem::size_of::<T>(), AllocationType::Global).expect("create d allocation")};+ let rht_allocation =+ input.rht_factors.as_ref().map(|factors| alloc_allocation_with_data::<B, i32>(context, factors));+ let bias_allocation = input.bias.as_ref().map(|bias| alloc_allocation_with_data::<B, T>(context, bias));let d_transform = MatmulDOps::<'_, B> {ab_scale: input.case.ab_scale,accumulate: input.case.accumulate,- bias: None,- rht_factors: None,+ bias: bias_allocation.as_ref(),+ rht_factors: rht_allocation.as_ref(),};let mut encoder = Encoder::new(context).expect("encoder");diff --git a/crates/backend-uzu/tests/common/mod.rs b/crates/backend-uzu/tests/common/mod.rsindex c9a00c538..30a73dfe6 100644--- a/crates/backend-uzu/tests/common/mod.rs+++ b/crates/backend-uzu/tests/common/mod.rs@@ -14,6 +14,15 @@ pub fn type_short_name<T>() -> &'static str {std::any::type_name::<T>().rsplit("::").next().unwrap()}+#[cfg(metal_backend)]+pub fn shared_metal_context() -> std::rc::Rc<backend_uzu::backends::metal::MetalContext> {+ use backend_uzu::backends::{common::Context, metal::MetalContext};+ thread_local! {+ static CTX: std::cell::OnceCell<std::rc::Rc<MetalContext>> = const { std::cell::OnceCell::new() };+ }+ CTX.with(|cell| cell.get_or_init(|| MetalContext::new().expect("Metal context")).clone())+}+/// Invokes `$body` once per available backend, with `$B` bound to each backend type.macro_rules! for_each_backend {(|$B:ident| $body:expr) => {{diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rsindex 5251fa6c4..23c79c7a5 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs@@ -10,7 +10,7 @@ use backend_uzu::{matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},},},- metal::{GemmDispatchPath, Metal, MetalContext},+ metal::{GemmDispatchPath, Metal},},};use criterion::{BenchmarkId, Criterion, Throughput};@@ -26,7 +26,7 @@ use crate::{#[uzu_bench]fn bench_gemm(c: &mut Criterion) {- let context = MetalContext::new().expect("Metal context");+ let context = crate::common::shared_metal_context();let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(&context,bf16::data_type(),@@ -35,44 +35,52 @@ fn bench_gemm(c: &mut Criterion) {).expect("MatmulKernel");- let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));+ let paths: &[(&str, GemmDispatchPath)] = if context.supports_mxu() {+ &[("GEMM", GemmDispatchPath::Simdgroup), ("GEMM_MXU", GemmDispatchPath::Mxu)]+ } else {+ &[("GEMM", GemmDispatchPath::Simdgroup)]+ };- for shape in bench_fp_gemm_shapes() {- let (m, k, n) = (shape.m, shape.k, shape.n);- let a = context- .create_allocation(m * k * std::mem::size_of::<bf16>(), AllocationType::Global)- .expect("a allocation");- let b_array = context.create_array_uninitialized(&[n, k], bf16::data_type());- let mut d = context- .create_allocation(m * n * std::mem::size_of::<bf16>(), AllocationType::Global)- .expect("d allocation");+ for &(group_label, path) in paths {+ let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/{}", type_short_name::<Metal>(), group_label));- group.throughput(Throughput::Elements((2 * m * k * n) as u64));- group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {- iter_encode_loop::<Metal, _>(&context, b, |encoder| {- kernel- .gemm- .encode_dispatch_path(- MatmulArguments {- a: &a,- a_offset: 0,- b: MatmulB::FullPrecision {- b: b_array.allocation(),+ for shape in bench_fp_gemm_shapes() {+ let (m, k, n) = (shape.m, shape.k, shape.n);+ let a = context+ .create_allocation(m * k * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("a allocation");+ let b_array = context.create_array_uninitialized(&[n, k], bf16::data_type());+ let mut d = context+ .create_allocation(m * n * std::mem::size_of::<bf16>(), AllocationType::Global)+ .expect("d allocation");++ group.throughput(Throughput::Elements((2 * m * k * n) as u64));+ group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {+ iter_encode_loop::<Metal, _>(&context, b, |encoder| {+ kernel+ .gemm+ .encode_dispatch_path(+ MatmulArguments {+ a: &a,+ a_offset: 0,+ b: MatmulB::FullPrecision {+ b: b_array.allocation(),+ },+ b_offset: 0,+ b_leading_dimension: None,+ b_transpose: true,+ d: &mut d,+ d_transform: MatmulDOps::none(),+ m: m as u32,+ n: n as u32,+ k: k as u32,},- b_offset: 0,- b_leading_dimension: None,- b_transpose: true,- d: &mut d,- d_transform: MatmulDOps::none(),- m: m as u32,- n: n as u32,- k: k as u32,- },- GemmDispatchPath::Mxu,- encoder,- )- .expect("encode_dispatch_path failed");+ path,+ encoder,+ )+ .expect("encode_dispatch_path failed");+ });});- });+ }}}diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rsindex 5b3a6f52f..0a6a1550f 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs@@ -4,7 +4,7 @@ use backend_uzu::{array::ArrayElement,backends::{common::{- Backend, Context,+ Backend,gpu_types::QuantizationMethod,kernel::{Kernels, matmul::MatmulKernel},},@@ -31,6 +31,13 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>(bits: u32,quant_method: QuantizationMethod,) {+ let supports_mxu = context.supports_mxu();+ let paths: &[(&str, GemmDispatchPath)] = if supports_mxu {+ &[("Simdgroup", GemmDispatchPath::Simdgroup), ("Mxu", GemmDispatchPath::Mxu)]+ } else {+ &[("Simdgroup", GemmDispatchPath::Simdgroup)]+ };+for shape in bench_quant_gemm_shapes(bits) {let (m, k, n) = (shape.m, shape.k, shape.n);let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);@@ -43,24 +50,30 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>().unwrap();- let mut group =- c.benchmark_group(format!("{}/Kernel/UnifiedQuantizedGemm/{}", type_short_name::<Metal>(), label));- group.throughput(Throughput::Elements((m * n * k) as u64));- group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {- iter_encode_loop::<Metal, _>(context, b, |encoder| {- matmul- .gemm- .encode_dispatch_path(quant_arguments(&mut buffers, &input), GemmDispatchPath::Simdgroup, encoder)- .expect("encode unified quant matmul");+ for &(path_label, path) in paths {+ let mut group = c.benchmark_group(format!(+ "{}/Kernel/UnifiedQuantizedGemm/{}/{}",+ type_short_name::<Metal>(),+ path_label,+ label+ ));+ group.throughput(Throughput::Elements((m * n * k) as u64));+ group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {+ iter_encode_loop::<Metal, _>(context, b, |encoder| {+ matmul+ .gemm+ .encode_dispatch_path(quant_arguments(&mut buffers, &input), path, encoder)+ .expect("encode unified quant matmul");+ });});- });- drop(group);+ drop(group);+ }}}#[uzu_bench]fn bench_unified_quantized_gemm(c: &mut Criterion) {- let context = MetalContext::new().expect("Metal context");+ let context = crate::common::shared_metal_context();bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rsindex 3111d9687..85b56a68b 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs@@ -1,7 +1,14 @@+#![cfg(metal_backend)]+use backend_uzu::{array::ArrayElement,- backends::common::{- Allocation, Backend, Context, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvFastKernel,+ backends::{+ common::{+ Backend,+ gpu_types::QuantizationMethod,+ kernel::{Kernels, matmul::MatmulKernel},+ },+ metal::{Metal, MetalContext},},};use criterion::{BenchmarkId, Criterion, Throughput};@@ -10,70 +17,48 @@ use num_traits::Float;use crate::{common::{- matmul::{QuantBuffers, QuantInput, bench_quant_gemv_shapes, iter_encode_loop},+ matmul::{QuantBuffers, QuantInput, bench_quant_gemv_shapes, iter_encode_loop, quant_arguments},type_short_name,},uzu_bench,};-fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(+fn bench_quant_gemv_typed<T: ArrayElement + Float>(+ context: &MetalContext,c: &mut Criterion,- context: &B::Context,label: &str,group_size: u32,bits: u32,quant_method: QuantizationMethod,) {- let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));+ let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/QuantGemv/{}", type_short_name::<Metal>(), label));for shape in bench_quant_gemv_shapes(bits) {let (m, k, n) = (shape.m, shape.k, shape.n);let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);- let mut buffers = QuantBuffers::<B, T>::allocate(context, &input);-- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(context,T::data_type(),T::data_type(),T::data_type(),- group_size,- bits,- quant_method,- false,).unwrap();group.throughput(Throughput::Elements((m * n * k) as u64));group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {- iter_encode_loop::<B, _>(context, b, |encoder| {- kernel.encode(- &buffers.w,- &buffers.scales,- buffers.zp.as_ref(),- buffers.bias.as_ref(),- &buffers.x,- &mut buffers.y,- None::<&Allocation<B>>,- k as u32,- n as u32,- m as u32,- encoder,- );+ iter_encode_loop::<Metal, _>(context, b, |encoder| {+ matmul.quant_gemv.encode(encoder, quant_arguments(&mut buffers, &input)).expect("encode quant gemv");});});}}#[uzu_bench]-fn bench_qmv_fast(c: &mut Criterion) {- for_each_backend!(|B| {- let context = <B as Backend>::Context::new().unwrap();- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);- });+fn bench_quant_gemv(c: &mut Criterion) {+ let context = crate::common::shared_metal_context();+ bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);+ bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);+ bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);+ bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);}diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rsindex dacfa71fe..e80110a40 100644--- a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs+++ b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs@@ -4,7 +4,7 @@ use backend_uzu::{array::ArrayElement,backends::{common::{- Backend, Context,+ Backend,gpu_types::QuantizationMethod,kernel::{Kernels, matmul::MatmulKernel},},@@ -55,7 +55,7 @@ fn bench_qwen3_layers_typed<T: ArrayElement + Float>(#[uzu_bench]fn bench_qwen3_layers(c: &mut Criterion) {- let context = MetalContext::new().expect("Metal context");+ let context = crate::common::shared_metal_context();bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleBias);bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleZeroPoint);}diff --git a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rsindex 1283c48b4..0a147e02d 100644--- a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs@@ -6,7 +6,7 @@ use backend_uzu::{Backend, Context, Encoder, Kernels, gpu_types::HadamardTransformOrder, kernel::HadamardTransformKernel,},};-use half::{bf16, f16};+use half::bf16;use num_traits::Float;use crate::uzu_test;@@ -154,11 +154,6 @@ fn test_f32() {test_hadamard_transform::<f32>(1e-4);}-#[uzu_test]-fn test_f16() {- test_hadamard_transform::<f16>(0.05);-}-#[uzu_test]fn test_bf16() {test_hadamard_transform::<bf16>(0.1);diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rsindex 8dedff722..d62f0fed8 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs@@ -89,3 +89,46 @@ fn b_transpose_false_bf16() {1.0,);}++fn rht_shapes() -> impl Iterator<Item = crate::common::matmul::Shape> {+ use crate::common::matmul::Shape;+ [Shape::new(8, 128, 64), Shape::new(64, 128, 128), Shape::new(128, 2048, 256), Shape::new(33, 128, 64)].into_iter()+}++#[test]+fn rht_parity_bf16() {+ let context = MetalContext::new().expect("Metal context");+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulKernel");+ for path in gemm_paths_for_hw(&context) {+ for shape in rht_shapes() {+ let case = Case::new(shape).with_rht(true);+ check_case::<bf16>(&context, &mut kernel, Some(path), case, 1.0);+ }+ }+}++#[test]+fn bias_parity_bf16() {+ use crate::common::matmul::Shape;+ let context = MetalContext::new().expect("Metal context");+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(+ &context,+ bf16::data_type(),+ bf16::data_type(),+ bf16::data_type(),+ )+ .expect("MatmulKernel");+ let shapes = [Shape::new(64, 128, 64), Shape::new(128, 2048, 128), Shape::new(33, 128, 64)];+ for path in gemm_paths_for_hw(&context) {+ for shape in shapes {+ let case = Case::new(shape).with_bias(true);+ check_case::<bf16>(&context, &mut kernel, Some(path), case, 1.0);+ }+ }+}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rsindex e53eeb816..0d1e0fbd6 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs@@ -227,3 +227,34 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {"got {matmul:?}");}++#[rstest]+#[case::gs32_4bit_mlx(128, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]+#[case::gs64_4bit_mlx(128, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]+#[case::gs128_4bit_mlx(128, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]+#[case::gs32_4bit_zp(128, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]+#[case::gs64_8bit_zp(128, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]+#[case::gs128_8bit_zp(128, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]+fn mxu_quant_parity_bf16(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+ #[case] gs: u32,+ #[case] bits: u32,+ #[case] method: QuantizationMethod,+) {+ let context = MetalContext::new().expect("Metal context");+ if !context.supports_mxu() {+ return;+ }+ let input = QuantInput::<bf16>::new(m, k, n, gs, bits, method, 0);+ let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmDispatchPath::Mxu));+ let reference = run_quant_cpu::<bf16>(&input);+ assert_parity::<bf16>(+ &format!("MXU m={m} k={k} n={n} gs={gs} bits={bits} method={method:?}"),+ &reference,+ &actual,+ 0.05,+ 0.5,+ );+}diff --git a/crates/backend-uzu/tests/unit/kernel/mod.rs b/crates/backend-uzu/tests/unit/kernel/mod.rsindex ef7419d7d..e4675a0c5 100644--- a/crates/backend-uzu/tests/unit/kernel/mod.rs+++ b/crates/backend-uzu/tests/unit/kernel/mod.rs@@ -17,7 +17,6 @@ mod moe;mod pooling;mod qk_unpack_test;mod qkv_norm_test;-mod quant_matmul;mod rms_norm_test;mod rope_test;mod sampling;diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rsdeleted file mode 100644index 243c6b7f7..000000000--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs+++ /dev/null@@ -1,98 +0,0 @@-#[macro_use]-#[path = "../../../common/mod.rs"]-mod common;--macro_rules! for_each_qmv_float_type {- (|$F:ident| $body:expr) => {{- {- type $F = f32;- $body- }- {- type $F = half::bf16;- $body- }- }};-}--mod qmv_fast_test;-mod qmv_test;--use backend_uzu::{array::ArrayElement, backends::common::gpu_types::QuantizationMethod};-use num_traits::Float;--pub(super) struct Input<T: ArrayElement + Float> {- pub w_packed: Vec<u32>,- pub scales: Vec<T>,- pub zero_points: Option<Vec<u8>>,- pub biases: Option<Vec<T>>,- pub x: Vec<T>,- pub k: u32,- pub n: u32,- pub m: u32,- pub group_size: u32,- pub bits: u32,- pub quant_method: QuantizationMethod,-}--pub(super) fn pack_weights_u32(- values: &[u8],- bits: u32,-) -> Vec<u32> {- if bits == 4 {- values- .chunks(8)- .map(|chunk| {- let mut word = 0u32;- for (i, &v) in chunk.iter().enumerate() {- word |= ((v & 0xF) as u32) << (i * 4);- }- word- })- .collect()- } else {- values- .chunks(4)- .map(|chunk| {- let mut word = 0u32;- for (i, &v) in chunk.iter().enumerate() {- word |= (v as u32) << (i * 8);- }- word- })- .collect()- }-}--pub(super) fn pack_zero_points(- values: &[u8],- bits: u32,-) -> Vec<u8> {- if bits == 4 {- values- .chunks(2)- .map(|chunk| {- let lo = chunk[0] & 0x0F;- let hi = if chunk.len() > 1 {- chunk[1] & 0x0F- } else {- 0- };- lo | (hi << 4)- })- .collect()- } else {- values.to_vec()- }-}--pub(super) fn check_tolerance(- expected: f32,- actual: f32,- rel_tol: f64,- abs_tol: f64,-) -> bool {- let diff = (expected - actual).abs() as f64;- let tol = abs_tol.max(expected.abs() as f64 * rel_tol);- diff <= tol-}diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsdeleted file mode 100644index 22d6b0944..000000000--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ /dev/null@@ -1,302 +0,0 @@-use std::fmt::{Debug, Display};--use backend_uzu::{- array::ArrayElement,- backends::{- common::{- Allocation, Backend, Context, Encoder, Kernels,- gpu_types::QuantizationMethod,- kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},- },- cpu::Cpu,- },- data_type::DataType,-};-use num_traits::Float;--use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};-use crate::{- common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},- uzu_test,-};--fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");-- let w_buf = alloc_allocation_with_data::<Cpu, u32>(&context, &input.w_packed);- let scales_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.scales);- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<Cpu, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<Cpu, T>(&context, b));- let x_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.x);- let mut y_buf = alloc_allocation::<Cpu, T>(&context, (input.m as usize) * (input.n as usize));-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");-- let kernel = <<Cpu as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(- &context,- T::data_type(),- T::data_type(),- T::data_type(),- input.group_size,- input.bits,- input.quant_method,- )- .expect("Failed to create QuantizedMatmulQmvKernel");- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- input.k,- input.n,- input.m,- &mut encoder,- );-- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- allocation_to_vec(&y_buf)-}--fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");-- let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));- let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);- let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");-- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(- &context,- T::data_type(),- T::data_type(),- T::data_type(),- input.group_size,- input.bits,- input.quant_method,- false,- )- .expect("Failed to create QuantizedMatmulQmvFastKernel");- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- None::<&Allocation<B>>,- input.k,- input.n,- input.m,- &mut encoder,- );-- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- allocation_to_vec(&y_buf)-}--fn get_test_data<T: ArrayElement + Float>(- m: usize,- k: usize,- n: usize,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) -> (Input<T>, Vec<T>) {- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;-- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);- let max_val = if bits == 4 {- 15u8- } else {- 255u8- };- for j in 0..n {- for l in 0..k {- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);- }- }- let w_packed = pack_weights_u32(&weights_raw, bits);-- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);- }- }- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = match quant_method {- QuantizationMethod::ScaleBias => {- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);- }- }- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();- (None, Some(biases))- },- QuantizationMethod::ScaleZeroPoint => {- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;- zp_raw.push(zp_val);- }- }- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);- for j in 0..n {- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];- let packed_row = pack_zero_points(row, bits);- let mut padded = packed_row;- padded.resize(zp_stride, 0);- zp_packed.extend_from_slice(&padded);- }- (Some(zp_packed), None)- },- QuantizationMethod::ScaleSymmetric => (None, None),- };-- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);- for i in 0..m {- for l in 0..k {- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);- }- }- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let input = Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- quant_method,- };-- let expected = get_expected::<T>(&input);- (input, expected)-}--fn test_internal<T: ArrayElement + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],-) {- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {- DataType::BF16 => (0.05, 0.5),- DataType::F16 => (0.02, 0.5),- _ => (0.01, 0.1),- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- assert_eq!(expected.len(), output.len(), "Output length mismatch");-- let mut errors = 0;- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {- let exp_f32 = exp.to_f32().unwrap();- let got_f32 = got.to_f32().unwrap();- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {- if errors < 5 {- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());- }- errors += 1;- }- }- assert_eq!(- errors,- 0,- "QMV fast kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, format={:?}: {} mismatches",- std::any::type_name::<B>(),- input.m,- input.k,- input.n,- input.group_size,- input.bits,- input.quant_method,- errors,- );- });-}--fn get_test_dims(- group_size: u32,- bits: u32,- is_half_precision: bool,-) -> Vec<(usize, usize, usize)> {- let gs = group_size as usize;- let block_size = if bits == 4 {- 512- } else {- 256- };- let k = block_size;- let n = gs.max(32);- if is_half_precision && bits == 8 {- vec![(1, k, n), (2, k, n)]- } else {- vec![(1, k, n), (2, k, n * 2), (1, k, n * 2)]- }-}--fn test_basic<T: ArrayElement + Float + Debug + Display>(- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);- let dims = get_test_dims(group_size, bits, is_half);-- for (m, k, n) in dims {- let (input, expected) = get_test_data::<T>(m, k, n, group_size, bits, quant_method);- test_internal::<T>(&input, &expected);- }-}--macro_rules! qmv_fast_test {- ($name:ident, gs=$gs:expr, bits=$bits:expr, format=$format:expr) => {- #[uzu_test]- fn $name() {- for_each_qmv_float_type!(|F| {- test_basic::<F>($gs, $bits, $format);- })- }- };-}--qmv_fast_test!(test_gs16_4bit_mlx, gs = 16, bits = 4, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs32_4bit_mlx, gs = 32, bits = 4, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs64_4bit_mlx, gs = 64, bits = 4, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs128_4bit_mlx, gs = 128, bits = 4, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs16_8bit_mlx, gs = 16, bits = 8, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, format = QuantizationMethod::ScaleBias);-qmv_fast_test!(test_gs16_4bit_zp, gs = 16, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs16_8bit_zp, gs = 16, bits = 8, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, format = QuantizationMethod::ScaleZeroPoint);-qmv_fast_test!(test_gs32_8bit_symmetric, gs = 32, bits = 8, format = QuantizationMethod::ScaleSymmetric);diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rsdeleted file mode 100644index 2eed46cb5..000000000--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs+++ /dev/null@@ -1,459 +0,0 @@-use std::fmt::{Debug, Display};--use backend_uzu::{- array::ArrayElement,- backends::{- common::{Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvKernel},- cpu::Cpu,- },- data_type::DataType,-};-use num_traits::Float;--use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};-use crate::{- common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},- uzu_test,-};--fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {- let context = B::Context::new().expect("Failed to create Context");-- let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);- let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);-- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));-- let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);- let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");-- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(- &context,- T::data_type(),- T::data_type(),- T::data_type(),- input.group_size,- input.bits,- input.quant_method,- )- .expect("Failed to create QuantizedMatmulQmvKernel");- kernel.encode(- &w_buf,- &scales_buf,- zp_buf.as_ref(),- bias_buf.as_ref(),- &x_buf,- &mut y_buf,- input.k,- input.n,- input.m,- &mut encoder,- );-- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- allocation_to_vec(&y_buf)-}--/// Create test data for a qmv scenario: m×k * k×n = m×n-/// Weights are in transposed [N × K] layout (each output row stores K input weights).-/// Scales: [N × num_groups_k], zero_points: [N × zp_stride_k], biases: [N × num_groups_k].-/// Groups are along the K dimension.-fn get_test_data_basic<T: ArrayElement + Float>(- m: usize,- k: usize,- n: usize,- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) -> (Input<T>, Vec<T>) {- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;-- // Generate quantized weights [N × K]- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);- let max_val = if bits == 4 {- 15u8- } else {- 255u8- };- for j in 0..n {- for l in 0..k {- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);- }- }- let w_packed = pack_weights_u32(&weights_raw, bits);-- // Scales: [N × num_groups_k]- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);- }- }- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- // Zero points or biases- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = match quant_method {- QuantizationMethod::ScaleBias => {- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);- }- }- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();- (None, Some(biases))- },- QuantizationMethod::ScaleZeroPoint => {- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);- for j in 0..n {- for g in 0..num_groups_k {- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;- zp_raw.push(zp_val);- }- }- // Pack zero points per row (row = output neuron)- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);- for j in 0..n {- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];- let packed_row = pack_zero_points(row, bits);- let mut padded = packed_row;- padded.resize(zp_stride, 0);- zp_packed.extend_from_slice(&padded);- }- (Some(zp_packed), None)- },- QuantizationMethod::ScaleSymmetric => (None, None),- };-- // X: [m × k]- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);- for i in 0..m {- for l in 0..k {- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);- }- }- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let input = Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- quant_method,- };-- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--/// Small edge-case test data with known values-fn get_test_data_edge<T: ArrayElement + Float>(- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) -> (Input<T>, Vec<T>) {- let m = 1usize;- let k = 4usize;- let n = 4usize;- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;-- // Simple weights [N × K]: all 1s- let weights_raw: Vec<u8> = vec![1u8; n * k];- let w_packed = pack_weights_u32(&weights_raw, bits);-- // Unit scales [N × num_groups_k]- let scales: Vec<T> = vec![T::one(); n * num_groups_k];-- let zp_stride = if bits == 4 {- (num_groups_k + 1) / 2- } else {- num_groups_k- };-- let (zero_points, biases) = match quant_method {- QuantizationMethod::ScaleBias => {- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];- (None, Some(biases))- },- QuantizationMethod::ScaleZeroPoint => {- // Zero points all 0 → bias = 0- let zp_packed = vec![0u8; n * zp_stride];- (Some(zp_packed), None)- },- QuantizationMethod::ScaleSymmetric => (None, None),- };-- // X: simple increasing values- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();-- let input = Input {- w_packed,- scales,- zero_points,- biases,- x,- k: k as u32,- n: n as u32,- m: m as u32,- group_size,- bits,- quant_method,- };-- let expected = get_output::<Cpu, T>(&input);- (input, expected)-}--fn test_internal<T: ArrayElement + Float + Debug + Display>(- input: &Input<T>,- expected: &[T],-) {- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {- DataType::BF16 => (0.05, 0.5),- DataType::F16 => (0.02, 0.5),- _ => (0.01, 0.1),- };-- for_each_non_cpu_backend!(|B| {- let output = get_output::<B, T>(input);- assert_eq!(expected.len(), output.len(), "Output length mismatch");-- let mut errors = 0;- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {- let exp_f32 = exp.to_f32().unwrap();- let got_f32 = got.to_f32().unwrap();- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {- if errors < 5 {- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());- }- errors += 1;- }- }- assert_eq!(- errors,- 0,- "QMV kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, format={:?}: {} mismatches",- std::any::type_name::<B>(),- input.m,- input.k,- input.n,- input.group_size,- input.bits,- input.quant_method,- errors,- );- });-}--fn get_test_dims(- group_size: u32,- bits: u32,- is_half_precision: bool,-) -> Vec<(usize, usize, usize)> {- let gs = group_size as usize;- // N must be >= group_size for the Metal tiled loader's stride calculations.- // For half-precision 8-bit, limit k to avoid catastrophic cancellation- // from large weight values (0-255) and ZP bias subtraction.- if is_half_precision && bits == 8 {- let k = gs.min(64);- vec![(1, k, gs), (2, k, gs)]- } else {- vec![(1, gs * 2, gs), (2, gs * 2, gs * 2), (1, gs, gs * 2)]- }-}--fn test_basic<T: ArrayElement + Float + Debug + Display>(- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);- let dims = get_test_dims(group_size, bits, is_half);-- for (m, k, n) in dims {- let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, quant_method);- test_internal::<T>(&input, &expected);- }-}--fn test_edge<T: ArrayElement + Float + Debug + Display>(- group_size: u32,- bits: u32,- quant_method: QuantizationMethod,-) {- let (input, expected) = get_test_data_edge::<T>(group_size, bits, quant_method);- test_internal::<T>(&input, &expected);-}--// ── 4-bit, mlx quant ───────────────────────────────────────────────────────-#[uzu_test]-fn test_gs16_4bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(16, 4, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_gs32_4bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(32, 4, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_gs64_4bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(64, 4, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_gs128_4bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(128, 4, QuantizationMethod::ScaleBias);- })-}--// ── 8-bit, mlx quant ───────────────────────────────────────────────────────--#[uzu_test]-fn test_gs16_8bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(16, 8, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_gs32_8bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(32, 8, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_gs64_8bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(64, 8, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_gs128_8bit_mlx() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(128, 8, QuantizationMethod::ScaleBias);- })-}--// ── 4-bit, zero points ─────────────────────────────────────────────────────--#[uzu_test]-fn test_gs16_4bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(16, 4, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_gs32_4bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_gs64_4bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(64, 4, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_gs128_4bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(128, 4, QuantizationMethod::ScaleZeroPoint);- })-}--// ── 8-bit, zero points ─────────────────────────────────────────────────────--#[uzu_test]-fn test_gs16_8bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(16, 8, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_gs32_8bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_gs64_8bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(64, 8, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_gs128_8bit_zp() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(128, 8, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_gs32_8bit_symmetric() {- for_each_qmv_float_type!(|F| {- test_basic::<F>(32, 8, QuantizationMethod::ScaleSymmetric);- })-}--// ── Edge cases ──────────────────────────────────────────────────────────────--#[uzu_test]-fn test_edge_4bit_mlx() {- for_each_qmv_float_type!(|F| {- test_edge::<F>(32, 4, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_edge_8bit_mlx() {- for_each_qmv_float_type!(|F| {- test_edge::<F>(32, 8, QuantizationMethod::ScaleBias);- })-}--#[uzu_test]-fn test_edge_4bit_zp() {- for_each_qmv_float_type!(|F| {- test_edge::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);- })-}--#[uzu_test]-fn test_edge_8bit_zp() {- for_each_qmv_float_type!(|F| {- test_edge::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);- })-}