Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

add qmm mxu

eugened32e99756973
diff --git a/.gitignore b/.gitignore
index d3bf51e56..1cedb0c35 100644
--- a/.gitignore
+++ b/.gitignore
@@ -34,3 +34,4 @@ tools/gpu_trace/*.json
.build/
.swiftpm/
Package.resolved
+.dinghy.toml
diff --git a/Cargo.lock b/Cargo.lock
index 624222622..e18d21558 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -364,9 +364,9 @@ dependencies = [
[[package]]
name = "autocfg"
-version = "1.5.0"
+version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
+checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
[[package]]
name = "autocxx"
@@ -395,7 +395,7 @@ dependencies = [
"quote",
"regex",
"rustc-hash 2.1.2",
- "shlex",
+ "shlex 1.3.0",
"syn 2.0.117",
]
@@ -517,9 +517,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.16.3"
+version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f"
+checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -527,9 +527,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.40.0"
+version = "0.41.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7"
+checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
dependencies = [
"cc",
"cmake",
@@ -544,7 +544,7 @@ dependencies = [
"async-openai",
"futures",
"indexmap 2.14.0",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
"serde",
"serde_json",
"shoji",
@@ -688,7 +688,7 @@ dependencies = [
"quote",
"regex",
"rustc-hash 2.1.2",
- "shlex",
+ "shlex 1.3.0",
"syn 2.0.117",
]
@@ -838,15 +838,15 @@ dependencies = [
[[package]]
name = "built"
-version = "0.8.0"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64"
+checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9"
[[package]]
name = "bumpalo"
-version = "3.20.2"
+version = "3.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
+checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
[[package]]
name = "bytecount"
@@ -947,9 +947,9 @@ dependencies = [
[[package]]
name = "cbindgen"
-version = "0.29.2"
+version = "0.29.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799"
+checksum = "c95537b45400390270fae69ac098d057c8f5399001cde9d04f700c105ddfff2d"
dependencies = [
"clap",
"heck",
@@ -966,14 +966,14 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.62"
+version = "1.2.63"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98"
+checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
dependencies = [
"find-msvc-tools",
"jobserver",
"libc",
- "shlex",
+ "shlex 2.0.1",
]
[[package]]
@@ -1230,9 +1230,9 @@ dependencies = [
[[package]]
name = "compact_str"
-version = "0.9.0"
+version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a"
+checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab"
dependencies = [
"castaway",
"cfg-if",
@@ -1582,9 +1582,9 @@ dependencies = [
[[package]]
name = "ctor"
-version = "0.11.1"
+version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "400a21f1014a968ec518c7ccdf9b4a4ed0cac8c56ccb6d604f8b91f00110501e"
+checksum = "01334b89b69ff726750c5ce5073fc8bd860e99aa9a8fc5ca11b04730e3aee97a"
[[package]]
name = "cxx"
@@ -1903,9 +1903,9 @@ dependencies = [
[[package]]
name = "displaydoc"
-version = "0.2.5"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
+checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f"
dependencies = [
"proc-macro2",
"quote",
@@ -1942,7 +1942,7 @@ dependencies = [
"objc-sys",
"objc2",
"objc2-foundation",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
"rstest",
"serde",
"serde_json",
@@ -1969,9 +1969,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "either"
-version = "1.15.0"
+version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e"
[[package]]
name = "email_address"
@@ -2372,9 +2372,9 @@ checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393"
[[package]]
name = "futures-timer"
-version = "3.0.3"
+version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24"
+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
[[package]]
name = "futures-util"
@@ -2673,9 +2673,9 @@ checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
[[package]]
name = "http"
-version = "1.4.0"
+version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a"
+checksum = "8be7462df143984c4598a256ef469b251d7d7f9e271135073e78fc535414f3d0"
dependencies = [
"bytes",
"itoa",
@@ -2724,9 +2724,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
[[package]]
name = "hyper"
-version = "1.9.0"
+version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
dependencies = [
"atomic-waker",
"bytes",
@@ -3092,9 +3092,9 @@ dependencies = [
[[package]]
name = "iocraft"
-version = "0.8.2"
+version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58c8c25269f0eae8c9791eed1519d04ca1ebcd7e11c8a10fc6c15091a4ed0019"
+checksum = "ec09d74eb53878be6a7bd49f0ecb0e68d7322963d0a4fbbade67465083f38dbd"
dependencies = [
"bitflags 2.11.1",
"crossterm",
@@ -3287,9 +3287,9 @@ dependencies = [
[[package]]
name = "js-sys"
-version = "0.3.98"
+version = "0.3.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08"
+checksum = "142bc4740e452c1e57ade0cbc129f139c9093e354346f0872ef985f4f5cf5f11"
dependencies = [
"cfg-if",
"futures-util",
@@ -3315,9 +3315,9 @@ dependencies = [
[[package]]
name = "jsonschema"
-version = "0.46.4"
+version = "0.46.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc59d2432e047d6090ba1d83c782d0128bd6203857978218f5614dbd3287281f"
+checksum = "6a5fe5206f06e589caf25e79fc05ccdf91fca745685fe9fe1a13bbdfb479a631"
dependencies = [
"ahash",
"bytecount",
@@ -3334,7 +3334,7 @@ dependencies = [
"referencing",
"regex",
"regex-syntax 0.8.10",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
"rustls",
"serde",
"serde_json",
@@ -3436,9 +3436,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
[[package]]
name = "libredox"
-version = "0.1.16"
+version = "0.1.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e02f3bb43d335493c96bf3fd3a321600bf6bd07ed34bc64118e9293bdffea46c"
+checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3"
dependencies = [
"libc",
]
@@ -3501,9 +3501,9 @@ dependencies = [
[[package]]
name = "log"
-version = "0.4.29"
+version = "0.4.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5"
[[package]]
name = "logos"
@@ -3635,9 +3635,9 @@ dependencies = [
[[package]]
name = "memchr"
-version = "2.8.0"
+version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+checksum = "6b947ae49db0d222b1dbc6b113ce7248a3fc3a6ca21b696717bfc000ba4484d8"
[[package]]
name = "memmap2"
@@ -3716,9 +3716,9 @@ dependencies = [
[[package]]
name = "minijinja"
-version = "2.19.0"
+version = "2.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "805bfd7352166bae857ee569628b52bcd85a1cecf7810861ebceb1686b72b75d"
+checksum = "2929e494b2280e1e18959bb2e121da03347ae896896fdfaceaab43c88a02803f"
dependencies = [
"indexmap 2.14.0",
"memo-map",
@@ -3728,9 +3728,9 @@ dependencies = [
[[package]]
name = "minijinja-contrib"
-version = "2.19.0"
+version = "2.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "45092d80391870622fcf3bd82f5d2af18f99533ea60debb4bc9db0c76f0e809a"
+checksum = "99df5123c54391e2a228014c1dbbd85a3dab08a25e776c810526f2f47542b3de"
dependencies = [
"minijinja",
"serde",
@@ -3754,9 +3754,9 @@ dependencies = [
[[package]]
name = "mio"
-version = "1.2.0"
+version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
+checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
dependencies = [
"libc",
"log",
@@ -3856,7 +3856,7 @@ dependencies = [
"pyo3-async-runtimes",
"pyo3-build-config",
"pyo3-stub-gen",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
"serde",
"serde-wasm-bindgen",
"serde_json",
@@ -3873,9 +3873,9 @@ dependencies = [
[[package]]
name = "napi"
-version = "3.8.6"
+version = "3.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e55037284865448ecf329baa86a4d05401f647ebde99f5747b640d32c2c5226"
+checksum = "f1d395473824516f38dd1071a1a37bc57daa7be65b293ebba4ead5f7abb017a2"
dependencies = [
"bitflags 2.11.1",
"ctor",
@@ -3890,15 +3890,15 @@ dependencies = [
[[package]]
name = "napi-build"
-version = "2.3.1"
+version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d376940fd5b723c6893cd1ee3f33abbfd86acb1cd1ec079f3ab04a2a3bc4d3b1"
+checksum = "c9c366d2c8c60b86fa632df75f745509b52f9128f91a6bad4c796e44abb505e1"
[[package]]
name = "napi-derive"
-version = "3.5.5"
+version = "3.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4ba740fe4c9524d86fd90798fd8ccdb23402b3eef7e7c30897a8a369b529fcf"
+checksum = "89b3f766e04667e6da0e181e2da4f85475d5a6513b7cf6a80bea184e224a5b42"
dependencies = [
"convert_case 0.11.0",
"ctor",
@@ -4109,9 +4109,9 @@ dependencies = [
[[package]]
name = "num-conv"
-version = "0.2.1"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6673768db2d862beb9b39a78fdcb1a69439615d5794a1be50caa9bc92c81967"
+checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441"
[[package]]
name = "num-derive"
@@ -4422,9 +4422,9 @@ dependencies = [
[[package]]
name = "openssl"
-version = "0.10.79"
+version = "0.10.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542"
+checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"
dependencies = [
"bitflags 2.11.1",
"cfg-if",
@@ -4453,9 +4453,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "openssl-sys"
-version = "0.9.115"
+version = "0.9.116"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "158fe5b292746440aa6e7a7e690e55aeb72d41505e2804c23c6973ad0e9c9781"
+checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4"
dependencies = [
"cc",
"libc",
@@ -4932,9 +4932,9 @@ dependencies = [
[[package]]
name = "pyo3-stub-gen"
-version = "0.22.2"
+version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b695173a33bec37b6acd288efe74f902f123f5891e5ff5ad53ff429e18833c14"
+checksum = "1242a6080f45edcfaf3be3b9968914dd47a408c5590aea22770b978f7c36ffa7"
dependencies = [
"anyhow",
"chrono",
@@ -4958,9 +4958,9 @@ dependencies = [
[[package]]
name = "pyo3-stub-gen-derive"
-version = "0.22.2"
+version = "0.22.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2f901834d55c74f36be3353994062e3c3b3d47fda25572f5535e99434612ac8"
+checksum = "fd9a73abe6f57890e657c265b86805637138f83203e1a56bc156078abf215e36"
dependencies = [
"heck",
"indexmap 2.14.0",
@@ -5383,9 +5383,9 @@ dependencies = [
[[package]]
name = "referencing"
-version = "0.46.4"
+version = "0.46.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb674900ca31acd75c4aaf63f48e43e719631c0539ea5a9e64163d1296bcb730"
+checksum = "69e4e17ef386c5383591d07623d3de49cbc601156e7582973e6db98d66a57de2"
dependencies = [
"ahash",
"fluent-uri",
@@ -5525,9 +5525,9 @@ dependencies = [
[[package]]
name = "reqwest"
-version = "0.13.3"
+version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0"
+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -5592,9 +5592,9 @@ checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4"
[[package]]
name = "rhai"
-version = "1.24.0"
+version = "1.25.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f9ef5dabe4c0b43d8f1187dc6beb67b53fe607fff7e30c5eb7f71b814b8c2c1"
+checksum = "dd4dd0f8c36625202a4ba553c416c19b719947cd2a31d1bda06126e4a5727daf"
dependencies = [
"ahash",
"bitflags 2.11.1",
@@ -5609,9 +5609,9 @@ dependencies = [
[[package]]
name = "rhai_codegen"
-version = "3.1.0"
+version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4322a2a4e8cf30771dd9f27f7f37ca9ac8fe812dddd811096a98483080dabe6"
+checksum = "3cd3a7535e50bf36857e7be7bec276d334e8c2dfa469c2201226fd01638ea5ca"
dependencies = [
"proc-macro2",
"quote",
@@ -6050,9 +6050,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.149"
+version = "1.0.150"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
+checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9"
dependencies = [
"indexmap 2.14.0",
"itoa",
@@ -6158,6 +6158,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+[[package]]
+name = "shlex"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
+
[[package]]
name = "shoji"
version = "0.5.3"
@@ -6294,9 +6300,9 @@ checksum = "b7c388c1b5e93756d0c740965c41e8822f866621d41acbdf6336a6a168f8840c"
[[package]]
name = "socket2"
-version = "0.6.3"
+version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
+checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51"
dependencies = [
"libc",
"windows-sys 0.61.2",
@@ -6947,7 +6953,7 @@ dependencies = [
"toml_datetime 1.1.1+spec-1.1.0",
"toml_parser",
"toml_writer",
- "winnow 1.0.2",
+ "winnow 1.0.3",
]
[[package]]
@@ -6970,15 +6976,15 @@ dependencies = [
[[package]]
name = "toml_edit"
-version = "0.25.11+spec-1.1.0"
+version = "0.25.12+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b"
+checksum = "d2153edc6955a6c354fad8f5efd38b6a8769bdccf9fe50f8e1329f81b0baa5d7"
dependencies = [
"indexmap 2.14.0",
"toml_datetime 1.1.1+spec-1.1.0",
"toml_parser",
"toml_writer",
- "winnow 1.0.2",
+ "winnow 1.0.3",
]
[[package]]
@@ -6987,7 +6993,7 @@ version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
dependencies = [
- "winnow 1.0.2",
+ "winnow 1.0.3",
]
[[package]]
@@ -7013,9 +7019,9 @@ dependencies = [
[[package]]
name = "tower-http"
-version = "0.6.10"
+version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"
+checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
dependencies = [
"bitflags 2.11.1",
"bytes",
@@ -7161,9 +7167,9 @@ dependencies = [
[[package]]
name = "typenum"
-version = "1.20.0"
+version = "1.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
+checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
[[package]]
name = "ucd-trie"
@@ -7519,9 +7525,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.23.1"
+version = "1.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
+checksum = "d258b83ceec21034727ecee8c382cfa6c3e133699b0742c64571814fb420c9f7"
dependencies = [
"atomic",
"getrandom 0.4.2",
@@ -7578,7 +7584,7 @@ dependencies = [
"pyo3-async-runtimes",
"pyo3-build-config",
"pyo3-stub-gen",
- "reqwest 0.13.3",
+ "reqwest 0.13.4",
"rodio",
"rstest",
"schemars 1.2.1",
@@ -7701,9 +7707,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
-version = "0.2.121"
+version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790"
+checksum = "3ed04576f974d2b2fba0f38c51dbc5518011e38c36bf1143164be765528fd409"
dependencies = [
"cfg-if",
"once_cell",
@@ -7716,9 +7722,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.71"
+version = "0.4.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96492d0d3ffba25305a7dc88720d250b1401d7edca02cc3bcd50633b424673b8"
+checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -7726,9 +7732,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.121"
+version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578"
+checksum = "916151b09da36bd82f6615cbf3a419e2f0ba23a03c6160e8e92eb6bd4aa1dec6"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -7736,9 +7742,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.121"
+version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2"
+checksum = "299047362ccbfce148b67ab7e73349f77748e00c8296f9542adfad2ad82c5c5e"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -7749,9 +7755,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.121"
+version = "0.2.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441"
+checksum = "9a929b2c61f11ba3e9bc35b50c1f25cb38e0e892c0c231ae2b8cf78d5dad4437"
dependencies = [
"unicode-ident",
]
@@ -7818,9 +7824,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.98"
+version = "0.3.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
+checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -8335,9 +8341,9 @@ dependencies = [
[[package]]
name = "winnow"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0"
+checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1"
dependencies = [
"memchr",
]
@@ -8518,18 +8524,18 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.8.48"
+version = "0.8.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
+checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.48"
+version = "0.8.50"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
+checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
dependencies = [
"proc-macro2",
"quote",
@@ -8538,9 +8544,9 @@ dependencies = [
[[package]]
name = "zerofrom"
-version = "0.1.7"
+version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
+checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
dependencies = [
"zerofrom-derive",
]
diff --git a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
index f71a9c7e8..57d6b6007 100644
--- a/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
+++ b/crates/backend-uzu/src/backends/common/gpu_types/gemm.rs
@@ -3,7 +3,7 @@ use derive_more::Display;
#[repr(C)]
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum GemmWeightPrologueKind {
+pub enum GemmBPrologueKind {
FullPrecision,
ScaleBiasDequant,
ScaleZeroPointDequant,
@@ -23,80 +23,117 @@ bitflags! {
#[repr(C)]
#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, Hash)]
+#[allow(non_camel_case_types)]
pub enum GemmTiling {
- T8x32x32_1x1,
- T64x32x32_2x2,
- T64x64x16_2x2,
- T64x64x32_2x2,
- T32x32x32_2x2,
- T32x64x32_2x2,
- T64x32x32_4x1,
- T128x128x32_4x4,
+ Tile8x32x32_Simdgroups1x1,
+ Tile64x32x32_Simdgroups2x2,
+ Tile64x64x16_Simdgroups2x2,
+ Tile64x64x32_Simdgroups2x2,
+ Tile32x32x32_Simdgroups2x2,
+ Tile32x64x256_Simdgroups2x2,
+ Tile64x32x256_Simdgroups4x1,
+ Tile64x64x256_Simdgroups2x2,
+ Tile128x128x256_Simdgroups4x4,
}
+const MXU_SIMDGROUP_BLOCK_K: u32 = 32;
+
impl GemmTiling {
pub const fn block_m(self) -> u32 {
match self {
- Self::T8x32x32_1x1 => 8,
- Self::T64x32x32_2x2 => 64,
- Self::T64x64x16_2x2 => 64,
- Self::T64x64x32_2x2 => 64,
- Self::T32x32x32_2x2 => 32,
- Self::T32x64x32_2x2 => 32,
- Self::T64x32x32_4x1 => 64,
- Self::T128x128x32_4x4 => 128,
+ Self::Tile8x32x32_Simdgroups1x1 => 8,
+ Self::Tile64x32x32_Simdgroups2x2 => 64,
+ Self::Tile64x64x16_Simdgroups2x2 => 64,
+ Self::Tile64x64x32_Simdgroups2x2 => 64,
+ Self::Tile32x32x32_Simdgroups2x2 => 32,
+ Self::Tile32x64x256_Simdgroups2x2 => 32,
+ Self::Tile64x32x256_Simdgroups4x1 => 64,
+ Self::Tile64x64x256_Simdgroups2x2 => 64,
+ Self::Tile128x128x256_Simdgroups4x4 => 128,
}
}
pub const fn block_n(self) -> u32 {
match self {
- Self::T8x32x32_1x1 => 32,
- Self::T64x32x32_2x2 => 32,
- Self::T64x64x16_2x2 => 64,
- Self::T64x64x32_2x2 => 64,
- Self::T32x32x32_2x2 => 32,
- Self::T32x64x32_2x2 => 64,
- Self::T64x32x32_4x1 => 32,
- Self::T128x128x32_4x4 => 128,
+ Self::Tile8x32x32_Simdgroups1x1 => 32,
+ Self::Tile64x32x32_Simdgroups2x2 => 32,
+ Self::Tile64x64x16_Simdgroups2x2 => 64,
+ Self::Tile64x64x32_Simdgroups2x2 => 64,
+ Self::Tile32x32x32_Simdgroups2x2 => 32,
+ Self::Tile32x64x256_Simdgroups2x2 => 64,
+ Self::Tile64x32x256_Simdgroups4x1 => 32,
+ Self::Tile64x64x256_Simdgroups2x2 => 64,
+ Self::Tile128x128x256_Simdgroups4x4 => 128,
}
}
pub const fn block_k(self) -> u32 {
match self {
- Self::T8x32x32_1x1 => 32,
- Self::T64x32x32_2x2 => 32,
- Self::T64x64x16_2x2 => 16,
- Self::T64x64x32_2x2 => 32,
- Self::T32x32x32_2x2 => 32,
- Self::T32x64x32_2x2 => 32,
- Self::T64x32x32_4x1 => 32,
- Self::T128x128x32_4x4 => 32,
+ Self::Tile8x32x32_Simdgroups1x1 => 32,
+ Self::Tile64x32x32_Simdgroups2x2 => 32,
+ Self::Tile64x64x16_Simdgroups2x2 => 16,
+ Self::Tile64x64x32_Simdgroups2x2 => 32,
+ Self::Tile32x32x32_Simdgroups2x2 => 32,
+ Self::Tile32x64x256_Simdgroups2x2 => 256,
+ Self::Tile64x32x256_Simdgroups4x1 => 256,
+ Self::Tile64x64x256_Simdgroups2x2 => 256,
+ Self::Tile128x128x256_Simdgroups4x4 => 256,
}
}
pub const fn simdgroups_m(self) -> u32 {
match self {
- Self::T8x32x32_1x1 => 1,
- Self::T64x32x32_2x2 => 2,
- Self::T64x64x16_2x2 => 2,
- Self::T64x64x32_2x2 => 2,
- Self::T32x32x32_2x2 => 2,
- Self::T32x64x32_2x2 => 2,
- Self::T64x32x32_4x1 => 4,
- Self::T128x128x32_4x4 => 4,
+ Self::Tile8x32x32_Simdgroups1x1 => 1,
+ Self::Tile64x32x32_Simdgroups2x2 => 2,
+ Self::Tile64x64x16_Simdgroups2x2 => 2,
+ Self::Tile64x64x32_Simdgroups2x2 => 2,
+ Self::Tile32x32x32_Simdgroups2x2 => 2,
+ Self::Tile32x64x256_Simdgroups2x2 => 2,
+ Self::Tile64x32x256_Simdgroups4x1 => 4,
+ Self::Tile64x64x256_Simdgroups2x2 => 2,
+ Self::Tile128x128x256_Simdgroups4x4 => 4,
}
}
pub const fn simdgroups_n(self) -> u32 {
match self {
- Self::T8x32x32_1x1 => 1,
- Self::T64x32x32_2x2 => 2,
- Self::T64x64x16_2x2 => 2,
- Self::T64x64x32_2x2 => 2,
- Self::T32x32x32_2x2 => 2,
- Self::T32x64x32_2x2 => 2,
- Self::T64x32x32_4x1 => 1,
- Self::T128x128x32_4x4 => 4,
+ Self::Tile8x32x32_Simdgroups1x1 => 1,
+ Self::Tile64x32x32_Simdgroups2x2 => 2,
+ Self::Tile64x64x16_Simdgroups2x2 => 2,
+ Self::Tile64x64x32_Simdgroups2x2 => 2,
+ Self::Tile32x32x32_Simdgroups2x2 => 2,
+ Self::Tile32x64x256_Simdgroups2x2 => 2,
+ Self::Tile64x32x256_Simdgroups4x1 => 1,
+ Self::Tile64x64x256_Simdgroups2x2 => 2,
+ Self::Tile128x128x256_Simdgroups4x4 => 4,
+ }
+ }
+
+ pub const fn is_mxu_variant(self) -> bool {
+ matches!(
+ self,
+ Self::Tile32x64x256_Simdgroups2x2
+ | Self::Tile64x32x256_Simdgroups4x1
+ | Self::Tile64x64x256_Simdgroups2x2
+ | Self::Tile128x128x256_Simdgroups4x4
+ )
+ }
+
+ pub const fn simdgroup_block_k(self) -> u32 {
+ if self.is_mxu_variant() {
+ MXU_SIMDGROUP_BLOCK_K
+ } else {
+ self.block_k()
+ }
+ }
+
+ pub const fn fits_quant_group_size(
+ self,
+ group_size: u32,
+ ) -> bool {
+ match self {
+ Self::Tile128x128x256_Simdgroups4x4 => group_size <= 64,
+ _ => true,
}
}
}
diff --git a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
index fa61fb877..6e9173281 100644
--- a/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
+++ b/crates/backend-uzu/src/backends/common/kernel/matmul/matmul_b.rs
@@ -1,7 +1,7 @@
use crate::{
backends::common::{
Allocation, AsBufferRangeRef, Backend,
- gpu_types::{QuantizationMode, gemm::GemmWeightPrologueKind},
+ gpu_types::{QuantizationMode, gemm::GemmBPrologueKind},
},
data_type::DataType,
};
@@ -33,24 +33,24 @@ pub enum MatmulB<'a, B: Backend, TB: AsBufferRangeRef = Allocation<B>> {
}
impl<B: Backend, TB: AsBufferRangeRef> MatmulB<'_, B, TB> {
- pub fn weight_prologue(&self) -> GemmWeightPrologueKind {
+ pub fn b_prologue(&self) -> GemmBPrologueKind {
match self {
Self::FullPrecision {
..
- } => GemmWeightPrologueKind::FullPrecision,
+ } => GemmBPrologueKind::FullPrecision,
Self::ScaleBiasDequant {
..
- } => GemmWeightPrologueKind::ScaleBiasDequant,
+ } => GemmBPrologueKind::ScaleBiasDequant,
Self::ScaleZeroPointDequant {
..
- } => GemmWeightPrologueKind::ScaleZeroPointDequant,
+ } => GemmBPrologueKind::ScaleZeroPointDequant,
Self::ScaleSymmetricDequant {
..
- } => GemmWeightPrologueKind::ScaleSymmetricDequant,
+ } => GemmBPrologueKind::ScaleSymmetricDequant,
}
}
- pub fn bits_per_weight(&self) -> Option<u32> {
+ pub fn bits_per_b(&self) -> Option<u32> {
match self {
Self::FullPrecision {
..
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
index 0dd4b332b..87213c086 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/hadamard_transform/hadamard_transform.rs
@@ -1,19 +1,66 @@
-use half::{bf16, f16};
-use num_traits::Float;
+use half::bf16;
+use num_traits::{Float, NumCast};
use proc_macros::kernel;
use crate::{array::ArrayElement, backends::common::gpu_types::HadamardTransformOrder};
+const SIMD_SIZE: usize = 32;
+
+fn hadamard_transform(values: &mut [f32; SIMD_SIZE]) {
+ let mut stride = 1;
+ while stride < SIMD_SIZE {
+ for lane in 0..SIMD_SIZE {
+ if lane & stride == 0 {
+ let a = values[lane];
+ let b = values[lane | stride];
+ values[lane] = a + b;
+ values[lane | stride] = a - b;
+ }
+ }
+ stride <<= 1;
+ }
+ let scale = 1.0 / (SIMD_SIZE as f32).sqrt();
+ for v in values.iter_mut() {
+ *v *= scale;
+ }
+}
+
#[kernel(HadamardTransform)]
-#[variants(T, f32, f16, bf16)]
+#[variants(T, f32, bf16)]
pub fn hadamard_transform_mul<T: ArrayElement + Float>(
- #[allow(unused)] data: *mut T,
- #[allow(unused)] factors: *const i32,
- #[allow(unused)] hidden_dim: u32,
- #[allow(unused)] batch_size: u32,
- #[allow(unused)]
- #[specialize]
- transform_order: HadamardTransformOrder,
+ data: *mut T,
+ factors: *const i32,
+ hidden_dim: u32,
+ batch_size: u32,
+ #[specialize] transform_order: HadamardTransformOrder,
) {
- todo!()
+ let hidden_dim = hidden_dim as usize;
+ let batch_size = batch_size as usize;
+ assert_eq!(hidden_dim % SIMD_SIZE, 0, "hidden_dim must be a multiple of {SIMD_SIZE}");
+
+ for batch in 0..batch_size {
+ let row_offset = batch * hidden_dim;
+ for stripe_start in (0..hidden_dim).step_by(SIMD_SIZE) {
+ let mut stripe = [0.0f32; SIMD_SIZE];
+ for lane in 0..SIMD_SIZE {
+ let v: f32 = NumCast::from(unsafe { *data.add(row_offset + stripe_start + lane) }).unwrap();
+ let f = unsafe { *factors.add(stripe_start + lane) } as f32;
+ stripe[lane] = match transform_order {
+ HadamardTransformOrder::Input => v * f,
+ HadamardTransformOrder::Output => v,
+ };
+ }
+
+ hadamard_transform(&mut stripe);
+
+ for lane in 0..SIMD_SIZE {
+ let f = unsafe { *factors.add(stripe_start + lane) } as f32;
+ let result = match transform_order {
+ HadamardTransformOrder::Input => stripe[lane],
+ HadamardTransformOrder::Output => stripe[lane] * f,
+ };
+ unsafe { *data.add(row_offset + stripe_start + lane) = <T as NumCast>::from(result).unwrap() };
+ }
+ }
+ }
}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs
deleted file mode 100644
index a20050452..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/mod.rs
+++ /dev/null
@@ -1,2 +0,0 @@
-pub mod qmm_transposed;
-pub mod shaders;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
deleted file mode 100644
index 4741e74a1..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/qmm_transposed.rs
+++ /dev/null
@@ -1,82 +0,0 @@
-use num_traits::Float;
-
-use crate::{array::ArrayElement, backends::common::gpu_types::QuantizationMethod};
-
-pub fn qmm_transposed<T: ArrayElement + Float>(
- weights: *const u32,
- scales: *const T,
- zero_points: Option<*const u8>,
- biases: Option<*const T>,
- input: *const T,
- output: *mut T,
- in_vec_size: usize,
- out_vec_size: usize,
- batch_size: usize,
- quant_method: QuantizationMethod,
- group_size: usize,
- bits: usize,
-) {
- let num_groups_k = in_vec_size.div_ceil(group_size);
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- unsafe {
- for i in 0..batch_size {
- for j in 0..out_vec_size {
- let mut acc = 0.0f32;
-
- for l in 0..in_vec_size {
- let group_idx = l / group_size;
-
- let weight_linear_idx = j * in_vec_size + l;
-
- let val_q = if bits == 4 {
- let u32_idx = weight_linear_idx / 8;
- let bit_offset = (weight_linear_idx % 8) * 4;
- ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32
- } else {
- let u32_idx = weight_linear_idx / 4;
- let byte_offset = (weight_linear_idx % 4) * 8;
- ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32
- };
-
- let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();
- let scale_t = *scales.add(j * num_groups_k + group_idx);
-
- let w_dequant_f32 = match quant_method {
- QuantizationMethod::ScaleBias => {
- let bias_t = *biases.unwrap().add(j * num_groups_k + group_idx);
- scale_t.to_f32().unwrap() * val_q + bias_t.to_f32().unwrap()
- },
- QuantizationMethod::ScaleZeroPoint => {
- let zp = zero_points.unwrap();
- let zp_val = if bits == 4 {
- let byte_index = j * zp_stride + (group_idx >> 1);
- let byte_val = *zp.add(byte_index);
- if (group_idx & 1) == 0 {
- (byte_val & 0x0F) as f32
- } else {
- ((byte_val >> 4) & 0x0F) as f32
- }
- } else {
- *zp.add(j * zp_stride + group_idx) as f32
- };
- scale_t.to_f32().unwrap() * (val_q - zp_val)
- },
- QuantizationMethod::ScaleSymmetric => {
- let midpoint = (1 << (bits - 1)) as f32;
- scale_t.to_f32().unwrap() * (val_q - midpoint)
- },
- };
-
- acc += val_a * w_dequant_f32;
- }
-
- *output.add(i * out_vec_size + j) = T::from(acc).unwrap();
- }
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs
deleted file mode 100644
index c55461396..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/mod.rs
+++ /dev/null
@@ -1 +0,0 @@
-pub mod steel_gemm;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
deleted file mode 100644
index 472476e88..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemm/shaders/steel_gemm.rs
+++ /dev/null
@@ -1,100 +0,0 @@
-use num_traits::Float;
-
-use crate::{
- array::ArrayElement,
- utils::pointers::{SendPtr, SendPtrMut},
-};
-
-#[inline(always)]
-unsafe fn matmul_gemm_compute_row<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(
- a: SendPtr<TA>,
- b: SendPtr<TB>,
- ab_scale: f32,
- d: SendPtrMut<TD>,
- row: usize,
- n: usize,
- k: usize,
- leading_dimension_a: usize,
- leading_dimension_b: usize,
- leading_dimension_d: usize,
- b_transpose: bool,
- is_accumulate: bool,
-) {
- unsafe {
- for col in 0..n {
- let mut acc: f32 = 0.0;
- for i in 0..k {
- let a_val = *a.0.add(row * leading_dimension_a + i);
- let b_val = if b_transpose {
- *b.0.add(col * leading_dimension_b + i)
- } else {
- *b.0.add(i * leading_dimension_b + col)
- };
- acc = acc + a_val.to_f32().unwrap() * b_val.to_f32().unwrap();
- }
- let d_element = d.0.add(row * leading_dimension_d + col);
- acc *= ab_scale;
- if is_accumulate {
- acc += (*d_element).to_f32().unwrap();
- }
- *d_element = TD::from(acc).unwrap();
- }
- }
-}
-
-pub fn matmul_gemm_impl<TA: ArrayElement + Float, TB: ArrayElement + Float, TD: ArrayElement + Float>(
- a: *const TA,
- b: *const TB,
- ab_scale: f32,
- d: *mut TD,
- m: usize,
- n: usize,
- k: usize,
- leading_dimension_a: usize,
- leading_dimension_b: usize,
- leading_dimension_d: usize,
- b_transpose: bool,
- is_accumulate: bool,
-) {
- let num_threads = std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1);
- let a = SendPtr(a);
- let b = SendPtr(b);
- let d = SendPtrMut(d);
-
- std::thread::scope(|s| {
- let rows_per_thread = m / num_threads;
- let rows_per_thread_remainder = m % num_threads;
- let mut start = 0;
-
- for t in 0..num_threads {
- let chunk_add = if t < rows_per_thread_remainder {
- 1
- } else {
- 0
- };
- let chunk = rows_per_thread + chunk_add;
- let end = start + chunk;
-
- s.spawn(move || unsafe {
- for row in start..end {
- matmul_gemm_compute_row(
- a,
- b,
- ab_scale,
- d,
- row,
- n,
- k,
- leading_dimension_a,
- leading_dimension_b,
- leading_dimension_d,
- b_transpose,
- is_accumulate,
- );
- }
- });
-
- start = end;
- }
- });
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs
deleted file mode 100644
index e4e00c1e2..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/mod.rs
+++ /dev/null
@@ -1,2 +0,0 @@
-pub mod qmv;
-pub mod qmv_fast;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
deleted file mode 100644
index a0fa1d2e8..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv.rs
+++ /dev/null
@@ -1,141 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use proc_macros::kernel;
-
-use crate::{array::ArrayElement, backends::common::gpu_types::QuantizationMethod};
-
-/// CPU reference implementation of quantized matrix-vector multiply.
-///
-/// Weight layout is transposed compared to qmm:
-/// weights: [N, K], scales: [N, num_groups_k], zero_points: [N, zp_stride_k]
-/// where groups are along the K dimension.
-///
-/// Computes y[i, j] = sum_l( x[i, l] * dequant(w[j, l]) ) for each batch i and output j.
-pub fn qmv<WeightT: ArrayElement + Float, InputT: ArrayElement + Float, OutputT: ArrayElement + Float>(
- weights: *const u32,
- scales: *const WeightT,
- zero_points: Option<*const u8>,
- biases: Option<*const WeightT>,
- input: *const InputT,
- output: *mut OutputT,
- in_vec_size: usize,
- out_vec_size: usize,
- batch_size: usize,
- quant_method: QuantizationMethod,
- group_size: usize,
- bits: usize,
-) {
- let num_groups_k = in_vec_size.div_ceil(group_size);
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- // Number of weight elements packed per u32
- let pack_factor = if bits == 4 {
- 8
- } else {
- 4
- };
-
- unsafe {
- for i in 0..batch_size {
- for j in 0..out_vec_size {
- let mut acc = 0.0f32;
-
- for l in 0..in_vec_size {
- // Weights are [N, K]: row j, column l
- let weight_linear_idx = j * in_vec_size + l;
-
- let val_q = if bits == 4 {
- let u32_idx = weight_linear_idx / pack_factor;
- let bit_offset = (weight_linear_idx % pack_factor) * 4;
- ((weights.add(u32_idx).read_unaligned() >> bit_offset) & 0xF) as f32
- } else {
- let u32_idx = weight_linear_idx / pack_factor;
- let byte_offset = (weight_linear_idx % pack_factor) * 8;
- ((weights.add(u32_idx).read_unaligned() >> byte_offset) & 0xFF) as f32
- };
-
- let val_a = (*input.add(i * in_vec_size + l)).to_f32().unwrap();
-
- // Scales are [N, num_groups_k]: groups along K
- let group_idx = l / group_size;
- let scale = (*scales.add(j * num_groups_k + group_idx)).to_f32().unwrap();
-
- let bias = match quant_method {
- QuantizationMethod::ScaleBias => {
- // Biases are [N, num_groups_k]
- (*biases.unwrap().add(j * num_groups_k + group_idx)).to_f32().unwrap()
- },
- QuantizationMethod::ScaleZeroPoint => {
- let zp = zero_points.unwrap();
- // Zero points are [N, zp_stride_k]
- let zp_val = if bits == 4 {
- let byte_index = j * zp_stride + (group_idx >> 1);
- let byte_val = *zp.add(byte_index);
- if (group_idx & 1) == 0 {
- (byte_val & 0x0F) as f32
- } else {
- ((byte_val >> 4) & 0x0F) as f32
- }
- } else {
- *zp.add(j * zp_stride + group_idx) as f32
- };
- -scale * zp_val
- },
- QuantizationMethod::ScaleSymmetric => {
- let midpoint = 1 << (bits - 1);
- -scale * midpoint as f32
- },
- };
-
- acc += val_a * (scale * val_q + bias);
- }
-
- *output.add(i * out_vec_size + j) = OutputT::from(acc).unwrap();
- }
- }
- }
-}
-
-#[kernel(QuantizedMatmulQmv)]
-#[variants(WeightT, f32, bf16)]
-#[variants(InputT, f32, bf16)]
-#[variants(OutputT, f32, bf16)]
-#[variants(GROUP_SIZE, 16, 32, 64, 128)]
-#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmv<
- WeightT: ArrayElement + Float,
- InputT: ArrayElement + Float,
- OutputT: ArrayElement + Float,
- const GROUP_SIZE: u32,
- const BITS: u32,
->(
- weights: *const u32,
- scales: *const WeightT,
- #[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,
- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,
- input: *const InputT,
- output: *mut OutputT,
- in_vec_size: u32,
- out_vec_size: u32,
- batch_size: u32,
- #[specialize] quant_method: QuantizationMethod,
-) {
- qmv::<WeightT, InputT, OutputT>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- in_vec_size as usize,
- out_vec_size as usize,
- batch_size as usize,
- quant_method,
- GROUP_SIZE as usize,
- BITS as usize,
- );
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
deleted file mode 100644
index 93f004b8b..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/gemv/qmv_fast.rs
+++ /dev/null
@@ -1,55 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-use proc_macros::kernel;
-
-use crate::{
- array::ArrayElement,
- backends::{common::gpu_types::QuantizationMethod, cpu::kernel::matmul::gemv::qmv::qmv},
-};
-
-#[kernel(QuantizedMatmulQmvFast)]
-#[variants(WeightT, f32, bf16)]
-#[variants(InputT, f32, bf16)]
-#[variants(OutputT, f32, bf16)]
-#[variants(GROUP_SIZE, 16, 32, 64, 128)]
-#[variants(BITS, 4, 8)]
-pub fn quantized_matmul_qmv_fast<
- WeightT: ArrayElement + Float,
- InputT: ArrayElement + Float,
- OutputT: ArrayElement + Float,
- const GROUP_SIZE: u32,
- const BITS: u32,
->(
- weights: *const u32,
- scales: *const WeightT,
- #[optional(quant_method == QuantizationMethod::ScaleZeroPoint)] zero_points: Option<*const u8>,
- #[optional(quant_method == QuantizationMethod::ScaleBias)] biases: Option<*const WeightT>,
- input: *const InputT,
- output: *mut OutputT,
- #[optional(use_hadamard)] hadamard_factors: Option<*const i32>,
- in_vec_size: u32,
- out_vec_size: u32,
- batch_size: u32,
- #[specialize] quant_method: QuantizationMethod,
- #[specialize] use_hadamard: bool,
-) {
- assert_eq!(hadamard_factors.is_some(), use_hadamard);
-
- if use_hadamard {
- unimplemented!("not supported yet");
- }
- qmv::<WeightT, InputT, OutputT>(
- weights,
- scales,
- zero_points,
- biases,
- input,
- output,
- in_vec_size as usize,
- out_vec_size as usize,
- batch_size as usize,
- quant_method,
- GROUP_SIZE as usize,
- BITS as usize,
- );
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
new file mode 100644
index 000000000..23a351825
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/kernel.rs
@@ -0,0 +1,212 @@
+use super::reference::{WeightData, read_f32, write_f32};
+use crate::{
+ backends::{
+ common::{
+ AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
+ gpu_types::HadamardTransformOrder,
+ kernel::{
+ HadamardTransformKernel,
+ matmul::{MatmulArguments, MatmulError, MatmulKernel},
+ },
+ },
+ cpu::{Cpu, context::CpuContext, error::CpuError},
+ },
+ data_type::DataType,
+ utils::pointers::{SendPtr, SendPtrMut},
+};
+
+pub struct MatmulCpuKernel {
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,
+}
+
+impl MatmulKernel for MatmulCpuKernel {
+ type Backend = Cpu;
+
+ fn new(
+ context: &CpuContext,
+ weights_data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
+ ) -> Result<Self, CpuError> {
+ for data_type in [weights_data_type, input_data_type, output_data_type] {
+ if !matches!(data_type, DataType::F16 | DataType::BF16 | DataType::F32) {
+ return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
+ }
+ }
+ let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
+ context,
+ output_data_type,
+ HadamardTransformOrder::Output,
+ )?;
+ Ok(Self {
+ weights_data_type,
+ input_data_type,
+ output_data_type,
+ hadamard,
+ })
+ }
+
+ fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
+ &mut self,
+ arguments: MatmulArguments<Cpu, TB>,
+ encoder: &mut Encoder<Cpu>,
+ ) -> Result<(), CpuError> {
+ let output_scale = arguments.d_transform.ab_scale;
+ let accumulate = arguments.d_transform.accumulate;
+ let bias_alloc = arguments.d_transform.bias;
+ let post_rht = arguments.d_transform.rht_factors;
+
+ let MatmulArguments {
+ a,
+ a_offset,
+ b,
+ b_offset,
+ b_leading_dimension,
+ b_transpose,
+ d,
+ m,
+ n,
+ k,
+ ..
+ } = arguments;
+
+ let m_u = m as usize;
+ let n_u = n as usize;
+ let k_u = k as usize;
+ let weights_data_type = self.weights_data_type;
+ let input_data_type = self.input_data_type;
+ let output_data_type = self.output_data_type;
+
+ let a_buffer_range = a.as_buffer_range_ref();
+ let a_byte_off = a_buffer_range.range().start + a_offset * input_data_type.size_in_bytes();
+ let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));
+ let bias_ptr = bias_alloc.map(|bias| {
+ let r = bias.as_buffer_range_ref();
+ SendPtr(unsafe { &*r.buffer().get() }.as_ptr().wrapping_byte_add(r.range().start))
+ });
+ let d_buffer_range = d.as_buffer_range_mut();
+ let d_ptr = SendPtrMut(unsafe {
+ (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_buffer_range.range().start) as *mut u8
+ });
+
+ let weight_data =
+ WeightData::from_b(b, b_offset, b_leading_dimension, b_transpose, weights_data_type, k_u, n_u);
+
+ let command_buffer = encoder.as_command_buffer_mut();
+ command_buffer.push_command(move || {
+ let quant_layout = match &weight_data {
+ WeightData::Quantized {
+ bits,
+ group_size,
+ ..
+ } => {
+ let num_groups_k = k_u.div_ceil(*group_size);
+ let zero_point_stride = if *bits == 4 {
+ num_groups_k.div_ceil(2)
+ } else {
+ num_groups_k
+ };
+ let pack_factor = if *bits == 4 {
+ 8
+ } else {
+ 4
+ };
+ Some((num_groups_k, zero_point_stride, pack_factor))
+ },
+ WeightData::FullPrecision {
+ ..
+ } => None,
+ };
+
+ unsafe {
+ for row in 0..m_u {
+ for col in 0..n_u {
+ let mut accumulator = 0.0f32;
+ for inner in 0..k_u {
+ let a_value = read_f32(a_ptr.as_ptr(), input_data_type, row * k_u + inner);
+ let b_value = match &weight_data {
+ WeightData::FullPrecision {
+ ptr,
+ leading_dimension,
+ transpose,
+ } => {
+ let index = if *transpose {
+ col * leading_dimension + inner
+ } else {
+ inner * leading_dimension + col
+ };
+ read_f32(ptr.as_ptr(), weights_data_type, index)
+ },
+ WeightData::Quantized {
+ weights,
+ scales,
+ zero_points,
+ biases,
+ bits,
+ group_size,
+ } => {
+ let (num_groups_k, zero_point_stride, pack_factor) = quant_layout.unwrap();
+ let weight_linear_index = col * k_u + inner;
+ let quantized_value = if *bits == 4 {
+ let word_index = weight_linear_index / pack_factor;
+ let bit_offset = (weight_linear_index % pack_factor) * 4;
+ let w = weights.as_ptr() as *const u32;
+ ((w.add(word_index).read_unaligned() >> bit_offset) & 0xF) as f32
+ } else {
+ let word_index = weight_linear_index / pack_factor;
+ let bit_offset = (weight_linear_index % pack_factor) * 8;
+ let w = weights.as_ptr() as *const u32;
+ ((w.add(word_index).read_unaligned() >> bit_offset) & 0xFF) as f32
+ };
+ let group_index = inner / group_size;
+ let scale =
+ read_f32(scales.as_ptr(), weights_data_type, col * num_groups_k + group_index);
+ let bias_term = if let Some(zp) = zero_points {
+ let zero_point = if *bits == 4 {
+ let byte_index = col * zero_point_stride + (group_index >> 1);
+ let byte_value = *zp.as_ptr().add(byte_index);
+ if (group_index & 1) == 0 {
+ (byte_value & 0x0F) as f32
+ } else {
+ ((byte_value >> 4) & 0x0F) as f32
+ }
+ } else {
+ *zp.as_ptr().add(col * zero_point_stride + group_index) as f32
+ };
+ -scale * zero_point
+ } else if let Some(b) = biases {
+ read_f32(b.as_ptr(), weights_data_type, col * num_groups_k + group_index)
+ } else {
+ let midpoint = (1u32 << (bits - 1)) as f32;
+ -scale * midpoint
+ };
+ scale * quantized_value + bias_term
+ },
+ };
+ accumulator += a_value * b_value;
+ }
+
+ let output_index = row * n_u + col;
+ let mut value = output_scale * accumulator;
+ if accumulate {
+ value += read_f32(d_ptr.as_ptr(), output_data_type, output_index);
+ }
+ if let Some(bias) = bias_ptr {
+ value += read_f32(bias.as_ptr(), weights_data_type, col);
+ }
+ write_f32(d_ptr.as_ptr(), output_data_type, output_index, value);
+ }
+ }
+ }
+ });
+
+ if let Some(factors) = post_rht {
+ self.hadamard.encode(d, factors, n, m, encoder);
+ }
+
+ Ok(())
+ }
+}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
deleted file mode 100644
index 733c30139..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/matmul.rs
+++ /dev/null
@@ -1,406 +0,0 @@
-use crate::{
- backends::{
- common::{
- Allocation, AsBufferRangeMut, AsBufferRangeRef, Backend, Buffer, Encoder, Kernels,
- gpu_types::{HadamardTransformOrder, QuantizationMethod, QuantizationMode, gemm::GemmDTransform},
- kernel::{
- HadamardTransformKernel, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, TensorAddBiasKernel,
- matmul::{MatmulArguments, MatmulB, MatmulError, MatmulKernel},
- },
- },
- cpu::{
- Cpu, buffer::BufferDowncastExt, context::CpuContext, error::CpuError,
- kernel::matmul::quant::encode_quantized_gemm,
- },
- },
- data_type::DataType,
- dispatch_dtype,
- utils::pointers::{SendPtr, SendPtrMut},
-};
-
-pub struct MatmulCpuKernel {
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
- hadamard: <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel,
- bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,
-}
-
-impl MatmulKernel for MatmulCpuKernel {
- type Backend = Cpu;
-
- fn new(
- context: &CpuContext,
- weights_data_type: DataType,
- input_data_type: DataType,
- output_data_type: DataType,
- ) -> Result<Self, CpuError> {
- for data_type in [weights_data_type, input_data_type, output_data_type] {
- if !matches!(data_type, DataType::BF16 | DataType::F32) {
- return Err(MatmulError::<Cpu>::UnsupportedDataType(data_type).into());
- }
- }
- let hadamard = <<Cpu as Backend>::Kernels as Kernels>::HadamardTransformKernel::new(
- context,
- output_data_type,
- HadamardTransformOrder::Output,
- )?;
- let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(
- context,
- output_data_type,
- weights_data_type,
- true,
- )?;
- Ok(Self {
- weights_data_type,
- input_data_type,
- output_data_type,
- hadamard,
- bias_add,
- })
- }
-
- fn encode<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
- &mut self,
- arguments: MatmulArguments<Cpu, TB>,
- encoder: &mut Encoder<Cpu>,
- ) -> Result<(), CpuError> {
- match arguments.b {
- MatmulB::FullPrecision {
- ..
- } => Ok(self.encode_fp(arguments, encoder)?),
- MatmulB::ScaleBiasDequant {
- ..
- }
- | MatmulB::ScaleZeroPointDequant {
- ..
- }
- | MatmulB::ScaleSymmetricDequant {
- ..
- } => Ok(self.encode_quant(arguments, encoder)?),
- }
- }
-}
-
-impl MatmulCpuKernel {
- fn encode_fp<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
- &mut self,
- arguments: MatmulArguments<Cpu, TB>,
- encoder: &mut Encoder<Cpu>,
- ) -> Result<(), MatmulError<Cpu>> {
- if arguments.d_transform.mask().contains(GemmDTransform::RHT) {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::RHT,
- path: "MatmulCpuKernel",
- });
- }
-
- let ab_scale = arguments.d_transform.ab_scale;
- let bias_alloc = arguments.d_transform.bias;
- let is_accumulate = arguments.d_transform.accumulate;
-
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d,
- m,
- n,
- k,
- ..
- } = arguments;
- let weights = match b {
- MatmulB::FullPrecision {
- b: w,
- } => w,
- _ => unreachable!(),
- };
-
- let command_buffer = encoder.as_command_buffer_mut();
- let m_u = m as usize;
- let n_u = n as usize;
- let k_u = k as usize;
- let lda = k_u;
- let ldb = b_leading_dimension.map(|n| n as usize).unwrap_or(if b_transpose {
- k_u
- } else {
- n_u
- });
- let ldd = n_u;
- let weights_data_type = self.weights_data_type;
- let input_data_type = self.input_data_type;
- let output_data_type = self.output_data_type;
- let a_buffer_range = a.as_buffer_range_ref();
- let b_buffer_range = weights.as_buffer_range_ref();
- let d_buffer_range = d.as_buffer_range_mut();
- let a_byte_off = a_buffer_range.range().start + a_offset;
- let b_byte_off = b_buffer_range.range().start + b_offset;
- let d_byte_off = d_buffer_range.range().start;
-
- let a_ptr = SendPtr(unsafe { &*a_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));
- let b_ptr =
- SendPtr(unsafe { &*b_buffer_range.buffer().downcast().get() }.as_ptr().wrapping_byte_add(b_byte_off));
- let d_ptr =
- SendPtrMut(unsafe { (&*d_buffer_range.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });
-
- let bias_ptr = bias_alloc.map(|bias| {
- let bias_buffer_range = bias.as_buffer_range_ref();
- let bias_range = bias_buffer_range.range();
- SendPtr(unsafe { &*bias_buffer_range.buffer().get() }.as_ptr().wrapping_byte_add(bias_range.start))
- });
-
- command_buffer.push_command(move || {
- dispatch_dtype!(|(TW: weights_data_type, TA: input_data_type, TD: output_data_type)| {
- super::gemm::shaders::steel_gemm::matmul_gemm_impl::<TA, TW, TD>(
- a_ptr.as_ptr() as *const TA,
- b_ptr.as_ptr() as *const TW,
- ab_scale,
- d_ptr.as_ptr() as *mut TD,
- m_u,
- n_u,
- k_u,
- lda,
- ldb,
- ldd,
- b_transpose,
- is_accumulate,
- );
- if let Some(bias) = bias_ptr {
- assert_eq!(
- weights_data_type, output_data_type,
- "mixed precision matmul with bias is not supported until bias dtype ownership is settled",
- );
- apply_bias::<TD>(d_ptr.as_ptr() as *mut TD, bias.as_ptr() as *const TD, m_u, n_u);
- }
- })
- });
-
- Ok(())
- }
-
- fn encode_quant<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
- &mut self,
- arguments: MatmulArguments<Cpu, TB>,
- encoder: &mut Encoder<Cpu>,
- ) -> Result<(), MatmulError<Cpu>> {
- let d_mask = arguments.d_transform.mask();
- if d_mask.contains(GemmDTransform::SCALE) {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::SCALE,
- path: "MatmulCpuKernel/Quant",
- });
- }
- if d_mask.contains(GemmDTransform::ACCUMULATE) {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::ACCUMULATE,
- path: "MatmulCpuKernel/Quant",
- });
- }
- if !arguments.b_transpose || arguments.b_leading_dimension.is_some() {
- return Err(MatmulError::UnsupportedLayout {
- path: "MatmulCpuKernel/Quant",
- });
- }
-
- let group_size = match arguments.b {
- MatmulB::ScaleBiasDequant {
- group_size,
- ..
- }
- | MatmulB::ScaleZeroPointDequant {
- group_size,
- ..
- }
- | MatmulB::ScaleSymmetricDequant {
- group_size,
- ..
- } => group_size,
- MatmulB::FullPrecision {
- ..
- } => unreachable!(),
- };
- if !matches!(group_size, 16 | 32 | 64 | 128) {
- return Err(MatmulError::UnsupportedGroupSize(group_size as usize));
- }
-
- let post_rht = arguments.d_transform.rht_factors;
- let post_bias = arguments.d_transform.bias;
-
- if arguments.m >= 5 && arguments.n > 1 {
- assert_eq!(
- self.input_data_type, self.weights_data_type,
- "mixed precision quantized GEMM input dtype is not supported yet",
- );
- assert_eq!(
- self.output_data_type, self.weights_data_type,
- "mixed precision quantized GEMM output dtype is not supported yet",
- );
- assert!(
- !(post_bias.is_some() && post_rht.is_some()),
- "MatmulCpuKernel/Quant GEMM with both output bias and output RHT is not supported: bias must be applied after RHT",
- );
- let MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d,
- d_transform,
- m,
- n,
- k,
- } = arguments;
- let d_transform = d_transform.without(GemmDTransform::RHT | GemmDTransform::BIAS);
- encode_quantized_gemm(
- encoder,
- MatmulArguments {
- a,
- a_offset,
- b,
- b_offset,
- b_leading_dimension,
- b_transpose,
- d: &mut *d,
- d_transform,
- m,
- n,
- k,
- },
- self.weights_data_type,
- );
- if let Some(bias) = post_bias {
- self.bias_add.encode(None::<&Allocation<Cpu>>, bias, &mut *d, n, m * n, encoder);
- }
- if let Some(factors) = post_rht {
- self.hadamard.encode(d, factors, n, m, encoder);
- }
- return Ok(());
- }
-
- let hadamard_factors = post_rht;
-
- let MatmulArguments {
- a,
- a_offset,
- b,
- d,
- m,
- n,
- k,
- ..
- } = arguments;
- let (weights, scales, zp_or_bias, method, mode, group_size) = match b {
- MatmulB::ScaleBiasDequant {
- b: w,
- scales,
- biases,
- mode,
- group_size,
- } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),
- MatmulB::ScaleZeroPointDequant {
- b: w,
- scales,
- zero_points,
- mode,
- group_size,
- } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),
- MatmulB::ScaleSymmetricDequant {
- b: w,
- scales,
- mode,
- group_size,
- } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),
- MatmulB::FullPrecision {
- ..
- } => unreachable!(),
- };
-
- let bits = match mode {
- QuantizationMode::U4 => 4u32,
- QuantizationMode::I8 | QuantizationMode::U8 => 8u32,
- };
- let use_fast = n % 8 == 0 && k % 512 == 0;
- let (zero_points, biases) = match method {
- QuantizationMethod::ScaleBias => {
- (None, Some(zp_or_bias.expect("ScaleBias quantized matmul requires biases")))
- },
- QuantizationMethod::ScaleZeroPoint => {
- (Some(zp_or_bias.expect("ScaleZeroPoint quantized matmul requires zero_points")), None)
- },
- QuantizationMethod::ScaleSymmetric => (None, None),
- };
-
- let context = encoder.context();
- if use_fast {
- let kernel =
- <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
- context,
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- group_size,
- bits,
- method,
- hadamard_factors.is_some(),
- )
- .map_err(MatmulError::BackendError)?;
- kernel.encode(
- weights,
- scales,
- zero_points,
- biases,
- (a, a_offset),
- &mut *d,
- hadamard_factors,
- k,
- n,
- m,
- encoder,
- );
- } else {
- if hadamard_factors.is_some() {
- return Err(MatmulError::UnsupportedDOp {
- bit: GemmDTransform::RHT,
- path: "MatmulCpuKernel/Quant",
- });
- }
- let kernel =
- <<Cpu as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
- context,
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- group_size,
- bits,
- method,
- )
- .map_err(MatmulError::BackendError)?;
- kernel.encode(weights, scales, zero_points, biases, (a, a_offset), &mut *d, k, n, m, encoder);
- }
- if let Some(bias) = post_bias {
- self.bias_add.encode(None::<&Allocation<Cpu>>, bias, d, n, m * n, encoder);
- }
- Ok(())
- }
-}
-
-fn apply_bias<T: num_traits::Float + Copy>(
- d: *mut T,
- bias: *const T,
- m: usize,
- n: usize,
-) {
- unsafe {
- for row in 0..m {
- for col in 0..n {
- let idx = row * n + col;
- *d.add(idx) = *d.add(idx) + *bias.add(col);
- }
- }
- }
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
index 7c21054fd..aa648840e 100644
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/mod.rs
@@ -1,6 +1,4 @@
-pub mod gemm;
-pub mod gemv;
-mod matmul;
-pub(crate) mod quant;
+mod kernel;
+mod reference;
-pub use matmul::MatmulCpuKernel;
+pub use kernel::MatmulCpuKernel;
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs
deleted file mode 100644
index 76eb56b89..000000000
--- a/crates/backend-uzu/src/backends/cpu/kernel/matmul/quant.rs
+++ /dev/null
@@ -1,156 +0,0 @@
-use half::bf16;
-use num_traits::Float;
-
-use crate::{
- array::ArrayElement,
- backends::{
- common::{
- AsBufferRangeMut, AsBufferRangeRef, Buffer, Encoder,
- gpu_types::{QuantizationMethod, QuantizationMode},
- kernel::matmul::{MatmulArguments, MatmulB},
- },
- cpu::{Cpu, kernel::matmul::gemm::qmm_transposed::qmm_transposed},
- },
- data_type::DataType,
- utils::pointers::{SendPtr, SendPtrMut},
-};
-
-pub(crate) fn encode_quantized_gemm<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
- encoder: &mut Encoder<Cpu>,
- arguments: MatmulArguments<Cpu, TB>,
- data_type: DataType,
-) {
- let MatmulArguments {
- a,
- a_offset,
- b,
- d,
- m,
- n,
- k,
- ..
- } = arguments;
- let (weights, scales, zp_or_bias, method, mode, group_size) = match b {
- MatmulB::ScaleBiasDequant {
- b: w,
- scales,
- biases,
- mode,
- group_size,
- } => (w, scales, Some(biases), QuantizationMethod::ScaleBias, mode, group_size),
- MatmulB::ScaleZeroPointDequant {
- b: w,
- scales,
- zero_points,
- mode,
- group_size,
- } => (w, scales, Some(zero_points), QuantizationMethod::ScaleZeroPoint, mode, group_size),
- MatmulB::ScaleSymmetricDequant {
- b: w,
- scales,
- mode,
- group_size,
- } => (w, scales, None, QuantizationMethod::ScaleSymmetric, mode, group_size),
- MatmulB::FullPrecision {
- ..
- } => unreachable!(),
- };
-
- let bits: usize = match mode {
- QuantizationMode::U4 => 4,
- QuantizationMode::I8 | QuantizationMode::U8 => 8,
- };
- let in_vec_size = k as usize;
- let out_vec_size = n as usize;
- let batch_size = m as usize;
- let group_size = group_size as usize;
-
- let a_buf = a.as_buffer_range_ref();
- let b_buf = weights.as_buffer_range_ref();
- let scales_buf = scales.as_buffer_range_ref();
- let zp_buf = zp_or_bias.map(|zp_or_bias| zp_or_bias.as_buffer_range_ref());
- let d_buf = d.as_buffer_range_mut();
-
- let a_byte_off = a_buf.range().start + a_offset * data_type.size_in_bytes();
- let b_byte_off = b_buf.range().start;
- let scales_byte_off = scales_buf.range().start;
- let zp_byte_off = zp_buf.as_ref().map(|zp_buf| zp_buf.range().start);
- let d_byte_off = d_buf.range().start;
-
- let a_ptr = SendPtr(unsafe { &*a_buf.buffer().get() }.as_ptr().wrapping_byte_add(a_byte_off));
- let b_ptr = SendPtr(unsafe { &*b_buf.buffer().get() }.as_ptr().wrapping_byte_add(b_byte_off));
- let scales_ptr = SendPtr(unsafe { &*scales_buf.buffer().get() }.as_ptr().wrapping_byte_add(scales_byte_off));
- let zp_ptr = zp_buf
- .map(|zp_buf| SendPtr(unsafe { &*zp_buf.buffer().get() }.as_ptr().wrapping_byte_add(zp_byte_off.unwrap())));
- let d_ptr = SendPtrMut(unsafe { (&*d_buf.buffer().get()).as_ptr().wrapping_byte_add(d_byte_off) as *mut u8 });
-
- let command_buffer = encoder.as_command_buffer_mut();
- command_buffer.push_command(move || match data_type {
- DataType::F32 => run::<f32>(
- a_ptr,
- b_ptr,
- scales_ptr,
- zp_ptr,
- d_ptr,
- in_vec_size,
- out_vec_size,
- batch_size,
- method,
- group_size,
- bits,
- ),
- DataType::BF16 => run::<bf16>(
- a_ptr,
- b_ptr,
- scales_ptr,
- zp_ptr,
- d_ptr,
- in_vec_size,
- out_vec_size,
- batch_size,
- method,
- group_size,
- bits,
- ),
- _ => unreachable!(),
- });
-}
-
-fn run<T: ArrayElement + Float>(
- a_ptr: SendPtr<u8>,
- b_ptr: SendPtr<u8>,
- scales_ptr: SendPtr<u8>,
- zp_or_bias_ptr: Option<SendPtr<u8>>,
- d_ptr: SendPtrMut<u8>,
- in_vec_size: usize,
- out_vec_size: usize,
- batch_size: usize,
- quant_method: QuantizationMethod,
- group_size: usize,
- bits: usize,
-) {
- let (zp, bias) = match quant_method {
- QuantizationMethod::ScaleBias => {
- (None, Some(zp_or_bias_ptr.expect("ScaleBias quantized matmul requires biases").as_ptr() as *const T))
- },
- QuantizationMethod::ScaleZeroPoint => (
- Some(zp_or_bias_ptr.expect("ScaleZeroPoint quantized matmul requires zero_points").as_ptr() as *const u8),
- None,
- ),
- QuantizationMethod::ScaleSymmetric => (None, None),
- };
- qmm_transposed::<T>(
- b_ptr.as_ptr() as *const u32,
- scales_ptr.as_ptr() as *const T,
- zp,
- bias,
- a_ptr.as_ptr() as *const T,
- d_ptr.as_ptr() as *mut T,
- in_vec_size,
- out_vec_size,
- batch_size,
- quant_method,
- group_size,
- bits,
- );
-}
diff --git a/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs
new file mode 100644
index 000000000..5076857f0
--- /dev/null
+++ b/crates/backend-uzu/src/backends/cpu/kernel/matmul/reference.rs
@@ -0,0 +1,139 @@
+use half::{bf16, f16};
+
+use crate::{
+ backends::{
+ common::{AsBufferRangeRef, Buffer, gpu_types::QuantizationMode, kernel::matmul::MatmulB},
+ cpu::{Cpu, buffer::BufferDowncastExt},
+ },
+ data_type::DataType,
+ utils::pointers::SendPtr,
+};
+
+pub(super) enum WeightData {
+ FullPrecision {
+ ptr: SendPtr<u8>,
+ leading_dimension: usize,
+ transpose: bool,
+ },
+ Quantized {
+ weights: SendPtr<u8>,
+ scales: SendPtr<u8>,
+ zero_points: Option<SendPtr<u8>>,
+ biases: Option<SendPtr<u8>>,
+ bits: usize,
+ group_size: usize,
+ },
+}
+
+impl WeightData {
+ pub(super) fn from_b<TB: AsBufferRangeRef<Buffer: Buffer<Backend = Cpu>>>(
+ b: MatmulB<'_, Cpu, TB>,
+ b_offset: usize,
+ b_leading_dimension: Option<u32>,
+ b_transpose: bool,
+ weights_data_type: DataType,
+ k: usize,
+ n: usize,
+ ) -> Self {
+ let alloc_ptr = |a: &crate::backends::common::Allocation<Cpu>| {
+ let r = a.as_buffer_range_ref();
+ SendPtr(unsafe { &*r.buffer().get() }.as_ptr().wrapping_byte_add(r.range().start))
+ };
+ let bits_of = |mode| match mode {
+ QuantizationMode::U4 => 4usize,
+ _ => 8usize,
+ };
+ match b {
+ MatmulB::FullPrecision {
+ b: weights,
+ } => {
+ let leading_dimension = b_leading_dimension.map(|ld| ld as usize).unwrap_or(if b_transpose {
+ k
+ } else {
+ n
+ });
+ let r = weights.as_buffer_range_ref();
+ let byte_off = r.range().start + b_offset * weights_data_type.size_in_bytes();
+ WeightData::FullPrecision {
+ ptr: SendPtr(unsafe { &*r.buffer().downcast().get() }.as_ptr().wrapping_byte_add(byte_off)),
+ leading_dimension,
+ transpose: b_transpose,
+ }
+ },
+ MatmulB::ScaleBiasDequant {
+ b: weights,
+ scales,
+ biases,
+ mode,
+ group_size,
+ } => WeightData::Quantized {
+ weights: alloc_ptr(weights),
+ scales: alloc_ptr(scales),
+ zero_points: None,
+ biases: Some(alloc_ptr(biases)),
+ bits: bits_of(mode),
+ group_size: group_size as usize,
+ },
+ MatmulB::ScaleZeroPointDequant {
+ b: weights,
+ scales,
+ zero_points,
+ mode,
+ group_size,
+ } => WeightData::Quantized {
+ weights: alloc_ptr(weights),
+ scales: alloc_ptr(scales),
+ zero_points: Some(alloc_ptr(zero_points)),
+ biases: None,
+ bits: bits_of(mode),
+ group_size: group_size as usize,
+ },
+ MatmulB::ScaleSymmetricDequant {
+ b: weights,
+ scales,
+ mode,
+ group_size,
+ } => WeightData::Quantized {
+ weights: alloc_ptr(weights),
+ scales: alloc_ptr(scales),
+ zero_points: None,
+ biases: None,
+ bits: bits_of(mode),
+ group_size: group_size as usize,
+ },
+ }
+ }
+}
+
+#[inline]
+pub(super) unsafe fn read_f32(
+ base: *const u8,
+ data_type: DataType,
+ index: usize,
+) -> f32 {
+ unsafe {
+ match data_type {
+ DataType::F32 => *(base as *const f32).add(index),
+ DataType::F16 => (*(base as *const f16).add(index)).to_f32(),
+ DataType::BF16 => (*(base as *const bf16).add(index)).to_f32(),
+ _ => unreachable!(),
+ }
+ }
+}
+
+#[inline]
+pub(super) unsafe fn write_f32(
+ base: *mut u8,
+ data_type: DataType,
+ index: usize,
+ value: f32,
+) {
+ unsafe {
+ match data_type {
+ DataType::F32 => *(base as *mut f32).add(index) = value,
+ DataType::F16 => *(base as *mut f16).add(index) = f16::from_f32(value),
+ DataType::BF16 => *(base as *mut bf16).add(index) = bf16::from_f32(value),
+ _ => unreachable!(),
+ }
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
index f165a6585..a2e90ab48 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/generated/gemm.h
@@ -5,7 +5,7 @@
using namespace metal;
namespace uzu::gemm {
-enum class GemmWeightPrologueKind : uint32_t {
+enum class GemmBPrologueKind : uint32_t {
FullPrecision = 0,
ScaleBiasDequant = 1,
ScaleZeroPointDequant = 2,
@@ -31,14 +31,15 @@ struct GemmDTransform {
};
enum class GemmTiling : uint32_t {
- T8x32x32_1x1 = 0,
- T64x32x32_2x2 = 1,
- T64x64x16_2x2 = 2,
- T64x64x32_2x2 = 3,
- T32x32x32_2x2 = 4,
- T32x64x32_2x2 = 5,
- T64x32x32_4x1 = 6,
- T128x128x32_4x4 = 7,
+ Tile8x32x32_Simdgroups1x1 = 0,
+ Tile64x32x32_Simdgroups2x2 = 1,
+ Tile64x64x16_Simdgroups2x2 = 2,
+ Tile64x64x32_Simdgroups2x2 = 3,
+ Tile32x32x32_Simdgroups2x2 = 4,
+ Tile32x64x256_Simdgroups2x2 = 5,
+ Tile64x32x256_Simdgroups4x1 = 6,
+ Tile64x64x256_Simdgroups2x2 = 7,
+ Tile128x128x256_Simdgroups4x4 = 8,
};
struct GemmAlignment {
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
index 314628029..10bc5ce2f 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.h
@@ -1,3 +1,5 @@
+#pragma once
+
#include <metal_stdlib>
#include "../common/defines.h"
#include "../common/thread_context.h"
diff --git a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
index 78b038b02..6678f8060 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/hadamard_transform/hadamard_transform.metal
@@ -8,7 +8,7 @@ using namespace metal;
using namespace uzu::hadamard_order;
template <typename T>
-VARIANTS(T, half, bfloat, float)
+VARIANTS(T, bfloat, float)
PUBLIC KERNEL(HadamardTransform)(
device T* data,
const device int32_t* factors,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index 9b64d6989..2552c3a0a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -117,8 +117,6 @@ struct ThreadgroupTile {
METAL_PRAGMA_UNROLL
for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH;
k_block_index += SIMDGROUP_BLOCK_SIZE) {
- simdgroup_barrier(mem_flags::mem_none);
-
a_fragment.template load<
AT,
SIMDGROUPS_PER_ROW,
@@ -126,8 +124,6 @@ struct ThreadgroupTile {
A_STRIDE_ROW,
A_STRIDE_INNER>(a_shared);
- simdgroup_barrier(mem_flags::mem_none);
-
b_fragment.template load<
BT,
1,
@@ -135,8 +131,6 @@ struct ThreadgroupTile {
B_STRIDE_INNER,
B_STRIDE_COL>(b_shared);
- simdgroup_barrier(mem_flags::mem_none);
-
tile_multiply_accumulate(c_fragment, a_fragment, b_fragment, c_fragment);
a_shared += TILE_STRIDE_A;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
new file mode 100644
index 000000000..864a0af1c
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_rht.h
@@ -0,0 +1,34 @@
+#pragma once
+
+#include "../../../common/thread_context.h"
+#include "../../../hadamard_transform/hadamard_transform.h"
+
+using namespace metal;
+
+template <typename T>
+static METAL_FUNC void apply_output_random_hadamard_transform(
+ device T* output_block,
+ const device int32_t* rht_factors_block,
+ ushort tile_block_rows,
+ ushort tile_block_cols,
+ uint leading_dimension_d,
+ ushort simdgroup_count,
+ const thread ThreadContext& thread_context
+) {
+ const ushort stripes_per_row = tile_block_cols / METAL_SIMD_SIZE;
+ const ushort simd_lane = thread_context.simd_lane_id;
+ const uint total_work = uint(tile_block_rows) * uint(stripes_per_row);
+ for (uint cell = thread_context.simdgroup_index; cell < total_work;
+ cell += simdgroup_count) {
+ const ushort row_local = ushort(cell / stripes_per_row);
+ const ushort stripe = ushort(cell % stripes_per_row);
+ const ushort col_local = stripe * ushort(METAL_SIMD_SIZE) + simd_lane;
+ const size_t output_index =
+ size_t(row_local) * size_t(leading_dimension_d) + size_t(col_local);
+ output_block[output_index] = simdgroup_output_random_hadamard_transform(
+ simd_lane,
+ output_block[output_index],
+ rht_factors_block[col_local]
+ );
+ }
+}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
index 0ee3354d6..7556db8c7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/gemm_tiling.h
@@ -6,63 +6,68 @@ namespace uzu {
namespace gemm {
constexpr uint gemm_tiling_block_m(GemmTiling t) {
- return t == GemmTiling::T8x32x32_1x1 ? 8
- : t == GemmTiling::T64x32x32_2x2 ? 64
- : t == GemmTiling::T64x64x16_2x2 ? 64
- : t == GemmTiling::T64x64x32_2x2 ? 64
- : t == GemmTiling::T32x32x32_2x2 ? 32
- : t == GemmTiling::T32x64x32_2x2 ? 32
- : t == GemmTiling::T64x32x32_4x1 ? 64
- : t == GemmTiling::T128x128x32_4x4 ? 128
- : 0;
+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 8
+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 32
+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 64
+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128
+ : 0;
}
constexpr uint gemm_tiling_block_n(GemmTiling t) {
- return t == GemmTiling::T8x32x32_1x1 ? 32
- : t == GemmTiling::T64x32x32_2x2 ? 32
- : t == GemmTiling::T64x64x16_2x2 ? 64
- : t == GemmTiling::T64x64x32_2x2 ? 64
- : t == GemmTiling::T32x32x32_2x2 ? 32
- : t == GemmTiling::T32x64x32_2x2 ? 64
- : t == GemmTiling::T64x32x32_4x1 ? 32
- : t == GemmTiling::T128x128x32_4x4 ? 128
- : 0;
+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32
+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32
+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 32
+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 64
+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 128
+ : 0;
}
constexpr uint gemm_tiling_block_k(GemmTiling t) {
- return t == GemmTiling::T8x32x32_1x1 ? 32
- : t == GemmTiling::T64x32x32_2x2 ? 32
- : t == GemmTiling::T64x64x16_2x2 ? 16
- : t == GemmTiling::T64x64x32_2x2 ? 32
- : t == GemmTiling::T32x32x32_2x2 ? 32
- : t == GemmTiling::T32x64x32_2x2 ? 32
- : t == GemmTiling::T64x32x32_4x1 ? 32
- : t == GemmTiling::T128x128x32_4x4 ? 32
- : 0;
+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 32
+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 32
+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 16
+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 32
+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 32
+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 256
+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 256
+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 256
+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 256
+ : 0;
}
constexpr uint gemm_tiling_simdgroups_per_row(GemmTiling t) {
- return t == GemmTiling::T8x32x32_1x1 ? 1
- : t == GemmTiling::T64x32x32_2x2 ? 2
- : t == GemmTiling::T64x64x16_2x2 ? 2
- : t == GemmTiling::T64x64x32_2x2 ? 2
- : t == GemmTiling::T32x32x32_2x2 ? 2
- : t == GemmTiling::T32x64x32_2x2 ? 2
- : t == GemmTiling::T64x32x32_4x1 ? 4
- : t == GemmTiling::T128x128x32_4x4 ? 4
- : 0;
+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1
+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 4
+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4
+ : 0;
}
constexpr uint gemm_tiling_simdgroups_per_column(GemmTiling t) {
- return t == GemmTiling::T8x32x32_1x1 ? 1
- : t == GemmTiling::T64x32x32_2x2 ? 2
- : t == GemmTiling::T64x64x16_2x2 ? 2
- : t == GemmTiling::T64x64x32_2x2 ? 2
- : t == GemmTiling::T32x32x32_2x2 ? 2
- : t == GemmTiling::T32x64x32_2x2 ? 2
- : t == GemmTiling::T64x32x32_4x1 ? 1
- : t == GemmTiling::T128x128x32_4x4 ? 4
- : 0;
+ return t == GemmTiling::Tile8x32x32_Simdgroups1x1 ? 1
+ : t == GemmTiling::Tile64x32x32_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile64x64x16_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile64x64x32_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile32x32x32_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile32x64x256_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile64x32x256_Simdgroups4x1 ? 1
+ : t == GemmTiling::Tile64x64x256_Simdgroups2x2 ? 2
+ : t == GemmTiling::Tile128x128x256_Simdgroups4x4 ? 4
+ : 0;
}
} // namespace gemm
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
index cca0830a2..55b8ad1cb 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/mxu_mma_core.h
@@ -2,13 +2,18 @@
#include "../../../common/integral_constant.h"
#include "../../../common/thread_context.h"
+#include "../../../hadamard_transform/hadamard_transform.h"
#include "../../common/fragment.h"
+#include "gemm_rht.h"
#include "../../common/mxu_fragment_ops.h"
#include "../../common/mxu_gemm_loop.h"
#include "../../../generated/matmul.h"
#include "../generated/gemm.h"
#include "block_geometry.h"
#include "gemm_tiling.h"
+#include "quant_pack.h"
+#include "quant_scale_bias.h"
+#include "quant_scale_zero_point.h"
using namespace metal;
@@ -20,7 +25,10 @@ template <
typename BT,
typename DT,
GemmTiling GEMM_TILING,
- bool TRANSPOSE_B>
+ bool TRANSPOSE_B,
+ GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,
+ int BITS = 0,
+ int GROUP_SIZE = 0>
struct MxuMmaCore {
METAL_CONST ushort THREADGROUP_BLOCK_M = gemm_tiling_block_m(GEMM_TILING);
METAL_CONST ushort THREADGROUP_BLOCK_N = gemm_tiling_block_n(GEMM_TILING);
@@ -33,14 +41,134 @@ struct MxuMmaCore {
METAL_CONST ushort SIMDGROUP_BLOCK_N =
THREADGROUP_BLOCK_N / SIMDGROUPS_PER_COLUMN;
METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;
- METAL_CONST ushort THREADGROUP_BLOCK_K = 256;
+ METAL_CONST ushort THREADGROUP_BLOCK_K_FP = gemm_tiling_block_k(GEMM_TILING);
+ static_assert(
+ THREADGROUP_BLOCK_K_FP % SIMDGROUP_BLOCK_K == 0,
+ "FP THREADGROUP_BLOCK_K must be a multiple of SIMDGROUP_BLOCK_K"
+ );
METAL_CONST ushort TILES_M =
SIMDGROUP_BLOCK_M / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
METAL_CONST ushort TILES_N =
SIMDGROUP_BLOCK_N / uzu::matmul::MxuFragmentOps::FRAGMENT_COLS;
+ METAL_CONST ushort TILES_K =
+ SIMDGROUP_BLOCK_K / uzu::matmul::MxuFragmentOps::FRAGMENT_ROWS;
+
+ METAL_CONST ushort QUANT_BK =
+ (B_PROLOGUE == GemmBPrologueKind::FullPrecision) ? 0 : GROUP_SIZE;
+ METAL_CONST ushort PADDING_B = 16 / sizeof(BT);
+ METAL_CONST ushort SHARED_STRIDE_B =
+ (QUANT_BK > 0) ? (QUANT_BK + PADDING_B) : 1;
+ METAL_CONST ushort THREADGROUP_THREADS =
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
+ static_assert(
+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
+ QUANT_BK % SIMDGROUP_BLOCK_K == 0,
+ "QUANT_BK must be a multiple of SIMDGROUP_BLOCK_K"
+ );
+ static_assert(
+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
+ THREADGROUP_BLOCK_K_FP % QUANT_BK == 0,
+ "Tile block_k must be a multiple of QUANT_BK"
+ );
using AccumulatorType = float;
+ using BLoaderScaleBias = QuantizedBlockLoaderScaleBias<
+ BT,
+ THREADGROUP_BLOCK_N,
+ (QUANT_BK > 0) ? QUANT_BK : 1,
+ SHARED_STRIDE_B,
+ 1,
+ THREADGROUP_THREADS,
+ (GROUP_SIZE > 0) ? GROUP_SIZE : 1,
+ (BITS > 0) ? BITS : 4>;
+ using BLoaderScaleZeroPoint = QuantizedBlockLoaderScaleZeroPoint<
+ BT,
+ THREADGROUP_BLOCK_N,
+ (QUANT_BK > 0) ? QUANT_BK : 1,
+ SHARED_STRIDE_B,
+ 1,
+ THREADGROUP_THREADS,
+ (GROUP_SIZE > 0) ? GROUP_SIZE : 1,
+ (BITS > 0) ? BITS : 4>;
+ using BLoaderScaleSymmetric = QuantizedBlockLoaderScaleZeroPoint<
+ BT,
+ THREADGROUP_BLOCK_N,
+ (QUANT_BK > 0) ? QUANT_BK : 1,
+ SHARED_STRIDE_B,
+ 1,
+ THREADGROUP_THREADS,
+ (GROUP_SIZE > 0) ? GROUP_SIZE : 1,
+ (BITS > 0) ? BITS : 4,
+ true>;
+
+ using AccumFragment = uzu::matmul::
+ Fragment<AccumulatorType, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
+
+ template <bool ALIGNED_M, bool ALIGNED_N, typename Loader>
+ static METAL_FUNC AccumFragment quant_k_loop(
+ const device AT* a_simdgroup,
+ threadgroup BT* b_shared,
+ const int leading_dimension_a,
+ const int aligned_k_iterations,
+ const short simdgroup_limit_m,
+ const short simdgroup_limit_n,
+ const ushort tile_col_offset,
+ const ushort tile_block_cols,
+ thread Loader& loader_b,
+ const thread ThreadContext& thread_context
+ ) {
+ AccumFragment accumulator(thread_context);
+ accumulator.clear();
+
+ threadgroup BT* b_shared_simdgroup =
+ b_shared + tile_col_offset * SHARED_STRIDE_B;
+ const short2 tile_dimensions_b = short2(QUANT_BK, tile_block_cols);
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int outer_k = 0; outer_k < aligned_k_iterations; ++outer_k) {
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ if constexpr (ALIGNED_N) {
+ loader_b.load_unsafe();
+ } else {
+ loader_b.load_safe(tile_dimensions_b);
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int inner_k = 0; inner_k < QUANT_BK; inner_k += SIMDGROUP_BLOCK_K) {
+ uzu::matmul::Fragment<AT, TILES_M, TILES_K, uzu::matmul::MxuFragmentOps>
+ left_tile(thread_context);
+ uzu::matmul::Fragment<BT, TILES_N, TILES_K, uzu::matmul::MxuFragmentOps>
+ right_tile(thread_context);
+
+ const int left_offset = inner_k;
+ if constexpr (ALIGNED_M) {
+ left_tile.load(a_simdgroup + left_offset, leading_dimension_a);
+ } else {
+ left_tile.load_safe(
+ a_simdgroup + left_offset,
+ leading_dimension_a,
+ short2(SIMDGROUP_BLOCK_K, simdgroup_limit_m)
+ );
+ }
+
+ right_tile.load(b_shared_simdgroup + inner_k, int(SHARED_STRIDE_B));
+
+ uzu::matmul::MxuFragmentOps::template tile_matmul<false, true>(
+ accumulator,
+ left_tile,
+ right_tile
+ );
+ }
+
+ a_simdgroup += QUANT_BK;
+ loader_b.next();
+ }
+
+ return accumulator;
+ }
+
static METAL_FUNC void run(
const device AT* a,
const device BT* b,
@@ -48,10 +176,14 @@ struct MxuMmaCore {
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
GemmDTransform output_transform,
+ const device BT* scales,
+ const device BT* biases,
+ const device uint8_t* zero_points,
const device BT* output_bias,
+ const device int32_t* rht_factors,
+ threadgroup BT* b_shared,
const thread ThreadContext& thread_context
) {
- (void)output_bias;
const uint2 tile = tile_id(thread_context.threadgroup_position.xy, params);
const auto geometry =
ThreadgroupTileGeometry<THREADGROUP_BLOCK_M, THREADGROUP_BLOCK_N>::
@@ -64,7 +196,7 @@ struct MxuMmaCore {
const size_t block_col = size_t(geometry.block_col_start);
const device AT* a_block = a + block_row * params->leading_dimension_a;
- const device BT* b_block =
+ const device BT* b_block_fp =
b + (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);
const ushort tile_row_offset =
@@ -97,18 +229,35 @@ struct MxuMmaCore {
const device AT* a_simdgroup =
a_block + size_t(tile_row_offset) * params->leading_dimension_a;
- const device BT* b_simdgroup =
- b_block + (TRANSPOSE_B ? size_t(tile_col_offset) *
- int(params->leading_dimension_b)
- : size_t(tile_col_offset));
+ const device BT* b_simdgroup_fp =
+ b_block_fp + (TRANSPOSE_B ? size_t(tile_col_offset) *
+ int(params->leading_dimension_b)
+ : size_t(tile_col_offset));
- const int aligned_k_iterations = int(params->K) / int(THREADGROUP_BLOCK_K);
+ const ushort tile_block_cols = ushort(
+ min(int(THREADGROUP_BLOCK_N),
+ int(params->N) - int(geometry.block_col_start))
+ );
const bool apply_scale = output_transform.contains(GemmDTransform::SCALE);
const bool apply_accumulate =
output_transform.contains(GemmDTransform::ACCUMULATE);
+ const bool apply_bias = output_transform.contains(GemmDTransform::BIAS);
- dispatch_bool(alignment.contains(GemmAlignment::K), [&](auto aligned_k) {
+ const device BT* bias_simdgroup =
+ output_bias + size_t(block_col) + size_t(tile_col_offset);
+ using FragType = uzu::matmul::
+ Fragment<DT, TILES_M, TILES_N, uzu::matmul::MxuFragmentOps>;
+ const short2 thread_position = FragType::get_position(thread_context);
+
+ auto dispatch_aligned_k = [&](auto body) {
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
+ dispatch_bool(alignment.contains(GemmAlignment::K), body);
+ } else {
+ body(true_type{});
+ }
+ };
+ dispatch_aligned_k([&](auto aligned_k) {
dispatch_bool(
alignment.contains(GemmAlignment::M) ||
(simdgroup_limit_m == SIMDGROUP_BLOCK_M),
@@ -117,29 +266,114 @@ struct MxuMmaCore {
alignment.contains(GemmAlignment::N) ||
(simdgroup_limit_n == SIMDGROUP_BLOCK_N),
[&](auto aligned_n) {
- auto accumulator_tile = uzu::matmul::gemm_loop<
- AT,
- BT,
- SIMDGROUP_BLOCK_M,
- SIMDGROUP_BLOCK_N,
- SIMDGROUP_BLOCK_K,
- THREADGROUP_BLOCK_K,
- false,
- TRANSPOSE_B,
- aligned_m.value,
- aligned_n.value,
- aligned_k.value,
- AccumulatorType>(
- a_simdgroup,
- b_simdgroup,
- int(params->leading_dimension_a),
- int(params->leading_dimension_b),
- int(params->K),
- aligned_k_iterations,
- simdgroup_limit_m,
- simdgroup_limit_n,
- thread_context
- );
+ auto accumulator_tile = [&]() {
+ if constexpr (
+ B_PROLOGUE == GemmBPrologueKind::FullPrecision
+ ) {
+ const int aligned_k_iterations_fp =
+ int(params->K) / int(THREADGROUP_BLOCK_K_FP);
+ return uzu::matmul::gemm_loop<
+ AT,
+ BT,
+ SIMDGROUP_BLOCK_M,
+ SIMDGROUP_BLOCK_N,
+ SIMDGROUP_BLOCK_K,
+ THREADGROUP_BLOCK_K_FP,
+ false,
+ TRANSPOSE_B,
+ aligned_m.value,
+ aligned_n.value,
+ aligned_k.value,
+ AccumulatorType>(
+ a_simdgroup,
+ b_simdgroup_fp,
+ int(params->leading_dimension_a),
+ int(params->leading_dimension_b),
+ int(params->K),
+ aligned_k_iterations_fp,
+ simdgroup_limit_m,
+ simdgroup_limit_n,
+ thread_context
+ );
+ } else {
+ const int aligned_k_iterations_q =
+ int(params->K) / int(QUANT_BK);
+ constexpr int pack_factor = get_pack_factor<BITS, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
+ const int k_elements = int(params->K);
+ const int weights_row_stride_bytes =
+ k_elements * bytes_per_pack / pack_factor;
+ const int groups_per_row =
+ (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const device uint8_t* weights_block =
+ reinterpret_cast<const device uint8_t*>(b) +
+ block_col * weights_row_stride_bytes;
+ const device BT* scales_offset =
+ scales + block_col * groups_per_row;
+
+ auto loader_b = [&]() {
+ if constexpr (
+ B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant
+ ) {
+ const device BT* biases_offset =
+ biases + block_col * groups_per_row;
+ return BLoaderScaleBias(
+ weights_block,
+ scales_offset,
+ biases_offset,
+ k_elements,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ } else if constexpr (
+ B_PROLOGUE ==
+ GemmBPrologueKind::ScaleZeroPointDequant
+ ) {
+ const int zero_point_stride_per_row =
+ (BITS == 4) ? ((groups_per_row + 1) / 2)
+ : groups_per_row;
+ const device uint8_t* zero_points_row_start =
+ zero_points +
+ block_col * zero_point_stride_per_row;
+ return BLoaderScaleZeroPoint(
+ weights_block,
+ scales_offset,
+ zero_points_row_start,
+ k_elements,
+ groups_per_row,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ } else {
+ return BLoaderScaleSymmetric(
+ weights_block,
+ scales_offset,
+ nullptr,
+ k_elements,
+ groups_per_row,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ }
+ }();
+
+ return quant_k_loop<aligned_m.value, aligned_n.value>(
+ a_simdgroup,
+ b_shared,
+ int(params->leading_dimension_a),
+ aligned_k_iterations_q,
+ simdgroup_limit_m,
+ simdgroup_limit_n,
+ tile_col_offset,
+ tile_block_cols,
+ loader_b,
+ thread_context
+ );
+ }
+ }();
if (apply_scale) {
const AccumulatorType scale =
@@ -178,6 +412,49 @@ struct MxuMmaCore {
}
}
+ if (apply_bias) {
+ METAL_PRAGMA_UNROLL
+ for (ushort tile_row = 0; tile_row < TILES_M; ++tile_row) {
+ METAL_PRAGMA_UNROLL
+ for (ushort tile_col = 0; tile_col < TILES_N;
+ ++tile_col) {
+ thread auto& frag =
+ accumulator_tile.fragment_at(tile_row, tile_col);
+ const short col_base = short(
+ tile_col *
+ uzu::matmul::MxuFragmentOps::FRAGMENT_COLS +
+ thread_position.x
+ );
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0;
+ i <
+ uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_ROWS;
+ ++i) {
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0;
+ j <
+ uzu::matmul::MxuFragmentOps::THREAD_ELEMENT_COLS;
+ ++j) {
+ const ushort element_index =
+ i * uzu::matmul::MxuFragmentOps::
+ THREAD_ELEMENT_COLS +
+ j;
+ const short col_local = short(col_base + j);
+ if constexpr (aligned_n.value) {
+ frag[element_index] +=
+ AccumulatorType(bias_simdgroup[col_local]);
+ } else {
+ if (col_local < simdgroup_limit_n) {
+ frag[element_index] +=
+ AccumulatorType(bias_simdgroup[col_local]);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
if constexpr (aligned_m.value && aligned_n.value) {
accumulator_tile.store(
d_simdgroup,
@@ -195,6 +472,27 @@ struct MxuMmaCore {
}
);
});
+
+ if (output_transform.contains(GemmDTransform::RHT)) {
+ threadgroup_barrier(mem_flags::mem_device);
+ device DT* d_block =
+ d + block_row * params->leading_dimension_d + block_col;
+ const ushort tile_block_rows = ushort(
+ min(int(THREADGROUP_BLOCK_M), int(params->M) - int(block_row))
+ );
+ const ushort tile_block_cols = ushort(
+ min(int(THREADGROUP_BLOCK_N), int(params->N) - int(block_col))
+ );
+ apply_output_random_hadamard_transform(
+ d_block,
+ rht_factors + block_col,
+ tile_block_rows,
+ tile_block_cols,
+ params->leading_dimension_d,
+ ushort(SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN),
+ thread_context
+ );
+ }
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
index 88c5dcbbb..e329b85a8 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/common/simdgroup_mma_core.h
@@ -7,7 +7,9 @@
#include "../../common/threadgroup_tile.h"
#include "../../../generated/matmul.h"
#include "../generated/gemm.h"
+#include "../../../hadamard_transform/hadamard_transform.h"
#include "block_geometry.h"
+#include "gemm_rht.h"
#include "gemm_alignment.h"
#include "gemm_tiling.h"
#include "quant_scale_bias.h"
@@ -24,8 +26,7 @@ template <
typename DT,
GemmTiling GEMM_TILING,
bool TRANSPOSE_B,
- GemmWeightPrologueKind WEIGHT_PROLOGUE =
- GemmWeightPrologueKind::FullPrecision,
+ GemmBPrologueKind B_PROLOGUE = GemmBPrologueKind::FullPrecision,
int BITS = 0,
int GROUP_SIZE = 0>
struct SimdgroupMmaCore {
@@ -168,7 +169,10 @@ struct SimdgroupMmaCore {
const thread uzu::matmul::TransformScaleAccumulate<float, float>&
epilogue,
const device BT* bias_block,
- const bool needs_bias
+ const bool needs_bias,
+ const device int32_t* rht_factors_block,
+ const bool needs_rht,
+ const thread ThreadContext& thread_context
) {
constexpr GemmAlignment gemm_alignment{GEMM_ALIGNMENT_RAW};
if constexpr (
@@ -204,11 +208,24 @@ struct SimdgroupMmaCore {
short2(tile_block_cols, tile_block_rows)
);
}
+
+ if (needs_rht) {
+ threadgroup_barrier(mem_flags::mem_device);
+ apply_output_random_hadamard_transform(
+ d,
+ rht_factors_block,
+ tile_block_rows,
+ tile_block_cols,
+ params->leading_dimension_d,
+ ushort(SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN),
+ thread_context
+ );
+ }
}
static METAL_FUNC void run(
const device AT* a,
- const device uint8_t* b_packed,
+ const device BT* b,
device DT* d,
const constant uzu::matmul::GemmParams* params,
GemmAlignment alignment,
@@ -217,6 +234,7 @@ struct SimdgroupMmaCore {
const device BT* biases,
const device uint8_t* zero_points,
const device BT* output_bias,
+ const device int32_t* rht_factors,
threadgroup AT* a_shared,
threadgroup BT* b_shared,
const thread ThreadContext& thread_context
@@ -256,22 +274,88 @@ struct SimdgroupMmaCore {
const bool needs_accumulate =
output_transform.contains(GemmDTransform::ACCUMULATE);
const bool needs_bias = output_transform.contains(GemmDTransform::BIAS);
+ const bool needs_rht = output_transform.contains(GemmDTransform::RHT);
const bool needs_epilogue = needs_scale || needs_accumulate;
const float alpha = needs_scale ? params->ab_scale : 1.0f;
const float beta = needs_accumulate ? 1.0f : 0.0f;
uzu::matmul::TransformScaleAccumulate<float, float> epilogue(alpha, beta);
const device BT* bias_block = output_bias + block_col;
+ const device int32_t* rht_factors_block = rht_factors + block_col;
+
+ auto loader_b = [&]() {
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::FullPrecision) {
+ const device BT* b_block_fp =
+ b +
+ (TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col);
+ return BLoaderFp(
+ b_block_fp,
+ params->leading_dimension_b,
+ b_shared,
+ thread_context
+ );
+ } else {
+ constexpr int pack_factor = get_pack_factor<BITS, 8>();
+ constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
+ const int k_elements = int(params->K);
+ const int weights_row_stride_bytes =
+ k_elements * bytes_per_pack / pack_factor;
+ const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
+ const device uint8_t* weights_block =
+ reinterpret_cast<const device uint8_t*>(b) +
+ block_col * weights_row_stride_bytes;
+ const device BT* scales_offset = scales + block_col * groups_per_row;
+
+ if constexpr (B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant) {
+ const device BT* biases_offset = biases + block_col * groups_per_row;
+ return BLoaderScaleBias(
+ weights_block,
+ scales_offset,
+ biases_offset,
+ k_elements,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ } else if constexpr (
+ B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant
+ ) {
+ const int zero_point_stride_per_row =
+ (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
+ const device uint8_t* zero_points_row_start =
+ zero_points + block_col * zero_point_stride_per_row;
+ return BLoaderScaleZeroPoint(
+ weights_block,
+ scales_offset,
+ zero_points_row_start,
+ k_elements,
+ groups_per_row,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ } else {
+ return BLoaderScaleSymmetric(
+ weights_block,
+ scales_offset,
+ nullptr,
+ k_elements,
+ groups_per_row,
+ b_shared,
+ thread_context.simdgroup_index,
+ thread_context.simd_lane_id
+ );
+ }
+ }
+ }();
const bool all_aligned = ((alignment.contains(GemmAlignment::M)) ||
(tile_block_rows == THREADGROUP_BLOCK_M)) &&
((alignment.contains(GemmAlignment::N)) ||
(tile_block_cols == THREADGROUP_BLOCK_N)) &&
alignment.contains(GemmAlignment::K);
-
constexpr uint MASK_ALL = static_cast<uint>(GemmAlignment::M) |
static_cast<uint>(GemmAlignment::N) |
static_cast<uint>(GemmAlignment::K);
-
const uint dynamic_alignment_mask =
alignment.raw_value |
((tile_block_rows == THREADGROUP_BLOCK_M)
@@ -281,196 +365,39 @@ struct SimdgroupMmaCore {
? static_cast<uint>(GemmAlignment::N)
: 0u);
- auto dispatch_kernel = [&](auto kernel_invoke) {
- if (all_aligned) {
- kernel_invoke(integral_constant<uint, MASK_ALL>{});
- } else {
- dispatch_gemm_alignment(dynamic_alignment_mask, kernel_invoke);
- }
- };
-
- if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) {
- const device BT* b = reinterpret_cast<const device BT*>(b_packed);
- b += TRANSPOSE_B ? block_col * params->leading_dimension_b : block_col;
- thread BLoaderFp
- loader_b(b, params->leading_dimension_b, b_shared, thread_context);
- dispatch_kernel([&](auto gemm_alignment_mask) {
- constexpr uint mask = gemm_alignment_mask.value;
- k_loop<mask, BLoaderFp>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- accumulator,
- tile_block_rows,
- tile_block_cols,
- leftover_block_depth
- );
- finalize<mask>(
- accumulator,
- d,
- params,
- tile_block_rows,
- tile_block_cols,
- needs_epilogue,
- epilogue,
- bias_block,
- needs_bias
- );
- });
- return;
- }
-
- if constexpr (WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant) {
- constexpr int pack_factor = get_pack_factor<BITS, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
- const int k_elements = int(params->K);
- const int weights_row_stride_bytes =
- k_elements * bytes_per_pack / pack_factor;
- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
- const device uint8_t* weights_block =
- b_packed + block_col * weights_row_stride_bytes;
- const device BT* scales_offset = scales + block_col * groups_per_row;
- const device BT* biases_offset = biases + block_col * groups_per_row;
- thread BLoaderScaleBias loader_b(
- weights_block,
- scales_offset,
- biases_offset,
- k_elements,
+ auto kernel_invoke = [&](auto gemm_alignment_mask) {
+ constexpr uint gemm_alignment = gemm_alignment_mask.value;
+ k_loop<gemm_alignment>(
+ a_shared,
b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
+ params->aligned_inner_iterations,
+ loader_a,
+ loader_b,
+ accumulator,
+ tile_block_rows,
+ tile_block_cols,
+ leftover_block_depth
);
- dispatch_kernel([&](auto gemm_alignment_mask) {
- constexpr uint mask = gemm_alignment_mask.value;
- k_loop<mask, BLoaderScaleBias>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- accumulator,
- tile_block_rows,
- tile_block_cols,
- leftover_block_depth
- );
- finalize<mask>(
- accumulator,
- d,
- params,
- tile_block_rows,
- tile_block_cols,
- needs_epilogue,
- epilogue,
- bias_block,
- needs_bias
- );
- });
- return;
- }
-
- if constexpr (
- WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant
- ) {
- constexpr int pack_factor = get_pack_factor<BITS, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
- const int k_elements = int(params->K);
- const int weights_row_stride_bytes =
- k_elements * bytes_per_pack / pack_factor;
- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
- const device uint8_t* weights_block =
- b_packed + block_col * weights_row_stride_bytes;
- const device BT* scales_offset = scales + block_col * groups_per_row;
- const int zero_point_stride_per_row =
- (BITS == 4) ? ((groups_per_row + 1) / 2) : groups_per_row;
- const device uint8_t* zero_points_row_start =
- zero_points + block_col * zero_point_stride_per_row;
- thread BLoaderScaleZeroPoint loader_b(
- weights_block,
- scales_offset,
- zero_points_row_start,
- k_elements,
- groups_per_row,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
+ finalize<gemm_alignment>(
+ accumulator,
+ d,
+ params,
+ tile_block_rows,
+ tile_block_cols,
+ needs_epilogue,
+ epilogue,
+ bias_block,
+ needs_bias,
+ rht_factors_block,
+ needs_rht,
+ thread_context
);
- dispatch_kernel([&](auto gemm_alignment_mask) {
- constexpr uint mask = gemm_alignment_mask.value;
- k_loop<mask, BLoaderScaleZeroPoint>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- accumulator,
- tile_block_rows,
- tile_block_cols,
- leftover_block_depth
- );
- finalize<mask>(
- accumulator,
- d,
- params,
- tile_block_rows,
- tile_block_cols,
- needs_epilogue,
- epilogue,
- bias_block,
- needs_bias
- );
- });
- return;
- }
+ };
- if constexpr (
- WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleSymmetricDequant
- ) {
- constexpr int pack_factor = get_pack_factor<BITS, 8>();
- constexpr int bytes_per_pack = get_bytes_per_pack<BITS>();
- const int k_elements = int(params->K);
- const int weights_row_stride_bytes =
- k_elements * bytes_per_pack / pack_factor;
- const int groups_per_row = (k_elements + GROUP_SIZE - 1) / GROUP_SIZE;
- const device uint8_t* weights_block =
- b_packed + block_col * weights_row_stride_bytes;
- const device BT* scales_offset = scales + block_col * groups_per_row;
- thread BLoaderScaleSymmetric loader_b(
- weights_block,
- scales_offset,
- nullptr,
- k_elements,
- groups_per_row,
- b_shared,
- thread_context.simdgroup_index,
- thread_context.simd_lane_id
- );
- dispatch_kernel([&](auto gemm_alignment_mask) {
- constexpr uint mask = gemm_alignment_mask.value;
- k_loop<mask, BLoaderScaleSymmetric>(
- a_shared,
- b_shared,
- params->aligned_inner_iterations,
- loader_a,
- loader_b,
- accumulator,
- tile_block_rows,
- tile_block_cols,
- leftover_block_depth
- );
- finalize<mask>(
- accumulator,
- d,
- params,
- tile_block_rows,
- tile_block_cols,
- needs_epilogue,
- epilogue,
- bias_block,
- needs_bias
- );
- });
+ if (all_aligned) {
+ kernel_invoke(integral_constant<uint, MASK_ALL>{});
+ } else {
+ dispatch_gemm_alignment(dynamic_alignment_mask, kernel_invoke);
}
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs
index 5663f7268..f80edad41 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/error.rs
@@ -1,14 +1,26 @@
use thiserror::Error;
+use crate::backends::common::gpu_types::gemm::GemmTiling;
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
pub enum GemmSpecializationError {
- #[error("threadgroup K={threadgroup_k} exceeds group size {group_size}")]
- ThreadgroupKExceedsGroupSize {
- threadgroup_k: u32,
+ #[error("simdgroup K={simdgroup_k} exceeds group size {group_size}")]
+ SimdgroupKExceedsGroupSize {
+ simdgroup_k: u32,
group_size: u32,
},
- #[error("quantized B requires the simdgroup path")]
- QuantizedRequiresSimdgroup,
#[error("quantized B requires transposed layout")]
QuantizedRequiresTransposedB,
+ #[error("tiling {tiling} does not match use_mxu={use_mxu}")]
+ TilingUseMxuMismatch {
+ tiling: GemmTiling,
+ use_mxu: bool,
+ },
+ #[error(
+ "MXU quantized GEMM with tile {tiling} requires group_size <= 64 (got {group_size}) due to threadgroup memory budget"
+ )]
+ MxuQuantTileTooLarge {
+ tiling: GemmTiling,
+ group_size: u32,
+ },
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
index 3e94d9973..70ac6cb24 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/gemm.metal
@@ -10,14 +10,18 @@
using namespace metal;
using namespace uzu::gemm;
+#define GEMM_MXU_QUANT \
+ (USE_MXU && B_PROLOGUE != GemmBPrologueKind::FullPrecision)
#define GEMM_TGA_ELEMENTS \
- (USE_MXU ? 1 \
- : (gemm_tiling_block_m(GEMM_TILING) * \
- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
+ ((USE_MXU) ? 1 \
+ : (gemm_tiling_block_m(GEMM_TILING) * \
+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(AT)))))
#define GEMM_TGB_ELEMENTS \
- (USE_MXU ? 1 \
- : (gemm_tiling_block_n(GEMM_TILING) * \
- (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
+ ((USE_MXU) ? (GEMM_MXU_QUANT ? (gemm_tiling_block_n(GEMM_TILING) * \
+ (int(GROUP_SIZE) + 16 / int(sizeof(BT)))) \
+ : 1) \
+ : (gemm_tiling_block_n(GEMM_TILING) * \
+ (gemm_tiling_block_k(GEMM_TILING) + 16 / int(sizeof(BT)))))
template <
typename AT,
@@ -26,7 +30,7 @@ template <
GemmTiling GEMM_TILING,
bool TRANSPOSE_B,
bool USE_MXU,
- GemmWeightPrologueKind WEIGHT_PROLOGUE,
+ GemmBPrologueKind B_PROLOGUE,
uint BITS,
uint GROUP_SIZE>
VARIANTS(AT, bfloat, float)
@@ -35,69 +39,60 @@ VARIANTS(DT, bfloat, float)
CONSTRAINT(BT != "float" || (AT == "float" && DT == "float"))
VARIANTS(
GEMM_TILING,
- GemmTiling::T8x32x32_1x1,
- GemmTiling::T64x32x32_2x2,
- GemmTiling::T64x64x16_2x2,
- GemmTiling::T64x64x32_2x2,
- GemmTiling::T32x32x32_2x2,
- GemmTiling::T32x64x32_2x2,
- GemmTiling::T64x32x32_4x1,
- GemmTiling::T128x128x32_4x4)
+ GemmTiling::Tile8x32x32_Simdgroups1x1,
+ GemmTiling::Tile64x32x32_Simdgroups2x2,
+ GemmTiling::Tile64x64x16_Simdgroups2x2,
+ GemmTiling::Tile64x64x32_Simdgroups2x2,
+ GemmTiling::Tile32x32x32_Simdgroups2x2,
+ GemmTiling::Tile32x64x256_Simdgroups2x2,
+ GemmTiling::Tile64x32x256_Simdgroups4x1,
+ GemmTiling::Tile64x64x256_Simdgroups2x2,
+ GemmTiling::Tile128x128x256_Simdgroups4x4)
VARIANTS(TRANSPOSE_B, false, true)
VARIANTS(USE_MXU, false, true)
VARIANTS(
- WEIGHT_PROLOGUE,
- GemmWeightPrologueKind::FullPrecision,
- GemmWeightPrologueKind::ScaleBiasDequant,
- GemmWeightPrologueKind::ScaleZeroPointDequant,
- GemmWeightPrologueKind::ScaleSymmetricDequant)
+ B_PROLOGUE,
+ GemmBPrologueKind::FullPrecision,
+ GemmBPrologueKind::ScaleBiasDequant,
+ GemmBPrologueKind::ScaleZeroPointDequant,
+ GemmBPrologueKind::ScaleSymmetricDequant)
VARIANTS(BITS, 0, 4, 8)
VARIANTS(GROUP_SIZE, 0, 16, 32, 64, 128)
CONSTRAINT(
- !USE_MXU || GEMM_TILING == GemmTiling::T64x64x32_2x2 ||
- GEMM_TILING == GemmTiling::T32x64x32_2x2 ||
- GEMM_TILING == GemmTiling::T64x32x32_4x1 ||
- GEMM_TILING == GemmTiling::T128x128x32_4x4)
-CONSTRAINT(
- WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision || USE_MXU ||
- GEMM_TILING == GemmTiling::T64x64x16_2x2 ||
- GEMM_TILING == GemmTiling::T64x32x32_2x2)
-CONSTRAINT(
- TRANSPOSE_B ||
- (WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision &&
- ((!USE_MXU &&
- (GEMM_TILING == GemmTiling::T64x64x16_2x2 ||
- GEMM_TILING == GemmTiling::T64x32x32_2x2)) ||
- (USE_MXU &&
- (GEMM_TILING == GemmTiling::T32x64x32_2x2 ||
- GEMM_TILING == GemmTiling::T64x64x32_2x2 ||
- GEMM_TILING == GemmTiling::T128x128x32_4x4)))))
-CONSTRAINT((WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision) == (BITS == 0))
+ USE_MXU ==
+ (GEMM_TILING == GemmTiling::Tile32x64x256_Simdgroups2x2 ||
+ GEMM_TILING == GemmTiling::Tile64x32x256_Simdgroups4x1 ||
+ GEMM_TILING == GemmTiling::Tile64x64x256_Simdgroups2x2 ||
+ GEMM_TILING == GemmTiling::Tile128x128x256_Simdgroups4x4))
+CONSTRAINT((B_PROLOGUE == GemmBPrologueKind::FullPrecision) == (BITS == 0))
CONSTRAINT((BITS == 0) == (GROUP_SIZE == 0))
-CONSTRAINT(WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision || BT != "float")
+CONSTRAINT(B_PROLOGUE == GemmBPrologueKind::FullPrecision || BT != "float")
+CONSTRAINT(
+ GROUP_SIZE != 16 ||
+ GEMM_TILING == GemmTiling::Tile64x64x16_Simdgroups2x2)
CONSTRAINT(
- WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||
- (!USE_MXU && TRANSPOSE_B &&
- (GEMM_TILING == GemmTiling::T8x32x32_1x1 ||
- GEMM_TILING == GemmTiling::T64x64x16_2x2 ||
- GEMM_TILING == GemmTiling::T32x32x32_2x2 ||
- GEMM_TILING == GemmTiling::T64x32x32_2x2 ||
- GEMM_TILING == GemmTiling::T64x64x32_2x2)))
+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
+ (TRANSPOSE_B &&
+ (GEMM_TILING != GemmTiling::Tile64x64x16_Simdgroups2x2 ||
+ GROUP_SIZE == 16)))
CONSTRAINT(
- WEIGHT_PROLOGUE == GemmWeightPrologueKind::FullPrecision ||
- GROUP_SIZE != 16 || GEMM_TILING == GemmTiling::T64x64x16_2x2)
+ B_PROLOGUE == GemmBPrologueKind::FullPrecision ||
+ GEMM_TILING != GemmTiling::Tile128x128x256_Simdgroups4x4 ||
+ GROUP_SIZE <= 64)
KERNEL(Gemm)(
const device AT* a,
- const device uint8_t* b_packed,
+ const device BT* b,
device DT* d,
const device BT* scales
- OPTIONAL(WEIGHT_PROLOGUE != GemmWeightPrologueKind::FullPrecision),
+ OPTIONAL(B_PROLOGUE != GemmBPrologueKind::FullPrecision),
const device BT* biases
- OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleBiasDequant),
+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleBiasDequant),
const device uint8_t* zero_points
- OPTIONAL(WEIGHT_PROLOGUE == GemmWeightPrologueKind::ScaleZeroPointDequant),
+ OPTIONAL(B_PROLOGUE == GemmBPrologueKind::ScaleZeroPointDequant),
const device BT* output_bias
OPTIONAL(output_transform.contains(GemmDTransform::BIAS)),
+ const device int32_t* rht_factors
+ OPTIONAL(output_transform.contains(GemmDTransform::RHT)),
const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
@@ -119,20 +114,28 @@ KERNEL(Gemm)(
(void)thread_z;
if constexpr (USE_MXU) {
- (void)scales;
- (void)biases;
- (void)zero_points;
- const device BT* b = reinterpret_cast<const device BT*>(b_packed);
- MxuMmaCore<AT, BT, DT, GEMM_TILING, TRANSPOSE_B>::run(
- a,
- b,
- d,
- params,
- alignment,
- output_transform,
- output_bias,
- thread_context
- );
+ MxuMmaCore<
+ AT,
+ BT,
+ DT,
+ GEMM_TILING,
+ TRANSPOSE_B,
+ B_PROLOGUE,
+ BITS,
+ GROUP_SIZE>::
+ run(a,
+ b,
+ d,
+ params,
+ alignment,
+ output_transform,
+ scales,
+ biases,
+ zero_points,
+ output_bias,
+ rht_factors,
+ b_shared,
+ thread_context);
} else {
SimdgroupMmaCore<
AT,
@@ -140,11 +143,11 @@ KERNEL(Gemm)(
DT,
GEMM_TILING,
TRANSPOSE_B,
- WEIGHT_PROLOGUE,
+ B_PROLOGUE,
BITS,
GROUP_SIZE>::
run(a,
- b_packed,
+ b,
d,
params,
alignment,
@@ -153,6 +156,7 @@ KERNEL(Gemm)(
biases,
zero_points,
output_bias,
+ rht_factors,
a_shared,
b_shared,
thread_context);
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
index 6b52829bd..0a19a29a2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/kernel.rs
@@ -1,6 +1,6 @@
use std::collections::{HashMap, hash_map::Entry};
-use super::{MXU_THREADGROUP_BLOCK_K, specialization::GemmSpecialization};
+use super::specialization::GemmSpecialization;
use crate::{
backends::{
common::{
@@ -94,8 +94,8 @@ impl GemmKernel {
specialization.tiling,
specialization.transpose_b,
specialization.use_mxu,
- specialization.weight_prologue,
- specialization.bits_per_weight.unwrap_or(0),
+ specialization.b_prologue,
+ specialization.bits_per_b.unwrap_or(0),
specialization.group_size.unwrap_or(0),
specialization.output_transform,
specialization.alignment,
@@ -110,11 +110,30 @@ impl GemmKernel {
arguments: MatmulArguments<'a, Metal, TB>,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
+ let mxu_eligible_for_quant = match &arguments.b {
+ MatmulB::FullPrecision {
+ ..
+ } => true,
+ MatmulB::ScaleBiasDequant {
+ ..
+ }
+ | MatmulB::ScaleZeroPointDequant {
+ ..
+ }
+ | MatmulB::ScaleSymmetricDequant {
+ ..
+ } => {
+ arguments.b_transpose
+ && arguments.b_leading_dimension.is_none_or(|ld| ld == arguments.k)
+ && arguments.b_offset == 0
+ && arguments.k % select_mxu_tiling(arguments.m, arguments.n).block_k() == 0
+ },
+ };
let path = if encoder.context().device.supports_mxu()
&& [self.weights_data_type, self.input_data_type, self.output_data_type]
.into_iter()
.all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32))
- && matches!(arguments.b, MatmulB::FullPrecision { .. })
+ && mxu_eligible_for_quant
{
GemmDispatchPath::Mxu
} else {
@@ -129,29 +148,21 @@ impl GemmKernel {
path: GemmDispatchPath,
encoder: &mut Encoder<Metal>,
) -> Result<(), MetalError> {
- let use_mxu = match path {
- GemmDispatchPath::Mxu => {
- assert!(
- encoder.context().device.supports_mxu(),
- "GemmDispatchPath::Mxu requested on hardware without MXU support",
- );
- assert!(
- [self.weights_data_type, self.input_data_type, self.output_data_type]
- .into_iter()
- .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32)),
- "GemmDispatchPath::Mxu requires BF16 or F32 data types, got weights {:?}, input {:?}, output {:?}",
- self.weights_data_type,
- self.input_data_type,
- self.output_data_type,
- );
- assert!(
- matches!(arguments.b, MatmulB::FullPrecision { .. }),
- "GemmDispatchPath::Mxu requires FullPrecision B",
- );
- true
- },
- GemmDispatchPath::Simdgroup => false,
- };
+ if matches!(path, GemmDispatchPath::Mxu) {
+ assert!(
+ encoder.context().device.supports_mxu(),
+ "GemmDispatchPath::Mxu requested on hardware without MXU support",
+ );
+ assert!(
+ [self.weights_data_type, self.input_data_type, self.output_data_type]
+ .into_iter()
+ .all(|data_type| matches!(data_type, DataType::BF16 | DataType::F32)),
+ "GemmDispatchPath::Mxu requires BF16 or F32 data types, got weights {:?}, input {:?}, output {:?}",
+ self.weights_data_type,
+ self.input_data_type,
+ self.output_data_type,
+ );
+ }
let is_quant = !matches!(arguments.b, MatmulB::FullPrecision { .. });
if is_quant {
@@ -176,25 +187,12 @@ impl GemmKernel {
}
let ab_scale = arguments.d_transform.ab_scale;
- let post_bias = if use_mxu {
- arguments.d_transform.bias
- } else {
- None
- };
- let output_bias = if use_mxu {
- None
- } else {
- arguments.d_transform.bias
- };
- let post_rht = arguments.d_transform.rht_factors;
- let mut output_transform = arguments.d_transform.mask();
- output_transform.remove(GemmDTransform::RHT);
- if use_mxu {
- output_transform.remove(GemmDTransform::BIAS);
- }
+ let output_bias = arguments.d_transform.bias;
+ let rht_factors = arguments.d_transform.rht_factors;
+ let output_transform = arguments.d_transform.mask();
- let weight_prologue = arguments.b.weight_prologue();
- let bits_per_weight = arguments.b.bits_per_weight();
+ let b_prologue = arguments.b.b_prologue();
+ let bits_per_b = arguments.b.bits_per_b();
let group_size = arguments.b.group_size();
let MatmulArguments {
@@ -211,6 +209,8 @@ impl GemmKernel {
..
} = arguments;
+ let use_mxu = matches!(path, GemmDispatchPath::Mxu);
+
match b {
MatmulB::FullPrecision {
b: weights,
@@ -220,11 +220,6 @@ impl GemmKernel {
} else {
select_simdgroup_tiling(m, n, k)
};
- let k_block = if use_mxu {
- MXU_THREADGROUP_BLOCK_K
- } else {
- tiling.block_k()
- };
let threadgroups_per_row = n.div_ceil(tiling.block_n());
let threadgroups_per_column = m.div_ceil(tiling.block_m());
@@ -246,7 +241,7 @@ impl GemmKernel {
};
let alignment =
- GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % k_block == 0);
+ GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
let default_ldb = if b_transpose {
k
@@ -262,7 +257,7 @@ impl GemmKernel {
leading_dimension_d: n,
threadgroups_per_row,
threadgroups_per_column,
- aligned_inner_iterations: k / k_block,
+ aligned_inner_iterations: k / tiling.block_k(),
use_morton,
ab_scale,
};
@@ -274,8 +269,8 @@ impl GemmKernel {
output_transform,
alignment,
transpose_b: b_transpose,
- weight_prologue,
- bits_per_weight,
+ b_prologue,
+ bits_per_b,
group_size,
};
specialization.validate()?;
@@ -288,6 +283,7 @@ impl GemmKernel {
None::<&Allocation<Metal>>,
None::<&Allocation<Metal>>,
output_bias,
+ rht_factors,
std::slice::from_ref(&params),
group_count_x,
group_count_y,
@@ -324,7 +320,11 @@ impl GemmKernel {
_ => unreachable!(),
};
- let tiling = select_quant_tiling(m, n, group_size.unwrap());
+ let tiling = if use_mxu {
+ select_mxu_quant_tiling(m, n, group_size.unwrap_or(0))
+ } else {
+ select_quant_tiling(m, n, group_size.unwrap_or(0))
+ };
let alignment =
GemmAlignment::new(m % tiling.block_m() == 0, n % tiling.block_n() == 0, k % tiling.block_k() == 0);
let params = quant_params(m, n, k, tiling, ab_scale);
@@ -338,8 +338,8 @@ impl GemmKernel {
output_transform,
alignment,
transpose_b: true,
- weight_prologue,
- bits_per_weight,
+ b_prologue,
+ bits_per_b,
group_size,
};
specialization.validate()?;
@@ -352,6 +352,7 @@ impl GemmKernel {
biases,
zero_points,
output_bias,
+ rht_factors,
std::slice::from_ref(&params),
group_count_x,
group_count_y,
@@ -360,12 +361,6 @@ impl GemmKernel {
},
}
- if let Some(bias) = post_bias {
- self.bias_add.encode(None::<&<Metal as Backend>::DenseBuffer>, bias, &mut *d, n, m * n, encoder);
- }
- if let Some(factors) = post_rht {
- self.hadamard.encode(d, factors, n, m, encoder);
- }
Ok(())
}
}
@@ -398,9 +393,9 @@ fn select_simdgroup_tiling(
k: u32,
) -> GemmTiling {
if 2 * m.max(n) > k {
- GemmTiling::T64x64x16_2x2
+ GemmTiling::Tile64x64x16_Simdgroups2x2
} else {
- GemmTiling::T64x32x32_2x2
+ GemmTiling::Tile64x32x32_Simdgroups2x2
}
}
@@ -409,13 +404,29 @@ fn select_mxu_tiling(
n: u32,
) -> GemmTiling {
if m >= 256 && n >= 128 {
- GemmTiling::T128x128x32_4x4
+ GemmTiling::Tile128x128x256_Simdgroups4x4
+ } else if n < 64 {
+ GemmTiling::Tile64x32x256_Simdgroups4x1
+ } else if m < 64 {
+ GemmTiling::Tile32x64x256_Simdgroups2x2
+ } else {
+ GemmTiling::Tile64x64x256_Simdgroups2x2
+ }
+}
+
+fn select_mxu_quant_tiling(
+ m: u32,
+ n: u32,
+ group_size: u32,
+) -> GemmTiling {
+ if m >= 256 && n >= 128 && GemmTiling::Tile128x128x256_Simdgroups4x4.fits_quant_group_size(group_size) {
+ GemmTiling::Tile128x128x256_Simdgroups4x4
} else if n < 64 {
- GemmTiling::T64x32x32_4x1
+ GemmTiling::Tile64x32x256_Simdgroups4x1
} else if m < 64 {
- GemmTiling::T32x64x32_2x2
+ GemmTiling::Tile32x64x256_Simdgroups2x2
} else {
- GemmTiling::T64x64x32_2x2
+ GemmTiling::Tile64x64x256_Simdgroups2x2
}
}
@@ -425,14 +436,14 @@ fn select_quant_tiling(
group_size: u32,
) -> GemmTiling {
if group_size < 32 {
- GemmTiling::T64x64x16_2x2
+ GemmTiling::Tile64x64x16_Simdgroups2x2
} else if m < 32 {
- GemmTiling::T8x32x32_1x1
+ GemmTiling::Tile8x32x32_Simdgroups1x1
} else if m >= 64 && n <= 2048 {
- GemmTiling::T64x32x32_2x2
+ GemmTiling::Tile64x32x32_Simdgroups2x2
} else if m >= 64 && n >= 6144 && n % 64 == 0 {
- GemmTiling::T64x64x32_2x2
+ GemmTiling::Tile64x64x32_Simdgroups2x2
} else {
- GemmTiling::T32x32x32_2x2
+ GemmTiling::Tile32x32x32_Simdgroups2x2
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
index 05ecef5a5..3841e73d9 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/mod.rs
@@ -4,6 +4,3 @@ mod specialization;
pub use error::GemmSpecializationError;
pub use kernel::{GemmDispatchPath, GemmKernel};
-
-// Matches `MxuMmaCore::THREADGROUP_BLOCK_K` in `common/mxu_mma_core.h`.
-pub(crate) const MXU_THREADGROUP_BLOCK_K: u32 = 256;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
index 4e6d594c1..78cafb8cc 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm/specialization.rs
@@ -3,7 +3,7 @@ use crate::{
backends::common::{
gpu_types::{
QuantizationMethod,
- gemm::{GemmAlignment, GemmDTransform, GemmTiling, GemmWeightPrologueKind},
+ gemm::{GemmAlignment, GemmBPrologueKind, GemmDTransform, GemmTiling},
},
kernel::matmul::MatmulQuantCombo,
},
@@ -18,29 +18,43 @@ pub(crate) struct GemmSpecialization {
pub(crate) output_transform: GemmDTransform,
pub(crate) alignment: GemmAlignment,
pub(crate) transpose_b: bool,
- pub(crate) weight_prologue: GemmWeightPrologueKind,
- pub(crate) bits_per_weight: Option<u32>,
+ pub(crate) b_prologue: GemmBPrologueKind,
+ pub(crate) bits_per_b: Option<u32>,
pub(crate) group_size: Option<u32>,
}
impl GemmSpecialization {
pub(crate) fn validate(&self) -> Result<(), GemmSpecializationError> {
- if let Some(group_size) = self.group_size {
- if self.tiling.block_k() > group_size {
- return Err(GemmSpecializationError::ThreadgroupKExceedsGroupSize {
- threadgroup_k: self.tiling.block_k(),
- group_size,
- });
- }
+ if self.use_mxu != self.tiling.is_mxu_variant() {
+ return Err(GemmSpecializationError::TilingUseMxuMismatch {
+ tiling: self.tiling,
+ use_mxu: self.use_mxu,
+ });
}
- if self.weight_prologue != GemmWeightPrologueKind::FullPrecision {
- if self.use_mxu {
- return Err(GemmSpecializationError::QuantizedRequiresSimdgroup);
+ if self.use_mxu && self.b_prologue != GemmBPrologueKind::FullPrecision {
+ if let Some(group_size) = self.group_size {
+ if !self.tiling.fits_quant_group_size(group_size) {
+ return Err(GemmSpecializationError::MxuQuantTileTooLarge {
+ tiling: self.tiling,
+ group_size,
+ });
+ }
}
- if !self.transpose_b {
- return Err(GemmSpecializationError::QuantizedRequiresTransposedB);
+ }
+ if !self.use_mxu {
+ if let Some(group_size) = self.group_size {
+ let simdgroup_block_k = self.tiling.simdgroup_block_k();
+ if simdgroup_block_k > group_size {
+ return Err(GemmSpecializationError::SimdgroupKExceedsGroupSize {
+ simdgroup_k: simdgroup_block_k,
+ group_size,
+ });
+ }
}
}
+ if self.b_prologue != GemmBPrologueKind::FullPrecision && !self.transpose_b {
+ return Err(GemmSpecializationError::QuantizedRequiresTransposedB);
+ }
Ok(())
}
@@ -48,7 +62,12 @@ impl GemmSpecialization {
let mut out = Vec::new();
for &tiling in simdgroup_tiling_set(weights_data_type) {
for align_mn in [true, false] {
- for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {
+ for output_transform in [
+ GemmDTransform::empty(),
+ GemmDTransform::BIAS,
+ GemmDTransform::RHT,
+ GemmDTransform::BIAS | GemmDTransform::RHT,
+ ] {
out.push(Self {
weights_data_type,
tiling,
@@ -56,8 +75,8 @@ impl GemmSpecialization {
output_transform,
alignment: GemmAlignment::new(align_mn, align_mn, true),
transpose_b: true,
- weight_prologue: GemmWeightPrologueKind::FullPrecision,
- bits_per_weight: None,
+ b_prologue: GemmBPrologueKind::FullPrecision,
+ bits_per_b: None,
group_size: None,
});
}
@@ -73,6 +92,21 @@ impl GemmSpecialization {
GemmDTransform::SCALE,
GemmDTransform::ACCUMULATE,
GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,
+ GemmDTransform::BIAS,
+ GemmDTransform::BIAS | GemmDTransform::SCALE,
+ GemmDTransform::BIAS | GemmDTransform::ACCUMULATE,
+ GemmDTransform::BIAS | GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,
+ GemmDTransform::RHT,
+ GemmDTransform::RHT | GemmDTransform::SCALE,
+ GemmDTransform::RHT | GemmDTransform::ACCUMULATE,
+ GemmDTransform::RHT | GemmDTransform::SCALE | GemmDTransform::ACCUMULATE,
+ GemmDTransform::BIAS | GemmDTransform::RHT,
+ GemmDTransform::BIAS | GemmDTransform::RHT | GemmDTransform::SCALE,
+ GemmDTransform::BIAS | GemmDTransform::RHT | GemmDTransform::ACCUMULATE,
+ GemmDTransform::BIAS
+ | GemmDTransform::RHT
+ | GemmDTransform::SCALE
+ | GemmDTransform::ACCUMULATE,
] {
out.push(Self {
weights_data_type,
@@ -81,8 +115,8 @@ impl GemmSpecialization {
output_transform,
alignment: GemmAlignment::new(align_m, align_n, align_k),
transpose_b: true,
- weight_prologue: GemmWeightPrologueKind::FullPrecision,
- bits_per_weight: None,
+ b_prologue: GemmBPrologueKind::FullPrecision,
+ bits_per_b: None,
group_size: None,
});
}
@@ -99,18 +133,23 @@ impl GemmSpecialization {
) -> Vec<Self> {
let bits = DataType::from(combo.mode).size_in_bits() as u32;
let group_size = combo.group_size;
- let weight_prologue = match combo.method {
- QuantizationMethod::ScaleBias => GemmWeightPrologueKind::ScaleBiasDequant,
- QuantizationMethod::ScaleZeroPoint => GemmWeightPrologueKind::ScaleZeroPointDequant,
- QuantizationMethod::ScaleSymmetric => GemmWeightPrologueKind::ScaleSymmetricDequant,
+ let b_prologue = match combo.method {
+ QuantizationMethod::ScaleBias => GemmBPrologueKind::ScaleBiasDequant,
+ QuantizationMethod::ScaleZeroPoint => GemmBPrologueKind::ScaleZeroPointDequant,
+ QuantizationMethod::ScaleSymmetric => GemmBPrologueKind::ScaleSymmetricDequant,
};
let mut out = Vec::new();
for &tiling in quant_tiling_set(weights_data_type) {
- if tiling.block_k() > group_size {
+ if tiling.simdgroup_block_k() > group_size {
continue;
}
for align_n in [true, false] {
- for output_transform in [GemmDTransform::empty(), GemmDTransform::BIAS] {
+ for output_transform in [
+ GemmDTransform::empty(),
+ GemmDTransform::BIAS,
+ GemmDTransform::RHT,
+ GemmDTransform::BIAS | GemmDTransform::RHT,
+ ] {
out.push(Self {
weights_data_type,
tiling,
@@ -118,20 +157,54 @@ impl GemmSpecialization {
output_transform,
alignment: GemmAlignment::new(true, align_n, true),
transpose_b: true,
- weight_prologue,
- bits_per_weight: Some(bits),
+ b_prologue,
+ bits_per_b: Some(bits),
group_size: Some(group_size),
});
}
}
}
+ for &tiling in mxu_tiling_set(weights_data_type) {
+ if group_size % 32 != 0 {
+ continue;
+ }
+ if !tiling.fits_quant_group_size(group_size) {
+ continue;
+ }
+ for align_m in [true, false] {
+ for align_n in [true, false] {
+ for output_transform in [
+ GemmDTransform::empty(),
+ GemmDTransform::SCALE,
+ GemmDTransform::BIAS,
+ GemmDTransform::RHT,
+ GemmDTransform::BIAS | GemmDTransform::RHT,
+ GemmDTransform::BIAS | GemmDTransform::SCALE,
+ ] {
+ out.push(Self {
+ weights_data_type,
+ tiling,
+ use_mxu: true,
+ output_transform,
+ alignment: GemmAlignment::new(align_m, align_n, true),
+ transpose_b: true,
+ b_prologue,
+ bits_per_b: Some(bits),
+ group_size: Some(group_size),
+ });
+ }
+ }
+ }
+ }
out
}
}
fn simdgroup_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
match data_type {
- DataType::BF16 | DataType::F32 => &[GemmTiling::T64x32x32_2x2, GemmTiling::T64x64x16_2x2],
+ DataType::BF16 | DataType::F32 => {
+ &[GemmTiling::Tile64x32x32_Simdgroups2x2, GemmTiling::Tile64x64x16_Simdgroups2x2]
+ },
_ => &[],
}
}
@@ -140,17 +213,21 @@ fn mxu_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
if !matches!(data_type, DataType::BF16 | DataType::F32) {
return &[];
}
- &[GemmTiling::T64x64x32_2x2, GemmTiling::T32x64x32_2x2, GemmTiling::T64x32x32_4x1, GemmTiling::T128x128x32_4x4]
+ &[
+ GemmTiling::Tile64x64x256_Simdgroups2x2,
+ GemmTiling::Tile32x64x256_Simdgroups2x2,
+ GemmTiling::Tile64x32x256_Simdgroups4x1,
+ GemmTiling::Tile128x128x256_Simdgroups4x4,
+ ]
}
pub(crate) fn quant_tiling_set(data_type: DataType) -> &'static [GemmTiling] {
match data_type {
DataType::BF16 => &[
- GemmTiling::T64x64x16_2x2,
- GemmTiling::T8x32x32_1x1,
- GemmTiling::T32x32x32_2x2,
- GemmTiling::T64x32x32_2x2,
- GemmTiling::T64x64x32_2x2,
+ GemmTiling::Tile8x32x32_Simdgroups1x1,
+ GemmTiling::Tile32x32x32_Simdgroups2x2,
+ GemmTiling::Tile64x32x32_Simdgroups2x2,
+ GemmTiling::Tile64x64x32_Simdgroups2x2,
],
_ => &[],
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
index ca75e02b7..d396315d2 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/mod.rs
@@ -4,4 +4,4 @@ mod quant_kernel;
mod spec;
pub(crate) use kernel::GemvKernel;
-pub(crate) use quant_kernel::QuantGemvKernel;
+pub use quant_kernel::QuantGemvKernel;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
index daa728584..7fbc08019 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv.metal
@@ -18,7 +18,7 @@ VARIANTS(InputT, float, bfloat)
VARIANTS(OutputT, float, bfloat)
VARIANTS(GROUP_SIZE, 16, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-PUBLIC KERNEL(QuantizedMatmulQmv)(
+KERNEL(QuantizedMatmulQmv)(
const device uint32_t* weights,
const device WeightT* scales,
const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
index 0ce1b822f..8fe649a8a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/qmv_fast.metal
@@ -19,7 +19,7 @@ VARIANTS(InputT, float, bfloat)
VARIANTS(OutputT, float, bfloat)
VARIANTS(GROUP_SIZE, 16, 32, 64, 128)
VARIANTS(BITS, 4, 8)
-PUBLIC KERNEL(QuantizedMatmulQmvFast)(
+KERNEL(QuantizedMatmulQmvFast)(
const device uint32_t* weights,
const device WeightT* scales,
const device uint8_t* zero_points OPTIONAL(quant_method == QuantizationMethod::ScaleZeroPoint),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
index c1e769f08..9e456cd82 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemv/quant_kernel.rs
@@ -5,12 +5,13 @@ use crate::{
common::{
AsBufferRangeRef, Buffer, Encoder,
gpu_types::{QuantizationMethod, QuantizationMode, gemm::GemmDTransform},
- kernel::{
- Kernels, QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel,
- matmul::{MatmulArguments, MatmulB, MatmulError},
- },
+ kernel::matmul::{MatmulArguments, MatmulB, MatmulError},
+ },
+ metal::{
+ Metal,
+ context::MetalContext,
+ kernel::{QuantizedMatmulQmvFastMetalKernel, QuantizedMatmulQmvMetalKernel},
},
- metal::{Metal, context::MetalContext},
},
data_type::DataType,
};
@@ -25,19 +26,16 @@ struct QmvKey {
use_hadamard: bool,
}
-pub(crate) struct QuantGemvKernel {
+pub struct QuantGemvKernel {
weights_data_type: DataType,
input_data_type: DataType,
output_data_type: DataType,
- qmv: HashMap<QmvKey, <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel>,
- qmv_fast: HashMap<
- QmvKey,
- <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel,
- >,
+ qmv: HashMap<QmvKey, QuantizedMatmulQmvMetalKernel>,
+ qmv_fast: HashMap<QmvKey, QuantizedMatmulQmvFastMetalKernel>,
}
impl QuantGemvKernel {
- pub(crate) fn new(
+ pub fn new(
_context: &MetalContext,
weights_data_type: DataType,
input_data_type: DataType,
@@ -52,7 +50,7 @@ impl QuantGemvKernel {
}
}
- pub(crate) fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
+ pub fn encode<'a, TB: AsBufferRangeRef<Buffer: Buffer<Backend = Metal>>>(
&mut self,
encoder: &mut Encoder<Metal>,
arguments: MatmulArguments<'a, Metal, TB>,
@@ -141,7 +139,7 @@ impl QuantGemvKernel {
let kernel = match self.qmv_fast.entry(key) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
- let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ let kernel = QuantizedMatmulQmvFastMetalKernel::new(
context,
self.weights_data_type,
self.input_data_type,
@@ -173,7 +171,7 @@ impl QuantGemvKernel {
let kernel = match self.qmv.entry(key) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
- let kernel = <<Metal as crate::backends::common::Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
+ let kernel = QuantizedMatmulQmvMetalKernel::new(
context,
self.weights_data_type,
self.input_data_type,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index 562a76ab3..982c80ca7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -3,8 +3,11 @@ pub mod gemv;
use std::sync::OnceLock;
-pub use self::gemm::{GemmDispatchPath, GemmKernel};
-use self::gemv::{GemvKernel, QuantGemvKernel};
+use self::gemv::GemvKernel;
+pub use self::{
+ gemm::{GemmDispatchPath, GemmKernel},
+ gemv::QuantGemvKernel,
+};
use crate::{
backends::{
common::{
@@ -22,7 +25,7 @@ use crate::{
pub struct MatmulMetalKernel {
gemv: GemvKernel,
- quant_gemv: QuantGemvKernel,
+ pub quant_gemv: QuantGemvKernel,
pub gemm: GemmKernel,
}
diff --git a/crates/backend-uzu/tests/common/matmul/harness.rs b/crates/backend-uzu/tests/common/matmul/harness.rs
index 899bccb05..512debeee 100644
--- a/crates/backend-uzu/tests/common/matmul/harness.rs
+++ b/crates/backend-uzu/tests/common/matmul/harness.rs
@@ -29,6 +29,8 @@ pub struct Case {
pub ab_scale: f32,
pub accumulate: bool,
pub b_transpose: bool,
+ pub enable_rht: bool,
+ pub enable_bias: bool,
}
impl Case {
@@ -38,6 +40,8 @@ impl Case {
ab_scale: 1.0,
accumulate: false,
b_transpose: true,
+ enable_rht: false,
+ enable_bias: false,
}
}
@@ -56,12 +60,30 @@ impl Case {
self.accumulate = accumulate;
self
}
+
+ pub const fn with_rht(
+ mut self,
+ enable_rht: bool,
+ ) -> Self {
+ self.enable_rht = enable_rht;
+ self
+ }
+
+ pub const fn with_bias(
+ mut self,
+ enable_bias: bool,
+ ) -> Self {
+ self.enable_bias = enable_bias;
+ self
+ }
}
pub struct Input<T: ArrayElement + Float> {
pub a: Box<[T]>,
pub b: Box<[T]>,
pub d_prefill: Option<Box<[T]>>,
+ pub rht_factors: Option<Box<[i32]>>,
+ pub bias: Option<Box<[T]>>,
pub case: Case,
}
@@ -76,10 +98,27 @@ pub fn deterministic_input<T: ArrayElement + Float>(case: Case) -> Input<T> {
let d_prefill = case.accumulate.then(|| {
(0..m * n).map(|i| T::from(((i % 7) as f32) * 0.03 - 0.09).unwrap()).collect::<Vec<_>>().into_boxed_slice()
});
+ let rht_factors = (case.enable_rht && n % 32 == 0).then(|| {
+ (0..n)
+ .map(|i| {
+ if (i % 2) == 0 {
+ 1i32
+ } else {
+ -1i32
+ }
+ })
+ .collect::<Vec<_>>()
+ .into_boxed_slice()
+ });
+ let bias = case.enable_bias.then(|| {
+ (0..n).map(|i| T::from(((i % 11) as f32) * 0.05 - 0.25).unwrap()).collect::<Vec<_>>().into_boxed_slice()
+ });
Input {
a: a.into_boxed_slice(),
b: b.into_boxed_slice(),
d_prefill,
+ rht_factors,
+ bias,
case,
}
}
@@ -108,12 +147,15 @@ fn run<B: Backend, T: ArrayElement + Float>(
.create_allocation(m * n * std::mem::size_of::<T>(), AllocationType::Global)
.expect("create d allocation")
};
+ let rht_allocation =
+ input.rht_factors.as_ref().map(|factors| alloc_allocation_with_data::<B, i32>(context, factors));
+ let bias_allocation = input.bias.as_ref().map(|bias| alloc_allocation_with_data::<B, T>(context, bias));
let d_transform = MatmulDOps::<'_, B> {
ab_scale: input.case.ab_scale,
accumulate: input.case.accumulate,
- bias: None,
- rht_factors: None,
+ bias: bias_allocation.as_ref(),
+ rht_factors: rht_allocation.as_ref(),
};
let mut encoder = Encoder::new(context).expect("encoder");
diff --git a/crates/backend-uzu/tests/common/mod.rs b/crates/backend-uzu/tests/common/mod.rs
index c9a00c538..30a73dfe6 100644
--- a/crates/backend-uzu/tests/common/mod.rs
+++ b/crates/backend-uzu/tests/common/mod.rs
@@ -14,6 +14,15 @@ pub fn type_short_name<T>() -> &'static str {
std::any::type_name::<T>().rsplit("::").next().unwrap()
}
+#[cfg(metal_backend)]
+pub fn shared_metal_context() -> std::rc::Rc<backend_uzu::backends::metal::MetalContext> {
+ use backend_uzu::backends::{common::Context, metal::MetalContext};
+ thread_local! {
+ static CTX: std::cell::OnceCell<std::rc::Rc<MetalContext>> = const { std::cell::OnceCell::new() };
+ }
+ CTX.with(|cell| cell.get_or_init(|| MetalContext::new().expect("Metal context")).clone())
+}
+
/// Invokes `$body` once per available backend, with `$B` bound to each backend type.
macro_rules! for_each_backend {
(|$B:ident| $body:expr) => {{
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
index 5251fa6c4..23c79c7a5 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/gemm_bench.rs
@@ -10,7 +10,7 @@ use backend_uzu::{
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel},
},
},
- metal::{GemmDispatchPath, Metal, MetalContext},
+ metal::{GemmDispatchPath, Metal},
},
};
use criterion::{BenchmarkId, Criterion, Throughput};
@@ -26,7 +26,7 @@ use crate::{
#[uzu_bench]
fn bench_gemm(c: &mut Criterion) {
- let context = MetalContext::new().expect("Metal context");
+ let context = crate::common::shared_metal_context();
let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
&context,
bf16::data_type(),
@@ -35,44 +35,52 @@ fn bench_gemm(c: &mut Criterion) {
)
.expect("MatmulKernel");
- let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));
+ let paths: &[(&str, GemmDispatchPath)] = if context.supports_mxu() {
+ &[("GEMM", GemmDispatchPath::Simdgroup), ("GEMM_MXU", GemmDispatchPath::Mxu)]
+ } else {
+ &[("GEMM", GemmDispatchPath::Simdgroup)]
+ };
- for shape in bench_fp_gemm_shapes() {
- let (m, k, n) = (shape.m, shape.k, shape.n);
- let a = context
- .create_allocation(m * k * std::mem::size_of::<bf16>(), AllocationType::Global)
- .expect("a allocation");
- let b_array = context.create_array_uninitialized(&[n, k], bf16::data_type());
- let mut d = context
- .create_allocation(m * n * std::mem::size_of::<bf16>(), AllocationType::Global)
- .expect("d allocation");
+ for &(group_label, path) in paths {
+ let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/{}", type_short_name::<Metal>(), group_label));
- group.throughput(Throughput::Elements((2 * m * k * n) as u64));
- group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {
- iter_encode_loop::<Metal, _>(&context, b, |encoder| {
- kernel
- .gemm
- .encode_dispatch_path(
- MatmulArguments {
- a: &a,
- a_offset: 0,
- b: MatmulB::FullPrecision {
- b: b_array.allocation(),
+ for shape in bench_fp_gemm_shapes() {
+ let (m, k, n) = (shape.m, shape.k, shape.n);
+ let a = context
+ .create_allocation(m * k * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("a allocation");
+ let b_array = context.create_array_uninitialized(&[n, k], bf16::data_type());
+ let mut d = context
+ .create_allocation(m * n * std::mem::size_of::<bf16>(), AllocationType::Global)
+ .expect("d allocation");
+
+ group.throughput(Throughput::Elements((2 * m * k * n) as u64));
+ group.bench_function(BenchmarkId::new("BF16", shape.to_string()), |b| {
+ iter_encode_loop::<Metal, _>(&context, b, |encoder| {
+ kernel
+ .gemm
+ .encode_dispatch_path(
+ MatmulArguments {
+ a: &a,
+ a_offset: 0,
+ b: MatmulB::FullPrecision {
+ b: b_array.allocation(),
+ },
+ b_offset: 0,
+ b_leading_dimension: None,
+ b_transpose: true,
+ d: &mut d,
+ d_transform: MatmulDOps::none(),
+ m: m as u32,
+ n: n as u32,
+ k: k as u32,
},
- b_offset: 0,
- b_leading_dimension: None,
- b_transpose: true,
- d: &mut d,
- d_transform: MatmulDOps::none(),
- m: m as u32,
- n: n as u32,
- k: k as u32,
- },
- GemmDispatchPath::Mxu,
- encoder,
- )
- .expect("encode_dispatch_path failed");
+ path,
+ encoder,
+ )
+ .expect("encode_dispatch_path failed");
+ });
});
- });
+ }
}
}
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
index 5b3a6f52f..0a6a1550f 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemm_bench.rs
@@ -4,7 +4,7 @@ use backend_uzu::{
array::ArrayElement,
backends::{
common::{
- Backend, Context,
+ Backend,
gpu_types::QuantizationMethod,
kernel::{Kernels, matmul::MatmulKernel},
},
@@ -31,6 +31,13 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>(
bits: u32,
quant_method: QuantizationMethod,
) {
+ let supports_mxu = context.supports_mxu();
+ let paths: &[(&str, GemmDispatchPath)] = if supports_mxu {
+ &[("Simdgroup", GemmDispatchPath::Simdgroup), ("Mxu", GemmDispatchPath::Mxu)]
+ } else {
+ &[("Simdgroup", GemmDispatchPath::Simdgroup)]
+ };
+
for shape in bench_quant_gemm_shapes(bits) {
let (m, k, n) = (shape.m, shape.k, shape.n);
let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
@@ -43,24 +50,30 @@ fn bench_unified_quant_typed<T: ArrayElement + Float>(
)
.unwrap();
- let mut group =
- c.benchmark_group(format!("{}/Kernel/UnifiedQuantizedGemm/{}", type_short_name::<Metal>(), label));
- group.throughput(Throughput::Elements((m * n * k) as u64));
- group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {
- iter_encode_loop::<Metal, _>(context, b, |encoder| {
- matmul
- .gemm
- .encode_dispatch_path(quant_arguments(&mut buffers, &input), GemmDispatchPath::Simdgroup, encoder)
- .expect("encode unified quant matmul");
+ for &(path_label, path) in paths {
+ let mut group = c.benchmark_group(format!(
+ "{}/Kernel/UnifiedQuantizedGemm/{}/{}",
+ type_short_name::<Metal>(),
+ path_label,
+ label
+ ));
+ group.throughput(Throughput::Elements((m * n * k) as u64));
+ group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {
+ iter_encode_loop::<Metal, _>(context, b, |encoder| {
+ matmul
+ .gemm
+ .encode_dispatch_path(quant_arguments(&mut buffers, &input), path, encoder)
+ .expect("encode unified quant matmul");
+ });
});
- });
- drop(group);
+ drop(group);
+ }
}
}
#[uzu_bench]
fn bench_unified_quantized_gemm(c: &mut Criterion) {
- let context = MetalContext::new().expect("Metal context");
+ let context = crate::common::shared_metal_context();
bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
bench_unified_quant_typed::<bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
bench_unified_quant_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
index 3111d9687..85b56a68b 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/quant_gemv_bench.rs
@@ -1,7 +1,14 @@
+#![cfg(metal_backend)]
+
use backend_uzu::{
array::ArrayElement,
- backends::common::{
- Allocation, Backend, Context, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvFastKernel,
+ backends::{
+ common::{
+ Backend,
+ gpu_types::QuantizationMethod,
+ kernel::{Kernels, matmul::MatmulKernel},
+ },
+ metal::{Metal, MetalContext},
},
};
use criterion::{BenchmarkId, Criterion, Throughput};
@@ -10,70 +17,48 @@ use num_traits::Float;
use crate::{
common::{
- matmul::{QuantBuffers, QuantInput, bench_quant_gemv_shapes, iter_encode_loop},
+ matmul::{QuantBuffers, QuantInput, bench_quant_gemv_shapes, iter_encode_loop, quant_arguments},
type_short_name,
},
uzu_bench,
};
-fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
+fn bench_quant_gemv_typed<T: ArrayElement + Float>(
+ context: &MetalContext,
c: &mut Criterion,
- context: &B::Context,
label: &str,
group_size: u32,
bits: u32,
quant_method: QuantizationMethod,
) {
- let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));
+ let mut group = c.benchmark_group(format!("{}/Kernel/Matmul/QuantGemv/{}", type_short_name::<Metal>(), label));
for shape in bench_quant_gemv_shapes(bits) {
let (m, k, n) = (shape.m, shape.k, shape.n);
let input = QuantInput::<T>::new(m, k, n, group_size, bits, quant_method, 42);
- let mut buffers = QuantBuffers::<B, T>::allocate(context, &input);
-
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
+ let mut buffers = QuantBuffers::<Metal, T>::allocate(context, &input);
+ let mut matmul = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
context,
T::data_type(),
T::data_type(),
T::data_type(),
- group_size,
- bits,
- quant_method,
- false,
)
.unwrap();
group.throughput(Throughput::Elements((m * n * k) as u64));
group.bench_function(BenchmarkId::from_parameter(shape.to_string()), |b| {
- iter_encode_loop::<B, _>(context, b, |encoder| {
- kernel.encode(
- &buffers.w,
- &buffers.scales,
- buffers.zp.as_ref(),
- buffers.bias.as_ref(),
- &buffers.x,
- &mut buffers.y,
- None::<&Allocation<B>>,
- k as u32,
- n as u32,
- m as u32,
- encoder,
- );
+ iter_encode_loop::<Metal, _>(context, b, |encoder| {
+ matmul.quant_gemv.encode(encoder, quant_arguments(&mut buffers, &input)).expect("encode quant gemv");
});
});
}
}
#[uzu_bench]
-fn bench_qmv_fast(c: &mut Criterion) {
- for_each_backend!(|B| {
- let context = <B as Backend>::Context::new().unwrap();
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs32", 32, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs32", 32, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
- bench_qmv_fast_typed::<B, bf16>(c, &context, "ZP_BF16_gs64_8b", 64, 8, QuantizationMethod::ScaleZeroPoint);
- });
+fn bench_quant_gemv(c: &mut Criterion) {
+ let context = crate::common::shared_metal_context();
+ bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs64", 64, 4, QuantizationMethod::ScaleBias);
+ bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs64", 64, 4, QuantizationMethod::ScaleZeroPoint);
+ bench_quant_gemv_typed::<bf16>(&context, c, "ScaleBias_BF16_gs128", 128, 4, QuantizationMethod::ScaleBias);
+ bench_quant_gemv_typed::<bf16>(&context, c, "ZP_BF16_gs128", 128, 4, QuantizationMethod::ScaleZeroPoint);
}
diff --git a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
index dacfa71fe..e80110a40 100644
--- a/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
+++ b/crates/backend-uzu/tests/performance/kernel/matmul/qwen3_bench.rs
@@ -4,7 +4,7 @@ use backend_uzu::{
array::ArrayElement,
backends::{
common::{
- Backend, Context,
+ Backend,
gpu_types::QuantizationMethod,
kernel::{Kernels, matmul::MatmulKernel},
},
@@ -55,7 +55,7 @@ fn bench_qwen3_layers_typed<T: ArrayElement + Float>(
#[uzu_bench]
fn bench_qwen3_layers(c: &mut Criterion) {
- let context = MetalContext::new().expect("Metal context");
+ let context = crate::common::shared_metal_context();
bench_qwen3_layers_typed::<bf16>(c, &context, "ScaleBias_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleBias);
bench_qwen3_layers_typed::<bf16>(c, &context, "ZP_BF16_gs128_4b", 128, 4, QuantizationMethod::ScaleZeroPoint);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
index 1283c48b4..0a147e02d 100644
--- a/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/hadamard_transform_test.rs
@@ -6,7 +6,7 @@ use backend_uzu::{
Backend, Context, Encoder, Kernels, gpu_types::HadamardTransformOrder, kernel::HadamardTransformKernel,
},
};
-use half::{bf16, f16};
+use half::bf16;
use num_traits::Float;
use crate::uzu_test;
@@ -154,11 +154,6 @@ fn test_f32() {
test_hadamard_transform::<f32>(1e-4);
}
-#[uzu_test]
-fn test_f16() {
- test_hadamard_transform::<f16>(0.05);
-}
-
#[uzu_test]
fn test_bf16() {
test_hadamard_transform::<bf16>(0.1);
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
index 8dedff722..d62f0fed8 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/dispatch_paths_test.rs
@@ -89,3 +89,46 @@ fn b_transpose_false_bf16() {
1.0,
);
}
+
+fn rht_shapes() -> impl Iterator<Item = crate::common::matmul::Shape> {
+ use crate::common::matmul::Shape;
+ [Shape::new(8, 128, 64), Shape::new(64, 128, 128), Shape::new(128, 2048, 256), Shape::new(33, 128, 64)].into_iter()
+}
+
+#[test]
+fn rht_parity_bf16() {
+ let context = MetalContext::new().expect("Metal context");
+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulKernel");
+ for path in gemm_paths_for_hw(&context) {
+ for shape in rht_shapes() {
+ let case = Case::new(shape).with_rht(true);
+ check_case::<bf16>(&context, &mut kernel, Some(path), case, 1.0);
+ }
+ }
+}
+
+#[test]
+fn bias_parity_bf16() {
+ use crate::common::matmul::Shape;
+ let context = MetalContext::new().expect("Metal context");
+ let mut kernel = <<Metal as Backend>::Kernels as Kernels>::MatmulKernel::new(
+ &context,
+ bf16::data_type(),
+ bf16::data_type(),
+ bf16::data_type(),
+ )
+ .expect("MatmulKernel");
+ let shapes = [Shape::new(64, 128, 64), Shape::new(128, 2048, 128), Shape::new(33, 128, 64)];
+ for path in gemm_paths_for_hw(&context) {
+ for shape in shapes {
+ let case = Case::new(shape).with_bias(true);
+ check_case::<bf16>(&context, &mut kernel, Some(path), case, 1.0);
+ }
+ }
+}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
index e53eeb816..0d1e0fbd6 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/quant_dispatch_test.rs
@@ -227,3 +227,34 @@ fn quant_gemm_accumulate_returns_unsupported_dop() {
"got {matmul:?}"
);
}
+
+#[rstest]
+#[case::gs32_4bit_mlx(128, 256, 64, 32, 4, QuantizationMethod::ScaleBias)]
+#[case::gs64_4bit_mlx(128, 256, 64, 64, 4, QuantizationMethod::ScaleBias)]
+#[case::gs128_4bit_mlx(128, 256, 64, 128, 4, QuantizationMethod::ScaleBias)]
+#[case::gs32_4bit_zp(128, 256, 64, 32, 4, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs64_8bit_zp(128, 256, 64, 64, 8, QuantizationMethod::ScaleZeroPoint)]
+#[case::gs128_8bit_zp(128, 256, 64, 128, 8, QuantizationMethod::ScaleZeroPoint)]
+fn mxu_quant_parity_bf16(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+ #[case] gs: u32,
+ #[case] bits: u32,
+ #[case] method: QuantizationMethod,
+) {
+ let context = MetalContext::new().expect("Metal context");
+ if !context.supports_mxu() {
+ return;
+ }
+ let input = QuantInput::<bf16>::new(m, k, n, gs, bits, method, 0);
+ let actual = run_quant_metal::<bf16>(&context, &input, Some(GemmDispatchPath::Mxu));
+ let reference = run_quant_cpu::<bf16>(&input);
+ assert_parity::<bf16>(
+ &format!("MXU m={m} k={k} n={n} gs={gs} bits={bits} method={method:?}"),
+ &reference,
+ &actual,
+ 0.05,
+ 0.5,
+ );
+}
diff --git a/crates/backend-uzu/tests/unit/kernel/mod.rs b/crates/backend-uzu/tests/unit/kernel/mod.rs
index ef7419d7d..e4675a0c5 100644
--- a/crates/backend-uzu/tests/unit/kernel/mod.rs
+++ b/crates/backend-uzu/tests/unit/kernel/mod.rs
@@ -17,7 +17,6 @@ mod moe;
mod pooling;
mod qk_unpack_test;
mod qkv_norm_test;
-mod quant_matmul;
mod rms_norm_test;
mod rope_test;
mod sampling;
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs
deleted file mode 100644
index 243c6b7f7..000000000
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/mod.rs
+++ /dev/null
@@ -1,98 +0,0 @@
-#[macro_use]
-#[path = "../../../common/mod.rs"]
-mod common;
-
-macro_rules! for_each_qmv_float_type {
- (|$F:ident| $body:expr) => {{
- {
- type $F = f32;
- $body
- }
- {
- type $F = half::bf16;
- $body
- }
- }};
-}
-
-mod qmv_fast_test;
-mod qmv_test;
-
-use backend_uzu::{array::ArrayElement, backends::common::gpu_types::QuantizationMethod};
-use num_traits::Float;
-
-pub(super) struct Input<T: ArrayElement + Float> {
- pub w_packed: Vec<u32>,
- pub scales: Vec<T>,
- pub zero_points: Option<Vec<u8>>,
- pub biases: Option<Vec<T>>,
- pub x: Vec<T>,
- pub k: u32,
- pub n: u32,
- pub m: u32,
- pub group_size: u32,
- pub bits: u32,
- pub quant_method: QuantizationMethod,
-}
-
-pub(super) fn pack_weights_u32(
- values: &[u8],
- bits: u32,
-) -> Vec<u32> {
- if bits == 4 {
- values
- .chunks(8)
- .map(|chunk| {
- let mut word = 0u32;
- for (i, &v) in chunk.iter().enumerate() {
- word |= ((v & 0xF) as u32) << (i * 4);
- }
- word
- })
- .collect()
- } else {
- values
- .chunks(4)
- .map(|chunk| {
- let mut word = 0u32;
- for (i, &v) in chunk.iter().enumerate() {
- word |= (v as u32) << (i * 8);
- }
- word
- })
- .collect()
- }
-}
-
-pub(super) fn pack_zero_points(
- values: &[u8],
- bits: u32,
-) -> Vec<u8> {
- if bits == 4 {
- values
- .chunks(2)
- .map(|chunk| {
- let lo = chunk[0] & 0x0F;
- let hi = if chunk.len() > 1 {
- chunk[1] & 0x0F
- } else {
- 0
- };
- lo | (hi << 4)
- })
- .collect()
- } else {
- values.to_vec()
- }
-}
-
-pub(super) fn check_tolerance(
- expected: f32,
- actual: f32,
- rel_tol: f64,
- abs_tol: f64,
-) -> bool {
- let diff = (expected - actual).abs() as f64;
- let tol = abs_tol.max(expected.abs() as f64 * rel_tol);
- diff <= tol
-}
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
deleted file mode 100644
index 22d6b0944..000000000
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ /dev/null
@@ -1,302 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use backend_uzu::{
- array::ArrayElement,
- backends::{
- common::{
- Allocation, Backend, Context, Encoder, Kernels,
- gpu_types::QuantizationMethod,
- kernel::{QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel},
- },
- cpu::Cpu,
- },
- data_type::DataType,
-};
-use num_traits::Float;
-
-use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
-use crate::{
- common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
- uzu_test,
-};
-
-fn get_expected<T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = <Cpu as Backend>::Context::new().expect("Failed to create Context");
-
- let w_buf = alloc_allocation_with_data::<Cpu, u32>(&context, &input.w_packed);
- let scales_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.scales);
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<Cpu, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<Cpu, T>(&context, b));
- let x_buf = alloc_allocation_with_data::<Cpu, T>(&context, &input.x);
- let mut y_buf = alloc_allocation::<Cpu, T>(&context, (input.m as usize) * (input.n as usize));
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
-
- let kernel = <<Cpu as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
- &context,
- T::data_type(),
- T::data_type(),
- T::data_type(),
- input.group_size,
- input.bits,
- input.quant_method,
- )
- .expect("Failed to create QuantizedMatmulQmvKernel");
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- input.k,
- input.n,
- input.m,
- &mut encoder,
- );
-
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- allocation_to_vec(&y_buf)
-}
-
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
-
- let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));
- let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);
- let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
-
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
- &context,
- T::data_type(),
- T::data_type(),
- T::data_type(),
- input.group_size,
- input.bits,
- input.quant_method,
- false,
- )
- .expect("Failed to create QuantizedMatmulQmvFastKernel");
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- None::<&Allocation<B>>,
- input.k,
- input.n,
- input.m,
- &mut encoder,
- );
-
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- allocation_to_vec(&y_buf)
-}
-
-fn get_test_data<T: ArrayElement + Float>(
- m: usize,
- k: usize,
- n: usize,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) -> (Input<T>, Vec<T>) {
- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
-
- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
- let max_val = if bits == 4 {
- 15u8
- } else {
- 255u8
- };
- for j in 0..n {
- for l in 0..k {
- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
- }
- }
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
- }
- }
- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = match quant_method {
- QuantizationMethod::ScaleBias => {
- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
- }
- }
- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- (None, Some(biases))
- },
- QuantizationMethod::ScaleZeroPoint => {
- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
- zp_raw.push(zp_val);
- }
- }
- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
- for j in 0..n {
- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
- let packed_row = pack_zero_points(row, bits);
- let mut padded = packed_row;
- padded.resize(zp_stride, 0);
- zp_packed.extend_from_slice(&padded);
- }
- (Some(zp_packed), None)
- },
- QuantizationMethod::ScaleSymmetric => (None, None),
- };
-
- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
- for i in 0..m {
- for l in 0..k {
- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
- }
- }
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let input = Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- quant_method,
- };
-
- let expected = get_expected::<T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
-) {
- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
- DataType::BF16 => (0.05, 0.5),
- DataType::F16 => (0.02, 0.5),
- _ => (0.01, 0.1),
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- assert_eq!(expected.len(), output.len(), "Output length mismatch");
-
- let mut errors = 0;
- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
- let exp_f32 = exp.to_f32().unwrap();
- let got_f32 = got.to_f32().unwrap();
- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
- if errors < 5 {
- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
- }
- errors += 1;
- }
- }
- assert_eq!(
- errors,
- 0,
- "QMV fast kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, format={:?}: {} mismatches",
- std::any::type_name::<B>(),
- input.m,
- input.k,
- input.n,
- input.group_size,
- input.bits,
- input.quant_method,
- errors,
- );
- });
-}
-
-fn get_test_dims(
- group_size: u32,
- bits: u32,
- is_half_precision: bool,
-) -> Vec<(usize, usize, usize)> {
- let gs = group_size as usize;
- let block_size = if bits == 4 {
- 512
- } else {
- 256
- };
- let k = block_size;
- let n = gs.max(32);
- if is_half_precision && bits == 8 {
- vec![(1, k, n), (2, k, n)]
- } else {
- vec![(1, k, n), (2, k, n * 2), (1, k, n * 2)]
- }
-}
-
-fn test_basic<T: ArrayElement + Float + Debug + Display>(
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);
- let dims = get_test_dims(group_size, bits, is_half);
-
- for (m, k, n) in dims {
- let (input, expected) = get_test_data::<T>(m, k, n, group_size, bits, quant_method);
- test_internal::<T>(&input, &expected);
- }
-}
-
-macro_rules! qmv_fast_test {
- ($name:ident, gs=$gs:expr, bits=$bits:expr, format=$format:expr) => {
- #[uzu_test]
- fn $name() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>($gs, $bits, $format);
- })
- }
- };
-}
-
-qmv_fast_test!(test_gs16_4bit_mlx, gs = 16, bits = 4, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs32_4bit_mlx, gs = 32, bits = 4, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs64_4bit_mlx, gs = 64, bits = 4, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs128_4bit_mlx, gs = 128, bits = 4, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs16_8bit_mlx, gs = 16, bits = 8, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, format = QuantizationMethod::ScaleBias);
-qmv_fast_test!(test_gs16_4bit_zp, gs = 16, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs16_8bit_zp, gs = 16, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, format = QuantizationMethod::ScaleZeroPoint);
-qmv_fast_test!(test_gs32_8bit_symmetric, gs = 32, bits = 8, format = QuantizationMethod::ScaleSymmetric);
diff --git a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs b/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
deleted file mode 100644
index 2eed46cb5..000000000
--- a/crates/backend-uzu/tests/unit/kernel/quant_matmul/qmv_test.rs
+++ /dev/null
@@ -1,459 +0,0 @@
-use std::fmt::{Debug, Display};
-
-use backend_uzu::{
- array::ArrayElement,
- backends::{
- common::{Backend, Context, Encoder, Kernels, gpu_types::QuantizationMethod, kernel::QuantizedMatmulQmvKernel},
- cpu::Cpu,
- },
- data_type::DataType,
-};
-use num_traits::Float;
-
-use super::{Input, check_tolerance, pack_weights_u32, pack_zero_points};
-use crate::{
- common::helpers::{alloc_allocation, alloc_allocation_with_data, allocation_to_vec},
- uzu_test,
-};
-
-fn get_output<B: Backend, T: ArrayElement + Float>(input: &Input<T>) -> Vec<T> {
- let context = B::Context::new().expect("Failed to create Context");
-
- let w_buf = alloc_allocation_with_data::<B, u32>(&context, &input.w_packed);
- let scales_buf = alloc_allocation_with_data::<B, T>(&context, &input.scales);
-
- let zp_buf = input.zero_points.as_ref().map(|zp| alloc_allocation_with_data::<B, u8>(&context, zp));
- let bias_buf = input.biases.as_ref().map(|b| alloc_allocation_with_data::<B, T>(&context, b));
-
- let x_buf = alloc_allocation_with_data::<B, T>(&context, &input.x);
- let mut y_buf = alloc_allocation::<B, T>(&context, (input.m as usize) * (input.n as usize));
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
-
- let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvKernel::new(
- &context,
- T::data_type(),
- T::data_type(),
- T::data_type(),
- input.group_size,
- input.bits,
- input.quant_method,
- )
- .expect("Failed to create QuantizedMatmulQmvKernel");
- kernel.encode(
- &w_buf,
- &scales_buf,
- zp_buf.as_ref(),
- bias_buf.as_ref(),
- &x_buf,
- &mut y_buf,
- input.k,
- input.n,
- input.m,
- &mut encoder,
- );
-
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- allocation_to_vec(&y_buf)
-}
-
-/// Create test data for a qmv scenario: m×k * k×n = m×n
-/// Weights are in transposed [N × K] layout (each output row stores K input weights).
-/// Scales: [N × num_groups_k], zero_points: [N × zp_stride_k], biases: [N × num_groups_k].
-/// Groups are along the K dimension.
-fn get_test_data_basic<T: ArrayElement + Float>(
- m: usize,
- k: usize,
- n: usize,
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) -> (Input<T>, Vec<T>) {
- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
-
- // Generate quantized weights [N × K]
- let mut weights_raw: Vec<u8> = Vec::with_capacity(n * k);
- let max_val = if bits == 4 {
- 15u8
- } else {
- 255u8
- };
- for j in 0..n {
- for l in 0..k {
- weights_raw.push(((j * 3 + l * 7 + 1) % (max_val as usize + 1)) as u8);
- }
- }
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- // Scales: [N × num_groups_k]
- let mut scales_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- scales_f32.push(0.5 + 0.1 * ((j + g) % 5) as f32);
- }
- }
- let scales: Vec<T> = scales_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- // Zero points or biases
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = match quant_method {
- QuantizationMethod::ScaleBias => {
- let mut biases_f32: Vec<f32> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- biases_f32.push(0.01 * ((j + g * 2) % 7) as f32);
- }
- }
- let biases: Vec<T> = biases_f32.iter().map(|&v| T::from(v).unwrap()).collect();
- (None, Some(biases))
- },
- QuantizationMethod::ScaleZeroPoint => {
- let mut zp_raw: Vec<u8> = Vec::with_capacity(n * num_groups_k);
- for j in 0..n {
- for g in 0..num_groups_k {
- let zp_val = ((j * 2 + g * 3) % (max_val as usize + 1)) as u8;
- zp_raw.push(zp_val);
- }
- }
- // Pack zero points per row (row = output neuron)
- let mut zp_packed: Vec<u8> = Vec::with_capacity(n * zp_stride);
- for j in 0..n {
- let row = &zp_raw[j * num_groups_k..(j + 1) * num_groups_k];
- let packed_row = pack_zero_points(row, bits);
- let mut padded = packed_row;
- padded.resize(zp_stride, 0);
- zp_packed.extend_from_slice(&padded);
- }
- (Some(zp_packed), None)
- },
- QuantizationMethod::ScaleSymmetric => (None, None),
- };
-
- // X: [m × k]
- let mut x_f32: Vec<f32> = Vec::with_capacity(m * k);
- for i in 0..m {
- for l in 0..k {
- x_f32.push(0.1 * f32::sin((i * k + l) as f32 * 0.05) + 0.5);
- }
- }
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let input = Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- quant_method,
- };
-
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-/// Small edge-case test data with known values
-fn get_test_data_edge<T: ArrayElement + Float>(
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) -> (Input<T>, Vec<T>) {
- let m = 1usize;
- let k = 4usize;
- let n = 4usize;
- let num_groups_k = (k + group_size as usize - 1) / group_size as usize;
-
- // Simple weights [N × K]: all 1s
- let weights_raw: Vec<u8> = vec![1u8; n * k];
- let w_packed = pack_weights_u32(&weights_raw, bits);
-
- // Unit scales [N × num_groups_k]
- let scales: Vec<T> = vec![T::one(); n * num_groups_k];
-
- let zp_stride = if bits == 4 {
- (num_groups_k + 1) / 2
- } else {
- num_groups_k
- };
-
- let (zero_points, biases) = match quant_method {
- QuantizationMethod::ScaleBias => {
- let biases: Vec<T> = vec![T::zero(); n * num_groups_k];
- (None, Some(biases))
- },
- QuantizationMethod::ScaleZeroPoint => {
- // Zero points all 0 → bias = 0
- let zp_packed = vec![0u8; n * zp_stride];
- (Some(zp_packed), None)
- },
- QuantizationMethod::ScaleSymmetric => (None, None),
- };
-
- // X: simple increasing values
- let x_f32: Vec<f32> = (0..m * k).map(|i| (i + 1) as f32 * 0.1).collect();
- let x: Vec<T> = x_f32.iter().map(|&v| T::from(v).unwrap()).collect();
-
- let input = Input {
- w_packed,
- scales,
- zero_points,
- biases,
- x,
- k: k as u32,
- n: n as u32,
- m: m as u32,
- group_size,
- bits,
- quant_method,
- };
-
- let expected = get_output::<Cpu, T>(&input);
- (input, expected)
-}
-
-fn test_internal<T: ArrayElement + Float + Debug + Display>(
- input: &Input<T>,
- expected: &[T],
-) {
- let (rel_tol, abs_tol): (f64, f64) = match T::data_type() {
- DataType::BF16 => (0.05, 0.5),
- DataType::F16 => (0.02, 0.5),
- _ => (0.01, 0.1),
- };
-
- for_each_non_cpu_backend!(|B| {
- let output = get_output::<B, T>(input);
- assert_eq!(expected.len(), output.len(), "Output length mismatch");
-
- let mut errors = 0;
- for (i, (&exp, &got)) in expected.iter().zip(output.iter()).enumerate() {
- let exp_f32 = exp.to_f32().unwrap();
- let got_f32 = got.to_f32().unwrap();
- if !check_tolerance(exp_f32, got_f32, rel_tol, abs_tol) {
- if errors < 5 {
- eprintln!(" idx={} expected={} got={} diff={}", i, exp_f32, got_f32, (exp_f32 - got_f32).abs());
- }
- errors += 1;
- }
- }
- assert_eq!(
- errors,
- 0,
- "QMV kernel: backend={}, m={}, k={}, n={}, gs={}, bits={}, format={:?}: {} mismatches",
- std::any::type_name::<B>(),
- input.m,
- input.k,
- input.n,
- input.group_size,
- input.bits,
- input.quant_method,
- errors,
- );
- });
-}
-
-fn get_test_dims(
- group_size: u32,
- bits: u32,
- is_half_precision: bool,
-) -> Vec<(usize, usize, usize)> {
- let gs = group_size as usize;
- // N must be >= group_size for the Metal tiled loader's stride calculations.
- // For half-precision 8-bit, limit k to avoid catastrophic cancellation
- // from large weight values (0-255) and ZP bias subtraction.
- if is_half_precision && bits == 8 {
- let k = gs.min(64);
- vec![(1, k, gs), (2, k, gs)]
- } else {
- vec![(1, gs * 2, gs), (2, gs * 2, gs * 2), (1, gs, gs * 2)]
- }
-}
-
-fn test_basic<T: ArrayElement + Float + Debug + Display>(
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- let is_half = matches!(T::data_type(), DataType::F16 | DataType::BF16);
- let dims = get_test_dims(group_size, bits, is_half);
-
- for (m, k, n) in dims {
- let (input, expected) = get_test_data_basic::<T>(m, k, n, group_size, bits, quant_method);
- test_internal::<T>(&input, &expected);
- }
-}
-
-fn test_edge<T: ArrayElement + Float + Debug + Display>(
- group_size: u32,
- bits: u32,
- quant_method: QuantizationMethod,
-) {
- let (input, expected) = get_test_data_edge::<T>(group_size, bits, quant_method);
- test_internal::<T>(&input, &expected);
-}
-
-// ── 4-bit, mlx quant ───────────────────────────────────────────────────────
-#[uzu_test]
-fn test_gs16_4bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(16, 4, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_gs32_4bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(32, 4, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_gs64_4bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(64, 4, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_gs128_4bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(128, 4, QuantizationMethod::ScaleBias);
- })
-}
-
-// ── 8-bit, mlx quant ───────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_gs16_8bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(16, 8, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_gs32_8bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(32, 8, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_gs64_8bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(64, 8, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_gs128_8bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(128, 8, QuantizationMethod::ScaleBias);
- })
-}
-
-// ── 4-bit, zero points ─────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_gs16_4bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(16, 4, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_gs32_4bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_gs64_4bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(64, 4, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_gs128_4bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(128, 4, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-// ── 8-bit, zero points ─────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_gs16_8bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(16, 8, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_gs32_8bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_gs64_8bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(64, 8, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_gs128_8bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(128, 8, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_gs32_8bit_symmetric() {
- for_each_qmv_float_type!(|F| {
- test_basic::<F>(32, 8, QuantizationMethod::ScaleSymmetric);
- })
-}
-
-// ── Edge cases ──────────────────────────────────────────────────────────────
-
-#[uzu_test]
-fn test_edge_4bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_edge::<F>(32, 4, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_edge_8bit_mlx() {
- for_each_qmv_float_type!(|F| {
- test_edge::<F>(32, 8, QuantizationMethod::ScaleBias);
- })
-}
-
-#[uzu_test]
-fn test_edge_4bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_edge::<F>(32, 4, QuantizationMethod::ScaleZeroPoint);
- })
-}
-
-#[uzu_test]
-fn test_edge_8bit_zp() {
- for_each_qmv_float_type!(|F| {
- test_edge::<F>(32, 8, QuantizationMethod::ScaleZeroPoint);
- })
-}