Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

remove thread group mem in mpp (#350)

eugene4971bc88ff46
diff --git a/Cargo.lock b/Cargo.lock
index 573b3aa96..ddcd3f4e7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -744,7 +744,7 @@ dependencies = [
"getrandom 0.2.17",
"instant",
"pin-project-lite",
- "rand 0.8.5",
+ "rand 0.8.6",
"tokio",
]
@@ -1227,9 +1227,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.6.0"
+version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
+checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1249,9 +1249,9 @@ dependencies = [
[[package]]
name = "clap_derive"
-version = "4.6.0"
+version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
+checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
dependencies = [
"heck",
"proc-macro2",
@@ -1329,7 +1329,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"
dependencies = [
"serde",
"termcolor",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
]
[[package]]
@@ -1361,7 +1361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"
dependencies = [
"unicode-segmentation",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
]
[[package]]
@@ -1396,7 +1396,7 @@ checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87"
dependencies = [
"encode_unicode",
"libc",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
"windows-sys 0.61.2",
]
@@ -1678,9 +1678,9 @@ dependencies = [
[[package]]
name = "ctor"
-version = "0.10.0"
+version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95d0d11eb38e7642efca359c3cf6eb7b2e528182d09110165de70192b0352775"
+checksum = "83cf0d42651b16c6dfe68685716d18480d18a9c39c62d76e8cf3eb6ed5d8bcbf"
dependencies = [
"ctor-proc-macro",
"dtor",
@@ -1688,9 +1688,9 @@ dependencies = [
[[package]]
name = "ctor-proc-macro"
-version = "0.0.12"
+version = "0.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7ab264ea985f1bd27887d7b21ea2bb046728e05d11909ca138d700c494730db"
+checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483"
[[package]]
name = "ctrlc"
@@ -1884,9 +1884,9 @@ dependencies = [
[[package]]
name = "dary_heap"
-version = "0.3.8"
+version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04"
+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
dependencies = [
"serde",
]
@@ -2183,18 +2183,18 @@ dependencies = [
[[package]]
name = "dtor"
-version = "0.7.0"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17f72721db8027a4e96dd6fb50d2a1d32259c9d3da1b63dee612ccd981e14293"
+checksum = "edf234dd1594d6dd434a8fb8cada51ddbbc593e40e4a01556a0b31c62da2775b"
dependencies = [
"dtor-proc-macro",
]
[[package]]
name = "dtor-proc-macro"
-version = "0.0.12"
+version = "0.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c98b077c7463d01d22dde8a24378ddf1ca7263dc687cffbed38819ea6c21131"
+checksum = "2647271c92754afcb174e758003cfd1cbf1e43e5a7853d7b1813e63e19e39a73"
[[package]]
name = "dunce"
@@ -2576,9 +2576,9 @@ dependencies = [
[[package]]
name = "fraction"
-version = "0.15.3"
+version = "0.15.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7"
+checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872"
dependencies = [
"lazy_static",
"num",
@@ -3431,7 +3431,7 @@ checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb"
dependencies = [
"console",
"portable-atomic",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
"unit-prefix",
"web-time",
]
@@ -3478,7 +3478,7 @@ dependencies = [
"dyn-clone",
"fuzzy-matcher",
"unicode-segmentation",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
]
[[package]]
@@ -3691,9 +3691,9 @@ dependencies = [
[[package]]
name = "jsonschema"
-version = "0.46.0"
+version = "0.46.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84695c6689b01384700a3d93acecbd07231ee6fff1bf22ae980b4c307e6ddfd5"
+checksum = "50180452e7808015fe083eae3efcf1ec98b89b45dd8cc204f7b4a6b7b81ea675"
dependencies = [
"ahash",
"bytecount",
@@ -3772,9 +3772,9 @@ checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8"
[[package]]
name = "libc"
-version = "0.2.185"
+version = "0.2.186"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"
+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
[[package]]
name = "libdbus-sys"
@@ -4748,9 +4748,9 @@ dependencies = [
[[package]]
name = "openssl"
-version = "0.10.77"
+version = "0.10.78"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f"
+checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222"
dependencies = [
"bitflags 2.11.1",
"cfg-if 1.0.4",
@@ -4780,9 +4780,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "openssl-sys"
-version = "0.9.113"
+version = "0.9.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644"
+checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6"
dependencies = [
"cc",
"libc",
@@ -4980,7 +4980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
dependencies = [
"phf_shared",
- "rand 0.8.5",
+ "rand 0.8.6",
]
[[package]]
@@ -5122,9 +5122,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
[[package]]
name = "portable-atomic-util"
-version = "0.2.6"
+version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3"
+checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"
dependencies = [
"portable-atomic",
]
@@ -5395,7 +5395,7 @@ dependencies = [
"once_cell",
"socket2 0.6.3",
"tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -5421,9 +5421,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
[[package]]
name = "rand"
-version = "0.8.5"
+version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
dependencies = [
"libc",
"rand_chacha 0.3.1",
@@ -5551,7 +5551,7 @@ dependencies = [
"thiserror 2.0.18",
"unicode-segmentation",
"unicode-truncate",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
]
[[package]]
@@ -5602,7 +5602,7 @@ dependencies = [
"strum",
"time",
"unicode-segmentation",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
]
[[package]]
@@ -5743,9 +5743,9 @@ dependencies = [
[[package]]
name = "referencing"
-version = "0.46.0"
+version = "0.46.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2d5554bf79f4acf770dc3193b44b2d63b348f5f7b7448a0ea1191b37b620728"
+checksum = "acb0c66c7b78c1da928bee668b5cc638c678642ff587faff6e6222f797be9d4c"
dependencies = [
"ahash",
"fluent-uri",
@@ -6007,7 +6007,7 @@ dependencies = [
"num_cpus",
"parking_lot",
"pin-project-lite",
- "rand 0.8.5",
+ "rand 0.8.6",
"ref-cast",
"rocket_codegen",
"rocket_http",
@@ -6104,9 +6104,9 @@ dependencies = [
[[package]]
name = "rustls"
-version = "0.23.38"
+version = "0.23.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"
+checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e"
dependencies = [
"aws-lc-rs",
"log",
@@ -6169,9 +6169,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
[[package]]
name = "rustls-webpki"
-version = "0.103.12"
+version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
"aws-lc-rs",
"ring",
@@ -6319,7 +6319,7 @@ dependencies = [
"hkdf",
"num",
"once_cell",
- "rand 0.8.5",
+ "rand 0.8.6",
"serde",
"sha2",
"zbus",
@@ -6779,6 +6779,12 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
+[[package]]
+name = "symlink"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"
+
[[package]]
name = "syn"
version = "1.0.109"
@@ -7166,9 +7172,9 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.52.0"
+version = "1.52.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776"
+checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6"
dependencies = [
"bytes",
"libc",
@@ -7315,7 +7321,7 @@ dependencies = [
"indexmap 2.14.0",
"toml_datetime 1.1.1+spec-1.1.0",
"toml_parser",
- "winnow 1.0.1",
+ "winnow 1.0.2",
]
[[package]]
@@ -7324,7 +7330,7 @@ version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
dependencies = [
- "winnow 1.0.1",
+ "winnow 1.0.2",
]
[[package]]
@@ -7397,11 +7403,12 @@ dependencies = [
[[package]]
name = "tracing-appender"
-version = "0.2.4"
+version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf"
+checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"
dependencies = [
"crossbeam-channel",
+ "symlink",
"thiserror 2.0.18",
"time",
"tracing-subscriber",
@@ -7513,9 +7520,9 @@ dependencies = [
[[package]]
name = "typenum"
-version = "1.19.0"
+version = "1.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
+checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"
[[package]]
name = "ubyte"
@@ -7600,7 +7607,7 @@ checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5"
dependencies = [
"itertools 0.14.0",
"unicode-segmentation",
- "unicode-width 0.2.0",
+ "unicode-width 0.2.2",
]
[[package]]
@@ -7611,9 +7618,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
[[package]]
name = "unicode-width"
-version = "0.2.0"
+version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "unicode-xid"
@@ -7826,9 +7833,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.23.0"
+version = "1.23.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
+checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"
dependencies = [
"atomic 0.6.1",
"getrandom 0.4.2",
@@ -7979,11 +7986,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
[[package]]
name = "wasip2"
-version = "1.0.2+wasi-0.2.9"
+version = "1.0.3+wasi-0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"
+checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
dependencies = [
- "wit-bindgen",
+ "wit-bindgen 0.57.1",
]
[[package]]
@@ -7992,7 +7999,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"
dependencies = [
- "wit-bindgen",
+ "wit-bindgen 0.51.0",
]
[[package]]
@@ -8134,18 +8141,18 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
-version = "1.0.6"
+version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca"
+checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
-version = "1.0.6"
+version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"
+checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"
dependencies = [
"rustls-pki-types",
]
@@ -8697,9 +8704,9 @@ dependencies = [
[[package]]
name = "winnow"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"
+checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0"
dependencies = [
"memchr",
]
@@ -8713,6 +8720,12 @@ dependencies = [
"wit-bindgen-rust-macro",
]
+[[package]]
+name = "wit-bindgen"
+version = "0.57.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
+
[[package]]
name = "wit-bindgen-core"
version = "0.51.0"
@@ -8886,7 +8899,7 @@ dependencies = [
"hex",
"nix 0.29.0",
"ordered-stream",
- "rand 0.8.5",
+ "rand 0.8.6",
"serde",
"serde_repr",
"sha1",
diff --git a/Cargo.toml b/Cargo.toml
index 8ddbd743a..ffbdbe89f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -167,6 +167,13 @@ keyring = { version = "3.6.3", features = [
"crypto-rust",
] }
+[workspace.metadata.dinghy]
+forward-cfgs = ["metal_backend", "grammar_xgrammar"]
+
+[workspace.metadata.dinghy.test-attribute-aliases]
+uzu_test = "dinghy_apple_runner_macros::test_case"
+uzu_bench = "dinghy_apple_runner_macros::bench_case"
+
[profile.dev.build-override]
opt-level = 3
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/common/defines.h
index 602984747..904411eac 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/defines.h
@@ -2,6 +2,7 @@
#define METAL_CONST static constant constexpr
#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")
+#define METAL_PRAGMA_NO_UNROLL _Pragma("clang loop unroll(disable)")
#define METAL_SIMD_SIZE 32
METAL_FUNC int pow2(int n) { return 1 << n; }
diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h
new file mode 100644
index 000000000..4263ddeba
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h
@@ -0,0 +1,72 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#pragma METAL internals : enable
+
+namespace uzu {
+
+template <typename T, T v>
+struct integral_constant {
+ static constexpr constant T value = v;
+ using value_type = T;
+ using type = integral_constant;
+
+ METAL_FUNC constexpr operator value_type() const noexcept { return value; }
+};
+
+template <bool B>
+using bool_constant = integral_constant<bool, B>;
+using true_type = bool_constant<true>;
+using false_type = bool_constant<false>;
+
+template <int val>
+using Int = integral_constant<int, val>;
+
+#define METAL_INTEGRAL_CONST_BINOP(op, fn) \
+ template <typename T, T tv, typename U, U uv> \
+ METAL_FUNC constexpr auto fn( \
+ integral_constant<T, tv>, \
+ integral_constant<U, uv> \
+ ) { \
+ constexpr auto res = tv op uv; \
+ return integral_constant<decltype(res), res>{}; \
+ }
+
+METAL_INTEGRAL_CONST_BINOP(+, operator+)
+METAL_INTEGRAL_CONST_BINOP(-, operator-)
+METAL_INTEGRAL_CONST_BINOP(*, operator*)
+METAL_INTEGRAL_CONST_BINOP(/, operator/)
+
+#undef METAL_INTEGRAL_CONST_BINOP
+
+template <int start, int step, typename F, int... Is>
+METAL_FUNC constexpr void const_for_loop_impl(
+ F f,
+ metal::integer_sequence<int, Is...>
+) {
+ (f(Int<start + Is * step>{}), ...);
+}
+
+template <int start, int stop, int step, typename F>
+METAL_FUNC constexpr void const_for_loop(F f) {
+ static_assert(step > 0 && start <= stop);
+ constexpr int count = (stop - start + step - 1) / step;
+ const_for_loop_impl<start, step>(
+ f,
+ metal::make_integer_sequence<int, count>{}
+ );
+}
+
+template <typename F>
+METAL_FUNC void dispatch_bool(bool v, F f) {
+ if (v) {
+ f(true_type{});
+ } else {
+ f(false_type{});
+ }
+}
+
+} // namespace uzu
+
+#pragma METAL internals : disable
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h
new file mode 100644
index 000000000..278600759
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h
@@ -0,0 +1,100 @@
+#pragma once
+
+#include "../../common/integral_constant.h"
+#include "../../common/thread_context.h"
+#include "defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// AluFragmentOps - stateless traits/ops for 8x8 ALU matmul (simdgroup_matrix)
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T>
+struct AluFragmentOps {
+ METAL_CONST ushort FRAGMENT_ROWS = 8;
+ METAL_CONST ushort FRAGMENT_COLS = 8;
+ METAL_CONST ushort ELEMENTS_PER_THREAD =
+ (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
+ METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;
+ METAL_CONST ushort THREAD_ELEMENT_COLS = 2;
+ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 1;
+
+ static_assert(
+ THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,
+ "AluFragmentOps shape is not consistent with element count"
+ );
+
+ typedef metal::simdgroup_matrix<T, FRAGMENT_ROWS, FRAGMENT_COLS>
+ SimdgroupMatrixType;
+
+ template <typename U>
+ using ThreadVector = metal::vec<U, ELEMENTS_PER_THREAD>;
+
+ METAL_FUNC static constexpr void multiply_accumulate(
+ thread ThreadVector<T>& D,
+ thread ThreadVector<T>& A,
+ thread ThreadVector<T>& B,
+ thread ThreadVector<T>& C
+ ) {
+ SimdgroupMatrixType D_mat;
+ SimdgroupMatrixType A_mat;
+ SimdgroupMatrixType B_mat;
+ SimdgroupMatrixType C_mat;
+
+ reinterpret_cast<thread ThreadVector<T>&>(A_mat.thread_elements()) = A;
+ reinterpret_cast<thread ThreadVector<T>&>(B_mat.thread_elements()) = B;
+ reinterpret_cast<thread ThreadVector<T>&>(C_mat.thread_elements()) = C;
+
+ simdgroup_multiply_accumulate(D_mat, A_mat, B_mat, C_mat);
+
+ D = reinterpret_cast<thread ThreadVector<T>&>(D_mat.thread_elements());
+ }
+
+ template <class OutputTile, class LeftTile, class RightTile>
+ METAL_FUNC static void tile_matmul(
+ thread OutputTile& output,
+ thread LeftTile& left,
+ thread RightTile& right
+ ) {
+ constexpr ushort tile_m = OutputTile::TILE_ROWS;
+ constexpr ushort tile_n = OutputTile::TILE_COLS;
+ constexpr ushort tile_k = LeftTile::TILE_COLS;
+
+ static_assert(
+ tile_m == LeftTile::TILE_ROWS,
+ "tile matmul: M dimensions do not match"
+ );
+ static_assert(
+ tile_n == RightTile::TILE_COLS,
+ "tile matmul: N dimensions do not match"
+ );
+ static_assert(
+ tile_k == RightTile::TILE_ROWS,
+ "tile matmul: K dimensions do not match"
+ );
+
+ METAL_PRAGMA_UNROLL
+ for (ushort row = 0; row < tile_m; ++row) {
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < tile_n; ++col) {
+ const ushort column_serpentine = (row % 2) ? (tile_n - 1 - col) : col;
+ METAL_PRAGMA_UNROLL
+ for (ushort k = 0; k < tile_k; ++k) {
+ multiply_accumulate(
+ output.fragment_at(row, column_serpentine),
+ left.fragment_at(row, k),
+ right.fragment_at(k, column_serpentine),
+ output.fragment_at(row, column_serpentine)
+ );
+ }
+ }
+ }
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h
index 1b56d03b3..5781cdf8c 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h
@@ -1,8 +1,6 @@
#pragma once
#include "../../common/defines.h"
-#include <metal_simdgroup>
-#include <metal_simdgroup_matrix>
#include <metal_stdlib>
using namespace metal;
@@ -33,15 +31,19 @@ struct PointerElementTypeImpl<threadgroup T*> {
template <typename T>
using PointerElementType = typename PointerElementTypeImpl<T>::type;
-template <typename OutT, typename InT>
+template <typename OutputType, typename InputType>
struct TransformNone {
TransformNone(float = 1.0f, float = 0.0f) {}
- static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }
- METAL_FUNC OutT apply(InT x, OutT) const { return static_cast<OutT>(x); }
+ static METAL_FUNC OutputType apply(InputType x) {
+ return static_cast<OutputType>(x);
+ }
+ METAL_FUNC OutputType apply(InputType x, OutputType) const {
+ return static_cast<OutputType>(x);
+ }
};
-template <typename OutT, typename InT>
+template <typename OutputType, typename InputType>
struct TransformScaleAccumulate {
const float alpha;
const float beta;
@@ -49,12 +51,15 @@ struct TransformScaleAccumulate {
TransformScaleAccumulate(float alpha, float beta)
: alpha(alpha), beta(beta) {}
- static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }
+ static METAL_FUNC OutputType apply(InputType x) {
+ return static_cast<OutputType>(x);
+ }
- METAL_FUNC OutT apply(InT x, OutT c) const {
- return static_cast<OutT>(
- x * static_cast<InT>(alpha) +
- (beta != 0.0f ? (static_cast<OutT>(beta) * c) : static_cast<OutT>(0.0))
+ METAL_FUNC OutputType apply(InputType x, OutputType c) const {
+ return static_cast<OutputType>(
+ x * static_cast<InputType>(alpha) +
+ (beta != 0.0f ? (static_cast<OutputType>(beta) * c)
+ : static_cast<OutputType>(0.0))
);
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
new file mode 100644
index 000000000..b67370a2a
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h
@@ -0,0 +1,298 @@
+#pragma once
+
+#include <metal_stdlib>
+
+#include "../../common/integral_constant.h"
+#include "../../common/thread_context.h"
+#include "defines.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+///////////////////////////////////////////////////////////////////////////////
+// Fragment - a thread-private tile of values arranged as TILE_ROWS x TILE_COLS
+// sub-tiles. Ops provides the sub-tile shape constants and the MMA primitive;
+// Fragment itself owns lane positioning and device<->register transfer.
+///////////////////////////////////////////////////////////////////////////////
+
+template <typename T, ushort TILE_ROWS_, ushort TILE_COLS_, class Ops>
+struct Fragment {
+ using FragmentOpsType = Ops;
+ using ElementType = T;
+ using ThreadVectorType = typename Ops::template ThreadVector<T>;
+
+ static_assert(
+ Ops::FRAGMENT_ROWS > 0 && Ops::FRAGMENT_COLS > 0,
+ "Ops must expose positive FRAGMENT_ROWS/FRAGMENT_COLS"
+ );
+ static_assert(
+ Ops::ELEMENTS_PER_THREAD ==
+ (Ops::FRAGMENT_ROWS * Ops::FRAGMENT_COLS) / METAL_SIMD_SIZE,
+ "Ops::ELEMENTS_PER_THREAD must equal "
+ "(FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE"
+ );
+ static_assert(
+ sizeof(ThreadVectorType) == Ops::ELEMENTS_PER_THREAD * sizeof(T),
+ "Ops::ThreadVector<T> must be tightly packed "
+ "(elements() relies on it)"
+ );
+
+ METAL_CONST ushort TILE_ROWS = TILE_ROWS_;
+ METAL_CONST ushort TILE_COLS = TILE_COLS_;
+
+ METAL_CONST ushort FRAGMENT_ROWS = ushort(Ops::FRAGMENT_ROWS);
+ METAL_CONST ushort FRAGMENT_COLS = ushort(Ops::FRAGMENT_COLS);
+
+ METAL_CONST ushort NUM_FRAGS = TILE_ROWS * TILE_COLS;
+ METAL_CONST ushort ELEMENTS_PER_TILE = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;
+
+ ThreadVectorType fragment_data[NUM_FRAGS];
+ ThreadContext thread_context;
+
+ METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread
+ : thread_context(thread_context) {}
+
+ // Lane origin (row, col) within a single FRAGMENT_ROWS x FRAGMENT_COLS
+ // sub-tile. The mapping depends only on Ops::THREAD_ELEMENT_COLS, so it is
+ // identical for every Fragment sharing that Ops type.
+ METAL_FUNC static constexpr short2 get_position(
+ const thread ThreadContext& thread_context
+ ) {
+ const ushort simdgroup_index = ushort(thread_context.simdgroup_index);
+ const short quad = simdgroup_index / 4;
+ const short row = (quad & 4) + (simdgroup_index / 2) % 4;
+ const short col =
+ ((quad & 2) + simdgroup_index % 2) * Ops::THREAD_ELEMENT_COLS;
+ return short2{col, row};
+ }
+
+ METAL_FUNC short2 get_position() const thread {
+ return get_position(thread_context);
+ }
+
+ METAL_FUNC constexpr void clear() {
+ METAL_PRAGMA_UNROLL
+ for (ushort index = 0; index < NUM_FRAGS; ++index) {
+ fragment_data[index] = ThreadVectorType(0);
+ }
+ }
+
+ METAL_FUNC constexpr thread ThreadVectorType& fragment_at(
+ const ushort row_index,
+ const ushort col_index
+ ) {
+ return fragment_data[row_index * TILE_COLS + col_index];
+ }
+
+ template <bool transpose>
+ METAL_FUNC constexpr thread ThreadVectorType& fragment_at(
+ const ushort row_index,
+ const ushort col_index,
+ metal::bool_constant<transpose>
+ ) {
+ if constexpr (transpose) {
+ return fragment_at(col_index, row_index);
+ } else {
+ return fragment_at(row_index, col_index);
+ }
+ }
+
+ METAL_FUNC thread ElementType* elements() {
+ return reinterpret_cast<thread ElementType*>(fragment_data);
+ }
+
+ // Unsafe load: copy a (TILE_ROWS * FRAGMENT_ROWS) x (TILE_COLS *
+ // FRAGMENT_COLS) block from device memory into fragment registers. ColStride
+ // defaults to the compile-time constant 1 so that the row-major fast path
+ // triggers when the caller omits it. Tile strides control the gap between
+ // sub-tiles.
+ template <
+ class Ptr,
+ class RowStride,
+ class ColStride = Int<1>,
+ class TileRowStride = Int<1>,
+ class TileColStride = Int<1>>
+ METAL_FUNC void load(
+ Ptr source,
+ RowStride row_stride,
+ ColStride col_stride = {},
+ TileRowStride tile_row_stride = {},
+ TileColStride tile_col_stride = {}
+ ) thread {
+ transfer<LOAD, UNSAFE>(
+ source,
+ row_stride,
+ col_stride,
+ Int<0>{},
+ Int<0>{},
+ tile_row_stride,
+ tile_col_stride
+ );
+ }
+
+ // Safe load: col_stride is implicitly 1; out-of-bounds elements become T(0).
+ template <
+ class Ptr,
+ class TileRowStride = Int<1>,
+ class TileColStride = Int<1>>
+ METAL_FUNC void load_safe(
+ Ptr source,
+ const int leading_dimension,
+ const short2 tile_dimensions,
+ TileRowStride tile_row_stride = {},
+ TileColStride tile_col_stride = {}
+ ) thread {
+ transfer<LOAD, SAFE>(
+ source,
+ leading_dimension,
+ Int<1>{},
+ tile_dimensions.y,
+ tile_dimensions.x,
+ tile_row_stride,
+ tile_col_stride
+ );
+ }
+
+ // Unsafe store: mirror of load. Same defaults apply.
+ template <
+ class Ptr,
+ class RowStride,
+ class ColStride = Int<1>,
+ class TileRowStride = Int<1>,
+ class TileColStride = Int<1>>
+ METAL_FUNC void store(
+ Ptr destination,
+ RowStride row_stride,
+ ColStride col_stride = {},
+ TileRowStride tile_row_stride = {},
+ TileColStride tile_col_stride = {}
+ ) thread {
+ transfer<STORE, UNSAFE>(
+ destination,
+ row_stride,
+ col_stride,
+ Int<0>{},
+ Int<0>{},
+ tile_row_stride,
+ tile_col_stride
+ );
+ }
+
+ // Safe store: col_stride is implicitly 1; out-of-bounds elements are skipped.
+ template <
+ class Ptr,
+ class TileRowStride = Int<1>,
+ class TileColStride = Int<1>>
+ METAL_FUNC void store_safe(
+ Ptr destination,
+ const int leading_dimension,
+ const short2 tile_dimensions,
+ TileRowStride tile_row_stride = {},
+ TileColStride tile_col_stride = {}
+ ) thread {
+ transfer<STORE, SAFE>(
+ destination,
+ leading_dimension,
+ Int<1>{},
+ tile_dimensions.y,
+ tile_dimensions.x,
+ tile_row_stride,
+ tile_col_stride
+ );
+ }
+
+private:
+ METAL_CONST bool LOAD = true;
+ METAL_CONST bool STORE = false;
+ METAL_CONST bool SAFE = true;
+ METAL_CONST bool UNSAFE = false;
+
+ template <class Fn>
+ METAL_FUNC void for_each_fragment(Fn fn) thread {
+ const_for_loop<0, TILE_ROWS, 1>([&](auto row_index) {
+ const_for_loop<0, TILE_COLS, 1>([&](auto col_index) {
+ fn(row_index, col_index);
+ });
+ });
+ }
+
+ // Unified memory transfer between device memory and fragment registers.
+ //
+ // IS_LOAD - true to read into fragment_data, false to write back.
+ // IS_SAFE - true to bounds-check each element against (row_limit,
+ // col_limit);
+ // on load, out-of-bounds elements become T(0); on store, skipped.
+ //
+ // When ColStride is a compile-time 1, the column-stride multiplication is
+ // elided to help the compiler collapse strength-reduced addressing.
+ template <
+ bool IS_LOAD,
+ bool IS_SAFE,
+ class Ptr,
+ class RowStride,
+ class ColStride,
+ class RowLimit,
+ class ColLimit,
+ class TileRowStride,
+ class TileColStride>
+ METAL_FUNC void transfer(
+ Ptr ptr,
+ RowStride row_stride,
+ ColStride col_stride,
+ RowLimit row_limit,
+ ColLimit col_limit,
+ TileRowStride tile_row_stride,
+ TileColStride tile_col_stride
+ ) thread {
+ using U = PointerElementType<Ptr>;
+ constexpr bool col_stride_is_one = metal::is_same_v<ColStride, Int<1>>;
+
+ const short2 position = get_position();
+ ptr += position.y * row_stride + position.x * col_stride;
+ const auto local_row_limit = row_limit - position.y;
+ const auto local_col_limit = col_limit - position.x;
+
+ for_each_fragment([&](auto tile_row, auto tile_col) {
+ thread auto& frag = fragment_at(tile_row.value, tile_col.value);
+ const auto row_base = tile_row.value * FRAGMENT_ROWS * tile_row_stride;
+ const auto col_base = tile_col.value * FRAGMENT_COLS * tile_col_stride;
+
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; i++) {
+ const auto row = row_base + i * Ops::THREAD_ELEMENT_ROW_STRIDE;
+ METAL_PRAGMA_UNROLL
+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; j++) {
+ const ushort element_index = i * Ops::THREAD_ELEMENT_COLS + j;
+ const auto col = col_base + j;
+ const auto offset = col_stride_is_one
+ ? (row * row_stride + col)
+ : (row * row_stride + col * col_stride);
+
+ if constexpr (IS_LOAD) {
+ if constexpr (IS_SAFE) {
+ const bool in_bounds =
+ (row < local_row_limit) && (col < local_col_limit);
+ frag[element_index] =
+ in_bounds ? static_cast<T>(ptr[offset]) : T(0);
+ } else {
+ frag[element_index] = static_cast<T>(ptr[offset]);
+ }
+ } else {
+ if constexpr (IS_SAFE) {
+ if ((row < local_row_limit) && (col < local_col_limit)) {
+ ptr[offset] = static_cast<U>(frag[element_index]);
+ }
+ } else {
+ ptr[offset] = static_cast<U>(frag[element_index]);
+ }
+ }
+ }
+ }
+ });
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
index 37b5673bd..6b1251c0a 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h
@@ -9,13 +9,6 @@ using namespace metal;
namespace uzu {
namespace matmul {
-///////////////////////////////////////////////////////////////////////////////
-// Loop Alignment Helper
-///////////////////////////////////////////////////////////////////////////////
-
-template <bool M_aligned, bool N_aligned, bool K_aligned>
-struct LoopAlignment {};
-
///////////////////////////////////////////////////////////////////////////////
// ThreadgroupGemm Kernel
///////////////////////////////////////////////////////////////////////////////
@@ -32,22 +25,22 @@ template <
bool transpose_b,
bool MN_aligned,
bool K_aligned,
- typename AccumType = float,
- typename Epilogue = TransformNone<U, AccumType>>
+ typename AccumulatorType = float,
+ typename Epilogue = TransformNone<U, AccumulatorType>>
struct ThreadgroupGemm {
- METAL_CONST short THREADGROUP_PADDING_A = 16 / sizeof(T);
- METAL_CONST short THREADGROUP_PADDING_B = 16 / sizeof(T);
- METAL_CONST short THREADGROUP_MEMORY_SIZE_A =
+ METAL_CONST ushort THREADGROUP_PADDING_A = 16 / sizeof(T);
+ METAL_CONST ushort THREADGROUP_PADDING_B = 16 / sizeof(T);
+ METAL_CONST ushort THREADGROUP_MEMORY_SIZE_A =
transpose_a ? BLOCK_DEPTH * (BLOCK_ROWS + THREADGROUP_PADDING_A)
: BLOCK_ROWS * (BLOCK_DEPTH + THREADGROUP_PADDING_A);
- METAL_CONST short THREADGROUP_MEMORY_SIZE_B =
+ METAL_CONST ushort THREADGROUP_MEMORY_SIZE_B =
transpose_b ? BLOCK_COLS * (BLOCK_DEPTH + THREADGROUP_PADDING_B)
: BLOCK_DEPTH * (BLOCK_COLS + THREADGROUP_PADDING_B);
- METAL_CONST short THREADGROUP_MEMORY_SIZE =
+ METAL_CONST ushort THREADGROUP_MEMORY_SIZE =
THREADGROUP_MEMORY_SIZE_A + THREADGROUP_MEMORY_SIZE_B;
- METAL_CONST short THREADGROUP_SIZE =
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32;
+ METAL_CONST ushort THREADGROUP_SIZE =
+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
using LoaderAType = ThreadgroupLoader<
T,
@@ -79,24 +72,21 @@ struct ThreadgroupGemm {
: BLOCK_DEPTH + THREADGROUP_PADDING_A,
transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B
: BLOCK_COLS + THREADGROUP_PADDING_B,
- AccumType,
+ AccumulatorType,
Epilogue>;
template <bool M_aligned, bool N_aligned, bool K_aligned_>
static METAL_FUNC void gemm_loop(
threadgroup T* a_shared,
threadgroup T* b_shared,
- const int gemm_k_iterations,
+ const int aligned_k_iterations,
thread LoaderAType& loader_a,
thread LoaderBType& loader_b,
thread ThreadgroupTileType& threadgroup_tile,
- thread const short& threadgroup_block_rows,
- thread const short& threadgroup_block_cols,
- thread const short& leftover_block_depth,
- LoopAlignment<M_aligned, N_aligned, K_aligned_> alignment = {}
+ thread const ushort& threadgroup_block_rows,
+ thread const ushort& threadgroup_block_cols,
+ thread const ushort& leftover_block_depth
) {
- (void)alignment;
-
short2 tile_dimensions_a =
transpose_a ? short2(threadgroup_block_rows, BLOCK_DEPTH)
: short2(BLOCK_DEPTH, threadgroup_block_rows);
@@ -104,7 +94,7 @@ struct ThreadgroupGemm {
transpose_b ? short2(BLOCK_DEPTH, threadgroup_block_cols)
: short2(threadgroup_block_cols, BLOCK_DEPTH);
- for (int k = 0; k < gemm_k_iterations; k++) {
+ for (int k = 0; k < aligned_k_iterations; k++) {
threadgroup_barrier(mem_flags::mem_threadgroup);
if (M_aligned) {
loader_a.load_unsafe();
@@ -149,25 +139,25 @@ struct ThreadgroupGemm {
const device T* a,
const device T* b,
device U* d,
- const constant GemmParams& params,
+ const constant GemmParams* params,
const constant float& ab_scale,
const bool is_accumulate,
threadgroup T* a_shared,
threadgroup T* b_shared,
uint simd_lane_id [[thread_index_in_simdgroup]],
uint simd_group_id [[simdgroup_index_in_threadgroup]],
- uint2 tid [[threadgroup_position_in_grid]],
- uint3 lid [[thread_position_in_threadgroup]]
+ uint2 threadgroup_position [[threadgroup_position_in_grid]],
+ uint3 thread_position [[thread_position_in_threadgroup]]
) {
- (void)lid;
+ (void)thread_position;
- const int swizzle_stride = pow2(params.swizzle_log);
- const int swizzled_row_id =
- tid.y * swizzle_stride + (tid.x % swizzle_stride);
- const int swizzled_col_id = tid.x / swizzle_stride;
+ const int swizzle_stride = pow2(params->swizzle_log);
+ const int swizzled_row_id = threadgroup_position.y * swizzle_stride +
+ (threadgroup_position.x % swizzle_stride);
+ const int swizzled_col_id = threadgroup_position.x / swizzle_stride;
- if (params.threadgroups_per_row <= swizzled_col_id ||
- params.threadgroups_per_column <= swizzled_row_id) {
+ if (params->threadgroups_per_row <= swizzled_col_id ||
+ params->threadgroups_per_column <= swizzled_row_id) {
return;
}
@@ -179,21 +169,21 @@ struct ThreadgroupGemm {
const size_t output_col_long = size_t(output_col);
a += transpose_a ? output_row_long
- : output_row_long * params.leading_dimension_a;
- b += transpose_b ? output_col_long * params.leading_dimension_b
+ : output_row_long * params->leading_dimension_a;
+ b += transpose_b ? output_col_long * params->leading_dimension_b
: output_col_long;
- d += output_row_long * params.leading_dimension_d + output_col_long;
+ d += output_row_long * params->leading_dimension_d + output_col_long;
thread LoaderAType loader_a(
a,
- params.leading_dimension_a,
+ params->leading_dimension_a,
a_shared,
simd_group_id,
simd_lane_id
);
thread LoaderBType loader_b(
b,
- params.leading_dimension_b,
+ params->leading_dimension_b,
b_shared,
simd_group_id,
simd_lane_id
@@ -201,17 +191,15 @@ struct ThreadgroupGemm {
thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);
- TransformScaleAccumulate<AccumType, AccumType> epilogue(
+ TransformScaleAccumulate<AccumulatorType, AccumulatorType> epilogue(
ab_scale,
is_accumulate ? 1.0f : 0.0f
);
- int gemm_k_iterations = params.aligned_inner_iterations;
-
///////////////////////////////////////////////////////////////////////////
// MNK aligned loop
if (MN_aligned) {
- for (int k = 0; k < gemm_k_iterations; k++) {
+ for (int k = 0; k < params->aligned_inner_iterations; k++) {
threadgroup_barrier(mem_flags::mem_threadgroup);
loader_a.load_unsafe();
loader_b.load_unsafe();
@@ -228,7 +216,7 @@ struct ThreadgroupGemm {
if (!K_aligned) {
int leftover_block_depth =
- params.K - params.aligned_inner_iterations * BLOCK_DEPTH;
+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
short2 tile_dimensions_a =
transpose_a ? short2(BLOCK_ROWS, leftover_block_depth)
: short2(leftover_block_depth, BLOCK_ROWS);
@@ -245,26 +233,26 @@ struct ThreadgroupGemm {
}
threadgroup_tile
- .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);
- threadgroup_tile.store_result(d, params.leading_dimension_d);
+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
+ threadgroup_tile.store_result(d, params->leading_dimension_d);
return;
}
///////////////////////////////////////////////////////////////////////////
// MN unaligned loop
else {
- short threadgroup_block_rows =
- min(BLOCK_ROWS, ((int)params.M) - output_row);
- short threadgroup_block_cols =
- min(BLOCK_COLS, ((int)params.N) - output_col);
- short leftover_block_depth =
- params.K - params.aligned_inner_iterations * BLOCK_DEPTH;
+ ushort threadgroup_block_rows =
+ min(BLOCK_ROWS, ((int)params->M) - output_row);
+ ushort threadgroup_block_cols =
+ min(BLOCK_COLS, ((int)params->N) - output_col);
+ ushort leftover_block_depth =
+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;
if (threadgroup_block_rows == BLOCK_ROWS &&
threadgroup_block_cols == BLOCK_COLS) {
gemm_loop<true, true, K_aligned>(
a_shared,
b_shared,
- gemm_k_iterations,
+ params->aligned_inner_iterations,
loader_a,
loader_b,
threadgroup_tile,
@@ -274,15 +262,15 @@ struct ThreadgroupGemm {
);
threadgroup_tile
- .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);
- threadgroup_tile.store_result(d, params.leading_dimension_d);
+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);
+ threadgroup_tile.store_result(d, params->leading_dimension_d);
return;
} else if (threadgroup_block_cols == BLOCK_COLS) {
gemm_loop<false, true, K_aligned>(
a_shared,
b_shared,
- gemm_k_iterations,
+ params->aligned_inner_iterations,
loader_a,
loader_b,
threadgroup_tile,
@@ -293,14 +281,14 @@ struct ThreadgroupGemm {
threadgroup_tile.apply_epilogue_safe(
d,
- params.leading_dimension_d,
+ params->leading_dimension_d,
1,
short2(threadgroup_block_cols, threadgroup_block_rows),
epilogue
);
threadgroup_tile.store_result_safe(
d,
- params.leading_dimension_d,
+ params->leading_dimension_d,
short2(threadgroup_block_cols, threadgroup_block_rows)
);
return;
@@ -309,7 +297,7 @@ struct ThreadgroupGemm {
gemm_loop<true, false, K_aligned>(
a_shared,
b_shared,
- gemm_k_iterations,
+ params->aligned_inner_iterations,
loader_a,
loader_b,
threadgroup_tile,
@@ -320,14 +308,14 @@ struct ThreadgroupGemm {
threadgroup_tile.apply_epilogue_safe(
d,
- params.leading_dimension_d,
+ params->leading_dimension_d,
1,
short2(threadgroup_block_cols, threadgroup_block_rows),
epilogue
);
threadgroup_tile.store_result_safe(
d,
- params.leading_dimension_d,
+ params->leading_dimension_d,
short2(threadgroup_block_cols, threadgroup_block_rows)
);
return;
@@ -336,7 +324,7 @@ struct ThreadgroupGemm {
gemm_loop<false, false, K_aligned>(
a_shared,
b_shared,
- gemm_k_iterations,
+ params->aligned_inner_iterations,
loader_a,
loader_b,
threadgroup_tile,
@@ -347,14 +335,14 @@ struct ThreadgroupGemm {
threadgroup_tile.apply_epilogue_safe(
d,
- params.leading_dimension_d,
+ params->leading_dimension_d,
1,
short2(threadgroup_block_cols, threadgroup_block_rows),
epilogue
);
threadgroup_tile.store_result_safe(
d,
- params.leading_dimension_d,
+ params->leading_dimension_d,
short2(threadgroup_block_cols, threadgroup_block_rows)
);
return;
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
index e51142ad7..77cc9092d 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h
@@ -1,11 +1,13 @@
#pragma once
-#include "defines.h"
-#include "loader.h"
+#include "../../common/defines.h"
+#include "../../common/integral_constant.h"
+using namespace uzu;
+#include "fragment.h"
+#include "mxu_fragment_ops.h"
+#include "mxu_gemm_loop.h"
#include "../../generated/matmul.h"
-#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>
-
using namespace metal;
namespace uzu {
@@ -13,109 +15,41 @@ namespace matmul {
template <
typename T,
- short BLOCK_ROWS,
- short BLOCK_COLS,
- short SIMDGROUPS_PER_ROW,
- short SIMDGROUPS_PER_COLUMN,
- short SUBTILE_ROWS = 16,
- short SUBTILE_COLS = 32,
- short MATMUL_K_STEP = 16,
- // Number of cooperative tensor elements owned by each thread.
- // Determined by the MPP compiler intrinsic get_capacity().
- // Empirically: SUBTILE_ROWS * SUBTILE_COLS / SIMD_SIZE = 16 * 32 / 32 = 16.
- short ACCUMULATOR_CAPACITY = 16>
-struct ThreadgroupGemmMpp {
- METAL_CONST short PREFETCH_K = 32;
- METAL_CONST short THREADGROUP_PADDING = short(16 / sizeof(T));
- METAL_CONST short THREADGROUP_LD = PREFETCH_K + THREADGROUP_PADDING;
- METAL_CONST short THREADGROUP_SIZE =
- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;
- METAL_CONST short SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;
- METAL_CONST short SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;
- METAL_CONST short TILES_M = SIMDGROUP_BLOCK_M / SUBTILE_ROWS;
- METAL_CONST short TILES_N = SIMDGROUP_BLOCK_N / SUBTILE_COLS;
- METAL_CONST short INNER_K_STEPS = PREFETCH_K / MATMUL_K_STEP;
+ ushort BLOCK_ROWS,
+ ushort BLOCK_COLS,
+ ushort SIMDGROUPS_PER_ROW,
+ ushort SIMDGROUPS_PER_COLUMN,
+ bool APPLY_AB_SCALE,
+ bool IS_ACCUMULATE>
+struct GemmMppCore {
+ METAL_CONST ushort SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;
+ METAL_CONST ushort SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;
+
+ METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;
+ METAL_CONST ushort BLOCK_K = 256;
+
+ METAL_CONST ushort TILES_M =
+ SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;
+ METAL_CONST ushort TILES_N =
+ SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;
using AccumulatorType = float;
- template <
- typename MatmulOperation,
- typename LeftTensor,
- typename RightTensor,
- typename AccumulatorTensor>
- static METAL_FUNC void accumulate_tiles(
- const short k_steps,
- const ushort tile_row_offset,
- const ushort tile_col_offset,
- threadgroup T* left_shared,
- threadgroup T* right_shared,
- thread MatmulOperation& matmul_operation,
- thread LeftTensor& left_tensor,
- thread RightTensor& right_tensor,
- thread AccumulatorTensor& accumulator_tensor,
- thread AccumulatorType
- accumulator_storage[TILES_M * TILES_N][ACCUMULATOR_CAPACITY]
- ) {
- METAL_PRAGMA_UNROLL
- for (short tile_m = 0; tile_m < TILES_M; tile_m++) {
- METAL_PRAGMA_UNROLL
- for (short tile_n = 0; tile_n < TILES_N; tile_n++) {
- const short subtile_index = tile_m * TILES_N + tile_n;
- const short left_row_base = tile_row_offset + tile_m * SUBTILE_ROWS;
- const short right_col_base = tile_col_offset + tile_n * SUBTILE_COLS;
-
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {
- accumulator_tensor[i] = accumulator_storage[subtile_index][i];
- }
-
- for (short k_step = 0; k_step < k_steps; k_step++) {
- const short k_offset = k_step * MATMUL_K_STEP;
-
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < left_tensor.get_capacity(); i++) {
- auto coord = left_tensor.get_multidimensional_index(i);
- left_tensor[i] = left_shared
- [(left_row_base + coord[1]) * THREADGROUP_LD + coord[0] +
- k_offset];
- }
-
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < right_tensor.get_capacity(); i++) {
- auto coord = right_tensor.get_multidimensional_index(i);
- right_tensor[i] = right_shared
- [(right_col_base + coord[1]) * THREADGROUP_LD + coord[0] +
- k_offset];
- }
-
- matmul_operation.run(left_tensor, right_tensor, accumulator_tensor);
- }
-
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {
- accumulator_storage[subtile_index][i] = accumulator_tensor[i];
- }
- }
- }
- }
-
static METAL_FUNC void run(
const device T* left_matrix,
const device T* right_matrix,
device T* output_matrix,
- const constant GemmParams& params,
+ const constant GemmParams* params,
const bool align_m,
const bool align_n,
- const bool is_accumulate,
+ const bool align_k,
const float ab_scale,
- threadgroup T* left_shared,
- threadgroup T* right_shared,
- uint simd_lane_id,
uint simd_group_id,
- uint2 threadgroup_position
+ uint2 threadgroup_position,
+ const thread ThreadContext& thread_context
) {
uint tile_id_x, tile_id_y;
- if (params.use_morton) {
+ if (params->use_morton) {
uint linear_id = threadgroup_position.x;
uint morton_x = linear_id;
uint morton_y = linear_id >> 1;
@@ -136,8 +70,8 @@ struct ThreadgroupGemmMpp {
tile_id_y = threadgroup_position.y;
}
- if (tile_id_x >= params.threadgroups_per_row ||
- tile_id_y >= params.threadgroups_per_column) {
+ if (tile_id_x >= params->threadgroups_per_row ||
+ tile_id_y >= params->threadgroups_per_column) {
return;
}
@@ -147,38 +81,9 @@ struct ThreadgroupGemmMpp {
const size_t block_col_start_long = size_t(block_col_start);
const device T* left_block_ptr =
- left_matrix + block_row_start_long * params.leading_dimension_a;
+ left_matrix + block_row_start_long * params->leading_dimension_a;
const device T* right_block_ptr =
- right_matrix + block_col_start_long * params.leading_dimension_b;
-
- ThreadgroupLoader<
- T,
- BLOCK_ROWS,
- PREFETCH_K,
- THREADGROUP_LD,
- 1,
- THREADGROUP_SIZE>
- left_loader(
- left_block_ptr,
- params.leading_dimension_a,
- left_shared,
- ushort(simd_group_id),
- ushort(simd_lane_id)
- );
- ThreadgroupLoader<
- T,
- BLOCK_COLS,
- PREFETCH_K,
- THREADGROUP_LD,
- 1,
- THREADGROUP_SIZE>
- right_loader(
- right_block_ptr,
- params.leading_dimension_b,
- right_shared,
- ushort(simd_group_id),
- ushort(simd_lane_id)
- );
+ right_matrix + block_col_start_long * params->leading_dimension_b;
const ushort tile_row_offset =
SIMDGROUP_BLOCK_M * (simd_group_id / SIMDGROUPS_PER_COLUMN);
@@ -186,8 +91,8 @@ struct ThreadgroupGemmMpp {
SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);
device T* output_ptr =
- output_matrix + block_row_start_long * params.leading_dimension_d +
- block_col_start_long + tile_row_offset * params.leading_dimension_d +
+ output_matrix + block_row_start_long * params->leading_dimension_d +
+ block_col_start_long + tile_row_offset * params->leading_dimension_d +
tile_col_offset;
const short simdgroup_limit_m =
@@ -195,166 +100,103 @@ struct ThreadgroupGemmMpp {
? SIMDGROUP_BLOCK_M
: short(
min(int(SIMDGROUP_BLOCK_M),
- int(params.M) - int(block_row_start + tile_row_offset))
+ int(params->M) - int(block_row_start + tile_row_offset))
);
const short simdgroup_limit_n =
align_n
? SIMDGROUP_BLOCK_N
: short(
min(int(SIMDGROUP_BLOCK_N),
- int(params.N) - int(block_col_start + tile_col_offset))
+ int(params->N) - int(block_col_start + tile_col_offset))
);
- constexpr auto matmul_descriptor = mpp::tensor_ops::matmul2d_descriptor(
- SUBTILE_ROWS,
- SUBTILE_COLS,
- MATMUL_K_STEP,
- false,
- true,
- false,
- mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate
- );
-
- mpp::tensor_ops::matmul2d<matmul_descriptor, metal::execution_simdgroup>
- matmul_operation;
-
- auto left_tensor =
- matmul_operation.template get_left_input_cooperative_tensor<
- T,
- T,
- AccumulatorType>();
- auto right_tensor =
- matmul_operation.template get_right_input_cooperative_tensor<
- T,
- T,
- AccumulatorType>();
- auto accumulator_tensor =
- matmul_operation.template get_destination_cooperative_tensor<
- decltype(left_tensor),
- decltype(right_tensor),
- AccumulatorType>();
-
- const uint leading_dimension_d = params.leading_dimension_d;
-
- AccumulatorType accumulator_storage[TILES_M * TILES_N]
- [ACCUMULATOR_CAPACITY] = {{0}};
-
- const uint full_prefetch_iterations = params.K / PREFETCH_K;
- const uint k_remainder = params.K - full_prefetch_iterations * PREFETCH_K;
-
- const ushort actual_block_rows =
- align_m ? BLOCK_ROWS
- : ushort(min(uint(BLOCK_ROWS), params.M - block_row_start));
- const ushort actual_block_cols =
- align_n ? BLOCK_COLS
- : ushort(min(uint(BLOCK_COLS), params.N - block_col_start));
-
- // Main loop
- for (uint outer_k = 0; outer_k < full_prefetch_iterations; outer_k++) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- if (align_m) {
- left_loader.load_unsafe();
- } else {
- left_loader.load_safe(short2(PREFETCH_K, actual_block_rows));
- }
- if (align_n) {
- right_loader.load_unsafe();
- } else {
- right_loader.load_safe(short2(PREFETCH_K, actual_block_cols));
- }
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- accumulate_tiles(
- INNER_K_STEPS,
- tile_row_offset,
- tile_col_offset,
- left_shared,
- right_shared,
- matmul_operation,
- left_tensor,
- right_tensor,
- accumulator_tensor,
- accumulator_storage
- );
-
- left_loader.next();
- right_loader.next();
- }
-
- // Remainder loop
- if (k_remainder > 0) {
- threadgroup_barrier(mem_flags::mem_threadgroup);
- left_loader.load_safe(short2(k_remainder, actual_block_rows));
- right_loader.load_safe(short2(k_remainder, actual_block_cols));
- threadgroup_barrier(mem_flags::mem_threadgroup);
-
- const short remainder_steps =
- short((k_remainder + MATMUL_K_STEP - 1) / MATMUL_K_STEP);
-
- accumulate_tiles(
- remainder_steps,
- tile_row_offset,
- tile_col_offset,
- left_shared,
- right_shared,
- matmul_operation,
- left_tensor,
- right_tensor,
- accumulator_tensor,
- accumulator_storage
- );
- }
-
- // Store results
- METAL_PRAGMA_UNROLL
- for (short tile_m = 0; tile_m < TILES_M; tile_m++) {
- METAL_PRAGMA_UNROLL
- for (short tile_n = 0; tile_n < TILES_N; tile_n++) {
- const short row_offset = tile_m * SUBTILE_ROWS;
- const short col_offset = tile_n * SUBTILE_COLS;
- const short m_limit =
- align_m ? SUBTILE_ROWS
- : short(max(0, int(simdgroup_limit_m) - row_offset));
- const short n_limit =
- align_n ? SUBTILE_COLS
- : short(max(0, int(simdgroup_limit_n) - col_offset));
- if (m_limit <= 0 || n_limit <= 0)
- continue;
-
- device T* output_tile_ptr =
- output_ptr + row_offset * leading_dimension_d + col_offset;
-
- METAL_PRAGMA_UNROLL
- for (short i = 0; i < short(ACCUMULATOR_CAPACITY); i++) {
- auto coord = accumulator_tensor.get_multidimensional_index(i);
- const bool is_valid_element = accumulator_tensor.is_valid_element(i);
- AccumulatorType accumulated_value =
- accumulator_storage[tile_m * TILES_N + tile_n][i] *
- AccumulatorType(ab_scale);
- if (align_m && align_n) {
- if (is_valid_element) {
- if (is_accumulate) {
- accumulated_value += AccumulatorType(
- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]
- );
- }
- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =
- T(accumulated_value);
- }
- } else {
- if (is_valid_element && coord[1] < m_limit && coord[0] < n_limit) {
- if (is_accumulate) {
- accumulated_value += AccumulatorType(
- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]
- );
- }
- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =
- T(accumulated_value);
- }
+ const device T* left_simdgroup_ptr =
+ left_block_ptr + size_t(tile_row_offset) * params->leading_dimension_a;
+ const device T* right_simdgroup_ptr =
+ right_block_ptr +
+ size_t(tile_col_offset) * int(params->leading_dimension_b);
+
+ const int aligned_k_iterations = int(params->K) / int(BLOCK_K);
+
+ dispatch_bool(align_k, [&](auto aligned_k) {
+ dispatch_bool(
+ align_m || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),
+ [&](auto aligned_m) {
+ dispatch_bool(
+ align_n || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),
+ [&](auto aligned_n) {
+ auto accumulator_tile = gemm_loop<
+ T,
+ SIMDGROUP_BLOCK_M,
+ SIMDGROUP_BLOCK_N,
+ SIMDGROUP_BLOCK_K,
+ BLOCK_K,
+ false,
+ true,
+ aligned_m.value,
+ aligned_n.value,
+ aligned_k.value,
+ AccumulatorType>(
+ left_simdgroup_ptr,
+ right_simdgroup_ptr,
+ int(params->leading_dimension_a),
+ int(params->leading_dimension_b),
+ int(params->K),
+ aligned_k_iterations,
+ simdgroup_limit_m,
+ simdgroup_limit_n,
+ thread_context
+ );
+
+ if constexpr (APPLY_AB_SCALE) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
+ i++) {
+ accumulator_tile.elements()[i] *=
+ AccumulatorType(ab_scale);
+ }
+ }
+
+ if constexpr (IS_ACCUMULATE) {
+ Fragment<T, TILES_M, TILES_N, MxuFragmentOps>
+ existing_output(thread_context);
+ if constexpr (aligned_m.value && aligned_n.value) {
+ existing_output.load(
+ output_ptr,
+ int(params->leading_dimension_d)
+ );
+ } else {
+ existing_output.load_safe(
+ output_ptr,
+ int(params->leading_dimension_d),
+ short2(simdgroup_limit_n, simdgroup_limit_m)
+ );
+ }
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;
+ i++) {
+ accumulator_tile.elements()[i] +=
+ AccumulatorType(existing_output.elements()[i]);
+ }
+ }
+
+ if constexpr (aligned_m.value && aligned_n.value) {
+ accumulator_tile.store(
+ output_ptr,
+ int(params->leading_dimension_d)
+ );
+ } else {
+ accumulator_tile.store_safe(
+ output_ptr,
+ int(params->leading_dimension_d),
+ short2(simdgroup_limit_n, simdgroup_limit_m)
+ );
+ }
+ }
+ );
}
- }
- }
- }
+ );
+ });
}
};
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
index 33e45d2c8..9fa6d3048 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h
@@ -1,6 +1,6 @@
#pragma once
-#include "defines.h"
+#include "../../common/defines.h"
using namespace metal;
@@ -13,32 +13,31 @@ namespace matmul {
template <
typename T,
- short BLOCK_ROWS,
- short BLOCK_COLS,
- short DESTINATION_LEADING_DIMENSION,
- short REDUCTION_DIMENSION,
- short THREADGROUP_SIZE,
- short ALIGNMENT = 1,
- short READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),
- short THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,
- short THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>
+ ushort BLOCK_ROWS,
+ ushort BLOCK_COLS,
+ ushort DESTINATION_LEADING_DIMENSION,
+ ushort REDUCTION_DIMENSION,
+ ushort THREADGROUP_SIZE,
+ ushort ALIGNMENT = 1,
+ ushort READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),
+ ushort THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,
+ ushort THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>
struct ThreadgroupLoader {
- METAL_CONST short ROW_ITERATIONS =
+ METAL_CONST ushort ROW_ITERATIONS =
(BLOCK_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;
- METAL_CONST short VECTOR_SIZE = READS_PER_THREAD;
const int source_leading_dimension;
const int tile_stride;
- const short thread_index;
- const short block_row_index;
- const short block_col_index;
+ const ushort thread_index;
+ const ushort block_row_index;
+ const ushort block_col_index;
threadgroup T* destination;
const device T* source;
struct alignas(ALIGNMENT * sizeof(T)) ReadVector {
- uint8_t bytes[sizeof(T) * VECTOR_SIZE];
+ uint8_t bytes[sizeof(T) * READS_PER_THREAD];
};
METAL_FUNC ThreadgroupLoader(
@@ -52,9 +51,9 @@ struct ThreadgroupLoader {
tile_stride(
REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim
),
- thread_index(simd_group_id * 32 + simd_lane_id),
+ thread_index(simd_group_id * METAL_SIMD_SIZE + simd_lane_id),
block_row_index(thread_index / THREAD_COLS),
- block_col_index(VECTOR_SIZE * (thread_index % THREAD_COLS)),
+ block_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),
destination(
destination_ptr + block_row_index * DESTINATION_LEADING_DIMENSION +
block_col_index
@@ -66,9 +65,9 @@ struct ThreadgroupLoader {
template <typename UnaryOp>
METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < VECTOR_SIZE; j++) {
+ for (ushort j = 0; j < READS_PER_THREAD; j++) {
destination[i * DESTINATION_LEADING_DIMENSION + j] =
operation.apply(destination[i * DESTINATION_LEADING_DIMENSION + j]);
}
@@ -77,7 +76,7 @@ struct ThreadgroupLoader {
METAL_FUNC void load_unsafe() const {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
*((threadgroup ReadVector*)(&destination
[i * DESTINATION_LEADING_DIMENSION])) =
*((const device ReadVector*)(&source[i * source_leading_dimension]));
@@ -90,39 +89,39 @@ struct ThreadgroupLoader {
if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < VECTOR_SIZE; j++) {
+ for (ushort j = 0; j < READS_PER_THREAD; j++) {
destination[i * DESTINATION_LEADING_DIMENSION + j] = T(0);
}
}
return;
}
- bool valid_mask[VECTOR_SIZE];
- T loaded_values[VECTOR_SIZE];
+ bool valid_mask[READS_PER_THREAD];
+ T loaded_values[READS_PER_THREAD];
METAL_PRAGMA_UNROLL
- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < VECTOR_SIZE; j++) {
+ for (ushort j = 0; j < READS_PER_THREAD; j++) {
valid_mask[j] =
(i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);
}
METAL_PRAGMA_UNROLL
- for (short j = 0; j < VECTOR_SIZE; j++) {
+ for (ushort j = 0; j < READS_PER_THREAD; j++) {
loaded_values[j] =
source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];
}
METAL_PRAGMA_UNROLL
- for (short j = 0; j < VECTOR_SIZE; j++) {
+ for (ushort j = 0; j < READS_PER_THREAD; j++) {
loaded_values[j] = valid_mask[j] ? loaded_values[j] : T(0);
}
METAL_PRAGMA_UNROLL
- for (short j = 0; j < VECTOR_SIZE; j++) {
+ for (ushort j = 0; j < READS_PER_THREAD; j++) {
destination[i * DESTINATION_LEADING_DIMENSION + j] = loaded_values[j];
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
new file mode 100644
index 000000000..9b31e884a
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h
@@ -0,0 +1,247 @@
+#pragma once
+
+#include <metal_simdgroup>
+#include <metal_simdgroup_matrix>
+#include <metal_stdlib>
+
+#include "../../common/integral_constant.h"
+#include "../../common/thread_context.h"
+using namespace uzu;
+
+#include "defines.h"
+
+#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+struct MxuFragmentOps {
+ METAL_CONST ushort FRAGMENT_ROWS = 16;
+ METAL_CONST ushort FRAGMENT_COLS = 16;
+
+ METAL_CONST ushort ELEMENTS_PER_THREAD =
+ (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;
+
+ METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;
+ METAL_CONST ushort THREAD_ELEMENT_COLS = 4;
+
+ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 8;
+
+ static_assert(
+ THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,
+ "MxuFragment shape is not consistent with element count"
+ );
+
+ template <typename U>
+ using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;
+
+ // Runs a 16x32 MPP matmul accumulating into (output_0 | output_1). The caller
+ // supplies a `marshal_inputs` callable that copies its fragments into the
+ // cooperative left/right tensors; the descriptor and output marshalling are
+ // shared between all call shapes.
+ template <
+ typename CType,
+ typename AType,
+ typename BType,
+ bool transpose_a,
+ bool transpose_b,
+ typename MarshalInputs>
+ METAL_FUNC static void mma_impl(
+ thread ThreadVector<CType>& output_0,
+ thread ThreadVector<CType>& output_1,
+ MarshalInputs marshal_inputs
+ ) {
+ constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(
+ FRAGMENT_ROWS,
+ 2 * FRAGMENT_COLS,
+ FRAGMENT_COLS,
+ transpose_a,
+ transpose_b,
+ true,
+ mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate
+ );
+
+ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;
+
+ auto cooperative_left =
+ matmul_op
+ .template get_left_input_cooperative_tensor<AType, BType, CType>();
+ auto cooperative_right =
+ matmul_op
+ .template get_right_input_cooperative_tensor<AType, BType, CType>();
+ auto cooperative_output =
+ matmul_op.template get_destination_cooperative_tensor<
+ decltype(cooperative_left),
+ decltype(cooperative_right),
+ CType>();
+
+ marshal_inputs(cooperative_left, cooperative_right);
+
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ cooperative_output[i] = output_0[i];
+ cooperative_output[ELEMENTS_PER_THREAD + i] = output_1[i];
+ }
+
+ matmul_op.run(cooperative_left, cooperative_right, cooperative_output);
+
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ output_0[i] = cooperative_output[i];
+ output_1[i] = cooperative_output[ELEMENTS_PER_THREAD + i];
+ }
+ }
+
+ // N-paired: output is two columns, left is one fragment, right is two.
+ template <
+ typename CType,
+ typename AType,
+ typename BType,
+ bool transpose_a = false,
+ bool transpose_b = false>
+ METAL_FUNC static void mma(
+ thread ThreadVector<CType>& output_col_0,
+ thread ThreadVector<CType>& output_col_1,
+ const thread ThreadVector<AType>& left,
+ metal::bool_constant<transpose_a>,
+ const thread ThreadVector<BType>& right_col_0,
+ const thread ThreadVector<BType>& right_col_1,
+ metal::bool_constant<transpose_b>
+ ) {
+ mma_impl<CType, AType, BType, transpose_a, transpose_b>(
+ output_col_0,
+ output_col_1,
+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ cooperative_left[i] = left[i];
+ cooperative_right[i] = right_col_0[i];
+ cooperative_right[ELEMENTS_PER_THREAD + i] = right_col_1[i];
+ }
+ }
+ );
+ }
+
+ // M-paired: output is two rows, left is two fragments, right is one.
+ template <
+ typename CType,
+ typename AType,
+ typename BType,
+ bool transpose_a = false,
+ bool transpose_b = false>
+ METAL_FUNC static void mma(
+ thread ThreadVector<CType>& output_row_0,
+ thread ThreadVector<CType>& output_row_1,
+ const thread ThreadVector<AType>& left_row_0,
+ const thread ThreadVector<AType>& left_row_1,
+ metal::bool_constant<transpose_a>,
+ const thread ThreadVector<BType>& right,
+ metal::bool_constant<transpose_b>
+ ) {
+ mma_impl<CType, AType, BType, transpose_a, transpose_b>(
+ output_row_0,
+ output_row_1,
+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {
+ METAL_PRAGMA_UNROLL
+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {
+ cooperative_left[i] = left_row_0[i];
+ cooperative_left[ELEMENTS_PER_THREAD + i] = left_row_1[i];
+ cooperative_right[i] = right[i];
+ }
+ }
+ );
+ }
+
+ template <
+ bool transpose_a,
+ bool transpose_b,
+ class OutputTile,
+ class LeftTile,
+ class RightTile>
+ METAL_FUNC static void tile_matmul(
+ thread OutputTile& output,
+ thread LeftTile& left,
+ thread RightTile& right
+ ) {
+ constexpr ushort left_tile_m =
+ transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;
+ constexpr ushort tile_m = OutputTile::TILE_ROWS;
+ static_assert(
+ left_tile_m == tile_m,
+ "tile matmul: M dimensions do not match"
+ );
+
+ constexpr ushort right_tile_n =
+ transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;
+ constexpr ushort tile_n = OutputTile::TILE_COLS;
+ static_assert(
+ right_tile_n == tile_n,
+ "tile matmul: N dimensions do not match"
+ );
+
+ constexpr ushort left_tile_k =
+ transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;
+ constexpr ushort tile_k =
+ transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;
+ static_assert(
+ left_tile_k == tile_k,
+ "tile matmul: K dimensions do not match"
+ );
+
+ constexpr auto transpose_left = metal::bool_constant<transpose_a>{};
+ constexpr auto transpose_right = metal::bool_constant<transpose_b>{};
+
+ if constexpr (tile_n == 1 && tile_m % 2 == 0) {
+ METAL_PRAGMA_UNROLL
+ for (ushort row = 0; row < tile_m; row += 2) {
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < tile_n; ++col) {
+ METAL_PRAGMA_UNROLL
+ for (ushort k = 0; k < tile_k; ++k) {
+ mma<typename OutputTile::ElementType,
+ typename LeftTile::ElementType,
+ typename RightTile::ElementType,
+ transpose_a,
+ transpose_b>(
+ output.fragment_at(row, col),
+ output.fragment_at(row + 1, col),
+ left.fragment_at(row, k, transpose_left),
+ left.fragment_at(row + 1, k, transpose_left),
+ transpose_left,
+ right.fragment_at(k, col, transpose_right),
+ transpose_right
+ );
+ }
+ }
+ }
+ } else if constexpr (tile_n % 2 == 0) {
+ METAL_PRAGMA_UNROLL
+ for (ushort row = 0; row < tile_m; ++row) {
+ METAL_PRAGMA_UNROLL
+ for (ushort col = 0; col < tile_n; col += 2) {
+ METAL_PRAGMA_UNROLL
+ for (ushort k = 0; k < tile_k; ++k) {
+ mma<typename OutputTile::ElementType,
+ typename LeftTile::ElementType,
+ typename RightTile::ElementType,
+ transpose_a,
+ transpose_b>(
+ output.fragment_at(row, col),
+ output.fragment_at(row, col + 1),
+ left.fragment_at(row, k, transpose_left),
+ transpose_left,
+ right.fragment_at(k, col, transpose_right),
+ right.fragment_at(k, col + 1, transpose_right),
+ transpose_right
+ );
+ }
+ }
+ }
+ }
+ }
+};
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
new file mode 100644
index 000000000..a00c9b4ec
--- /dev/null
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h
@@ -0,0 +1,155 @@
+#pragma once
+
+#include "fragment.h"
+#include "mxu_fragment_ops.h"
+#include "../../generated/matmul.h"
+
+using namespace metal;
+
+namespace uzu {
+namespace matmul {
+
+template <
+ typename T,
+ ushort SIMDGROUP_BLOCK_M,
+ ushort SIMDGROUP_BLOCK_N,
+ ushort SIMDGROUP_BLOCK_K,
+ ushort BLOCK_K,
+ bool transpose_a,
+ bool transpose_b,
+ bool aligned_m,
+ bool aligned_n,
+ bool aligned_k,
+ typename AccumulatorType = float>
+METAL_FUNC auto gemm_loop(
+ const device T* left_ptr,
+ const device T* right_ptr,
+ int leading_dimension_a,
+ int leading_dimension_b,
+ int K,
+ int aligned_k_iterations,
+ const short simdgroup_limit_m,
+ const short simdgroup_limit_n,
+ const thread ThreadContext& thread_context
+) {
+ constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;
+ constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;
+ constexpr ushort TILES_K = SIMDGROUP_BLOCK_K / MxuFragmentOps::FRAGMENT_ROWS;
+
+ constexpr ushort LEFT_TILE_ROWS = transpose_a ? TILES_K : TILES_M;
+ constexpr ushort LEFT_TILE_COLS = transpose_a ? TILES_M : TILES_K;
+
+ constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;
+ constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;
+
+ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(
+ thread_context
+ );
+ accumulator.clear();
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int outer_k = 0; outer_k < aligned_k_iterations; outer_k++) {
+ threadgroup_barrier(mem_flags::mem_none);
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {
+ Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
+ thread_context
+ );
+ Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
+ thread_context
+ );
+
+ const int left_offset =
+ transpose_a ? inner_k * leading_dimension_a : inner_k;
+ const int right_offset =
+ transpose_b ? inner_k : inner_k * leading_dimension_b;
+
+ if constexpr (aligned_m) {
+ left_tile.load(left_ptr + left_offset, leading_dimension_a);
+ } else {
+ const short row_limit =
+ transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;
+ const short col_limit =
+ transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;
+ left_tile.load_safe(
+ left_ptr + left_offset,
+ leading_dimension_a,
+ short2(col_limit, row_limit)
+ );
+ }
+
+ if constexpr (aligned_n) {
+ right_tile.load(right_ptr + right_offset, leading_dimension_b);
+ } else {
+ const short row_limit =
+ transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;
+ const short col_limit =
+ transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;
+ right_tile.load_safe(
+ right_ptr + right_offset,
+ leading_dimension_b,
+ short2(col_limit, row_limit)
+ );
+ }
+
+ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(
+ accumulator,
+ left_tile,
+ right_tile
+ );
+ }
+
+ left_ptr += transpose_a ? (BLOCK_K * leading_dimension_a) : BLOCK_K;
+ right_ptr += transpose_b ? BLOCK_K : (BLOCK_K * leading_dimension_b);
+ }
+
+ if constexpr (!aligned_k) {
+ simdgroup_barrier(mem_flags::mem_none);
+
+ const short remaining_k = K - aligned_k_iterations * BLOCK_K;
+
+ METAL_PRAGMA_NO_UNROLL
+ for (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {
+ Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(
+ thread_context
+ );
+ Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(
+ thread_context
+ );
+
+ const short safe_k = max(short(0), short(remaining_k - inner_k));
+
+ const short2 left_limits = transpose_a
+ ? short2(simdgroup_limit_m, safe_k)
+ : short2(safe_k, simdgroup_limit_m);
+ const short2 right_limits = transpose_b
+ ? short2(safe_k, simdgroup_limit_n)
+ : short2(simdgroup_limit_n, safe_k);
+
+ const int left_offset =
+ transpose_a ? inner_k * leading_dimension_a : inner_k;
+ const int right_offset =
+ transpose_b ? inner_k : inner_k * leading_dimension_b;
+
+ left_tile
+ .load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);
+ right_tile.load_safe(
+ right_ptr + right_offset,
+ leading_dimension_b,
+ right_limits
+ );
+
+ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(
+ accumulator,
+ left_tile,
+ right_tile
+ );
+ }
+ }
+
+ return accumulator;
+}
+
+} // namespace matmul
+} // namespace uzu
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
index 5c8dafae6..2db44e7a1 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h
@@ -50,21 +50,21 @@ struct SimdgroupFragment {
METAL_FUNC constexpr void clear() {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {
+ for (ushort i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {
multiply_accumulate_data[i] = ThreadDataType(0);
}
}
METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(
- const short i,
- const short j
+ const ushort i,
+ const ushort j
) {
return multiply_accumulate_data[i * GRID_COLS + j];
}
METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(
- const short i,
- const short j
+ const ushort i,
+ const ushort j
) const {
return multiply_accumulate_data[i * GRID_COLS + j];
}
@@ -85,19 +85,18 @@ struct SimdgroupFragment {
int SIMDGROUP_STRIDE_Y,
int STRIDE_X,
int STRIDE_Y>
- METAL_FUNC void load(const threadgroup U* src) {
+ METAL_FUNC void load(const threadgroup U* source) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < GRID_ROWS; ++i) {
+ for (ushort i = 0; i < GRID_ROWS; ++i) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < GRID_COLS; ++j) {
+ for (ushort j = 0; j < GRID_COLS; ++j) {
SimdgroupMultiplyAccumulateOpsType::load(
multiply_accumulate_at(i, j),
- &(
- src[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
- STRIDE_X +
- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *
- STRIDE_Y]
- ),
+ &(source
+ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
+ STRIDE_X +
+ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *
+ STRIDE_Y]),
STRIDE_X,
STRIDE_Y
);
@@ -106,18 +105,20 @@ struct SimdgroupFragment {
}
template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>
- METAL_FUNC void store(device U* dst, const int leading_dimension) const {
+ METAL_FUNC void store(
+ device U* destination,
+ const int leading_dimension
+ ) const {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < GRID_ROWS; ++i) {
+ for (ushort i = 0; i < GRID_ROWS; ++i) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < GRID_COLS; ++j) {
+ for (ushort j = 0; j < GRID_COLS; ++j) {
SimdgroupMultiplyAccumulateOpsType::store(
multiply_accumulate_at(i, j),
- &(
- dst[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
- leading_dimension +
- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]
- ),
+ &(destination
+ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *
+ leading_dimension +
+ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]),
leading_dimension,
1
);
@@ -127,7 +128,7 @@ struct SimdgroupFragment {
template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>
METAL_FUNC void store_safe(
- device U* dst,
+ device U* destination,
const int leading_dimension,
const short2 destination_tile_dimensions
) const {
@@ -137,7 +138,7 @@ struct SimdgroupFragment {
for (int j = 0; j < GRID_COLS; ++j) {
SimdgroupMultiplyAccumulateOpsType::store_safe(
multiply_accumulate_at(i, j),
- dst,
+ destination,
leading_dimension,
1,
destination_tile_dimensions.y,
@@ -162,12 +163,12 @@ METAL_FUNC void tile_multiply_accumulate(
thread SimdgroupFragment<T, M, N>& C
) {
METAL_PRAGMA_UNROLL
- for (short m = 0; m < M; ++m) {
+ for (ushort m = 0; m < M; ++m) {
METAL_PRAGMA_UNROLL
- for (short n = 0; n < N; ++n) {
- short column_serpentine = (m % 2) ? (N - 1 - n) : n;
+ for (ushort n = 0; n < N; ++n) {
+ const ushort column_serpentine = (m % 2) ? (N - 1 - n) : n;
METAL_PRAGMA_UNROLL
- for (short k = 0; k < K; ++k) {
+ for (ushort k = 0; k < K; ++k) {
SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::
multiply_accumulate(
D.multiply_accumulate_at(m, column_serpentine),
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
index 6c4ddb93e..27860a3ac 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h
@@ -27,7 +27,7 @@ template <typename T>
struct SimdgroupMultiplyAccumulate<T, 8, 8> {
METAL_CONST int ROWS = 8;
METAL_CONST int COLS = 8;
- METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / 32;
+ METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;
METAL_CONST int THREAD_ELEMENT_ROWS = 1;
METAL_CONST int THREAD_ELEMENT_COLS = 2;
@@ -48,102 +48,107 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {
return short2{lane_col, lane_row};
}
- template <typename SrcPtrType, typename StrideX, typename StrideY>
+ template <typename SourcePointerType, typename RowStride, typename ColStride>
METAL_FUNC static constexpr void load(
- thread ThreadDataType& dst,
- SrcPtrType src,
- StrideX stride_x,
- StrideY stride_y
+ thread ThreadDataType& destination,
+ SourcePointerType source,
+ RowStride row_stride,
+ ColStride col_stride
) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
- dst[i * THREAD_ELEMENT_COLS + j] =
- static_cast<T>(src[i * stride_x + j * stride_y]);
+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ destination[i * THREAD_ELEMENT_COLS + j] =
+ static_cast<T>(source[i * row_stride + j * col_stride]);
}
}
}
template <
- typename SrcPtrType,
- typename StrideX,
- typename StrideY,
- typename LimitX,
- typename LimitY,
- typename OffsetX,
- typename OffsetY>
+ typename SourcePointerType,
+ typename RowStride,
+ typename ColStride,
+ typename RowLimit,
+ typename ColLimit,
+ typename RowOffset,
+ typename ColOffset>
METAL_FUNC static constexpr void load_safe(
- thread ThreadDataType& dst,
- SrcPtrType src,
- StrideX stride_x,
- StrideY stride_y,
- LimitX limit_x,
- LimitY limit_y,
- OffsetX offset_x = 0,
- OffsetY offset_y = 0
+ thread ThreadDataType& destination,
+ SourcePointerType source,
+ RowStride row_stride,
+ ColStride col_stride,
+ RowLimit row_limit,
+ ColLimit col_limit,
+ RowOffset row_offset = 0,
+ ColOffset col_offset = 0
) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
- if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {
- dst[i * THREAD_ELEMENT_COLS + j] = static_cast<T>(
- src[(offset_x + i) * stride_x + (offset_y + j) * stride_y]
- );
+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {
+ destination[i * THREAD_ELEMENT_COLS + j] =
+ static_cast<T>(source
+ [(row_offset + i) * row_stride +
+ (col_offset + j) * col_stride]);
} else {
- dst[i * THREAD_ELEMENT_COLS + j] = T(0);
+ destination[i * THREAD_ELEMENT_COLS + j] = T(0);
}
}
}
}
- template <typename DstPtrType, typename StrideX, typename StrideY>
+ template <
+ typename DestinationPointerType,
+ typename RowStride,
+ typename ColStride>
METAL_FUNC static constexpr void store(
- const thread ThreadDataType& src,
- DstPtrType dst,
- StrideX stride_x,
- StrideY stride_y
+ const thread ThreadDataType& source,
+ DestinationPointerType destination,
+ RowStride row_stride,
+ ColStride col_stride
) {
- using U = PointerElementType<DstPtrType>;
+ using U = PointerElementType<DestinationPointerType>;
METAL_PRAGMA_UNROLL
- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
- dst[i * stride_x + j * stride_y] =
- static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);
+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ destination[i * row_stride + j * col_stride] =
+ static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);
}
}
}
template <
- typename DstPtrType,
- typename StrideX,
- typename StrideY,
- typename LimitX,
- typename LimitY,
- typename OffsetX,
- typename OffsetY>
+ typename DestinationPointerType,
+ typename RowStride,
+ typename ColStride,
+ typename RowLimit,
+ typename ColLimit,
+ typename RowOffset,
+ typename ColOffset>
METAL_FUNC static constexpr void store_safe(
- const thread ThreadDataType& src,
- DstPtrType dst,
- StrideX stride_x,
- StrideY stride_y,
- LimitX limit_x,
- LimitY limit_y,
- OffsetX offset_x = 0,
- OffsetY offset_y = 0
+ const thread ThreadDataType& source,
+ DestinationPointerType destination,
+ RowStride row_stride,
+ ColStride col_stride,
+ RowLimit row_limit,
+ ColLimit col_limit,
+ RowOffset row_offset = 0,
+ ColOffset col_offset = 0
) {
- using U = PointerElementType<DstPtrType>;
+ using U = PointerElementType<DestinationPointerType>;
METAL_PRAGMA_UNROLL
- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {
+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {
- if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {
- dst[(offset_x + i) * stride_x + (offset_y + j) * stride_y] =
- static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);
+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {
+ if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {
+ destination
+ [(row_offset + i) * row_stride + (col_offset + j) * col_stride] =
+ static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);
}
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
index c323dd73e..082cd7284 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h
@@ -21,67 +21,77 @@ template <
int SIMDGROUPS_PER_COLUMN,
bool transpose_a,
bool transpose_b,
- short THREADGROUP_LEADING_DIMENSION_A,
- short THREADGROUP_LEADING_DIMENSION_B,
- typename AccumType = float,
- typename Epilogue = TransformNone<U, AccumType>>
+ ushort THREADGROUP_LEADING_DIMENSION_A,
+ ushort THREADGROUP_LEADING_DIMENSION_B,
+ typename AccumulatorType = float,
+ typename Epilogue = TransformNone<U, AccumulatorType>>
struct ThreadgroupTile {
- METAL_CONST short SIMDGROUP_BLOCK_SIZE = 8;
+ METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;
using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<
- AccumType,
+ AccumulatorType,
SIMDGROUP_BLOCK_SIZE,
SIMDGROUP_BLOCK_SIZE>;
- METAL_CONST short TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
- METAL_CONST short TILE_COL_STRIDE =
+ METAL_CONST ushort TILE_ROW_STRIDE =
+ SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
+ METAL_CONST ushort TILE_COL_STRIDE =
SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
- METAL_CONST short TILE_ROWS =
+ METAL_CONST ushort TILE_ROWS =
BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
- METAL_CONST short TILE_COLS =
+ METAL_CONST ushort TILE_COLS =
BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
- METAL_CONST short A_STRIDE_ROW =
+ METAL_CONST ushort A_STRIDE_ROW =
transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;
- METAL_CONST short A_STRIDE_INNER =
+ METAL_CONST ushort A_STRIDE_INNER =
transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;
- METAL_CONST short B_STRIDE_INNER =
+ METAL_CONST ushort B_STRIDE_INNER =
transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;
- METAL_CONST short B_STRIDE_COL =
+ METAL_CONST ushort B_STRIDE_COL =
transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;
- METAL_CONST short TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;
- METAL_CONST short TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;
+ METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;
+ METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;
- SimdgroupFragment<AccumType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType>
+ SimdgroupFragment<
+ AccumulatorType,
+ TILE_ROWS,
+ 1,
+ SimdgroupMultiplyAccumulateType>
a_fragment;
- SimdgroupFragment<AccumType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType>
+ SimdgroupFragment<
+ AccumulatorType,
+ 1,
+ TILE_COLS,
+ SimdgroupMultiplyAccumulateType>
b_fragment;
SimdgroupFragment<
- AccumType,
+ AccumulatorType,
TILE_ROWS,
TILE_COLS,
SimdgroupMultiplyAccumulateType>
c_fragment;
- short simdgroup_row_offset;
- short simdgroup_col_offset;
+ ushort simdgroup_row_offset;
+ ushort simdgroup_col_offset;
- short a_shared_offset;
- short b_shared_offset;
+ ushort a_shared_offset;
+ ushort b_shared_offset;
METAL_FUNC ThreadgroupTile(
ushort simd_group_id [[simdgroup_index_in_threadgroup]],
ushort simd_lane_id [[thread_index_in_simdgroup]]
) {
- short tile_row_base =
+ ushort tile_row_base =
SIMDGROUP_BLOCK_SIZE * (simd_group_id / SIMDGROUPS_PER_COLUMN);
- short tile_col_base =
+ ushort tile_col_base =
SIMDGROUP_BLOCK_SIZE * (simd_group_id % SIMDGROUPS_PER_COLUMN);
- short2 simdgroup_coordinates =
- SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id);
+ const ushort2 simdgroup_coordinates = ushort2(
+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id)
+ );
simdgroup_row_offset = simdgroup_coordinates.y;
simdgroup_col_offset = simdgroup_coordinates.x;
@@ -102,7 +112,7 @@ struct ThreadgroupTile {
b_shared += b_shared_offset;
METAL_PRAGMA_UNROLL
- for (short k_block_index = 0; k_block_index < BLOCK_DEPTH;
+ for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH;
k_block_index += SIMDGROUP_BLOCK_SIZE) {
simdgroup_barrier(mem_flags::mem_none);
@@ -133,7 +143,7 @@ struct ThreadgroupTile {
METAL_FUNC void store_result(device U* D, const int leading_dimension_d) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
+ for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
}
@@ -151,7 +161,7 @@ struct ThreadgroupTile {
short2 destination_tile_dimensions
) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
+ for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {
c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);
}
@@ -182,22 +192,24 @@ struct ThreadgroupTile {
simdgroup_col_offset * column_stride_c;
METAL_PRAGMA_UNROLL
- for (short i = 0; i < TILE_ROWS; i++) {
+ for (ushort i = 0; i < TILE_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < TILE_COLS; j++) {
+ for (ushort j = 0; j < TILE_COLS; j++) {
thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
METAL_PRAGMA_UNROLL
- for (short k = 0;
+ for (ushort k = 0;
k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
k++) {
- short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
- short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
+ const ushort row_offset =
+ i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
+ const ushort col_offset =
+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
U c_value = c_pointer
[row_offset * leading_dimension_c + col_offset * column_stride_c +
k];
block_data[k] = epilogue_operation.apply(
block_data[k],
- static_cast<AccumType>(c_value)
+ static_cast<AccumulatorType>(c_value)
);
}
}
@@ -217,14 +229,15 @@ struct ThreadgroupTile {
tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);
METAL_PRAGMA_UNROLL
- for (short i = 0; i < TILE_ROWS; i++) {
+ for (ushort i = 0; i < TILE_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < TILE_COLS; j++) {
+ for (ushort j = 0; j < TILE_COLS; j++) {
thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);
- short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);
- short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);
+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;
+ const ushort col_offset =
+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;
METAL_PRAGMA_UNROLL
- for (short k = 0;
+ for (ushort k = 0;
k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;
k++) {
if (row_offset < tile_dimensions.y &&
@@ -234,7 +247,7 @@ struct ThreadgroupTile {
col_offset * column_stride_c + k];
block_data[k] = epilogue_operation.apply(
block_data[k],
- static_cast<AccumType>(c_value)
+ static_cast<AccumulatorType>(c_value)
);
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
index 69313b32a..f790479f0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal
@@ -22,7 +22,7 @@ KERNEL(MatmulGemm)(
const device T* a,
const device T* b,
device T* d,
- const constant uzu::matmul::GemmParams& params,
+ const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
const constant float& ab_scale,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
index 0a1c579a3..6fc7399d7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal
@@ -5,52 +5,58 @@
using namespace uzu::matmul;
-#define GEMM_MPP_THREADGROUP_TILE_SIZE (128 * (32 + (short)(16 / sizeof(T))))
-
template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN>
VARIANTS(T, float, half, bfloat)
-VARIANTS(BLOCK_ROWS, 32, 64)
-VARIANTS(BLOCK_COLS, 32, 64)
+VARIANTS(BLOCK_ROWS, 32, 64, 128)
+VARIANTS(BLOCK_COLS, 32, 64, 128)
VARIANTS(SIMDGROUPS_PER_ROW, 2, 4)
-VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)
-CONSTRAINT(BLOCK_ROWS >= 16 * SIMDGROUPS_PER_ROW && BLOCK_COLS >= 32 * SIMDGROUPS_PER_COLUMN)
+VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2, 4)
+CONSTRAINT(
+ (BLOCK_ROWS == 32 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||
+ (BLOCK_ROWS == 64 && BLOCK_COLS == 32 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 1) ||
+ (BLOCK_ROWS == 64 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||
+ (BLOCK_ROWS == 128 && BLOCK_COLS == 128 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 4))
KERNEL(MatmulGemmMpp)(
const device T* left_matrix,
const device T* right_matrix,
device T* output_matrix,
- const constant uzu::matmul::GemmParams& params,
+ const constant uzu::matmul::GemmParams* params,
const constant uint& group_count_x,
const constant uint& group_count_y,
const constant float& ab_scale,
const bool align_m SPECIALIZE,
const bool align_n SPECIALIZE,
+ const bool align_k SPECIALIZE,
+ const bool apply_ab_scale SPECIALIZE,
const bool is_accumulate SPECIALIZE,
- threadgroup T left_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],
- threadgroup T right_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],
const uint group_x GROUPS(group_count_x),
const uint group_y GROUPS(group_count_y),
const uint thread_x THREADS(32),
- const uint thread_y THREADS(2),
- const uint thread_z THREADS(2),
+ const uint thread_y THREADS(SIMDGROUPS_PER_ROW),
+ const uint thread_z THREADS(SIMDGROUPS_PER_COLUMN),
const ThreadContext thread_context
) {
- ThreadgroupGemmMpp<
- T,
- BLOCK_ROWS,
- BLOCK_COLS,
- SIMDGROUPS_PER_ROW,
- SIMDGROUPS_PER_COLUMN>::
- run(left_matrix,
- right_matrix,
- output_matrix,
- params,
- align_m,
- align_n,
- is_accumulate,
- ab_scale,
- left_shared,
- right_shared,
- thread_context.simdgroup_index,
- thread_context.threadgroup_index,
- uint2(group_x, group_y));
+ dispatch_bool(apply_ab_scale, [&](auto scale_enabled) {
+ dispatch_bool(is_accumulate, [&](auto accumulate_enabled) {
+ GemmMppCore<
+ T,
+ BLOCK_ROWS,
+ BLOCK_COLS,
+ SIMDGROUPS_PER_ROW,
+ SIMDGROUPS_PER_COLUMN,
+ scale_enabled.value,
+ accumulate_enabled.value>::
+ run(left_matrix,
+ right_matrix,
+ output_matrix,
+ params,
+ align_m,
+ align_n,
+ align_k,
+ ab_scale,
+ thread_context.threadgroup_index,
+ uint2(group_x, group_y),
+ thread_context);
+ });
+ });
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs
index 03510f4e6..a957795d0 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs
@@ -1,3 +1,5 @@
+use std::sync::OnceLock;
+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct GemmMppSpecialization {
pub block_rows: u32,
@@ -6,169 +8,106 @@ pub struct GemmMppSpecialization {
pub simdgroups_per_column: u32,
pub align_m: bool,
pub align_n: bool,
+ pub align_k: bool,
+ pub apply_ab_scale: bool,
pub is_accumulate: bool,
}
+const fn config(
+ block_rows: u32,
+ block_cols: u32,
+ simdgroups_per_row: u32,
+ simdgroups_per_column: u32,
+ align_m: bool,
+ align_n: bool,
+ is_accumulate: bool,
+) -> GemmMppSpecialization {
+ GemmMppSpecialization {
+ block_rows,
+ block_cols,
+ simdgroups_per_row,
+ simdgroups_per_column,
+ align_m,
+ align_n,
+ align_k: false,
+ apply_ab_scale: false,
+ is_accumulate,
+ }
+}
+
+static GEMM_MPP_PRECOMPILE: OnceLock<&'static [GemmMppSpecialization]> = OnceLock::new();
+
impl GemmMppSpecialization {
pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {
if !matches!(data_type, crate::DataType::F16 | crate::DataType::BF16) {
return &[];
}
- &[
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: true,
- align_n: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: false,
- align_n: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: true,
- align_n: false,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: false,
- align_n: false,
- is_accumulate: false,
- },
- Self {
- block_rows: 32,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: true,
- align_n: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 32,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: false,
- align_n: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 32,
- simdgroups_per_row: 4,
- simdgroups_per_column: 1,
- align_m: true,
- align_n: true,
- is_accumulate: false,
- },
- Self {
- block_rows: 64,
- block_cols: 32,
- simdgroups_per_row: 4,
- simdgroups_per_column: 1,
- align_m: true,
- align_n: false,
- is_accumulate: false,
- },
- // Accumulate variants
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: true,
- align_n: true,
- is_accumulate: true,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: false,
- align_n: true,
- is_accumulate: true,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: true,
- align_n: false,
- is_accumulate: true,
- },
- Self {
- block_rows: 64,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: false,
- align_n: false,
- is_accumulate: true,
- },
- Self {
- block_rows: 32,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: true,
- align_n: true,
- is_accumulate: true,
- },
- Self {
- block_rows: 32,
- block_cols: 64,
- simdgroups_per_row: 2,
- simdgroups_per_column: 2,
- align_m: false,
- align_n: true,
- is_accumulate: true,
- },
- Self {
- block_rows: 64,
- block_cols: 32,
- simdgroups_per_row: 4,
- simdgroups_per_column: 1,
- align_m: true,
- align_n: true,
- is_accumulate: true,
- },
- Self {
- block_rows: 64,
- block_cols: 32,
- simdgroups_per_row: 4,
- simdgroups_per_column: 1,
- align_m: true,
- align_n: false,
- is_accumulate: true,
- },
- ]
+ GEMM_MPP_PRECOMPILE.get_or_init(|| {
+ const BASE: &[GemmMppSpecialization] = &[
+ config(64, 64, 2, 2, true, true, false),
+ config(64, 64, 2, 2, false, true, false),
+ config(64, 64, 2, 2, true, false, false),
+ config(64, 64, 2, 2, false, false, false),
+ config(32, 64, 2, 2, true, true, false),
+ config(32, 64, 2, 2, false, true, false),
+ config(64, 32, 4, 1, true, true, false),
+ config(64, 32, 4, 1, true, false, false),
+ config(128, 128, 4, 4, true, true, false),
+ config(128, 128, 4, 4, false, true, false),
+ config(128, 128, 4, 4, true, false, false),
+ config(128, 128, 4, 4, false, false, false),
+ config(64, 64, 2, 2, true, true, true),
+ config(64, 64, 2, 2, false, true, true),
+ config(64, 64, 2, 2, true, false, true),
+ config(64, 64, 2, 2, false, false, true),
+ config(32, 64, 2, 2, true, true, true),
+ config(32, 64, 2, 2, false, true, true),
+ config(64, 32, 4, 1, true, true, true),
+ config(64, 32, 4, 1, true, false, true),
+ config(128, 128, 4, 4, true, true, true),
+ config(128, 128, 4, 4, false, true, true),
+ config(128, 128, 4, 4, true, false, true),
+ config(128, 128, 4, 4, false, false, true),
+ ];
+
+ let mut expanded = Vec::with_capacity(BASE.len() * 4);
+ for &base in BASE {
+ expanded.push(Self {
+ align_k: true,
+ apply_ab_scale: true,
+ ..base
+ });
+ expanded.push(Self {
+ align_k: true,
+ apply_ab_scale: false,
+ ..base
+ });
+ expanded.push(Self {
+ align_k: false,
+ apply_ab_scale: true,
+ ..base
+ });
+ expanded.push(Self {
+ align_k: false,
+ apply_ab_scale: false,
+ ..base
+ });
+ }
+
+ Box::leak(expanded.into_boxed_slice())
+ })
}
pub fn select(
m: u32,
n: u32,
+ k: u32,
is_accumulate: bool,
+ apply_ab_scale: bool,
) -> Self {
- let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if n < 64 {
+ let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if m >= 128 && n >= 128 {
+ (128, 128, 4u32, 4u32)
+ } else if n < 64 {
(64, 32, 4u32, 1u32)
} else if m < 64 {
(32, 64, 2u32, 2u32)
@@ -183,6 +122,8 @@ impl GemmMppSpecialization {
simdgroups_per_column,
align_m: (m % block_rows) == 0,
align_n: (n % block_cols) == 0,
+ align_k: (k % 256) == 0,
+ apply_ab_scale,
is_accumulate,
}
}
diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
index c26498367..f9bffbda7 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs
@@ -124,6 +124,8 @@ impl MatmulMetalKernel {
specialization.simdgroups_per_column,
specialization.align_m,
specialization.align_n,
+ specialization.align_k,
+ specialization.apply_ab_scale,
specialization.is_accumulate,
)
.map_err(MatmulError::BackendError)?;
@@ -199,7 +201,7 @@ impl MatmulMetalKernel {
(arguments.a, arguments.a_offset as usize),
arguments.b,
&mut *arguments.d,
- params,
+ std::slice::from_ref(&params),
threadgroups_per_row,
threadgroups_per_column,
arguments.ab_scale,
@@ -227,8 +229,13 @@ impl MatmulMetalKernel {
arguments: MatmulArguments<Metal>,
) -> Result<(), MatmulError<Metal>> {
let is_accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);
- let specialization =
- gemm_mpp::GemmMppSpecialization::select(arguments.batch_dim, arguments.output_dim, is_accumulate);
+ let specialization = gemm_mpp::GemmMppSpecialization::select(
+ arguments.batch_dim,
+ arguments.output_dim,
+ arguments.input_dim,
+ is_accumulate,
+ arguments.ab_scale != 1.0,
+ );
let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);
let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);
@@ -267,7 +274,7 @@ impl MatmulMetalKernel {
(arguments.a, arguments.a_offset as usize),
arguments.b,
&mut *arguments.d,
- params,
+ std::slice::from_ref(&params),
group_count_x,
group_count_y,
arguments.ab_scale,
diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h
index 162e1ba18..f86a61e25 100644
--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h
+++ b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h
@@ -30,33 +30,33 @@ struct TransformNone {
METAL_FUNC static U apply(U val) { return val; }
};
-template <typename T, int kFragRows_, int kFragCols_>
+template <typename T, int FRAG_ROWS_, int FRAG_COLS_>
struct BaseMMAFrag {
static_assert(
- kFragRows_ == 8,
+ FRAG_ROWS_ == 8,
"Only 8 x 8 fragment matrices are currently supported"
);
static_assert(
- kFragCols_ == 8,
+ FRAG_COLS_ == 8,
"Only 8 x 8 fragment matrices are currently supported"
);
};
template <typename T>
struct BaseMMAFrag<T, 8, 8> {
- METAL_CONST int kFragRows = 8;
- METAL_CONST int kFragCols = 8;
- METAL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;
- METAL_CONST int kElemRows = 1;
- METAL_CONST int kElemCols = 2;
+ METAL_CONST int FRAG_ROWS = 8;
+ METAL_CONST int FRAG_COLS = 8;
+ METAL_CONST int ELEMS_PER_FRAG = (FRAG_ROWS * FRAG_COLS) / 32;
+ METAL_CONST int ELEM_ROWS = 1;
+ METAL_CONST int ELEM_COLS = 2;
static_assert(
- kElemRows * kElemCols == kElemsPerFrag,
+ ELEM_ROWS * ELEM_COLS == ELEMS_PER_FRAG,
"MMAFrag shape is not consistent with MMAFrag size"
);
- typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;
- typedef metal::vec<T, kElemsPerFrag> frag_type;
+ typedef metal::simdgroup_matrix<T, FRAG_ROWS, FRAG_COLS> mat_type;
+ typedef metal::vec<T, ELEMS_PER_FRAG> frag_type;
METAL_FUNC static constexpr short2 get_coord(
ushort simd_lane_id [[thread_index_in_simdgroup]]
@@ -75,10 +75,10 @@ struct BaseMMAFrag<T, 8, 8> {
StrY str_y
) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
+ for (short i = 0; i < ELEM_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);
+ for (short j = 0; j < ELEM_COLS; j++) {
+ dst[i * ELEM_COLS + j] = static_cast<T>(src[i * str_x + j * str_y]);
}
}
}
@@ -93,10 +93,10 @@ struct BaseMMAFrag<T, 8, 8> {
using U = pointer_element_t<DstPtrType>;
METAL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
+ for (short i = 0; i < ELEM_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
- dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);
+ for (short j = 0; j < ELEM_COLS; j++) {
+ dst[i * str_x + j * str_y] = static_cast<U>(src[i * ELEM_COLS + j]);
}
}
}
@@ -122,12 +122,12 @@ struct BaseMMAFrag<T, 8, 8> {
using U = pointer_element_t<DstPtrType>;
METAL_PRAGMA_UNROLL
- for (short i = 0; i < kElemRows; i++) {
+ for (short i = 0; i < ELEM_ROWS; i++) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < kElemCols; j++) {
+ for (short j = 0; j < ELEM_COLS; j++) {
if ((off_x + i) < lim_x && (off_y + j) < lim_y) {
dst[(off_x + i) * str_x + (off_y + j) * str_y] =
- static_cast<U>(src[i * kElemCols + j]);
+ static_cast<U>(src[i * ELEM_COLS + j]);
}
}
}
@@ -165,48 +165,48 @@ struct BaseMMAFrag<T, 8, 8> {
template <
typename T,
- int kTileRows_,
- int kTileCols_,
+ int TILE_ROWS_,
+ int TILE_COLS_,
class MMAFrag_ = BaseMMAFrag<T, 8, 8>>
struct MMATile {
using MMAFrag_t = MMAFrag_;
using elem_type = T;
- METAL_CONST int kFragRows = MMAFrag_t::kFragRows;
- METAL_CONST int kFragCols = MMAFrag_t::kFragCols;
- METAL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;
+ METAL_CONST int FRAG_ROWS = MMAFrag_t::FRAG_ROWS;
+ METAL_CONST int FRAG_COLS = MMAFrag_t::FRAG_COLS;
+ METAL_CONST int ELEMS_PER_FRAG = MMAFrag_t::ELEMS_PER_FRAG;
- METAL_CONST int kTileRows = kTileRows_;
- METAL_CONST int kTileCols = kTileCols_;
+ METAL_CONST int TILE_ROWS = TILE_ROWS_;
+ METAL_CONST int TILE_COLS = TILE_COLS_;
- METAL_CONST int kRows = kTileRows * kFragRows;
- METAL_CONST int kCols = kTileCols * kFragCols;
+ METAL_CONST int ROWS = TILE_ROWS * FRAG_ROWS;
+ METAL_CONST int COLS = TILE_COLS * FRAG_COLS;
- METAL_CONST int kNumFrags = kTileRows * kTileCols;
- METAL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;
+ METAL_CONST int NUM_FRAGS = TILE_ROWS * TILE_COLS;
+ METAL_CONST int ELEMS_PER_TILE = NUM_FRAGS * ELEMS_PER_FRAG;
typedef typename MMAFrag_t::mat_type mat_type;
typedef typename MMAFrag_t::frag_type frag_type;
- frag_type val_frags[kNumFrags] = {frag_type(0)};
+ frag_type val_frags[NUM_FRAGS] = {frag_type(0)};
METAL_FUNC MMATile() thread {}
METAL_FUNC constexpr void clear() {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < kNumFrags; ++i) {
+ for (short i = 0; i < NUM_FRAGS; ++i) {
val_frags[i] = frag_type(0);
}
}
METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {
- return val_frags[i * kTileCols + j];
+ return val_frags[i * TILE_COLS + j];
}
METAL_FUNC constexpr const thread frag_type& frag_at(
const short i,
const short j
) const {
- return val_frags[i * kTileCols + j];
+ return val_frags[i * TILE_COLS + j];
}
METAL_FUNC thread elem_type* elems() {
@@ -220,14 +220,14 @@ struct MMATile {
template <typename U, int w_x, int w_y, int str_x, int str_y>
METAL_FUNC void load(const threadgroup U* src) {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
+ for (short i = 0; i < TILE_ROWS; ++i) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
+ for (short j = 0; j < TILE_COLS; ++j) {
MMAFrag_t::load(
frag_at(i, j),
&(
- src[(i * kFragRows) * w_x * str_x +
- (j * kFragCols) * w_y * str_y]
+ src[(i * FRAG_ROWS) * w_x * str_x +
+ (j * FRAG_COLS) * w_y * str_y]
),
str_x,
str_y
@@ -239,12 +239,12 @@ struct MMATile {
template <typename U, int w_x, int w_y>
METAL_FUNC void store(device U* dst, const int ld) const {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
+ for (short i = 0; i < TILE_ROWS; ++i) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
+ for (short j = 0; j < TILE_COLS; ++j) {
MMAFrag_t::store(
frag_at(i, j),
- &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
+ &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),
ld,
1
);
@@ -256,12 +256,12 @@ struct MMATile {
template <typename U, int w_x, int w_y>
METAL_FUNC void store_tg(threadgroup U* dst, const int ld) const {
METAL_PRAGMA_UNROLL
- for (short i = 0; i < kTileRows; ++i) {
+ for (short i = 0; i < TILE_ROWS; ++i) {
METAL_PRAGMA_UNROLL
- for (short j = 0; j < kTileCols; ++j) {
+ for (short j = 0; j < TILE_COLS; ++j) {
MMAFrag_t::store(
frag_at(i, j),
- &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),
+ &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),
ld,
1
);
@@ -276,9 +276,9 @@ struct MMATile {
const short2 dst_tile_dims
) const {
METAL_PRAGMA_UNROLL
- for (int i = 0; i < kTileRows; ++i) {
+ for (int i = 0; i < TILE_ROWS; ++i) {
METAL_PRAGMA_UNROLL
- for (int j = 0; j < kTileCols; ++j) {
+ for (int j = 0; j < TILE_COLS; ++j) {
MMAFrag_t::store_safe(
frag_at(i, j),
dst,
@@ -286,8 +286,8 @@ struct MMATile {
1,
dst_tile_dims.y,
dst_tile_dims.x,
- (i * kFragRows) * w_x,
- (j * kFragCols) * w_y
+ (i * FRAG_ROWS) * w_x,
+ (j * FRAG_COLS) * w_y
);
}
}
@@ -335,18 +335,18 @@ template <
typename Epilogue = TransformNone<U, AccumType>>
struct BlockMMA {
// MMAFrag size
- METAL_CONST short kFragSize = 8;
- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;
+ METAL_CONST short FRAG_SIZE = 8;
+ using MMAFrag_acc_t = BaseMMAFrag<AccumType, FRAG_SIZE, FRAG_SIZE>;
// Warp tile simdgroup matrix strides along M
- METAL_CONST short TM_stride = kFragSize * WM;
+ METAL_CONST short TM_stride = FRAG_SIZE * WM;
// Warp tile simdgroup matrix strides along M
- METAL_CONST short TN_stride = kFragSize * WN;
+ METAL_CONST short TN_stride = FRAG_SIZE * WN;
// Warp tile size along M
- METAL_CONST short TM = BM / (kFragSize * WM);
+ METAL_CONST short TM = BM / (FRAG_SIZE * WM);
// Warp tile size along N
- METAL_CONST short TN = BN / (kFragSize * WN);
+ METAL_CONST short TN = BN / (FRAG_SIZE * WN);
// Threadgroup A strides
METAL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M
@@ -357,8 +357,8 @@ struct BlockMMA {
METAL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N
// Threadgroup strides along K
- METAL_CONST short tile_stride_a = kFragSize * A_str_k;
- METAL_CONST short tile_stride_b = kFragSize * B_str_k;
+ METAL_CONST short tile_stride_a = FRAG_SIZE * A_str_k;
+ METAL_CONST short tile_stride_b = FRAG_SIZE * B_str_k;
// Simdgroup matrices
MMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;
@@ -378,8 +378,8 @@ struct BlockMMA {
ushort simd_lane_id [[thread_index_in_simdgroup]]
) {
// Determine thread position in simdgroup matrix
- short tm = kFragSize * (simd_group_id / WN);
- short tn = kFragSize * (simd_group_id % WN);
+ short tm = FRAG_SIZE * (simd_group_id / WN);
+ short tn = FRAG_SIZE * (simd_group_id % WN);
short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);
sm = simd_coord.y;
@@ -399,9 +399,9 @@ struct BlockMMA {
As += As_offset;
Bs += Bs_offset;
- // Iterate over BK in blocks of kFragSize
+ // Iterate over BK in blocks of FRAG_SIZE
METAL_PRAGMA_UNROLL
- for (short kk = 0; kk < BK; kk += kFragSize) {
+ for (short kk = 0; kk < BK; kk += FRAG_SIZE) {
simdgroup_barrier(mem_flags::mem_none);
Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);
@@ -424,7 +424,7 @@ struct BlockMMA {
METAL_FUNC void store_result(device U* D, const int ldd) {
// Apply epilogue
METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
@@ -438,7 +438,7 @@ struct BlockMMA {
METAL_FUNC void store_result_tg(threadgroup U* D, const int ldd) {
// Apply epilogue
METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
@@ -455,7 +455,7 @@ struct BlockMMA {
) {
// Apply epilogue
METAL_PRAGMA_UNROLL
- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {
+ for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {
Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);
}
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
index d17da07be..19ec918c3 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
@@ -239,6 +239,38 @@ fn test_bf16_scale_and_accumulate() {
test_mpp_correctness(64, 2048, 2048, 0.5, true, 1.0);
}
+// 128x128 tile (M >= 128, N >= 128)
+
+#[uzu_test]
+fn test_bf16_128x128_aligned() {
+ test_mpp_correctness(128, 2048, 2048, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn test_bf16_128x128_unaligned_m() {
+ test_mpp_correctness(200, 2048, 2048, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn test_bf16_128x128_unaligned_n() {
+ test_mpp_correctness(128, 2048, 200, 1.0, false, 1.0);
+}
+
+#[uzu_test]
+fn test_bf16_128x128_large() {
+ test_mpp_correctness(256, 4096, 4096, 1.0, false, 2.0);
+}
+
+#[uzu_test]
+fn test_bf16_128x128_accumulate() {
+ test_mpp_correctness(128, 2048, 2048, 1.0, true, 1.0);
+}
+
+#[uzu_test]
+fn test_bf16_128x128_scale_and_accumulate() {
+ test_mpp_correctness(256, 2048, 2048, 0.5, true, 1.0);
+}
+
// Benchmarks
const BENCHMARK_SHAPES: &[(usize, usize, usize)] =
diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
index 4daadaf4b..2f197a59f 100644
--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs
@@ -16,9 +16,15 @@ use backend_uzu::{
cpu::Cpu,
},
};
+#[cfg(metal_backend)]
+use backend_uzu::backends::metal::{MatmulDispatchPath, Metal, MetalContext};
+#[cfg(metal_backend)]
+use criterion::{BenchmarkId, Criterion, Throughput};
use half::{bf16, f16};
use num_traits::Float;
+#[cfg(metal_backend)]
+use crate::{common::type_short_name, uzu_bench};
use crate::{common::assert::assert_eq_float, uzu_test};
struct Input<T: ArrayElement + Float> {
@@ -184,3 +190,69 @@ fn test_f32_ab_scale() {
fn test_bf16_ab_scale() {
test_with_scale::<bf16>(16, 128, 256, 0.5, 0.1);
}
+
+// Benchmarks (classic ALU GEMM path, Metal only)
+
+#[cfg(metal_backend)]
+const BENCHMARK_SHAPES: &[(usize, usize, usize)] =
+ &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];
+
+#[cfg(metal_backend)]
+#[uzu_bench]
+fn bench_gemm(criterion: &mut Criterion) {
+ let metal_context = MetalContext::new().unwrap();
+
+ let mut matmul_kernel =
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())
+ .expect("MatmulKernel");
+
+ let mut benchmark_group =
+ criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));
+
+ for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {
+ let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");
+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");
+ let destination_array =
+ metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");
+
+ let floating_point_operations = 2 * batch_dim * input_dim * output_dim;
+ benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));
+
+ benchmark_group.bench_function(
+ BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),
+ |bencher| {
+ let left_buffer = left_array.buffer();
+ let left_ref = left_buffer.borrow();
+ let right_buffer = right_array.buffer();
+ let right_ref = right_buffer.borrow();
+ let destination_buffer = destination_array.buffer();
+
+ bencher.iter_custom(|iteration_count| {
+ let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();
+
+ for _ in 0..iteration_count {
+ let mut destination_ref = destination_buffer.borrow_mut();
+ matmul_kernel.encode_with_path(
+ &metal_context,
+ MatmulArguments {
+ a: left_ref.deref(),
+ a_offset: 0,
+ b: right_ref.deref(),
+ ab_scale: 1.0,
+ c: MatmulArgumentC::None,
+ d: destination_ref.deref_mut(),
+ batch_dim: batch_dim as u32,
+ input_dim: input_dim as u32,
+ output_dim: output_dim as u32,
+ },
+ &mut encoder,
+ MatmulDispatchPath::Gemm,
+ );
+ }
+
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
+ })
+ },
+ );
+ }
+}