A high-performance inference engine for AI models
remove thread group mem in mpp (#350)
eugene4971bc88ff46
diff --git a/Cargo.lock b/Cargo.lockindex 573b3aa96..ddcd3f4e7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -744,7 +744,7 @@ dependencies = ["getrandom 0.2.17","instant","pin-project-lite",- "rand 0.8.5",+ "rand 0.8.6","tokio",]@@ -1227,9 +1227,9 @@ dependencies = [[[package]]name = "clap"-version = "4.6.0"+version = "4.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"+checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"dependencies = ["clap_builder","clap_derive",@@ -1249,9 +1249,9 @@ dependencies = [[[package]]name = "clap_derive"-version = "4.6.0"+version = "4.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"+checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"dependencies = ["heck","proc-macro2",@@ -1329,7 +1329,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681"dependencies = ["serde","termcolor",- "unicode-width 0.2.0",+ "unicode-width 0.2.2",][[package]]@@ -1361,7 +1361,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47"dependencies = ["unicode-segmentation",- "unicode-width 0.2.0",+ "unicode-width 0.2.2",][[package]]@@ -1396,7 +1396,7 @@ checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87"dependencies = ["encode_unicode","libc",- "unicode-width 0.2.0",+ "unicode-width 0.2.2","windows-sys 0.61.2",]@@ -1678,9 +1678,9 @@ dependencies = [[[package]]name = "ctor"-version = "0.10.0"+version = "0.10.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "95d0d11eb38e7642efca359c3cf6eb7b2e528182d09110165de70192b0352775"+checksum = "83cf0d42651b16c6dfe68685716d18480d18a9c39c62d76e8cf3eb6ed5d8bcbf"dependencies = ["ctor-proc-macro","dtor",@@ -1688,9 +1688,9 @@ dependencies = [[[package]]name = "ctor-proc-macro"-version = "0.0.12"+version = "0.0.13"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a7ab264ea985f1bd27887d7b21ea2bb046728e05d11909ca138d700c494730db"+checksum = "7a949c44fcacbbbb7ada007dc7acb34603dd97cd47de5d054f2b6493ecebb483"[[package]]name = "ctrlc"@@ -1884,9 +1884,9 @@ dependencies = [[[package]]name = "dary_heap"-version = "0.3.8"+version = "0.3.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "06d2e3287df1c007e74221c49ca10a95d557349e54b3a75dc2fb14712c751f04"+checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"dependencies = ["serde",]@@ -2183,18 +2183,18 @@ dependencies = [[[package]]name = "dtor"-version = "0.7.0"+version = "0.8.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "17f72721db8027a4e96dd6fb50d2a1d32259c9d3da1b63dee612ccd981e14293"+checksum = "edf234dd1594d6dd434a8fb8cada51ddbbc593e40e4a01556a0b31c62da2775b"dependencies = ["dtor-proc-macro",][[package]]name = "dtor-proc-macro"-version = "0.0.12"+version = "0.0.13"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8c98b077c7463d01d22dde8a24378ddf1ca7263dc687cffbed38819ea6c21131"+checksum = "2647271c92754afcb174e758003cfd1cbf1e43e5a7853d7b1813e63e19e39a73"[[package]]name = "dunce"@@ -2576,9 +2576,9 @@ dependencies = [[[package]]name = "fraction"-version = "0.15.3"+version = "0.15.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0f158e3ff0a1b334408dc9fb811cd99b446986f4d8b741bb08f9df1604085ae7"+checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872"dependencies = ["lazy_static","num",@@ -3431,7 +3431,7 @@ checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb"dependencies = ["console","portable-atomic",- "unicode-width 0.2.0",+ "unicode-width 0.2.2","unit-prefix","web-time",]@@ -3478,7 +3478,7 @@ dependencies = ["dyn-clone","fuzzy-matcher","unicode-segmentation",- "unicode-width 0.2.0",+ "unicode-width 0.2.2",][[package]]@@ -3691,9 +3691,9 @@ dependencies = [[[package]]name = "jsonschema"-version = "0.46.0"+version = "0.46.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "84695c6689b01384700a3d93acecbd07231ee6fff1bf22ae980b4c307e6ddfd5"+checksum = "50180452e7808015fe083eae3efcf1ec98b89b45dd8cc204f7b4a6b7b81ea675"dependencies = ["ahash","bytecount",@@ -3772,9 +3772,9 @@ checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8"[[package]]name = "libc"-version = "0.2.185"+version = "0.2.186"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f"+checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"[[package]]name = "libdbus-sys"@@ -4748,9 +4748,9 @@ dependencies = [[[package]]name = "openssl"-version = "0.10.77"+version = "0.10.78"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bfe4646e360ec77dff7dde40ed3d6c5fee52d156ef4a62f53973d38294dad87f"+checksum = "f38c4372413cdaaf3cc79dd92d29d7d9f5ab09b51b10dded508fb90bb70b9222"dependencies = ["bitflags 2.11.1","cfg-if 1.0.4",@@ -4780,9 +4780,9 @@ checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"[[package]]name = "openssl-sys"-version = "0.9.113"+version = "0.9.114"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ad2f2c0eba47118757e4c6d2bff2838f3e0523380021356e7875e858372ce644"+checksum = "13ce1245cd07fcc4cfdb438f7507b0c7e4f3849a69fd84d52374c66d83741bb6"dependencies = ["cc","libc",@@ -4980,7 +4980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"dependencies = ["phf_shared",- "rand 0.8.5",+ "rand 0.8.6",][[package]]@@ -5122,9 +5122,9 @@ checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"[[package]]name = "portable-atomic-util"-version = "0.2.6"+version = "0.2.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3"+checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618"dependencies = ["portable-atomic",]@@ -5395,7 +5395,7 @@ dependencies = ["once_cell","socket2 0.6.3","tracing",- "windows-sys 0.52.0",+ "windows-sys 0.60.2",][[package]]@@ -5421,9 +5421,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"[[package]]name = "rand"-version = "0.8.5"+version = "0.8.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"+checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"dependencies = ["libc","rand_chacha 0.3.1",@@ -5551,7 +5551,7 @@ dependencies = ["thiserror 2.0.18","unicode-segmentation","unicode-truncate",- "unicode-width 0.2.0",+ "unicode-width 0.2.2",][[package]]@@ -5602,7 +5602,7 @@ dependencies = ["strum","time","unicode-segmentation",- "unicode-width 0.2.0",+ "unicode-width 0.2.2",][[package]]@@ -5743,9 +5743,9 @@ dependencies = [[[package]]name = "referencing"-version = "0.46.0"+version = "0.46.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a2d5554bf79f4acf770dc3193b44b2d63b348f5f7b7448a0ea1191b37b620728"+checksum = "acb0c66c7b78c1da928bee668b5cc638c678642ff587faff6e6222f797be9d4c"dependencies = ["ahash","fluent-uri",@@ -6007,7 +6007,7 @@ dependencies = ["num_cpus","parking_lot","pin-project-lite",- "rand 0.8.5",+ "rand 0.8.6","ref-cast","rocket_codegen","rocket_http",@@ -6104,9 +6104,9 @@ dependencies = [[[package]]name = "rustls"-version = "0.23.38"+version = "0.23.39"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"+checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e"dependencies = ["aws-lc-rs","log",@@ -6169,9 +6169,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"[[package]]name = "rustls-webpki"-version = "0.103.12"+version = "0.103.13"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"dependencies = ["aws-lc-rs","ring",@@ -6319,7 +6319,7 @@ dependencies = ["hkdf","num","once_cell",- "rand 0.8.5",+ "rand 0.8.6","serde","sha2","zbus",@@ -6779,6 +6779,12 @@ version = "2.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"+[[package]]+name = "symlink"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a"+[[package]]name = "syn"version = "1.0.109"@@ -7166,9 +7172,9 @@ dependencies = [[[package]]name = "tokio"-version = "1.52.0"+version = "1.52.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776"+checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6"dependencies = ["bytes","libc",@@ -7315,7 +7321,7 @@ dependencies = ["indexmap 2.14.0","toml_datetime 1.1.1+spec-1.1.0","toml_parser",- "winnow 1.0.1",+ "winnow 1.0.2",][[package]]@@ -7324,7 +7330,7 @@ version = "1.1.2+spec-1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"dependencies = [- "winnow 1.0.1",+ "winnow 1.0.2",][[package]]@@ -7397,11 +7403,12 @@ dependencies = [[[package]]name = "tracing-appender"-version = "0.2.4"+version = "0.2.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf"+checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c"dependencies = ["crossbeam-channel",+ "symlink","thiserror 2.0.18","time","tracing-subscriber",@@ -7513,9 +7520,9 @@ dependencies = [[[package]]name = "typenum"-version = "1.19.0"+version = "1.20.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"+checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de"[[package]]name = "ubyte"@@ -7600,7 +7607,7 @@ checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5"dependencies = ["itertools 0.14.0","unicode-segmentation",- "unicode-width 0.2.0",+ "unicode-width 0.2.2",][[package]]@@ -7611,9 +7618,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"[[package]]name = "unicode-width"-version = "0.2.0"+version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"[[package]]name = "unicode-xid"@@ -7826,9 +7833,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"[[package]]name = "uuid"-version = "1.23.0"+version = "1.23.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"+checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76"dependencies = ["atomic 0.6.1","getrandom 0.4.2",@@ -7979,11 +7986,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"[[package]]name = "wasip2"-version = "1.0.2+wasi-0.2.9"+version = "1.0.3+wasi-0.2.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5"+checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"dependencies = [- "wit-bindgen",+ "wit-bindgen 0.57.1",][[package]]@@ -7992,7 +7999,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5"dependencies = [- "wit-bindgen",+ "wit-bindgen 0.51.0",][[package]]@@ -8134,18 +8141,18 @@ dependencies = [[[package]]name = "webpki-root-certs"-version = "1.0.6"+version = "1.0.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca"+checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"dependencies = ["rustls-pki-types",][[package]]name = "webpki-roots"-version = "1.0.6"+version = "1.0.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"+checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d"dependencies = ["rustls-pki-types",]@@ -8697,9 +8704,9 @@ dependencies = [[[package]]name = "winnow"-version = "1.0.1"+version = "1.0.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"+checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0"dependencies = ["memchr",]@@ -8713,6 +8720,12 @@ dependencies = ["wit-bindgen-rust-macro",]+[[package]]+name = "wit-bindgen"+version = "0.57.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"+[[package]]name = "wit-bindgen-core"version = "0.51.0"@@ -8886,7 +8899,7 @@ dependencies = ["hex","nix 0.29.0","ordered-stream",- "rand 0.8.5",+ "rand 0.8.6","serde","serde_repr","sha1",diff --git a/Cargo.toml b/Cargo.tomlindex 8ddbd743a..ffbdbe89f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -167,6 +167,13 @@ keyring = { version = "3.6.3", features = ["crypto-rust",] }+[workspace.metadata.dinghy]+forward-cfgs = ["metal_backend", "grammar_xgrammar"]++[workspace.metadata.dinghy.test-attribute-aliases]+uzu_test = "dinghy_apple_runner_macros::test_case"+uzu_bench = "dinghy_apple_runner_macros::bench_case"+[profile.dev.build-override]opt-level = 3diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/common/defines.hindex 602984747..904411eac 100644--- a/crates/backend-uzu/src/backends/metal/kernel/common/defines.h+++ b/crates/backend-uzu/src/backends/metal/kernel/common/defines.h@@ -2,6 +2,7 @@#define METAL_CONST static constant constexpr#define METAL_PRAGMA_UNROLL _Pragma("clang loop unroll(full)")+#define METAL_PRAGMA_NO_UNROLL _Pragma("clang loop unroll(disable)")#define METAL_SIMD_SIZE 32METAL_FUNC int pow2(int n) { return 1 << n; }diff --git a/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.hnew file mode 100644index 000000000..4263ddeba--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/common/integral_constant.h@@ -0,0 +1,72 @@+#pragma once++#include <metal_stdlib>++#pragma METAL internals : enable++namespace uzu {++template <typename T, T v>+struct integral_constant {+ static constexpr constant T value = v;+ using value_type = T;+ using type = integral_constant;++ METAL_FUNC constexpr operator value_type() const noexcept { return value; }+};++template <bool B>+using bool_constant = integral_constant<bool, B>;+using true_type = bool_constant<true>;+using false_type = bool_constant<false>;++template <int val>+using Int = integral_constant<int, val>;++#define METAL_INTEGRAL_CONST_BINOP(op, fn) \+ template <typename T, T tv, typename U, U uv> \+ METAL_FUNC constexpr auto fn( \+ integral_constant<T, tv>, \+ integral_constant<U, uv> \+ ) { \+ constexpr auto res = tv op uv; \+ return integral_constant<decltype(res), res>{}; \+ }++METAL_INTEGRAL_CONST_BINOP(+, operator+)+METAL_INTEGRAL_CONST_BINOP(-, operator-)+METAL_INTEGRAL_CONST_BINOP(*, operator*)+METAL_INTEGRAL_CONST_BINOP(/, operator/)++#undef METAL_INTEGRAL_CONST_BINOP++template <int start, int step, typename F, int... Is>+METAL_FUNC constexpr void const_for_loop_impl(+ F f,+ metal::integer_sequence<int, Is...>+) {+ (f(Int<start + Is * step>{}), ...);+}++template <int start, int stop, int step, typename F>+METAL_FUNC constexpr void const_for_loop(F f) {+ static_assert(step > 0 && start <= stop);+ constexpr int count = (stop - start + step - 1) / step;+ const_for_loop_impl<start, step>(+ f,+ metal::make_integer_sequence<int, count>{}+ );+}++template <typename F>+METAL_FUNC void dispatch_bool(bool v, F f) {+ if (v) {+ f(true_type{});+ } else {+ f(false_type{});+ }+}++} // namespace uzu++#pragma METAL internals : disablediff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.hnew file mode 100644index 000000000..278600759--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/alu_fragment_ops.h@@ -0,0 +1,100 @@+#pragma once++#include "../../common/integral_constant.h"+#include "../../common/thread_context.h"+#include "defines.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// AluFragmentOps - stateless traits/ops for 8x8 ALU matmul (simdgroup_matrix)+///////////////////////////////////////////////////////////////////////////////++template <typename T>+struct AluFragmentOps {+ METAL_CONST ushort FRAGMENT_ROWS = 8;+ METAL_CONST ushort FRAGMENT_COLS = 8;+ METAL_CONST ushort ELEMENTS_PER_THREAD =+ (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;+ METAL_CONST ushort THREAD_ELEMENT_ROWS = 1;+ METAL_CONST ushort THREAD_ELEMENT_COLS = 2;+ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 1;++ static_assert(+ THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,+ "AluFragmentOps shape is not consistent with element count"+ );++ typedef metal::simdgroup_matrix<T, FRAGMENT_ROWS, FRAGMENT_COLS>+ SimdgroupMatrixType;++ template <typename U>+ using ThreadVector = metal::vec<U, ELEMENTS_PER_THREAD>;++ METAL_FUNC static constexpr void multiply_accumulate(+ thread ThreadVector<T>& D,+ thread ThreadVector<T>& A,+ thread ThreadVector<T>& B,+ thread ThreadVector<T>& C+ ) {+ SimdgroupMatrixType D_mat;+ SimdgroupMatrixType A_mat;+ SimdgroupMatrixType B_mat;+ SimdgroupMatrixType C_mat;++ reinterpret_cast<thread ThreadVector<T>&>(A_mat.thread_elements()) = A;+ reinterpret_cast<thread ThreadVector<T>&>(B_mat.thread_elements()) = B;+ reinterpret_cast<thread ThreadVector<T>&>(C_mat.thread_elements()) = C;++ simdgroup_multiply_accumulate(D_mat, A_mat, B_mat, C_mat);++ D = reinterpret_cast<thread ThreadVector<T>&>(D_mat.thread_elements());+ }++ template <class OutputTile, class LeftTile, class RightTile>+ METAL_FUNC static void tile_matmul(+ thread OutputTile& output,+ thread LeftTile& left,+ thread RightTile& right+ ) {+ constexpr ushort tile_m = OutputTile::TILE_ROWS;+ constexpr ushort tile_n = OutputTile::TILE_COLS;+ constexpr ushort tile_k = LeftTile::TILE_COLS;++ static_assert(+ tile_m == LeftTile::TILE_ROWS,+ "tile matmul: M dimensions do not match"+ );+ static_assert(+ tile_n == RightTile::TILE_COLS,+ "tile matmul: N dimensions do not match"+ );+ static_assert(+ tile_k == RightTile::TILE_ROWS,+ "tile matmul: K dimensions do not match"+ );++ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < tile_m; ++row) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < tile_n; ++col) {+ const ushort column_serpentine = (row % 2) ? (tile_n - 1 - col) : col;+ METAL_PRAGMA_UNROLL+ for (ushort k = 0; k < tile_k; ++k) {+ multiply_accumulate(+ output.fragment_at(row, column_serpentine),+ left.fragment_at(row, k),+ right.fragment_at(k, column_serpentine),+ output.fragment_at(row, column_serpentine)+ );+ }+ }+ }+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.hindex 1b56d03b3..5781cdf8c 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/defines.h@@ -1,8 +1,6 @@#pragma once#include "../../common/defines.h"-#include <metal_simdgroup>-#include <metal_simdgroup_matrix>#include <metal_stdlib>using namespace metal;@@ -33,15 +31,19 @@ struct PointerElementTypeImpl<threadgroup T*> {template <typename T>using PointerElementType = typename PointerElementTypeImpl<T>::type;-template <typename OutT, typename InT>+template <typename OutputType, typename InputType>struct TransformNone {TransformNone(float = 1.0f, float = 0.0f) {}- static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }- METAL_FUNC OutT apply(InT x, OutT) const { return static_cast<OutT>(x); }+ static METAL_FUNC OutputType apply(InputType x) {+ return static_cast<OutputType>(x);+ }+ METAL_FUNC OutputType apply(InputType x, OutputType) const {+ return static_cast<OutputType>(x);+ }};-template <typename OutT, typename InT>+template <typename OutputType, typename InputType>struct TransformScaleAccumulate {const float alpha;const float beta;@@ -49,12 +51,15 @@ struct TransformScaleAccumulate {TransformScaleAccumulate(float alpha, float beta): alpha(alpha), beta(beta) {}- static METAL_FUNC OutT apply(InT x) { return static_cast<OutT>(x); }+ static METAL_FUNC OutputType apply(InputType x) {+ return static_cast<OutputType>(x);+ }- METAL_FUNC OutT apply(InT x, OutT c) const {- return static_cast<OutT>(- x * static_cast<InT>(alpha) +- (beta != 0.0f ? (static_cast<OutT>(beta) * c) : static_cast<OutT>(0.0))+ METAL_FUNC OutputType apply(InputType x, OutputType c) const {+ return static_cast<OutputType>(+ x * static_cast<InputType>(alpha) ++ (beta != 0.0f ? (static_cast<OutputType>(beta) * c)+ : static_cast<OutputType>(0.0)));}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.hnew file mode 100644index 000000000..b67370a2a--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/fragment.h@@ -0,0 +1,298 @@+#pragma once++#include <metal_stdlib>++#include "../../common/integral_constant.h"+#include "../../common/thread_context.h"+#include "defines.h"++using namespace metal;++namespace uzu {+namespace matmul {++///////////////////////////////////////////////////////////////////////////////+// Fragment - a thread-private tile of values arranged as TILE_ROWS x TILE_COLS+// sub-tiles. Ops provides the sub-tile shape constants and the MMA primitive;+// Fragment itself owns lane positioning and device<->register transfer.+///////////////////////////////////////////////////////////////////////////////++template <typename T, ushort TILE_ROWS_, ushort TILE_COLS_, class Ops>+struct Fragment {+ using FragmentOpsType = Ops;+ using ElementType = T;+ using ThreadVectorType = typename Ops::template ThreadVector<T>;++ static_assert(+ Ops::FRAGMENT_ROWS > 0 && Ops::FRAGMENT_COLS > 0,+ "Ops must expose positive FRAGMENT_ROWS/FRAGMENT_COLS"+ );+ static_assert(+ Ops::ELEMENTS_PER_THREAD ==+ (Ops::FRAGMENT_ROWS * Ops::FRAGMENT_COLS) / METAL_SIMD_SIZE,+ "Ops::ELEMENTS_PER_THREAD must equal "+ "(FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE"+ );+ static_assert(+ sizeof(ThreadVectorType) == Ops::ELEMENTS_PER_THREAD * sizeof(T),+ "Ops::ThreadVector<T> must be tightly packed "+ "(elements() relies on it)"+ );++ METAL_CONST ushort TILE_ROWS = TILE_ROWS_;+ METAL_CONST ushort TILE_COLS = TILE_COLS_;++ METAL_CONST ushort FRAGMENT_ROWS = ushort(Ops::FRAGMENT_ROWS);+ METAL_CONST ushort FRAGMENT_COLS = ushort(Ops::FRAGMENT_COLS);++ METAL_CONST ushort NUM_FRAGS = TILE_ROWS * TILE_COLS;+ METAL_CONST ushort ELEMENTS_PER_TILE = NUM_FRAGS * Ops::ELEMENTS_PER_THREAD;++ ThreadVectorType fragment_data[NUM_FRAGS];+ ThreadContext thread_context;++ METAL_FUNC Fragment(const thread ThreadContext& thread_context) thread+ : thread_context(thread_context) {}++ // Lane origin (row, col) within a single FRAGMENT_ROWS x FRAGMENT_COLS+ // sub-tile. The mapping depends only on Ops::THREAD_ELEMENT_COLS, so it is+ // identical for every Fragment sharing that Ops type.+ METAL_FUNC static constexpr short2 get_position(+ const thread ThreadContext& thread_context+ ) {+ const ushort simdgroup_index = ushort(thread_context.simdgroup_index);+ const short quad = simdgroup_index / 4;+ const short row = (quad & 4) + (simdgroup_index / 2) % 4;+ const short col =+ ((quad & 2) + simdgroup_index % 2) * Ops::THREAD_ELEMENT_COLS;+ return short2{col, row};+ }++ METAL_FUNC short2 get_position() const thread {+ return get_position(thread_context);+ }++ METAL_FUNC constexpr void clear() {+ METAL_PRAGMA_UNROLL+ for (ushort index = 0; index < NUM_FRAGS; ++index) {+ fragment_data[index] = ThreadVectorType(0);+ }+ }++ METAL_FUNC constexpr thread ThreadVectorType& fragment_at(+ const ushort row_index,+ const ushort col_index+ ) {+ return fragment_data[row_index * TILE_COLS + col_index];+ }++ template <bool transpose>+ METAL_FUNC constexpr thread ThreadVectorType& fragment_at(+ const ushort row_index,+ const ushort col_index,+ metal::bool_constant<transpose>+ ) {+ if constexpr (transpose) {+ return fragment_at(col_index, row_index);+ } else {+ return fragment_at(row_index, col_index);+ }+ }++ METAL_FUNC thread ElementType* elements() {+ return reinterpret_cast<thread ElementType*>(fragment_data);+ }++ // Unsafe load: copy a (TILE_ROWS * FRAGMENT_ROWS) x (TILE_COLS *+ // FRAGMENT_COLS) block from device memory into fragment registers. ColStride+ // defaults to the compile-time constant 1 so that the row-major fast path+ // triggers when the caller omits it. Tile strides control the gap between+ // sub-tiles.+ template <+ class Ptr,+ class RowStride,+ class ColStride = Int<1>,+ class TileRowStride = Int<1>,+ class TileColStride = Int<1>>+ METAL_FUNC void load(+ Ptr source,+ RowStride row_stride,+ ColStride col_stride = {},+ TileRowStride tile_row_stride = {},+ TileColStride tile_col_stride = {}+ ) thread {+ transfer<LOAD, UNSAFE>(+ source,+ row_stride,+ col_stride,+ Int<0>{},+ Int<0>{},+ tile_row_stride,+ tile_col_stride+ );+ }++ // Safe load: col_stride is implicitly 1; out-of-bounds elements become T(0).+ template <+ class Ptr,+ class TileRowStride = Int<1>,+ class TileColStride = Int<1>>+ METAL_FUNC void load_safe(+ Ptr source,+ const int leading_dimension,+ const short2 tile_dimensions,+ TileRowStride tile_row_stride = {},+ TileColStride tile_col_stride = {}+ ) thread {+ transfer<LOAD, SAFE>(+ source,+ leading_dimension,+ Int<1>{},+ tile_dimensions.y,+ tile_dimensions.x,+ tile_row_stride,+ tile_col_stride+ );+ }++ // Unsafe store: mirror of load. Same defaults apply.+ template <+ class Ptr,+ class RowStride,+ class ColStride = Int<1>,+ class TileRowStride = Int<1>,+ class TileColStride = Int<1>>+ METAL_FUNC void store(+ Ptr destination,+ RowStride row_stride,+ ColStride col_stride = {},+ TileRowStride tile_row_stride = {},+ TileColStride tile_col_stride = {}+ ) thread {+ transfer<STORE, UNSAFE>(+ destination,+ row_stride,+ col_stride,+ Int<0>{},+ Int<0>{},+ tile_row_stride,+ tile_col_stride+ );+ }++ // Safe store: col_stride is implicitly 1; out-of-bounds elements are skipped.+ template <+ class Ptr,+ class TileRowStride = Int<1>,+ class TileColStride = Int<1>>+ METAL_FUNC void store_safe(+ Ptr destination,+ const int leading_dimension,+ const short2 tile_dimensions,+ TileRowStride tile_row_stride = {},+ TileColStride tile_col_stride = {}+ ) thread {+ transfer<STORE, SAFE>(+ destination,+ leading_dimension,+ Int<1>{},+ tile_dimensions.y,+ tile_dimensions.x,+ tile_row_stride,+ tile_col_stride+ );+ }++private:+ METAL_CONST bool LOAD = true;+ METAL_CONST bool STORE = false;+ METAL_CONST bool SAFE = true;+ METAL_CONST bool UNSAFE = false;++ template <class Fn>+ METAL_FUNC void for_each_fragment(Fn fn) thread {+ const_for_loop<0, TILE_ROWS, 1>([&](auto row_index) {+ const_for_loop<0, TILE_COLS, 1>([&](auto col_index) {+ fn(row_index, col_index);+ });+ });+ }++ // Unified memory transfer between device memory and fragment registers.+ //+ // IS_LOAD - true to read into fragment_data, false to write back.+ // IS_SAFE - true to bounds-check each element against (row_limit,+ // col_limit);+ // on load, out-of-bounds elements become T(0); on store, skipped.+ //+ // When ColStride is a compile-time 1, the column-stride multiplication is+ // elided to help the compiler collapse strength-reduced addressing.+ template <+ bool IS_LOAD,+ bool IS_SAFE,+ class Ptr,+ class RowStride,+ class ColStride,+ class RowLimit,+ class ColLimit,+ class TileRowStride,+ class TileColStride>+ METAL_FUNC void transfer(+ Ptr ptr,+ RowStride row_stride,+ ColStride col_stride,+ RowLimit row_limit,+ ColLimit col_limit,+ TileRowStride tile_row_stride,+ TileColStride tile_col_stride+ ) thread {+ using U = PointerElementType<Ptr>;+ constexpr bool col_stride_is_one = metal::is_same_v<ColStride, Int<1>>;++ const short2 position = get_position();+ ptr += position.y * row_stride + position.x * col_stride;+ const auto local_row_limit = row_limit - position.y;+ const auto local_col_limit = col_limit - position.x;++ for_each_fragment([&](auto tile_row, auto tile_col) {+ thread auto& frag = fragment_at(tile_row.value, tile_col.value);+ const auto row_base = tile_row.value * FRAGMENT_ROWS * tile_row_stride;+ const auto col_base = tile_col.value * FRAGMENT_COLS * tile_col_stride;++ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < Ops::THREAD_ELEMENT_ROWS; i++) {+ const auto row = row_base + i * Ops::THREAD_ELEMENT_ROW_STRIDE;+ METAL_PRAGMA_UNROLL+ for (ushort j = 0; j < Ops::THREAD_ELEMENT_COLS; j++) {+ const ushort element_index = i * Ops::THREAD_ELEMENT_COLS + j;+ const auto col = col_base + j;+ const auto offset = col_stride_is_one+ ? (row * row_stride + col)+ : (row * row_stride + col * col_stride);++ if constexpr (IS_LOAD) {+ if constexpr (IS_SAFE) {+ const bool in_bounds =+ (row < local_row_limit) && (col < local_col_limit);+ frag[element_index] =+ in_bounds ? static_cast<T>(ptr[offset]) : T(0);+ } else {+ frag[element_index] = static_cast<T>(ptr[offset]);+ }+ } else {+ if constexpr (IS_SAFE) {+ if ((row < local_row_limit) && (col < local_col_limit)) {+ ptr[offset] = static_cast<U>(frag[element_index]);+ }+ } else {+ ptr[offset] = static_cast<U>(frag[element_index]);+ }+ }+ }+ }+ });+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.hindex 37b5673bd..6b1251c0a 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm.h@@ -9,13 +9,6 @@ using namespace metal;namespace uzu {namespace matmul {-///////////////////////////////////////////////////////////////////////////////-// Loop Alignment Helper-///////////////////////////////////////////////////////////////////////////////--template <bool M_aligned, bool N_aligned, bool K_aligned>-struct LoopAlignment {};-///////////////////////////////////////////////////////////////////////////////// ThreadgroupGemm Kernel///////////////////////////////////////////////////////////////////////////////@@ -32,22 +25,22 @@ template <bool transpose_b,bool MN_aligned,bool K_aligned,- typename AccumType = float,- typename Epilogue = TransformNone<U, AccumType>>+ typename AccumulatorType = float,+ typename Epilogue = TransformNone<U, AccumulatorType>>struct ThreadgroupGemm {- METAL_CONST short THREADGROUP_PADDING_A = 16 / sizeof(T);- METAL_CONST short THREADGROUP_PADDING_B = 16 / sizeof(T);- METAL_CONST short THREADGROUP_MEMORY_SIZE_A =+ METAL_CONST ushort THREADGROUP_PADDING_A = 16 / sizeof(T);+ METAL_CONST ushort THREADGROUP_PADDING_B = 16 / sizeof(T);+ METAL_CONST ushort THREADGROUP_MEMORY_SIZE_A =transpose_a ? BLOCK_DEPTH * (BLOCK_ROWS + THREADGROUP_PADDING_A): BLOCK_ROWS * (BLOCK_DEPTH + THREADGROUP_PADDING_A);- METAL_CONST short THREADGROUP_MEMORY_SIZE_B =+ METAL_CONST ushort THREADGROUP_MEMORY_SIZE_B =transpose_b ? BLOCK_COLS * (BLOCK_DEPTH + THREADGROUP_PADDING_B): BLOCK_DEPTH * (BLOCK_COLS + THREADGROUP_PADDING_B);- METAL_CONST short THREADGROUP_MEMORY_SIZE =+ METAL_CONST ushort THREADGROUP_MEMORY_SIZE =THREADGROUP_MEMORY_SIZE_A + THREADGROUP_MEMORY_SIZE_B;- METAL_CONST short THREADGROUP_SIZE =- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * 32;+ METAL_CONST ushort THREADGROUP_SIZE =+ SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;using LoaderAType = ThreadgroupLoader<T,@@ -79,24 +72,21 @@ struct ThreadgroupGemm {: BLOCK_DEPTH + THREADGROUP_PADDING_A,transpose_b ? BLOCK_DEPTH + THREADGROUP_PADDING_B: BLOCK_COLS + THREADGROUP_PADDING_B,- AccumType,+ AccumulatorType,Epilogue>;template <bool M_aligned, bool N_aligned, bool K_aligned_>static METAL_FUNC void gemm_loop(threadgroup T* a_shared,threadgroup T* b_shared,- const int gemm_k_iterations,+ const int aligned_k_iterations,thread LoaderAType& loader_a,thread LoaderBType& loader_b,thread ThreadgroupTileType& threadgroup_tile,- thread const short& threadgroup_block_rows,- thread const short& threadgroup_block_cols,- thread const short& leftover_block_depth,- LoopAlignment<M_aligned, N_aligned, K_aligned_> alignment = {}+ thread const ushort& threadgroup_block_rows,+ thread const ushort& threadgroup_block_cols,+ thread const ushort& leftover_block_depth) {- (void)alignment;-short2 tile_dimensions_a =transpose_a ? short2(threadgroup_block_rows, BLOCK_DEPTH): short2(BLOCK_DEPTH, threadgroup_block_rows);@@ -104,7 +94,7 @@ struct ThreadgroupGemm {transpose_b ? short2(BLOCK_DEPTH, threadgroup_block_cols): short2(threadgroup_block_cols, BLOCK_DEPTH);- for (int k = 0; k < gemm_k_iterations; k++) {+ for (int k = 0; k < aligned_k_iterations; k++) {threadgroup_barrier(mem_flags::mem_threadgroup);if (M_aligned) {loader_a.load_unsafe();@@ -149,25 +139,25 @@ struct ThreadgroupGemm {const device T* a,const device T* b,device U* d,- const constant GemmParams& params,+ const constant GemmParams* params,const constant float& ab_scale,const bool is_accumulate,threadgroup T* a_shared,threadgroup T* b_shared,uint simd_lane_id [[thread_index_in_simdgroup]],uint simd_group_id [[simdgroup_index_in_threadgroup]],- uint2 tid [[threadgroup_position_in_grid]],- uint3 lid [[thread_position_in_threadgroup]]+ uint2 threadgroup_position [[threadgroup_position_in_grid]],+ uint3 thread_position [[thread_position_in_threadgroup]]) {- (void)lid;+ (void)thread_position;- const int swizzle_stride = pow2(params.swizzle_log);- const int swizzled_row_id =- tid.y * swizzle_stride + (tid.x % swizzle_stride);- const int swizzled_col_id = tid.x / swizzle_stride;+ const int swizzle_stride = pow2(params->swizzle_log);+ const int swizzled_row_id = threadgroup_position.y * swizzle_stride ++ (threadgroup_position.x % swizzle_stride);+ const int swizzled_col_id = threadgroup_position.x / swizzle_stride;- if (params.threadgroups_per_row <= swizzled_col_id ||- params.threadgroups_per_column <= swizzled_row_id) {+ if (params->threadgroups_per_row <= swizzled_col_id ||+ params->threadgroups_per_column <= swizzled_row_id) {return;}@@ -179,21 +169,21 @@ struct ThreadgroupGemm {const size_t output_col_long = size_t(output_col);a += transpose_a ? output_row_long- : output_row_long * params.leading_dimension_a;- b += transpose_b ? output_col_long * params.leading_dimension_b+ : output_row_long * params->leading_dimension_a;+ b += transpose_b ? output_col_long * params->leading_dimension_b: output_col_long;- d += output_row_long * params.leading_dimension_d + output_col_long;+ d += output_row_long * params->leading_dimension_d + output_col_long;thread LoaderAType loader_a(a,- params.leading_dimension_a,+ params->leading_dimension_a,a_shared,simd_group_id,simd_lane_id);thread LoaderBType loader_b(b,- params.leading_dimension_b,+ params->leading_dimension_b,b_shared,simd_group_id,simd_lane_id@@ -201,17 +191,15 @@ struct ThreadgroupGemm {thread ThreadgroupTileType threadgroup_tile(simd_group_id, simd_lane_id);- TransformScaleAccumulate<AccumType, AccumType> epilogue(+ TransformScaleAccumulate<AccumulatorType, AccumulatorType> epilogue(ab_scale,is_accumulate ? 1.0f : 0.0f);- int gemm_k_iterations = params.aligned_inner_iterations;-///////////////////////////////////////////////////////////////////////////// MNK aligned loopif (MN_aligned) {- for (int k = 0; k < gemm_k_iterations; k++) {+ for (int k = 0; k < params->aligned_inner_iterations; k++) {threadgroup_barrier(mem_flags::mem_threadgroup);loader_a.load_unsafe();loader_b.load_unsafe();@@ -228,7 +216,7 @@ struct ThreadgroupGemm {if (!K_aligned) {int leftover_block_depth =- params.K - params.aligned_inner_iterations * BLOCK_DEPTH;+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;short2 tile_dimensions_a =transpose_a ? short2(BLOCK_ROWS, leftover_block_depth): short2(leftover_block_depth, BLOCK_ROWS);@@ -245,26 +233,26 @@ struct ThreadgroupGemm {}threadgroup_tile- .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);- threadgroup_tile.store_result(d, params.leading_dimension_d);+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);+ threadgroup_tile.store_result(d, params->leading_dimension_d);return;}///////////////////////////////////////////////////////////////////////////// MN unaligned loopelse {- short threadgroup_block_rows =- min(BLOCK_ROWS, ((int)params.M) - output_row);- short threadgroup_block_cols =- min(BLOCK_COLS, ((int)params.N) - output_col);- short leftover_block_depth =- params.K - params.aligned_inner_iterations * BLOCK_DEPTH;+ ushort threadgroup_block_rows =+ min(BLOCK_ROWS, ((int)params->M) - output_row);+ ushort threadgroup_block_cols =+ min(BLOCK_COLS, ((int)params->N) - output_col);+ ushort leftover_block_depth =+ params->K - params->aligned_inner_iterations * BLOCK_DEPTH;if (threadgroup_block_rows == BLOCK_ROWS &&threadgroup_block_cols == BLOCK_COLS) {gemm_loop<true, true, K_aligned>(a_shared,b_shared,- gemm_k_iterations,+ params->aligned_inner_iterations,loader_a,loader_b,threadgroup_tile,@@ -274,15 +262,15 @@ struct ThreadgroupGemm {);threadgroup_tile- .apply_epilogue(d, params.leading_dimension_d, 1, epilogue);- threadgroup_tile.store_result(d, params.leading_dimension_d);+ .apply_epilogue(d, params->leading_dimension_d, 1, epilogue);+ threadgroup_tile.store_result(d, params->leading_dimension_d);return;} else if (threadgroup_block_cols == BLOCK_COLS) {gemm_loop<false, true, K_aligned>(a_shared,b_shared,- gemm_k_iterations,+ params->aligned_inner_iterations,loader_a,loader_b,threadgroup_tile,@@ -293,14 +281,14 @@ struct ThreadgroupGemm {threadgroup_tile.apply_epilogue_safe(d,- params.leading_dimension_d,+ params->leading_dimension_d,1,short2(threadgroup_block_cols, threadgroup_block_rows),epilogue);threadgroup_tile.store_result_safe(d,- params.leading_dimension_d,+ params->leading_dimension_d,short2(threadgroup_block_cols, threadgroup_block_rows));return;@@ -309,7 +297,7 @@ struct ThreadgroupGemm {gemm_loop<true, false, K_aligned>(a_shared,b_shared,- gemm_k_iterations,+ params->aligned_inner_iterations,loader_a,loader_b,threadgroup_tile,@@ -320,14 +308,14 @@ struct ThreadgroupGemm {threadgroup_tile.apply_epilogue_safe(d,- params.leading_dimension_d,+ params->leading_dimension_d,1,short2(threadgroup_block_cols, threadgroup_block_rows),epilogue);threadgroup_tile.store_result_safe(d,- params.leading_dimension_d,+ params->leading_dimension_d,short2(threadgroup_block_cols, threadgroup_block_rows));return;@@ -336,7 +324,7 @@ struct ThreadgroupGemm {gemm_loop<false, false, K_aligned>(a_shared,b_shared,- gemm_k_iterations,+ params->aligned_inner_iterations,loader_a,loader_b,threadgroup_tile,@@ -347,14 +335,14 @@ struct ThreadgroupGemm {threadgroup_tile.apply_epilogue_safe(d,- params.leading_dimension_d,+ params->leading_dimension_d,1,short2(threadgroup_block_cols, threadgroup_block_rows),epilogue);threadgroup_tile.store_result_safe(d,- params.leading_dimension_d,+ params->leading_dimension_d,short2(threadgroup_block_cols, threadgroup_block_rows));return;diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.hindex e51142ad7..77cc9092d 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/gemm_mpp_core.h@@ -1,11 +1,13 @@#pragma once-#include "defines.h"-#include "loader.h"+#include "../../common/defines.h"+#include "../../common/integral_constant.h"+using namespace uzu;+#include "fragment.h"+#include "mxu_fragment_ops.h"+#include "mxu_gemm_loop.h"#include "../../generated/matmul.h"-#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>-using namespace metal;namespace uzu {@@ -13,109 +15,41 @@ namespace matmul {template <typename T,- short BLOCK_ROWS,- short BLOCK_COLS,- short SIMDGROUPS_PER_ROW,- short SIMDGROUPS_PER_COLUMN,- short SUBTILE_ROWS = 16,- short SUBTILE_COLS = 32,- short MATMUL_K_STEP = 16,- // Number of cooperative tensor elements owned by each thread.- // Determined by the MPP compiler intrinsic get_capacity().- // Empirically: SUBTILE_ROWS * SUBTILE_COLS / SIMD_SIZE = 16 * 32 / 32 = 16.- short ACCUMULATOR_CAPACITY = 16>-struct ThreadgroupGemmMpp {- METAL_CONST short PREFETCH_K = 32;- METAL_CONST short THREADGROUP_PADDING = short(16 / sizeof(T));- METAL_CONST short THREADGROUP_LD = PREFETCH_K + THREADGROUP_PADDING;- METAL_CONST short THREADGROUP_SIZE =- SIMDGROUPS_PER_ROW * SIMDGROUPS_PER_COLUMN * METAL_SIMD_SIZE;- METAL_CONST short SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;- METAL_CONST short SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;- METAL_CONST short TILES_M = SIMDGROUP_BLOCK_M / SUBTILE_ROWS;- METAL_CONST short TILES_N = SIMDGROUP_BLOCK_N / SUBTILE_COLS;- METAL_CONST short INNER_K_STEPS = PREFETCH_K / MATMUL_K_STEP;+ ushort BLOCK_ROWS,+ ushort BLOCK_COLS,+ ushort SIMDGROUPS_PER_ROW,+ ushort SIMDGROUPS_PER_COLUMN,+ bool APPLY_AB_SCALE,+ bool IS_ACCUMULATE>+struct GemmMppCore {+ METAL_CONST ushort SIMDGROUP_BLOCK_M = BLOCK_ROWS / SIMDGROUPS_PER_ROW;+ METAL_CONST ushort SIMDGROUP_BLOCK_N = BLOCK_COLS / SIMDGROUPS_PER_COLUMN;++ METAL_CONST ushort SIMDGROUP_BLOCK_K = 32;+ METAL_CONST ushort BLOCK_K = 256;++ METAL_CONST ushort TILES_M =+ SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;+ METAL_CONST ushort TILES_N =+ SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;using AccumulatorType = float;- template <- typename MatmulOperation,- typename LeftTensor,- typename RightTensor,- typename AccumulatorTensor>- static METAL_FUNC void accumulate_tiles(- const short k_steps,- const ushort tile_row_offset,- const ushort tile_col_offset,- threadgroup T* left_shared,- threadgroup T* right_shared,- thread MatmulOperation& matmul_operation,- thread LeftTensor& left_tensor,- thread RightTensor& right_tensor,- thread AccumulatorTensor& accumulator_tensor,- thread AccumulatorType- accumulator_storage[TILES_M * TILES_N][ACCUMULATOR_CAPACITY]- ) {- METAL_PRAGMA_UNROLL- for (short tile_m = 0; tile_m < TILES_M; tile_m++) {- METAL_PRAGMA_UNROLL- for (short tile_n = 0; tile_n < TILES_N; tile_n++) {- const short subtile_index = tile_m * TILES_N + tile_n;- const short left_row_base = tile_row_offset + tile_m * SUBTILE_ROWS;- const short right_col_base = tile_col_offset + tile_n * SUBTILE_COLS;-- METAL_PRAGMA_UNROLL- for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {- accumulator_tensor[i] = accumulator_storage[subtile_index][i];- }-- for (short k_step = 0; k_step < k_steps; k_step++) {- const short k_offset = k_step * MATMUL_K_STEP;-- METAL_PRAGMA_UNROLL- for (short i = 0; i < left_tensor.get_capacity(); i++) {- auto coord = left_tensor.get_multidimensional_index(i);- left_tensor[i] = left_shared- [(left_row_base + coord[1]) * THREADGROUP_LD + coord[0] +- k_offset];- }-- METAL_PRAGMA_UNROLL- for (short i = 0; i < right_tensor.get_capacity(); i++) {- auto coord = right_tensor.get_multidimensional_index(i);- right_tensor[i] = right_shared- [(right_col_base + coord[1]) * THREADGROUP_LD + coord[0] +- k_offset];- }-- matmul_operation.run(left_tensor, right_tensor, accumulator_tensor);- }-- METAL_PRAGMA_UNROLL- for (short i = 0; i < ACCUMULATOR_CAPACITY; i++) {- accumulator_storage[subtile_index][i] = accumulator_tensor[i];- }- }- }- }-static METAL_FUNC void run(const device T* left_matrix,const device T* right_matrix,device T* output_matrix,- const constant GemmParams& params,+ const constant GemmParams* params,const bool align_m,const bool align_n,- const bool is_accumulate,+ const bool align_k,const float ab_scale,- threadgroup T* left_shared,- threadgroup T* right_shared,- uint simd_lane_id,uint simd_group_id,- uint2 threadgroup_position+ uint2 threadgroup_position,+ const thread ThreadContext& thread_context) {uint tile_id_x, tile_id_y;- if (params.use_morton) {+ if (params->use_morton) {uint linear_id = threadgroup_position.x;uint morton_x = linear_id;uint morton_y = linear_id >> 1;@@ -136,8 +70,8 @@ struct ThreadgroupGemmMpp {tile_id_y = threadgroup_position.y;}- if (tile_id_x >= params.threadgroups_per_row ||- tile_id_y >= params.threadgroups_per_column) {+ if (tile_id_x >= params->threadgroups_per_row ||+ tile_id_y >= params->threadgroups_per_column) {return;}@@ -147,38 +81,9 @@ struct ThreadgroupGemmMpp {const size_t block_col_start_long = size_t(block_col_start);const device T* left_block_ptr =- left_matrix + block_row_start_long * params.leading_dimension_a;+ left_matrix + block_row_start_long * params->leading_dimension_a;const device T* right_block_ptr =- right_matrix + block_col_start_long * params.leading_dimension_b;-- ThreadgroupLoader<- T,- BLOCK_ROWS,- PREFETCH_K,- THREADGROUP_LD,- 1,- THREADGROUP_SIZE>- left_loader(- left_block_ptr,- params.leading_dimension_a,- left_shared,- ushort(simd_group_id),- ushort(simd_lane_id)- );- ThreadgroupLoader<- T,- BLOCK_COLS,- PREFETCH_K,- THREADGROUP_LD,- 1,- THREADGROUP_SIZE>- right_loader(- right_block_ptr,- params.leading_dimension_b,- right_shared,- ushort(simd_group_id),- ushort(simd_lane_id)- );+ right_matrix + block_col_start_long * params->leading_dimension_b;const ushort tile_row_offset =SIMDGROUP_BLOCK_M * (simd_group_id / SIMDGROUPS_PER_COLUMN);@@ -186,8 +91,8 @@ struct ThreadgroupGemmMpp {SIMDGROUP_BLOCK_N * (simd_group_id % SIMDGROUPS_PER_COLUMN);device T* output_ptr =- output_matrix + block_row_start_long * params.leading_dimension_d +- block_col_start_long + tile_row_offset * params.leading_dimension_d ++ output_matrix + block_row_start_long * params->leading_dimension_d ++ block_col_start_long + tile_row_offset * params->leading_dimension_d +tile_col_offset;const short simdgroup_limit_m =@@ -195,166 +100,103 @@ struct ThreadgroupGemmMpp {? SIMDGROUP_BLOCK_M: short(min(int(SIMDGROUP_BLOCK_M),- int(params.M) - int(block_row_start + tile_row_offset))+ int(params->M) - int(block_row_start + tile_row_offset)));const short simdgroup_limit_n =align_n? SIMDGROUP_BLOCK_N: short(min(int(SIMDGROUP_BLOCK_N),- int(params.N) - int(block_col_start + tile_col_offset))+ int(params->N) - int(block_col_start + tile_col_offset)));- constexpr auto matmul_descriptor = mpp::tensor_ops::matmul2d_descriptor(- SUBTILE_ROWS,- SUBTILE_COLS,- MATMUL_K_STEP,- false,- true,- false,- mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate- );-- mpp::tensor_ops::matmul2d<matmul_descriptor, metal::execution_simdgroup>- matmul_operation;-- auto left_tensor =- matmul_operation.template get_left_input_cooperative_tensor<- T,- T,- AccumulatorType>();- auto right_tensor =- matmul_operation.template get_right_input_cooperative_tensor<- T,- T,- AccumulatorType>();- auto accumulator_tensor =- matmul_operation.template get_destination_cooperative_tensor<- decltype(left_tensor),- decltype(right_tensor),- AccumulatorType>();-- const uint leading_dimension_d = params.leading_dimension_d;-- AccumulatorType accumulator_storage[TILES_M * TILES_N]- [ACCUMULATOR_CAPACITY] = {{0}};-- const uint full_prefetch_iterations = params.K / PREFETCH_K;- const uint k_remainder = params.K - full_prefetch_iterations * PREFETCH_K;-- const ushort actual_block_rows =- align_m ? BLOCK_ROWS- : ushort(min(uint(BLOCK_ROWS), params.M - block_row_start));- const ushort actual_block_cols =- align_n ? BLOCK_COLS- : ushort(min(uint(BLOCK_COLS), params.N - block_col_start));-- // Main loop- for (uint outer_k = 0; outer_k < full_prefetch_iterations; outer_k++) {- threadgroup_barrier(mem_flags::mem_threadgroup);- if (align_m) {- left_loader.load_unsafe();- } else {- left_loader.load_safe(short2(PREFETCH_K, actual_block_rows));- }- if (align_n) {- right_loader.load_unsafe();- } else {- right_loader.load_safe(short2(PREFETCH_K, actual_block_cols));- }- threadgroup_barrier(mem_flags::mem_threadgroup);-- accumulate_tiles(- INNER_K_STEPS,- tile_row_offset,- tile_col_offset,- left_shared,- right_shared,- matmul_operation,- left_tensor,- right_tensor,- accumulator_tensor,- accumulator_storage- );-- left_loader.next();- right_loader.next();- }-- // Remainder loop- if (k_remainder > 0) {- threadgroup_barrier(mem_flags::mem_threadgroup);- left_loader.load_safe(short2(k_remainder, actual_block_rows));- right_loader.load_safe(short2(k_remainder, actual_block_cols));- threadgroup_barrier(mem_flags::mem_threadgroup);-- const short remainder_steps =- short((k_remainder + MATMUL_K_STEP - 1) / MATMUL_K_STEP);-- accumulate_tiles(- remainder_steps,- tile_row_offset,- tile_col_offset,- left_shared,- right_shared,- matmul_operation,- left_tensor,- right_tensor,- accumulator_tensor,- accumulator_storage- );- }-- // Store results- METAL_PRAGMA_UNROLL- for (short tile_m = 0; tile_m < TILES_M; tile_m++) {- METAL_PRAGMA_UNROLL- for (short tile_n = 0; tile_n < TILES_N; tile_n++) {- const short row_offset = tile_m * SUBTILE_ROWS;- const short col_offset = tile_n * SUBTILE_COLS;- const short m_limit =- align_m ? SUBTILE_ROWS- : short(max(0, int(simdgroup_limit_m) - row_offset));- const short n_limit =- align_n ? SUBTILE_COLS- : short(max(0, int(simdgroup_limit_n) - col_offset));- if (m_limit <= 0 || n_limit <= 0)- continue;-- device T* output_tile_ptr =- output_ptr + row_offset * leading_dimension_d + col_offset;-- METAL_PRAGMA_UNROLL- for (short i = 0; i < short(ACCUMULATOR_CAPACITY); i++) {- auto coord = accumulator_tensor.get_multidimensional_index(i);- const bool is_valid_element = accumulator_tensor.is_valid_element(i);- AccumulatorType accumulated_value =- accumulator_storage[tile_m * TILES_N + tile_n][i] *- AccumulatorType(ab_scale);- if (align_m && align_n) {- if (is_valid_element) {- if (is_accumulate) {- accumulated_value += AccumulatorType(- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]- );- }- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =- T(accumulated_value);- }- } else {- if (is_valid_element && coord[1] < m_limit && coord[0] < n_limit) {- if (is_accumulate) {- accumulated_value += AccumulatorType(- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]]- );- }- output_tile_ptr[coord[1] * leading_dimension_d + coord[0]] =- T(accumulated_value);- }+ const device T* left_simdgroup_ptr =+ left_block_ptr + size_t(tile_row_offset) * params->leading_dimension_a;+ const device T* right_simdgroup_ptr =+ right_block_ptr ++ size_t(tile_col_offset) * int(params->leading_dimension_b);++ const int aligned_k_iterations = int(params->K) / int(BLOCK_K);++ dispatch_bool(align_k, [&](auto aligned_k) {+ dispatch_bool(+ align_m || (simdgroup_limit_m == SIMDGROUP_BLOCK_M),+ [&](auto aligned_m) {+ dispatch_bool(+ align_n || (simdgroup_limit_n == SIMDGROUP_BLOCK_N),+ [&](auto aligned_n) {+ auto accumulator_tile = gemm_loop<+ T,+ SIMDGROUP_BLOCK_M,+ SIMDGROUP_BLOCK_N,+ SIMDGROUP_BLOCK_K,+ BLOCK_K,+ false,+ true,+ aligned_m.value,+ aligned_n.value,+ aligned_k.value,+ AccumulatorType>(+ left_simdgroup_ptr,+ right_simdgroup_ptr,+ int(params->leading_dimension_a),+ int(params->leading_dimension_b),+ int(params->K),+ aligned_k_iterations,+ simdgroup_limit_m,+ simdgroup_limit_n,+ thread_context+ );++ if constexpr (APPLY_AB_SCALE) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;+ i++) {+ accumulator_tile.elements()[i] *=+ AccumulatorType(ab_scale);+ }+ }++ if constexpr (IS_ACCUMULATE) {+ Fragment<T, TILES_M, TILES_N, MxuFragmentOps>+ existing_output(thread_context);+ if constexpr (aligned_m.value && aligned_n.value) {+ existing_output.load(+ output_ptr,+ int(params->leading_dimension_d)+ );+ } else {+ existing_output.load_safe(+ output_ptr,+ int(params->leading_dimension_d),+ short2(simdgroup_limit_n, simdgroup_limit_m)+ );+ }+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < accumulator_tile.ELEMENTS_PER_TILE;+ i++) {+ accumulator_tile.elements()[i] +=+ AccumulatorType(existing_output.elements()[i]);+ }+ }++ if constexpr (aligned_m.value && aligned_n.value) {+ accumulator_tile.store(+ output_ptr,+ int(params->leading_dimension_d)+ );+ } else {+ accumulator_tile.store_safe(+ output_ptr,+ int(params->leading_dimension_d),+ short2(simdgroup_limit_n, simdgroup_limit_m)+ );+ }+ }+ );}- }- }- }+ );+ });}};diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.hindex 33e45d2c8..9fa6d3048 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/loader.h@@ -1,6 +1,6 @@#pragma once-#include "defines.h"+#include "../../common/defines.h"using namespace metal;@@ -13,32 +13,31 @@ namespace matmul {template <typename T,- short BLOCK_ROWS,- short BLOCK_COLS,- short DESTINATION_LEADING_DIMENSION,- short REDUCTION_DIMENSION,- short THREADGROUP_SIZE,- short ALIGNMENT = 1,- short READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),- short THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,- short THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>+ ushort BLOCK_ROWS,+ ushort BLOCK_COLS,+ ushort DESTINATION_LEADING_DIMENSION,+ ushort REDUCTION_DIMENSION,+ ushort THREADGROUP_SIZE,+ ushort ALIGNMENT = 1,+ ushort READS_PER_THREAD = (BLOCK_COLS * BLOCK_ROWS) / (THREADGROUP_SIZE),+ ushort THREAD_COLS = BLOCK_COLS / READS_PER_THREAD,+ ushort THREAD_ROWS = THREADGROUP_SIZE / THREAD_COLS>struct ThreadgroupLoader {- METAL_CONST short ROW_ITERATIONS =+ METAL_CONST ushort ROW_ITERATIONS =(BLOCK_ROWS + THREAD_ROWS - 1) / THREAD_ROWS;- METAL_CONST short VECTOR_SIZE = READS_PER_THREAD;const int source_leading_dimension;const int tile_stride;- const short thread_index;- const short block_row_index;- const short block_col_index;+ const ushort thread_index;+ const ushort block_row_index;+ const ushort block_col_index;threadgroup T* destination;const device T* source;struct alignas(ALIGNMENT * sizeof(T)) ReadVector {- uint8_t bytes[sizeof(T) * VECTOR_SIZE];+ uint8_t bytes[sizeof(T) * READS_PER_THREAD];};METAL_FUNC ThreadgroupLoader(@@ -52,9 +51,9 @@ struct ThreadgroupLoader {tile_stride(REDUCTION_DIMENSION ? BLOCK_COLS : BLOCK_ROWS * source_leading_dim),- thread_index(simd_group_id * 32 + simd_lane_id),+ thread_index(simd_group_id * METAL_SIMD_SIZE + simd_lane_id),block_row_index(thread_index / THREAD_COLS),- block_col_index(VECTOR_SIZE * (thread_index % THREAD_COLS)),+ block_col_index(READS_PER_THREAD * (thread_index % THREAD_COLS)),destination(destination_ptr + block_row_index * DESTINATION_LEADING_DIMENSION +block_col_index@@ -66,9 +65,9 @@ struct ThreadgroupLoader {template <typename UnaryOp>METAL_FUNC void apply_inplace_op(thread const UnaryOp& operation) const {METAL_PRAGMA_UNROLL- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {METAL_PRAGMA_UNROLL- for (short j = 0; j < VECTOR_SIZE; j++) {+ for (ushort j = 0; j < READS_PER_THREAD; j++) {destination[i * DESTINATION_LEADING_DIMENSION + j] =operation.apply(destination[i * DESTINATION_LEADING_DIMENSION + j]);}@@ -77,7 +76,7 @@ struct ThreadgroupLoader {METAL_FUNC void load_unsafe() const {METAL_PRAGMA_UNROLL- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {*((threadgroup ReadVector*)(&destination[i * DESTINATION_LEADING_DIMENSION])) =*((const device ReadVector*)(&source[i * source_leading_dimension]));@@ -90,39 +89,39 @@ struct ThreadgroupLoader {if (source_tile_dimensions.x <= 0 || source_tile_dimensions.y <= 0) {METAL_PRAGMA_UNROLL- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {METAL_PRAGMA_UNROLL- for (short j = 0; j < VECTOR_SIZE; j++) {+ for (ushort j = 0; j < READS_PER_THREAD; j++) {destination[i * DESTINATION_LEADING_DIMENSION + j] = T(0);}}return;}- bool valid_mask[VECTOR_SIZE];- T loaded_values[VECTOR_SIZE];+ bool valid_mask[READS_PER_THREAD];+ T loaded_values[READS_PER_THREAD];METAL_PRAGMA_UNROLL- for (short i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {+ for (ushort i = 0; i < BLOCK_ROWS; i += THREAD_ROWS) {METAL_PRAGMA_UNROLL- for (short j = 0; j < VECTOR_SIZE; j++) {+ for (ushort j = 0; j < READS_PER_THREAD; j++) {valid_mask[j] =(i < source_tile_dimensions.y) && (j < source_tile_dimensions.x);}METAL_PRAGMA_UNROLL- for (short j = 0; j < VECTOR_SIZE; j++) {+ for (ushort j = 0; j < READS_PER_THREAD; j++) {loaded_values[j] =source[(valid_mask[j] ? i * source_leading_dimension + j : 0)];}METAL_PRAGMA_UNROLL- for (short j = 0; j < VECTOR_SIZE; j++) {+ for (ushort j = 0; j < READS_PER_THREAD; j++) {loaded_values[j] = valid_mask[j] ? loaded_values[j] : T(0);}METAL_PRAGMA_UNROLL- for (short j = 0; j < VECTOR_SIZE; j++) {+ for (ushort j = 0; j < READS_PER_THREAD; j++) {destination[i * DESTINATION_LEADING_DIMENSION + j] = loaded_values[j];}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.hnew file mode 100644index 000000000..9b31e884a--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_fragment_ops.h@@ -0,0 +1,247 @@+#pragma once++#include <metal_simdgroup>+#include <metal_simdgroup_matrix>+#include <metal_stdlib>++#include "../../common/integral_constant.h"+#include "../../common/thread_context.h"+using namespace uzu;++#include "defines.h"++#include <MetalPerformancePrimitives/MetalPerformancePrimitives.h>++using namespace metal;++namespace uzu {+namespace matmul {++struct MxuFragmentOps {+ METAL_CONST ushort FRAGMENT_ROWS = 16;+ METAL_CONST ushort FRAGMENT_COLS = 16;++ METAL_CONST ushort ELEMENTS_PER_THREAD =+ (FRAGMENT_ROWS * FRAGMENT_COLS) / METAL_SIMD_SIZE;++ METAL_CONST ushort THREAD_ELEMENT_ROWS = 2;+ METAL_CONST ushort THREAD_ELEMENT_COLS = 4;++ METAL_CONST ushort THREAD_ELEMENT_ROW_STRIDE = 8;++ static_assert(+ THREAD_ELEMENT_ROWS * THREAD_ELEMENT_COLS == ELEMENTS_PER_THREAD,+ "MxuFragment shape is not consistent with element count"+ );++ template <typename U>+ using ThreadVector = typename metal::vec<U, ELEMENTS_PER_THREAD>;++ // Runs a 16x32 MPP matmul accumulating into (output_0 | output_1). The caller+ // supplies a `marshal_inputs` callable that copies its fragments into the+ // cooperative left/right tensors; the descriptor and output marshalling are+ // shared between all call shapes.+ template <+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a,+ bool transpose_b,+ typename MarshalInputs>+ METAL_FUNC static void mma_impl(+ thread ThreadVector<CType>& output_0,+ thread ThreadVector<CType>& output_1,+ MarshalInputs marshal_inputs+ ) {+ constexpr auto descriptor = mpp::tensor_ops::matmul2d_descriptor(+ FRAGMENT_ROWS,+ 2 * FRAGMENT_COLS,+ FRAGMENT_COLS,+ transpose_a,+ transpose_b,+ true,+ mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate+ );++ mpp::tensor_ops::matmul2d<descriptor, metal::execution_simdgroup> matmul_op;++ auto cooperative_left =+ matmul_op+ .template get_left_input_cooperative_tensor<AType, BType, CType>();+ auto cooperative_right =+ matmul_op+ .template get_right_input_cooperative_tensor<AType, BType, CType>();+ auto cooperative_output =+ matmul_op.template get_destination_cooperative_tensor<+ decltype(cooperative_left),+ decltype(cooperative_right),+ CType>();++ marshal_inputs(cooperative_left, cooperative_right);++ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_output[i] = output_0[i];+ cooperative_output[ELEMENTS_PER_THREAD + i] = output_1[i];+ }++ matmul_op.run(cooperative_left, cooperative_right, cooperative_output);++ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ output_0[i] = cooperative_output[i];+ output_1[i] = cooperative_output[ELEMENTS_PER_THREAD + i];+ }+ }++ // N-paired: output is two columns, left is one fragment, right is two.+ template <+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a = false,+ bool transpose_b = false>+ METAL_FUNC static void mma(+ thread ThreadVector<CType>& output_col_0,+ thread ThreadVector<CType>& output_col_1,+ const thread ThreadVector<AType>& left,+ metal::bool_constant<transpose_a>,+ const thread ThreadVector<BType>& right_col_0,+ const thread ThreadVector<BType>& right_col_1,+ metal::bool_constant<transpose_b>+ ) {+ mma_impl<CType, AType, BType, transpose_a, transpose_b>(+ output_col_0,+ output_col_1,+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_left[i] = left[i];+ cooperative_right[i] = right_col_0[i];+ cooperative_right[ELEMENTS_PER_THREAD + i] = right_col_1[i];+ }+ }+ );+ }++ // M-paired: output is two rows, left is two fragments, right is one.+ template <+ typename CType,+ typename AType,+ typename BType,+ bool transpose_a = false,+ bool transpose_b = false>+ METAL_FUNC static void mma(+ thread ThreadVector<CType>& output_row_0,+ thread ThreadVector<CType>& output_row_1,+ const thread ThreadVector<AType>& left_row_0,+ const thread ThreadVector<AType>& left_row_1,+ metal::bool_constant<transpose_a>,+ const thread ThreadVector<BType>& right,+ metal::bool_constant<transpose_b>+ ) {+ mma_impl<CType, AType, BType, transpose_a, transpose_b>(+ output_row_0,+ output_row_1,+ [&](thread auto& cooperative_left, thread auto& cooperative_right) {+ METAL_PRAGMA_UNROLL+ for (ushort i = 0; i < ELEMENTS_PER_THREAD; i++) {+ cooperative_left[i] = left_row_0[i];+ cooperative_left[ELEMENTS_PER_THREAD + i] = left_row_1[i];+ cooperative_right[i] = right[i];+ }+ }+ );+ }++ template <+ bool transpose_a,+ bool transpose_b,+ class OutputTile,+ class LeftTile,+ class RightTile>+ METAL_FUNC static void tile_matmul(+ thread OutputTile& output,+ thread LeftTile& left,+ thread RightTile& right+ ) {+ constexpr ushort left_tile_m =+ transpose_a ? LeftTile::TILE_COLS : LeftTile::TILE_ROWS;+ constexpr ushort tile_m = OutputTile::TILE_ROWS;+ static_assert(+ left_tile_m == tile_m,+ "tile matmul: M dimensions do not match"+ );++ constexpr ushort right_tile_n =+ transpose_b ? RightTile::TILE_ROWS : RightTile::TILE_COLS;+ constexpr ushort tile_n = OutputTile::TILE_COLS;+ static_assert(+ right_tile_n == tile_n,+ "tile matmul: N dimensions do not match"+ );++ constexpr ushort left_tile_k =+ transpose_a ? LeftTile::TILE_ROWS : LeftTile::TILE_COLS;+ constexpr ushort tile_k =+ transpose_b ? RightTile::TILE_COLS : RightTile::TILE_ROWS;+ static_assert(+ left_tile_k == tile_k,+ "tile matmul: K dimensions do not match"+ );++ constexpr auto transpose_left = metal::bool_constant<transpose_a>{};+ constexpr auto transpose_right = metal::bool_constant<transpose_b>{};++ if constexpr (tile_n == 1 && tile_m % 2 == 0) {+ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < tile_m; row += 2) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < tile_n; ++col) {+ METAL_PRAGMA_UNROLL+ for (ushort k = 0; k < tile_k; ++k) {+ mma<typename OutputTile::ElementType,+ typename LeftTile::ElementType,+ typename RightTile::ElementType,+ transpose_a,+ transpose_b>(+ output.fragment_at(row, col),+ output.fragment_at(row + 1, col),+ left.fragment_at(row, k, transpose_left),+ left.fragment_at(row + 1, k, transpose_left),+ transpose_left,+ right.fragment_at(k, col, transpose_right),+ transpose_right+ );+ }+ }+ }+ } else if constexpr (tile_n % 2 == 0) {+ METAL_PRAGMA_UNROLL+ for (ushort row = 0; row < tile_m; ++row) {+ METAL_PRAGMA_UNROLL+ for (ushort col = 0; col < tile_n; col += 2) {+ METAL_PRAGMA_UNROLL+ for (ushort k = 0; k < tile_k; ++k) {+ mma<typename OutputTile::ElementType,+ typename LeftTile::ElementType,+ typename RightTile::ElementType,+ transpose_a,+ transpose_b>(+ output.fragment_at(row, col),+ output.fragment_at(row, col + 1),+ left.fragment_at(row, k, transpose_left),+ transpose_left,+ right.fragment_at(k, col, transpose_right),+ right.fragment_at(k, col + 1, transpose_right),+ transpose_right+ );+ }+ }+ }+ }+ }+};++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.hnew file mode 100644index 000000000..a00c9b4ec--- /dev/null+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/mxu_gemm_loop.h@@ -0,0 +1,155 @@+#pragma once++#include "fragment.h"+#include "mxu_fragment_ops.h"+#include "../../generated/matmul.h"++using namespace metal;++namespace uzu {+namespace matmul {++template <+ typename T,+ ushort SIMDGROUP_BLOCK_M,+ ushort SIMDGROUP_BLOCK_N,+ ushort SIMDGROUP_BLOCK_K,+ ushort BLOCK_K,+ bool transpose_a,+ bool transpose_b,+ bool aligned_m,+ bool aligned_n,+ bool aligned_k,+ typename AccumulatorType = float>+METAL_FUNC auto gemm_loop(+ const device T* left_ptr,+ const device T* right_ptr,+ int leading_dimension_a,+ int leading_dimension_b,+ int K,+ int aligned_k_iterations,+ const short simdgroup_limit_m,+ const short simdgroup_limit_n,+ const thread ThreadContext& thread_context+) {+ constexpr ushort TILES_M = SIMDGROUP_BLOCK_M / MxuFragmentOps::FRAGMENT_ROWS;+ constexpr ushort TILES_N = SIMDGROUP_BLOCK_N / MxuFragmentOps::FRAGMENT_COLS;+ constexpr ushort TILES_K = SIMDGROUP_BLOCK_K / MxuFragmentOps::FRAGMENT_ROWS;++ constexpr ushort LEFT_TILE_ROWS = transpose_a ? TILES_K : TILES_M;+ constexpr ushort LEFT_TILE_COLS = transpose_a ? TILES_M : TILES_K;++ constexpr ushort RIGHT_TILE_ROWS = transpose_b ? TILES_N : TILES_K;+ constexpr ushort RIGHT_TILE_COLS = transpose_b ? TILES_K : TILES_N;++ Fragment<AccumulatorType, TILES_M, TILES_N, MxuFragmentOps> accumulator(+ thread_context+ );+ accumulator.clear();++ METAL_PRAGMA_NO_UNROLL+ for (int outer_k = 0; outer_k < aligned_k_iterations; outer_k++) {+ threadgroup_barrier(mem_flags::mem_none);++ METAL_PRAGMA_NO_UNROLL+ for (int inner_k = 0; inner_k < BLOCK_K; inner_k += SIMDGROUP_BLOCK_K) {+ Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(+ thread_context+ );+ Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(+ thread_context+ );++ const int left_offset =+ transpose_a ? inner_k * leading_dimension_a : inner_k;+ const int right_offset =+ transpose_b ? inner_k : inner_k * leading_dimension_b;++ if constexpr (aligned_m) {+ left_tile.load(left_ptr + left_offset, leading_dimension_a);+ } else {+ const short row_limit =+ transpose_a ? SIMDGROUP_BLOCK_K : simdgroup_limit_m;+ const short col_limit =+ transpose_a ? simdgroup_limit_m : SIMDGROUP_BLOCK_K;+ left_tile.load_safe(+ left_ptr + left_offset,+ leading_dimension_a,+ short2(col_limit, row_limit)+ );+ }++ if constexpr (aligned_n) {+ right_tile.load(right_ptr + right_offset, leading_dimension_b);+ } else {+ const short row_limit =+ transpose_b ? simdgroup_limit_n : SIMDGROUP_BLOCK_K;+ const short col_limit =+ transpose_b ? SIMDGROUP_BLOCK_K : simdgroup_limit_n;+ right_tile.load_safe(+ right_ptr + right_offset,+ leading_dimension_b,+ short2(col_limit, row_limit)+ );+ }++ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(+ accumulator,+ left_tile,+ right_tile+ );+ }++ left_ptr += transpose_a ? (BLOCK_K * leading_dimension_a) : BLOCK_K;+ right_ptr += transpose_b ? BLOCK_K : (BLOCK_K * leading_dimension_b);+ }++ if constexpr (!aligned_k) {+ simdgroup_barrier(mem_flags::mem_none);++ const short remaining_k = K - aligned_k_iterations * BLOCK_K;++ METAL_PRAGMA_NO_UNROLL+ for (int inner_k = 0; inner_k < remaining_k; inner_k += SIMDGROUP_BLOCK_K) {+ Fragment<T, LEFT_TILE_ROWS, LEFT_TILE_COLS, MxuFragmentOps> left_tile(+ thread_context+ );+ Fragment<T, RIGHT_TILE_ROWS, RIGHT_TILE_COLS, MxuFragmentOps> right_tile(+ thread_context+ );++ const short safe_k = max(short(0), short(remaining_k - inner_k));++ const short2 left_limits = transpose_a+ ? short2(simdgroup_limit_m, safe_k)+ : short2(safe_k, simdgroup_limit_m);+ const short2 right_limits = transpose_b+ ? short2(safe_k, simdgroup_limit_n)+ : short2(simdgroup_limit_n, safe_k);++ const int left_offset =+ transpose_a ? inner_k * leading_dimension_a : inner_k;+ const int right_offset =+ transpose_b ? inner_k : inner_k * leading_dimension_b;++ left_tile+ .load_safe(left_ptr + left_offset, leading_dimension_a, left_limits);+ right_tile.load_safe(+ right_ptr + right_offset,+ leading_dimension_b,+ right_limits+ );++ MxuFragmentOps::template tile_matmul<transpose_a, transpose_b>(+ accumulator,+ left_tile,+ right_tile+ );+ }+ }++ return accumulator;+}++} // namespace matmul+} // namespace uzudiff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.hindex 5c8dafae6..2db44e7a1 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_fragment.h@@ -50,21 +50,21 @@ struct SimdgroupFragment {METAL_FUNC constexpr void clear() {METAL_PRAGMA_UNROLL- for (short i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {+ for (ushort i = 0; i < MULTIPLY_ACCUMULATE_COUNT; ++i) {multiply_accumulate_data[i] = ThreadDataType(0);}}METAL_FUNC constexpr thread ThreadDataType& multiply_accumulate_at(- const short i,- const short j+ const ushort i,+ const ushort j) {return multiply_accumulate_data[i * GRID_COLS + j];}METAL_FUNC constexpr const thread ThreadDataType& multiply_accumulate_at(- const short i,- const short j+ const ushort i,+ const ushort j) const {return multiply_accumulate_data[i * GRID_COLS + j];}@@ -85,19 +85,18 @@ struct SimdgroupFragment {int SIMDGROUP_STRIDE_Y,int STRIDE_X,int STRIDE_Y>- METAL_FUNC void load(const threadgroup U* src) {+ METAL_FUNC void load(const threadgroup U* source) {METAL_PRAGMA_UNROLL- for (short i = 0; i < GRID_ROWS; ++i) {+ for (ushort i = 0; i < GRID_ROWS; ++i) {METAL_PRAGMA_UNROLL- for (short j = 0; j < GRID_COLS; ++j) {+ for (ushort j = 0; j < GRID_COLS; ++j) {SimdgroupMultiplyAccumulateOpsType::load(multiply_accumulate_at(i, j),- &(- src[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *- STRIDE_X +- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *- STRIDE_Y]- ),+ &(source+ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *+ STRIDE_X ++ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y *+ STRIDE_Y]),STRIDE_X,STRIDE_Y);@@ -106,18 +105,20 @@ struct SimdgroupFragment {}template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>- METAL_FUNC void store(device U* dst, const int leading_dimension) const {+ METAL_FUNC void store(+ device U* destination,+ const int leading_dimension+ ) const {METAL_PRAGMA_UNROLL- for (short i = 0; i < GRID_ROWS; ++i) {+ for (ushort i = 0; i < GRID_ROWS; ++i) {METAL_PRAGMA_UNROLL- for (short j = 0; j < GRID_COLS; ++j) {+ for (ushort j = 0; j < GRID_COLS; ++j) {SimdgroupMultiplyAccumulateOpsType::store(multiply_accumulate_at(i, j),- &(- dst[(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *- leading_dimension +- (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]- ),+ &(destination+ [(i * MULTIPLY_ACCUMULATE_ROWS) * SIMDGROUP_STRIDE_X *+ leading_dimension ++ (j * MULTIPLY_ACCUMULATE_COLS) * SIMDGROUP_STRIDE_Y]),leading_dimension,1);@@ -127,7 +128,7 @@ struct SimdgroupFragment {template <typename U, int SIMDGROUP_STRIDE_X, int SIMDGROUP_STRIDE_Y>METAL_FUNC void store_safe(- device U* dst,+ device U* destination,const int leading_dimension,const short2 destination_tile_dimensions) const {@@ -137,7 +138,7 @@ struct SimdgroupFragment {for (int j = 0; j < GRID_COLS; ++j) {SimdgroupMultiplyAccumulateOpsType::store_safe(multiply_accumulate_at(i, j),- dst,+ destination,leading_dimension,1,destination_tile_dimensions.y,@@ -162,12 +163,12 @@ METAL_FUNC void tile_multiply_accumulate(thread SimdgroupFragment<T, M, N>& C) {METAL_PRAGMA_UNROLL- for (short m = 0; m < M; ++m) {+ for (ushort m = 0; m < M; ++m) {METAL_PRAGMA_UNROLL- for (short n = 0; n < N; ++n) {- short column_serpentine = (m % 2) ? (N - 1 - n) : n;+ for (ushort n = 0; n < N; ++n) {+ const ushort column_serpentine = (m % 2) ? (N - 1 - n) : n;METAL_PRAGMA_UNROLL- for (short k = 0; k < K; ++k) {+ for (ushort k = 0; k < K; ++k) {SimdgroupFragment<T, M, N>::SimdgroupMultiplyAccumulateOpsType::multiply_accumulate(D.multiply_accumulate_at(m, column_serpentine),diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.hindex 6c4ddb93e..27860a3ac 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/simdgroup_multiply_accumulate.h@@ -27,7 +27,7 @@ template <typename T>struct SimdgroupMultiplyAccumulate<T, 8, 8> {METAL_CONST int ROWS = 8;METAL_CONST int COLS = 8;- METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / 32;+ METAL_CONST int THREAD_ELEMENT_COUNT = (ROWS * COLS) / METAL_SIMD_SIZE;METAL_CONST int THREAD_ELEMENT_ROWS = 1;METAL_CONST int THREAD_ELEMENT_COLS = 2;@@ -48,102 +48,107 @@ struct SimdgroupMultiplyAccumulate<T, 8, 8> {return short2{lane_col, lane_row};}- template <typename SrcPtrType, typename StrideX, typename StrideY>+ template <typename SourcePointerType, typename RowStride, typename ColStride>METAL_FUNC static constexpr void load(- thread ThreadDataType& dst,- SrcPtrType src,- StrideX stride_x,- StrideY stride_y+ thread ThreadDataType& destination,+ SourcePointerType source,+ RowStride row_stride,+ ColStride col_stride) {METAL_PRAGMA_UNROLL- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {- dst[i * THREAD_ELEMENT_COLS + j] =- static_cast<T>(src[i * stride_x + j * stride_y]);+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {+ destination[i * THREAD_ELEMENT_COLS + j] =+ static_cast<T>(source[i * row_stride + j * col_stride]);}}}template <- typename SrcPtrType,- typename StrideX,- typename StrideY,- typename LimitX,- typename LimitY,- typename OffsetX,- typename OffsetY>+ typename SourcePointerType,+ typename RowStride,+ typename ColStride,+ typename RowLimit,+ typename ColLimit,+ typename RowOffset,+ typename ColOffset>METAL_FUNC static constexpr void load_safe(- thread ThreadDataType& dst,- SrcPtrType src,- StrideX stride_x,- StrideY stride_y,- LimitX limit_x,- LimitY limit_y,- OffsetX offset_x = 0,- OffsetY offset_y = 0+ thread ThreadDataType& destination,+ SourcePointerType source,+ RowStride row_stride,+ ColStride col_stride,+ RowLimit row_limit,+ ColLimit col_limit,+ RowOffset row_offset = 0,+ ColOffset col_offset = 0) {METAL_PRAGMA_UNROLL- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {- if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {- dst[i * THREAD_ELEMENT_COLS + j] = static_cast<T>(- src[(offset_x + i) * stride_x + (offset_y + j) * stride_y]- );+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {+ if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {+ destination[i * THREAD_ELEMENT_COLS + j] =+ static_cast<T>(source+ [(row_offset + i) * row_stride ++ (col_offset + j) * col_stride]);} else {- dst[i * THREAD_ELEMENT_COLS + j] = T(0);+ destination[i * THREAD_ELEMENT_COLS + j] = T(0);}}}}- template <typename DstPtrType, typename StrideX, typename StrideY>+ template <+ typename DestinationPointerType,+ typename RowStride,+ typename ColStride>METAL_FUNC static constexpr void store(- const thread ThreadDataType& src,- DstPtrType dst,- StrideX stride_x,- StrideY stride_y+ const thread ThreadDataType& source,+ DestinationPointerType destination,+ RowStride row_stride,+ ColStride col_stride) {- using U = PointerElementType<DstPtrType>;+ using U = PointerElementType<DestinationPointerType>;METAL_PRAGMA_UNROLL- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {- dst[i * stride_x + j * stride_y] =- static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {+ destination[i * row_stride + j * col_stride] =+ static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);}}}template <- typename DstPtrType,- typename StrideX,- typename StrideY,- typename LimitX,- typename LimitY,- typename OffsetX,- typename OffsetY>+ typename DestinationPointerType,+ typename RowStride,+ typename ColStride,+ typename RowLimit,+ typename ColLimit,+ typename RowOffset,+ typename ColOffset>METAL_FUNC static constexpr void store_safe(- const thread ThreadDataType& src,- DstPtrType dst,- StrideX stride_x,- StrideY stride_y,- LimitX limit_x,- LimitY limit_y,- OffsetX offset_x = 0,- OffsetY offset_y = 0+ const thread ThreadDataType& source,+ DestinationPointerType destination,+ RowStride row_stride,+ ColStride col_stride,+ RowLimit row_limit,+ ColLimit col_limit,+ RowOffset row_offset = 0,+ ColOffset col_offset = 0) {- using U = PointerElementType<DstPtrType>;+ using U = PointerElementType<DestinationPointerType>;METAL_PRAGMA_UNROLL- for (short i = 0; i < THREAD_ELEMENT_ROWS; i++) {+ for (ushort i = 0; i < THREAD_ELEMENT_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < THREAD_ELEMENT_COLS; j++) {- if ((offset_x + i) < limit_x && (offset_y + j) < limit_y) {- dst[(offset_x + i) * stride_x + (offset_y + j) * stride_y] =- static_cast<U>(src[i * THREAD_ELEMENT_COLS + j]);+ for (ushort j = 0; j < THREAD_ELEMENT_COLS; j++) {+ if ((row_offset + i) < row_limit && (col_offset + j) < col_limit) {+ destination+ [(row_offset + i) * row_stride + (col_offset + j) * col_stride] =+ static_cast<U>(source[i * THREAD_ELEMENT_COLS + j]);}}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.hindex c323dd73e..082cd7284 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/common/threadgroup_tile.h@@ -21,67 +21,77 @@ template <int SIMDGROUPS_PER_COLUMN,bool transpose_a,bool transpose_b,- short THREADGROUP_LEADING_DIMENSION_A,- short THREADGROUP_LEADING_DIMENSION_B,- typename AccumType = float,- typename Epilogue = TransformNone<U, AccumType>>+ ushort THREADGROUP_LEADING_DIMENSION_A,+ ushort THREADGROUP_LEADING_DIMENSION_B,+ typename AccumulatorType = float,+ typename Epilogue = TransformNone<U, AccumulatorType>>struct ThreadgroupTile {- METAL_CONST short SIMDGROUP_BLOCK_SIZE = 8;+ METAL_CONST ushort SIMDGROUP_BLOCK_SIZE = 8;using SimdgroupMultiplyAccumulateType = SimdgroupMultiplyAccumulate<- AccumType,+ AccumulatorType,SIMDGROUP_BLOCK_SIZE,SIMDGROUP_BLOCK_SIZE>;- METAL_CONST short TILE_ROW_STRIDE = SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;- METAL_CONST short TILE_COL_STRIDE =+ METAL_CONST ushort TILE_ROW_STRIDE =+ SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;+ METAL_CONST ushort TILE_COL_STRIDE =SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;- METAL_CONST short TILE_ROWS =+ METAL_CONST ushort TILE_ROWS =BLOCK_ROWS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);- METAL_CONST short TILE_COLS =+ METAL_CONST ushort TILE_COLS =BLOCK_COLS / (SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);- METAL_CONST short A_STRIDE_ROW =+ METAL_CONST ushort A_STRIDE_ROW =transpose_a ? 1 : THREADGROUP_LEADING_DIMENSION_A;- METAL_CONST short A_STRIDE_INNER =+ METAL_CONST ushort A_STRIDE_INNER =transpose_a ? THREADGROUP_LEADING_DIMENSION_A : 1;- METAL_CONST short B_STRIDE_INNER =+ METAL_CONST ushort B_STRIDE_INNER =transpose_b ? 1 : THREADGROUP_LEADING_DIMENSION_B;- METAL_CONST short B_STRIDE_COL =+ METAL_CONST ushort B_STRIDE_COL =transpose_b ? THREADGROUP_LEADING_DIMENSION_B : 1;- METAL_CONST short TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;- METAL_CONST short TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;+ METAL_CONST ushort TILE_STRIDE_A = SIMDGROUP_BLOCK_SIZE * A_STRIDE_INNER;+ METAL_CONST ushort TILE_STRIDE_B = SIMDGROUP_BLOCK_SIZE * B_STRIDE_INNER;- SimdgroupFragment<AccumType, TILE_ROWS, 1, SimdgroupMultiplyAccumulateType>+ SimdgroupFragment<+ AccumulatorType,+ TILE_ROWS,+ 1,+ SimdgroupMultiplyAccumulateType>a_fragment;- SimdgroupFragment<AccumType, 1, TILE_COLS, SimdgroupMultiplyAccumulateType>+ SimdgroupFragment<+ AccumulatorType,+ 1,+ TILE_COLS,+ SimdgroupMultiplyAccumulateType>b_fragment;SimdgroupFragment<- AccumType,+ AccumulatorType,TILE_ROWS,TILE_COLS,SimdgroupMultiplyAccumulateType>c_fragment;- short simdgroup_row_offset;- short simdgroup_col_offset;+ ushort simdgroup_row_offset;+ ushort simdgroup_col_offset;- short a_shared_offset;- short b_shared_offset;+ ushort a_shared_offset;+ ushort b_shared_offset;METAL_FUNC ThreadgroupTile(ushort simd_group_id [[simdgroup_index_in_threadgroup]],ushort simd_lane_id [[thread_index_in_simdgroup]]) {- short tile_row_base =+ ushort tile_row_base =SIMDGROUP_BLOCK_SIZE * (simd_group_id / SIMDGROUPS_PER_COLUMN);- short tile_col_base =+ ushort tile_col_base =SIMDGROUP_BLOCK_SIZE * (simd_group_id % SIMDGROUPS_PER_COLUMN);- short2 simdgroup_coordinates =- SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id);+ const ushort2 simdgroup_coordinates = ushort2(+ SimdgroupMultiplyAccumulateType::get_lane_coordinates(simd_lane_id)+ );simdgroup_row_offset = simdgroup_coordinates.y;simdgroup_col_offset = simdgroup_coordinates.x;@@ -102,7 +112,7 @@ struct ThreadgroupTile {b_shared += b_shared_offset;METAL_PRAGMA_UNROLL- for (short k_block_index = 0; k_block_index < BLOCK_DEPTH;+ for (ushort k_block_index = 0; k_block_index < BLOCK_DEPTH;k_block_index += SIMDGROUP_BLOCK_SIZE) {simdgroup_barrier(mem_flags::mem_none);@@ -133,7 +143,7 @@ struct ThreadgroupTile {METAL_FUNC void store_result(device U* D, const int leading_dimension_d) {METAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {+ for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);}@@ -151,7 +161,7 @@ struct ThreadgroupTile {short2 destination_tile_dimensions) {METAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {+ for (ushort i = 0; i < decltype(c_fragment)::ELEMENTS_PER_FRAGMENT; i++) {c_fragment.elements()[i] = Epilogue::apply(c_fragment.elements()[i]);}@@ -182,22 +192,24 @@ struct ThreadgroupTile {simdgroup_col_offset * column_stride_c;METAL_PRAGMA_UNROLL- for (short i = 0; i < TILE_ROWS; i++) {+ for (ushort i = 0; i < TILE_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < TILE_COLS; j++) {+ for (ushort j = 0; j < TILE_COLS; j++) {thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);METAL_PRAGMA_UNROLL- for (short k = 0;+ for (ushort k = 0;k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;k++) {- short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);- short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);+ const ushort row_offset =+ i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;+ const ushort col_offset =+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;U c_value = c_pointer[row_offset * leading_dimension_c + col_offset * column_stride_c +k];block_data[k] = epilogue_operation.apply(block_data[k],- static_cast<AccumType>(c_value)+ static_cast<AccumulatorType>(c_value));}}@@ -217,14 +229,15 @@ struct ThreadgroupTile {tile_dimensions -= short2(simdgroup_col_offset, simdgroup_row_offset);METAL_PRAGMA_UNROLL- for (short i = 0; i < TILE_ROWS; i++) {+ for (ushort i = 0; i < TILE_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < TILE_COLS; j++) {+ for (ushort j = 0; j < TILE_COLS; j++) {thread auto& block_data = c_fragment.multiply_accumulate_at(i, j);- short row_offset = (i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW);- short col_offset = (j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN);+ const ushort row_offset = i * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_ROW;+ const ushort col_offset =+ j * SIMDGROUP_BLOCK_SIZE * SIMDGROUPS_PER_COLUMN;METAL_PRAGMA_UNROLL- for (short k = 0;+ for (ushort k = 0;k < decltype(c_fragment)::ELEMENTS_PER_MULTIPLY_ACCUMULATE;k++) {if (row_offset < tile_dimensions.y &&@@ -234,7 +247,7 @@ struct ThreadgroupTile {col_offset * column_stride_c + k];block_data[k] = epilogue_operation.apply(block_data[k],- static_cast<AccumType>(c_value)+ static_cast<AccumulatorType>(c_value));}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metalindex 69313b32a..f790479f0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm.metal@@ -22,7 +22,7 @@ KERNEL(MatmulGemm)(const device T* a,const device T* b,device T* d,- const constant uzu::matmul::GemmParams& params,+ const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,const constant float& ab_scale,diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metalindex 0a1c579a3..6fc7399d7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.metal@@ -5,52 +5,58 @@using namespace uzu::matmul;-#define GEMM_MPP_THREADGROUP_TILE_SIZE (128 * (32 + (short)(16 / sizeof(T))))-template <typename T, uint BLOCK_ROWS, uint BLOCK_COLS, uint SIMDGROUPS_PER_ROW, uint SIMDGROUPS_PER_COLUMN>VARIANTS(T, float, half, bfloat)-VARIANTS(BLOCK_ROWS, 32, 64)-VARIANTS(BLOCK_COLS, 32, 64)+VARIANTS(BLOCK_ROWS, 32, 64, 128)+VARIANTS(BLOCK_COLS, 32, 64, 128)VARIANTS(SIMDGROUPS_PER_ROW, 2, 4)-VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2)-CONSTRAINT(BLOCK_ROWS >= 16 * SIMDGROUPS_PER_ROW && BLOCK_COLS >= 32 * SIMDGROUPS_PER_COLUMN)+VARIANTS(SIMDGROUPS_PER_COLUMN, 1, 2, 4)+CONSTRAINT(+ (BLOCK_ROWS == 32 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||+ (BLOCK_ROWS == 64 && BLOCK_COLS == 32 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 1) ||+ (BLOCK_ROWS == 64 && BLOCK_COLS == 64 && SIMDGROUPS_PER_ROW == 2 && SIMDGROUPS_PER_COLUMN == 2) ||+ (BLOCK_ROWS == 128 && BLOCK_COLS == 128 && SIMDGROUPS_PER_ROW == 4 && SIMDGROUPS_PER_COLUMN == 4))KERNEL(MatmulGemmMpp)(const device T* left_matrix,const device T* right_matrix,device T* output_matrix,- const constant uzu::matmul::GemmParams& params,+ const constant uzu::matmul::GemmParams* params,const constant uint& group_count_x,const constant uint& group_count_y,const constant float& ab_scale,const bool align_m SPECIALIZE,const bool align_n SPECIALIZE,+ const bool align_k SPECIALIZE,+ const bool apply_ab_scale SPECIALIZE,const bool is_accumulate SPECIALIZE,- threadgroup T left_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],- threadgroup T right_shared[GEMM_MPP_THREADGROUP_TILE_SIZE],const uint group_x GROUPS(group_count_x),const uint group_y GROUPS(group_count_y),const uint thread_x THREADS(32),- const uint thread_y THREADS(2),- const uint thread_z THREADS(2),+ const uint thread_y THREADS(SIMDGROUPS_PER_ROW),+ const uint thread_z THREADS(SIMDGROUPS_PER_COLUMN),const ThreadContext thread_context) {- ThreadgroupGemmMpp<- T,- BLOCK_ROWS,- BLOCK_COLS,- SIMDGROUPS_PER_ROW,- SIMDGROUPS_PER_COLUMN>::- run(left_matrix,- right_matrix,- output_matrix,- params,- align_m,- align_n,- is_accumulate,- ab_scale,- left_shared,- right_shared,- thread_context.simdgroup_index,- thread_context.threadgroup_index,- uint2(group_x, group_y));+ dispatch_bool(apply_ab_scale, [&](auto scale_enabled) {+ dispatch_bool(is_accumulate, [&](auto accumulate_enabled) {+ GemmMppCore<+ T,+ BLOCK_ROWS,+ BLOCK_COLS,+ SIMDGROUPS_PER_ROW,+ SIMDGROUPS_PER_COLUMN,+ scale_enabled.value,+ accumulate_enabled.value>::+ run(left_matrix,+ right_matrix,+ output_matrix,+ params,+ align_m,+ align_n,+ align_k,+ ab_scale,+ thread_context.threadgroup_index,+ uint2(group_x, group_y),+ thread_context);+ });+ });}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rsindex 03510f4e6..a957795d0 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/gemm_mpp.rs@@ -1,3 +1,5 @@+use std::sync::OnceLock;+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct GemmMppSpecialization {pub block_rows: u32,@@ -6,169 +8,106 @@ pub struct GemmMppSpecialization {pub simdgroups_per_column: u32,pub align_m: bool,pub align_n: bool,+ pub align_k: bool,+ pub apply_ab_scale: bool,pub is_accumulate: bool,}+const fn config(+ block_rows: u32,+ block_cols: u32,+ simdgroups_per_row: u32,+ simdgroups_per_column: u32,+ align_m: bool,+ align_n: bool,+ is_accumulate: bool,+) -> GemmMppSpecialization {+ GemmMppSpecialization {+ block_rows,+ block_cols,+ simdgroups_per_row,+ simdgroups_per_column,+ align_m,+ align_n,+ align_k: false,+ apply_ab_scale: false,+ is_accumulate,+ }+}++static GEMM_MPP_PRECOMPILE: OnceLock<&'static [GemmMppSpecialization]> = OnceLock::new();+impl GemmMppSpecialization {pub fn precompile_configs(data_type: crate::DataType) -> &'static [Self] {if !matches!(data_type, crate::DataType::F16 | crate::DataType::BF16) {return &[];}- &[- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: true,- align_n: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: false,- align_n: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: true,- align_n: false,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: false,- align_n: false,- is_accumulate: false,- },- Self {- block_rows: 32,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: true,- align_n: true,- is_accumulate: false,- },- Self {- block_rows: 32,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: false,- align_n: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 32,- simdgroups_per_row: 4,- simdgroups_per_column: 1,- align_m: true,- align_n: true,- is_accumulate: false,- },- Self {- block_rows: 64,- block_cols: 32,- simdgroups_per_row: 4,- simdgroups_per_column: 1,- align_m: true,- align_n: false,- is_accumulate: false,- },- // Accumulate variants- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: true,- align_n: true,- is_accumulate: true,- },- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: false,- align_n: true,- is_accumulate: true,- },- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: true,- align_n: false,- is_accumulate: true,- },- Self {- block_rows: 64,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: false,- align_n: false,- is_accumulate: true,- },- Self {- block_rows: 32,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: true,- align_n: true,- is_accumulate: true,- },- Self {- block_rows: 32,- block_cols: 64,- simdgroups_per_row: 2,- simdgroups_per_column: 2,- align_m: false,- align_n: true,- is_accumulate: true,- },- Self {- block_rows: 64,- block_cols: 32,- simdgroups_per_row: 4,- simdgroups_per_column: 1,- align_m: true,- align_n: true,- is_accumulate: true,- },- Self {- block_rows: 64,- block_cols: 32,- simdgroups_per_row: 4,- simdgroups_per_column: 1,- align_m: true,- align_n: false,- is_accumulate: true,- },- ]+ GEMM_MPP_PRECOMPILE.get_or_init(|| {+ const BASE: &[GemmMppSpecialization] = &[+ config(64, 64, 2, 2, true, true, false),+ config(64, 64, 2, 2, false, true, false),+ config(64, 64, 2, 2, true, false, false),+ config(64, 64, 2, 2, false, false, false),+ config(32, 64, 2, 2, true, true, false),+ config(32, 64, 2, 2, false, true, false),+ config(64, 32, 4, 1, true, true, false),+ config(64, 32, 4, 1, true, false, false),+ config(128, 128, 4, 4, true, true, false),+ config(128, 128, 4, 4, false, true, false),+ config(128, 128, 4, 4, true, false, false),+ config(128, 128, 4, 4, false, false, false),+ config(64, 64, 2, 2, true, true, true),+ config(64, 64, 2, 2, false, true, true),+ config(64, 64, 2, 2, true, false, true),+ config(64, 64, 2, 2, false, false, true),+ config(32, 64, 2, 2, true, true, true),+ config(32, 64, 2, 2, false, true, true),+ config(64, 32, 4, 1, true, true, true),+ config(64, 32, 4, 1, true, false, true),+ config(128, 128, 4, 4, true, true, true),+ config(128, 128, 4, 4, false, true, true),+ config(128, 128, 4, 4, true, false, true),+ config(128, 128, 4, 4, false, false, true),+ ];++ let mut expanded = Vec::with_capacity(BASE.len() * 4);+ for &base in BASE {+ expanded.push(Self {+ align_k: true,+ apply_ab_scale: true,+ ..base+ });+ expanded.push(Self {+ align_k: true,+ apply_ab_scale: false,+ ..base+ });+ expanded.push(Self {+ align_k: false,+ apply_ab_scale: true,+ ..base+ });+ expanded.push(Self {+ align_k: false,+ apply_ab_scale: false,+ ..base+ });+ }++ Box::leak(expanded.into_boxed_slice())+ })}pub fn select(m: u32,n: u32,+ k: u32,is_accumulate: bool,+ apply_ab_scale: bool,) -> Self {- let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if n < 64 {+ let (block_rows, block_cols, simdgroups_per_row, simdgroups_per_column) = if m >= 128 && n >= 128 {+ (128, 128, 4u32, 4u32)+ } else if n < 64 {(64, 32, 4u32, 1u32)} else if m < 64 {(32, 64, 2u32, 2u32)@@ -183,6 +122,8 @@ impl GemmMppSpecialization {simdgroups_per_column,align_m: (m % block_rows) == 0,align_n: (n % block_cols) == 0,+ align_k: (k % 256) == 0,+ apply_ab_scale,is_accumulate,}}diff --git a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rsindex c26498367..f9bffbda7 100644--- a/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs+++ b/crates/backend-uzu/src/backends/metal/kernel/matmul/mod.rs@@ -124,6 +124,8 @@ impl MatmulMetalKernel {specialization.simdgroups_per_column,specialization.align_m,specialization.align_n,+ specialization.align_k,+ specialization.apply_ab_scale,specialization.is_accumulate,).map_err(MatmulError::BackendError)?;@@ -199,7 +201,7 @@ impl MatmulMetalKernel {(arguments.a, arguments.a_offset as usize),arguments.b,&mut *arguments.d,- params,+ std::slice::from_ref(¶ms),threadgroups_per_row,threadgroups_per_column,arguments.ab_scale,@@ -227,8 +229,13 @@ impl MatmulMetalKernel {arguments: MatmulArguments<Metal>,) -> Result<(), MatmulError<Metal>> {let is_accumulate = matches!(arguments.c, MatmulArgumentC::Accumulate);- let specialization =- gemm_mpp::GemmMppSpecialization::select(arguments.batch_dim, arguments.output_dim, is_accumulate);+ let specialization = gemm_mpp::GemmMppSpecialization::select(+ arguments.batch_dim,+ arguments.output_dim,+ arguments.input_dim,+ is_accumulate,+ arguments.ab_scale != 1.0,+ );let threadgroups_per_row = arguments.output_dim.div_ceil(specialization.block_cols);let threadgroups_per_column = arguments.batch_dim.div_ceil(specialization.block_rows);@@ -267,7 +274,7 @@ impl MatmulMetalKernel {(arguments.a, arguments.a_offset as usize),arguments.b,&mut *arguments.d,- params,+ std::slice::from_ref(¶ms),group_count_x,group_count_y,arguments.ab_scale,diff --git a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.hindex 162e1ba18..f86a61e25 100644--- a/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h+++ b/crates/backend-uzu/src/backends/metal/kernel/quant_matmul/mma.h@@ -30,33 +30,33 @@ struct TransformNone {METAL_FUNC static U apply(U val) { return val; }};-template <typename T, int kFragRows_, int kFragCols_>+template <typename T, int FRAG_ROWS_, int FRAG_COLS_>struct BaseMMAFrag {static_assert(- kFragRows_ == 8,+ FRAG_ROWS_ == 8,"Only 8 x 8 fragment matrices are currently supported");static_assert(- kFragCols_ == 8,+ FRAG_COLS_ == 8,"Only 8 x 8 fragment matrices are currently supported");};template <typename T>struct BaseMMAFrag<T, 8, 8> {- METAL_CONST int kFragRows = 8;- METAL_CONST int kFragCols = 8;- METAL_CONST int kElemsPerFrag = (kFragRows * kFragCols) / 32;- METAL_CONST int kElemRows = 1;- METAL_CONST int kElemCols = 2;+ METAL_CONST int FRAG_ROWS = 8;+ METAL_CONST int FRAG_COLS = 8;+ METAL_CONST int ELEMS_PER_FRAG = (FRAG_ROWS * FRAG_COLS) / 32;+ METAL_CONST int ELEM_ROWS = 1;+ METAL_CONST int ELEM_COLS = 2;static_assert(- kElemRows * kElemCols == kElemsPerFrag,+ ELEM_ROWS * ELEM_COLS == ELEMS_PER_FRAG,"MMAFrag shape is not consistent with MMAFrag size");- typedef metal::simdgroup_matrix<T, kFragRows, kFragCols> mat_type;- typedef metal::vec<T, kElemsPerFrag> frag_type;+ typedef metal::simdgroup_matrix<T, FRAG_ROWS, FRAG_COLS> mat_type;+ typedef metal::vec<T, ELEMS_PER_FRAG> frag_type;METAL_FUNC static constexpr short2 get_coord(ushort simd_lane_id [[thread_index_in_simdgroup]]@@ -75,10 +75,10 @@ struct BaseMMAFrag<T, 8, 8> {StrY str_y) {METAL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {+ for (short i = 0; i < ELEM_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * kElemCols + j] = static_cast<T>(src[i * str_x + j * str_y]);+ for (short j = 0; j < ELEM_COLS; j++) {+ dst[i * ELEM_COLS + j] = static_cast<T>(src[i * str_x + j * str_y]);}}}@@ -93,10 +93,10 @@ struct BaseMMAFrag<T, 8, 8> {using U = pointer_element_t<DstPtrType>;METAL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {+ for (short i = 0; i < ELEM_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {- dst[i * str_x + j * str_y] = static_cast<U>(src[i * kElemCols + j]);+ for (short j = 0; j < ELEM_COLS; j++) {+ dst[i * str_x + j * str_y] = static_cast<U>(src[i * ELEM_COLS + j]);}}}@@ -122,12 +122,12 @@ struct BaseMMAFrag<T, 8, 8> {using U = pointer_element_t<DstPtrType>;METAL_PRAGMA_UNROLL- for (short i = 0; i < kElemRows; i++) {+ for (short i = 0; i < ELEM_ROWS; i++) {METAL_PRAGMA_UNROLL- for (short j = 0; j < kElemCols; j++) {+ for (short j = 0; j < ELEM_COLS; j++) {if ((off_x + i) < lim_x && (off_y + j) < lim_y) {dst[(off_x + i) * str_x + (off_y + j) * str_y] =- static_cast<U>(src[i * kElemCols + j]);+ static_cast<U>(src[i * ELEM_COLS + j]);}}}@@ -165,48 +165,48 @@ struct BaseMMAFrag<T, 8, 8> {template <typename T,- int kTileRows_,- int kTileCols_,+ int TILE_ROWS_,+ int TILE_COLS_,class MMAFrag_ = BaseMMAFrag<T, 8, 8>>struct MMATile {using MMAFrag_t = MMAFrag_;using elem_type = T;- METAL_CONST int kFragRows = MMAFrag_t::kFragRows;- METAL_CONST int kFragCols = MMAFrag_t::kFragCols;- METAL_CONST int kElemsPerFrag = MMAFrag_t::kElemsPerFrag;+ METAL_CONST int FRAG_ROWS = MMAFrag_t::FRAG_ROWS;+ METAL_CONST int FRAG_COLS = MMAFrag_t::FRAG_COLS;+ METAL_CONST int ELEMS_PER_FRAG = MMAFrag_t::ELEMS_PER_FRAG;- METAL_CONST int kTileRows = kTileRows_;- METAL_CONST int kTileCols = kTileCols_;+ METAL_CONST int TILE_ROWS = TILE_ROWS_;+ METAL_CONST int TILE_COLS = TILE_COLS_;- METAL_CONST int kRows = kTileRows * kFragRows;- METAL_CONST int kCols = kTileCols * kFragCols;+ METAL_CONST int ROWS = TILE_ROWS * FRAG_ROWS;+ METAL_CONST int COLS = TILE_COLS * FRAG_COLS;- METAL_CONST int kNumFrags = kTileRows * kTileCols;- METAL_CONST int kElemsPerTile = kNumFrags * kElemsPerFrag;+ METAL_CONST int NUM_FRAGS = TILE_ROWS * TILE_COLS;+ METAL_CONST int ELEMS_PER_TILE = NUM_FRAGS * ELEMS_PER_FRAG;typedef typename MMAFrag_t::mat_type mat_type;typedef typename MMAFrag_t::frag_type frag_type;- frag_type val_frags[kNumFrags] = {frag_type(0)};+ frag_type val_frags[NUM_FRAGS] = {frag_type(0)};METAL_FUNC MMATile() thread {}METAL_FUNC constexpr void clear() {METAL_PRAGMA_UNROLL- for (short i = 0; i < kNumFrags; ++i) {+ for (short i = 0; i < NUM_FRAGS; ++i) {val_frags[i] = frag_type(0);}}METAL_FUNC constexpr thread frag_type& frag_at(const short i, const short j) {- return val_frags[i * kTileCols + j];+ return val_frags[i * TILE_COLS + j];}METAL_FUNC constexpr const thread frag_type& frag_at(const short i,const short j) const {- return val_frags[i * kTileCols + j];+ return val_frags[i * TILE_COLS + j];}METAL_FUNC thread elem_type* elems() {@@ -220,14 +220,14 @@ struct MMATile {template <typename U, int w_x, int w_y, int str_x, int str_y>METAL_FUNC void load(const threadgroup U* src) {METAL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {+ for (short i = 0; i < TILE_ROWS; ++i) {METAL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {+ for (short j = 0; j < TILE_COLS; ++j) {MMAFrag_t::load(frag_at(i, j),&(- src[(i * kFragRows) * w_x * str_x +- (j * kFragCols) * w_y * str_y]+ src[(i * FRAG_ROWS) * w_x * str_x ++ (j * FRAG_COLS) * w_y * str_y]),str_x,str_y@@ -239,12 +239,12 @@ struct MMATile {template <typename U, int w_x, int w_y>METAL_FUNC void store(device U* dst, const int ld) const {METAL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {+ for (short i = 0; i < TILE_ROWS; ++i) {METAL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {+ for (short j = 0; j < TILE_COLS; ++j) {MMAFrag_t::store(frag_at(i, j),- &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),+ &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),ld,1);@@ -256,12 +256,12 @@ struct MMATile {template <typename U, int w_x, int w_y>METAL_FUNC void store_tg(threadgroup U* dst, const int ld) const {METAL_PRAGMA_UNROLL- for (short i = 0; i < kTileRows; ++i) {+ for (short i = 0; i < TILE_ROWS; ++i) {METAL_PRAGMA_UNROLL- for (short j = 0; j < kTileCols; ++j) {+ for (short j = 0; j < TILE_COLS; ++j) {MMAFrag_t::store(frag_at(i, j),- &(dst[(i * kFragRows) * w_x * ld + (j * kFragCols) * w_y]),+ &(dst[(i * FRAG_ROWS) * w_x * ld + (j * FRAG_COLS) * w_y]),ld,1);@@ -276,9 +276,9 @@ struct MMATile {const short2 dst_tile_dims) const {METAL_PRAGMA_UNROLL- for (int i = 0; i < kTileRows; ++i) {+ for (int i = 0; i < TILE_ROWS; ++i) {METAL_PRAGMA_UNROLL- for (int j = 0; j < kTileCols; ++j) {+ for (int j = 0; j < TILE_COLS; ++j) {MMAFrag_t::store_safe(frag_at(i, j),dst,@@ -286,8 +286,8 @@ struct MMATile {1,dst_tile_dims.y,dst_tile_dims.x,- (i * kFragRows) * w_x,- (j * kFragCols) * w_y+ (i * FRAG_ROWS) * w_x,+ (j * FRAG_COLS) * w_y);}}@@ -335,18 +335,18 @@ template <typename Epilogue = TransformNone<U, AccumType>>struct BlockMMA {// MMAFrag size- METAL_CONST short kFragSize = 8;- using MMAFrag_acc_t = BaseMMAFrag<AccumType, kFragSize, kFragSize>;+ METAL_CONST short FRAG_SIZE = 8;+ using MMAFrag_acc_t = BaseMMAFrag<AccumType, FRAG_SIZE, FRAG_SIZE>;// Warp tile simdgroup matrix strides along M- METAL_CONST short TM_stride = kFragSize * WM;+ METAL_CONST short TM_stride = FRAG_SIZE * WM;// Warp tile simdgroup matrix strides along M- METAL_CONST short TN_stride = kFragSize * WN;+ METAL_CONST short TN_stride = FRAG_SIZE * WN;// Warp tile size along M- METAL_CONST short TM = BM / (kFragSize * WM);+ METAL_CONST short TM = BM / (FRAG_SIZE * WM);// Warp tile size along N- METAL_CONST short TN = BN / (kFragSize * WN);+ METAL_CONST short TN = BN / (FRAG_SIZE * WN);// Threadgroup A stridesMETAL_CONST short A_str_m = transpose_a ? 1 : lda_tgp; // M@@ -357,8 +357,8 @@ struct BlockMMA {METAL_CONST short B_str_n = transpose_b ? ldb_tgp : 1; // N// Threadgroup strides along K- METAL_CONST short tile_stride_a = kFragSize * A_str_k;- METAL_CONST short tile_stride_b = kFragSize * B_str_k;+ METAL_CONST short tile_stride_a = FRAG_SIZE * A_str_k;+ METAL_CONST short tile_stride_b = FRAG_SIZE * B_str_k;// Simdgroup matricesMMATile<AccumType, TM, 1, MMAFrag_acc_t> Atile;@@ -378,8 +378,8 @@ struct BlockMMA {ushort simd_lane_id [[thread_index_in_simdgroup]]) {// Determine thread position in simdgroup matrix- short tm = kFragSize * (simd_group_id / WN);- short tn = kFragSize * (simd_group_id % WN);+ short tm = FRAG_SIZE * (simd_group_id / WN);+ short tn = FRAG_SIZE * (simd_group_id % WN);short2 simd_coord = MMAFrag_acc_t::get_coord(simd_lane_id);sm = simd_coord.y;@@ -399,9 +399,9 @@ struct BlockMMA {As += As_offset;Bs += Bs_offset;- // Iterate over BK in blocks of kFragSize+ // Iterate over BK in blocks of FRAG_SIZEMETAL_PRAGMA_UNROLL- for (short kk = 0; kk < BK; kk += kFragSize) {+ for (short kk = 0; kk < BK; kk += FRAG_SIZE) {simdgroup_barrier(mem_flags::mem_none);Atile.template load<T, WM, 1, A_str_m, A_str_k>(As);@@ -424,7 +424,7 @@ struct BlockMMA {METAL_FUNC void store_result(device U* D, const int ldd) {// Apply epilogueMETAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}@@ -438,7 +438,7 @@ struct BlockMMA {METAL_FUNC void store_result_tg(threadgroup U* D, const int ldd) {// Apply epilogueMETAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}@@ -455,7 +455,7 @@ struct BlockMMA {) {// Apply epilogueMETAL_PRAGMA_UNROLL- for (short i = 0; i < decltype(Ctile)::kElemsPerTile; i++) {+ for (short i = 0; i < decltype(Ctile)::ELEMS_PER_TILE; i++) {Ctile.elems()[i] = Epilogue::apply(Ctile.elems()[i]);}diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rsindex d17da07be..19ec918c3 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs@@ -239,6 +239,38 @@ fn test_bf16_scale_and_accumulate() {test_mpp_correctness(64, 2048, 2048, 0.5, true, 1.0);}+// 128x128 tile (M >= 128, N >= 128)++#[uzu_test]+fn test_bf16_128x128_aligned() {+ test_mpp_correctness(128, 2048, 2048, 1.0, false, 1.0);+}++#[uzu_test]+fn test_bf16_128x128_unaligned_m() {+ test_mpp_correctness(200, 2048, 2048, 1.0, false, 1.0);+}++#[uzu_test]+fn test_bf16_128x128_unaligned_n() {+ test_mpp_correctness(128, 2048, 200, 1.0, false, 1.0);+}++#[uzu_test]+fn test_bf16_128x128_large() {+ test_mpp_correctness(256, 4096, 4096, 1.0, false, 2.0);+}++#[uzu_test]+fn test_bf16_128x128_accumulate() {+ test_mpp_correctness(128, 2048, 2048, 1.0, true, 1.0);+}++#[uzu_test]+fn test_bf16_128x128_scale_and_accumulate() {+ test_mpp_correctness(256, 2048, 2048, 0.5, true, 1.0);+}+// Benchmarksconst BENCHMARK_SHAPES: &[(usize, usize, usize)] =diff --git a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rsindex 4daadaf4b..2f197a59f 100644--- a/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs+++ b/crates/backend-uzu/tests/unit/kernel/matmul/gemm_test.rs@@ -16,9 +16,15 @@ use backend_uzu::{cpu::Cpu,},};+#[cfg(metal_backend)]+use backend_uzu::backends::metal::{MatmulDispatchPath, Metal, MetalContext};+#[cfg(metal_backend)]+use criterion::{BenchmarkId, Criterion, Throughput};use half::{bf16, f16};use num_traits::Float;+#[cfg(metal_backend)]+use crate::{common::type_short_name, uzu_bench};use crate::{common::assert::assert_eq_float, uzu_test};struct Input<T: ArrayElement + Float> {@@ -184,3 +190,69 @@ fn test_f32_ab_scale() {fn test_bf16_ab_scale() {test_with_scale::<bf16>(16, 128, 256, 0.5, 0.1);}++// Benchmarks (classic ALU GEMM path, Metal only)++#[cfg(metal_backend)]+const BENCHMARK_SHAPES: &[(usize, usize, usize)] =+ &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];++#[cfg(metal_backend)]+#[uzu_bench]+fn bench_gemm(criterion: &mut Criterion) {+ let metal_context = MetalContext::new().unwrap();++ let mut matmul_kernel =+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())+ .expect("MatmulKernel");++ let mut benchmark_group =+ criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM", type_short_name::<Metal>()));++ for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {+ let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");+ let destination_array =+ metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");++ let floating_point_operations = 2 * batch_dim * input_dim * output_dim;+ benchmark_group.throughput(Throughput::Elements(floating_point_operations as u64));++ benchmark_group.bench_function(+ BenchmarkId::new("BF16", format!("M[{batch_dim}]K[{input_dim}]N[{output_dim}]")),+ |bencher| {+ let left_buffer = left_array.buffer();+ let left_ref = left_buffer.borrow();+ let right_buffer = right_array.buffer();+ let right_ref = right_buffer.borrow();+ let destination_buffer = destination_array.buffer();++ bencher.iter_custom(|iteration_count| {+ let mut encoder = Encoder::<Metal>::new(&metal_context).unwrap();++ for _ in 0..iteration_count {+ let mut destination_ref = destination_buffer.borrow_mut();+ matmul_kernel.encode_with_path(+ &metal_context,+ MatmulArguments {+ a: left_ref.deref(),+ a_offset: 0,+ b: right_ref.deref(),+ ab_scale: 1.0,+ c: MatmulArgumentC::None,+ d: destination_ref.deref_mut(),+ batch_dim: batch_dim as u32,+ input_dim: input_dim as u32,+ output_dim: output_dim as u32,+ },+ &mut encoder,+ MatmulDispatchPath::Gemm,+ );+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()+ })+ },+ );+ }+}