Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

dsl (#57)

uuuvn9351dbd4e370
diff --git a/Cargo.lock b/Cargo.lock
index a8d408bb4..b0e8a95a1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,15 +2,6 @@
# It is not intended for manual editing.
version = 4
-[[package]]
-name = "addr2line"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
-dependencies = [
- "gimli",
-]
-
[[package]]
name = "adler2"
version = "2.0.1"
@@ -90,6 +81,12 @@ dependencies = [
"windows-sys 0.59.0",
]
+[[package]]
+name = "anyhow"
+version = "1.0.100"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
+
[[package]]
name = "aquamarine"
version = "0.1.12"
@@ -103,6 +100,18 @@ dependencies = [
"syn 1.0.109",
]
+[[package]]
+name = "arrayref"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"
+
+[[package]]
+name = "arrayvec"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
+
[[package]]
name = "async-stream"
version = "0.3.6"
@@ -122,7 +131,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -133,7 +142,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -202,7 +211,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -214,7 +223,7 @@ dependencies = [
"autocxx-engine",
"env_logger",
"indexmap 1.9.3",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -241,7 +250,7 @@ dependencies = [
"regex_static",
"rustversion",
"serde_json",
- "syn 2.0.102",
+ "syn 2.0.113",
"tempfile",
"thiserror 1.0.69",
"version_check",
@@ -257,7 +266,7 @@ dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -274,25 +283,10 @@ dependencies = [
"quote",
"serde",
"serde_json",
- "syn 2.0.102",
+ "syn 2.0.113",
"thiserror 1.0.69",
]
-[[package]]
-name = "backtrace"
-version = "0.3.75"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002"
-dependencies = [
- "addr2line",
- "cfg-if",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
- "windows-targets 0.52.6",
-]
-
[[package]]
name = "base64"
version = "0.13.1"
@@ -305,6 +299,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
+[[package]]
+name = "beef"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
+
[[package]]
name = "benchmarks"
version = "0.1.7"
@@ -343,7 +343,7 @@ dependencies = [
"regex",
"rustc-hash",
"shlex",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -358,6 +358,19 @@ version = "2.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394"
+[[package]]
+name = "blake3"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0"
+dependencies = [
+ "arrayref",
+ "arrayvec",
+ "cc",
+ "cfg-if",
+ "constant_time_eq",
+]
+
[[package]]
name = "block"
version = "0.1.6"
@@ -405,7 +418,7 @@ checksum = "4f154e572231cb6ba2bd1176980827e3d5dc04cc183a75dea38109fbdd672d29"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -460,6 +473,16 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+[[package]]
+name = "clang-ast"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d76fac0a007075fa8d38a6ec08df340bea2035cc0a6e9b0f6594bff1627692b"
+dependencies = [
+ "foldhash",
+ "serde_core",
+]
+
[[package]]
name = "clang-sys"
version = "1.8.1"
@@ -502,7 +525,7 @@ dependencies = [
"heck",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -608,6 +631,12 @@ dependencies = [
"windows-sys 0.60.2",
]
+[[package]]
+name = "constant_time_eq"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
+
[[package]]
name = "cookie"
version = "0.18.1"
@@ -791,7 +820,7 @@ dependencies = [
"proc-macro2",
"quote",
"scratch",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -804,7 +833,7 @@ dependencies = [
"indexmap 2.9.0",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -818,7 +847,7 @@ dependencies = [
"indexmap 2.9.0",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -836,7 +865,7 @@ dependencies = [
"indexmap 2.9.0",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -860,7 +889,7 @@ dependencies = [
"proc-macro2",
"quote",
"strsim",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -871,7 +900,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
dependencies = [
"darling_core",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -883,6 +912,15 @@ dependencies = [
"serde",
]
+[[package]]
+name = "depfile"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab7b3a8c29cb8b0b8e17acfcdb203b4a89a3e50718efe5f8e4f4d765be0dd1a1"
+dependencies = [
+ "logos",
+]
+
[[package]]
name = "deranged"
version = "0.4.0"
@@ -910,7 +948,7 @@ dependencies = [
"darling",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -920,7 +958,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
dependencies = [
"derive_builder_core",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -953,7 +991,7 @@ dependencies = [
"proc-macro2",
"proc-macro2-diagnostics",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -1005,7 +1043,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -1157,7 +1195,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -1237,7 +1275,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -1334,12 +1372,6 @@ dependencies = [
"wasi 0.14.2+wasi-0.2.4",
]
-[[package]]
-name = "gimli"
-version = "0.31.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
-
[[package]]
name = "glob"
version = "0.3.2"
@@ -1447,7 +1479,7 @@ dependencies = [
"reqwest",
"serde",
"serde_json",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
"tokio",
"ureq",
"windows-sys 0.60.2",
@@ -1544,7 +1576,7 @@ dependencies = [
"httpdate",
"itoa",
"pin-project-lite",
- "socket2",
+ "socket2 0.5.10",
"tokio",
"tower-service",
"tracing",
@@ -1623,7 +1655,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2",
+ "socket2 0.6.1",
"system-configuration",
"tokio",
"tower-service",
@@ -1959,9 +1991,9 @@ dependencies = [
[[package]]
name = "linux-raw-sys"
-version = "0.9.4"
+version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
+checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
[[package]]
name = "litemap"
@@ -1991,6 +2023,40 @@ version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
+[[package]]
+name = "logos"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff472f899b4ec2d99161c51f60ff7075eeb3097069a36050d8037a6325eb8154"
+dependencies = [
+ "logos-derive",
+]
+
+[[package]]
+name = "logos-codegen"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "192a3a2b90b0c05b27a0b2c43eecdb7c415e29243acc3f89cc8247a5b693045c"
+dependencies = [
+ "beef",
+ "fnv",
+ "lazy_static",
+ "proc-macro2",
+ "quote",
+ "regex-syntax 0.8.5",
+ "rustc_version",
+ "syn 2.0.113",
+]
+
+[[package]]
+name = "logos-derive"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "605d9697bcd5ef3a42d38efc51541aa3d6a4a25f7ab6d1ed0da5ac632a26b470"
+dependencies = [
+ "logos-codegen",
+]
+
[[package]]
name = "loom"
version = "0.5.6"
@@ -2115,7 +2181,7 @@ checksum = "49e7bc1560b95a3c4a25d03de42fe76ca718ab92d1a22a55b9b4cf67b3ae635c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -2200,7 +2266,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -2434,15 +2500,6 @@ dependencies = [
"objc2-core-foundation",
]
-[[package]]
-name = "object"
-version = "0.36.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
-dependencies = [
- "memchr",
-]
-
[[package]]
name = "once_cell"
version = "1.21.3"
@@ -2500,7 +2557,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -2582,7 +2639,7 @@ dependencies = [
"proc-macro2",
"proc-macro2-diagnostics",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -2598,7 +2655,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323"
dependencies = [
"memchr",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
"ucd-trie",
]
@@ -2622,7 +2679,7 @@ dependencies = [
"pest_meta",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -2694,12 +2751,12 @@ dependencies = [
[[package]]
name = "prettyplease"
-version = "0.2.34"
+version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6837b9e10d61f45f987d50808f83d1ee3d206c66acf650c3e4ae2e1f6ddedf55"
+checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
dependencies = [
"proc-macro2",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -2728,9 +2785,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
-version = "1.0.95"
+version = "1.0.104"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
+checksum = "9695f8df41bb4f3d222c95a67532365f569318332d03d5f3f67f37b20e6ebdf0"
dependencies = [
"unicode-ident",
]
@@ -2743,16 +2800,16 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
"version_check",
"yansi",
]
[[package]]
name = "quote"
-version = "1.0.40"
+version = "1.0.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
+checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
dependencies = [
"proc-macro2",
]
@@ -2876,7 +2933,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
dependencies = [
"getrandom 0.2.16",
"libredox",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
]
[[package]]
@@ -2896,7 +2953,7 @@ checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -3084,7 +3141,7 @@ dependencies = [
"proc-macro2",
"quote",
"rocket_http",
- "syn 2.0.102",
+ "syn 2.0.113",
"unicode-xid",
"version_check",
]
@@ -3116,29 +3173,32 @@ dependencies = [
"uncased",
]
-[[package]]
-name = "rustc-demangle"
-version = "0.1.25"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f"
-
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
[[package]]
name = "rustix"
-version = "1.0.7"
+version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266"
+checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34"
dependencies = [
"bitflags 2.9.4",
"errno",
"libc",
"linux-raw-sys",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3228,7 +3288,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -3272,6 +3332,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "semver"
+version = "1.0.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+
[[package]]
name = "serde"
version = "1.0.228"
@@ -3299,7 +3365,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -3310,7 +3376,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -3445,6 +3511,16 @@ dependencies = [
"windows-sys 0.52.0",
]
+[[package]]
+name = "socket2"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881"
+dependencies = [
+ "libc",
+ "windows-sys 0.60.2",
+]
+
[[package]]
name = "socks"
version = "0.3.4"
@@ -3529,9 +3605,9 @@ dependencies = [
[[package]]
name = "syn"
-version = "2.0.102"
+version = "2.0.113"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6397daf94fa90f058bd0fd88429dd9e5738999cca8d701813c80723add80462"
+checksum = "678faa00651c9eb72dd2020cbdf275d92eccb2400d568e419efdd64838145cb4"
dependencies = [
"proc-macro2",
"quote",
@@ -3555,7 +3631,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -3595,15 +3671,15 @@ dependencies = [
[[package]]
name = "tempfile"
-version = "3.20.0"
+version = "3.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
+checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c"
dependencies = [
"fastrand",
"getrandom 0.3.3",
"once_cell",
"rustix",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3626,11 +3702,11 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "2.0.16"
+version = "2.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0"
+checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
dependencies = [
- "thiserror-impl 2.0.16",
+ "thiserror-impl 2.0.17",
]
[[package]]
@@ -3641,18 +3717,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
name = "thiserror-impl"
-version = "2.0.16"
+version = "2.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960"
+checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -3734,7 +3810,7 @@ dependencies = [
"serde",
"serde_json",
"spm_precompiled",
- "thiserror 2.0.16",
+ "thiserror 2.0.17",
"unicode-normalization-alignments",
"unicode-segmentation",
"unicode_categories",
@@ -3742,31 +3818,30 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.45.1"
+version = "1.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779"
+checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408"
dependencies = [
- "backtrace",
"bytes",
"libc",
"mio 1.0.4",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
- "socket2",
+ "socket2 0.6.1",
"tokio-macros",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
]
[[package]]
name = "tokio-macros"
-version = "2.5.0"
+version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
+checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -3918,7 +3993,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -4112,11 +4187,16 @@ dependencies = [
name = "uzu"
version = "0.1.7"
dependencies = [
+ "anyhow",
"bindgen",
"bitflags 2.9.4",
+ "blake3",
"block",
"bytemuck",
+ "clang-ast",
"clap",
+ "depfile",
+ "futures",
"half",
"is_close",
"itertools 0.14.0",
@@ -4129,15 +4209,22 @@ dependencies = [
"num-traits",
"objc2",
"objc2-foundation",
+ "prettyplease",
+ "proc-macro2",
+ "quote",
"rand 0.9.2",
"regex",
"schemars",
"serde",
"serde_json",
"serde_json5",
+ "syn 2.0.113",
"sysinfo",
- "thiserror 2.0.16",
+ "tempfile",
+ "thiserror 2.0.17",
"tokenizers",
+ "tokio",
+ "walkdir",
"xgrammar-rs",
]
@@ -4215,7 +4302,7 @@ dependencies = [
"log",
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
"wasm-bindgen-shared",
]
@@ -4250,7 +4337,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -4409,7 +4496,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -4420,7 +4507,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -4800,7 +4887,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
"synstructure",
]
@@ -4821,7 +4908,7 @@ checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
@@ -4841,7 +4928,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
"synstructure",
]
@@ -4881,7 +4968,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
dependencies = [
"proc-macro2",
"quote",
- "syn 2.0.102",
+ "syn 2.0.113",
]
[[package]]
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index c7e47cf0f..a653c6832 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -2,7 +2,7 @@
name = "uzu"
version.workspace = true
edition = "2024"
-build = "build.rs"
+build = "build/main.rs"
[dependencies]
xgrammar = { package = "xgrammar-rs", version = "0.1.29", features = ["hf"] }
@@ -30,18 +30,33 @@ sysinfo = "0.37.2"
itertools = "0.14.0"
schemars = "1.0.5"
-[build-dependencies]
-bindgen = "0.72.1"
-
[dev-dependencies]
is_close = "0.1.3"
serde_json5 = "0.2.1"
schemars = { version = "1.0.5", features = ["derive"] }
mpsgraph = "0.2.0"
+[build-dependencies]
+anyhow = "1.0.100"
+bindgen = "0.72.1"
+walkdir = "2.5.0"
+depfile = "0.1.1"
+tempfile = "3.24.0"
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = "1.0.143"
+clang-ast = "0.1.35"
+futures = "0.3"
+tokio = { version = "1.48.0", features = ["full"] }
+blake3 = "1.8.2"
+quote = "1.0.42"
+proc-macro2 = "1.0.104"
+syn = "2.0.113"
+prettyplease = "0.2.37"
+itertools = "0.14.0"
+
[features]
-default = ["metal-shaders"]
-metal-shaders = []
+build-debug = []
+build-clean = []
tracing = []
[[test]]
diff --git a/crates/uzu/build/common/build_debug.rs b/crates/uzu/build/common/build_debug.rs
new file mode 100644
index 000000000..b3d0f3315
--- /dev/null
+++ b/crates/uzu/build/common/build_debug.rs
@@ -0,0 +1,19 @@
+use std::sync::OnceLock;
+use std::time::Instant;
+
+static START: OnceLock<Instant> = OnceLock::new();
+
+pub fn log(args: std::fmt::Arguments) {
+ let start = START.get_or_init(Instant::now);
+ let elapsed_ms = start.elapsed().as_millis();
+ println!("cargo::warning=(build-debug) [{elapsed_ms}ms] {args}");
+}
+
+#[macro_export]
+macro_rules! debug_log {
+ ($($arg:tt)*) => {{
+ if cfg!(feature = "build-debug") {
+ $crate::common::build_debug::log(format_args!($($arg)*));
+ }
+ }};
+}
diff --git a/crates/uzu/build/common/mod.rs b/crates/uzu/build/common/mod.rs
new file mode 100644
index 000000000..78c5959e9
--- /dev/null
+++ b/crates/uzu/build/common/mod.rs
@@ -0,0 +1 @@
+pub(crate) mod build_debug;
diff --git a/crates/uzu/build/main.rs b/crates/uzu/build/main.rs
new file mode 100644
index 000000000..6e86b0025
--- /dev/null
+++ b/crates/uzu/build/main.rs
@@ -0,0 +1,33 @@
+use std::{env, fs, path::PathBuf};
+
+use anyhow::Context;
+use tokio::try_join;
+
+mod common;
+mod metal;
+mod shared_types;
+
+#[tokio::main(flavor = "current_thread")]
+async fn main() -> anyhow::Result<()> {
+ debug_log!("build script started");
+
+ if cfg!(feature = "build-clean") {
+ let out_dir =
+ PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
+ if out_dir.exists() {
+ fs::remove_dir_all(&out_dir).with_context(|| {
+ format!("cannot clean {}", out_dir.display())
+ })?;
+ fs::create_dir_all(&out_dir).with_context(|| {
+ format!("cannot recreate {}", out_dir.display())
+ })?;
+ }
+ debug_log!("cleaned caches");
+ }
+
+ try_join!(metal::main(), shared_types::main())?;
+
+ debug_log!("build script ended");
+
+ Ok(())
+}
diff --git a/crates/uzu/build/metal/ast.rs b/crates/uzu/build/metal/ast.rs
new file mode 100644
index 000000000..c860d56fb
--- /dev/null
+++ b/crates/uzu/build/metal/ast.rs
@@ -0,0 +1,387 @@
+use std::collections::HashMap;
+
+use anyhow::{Context, bail};
+use serde::{Deserialize, Serialize};
+
+pub type MetalAstNode = clang_ast::Node<MetalAstKind>;
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct ParmVarDeclType {
+ #[serde(rename = "qualType")]
+ qual_type: Box<str>,
+ #[serde(rename = "desugaredQualType")]
+ desugared_qual_type: Option<Box<str>>,
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub enum MetalAstKind {
+ TranslationUnitDecl,
+ FunctionTemplateDecl,
+ TemplateTypeParmDecl,
+ FunctionDecl {
+ name: Box<str>,
+ },
+ ParmVarDecl {
+ name: Option<Box<str>>,
+ range: clang_ast::SourceRange,
+ #[serde(rename = "type")]
+ ty: ParmVarDeclType,
+ },
+ AnnotateAttr,
+ ConstantExpr,
+ ImplicitCastExpr,
+ StringLiteral {
+ value: Box<str>,
+ },
+ Other,
+}
+
+fn annotation_from_ast_node(
+ annotation_node: MetalAstNode
+) -> anyhow::Result<Box<[Box<str>]>> {
+ if !matches!(annotation_node.kind, MetalAstKind::AnnotateAttr) {
+ bail!(
+ "unexpected kind of root node: MetalAstKind::AnnotateAttr expected, but {:?} found",
+ annotation_node.kind
+ );
+ }
+
+ annotation_node
+ .inner
+ .into_iter()
+ .map(|mut constant_expr| {
+ let MetalAstKind::ConstantExpr = constant_expr.kind else {
+ bail!("expected ConstantExpr, found {:?}", constant_expr.kind);
+ };
+
+ if constant_expr.inner.len() != 1 {
+ bail!(
+ "ConstantExpr must have exactly one child, found {}",
+ constant_expr.inner.len()
+ );
+ }
+
+ let mut implicit_cast_expr = constant_expr.inner.pop().unwrap();
+
+ let MetalAstKind::ImplicitCastExpr = implicit_cast_expr.kind else {
+ bail!(
+ "expected ImplicitCastExpr, found {:?}",
+ implicit_cast_expr.kind
+ );
+ };
+
+ if implicit_cast_expr.inner.len() != 1 {
+ bail!(
+ "ImplicitCastExpr must have exactly one child, found {}",
+ implicit_cast_expr.inner.len()
+ );
+ }
+
+ let string_literal = implicit_cast_expr.inner.pop().unwrap();
+
+ let MetalAstKind::StringLiteral {
+ value,
+ } = string_literal.kind
+ else {
+ bail!(
+ "expected StringLiteral, found {:?}",
+ string_literal.kind
+ );
+ };
+
+ // NOTE: string literal includes "" (and is probably not parsed?), using json parse here for now
+ Ok(serde_json::from_str(&value)
+ .context("failed to parse string literal")?)
+ })
+ .collect()
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub enum MetalArgumentType {
+ Buffer,
+ Constant(Box<str>),
+ Shared(Box<str>),
+ Axis(Box<str>, Box<str>),
+ Groups(Box<str>),
+ Threads(Box<str>),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct MetalArgument {
+ pub name: Box<str>,
+ pub c_type: Box<str>,
+ pub annotation: Option<Box<[Box<str>]>>,
+ pub source: Box<str>,
+}
+
+impl MetalArgument {
+ fn from_ast_node_and_source(
+ argument_node: MetalAstNode,
+ source: &str,
+ ) -> anyhow::Result<Self> {
+ let MetalAstKind::ParmVarDecl {
+ name,
+ range,
+ ty:
+ ParmVarDeclType {
+ qual_type,
+ desugared_qual_type,
+ },
+ } = argument_node.kind
+ else {
+ bail!("argument isn't ParmVarDecl: {:?}", argument_node.kind);
+ };
+
+ let name = name.context("ParmVarDecl has no name")?;
+
+ let c_type = desugared_qual_type.unwrap_or(qual_type);
+
+ if argument_node.inner.len() > 1 {
+ bail!("more than one annotation on argument ast node");
+ }
+
+ let annotation =
+ if let Some(annotation_node) = argument_node.inner.first() {
+ Some(annotation_from_ast_node(annotation_node.clone())?)
+ } else {
+ None
+ };
+
+ let start_offset = range
+ .begin
+ .spelling_loc
+ .context("no start location in source range")?
+ .offset;
+ let end_offset = range
+ .end
+ .spelling_loc
+ .context("no end location in source range")?
+ .offset;
+ let source =
+ str::from_utf8(&source.as_bytes()[start_offset..=end_offset])
+ .context("source range is not utf-8")?
+ .into();
+
+ Ok(Self {
+ name,
+ c_type,
+ annotation,
+ source,
+ })
+ }
+
+ pub fn argument_type(&self) -> anyhow::Result<MetalArgumentType> {
+ if let Some(annotation) = self.annotation.as_ref() {
+ let mut annotation = annotation.to_vec();
+
+ if annotation.is_empty() {
+ bail!("empty annotation");
+ }
+ let annotation_key = annotation.remove(0);
+
+ match &*annotation_key {
+ "dsl.axis" => {
+ if annotation.len() != 2 {
+ bail!(
+ "dsl.axis requires 2 arguments, got {}",
+ annotation.len()
+ );
+ }
+ Ok(MetalArgumentType::Axis(
+ annotation.remove(0),
+ annotation.remove(0),
+ ))
+ },
+ "dsl.groups" => {
+ if annotation.len() != 1 {
+ bail!(
+ "dsl.groups requires 1 argument, got {}",
+ annotation.len()
+ );
+ }
+ Ok(MetalArgumentType::Groups(annotation.remove(0)))
+ },
+ "dsl.threads" => {
+ if annotation.len() != 1 {
+ bail!(
+ "dsl.threads requires 1 argument, got {}",
+ annotation.len()
+ );
+ }
+ Ok(MetalArgumentType::Threads(annotation.remove(0)))
+ },
+ _ => bail!("unknown annotation: {annotation_key}"),
+ }
+ } else if (self.c_type.contains("device")
+ || self.c_type.contains("constant"))
+ && self.c_type.contains('*')
+ && !self.c_type.contains('&')
+ {
+ Ok(MetalArgumentType::Buffer)
+ } else if let ["const", "constant", c_type_scalar, "&"] =
+ self.c_type.split_whitespace().collect::<Vec<_>>().as_slice()
+ {
+ let rust_type = match *c_type_scalar {
+ "uint" | "uint32_t" => "u32",
+ "int" | "int32_t" => "i32",
+ "float" => "f32",
+ _ => {
+ bail!("unknown scalar type: {c_type_scalar}")
+ },
+ };
+ Ok(MetalArgumentType::Constant(rust_type.into()))
+ } else if self.c_type.contains("threadgroup")
+ && self.c_type.contains('*')
+ {
+ let lbracket = self
+ .source
+ .rfind('[')
+ .context("threadgroup missing size bracket")?
+ + 1;
+ let rbracket = self
+ .source
+ .rfind(']')
+ .context("threadgroup missing size bracket")?;
+ let size_expr = &self.source[lbracket..rbracket];
+ Ok(MetalArgumentType::Shared(size_expr.into()))
+ } else {
+ bail!("cannot parse c type: {}", self.c_type);
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct MetalKernelInfo {
+ pub name: Box<str>,
+ pub arguments: Box<[MetalArgument]>,
+ pub specializations: Option<(Box<str>, Box<[Box<str>]>)>,
+}
+
+impl MetalKernelInfo {
+ pub fn has_axis(&self) -> bool {
+ self.arguments.iter().any(|a| {
+ matches!(a.argument_type(), Ok(MetalArgumentType::Axis(..)))
+ })
+ }
+
+ pub fn has_groups(&self) -> bool {
+ self.arguments.iter().any(|a| {
+ matches!(a.argument_type(), Ok(MetalArgumentType::Groups(_)))
+ })
+ }
+
+ pub fn has_threads(&self) -> bool {
+ self.arguments.iter().any(|a| {
+ matches!(a.argument_type(), Ok(MetalArgumentType::Threads(_)))
+ })
+ }
+}
+
+impl MetalKernelInfo {
+ pub fn from_ast_node_and_source(
+ node: MetalAstNode,
+ source: &str,
+ ) -> anyhow::Result<Option<Self>> {
+ let (is_template, node) =
+ if matches!(node.kind, MetalAstKind::FunctionTemplateDecl) {
+ let node = node
+ .inner
+ .into_iter()
+ .find(|c| {
+ matches!(
+ c.kind,
+ MetalAstKind::FunctionDecl {
+ name: _
+ }
+ )
+ })
+ .context(
+ "unexpected kind of root node: template without function",
+ )?;
+
+ (true, node)
+ } else if matches!(node.kind, MetalAstKind::FunctionDecl { .. }) {
+ (false, node)
+ } else {
+ return Ok(None);
+ };
+
+ let MetalAstKind::FunctionDecl {
+ name,
+ } = node.kind
+ else {
+ bail!(
+ "unexpected kind of root node: function expected, but {:?} found",
+ node.kind
+ );
+ };
+
+ let mut arg_nodes = Vec::new();
+ let mut annotations = Vec::new();
+
+ for node in node.inner {
+ match node.kind {
+ MetalAstKind::ParmVarDecl {
+ name: _,
+ range: _,
+ ty: _,
+ } => arg_nodes.push(node),
+ MetalAstKind::AnnotateAttr => {
+ annotations.push(annotation_from_ast_node(node)?)
+ },
+ _ => (),
+ }
+ }
+
+ let annotations = annotations
+ .into_iter()
+ .map(|a| {
+ if !a.is_empty() {
+ let mut a = a.into_vec();
+ Ok((a.remove(0), a.into_boxed_slice()))
+ } else {
+ bail!("zero length annotation");
+ }
+ })
+ .collect::<anyhow::Result<HashMap<_, _>>>()?;
+
+ if !annotations.contains_key("dsl.kernel") {
+ return Ok(None);
+ }
+
+ if annotations.contains_key("dsl.specialize") != is_template {
+ bail!("mismatch between AST nodes and specialization annotation");
+ }
+
+ let specializations = if is_template {
+ let specialize = annotations
+ .get("dsl.specialize")
+ .context("missing dsl.specialize annotation")?;
+ let [specialization_typename, specialization_variants] =
+ specialize.as_ref()
+ else {
+ bail!("malformed dsl.specialize annotation");
+ };
+
+ let specialization_variants = specialization_variants
+ .split(',')
+ .map(|v| v.trim().into())
+ .collect::<Box<[Box<str>]>>();
+
+ Some((specialization_typename.clone(), specialization_variants))
+ } else {
+ None
+ };
+
+ let arguments = arg_nodes
+ .into_iter()
+ .map(|an| MetalArgument::from_ast_node_and_source(an, source))
+ .collect::<anyhow::Result<Box<[_]>>>()?;
+
+ Ok(Some(MetalKernelInfo {
+ name,
+ arguments,
+ specializations,
+ }))
+ }
+}
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
new file mode 100644
index 000000000..bd651d5c7
--- /dev/null
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -0,0 +1,176 @@
+use std::iter::repeat_n;
+
+use anyhow::Context;
+use itertools::Itertools;
+use proc_macro2::TokenStream;
+use quote::{format_ident, quote};
+
+use super::ast::{MetalArgumentType, MetalKernelInfo};
+
+pub fn bindgen(kernel: &MetalKernelInfo) -> anyhow::Result<TokenStream> {
+ let kernel_name = kernel.name.as_ref();
+ let struct_name = format_ident!("{kernel_name}Kernel");
+
+ let parse_expr = |expr: &Box<str>| -> anyhow::Result<TokenStream> {
+ syn::parse_str(expr.as_ref()).with_context(|| {
+ format!(
+ "cannot parse rust expression `{}` in kernel `{}`",
+ expr, kernel_name
+ )
+ })
+ };
+
+ let (specialize_extra_argument, specialize_kernel_format) = if kernel
+ .specializations
+ .is_some()
+ {
+ (
+ quote! { , data_type: crate::backends::metal::KernelDataType },
+ quote! { &format!("{}_{}", #kernel_name, data_type.function_name_suffix()) },
+ )
+ } else {
+ (quote! {}, quote! { #kernel_name })
+ };
+
+ let (encode_args_defs, encode_args_sets, encode_args_names): (
+ Vec<TokenStream>,
+ Vec<TokenStream>,
+ Vec<TokenStream>,
+ ) = kernel
+ .arguments
+ .iter()
+ .filter(|k| {
+ matches!(
+ k.argument_type(),
+ Ok(MetalArgumentType::Buffer | MetalArgumentType::Constant(_))
+ )
+ })
+ .enumerate()
+ .map(|(i, ka)| {
+ let arg_index = i as u64;
+ let arg_name = format_ident!("{}", ka.name.as_ref());
+
+ match ka.argument_type().unwrap() {
+ MetalArgumentType::Buffer => {
+ let def = quote! { #arg_name: &metal::Buffer };
+ let set = quote! {
+ compute_encoder.set_buffer(#arg_index, Some(#arg_name), 0);
+ };
+
+ (def, set, quote! { #arg_name })
+ },
+ MetalArgumentType::Constant(r_type) => {
+ let arg_dtype = format_ident!("{r_type}");
+ let def = quote! { #arg_name: #arg_dtype };
+ let set = quote! {
+ compute_encoder.set_bytes(#arg_index, std::mem::size_of::<#arg_dtype>() as u64, std::ptr::addr_of!(#arg_name).cast::<std::ffi::c_void>());
+ };
+
+ (def, set, quote! { #arg_name })
+ }
+ _ => unreachable!(),
+ }
+ })
+ .multiunzip();
+
+ let dispatch = if kernel.has_axis() {
+ if kernel.has_groups() || kernel.has_threads() {
+ anyhow::bail!("mixing groups/threads and axis is not supported");
+ }
+
+ let mut axis = kernel
+ .arguments
+ .iter()
+ .filter_map(|a| match a.argument_type() {
+ Ok(MetalArgumentType::Axis(
+ threads_rexprs,
+ threads_per_group_rexprs,
+ )) => Some((threads_rexprs, threads_per_group_rexprs)),
+ _ => None,
+ })
+ .map(|(threads_rexprs, threads_per_group_rexprs)| {
+ let threads = parse_expr(&threads_rexprs)?;
+ let threads_per_group = parse_expr(&threads_per_group_rexprs)?;
+ Ok((threads, threads_per_group))
+ })
+ .collect::<anyhow::Result<Vec<(TokenStream, TokenStream)>>>()?;
+ axis.extend(repeat_n((quote! {1}, quote! {1}), 3 - axis.len()));
+
+ let (threads, threads_per_group): (Vec<TokenStream>, Vec<TokenStream>) =
+ axis.into_iter().unzip();
+
+ quote! {
+ compute_encoder.dispatch_threads(
+ metal::MTLSize::new(#((#threads) as u64, )*),
+ metal::MTLSize::new(#((#threads_per_group) as u64, )*),
+ );
+ }
+ } else {
+ let mut groups = kernel
+ .arguments
+ .iter()
+ .filter_map(|a| {
+ if let Ok(MetalArgumentType::Groups(rexprs)) = a.argument_type()
+ {
+ Some(parse_expr(&rexprs))
+ } else {
+ None
+ }
+ })
+ .collect::<anyhow::Result<Vec<TokenStream>>>()?;
+ groups.extend(repeat_n(quote! {1}, 3 - groups.len()));
+
+ let mut threads = kernel
+ .arguments
+ .iter()
+ .filter_map(|a| {
+ if let Ok(MetalArgumentType::Threads(rexprs)) =
+ a.argument_type()
+ {
+ Some(parse_expr(&rexprs))
+ } else {
+ None
+ }
+ })
+ .collect::<anyhow::Result<Vec<TokenStream>>>()?;
+ threads.extend(repeat_n(quote! {1}, 3 - threads.len()));
+
+ quote! {
+ compute_encoder.dispatch_thread_groups(
+ metal::MTLSize::new(#((#groups) as u64, )*),
+ metal::MTLSize::new(#((#threads) as u64, )*),
+ );
+ }
+ };
+
+ Ok(quote! {
+ pub struct #struct_name {
+ pipeline: metal::ComputePipelineState,
+ }
+
+ impl #struct_name {
+ pub fn new(context: &crate::backends::metal::MTLContext #specialize_extra_argument) -> Result<Self, crate::backends::metal::MTLError> {
+ let func = context.library.get_function(#specialize_kernel_format, None)?;
+ let pipeline = context.device.new_compute_pipeline_state_with_function(&func)?;
+ Ok(Self { pipeline })
+ }
+
+ pub fn encode(&self, #(#encode_args_defs, )*compute_encoder: &metal::ComputeCommandEncoderRef) {
+ compute_encoder.set_compute_pipeline_state(&self.pipeline);
+ #(#encode_args_sets)*
+ #dispatch
+ }
+
+ pub fn encode_if(&self, #(#encode_args_defs, )*compute_encoder: &metal::ComputeCommandEncoderRef, predicate: Option<&metal::BufferRef>) {
+ use crate::backends::metal::metal_extensions::ComputeEncoderConditional;
+ compute_encoder.condition(
+ predicate,
+ || {
+ self.encode(#(#encode_args_names, )*compute_encoder);
+ },
+ None::<fn()>,
+ );
+ }
+ }
+ })
+}
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
new file mode 100644
index 000000000..8e448b740
--- /dev/null
+++ b/crates/uzu/build/metal/compiler.rs
@@ -0,0 +1,376 @@
+use std::collections::HashMap;
+use std::iter::{once, zip};
+use std::{env, fs, path::PathBuf};
+
+use anyhow::{Context, bail};
+use futures::future::try_join_all;
+use serde::{Deserialize, Serialize};
+use tokio::io::AsyncReadExt;
+use tokio::task::spawn_blocking;
+
+use crate::debug_log;
+
+use super::ast::{MetalArgumentType, MetalKernelInfo};
+use super::toolchain::MetalToolchain;
+
+#[derive(Serialize, Deserialize, Debug)]
+pub struct ObjectInfo {
+ pub object_path: PathBuf,
+ pub kernels: Box<[MetalKernelInfo]>,
+ pub dependency_hashes: HashMap<Box<str>, [u8; blake3::OUT_LEN]>,
+}
+
+pub fn wrappers(kernel: &MetalKernelInfo) -> anyhow::Result<Box<[Box<str>]>> {
+ let mut kernel_wrappers = Vec::new();
+
+ for specialization_variant in kernel
+ .specializations
+ .as_ref()
+ .map(|(t, v)| zip(std::iter::repeat(t), v.iter()).map(Some).collect())
+ .unwrap_or(vec![None])
+ {
+ let (wrapper_name, underlying_name) =
+ if let Some((_, variant)) = specialization_variant {
+ (
+ format!("{}_{}", kernel.name, variant),
+ format!("{}<{}>", kernel.name, variant),
+ )
+ } else {
+ (kernel.name.to_string(), kernel.name.to_string())
+ };
+
+ let max_total_threads_per_threadgroup = kernel
+ .arguments
+ .iter()
+ .filter_map(|a| match a.argument_type() {
+ Ok(MetalArgumentType::Axis(_, l))
+ | Ok(MetalArgumentType::Threads(l)) => Some(format!("({l})")),
+ _ => None,
+ })
+ .collect::<Vec<_>>()
+ .join(" * ");
+
+ let mut wrapper_arguments = kernel
+ .arguments
+ .iter()
+ .filter_map(|a| match a.argument_type() {
+ Ok(MetalArgumentType::Buffer)
+ | Ok(MetalArgumentType::Constant(_)) => Some(format!(
+ "{} {}",
+ a.c_type
+ .split_whitespace()
+ .map(|token| {
+ if let Some((typename, variant)) =
+ specialization_variant
+ && token == typename.as_ref()
+ {
+ variant
+ } else {
+ token
+ }
+ })
+ .collect::<Vec<_>>()
+ .join(" "),
+ a.name
+ )),
+ _ => None,
+ })
+ .collect::<Vec<_>>();
+
+ if kernel.has_axis() {
+ if kernel.has_groups() || kernel.has_threads() {
+ bail!("mixing groups/threads and axis is not supported");
+ }
+
+ wrapper_arguments.push(
+ "uint3 __dsl_axis_idx [[thread_position_in_grid]]".into(),
+ );
+ }
+
+ if kernel.has_groups() {
+ wrapper_arguments.push(
+ "uint3 __dsl_group_idx [[threadgroup_position_in_grid]]".into(),
+ );
+ }
+
+ if kernel.has_threads() {
+ wrapper_arguments.push(
+ "uint3 __dsl_thread_idx [[thread_position_in_threadgroup]]"
+ .into(),
+ );
+ }
+
+ let wrapper_arguments = wrapper_arguments.join(", ");
+
+ let shared_definitions = kernel.arguments.iter().filter_map(|a| {
+ if let Ok(MetalArgumentType::Shared(len)) = a.argument_type() {
+ Some(format!(
+ "{} {}[{}]",
+ a.c_type.replace('*', ""),
+ a.name,
+ len.as_ref(),
+ ))
+ } else {
+ None
+ }
+ });
+
+ let underlying_arguments = {
+ let mut group_axis_letters = ["x", "y", "z"].iter();
+ let mut thread_axis_letters = ["x", "y", "z"].iter();
+
+ kernel
+ .arguments
+ .iter()
+ .map(|a| match a.argument_type().unwrap() {
+ MetalArgumentType::Buffer
+ | MetalArgumentType::Constant(_)
+ | MetalArgumentType::Shared(_) => a.name.to_string(),
+ MetalArgumentType::Axis(..) => {
+ format!(
+ "__dsl_axis_idx.{}",
+ group_axis_letters.next().unwrap()
+ )
+ },
+ MetalArgumentType::Groups(_) => {
+ format!(
+ "__dsl_group_idx.{}",
+ group_axis_letters.next().unwrap()
+ )
+ },
+ MetalArgumentType::Threads(_) => {
+ format!(
+ "__dsl_thread_idx.{}",
+ thread_axis_letters.next().unwrap()
+ )
+ },
+ })
+ .collect::<Vec<_>>()
+ .join(", ")
+ };
+
+ let underlying_call =
+ format!("{underlying_name}({underlying_arguments})");
+
+ let wrapper_body = shared_definitions
+ .chain(once(underlying_call))
+ .map(|l| format!(" {l};\n"))
+ .collect::<Vec<_>>()
+ .join("");
+
+ kernel_wrappers.push(
+ format!(
+ "\n[[kernel, max_total_threads_per_threadgroup({max_total_threads_per_threadgroup})]] void {wrapper_name}({wrapper_arguments}) {{\n{wrapper_body}}}\n"
+ )
+ .into(),
+ );
+ }
+
+ Ok(kernel_wrappers.into())
+}
+
+async fn hash_dependencies(
+ dependencies: impl Iterator<Item = Box<str>>
+) -> anyhow::Result<HashMap<Box<str>, [u8; blake3::OUT_LEN]>> {
+ let futures: Vec<_> = dependencies
+ .map(|path| async move {
+ let mut file = tokio::fs::File::open(path.as_ref())
+ .await
+ .with_context(|| format!("cannot open file {}", path))?;
+ let mut contents = Vec::new();
+ file.read_to_end(&mut contents)
+ .await
+ .with_context(|| format!("cannot read file {}", path))?;
+ let hash = spawn_blocking(move || blake3::hash(&contents))
+ .await
+ .context("hash task panicked")?;
+ Ok::<_, anyhow::Error>((path, hash.into()))
+ })
+ .collect();
+ try_join_all(futures).await.map(|v| v.into_iter().collect())
+}
+
+#[derive(Debug)]
+pub struct MetalCompiler {
+ out_dir: PathBuf,
+ build_dir: PathBuf,
+ toolchain: MetalToolchain,
+}
+
+impl MetalCompiler {
+ pub fn new() -> anyhow::Result<Self> {
+ let out_dir =
+ PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?);
+ let build_dir = out_dir.join("metal");
+ fs::create_dir_all(&build_dir).with_context(|| {
+ format!("cannot create {}", build_dir.display())
+ })?;
+
+ let toolchain =
+ MetalToolchain::from_env().context("cannot create toolchain")?;
+
+ Ok(Self {
+ out_dir,
+ build_dir,
+ toolchain,
+ })
+ }
+
+ pub async fn compile(
+ &self,
+ source_path: PathBuf,
+ ) -> anyhow::Result<(ObjectInfo, bool)> {
+ let source_path_pretty =
+ source_path.file_name().unwrap().to_str().unwrap();
+ debug_log!("compile start: {source_path_pretty}");
+
+ let source_path_display = source_path.display().to_string();
+ let source_path_hash =
+ blake3::hash(source_path.to_string_lossy().as_bytes());
+ let source_file_name = source_path
+ .with_extension("")
+ .file_name()
+ .context("source path has no file name")?
+ .to_os_string();
+
+ let build_dir = self.build_dir.join(source_path_hash.to_string());
+ let objectinfo_path =
+ build_dir.join(&source_file_name).with_extension("objectinfo");
+
+ // Check cache
+ if build_dir.exists() {
+ if let Ok(contents) = tokio::fs::read(&objectinfo_path).await
+ && let Ok(cached) =
+ serde_json::from_slice::<ObjectInfo>(&contents)
+ && let Ok(current_hashes) =
+ hash_dependencies(cached.dependency_hashes.keys().cloned())
+ .await
+ && current_hashes == cached.dependency_hashes
+ {
+ if cfg!(feature = "build-debug") {
+ let kernel_list = cached
+ .kernels
+ .iter()
+ .map(|k| k.name.as_ref())
+ .collect::<Vec<_>>()
+ .join(", ");
+ if kernel_list.is_empty() {
+ debug_log!("compile cached: {source_path_pretty}");
+ } else {
+ debug_log!(
+ "compile cached: {source_path_pretty} (kernels: [{kernel_list}])"
+ );
+ }
+ }
+ return Ok((cached, true));
+ }
+ fs::remove_dir_all(&build_dir).ok();
+ }
+
+ fs::create_dir_all(&build_dir).with_context(|| {
+ format!("cannot create build directory {}", build_dir.display())
+ })?;
+
+ // Analyze source
+ let (metal_kernel_infos, dependencies) =
+ self.toolchain.analyze(&source_path).await.with_context(|| {
+ format!("cannot analyze {}", source_path_display)
+ })?;
+
+ let kernel_infos: Vec<MetalKernelInfo> = metal_kernel_infos.collect();
+
+ // Generate footer with kernel wrappers
+ let mut footer = String::new();
+ for kernel_info in &kernel_infos {
+ for wrapper in wrappers(kernel_info)
+ .with_context(|| {
+ format!("cannot generate wrappers for {}", kernel_info.name)
+ })?
+ .iter()
+ {
+ footer.push_str(wrapper);
+ }
+ }
+
+ // Compile
+ let object_path =
+ build_dir.join(&source_file_name).with_extension("air");
+
+ let compile_output = self
+ .toolchain
+ .compile(&source_path, &footer, &object_path)
+ .await
+ .with_context(|| {
+ format!("cannot compile {}", source_path_display)
+ })?;
+
+ if let Some(warnings) = &compile_output {
+ for line in warnings.lines() {
+ println!("cargo::warning={line}");
+ }
+ }
+
+ let dependency_hashes = hash_dependencies(dependencies)
+ .await
+ .context("cannot hash dependencies")?;
+
+ let object_info = ObjectInfo {
+ object_path,
+ kernels: kernel_infos.into_boxed_slice(),
+ dependency_hashes,
+ };
+
+ fs::write(
+ &objectinfo_path,
+ serde_json::to_string_pretty(&object_info)
+ .context("failed to serialize object info")?
+ .as_bytes(),
+ )
+ .with_context(|| {
+ format!("cannot write object info {}", objectinfo_path.display())
+ })?;
+
+ if cfg!(feature = "build-debug") {
+ let kernel_list = object_info
+ .kernels
+ .iter()
+ .map(|k| k.name.as_ref())
+ .collect::<Vec<_>>()
+ .join(", ");
+ if kernel_list.is_empty() {
+ debug_log!("compile end: {source_path_pretty}");
+ } else {
+ debug_log!(
+ "compile end: {source_path_pretty} (kernels: [{kernel_list}])"
+ );
+ }
+ }
+
+ Ok((object_info, false))
+ }
+
+ pub async fn link(
+ &self,
+ objects: impl IntoIterator<Item = &ObjectInfo>,
+ ) -> anyhow::Result<PathBuf> {
+ debug_log!("linking start");
+
+ let library_path = self.out_dir.join("default.metallib");
+
+ let link_output = self
+ .toolchain
+ .link(objects.into_iter().map(|o| &o.object_path), &library_path)
+ .await
+ .context("cannot link objects")?;
+
+ if let Some(warnings) = &link_output {
+ for line in warnings.lines() {
+ println!("cargo::warning={line}");
+ }
+ }
+
+ debug_log!("linking end");
+
+ Ok(library_path)
+ }
+}
diff --git a/crates/uzu/build/metal/mod.rs b/crates/uzu/build/metal/mod.rs
new file mode 100644
index 000000000..e4f31303d
--- /dev/null
+++ b/crates/uzu/build/metal/mod.rs
@@ -0,0 +1,74 @@
+use std::{env, fs, path::PathBuf};
+
+use anyhow::Context;
+use compiler::{MetalCompiler, ObjectInfo};
+use futures::{StreamExt, TryStreamExt, stream};
+use proc_macro2::TokenStream;
+use quote::quote;
+use walkdir::WalkDir;
+
+mod ast;
+mod bindgen;
+mod compiler;
+mod toolchain;
+
+pub async fn main() -> anyhow::Result<()> {
+ let dsl = MetalCompiler::new().context("cannot create metal dsl")?;
+
+ let src_dir = PathBuf::from(
+ env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?,
+ )
+ .join("src");
+
+ let metal_sources: Vec<PathBuf> = WalkDir::new(&src_dir)
+ .into_iter()
+ .filter_map(|e| e.ok())
+ .filter(|e| {
+ e.file_type().is_file()
+ && e.path().extension().and_then(|s| s.to_str())
+ == Some("metal")
+ })
+ .map(|e| e.into_path())
+ .collect();
+
+ let num_concurrent_compiles =
+ std::thread::available_parallelism().map(|x| x.get()).unwrap_or(4) * 2;
+
+ let results: Vec<(ObjectInfo, bool)> = stream::iter(metal_sources)
+ .map(|p| dsl.compile(p))
+ .buffer_unordered(num_concurrent_compiles)
+ .try_collect()
+ .await
+ .context("cannot compile metal sources")?;
+
+ let all_cached = results.iter().all(|(_, cached)| *cached);
+ let kernels: Vec<ObjectInfo> =
+ results.into_iter().map(|(o, _)| o).collect();
+
+ if !all_cached {
+ dsl.link(&kernels).await.context("cannot link objects")?;
+
+ let bindings = kernels
+ .iter()
+ .flat_map(|k| &k.kernels)
+ .map(|k| {
+ bindgen::bindgen(k).with_context(|| {
+ format!("cannot generate bindings for {}", k.name)
+ })
+ })
+ .collect::<anyhow::Result<Vec<TokenStream>>>()?;
+
+ let tokens = quote! { #(#bindings)* };
+
+ let out_path =
+ PathBuf::from(env::var("OUT_DIR").context("missing OUT_DIR")?)
+ .join("dsl.rs");
+ let parsed =
+ syn::parse2(tokens).context("cannot parse generated bindings")?;
+ fs::write(&out_path, prettyplease::unparse(&parsed)).with_context(
+ || format!("cannot write bindings file {}", out_path.display()),
+ )?;
+ }
+
+ Ok(())
+}
diff --git a/crates/uzu/build/metal/toolchain.rs b/crates/uzu/build/metal/toolchain.rs
new file mode 100644
index 000000000..dc28f26cb
--- /dev/null
+++ b/crates/uzu/build/metal/toolchain.rs
@@ -0,0 +1,322 @@
+use std::{
+ env::{self},
+ ffi::OsString,
+ fs,
+ path::Path,
+ process::Stdio,
+};
+
+use anyhow::{Context, bail};
+use tempfile::NamedTempFile;
+use tokio::{io::AsyncWriteExt, process::Command};
+
+use super::ast::{MetalAstKind, MetalAstNode, MetalKernelInfo};
+
+#[derive(Debug)]
+pub enum MetalSdk {
+ MacOSX,
+ MacCatalyst,
+ IPhoneOS,
+ IPhoneSimulator,
+}
+
+impl MetalSdk {
+ pub fn from_parts(
+ target: &str,
+ target_os: &str,
+ target_env: &str,
+ ) -> anyhow::Result<Self> {
+ if target_os == "ios" {
+ if target.contains("macabi") || target_env == "macabi" {
+ Ok(Self::MacCatalyst)
+ } else if target.contains("ios")
+ && (target.contains("86_64") || target_env == "sim")
+ {
+ Ok(Self::IPhoneSimulator)
+ } else {
+ Ok(Self::IPhoneOS)
+ }
+ } else if target_os == "macos" {
+ Ok(Self::MacOSX)
+ } else {
+ bail!(
+ "cannot find matching metal sdk for ({target:?}, {target_os:?}, {target_env:?})"
+ );
+ }
+ }
+
+ pub fn from_env() -> anyhow::Result<Self> {
+ let target = env::var("TARGET").context("missing TARGET")?;
+ let target_os = env::var("CARGO_CFG_TARGET_OS")
+ .context("missing CARGO_CFG_TARGET_OS")?;
+ let target_env = env::var("CARGO_CFG_TARGET_ENV")
+ .context("missing CARGO_CFG_TARGET_ENV")?;
+
+ let sdk = Self::from_parts(&target, &target_os, &target_env)?;
+
+ Ok(sdk)
+ }
+
+ pub fn to_str(&self) -> &'static str {
+ match self {
+ Self::MacOSX => "macosx",
+ Self::MacCatalyst => "maccatalyst",
+ Self::IPhoneOS => "iphoneos",
+ Self::IPhoneSimulator => "iphonesimulator",
+ }
+ }
+
+ pub fn os(&self) -> &'static str {
+ match self {
+ Self::MacOSX | Self::MacCatalyst => "macosx",
+ Self::IPhoneOS | Self::IPhoneSimulator => "ios",
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum MetalStd {
+ Metal4_0,
+}
+
+impl MetalStd {
+ pub fn to_str(&self) -> &'static str {
+ match self {
+ MetalStd::Metal4_0 => "metal4.0",
+ }
+ }
+
+ pub fn min_os(&self) -> &'static str {
+ match self {
+ MetalStd::Metal4_0 => "26.0",
+ }
+ }
+}
+
+impl Default for MetalStd {
+ fn default() -> Self {
+ Self::Metal4_0
+ }
+}
+
+#[derive(Debug)]
+pub struct MetalToolchain {
+ sdk: MetalSdk,
+ std: MetalStd,
+ opt_flags: Box<[OsString]>,
+ extra_options: Box<[OsString]>,
+}
+
+impl MetalToolchain {
+ pub fn from_env() -> anyhow::Result<Self> {
+ let sdk = MetalSdk::from_env().context("cannot get sdk")?;
+ let std = MetalStd::default();
+
+ let opt_flags = match env::var("OPT_LEVEL")
+ .context("missing OPT_LEVEL")?
+ .as_str()
+ {
+ "0" => {
+ [
+ OsString::from("-O1"), // matmul kernels compiled with -O0 are broken and require a reboot to unfreeze the os
+ OsString::from("-gline-tables-only"), // debug with line tables only
+ OsString::from("-frecord-sources"), // include source code
+ ]
+ .into()
+ },
+ "1" => [OsString::from("-O1")].into(),
+ _ => [OsString::from("-O2")].into(), // treat levels 2,3,s,z as O2 for metal
+ };
+
+ let extra_options = Box::new([
+ OsString::from(format!(
+ "-m{}-version-min={}",
+ sdk.os(),
+ std.min_os()
+ )),
+ // NOTE: Previous build.rs script didn't forward metal compiler warnings to cargo warnings, the new one does. This is temporary to avoid warning spam without modifying unrelated things in the same pull request as introducing the dsl and converting kernels
+ OsString::from("-Wno-sign-compare"),
+ OsString::from("-Wno-macro-redefined"),
+ OsString::from("-Wno-unused-variable"),
+ ]);
+
+ Ok(Self {
+ sdk,
+ std,
+ opt_flags,
+ extra_options,
+ })
+ }
+
+ fn xcrun(&self) -> Command {
+ let mut cmd = Command::new("xcrun");
+ cmd.args(["-sdk", self.sdk.to_str()]);
+ cmd
+ }
+
+ pub async fn analyze(
+ &self,
+ path: impl AsRef<Path>,
+ ) -> anyhow::Result<(
+ impl Iterator<Item = MetalKernelInfo>,
+ impl Iterator<Item = Box<str>>,
+ )> {
+ let path = path.as_ref();
+
+ let depfile_path =
+ NamedTempFile::new().context("cannot create temporary file")?;
+
+ let analyze_output = self
+ .xcrun()
+ .arg("metal")
+ .args(["-x", "metal"])
+ .arg(format!("-std={}", self.std.to_str()))
+ .args(self.extra_options.as_ref())
+ .arg(path)
+ .arg("-fsyntax-only")
+ .args(["-MMD", "-MF"])
+ .arg(depfile_path.path())
+ .args(["-Xclang", "-ast-dump=json"])
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .output()
+ .await
+ .context("cannot execute metal analyzer")?;
+
+ if !analyze_output.status.success() {
+ bail!(
+ "metal analyzer failed: {}",
+ String::from_utf8_lossy(&analyze_output.stderr)
+ );
+ }
+
+ let ast_root =
+ serde_json::from_slice::<MetalAstNode>(&analyze_output.stdout)
+ .context("cannot deserialize ast dump")?;
+
+ if !matches!(&ast_root.kind, MetalAstKind::TranslationUnitDecl) {
+ bail!(
+ "unexpected kind of ast root: MetalAstKind::TranslationUnitDecl expected, but {:?} found",
+ ast_root.kind
+ );
+ }
+
+ let source_contents =
+ fs::read_to_string(path).context("cannot read source file")?;
+
+ let kernel_infos = ast_root
+ .inner
+ .into_iter()
+ .filter_map(|node| {
+ MetalKernelInfo::from_ast_node_and_source(
+ node,
+ &source_contents,
+ )
+ .transpose()
+ })
+ .collect::<anyhow::Result<Vec<_>>>()
+ .context("cannot parse kernel infos from AST")?
+ .into_iter();
+
+ let depfile_contents = fs::read_to_string(depfile_path.path())
+ .context("cannot read depfile")?;
+
+ let dependencies = depfile::parse(&depfile_contents)
+ .map_err(|e| anyhow::anyhow!("cannot parse depfile: {e}"))?
+ .iter()
+ .flat_map(|(_, d)| d)
+ .map(|f| f.as_ref().into())
+ .collect::<Vec<_>>()
+ .into_iter();
+
+ Ok((kernel_infos, dependencies))
+ }
+
+ pub async fn compile(
+ &self,
+ source: impl AsRef<Path>,
+ footer: impl AsRef<str>,
+ output: impl AsRef<Path>,
+ ) -> anyhow::Result<Option<Box<str>>> {
+ let mut compile_child = self
+ .xcrun()
+ .arg("metal")
+ .arg("-c")
+ .args(["-x", "metal"])
+ .arg(format!("-std={}", self.std.to_str()))
+ .args(self.extra_options.as_ref())
+ .args(self.opt_flags.as_ref())
+ .arg("-include")
+ .arg(source.as_ref())
+ .arg("-")
+ .arg("-o")
+ .arg(output.as_ref())
+ .stdin(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .context("cannot execute metal compiler")?;
+
+ compile_child
+ .stdin
+ .as_mut()
+ .context("metal compiler stdin missing")?
+ .write_all(footer.as_ref().as_bytes())
+ .await
+ .context("cannot write to metal compiler stdin")?;
+
+ let compile_output = compile_child
+ .wait_with_output()
+ .await
+ .context("cannot wait on metal compiler")?;
+
+ let stderr = String::from_utf8_lossy(&compile_output.stderr)
+ .into_owned()
+ .into_boxed_str();
+
+ if !compile_output.status.success() {
+ bail!("metal compiler failed: {stderr}");
+ }
+
+ let warnings = if !stderr.is_empty() {
+ Some(stderr)
+ } else {
+ None
+ };
+
+ Ok(warnings)
+ }
+
+ pub async fn link(
+ &self,
+ objects: impl IntoIterator<Item = impl AsRef<Path>>,
+ output: impl AsRef<Path>,
+ ) -> anyhow::Result<Option<Box<str>>> {
+ let link_output = self
+ .xcrun()
+ .arg("metallib")
+ .args(
+ objects.into_iter().map(|p| p.as_ref().as_os_str().to_owned()),
+ )
+ .arg("-o")
+ .arg(output.as_ref())
+ .output()
+ .await
+ .context("cannot execute metal linker")?;
+
+ let stderr = String::from_utf8_lossy(&link_output.stderr)
+ .into_owned()
+ .into_boxed_str();
+
+ if !link_output.status.success() {
+ bail!("metal linker failed: {stderr}");
+ }
+
+ let warnings = if !stderr.is_empty() {
+ Some(stderr)
+ } else {
+ None
+ };
+
+ Ok(warnings)
+ }
+}
diff --git a/crates/uzu/build/shared_types/mod.rs b/crates/uzu/build/shared_types/mod.rs
new file mode 100644
index 000000000..da478d473
--- /dev/null
+++ b/crates/uzu/build/shared_types/mod.rs
@@ -0,0 +1,114 @@
+use std::{env, fs, path::PathBuf};
+
+use crate::debug_log;
+use anyhow::Context;
+use tokio::{task::spawn_blocking, try_join};
+
+struct BindgenConfig {
+ name: &'static str,
+ kernel_subdir: &'static str,
+ header_file: &'static str,
+ output_file: &'static str,
+ allowlist_types: &'static [&'static str],
+ raw_lines: &'static [&'static str],
+}
+
+const MATMUL_CONFIG: BindgenConfig = BindgenConfig {
+ name: "matmul",
+ kernel_subdir: "matmul/common",
+ header_file: "shared_types.h",
+ output_file: "shared_types.rs",
+ allowlist_types: &[
+ "GEMMParams",
+ "GEMMAddMMParams",
+ "GEMMSpiltKParams",
+ "GEMMSpiltKMlpFusedParams",
+ ],
+ raw_lines: &["#![allow(non_snake_case)]"],
+};
+
+const ATTENTION_CONFIG: BindgenConfig = BindgenConfig {
+ name: "attention",
+ kernel_subdir: "attention",
+ header_file: "gemm_types.h",
+ output_file: "gemm_types.rs",
+ allowlist_types: &["AttnParams", "AttnMaskParams"],
+ raw_lines: &[],
+};
+
+fn autogen_shared_types(config: &BindgenConfig) -> anyhow::Result<()> {
+ let out_dir = env::var("OUT_DIR").context("missing OUT_DIR")?;
+ let build_dir = PathBuf::from(out_dir).join("shared_types");
+ fs::create_dir_all(&build_dir).with_context(|| {
+ format!("cannot create shared types build dir {}", build_dir.display())
+ })?;
+
+ let manifest_dir = PathBuf::from(
+ env::var("CARGO_MANIFEST_DIR").context("missing CARGO_MANIFEST_DIR")?,
+ );
+ let kernel_dir = manifest_dir
+ .join(format!("src/backends/metal/kernel/{}", config.kernel_subdir));
+ let types_header = kernel_dir.join(config.header_file);
+ let types_header_hash: [u8; blake3::OUT_LEN] =
+ blake3::hash(&fs::read(types_header.as_path()).with_context(|| {
+ format!("cannot read {}", types_header.display())
+ })?)
+ .into();
+ let cached_types_header_hash =
+ build_dir.join(format!("{}.hash", config.header_file));
+
+ if let Some(cached_hash) = fs::read(&cached_types_header_hash).ok()
+ && cached_hash == types_header_hash
+ {
+ debug_log!("{} bindgen cached", config.name);
+ return Ok(());
+ }
+ debug_log!("{} bindgen started", config.name);
+
+ let mut builder = bindgen::Builder::default()
+ .header(types_header.to_string_lossy())
+ .clang_arg("-x")
+ .clang_arg("c")
+ .derive_default(true)
+ .derive_copy(true)
+ .use_core();
+
+ for raw_line in config.raw_lines {
+ builder = builder.raw_line(*raw_line);
+ }
+
+ for ty in config.allowlist_types {
+ builder = builder.allowlist_type(*ty);
+ }
+
+ let bindings = builder.generate().map_err(|e| {
+ anyhow::anyhow!(
+ "failed to generate bindings from {}: {e}",
+ config.header_file
+ )
+ })?;
+
+ let bindings_path = kernel_dir.join(config.output_file);
+ bindings.write_to_file(&bindings_path).with_context(|| {
+ format!("failed to write {}", bindings_path.display())
+ })?;
+
+ fs::write(&cached_types_header_hash, &types_header_hash).with_context(
+ || format!("failed to write {}", cached_types_header_hash.display()),
+ )?;
+ debug_log!("{} bindgen done", config.name);
+
+ Ok(())
+}
+
+pub async fn main() -> anyhow::Result<()> {
+ let (matmul_result, attn_result) = try_join!(
+ spawn_blocking(|| autogen_shared_types(&MATMUL_CONFIG)),
+ spawn_blocking(|| autogen_shared_types(&ATTENTION_CONFIG))
+ )?;
+
+ matmul_result?;
+ attn_result?;
+
+ Ok(())
+}
diff --git a/crates/uzu/src/backends/metal/context.rs b/crates/uzu/src/backends/metal/context.rs
index c5b7d08f0..061f1af22 100644
--- a/crates/uzu/src/backends/metal/context.rs
+++ b/crates/uzu/src/backends/metal/context.rs
@@ -1,5 +1,5 @@
-#![allow(dead_code)]
-include!(concat!(env!("OUT_DIR"), "/metal_lib.rs"));
+const MTLB: &[u8] =
+ include_bytes!(concat!(env!("OUT_DIR"), "/default.metallib"));
use std::{cell::RefCell, collections::HashMap, env, rc::Rc};
@@ -148,7 +148,7 @@ pub struct MTLContext {
pub device: MTLDevice,
pub command_queue: MTLCommandQueue,
pub architecture: DeviceArchitecture,
- library: MTLLibrary,
+ pub library: MTLLibrary,
pipeline_cache: RefCell<HashMap<String, MTLComputePipelineState>>,
}
@@ -157,7 +157,7 @@ impl MTLContext {
device: MTLDevice,
command_queue: MTLCommandQueue,
) -> Result<Self, MTLError> {
- let library = match device.new_library_with_data(METAL_LIBRARY_DATA) {
+ let library = match device.new_library_with_data(MTLB) {
Ok(lib) => lib,
Err(e) => {
return Err(MTLError::Generic(format!(
diff --git a/crates/uzu/src/backends/metal/kernel/definitions.metal b/crates/uzu/src/backends/metal/kernel/definitions.metal
index 78a880369..af5f31887 100644
--- a/crates/uzu/src/backends/metal/kernel/definitions.metal
+++ b/crates/uzu/src/backends/metal/kernel/definitions.metal
@@ -460,6 +460,21 @@ static T threadgroup_cooperative_reduce_min(
return result;
}
+// MARK: - DSL Annotation Helpers
+
+#define DSL_META(...) [[clang::annotate("", __VA_ARGS__)]]
+#define DSL_STR(X) #X
+#define DSL_XSTR(X) DSL_STR(X)
+
+#define SPECIALIZE(TYPENAME, ...) \
+ template <typename TYPENAME> \
+ DSL_META("dsl.specialize", #TYPENAME, #__VA_ARGS__)
+#define KERNEL(NAME) DSL_META("dsl.kernel") void NAME
+
+#define AXIS(TDS, TPG) DSL_META("dsl.axis", DSL_XSTR(TDS), DSL_XSTR(TPG))
+#define GROUPS(EXPR) DSL_META("dsl.groups", DSL_XSTR(EXPR))
+#define THREADS(EXPR) DSL_META("dsl.threads", DSL_XSTR(EXPR))
+
// MARK: - Generate Template Kernels
#define generateKernel(functionName, scalarType, outerArgs, innerArgs) \
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 3caf3ae9b..8a6e72f52 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,6 +1,9 @@
pub mod activation;
pub mod attention;
mod data_type;
+pub mod dsl {
+ include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
+}
pub mod embedding;
pub mod kv_cache_update;
pub mod layer_norm;
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
index 0d2e127e4..af6db77a3 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/argmax.metal
@@ -153,16 +153,57 @@ static ArgmaxPair threadgroup_raking_argmax(
return result;
}
-template <typename T>
-void batched_argmax_main_impl(
+// Single-pass argmax
+
+SPECIALIZE(T, float, half, bfloat) KERNEL(ArgmaxSingle)(
+ const device T* logits_data,
+ device uint* final_tokens,
+ constant uint& batch_size,
+ constant uint& vocab_size,
+ threadgroup ArgmaxPair shared[BLOCK_SIZE],
+ uint batch_idx GROUPS(batch_size),
+ ushort local_id THREADS(BLOCK_SIZE)
+) {
+ if (batch_idx >= batch_size)
+ return;
+
+ const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;
+ ArgmaxPair best = ARGMAX_INIT;
+
+ for (uint vocab_offset = 0; vocab_offset < vocab_size;
+ vocab_offset += elements_per_tile) {
+ ArgmaxPair buf[GRAIN_SIZE];
+ load_blocked_argmax_batched<GRAIN_SIZE>(
+ buf,
+ logits_data,
+ local_id,
+ batch_idx,
+ vocab_offset,
+ vocab_size
+ );
+ ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);
+ best = argmax_combine(best, tile_best);
+ }
+
+ ArgmaxPair group_best =
+ threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);
+
+ if (local_id == 0) {
+ final_tokens[batch_idx] = group_best.index;
+ }
+}
+
+// Two-pass argmax
+
+SPECIALIZE(T, float, half, bfloat) KERNEL(ArgmaxMain)(
const device T* logits_data,
device ArgmaxPair* partial_results,
constant uint& batch_size,
constant uint& vocab_size,
- threadgroup ArgmaxPair* shared,
- uint batch_idx,
- uint vocab_group_idx,
- ushort local_id
+ threadgroup ArgmaxPair shared[BLOCK_SIZE],
+ uint batch_idx GROUPS(batch_size),
+ uint vocab_group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),
+ ushort local_id THREADS(BLOCK_SIZE)
) {
if (batch_idx >= batch_size)
@@ -196,14 +237,14 @@ void batched_argmax_main_impl(
}
}
-void batched_argmax_final_impl(
+KERNEL(ArgmaxFinal)(
const device ArgmaxPair* partial_results,
device uint* final_tokens,
constant uint& batch_size,
constant uint& vocab_size,
- threadgroup ArgmaxPair* shared,
- uint batch_idx,
- ushort local_id
+ threadgroup ArgmaxPair shared[BLOCK_SIZE],
+ uint batch_idx GROUPS(batch_size),
+ ushort local_id THREADS(BLOCK_SIZE)
) {
if (batch_idx >= batch_size)
@@ -231,158 +272,3 @@ void batched_argmax_final_impl(
final_tokens[batch_idx] = result.index;
}
}
-
-// Generate argmax main kernels using macro approach
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device ArgmaxPair* partial_results [[buffer(1)]], \
- constant uint& batch_size [[buffer(2)]], \
- constant uint& vocab_size [[buffer(3)]], \
- uint2 group_position [[threadgroup_position_in_grid]], \
- uint2 local_position [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- partial_results, \
- batch_size, \
- vocab_size, \
- shared, \
- group_position.x, \
- group_position.y, \
- (ushort)local_position.x)
-
-#define generateArgmaxMainKernel(functionName, T, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##T outerArgs { \
- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
- batched_argmax_main_impl<T> innerArgs; \
- }
-
-generateArgmaxMainKernel(batched_argmax_main, float, outerArguments(float), innerArguments) generateArgmaxMainKernel(
- batched_argmax_main,
- half,
- outerArguments(half),
- innerArguments
-) generateArgmaxMainKernel(batched_argmax_main, bfloat, outerArguments(bfloat), innerArguments)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateArgmaxMainKernel
-
-// Generate argmax final kernels using macro approach - note: no T parameter
-// needed for final stage
-#define outerArguments(T) \
- (device const ArgmaxPair* partial_results [[buffer(0)]], \
- device uint* final_tokens [[buffer(1)]], \
- constant uint& batch_size [[buffer(2)]], \
- constant uint& vocab_size [[buffer(3)]], \
- uint batch_idx [[threadgroup_position_in_grid]], \
- uint local_id [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (partial_results, \
- final_tokens, \
- batch_size, \
- vocab_size, \
- shared, \
- batch_idx, \
- (ushort)local_id)
-
-#define generateArgmaxFinalKernel(functionName, T, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##T outerArgs { \
- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
- batched_argmax_final_impl innerArgs; \
- }
-
- generateArgmaxFinalKernel(batched_argmax_final, float, outerArguments(float), innerArguments) generateArgmaxFinalKernel(
- batched_argmax_final,
- half,
- outerArguments(half),
- innerArguments
- ) generateArgmaxFinalKernel(batched_argmax_final, bfloat, outerArguments(bfloat), innerArguments)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateArgmaxFinalKernel
-
- template <typename T>
- void batched_argmax_single_impl(
- const device T* logits_data,
- device uint* final_tokens,
- constant uint& batch_size,
- constant uint& vocab_size,
- threadgroup ArgmaxPair* shared,
- uint batch_idx,
- ushort local_id
- ) {
- if (batch_idx >= batch_size)
- return;
-
- const uint elements_per_tile = BLOCK_SIZE * GRAIN_SIZE;
- ArgmaxPair best = ARGMAX_INIT;
-
- for (uint vocab_offset = 0; vocab_offset < vocab_size;
- vocab_offset += elements_per_tile) {
- ArgmaxPair buf[GRAIN_SIZE];
- load_blocked_argmax_batched<GRAIN_SIZE>(
- buf,
- logits_data,
- local_id,
- batch_idx,
- vocab_offset,
- vocab_size
- );
- ArgmaxPair tile_best = thread_argmax_reduce<GRAIN_SIZE>(buf);
- best = argmax_combine(best, tile_best);
- }
-
- ArgmaxPair group_best =
- threadgroup_raking_argmax<BLOCK_SIZE>(best, shared, local_id);
-
- if (local_id == 0) {
- final_tokens[batch_idx] = group_best.index;
- }
-}
-
-// Generate argmax single pass kernels using macro approach
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device uint* final_tokens [[buffer(1)]], \
- constant uint& batch_size [[buffer(2)]], \
- constant uint& vocab_size [[buffer(3)]], \
- uint batch_idx [[threadgroup_position_in_grid]], \
- uint local_id [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- final_tokens, \
- batch_size, \
- vocab_size, \
- shared, \
- batch_idx, \
- (ushort)local_id)
-
-#define generateArgmaxSingleKernel(functionName, T, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##T outerArgs { \
- threadgroup ArgmaxPair shared[BLOCK_SIZE]; \
- batched_argmax_single_impl<T> innerArgs; \
- }
-
-generateArgmaxSingleKernel(batched_argmax_single, float, outerArguments(float), innerArguments) generateArgmaxSingleKernel(
- batched_argmax_single,
- half,
- outerArguments(half),
- innerArguments
-) generateArgmaxSingleKernel(batched_argmax_single, bfloat, outerArguments(bfloat), innerArguments)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateArgmaxSingleKernel
-
-#undef BLOCK_SIZE
-#undef GRAIN_SIZE
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
index 8165d45f4..b4ee1d1a1 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/bitmask.metal
@@ -1,21 +1,22 @@
#include <metal_stdlib>
#include "../definitions.metal"
-#define GRAIN_SIZE 64
#define BLOCK_SIZE 1024
-
+#define GRAIN_SIZE 64
#define BITS_IN_U32 32
-template <typename T>
-void batched_bitmask(
- device const T* logits_data,
- device const uint32_t* bitmask_data,
+SPECIALIZE(T, float, half, bfloat) KERNEL(Bitmask) (
+ device const T* logits,
+ device const uint32_t* bitmask,
+ constant uint& bitmask_offset,
device T* processed_logits,
+ constant uint& batch_size,
constant uint& vocab_size,
- uint group_idx,
- uint batch_idx,
- uint thread_idx
+ uint group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),
+ uint batch_idx GROUPS(batch_size),
+ uint thread_idx THREADS(BLOCK_SIZE)
) {
+ bitmask += bitmask_offset / sizeof(uint32_t);
uint bitmask_size = (vocab_size + (BITS_IN_U32 - 1)) / BITS_IN_U32;
uint base_idx =
batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
@@ -27,61 +28,9 @@ void batched_bitmask(
if (global_idx < batch_end) {
uint token_idx = global_idx - batch_idx * vocab_size;
uint bitmask_idx = batch_idx * bitmask_size + (token_idx / BITS_IN_U32);
- bool mask =
- (bitmask_data[bitmask_idx] >> (token_idx % BITS_IN_U32)) & 0b1;
+ bool mask = (bitmask[bitmask_idx] >> (token_idx % BITS_IN_U32)) & 0b1;
processed_logits[global_idx] =
- select(T(-INFINITY), logits_data[global_idx], mask);
+ select(T(-INFINITY), logits[global_idx], mask);
}
}
}
-
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device const uint32_t* bitmask_data [[buffer(1)]], \
- device T* processed_logits [[buffer(2)]], \
- constant uint& vocab_size [[buffer(3)]], \
- uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
- uint3 thread_idx [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- bitmask_data, \
- processed_logits, \
- vocab_size, \
- threadgroup_idx.x, \
- threadgroup_idx.y, \
- thread_idx.x)
-
-#define generateBitmaskKernel(functionName, scalarType, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
- }
-
-#define generateBitmaskKernels(functionName) \
- generateBitmaskKernel( \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateBitmaskKernel( \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateBitmaskKernel( \
- functionName, \
- half, \
- outerArguments(half), \
- innerArguments \
- );
-
-generateBitmaskKernels(batched_bitmask)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateBitmaskKernel
-#undef generateBitmaskKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
index b51b9bb94..4ef098927 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/gumbel.metal
@@ -2,20 +2,20 @@
#include "../definitions.metal"
#include "../rng.metal"
-#define GRAIN_SIZE 64
#define BLOCK_SIZE 1024
-
+#define GRAIN_SIZE 64
#define WORDS_PER_OFFSET 4
-template <typename T>
-void batched_gumbel(
- device const T* logits_data,
+SPECIALIZE(T, float, half, bfloat) KERNEL(Gumbel) (
+ device const T* logits,
device const uint64_t* batch_seeds,
device T* processed_logits,
+ constant uint& batch_size,
constant uint& vocab_size,
- uint group_idx,
- uint batch_idx,
- uint thread_idx
+ constant uint& seeds_offset,
+ uint group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),
+ uint batch_idx GROUPS(batch_size),
+ uint thread_idx THREADS(BLOCK_SIZE)
) {
uint batch_start = vocab_size * batch_idx;
uint batch_end = batch_start + vocab_size;
@@ -24,7 +24,7 @@ void batched_gumbel(
uint grain_offset = batch_start + grain_offset_in_batch;
- uint64_t rng_seed = batch_seeds[batch_idx];
+ uint64_t rng_seed = batch_seeds[seeds_offset + batch_idx];
uint64_t rng_offset = (group_idx * BLOCK_SIZE + thread_idx) *
(GRAIN_SIZE + WORDS_PER_OFFSET - 1) / WORDS_PER_OFFSET;
PhiloxState rng;
@@ -35,59 +35,7 @@ void batched_gumbel(
uint global_idx = grain_offset + i * BLOCK_SIZE;
if (global_idx < batch_end) {
processed_logits[global_idx] =
- logits_data[global_idx] +
- T(-fast::log(-fast::log(uniform_float(&rng))));
+ logits[global_idx] + T(-fast::log(-fast::log(uniform_float(&rng))));
}
}
}
-
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device const uint64_t* batch_seeds [[buffer(1)]], \
- device T* processed_logits [[buffer(2)]], \
- constant uint& vocab_size [[buffer(3)]], \
- uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
- uint3 thread_idx [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- batch_seeds, \
- processed_logits, \
- vocab_size, \
- threadgroup_idx.x, \
- threadgroup_idx.y, \
- thread_idx.x)
-
-#define generateGumbelKernel(functionName, scalarType, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
- }
-
-#define generateGumbelKernels(functionName) \
- generateGumbelKernel( \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateGumbelKernel( \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateGumbelKernel( \
- functionName, \
- half, \
- outerArguments(half), \
- innerArguments \
- );
-
-generateGumbelKernels(batched_gumbel)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateGumbelKernel
-#undef generateGumbelKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
index a5fb3797d..6c845a38b 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/min_p.metal
@@ -4,15 +4,15 @@
#define BLOCK_SIZE 1024
#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
-template <typename T>
-void batched_minp(
- device const T* logits_data,
+SPECIALIZE(T, float, half, bfloat) KERNEL(MinP) (
+ device const T* logits,
device T* processed_logits,
threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
+ constant uint& batch_size,
constant uint& vocab_size,
constant float& min_p,
- uint batch_idx,
- uint thread_idx
+ uint batch_idx GROUPS(batch_size),
+ uint thread_idx THREADS(BLOCK_SIZE)
) {
uint batch_start = batch_idx * vocab_size;
@@ -20,7 +20,7 @@ void batched_minp(
float local_max = -INFINITY;
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
+ float logit_value = float(logits[batch_start + i]);
local_max = fmax(local_max, logit_value);
}
float max_logit = threadgroup_cooperative_reduce_max<BLOCK_SIZE>(
@@ -34,55 +34,8 @@ void batched_minp(
T t_threshold = T(max_logit + log(min_p));
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- T logit_value = logits_data[batch_start + i];
+ T logit_value = logits[batch_start + i];
processed_logits[batch_start + i] =
select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
-
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device T* processed_logits [[buffer(1)]], \
- constant uint& vocab_size [[buffer(2)]], \
- constant float& min_p [[buffer(3)]], \
- uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
- uint3 thread_idx [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- processed_logits, \
- shared_reduce_buffer, \
- vocab_size, \
- min_p, \
- threadgroup_idx.x, \
- thread_idx.x)
-
-#define generateMinpKernel(functionName, scalarType, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
- functionName innerArgs; \
- }
-
-#define generateMinpKernels(functionName) \
- generateMinpKernel( \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateMinpKernel( \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateMinpKernel(functionName, half, outerArguments(half), innerArguments);
-
-generateMinpKernels(batched_minp)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateMinpKernel
-#undef generateMinpKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
index 8014fc1f0..aaa944f97 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/sampling/mod.rs
@@ -2,25 +2,27 @@ use std::mem::size_of;
use metal::{
Buffer as MTLBuffer, CommandBuffer as MTLCommandBuffer,
- ComputeCommandEncoderRef, ComputePipelineState as MTLComputePipelineState,
- MTLResourceOptions, MTLSize,
+ ComputeCommandEncoderRef, MTLResourceOptions,
};
use thiserror::Error;
use crate::{
DataType,
- backends::metal::{KernelDataType, MTLContext, MTLError},
+ backends::metal::{
+ KernelDataType, MTLContext, MTLError,
+ kernel::dsl::{
+ ArgmaxFinalKernel, ArgmaxMainKernel, ArgmaxSingleKernel,
+ BitmaskKernel, GumbelKernel, MinPKernel, TemperatureKernel,
+ TopKKernel, TopPKernel,
+ },
+ },
session::parameter::SamplingMethod,
};
-const BLOCK_SIZE: usize = 1024;
-const ELEMENTWISE_GRAIN_SIZE: usize = 64;
-const TWOPASS_ARGMAX_GRAIN_SIZE: usize = 4;
-
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ArgmaxStrategy {
SinglePass, // One threadgroup per batch item
- TwoPass, // Multi-stage reduction
+ TwoPass, // Two-stage reduction
}
#[repr(C)]
@@ -41,27 +43,27 @@ impl Default for ArgmaxPair {
enum ArgmaxImplementation {
SinglePass {
- pipeline: MTLComputePipelineState,
+ kernel: ArgmaxSingleKernel,
},
TwoPass {
- main_pipeline: MTLComputePipelineState,
- final_pipeline: MTLComputePipelineState,
+ main_kernel: ArgmaxMainKernel,
+ final_kernel: ArgmaxFinalKernel,
partial_results_buffer: MTLBuffer,
},
}
pub struct SamplingKernel {
- bitmask_pipeline: MTLComputePipelineState,
+ bitmask: BitmaskKernel,
bitmask_applied_logits: MTLBuffer,
- temperature_pipeline: MTLComputePipelineState,
+ temperature: TemperatureKernel,
temperature_applied_logits: MTLBuffer,
- topk_pipeline: MTLComputePipelineState,
+ topk: TopKKernel,
topk_applied_logits: MTLBuffer,
- topp_pipeline: MTLComputePipelineState,
+ topp: TopPKernel,
topp_applied_logits: MTLBuffer,
- minp_pipeline: MTLComputePipelineState,
+ minp: MinPKernel,
minp_applied_logits: MTLBuffer,
- gumbel_pipeline: MTLComputePipelineState,
+ gumbel: GumbelKernel,
gumbel_applied_logits: MTLBuffer,
argmax_implementation: ArgmaxImplementation,
max_batch_size: usize,
@@ -105,16 +107,9 @@ impl SamplingKernel {
max_vocab_size: usize,
argmax_strategy: ArgmaxStrategy,
) -> Result<Self, SamplingError> {
- let data_suffix = data_type.function_name_suffix();
let max_elements = max_batch_size * max_vocab_size;
- let bitmask_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_bitmask_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
+ let bitmask = BitmaskKernel::new(context, data_type)?;
let bitmask_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
@@ -122,13 +117,7 @@ impl SamplingKernel {
MTLResourceOptions::StorageModeShared,
);
- let temperature_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_temperature_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
+ let temperature = TemperatureKernel::new(context, data_type)?;
let temperature_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
@@ -136,13 +125,7 @@ impl SamplingKernel {
MTLResourceOptions::StorageModeShared,
);
- let topk_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_topk_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
+ let topk = TopKKernel::new(context, data_type)?;
let topk_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
@@ -150,13 +133,7 @@ impl SamplingKernel {
MTLResourceOptions::StorageModeShared,
);
- let topp_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_topp_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
+ let topp = TopPKernel::new(context, data_type)?;
let topp_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
@@ -164,13 +141,7 @@ impl SamplingKernel {
MTLResourceOptions::StorageModeShared,
);
- let minp_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_minp_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
+ let minp = MinPKernel::new(context, data_type)?;
let minp_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
@@ -178,13 +149,7 @@ impl SamplingKernel {
MTLResourceOptions::StorageModeShared,
);
- let gumbel_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_gumbel_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
+ let gumbel = GumbelKernel::new(context, data_type)?;
let gumbel_applied_logits = context.device.new_buffer(
(max_elements * Into::<DataType>::into(data_type).size_in_bytes())
@@ -194,36 +159,20 @@ impl SamplingKernel {
let argmax_implementation = match argmax_strategy {
ArgmaxStrategy::SinglePass => {
- let pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_argmax_single_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
+ let kernel = ArgmaxSingleKernel::new(context, data_type)?;
ArgmaxImplementation::SinglePass {
- pipeline,
+ kernel,
}
},
ArgmaxStrategy::TwoPass => {
- let main_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_argmax_main_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
-
- let final_pipeline = context
- .compute_pipeline_state_with_reflection(
- &format!("batched_argmax_final_{}", data_suffix),
- None,
- )
- .map(|(pipeline, _)| pipeline)
- .map_err(SamplingError::MetalError)?;
-
- let elements_per_group = BLOCK_SIZE * TWOPASS_ARGMAX_GRAIN_SIZE;
+ let main_kernel = ArgmaxMainKernel::new(context, data_type)?;
+ let final_kernel = ArgmaxFinalKernel::new(context)?;
+
+ let block_size = 1024;
+ let grain_size = 4;
+
+ let elements_per_group = block_size * grain_size;
let max_vocab_groups_per_batch =
(max_vocab_size + elements_per_group - 1)
/ elements_per_group;
@@ -236,25 +185,25 @@ impl SamplingKernel {
);
ArgmaxImplementation::TwoPass {
- main_pipeline,
- final_pipeline,
+ main_kernel,
+ final_kernel,
partial_results_buffer,
}
},
};
Ok(Self {
- bitmask_pipeline,
+ bitmask,
bitmask_applied_logits,
- temperature_pipeline,
+ temperature,
temperature_applied_logits,
- topk_pipeline,
+ topk,
topk_applied_logits,
- topp_pipeline,
+ topp,
topp_applied_logits,
- minp_pipeline,
+ minp,
minp_applied_logits,
- gumbel_pipeline,
+ gumbel,
gumbel_applied_logits,
argmax_implementation,
max_batch_size,
@@ -321,15 +270,15 @@ impl SamplingKernel {
let mut last_logits_buffer = logits_buffer;
if let Some(bitmask_buffer) = bitmask_buffer {
- self.encode_bitmask(
+ self.bitmask.encode(
last_logits_buffer,
bitmask_buffer,
- bitmask_offset,
+ bitmask_offset as u32,
&self.bitmask_applied_logits,
batch_size as u32,
vocab_size as u32,
compute_encoder,
- )?;
+ );
last_logits_buffer = &self.bitmask_applied_logits;
}
@@ -341,386 +290,98 @@ impl SamplingKernel {
} = sampling_method
{
if let Some(temperature) = temperature {
- self.encode_temperature(
+ self.temperature.encode(
last_logits_buffer,
&self.temperature_applied_logits,
batch_size as u32,
vocab_size as u32,
temperature,
compute_encoder,
- )?;
+ );
last_logits_buffer = &self.temperature_applied_logits;
}
if let Some(top_k) = top_k {
- self.encode_topk(
+ self.topk.encode(
last_logits_buffer,
&self.topk_applied_logits,
batch_size as u32,
vocab_size as u32,
top_k,
compute_encoder,
- )?;
+ );
last_logits_buffer = &self.topk_applied_logits;
}
if let Some(top_p) = top_p {
- self.encode_topp(
+ self.topp.encode(
last_logits_buffer,
&self.topp_applied_logits,
batch_size as u32,
vocab_size as u32,
top_p,
compute_encoder,
- )?;
+ );
last_logits_buffer = &self.topp_applied_logits;
}
if let Some(min_p) = min_p {
- self.encode_minp(
+ self.minp.encode(
last_logits_buffer,
&self.minp_applied_logits,
batch_size as u32,
vocab_size as u32,
min_p,
compute_encoder,
- )?;
+ );
last_logits_buffer = &self.minp_applied_logits;
}
- self.encode_gumbel(
+ self.gumbel.encode(
last_logits_buffer,
seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?,
- seeds_offset,
&self.gumbel_applied_logits,
batch_size as u32,
vocab_size as u32,
+ (seeds_offset / size_of::<u64>()) as u32,
compute_encoder,
- )?;
+ );
last_logits_buffer = &self.gumbel_applied_logits;
}
match &self.argmax_implementation {
ArgmaxImplementation::SinglePass {
- pipeline,
- } => self.encode_argmax_single_pass(
- pipeline,
- last_logits_buffer,
- sampled_tokens_buffer,
- batch_size,
- vocab_size,
- compute_encoder,
- ),
+ kernel,
+ } => {
+ kernel.encode(
+ last_logits_buffer,
+ sampled_tokens_buffer,
+ batch_size as u32,
+ vocab_size as u32,
+ compute_encoder,
+ );
+ },
ArgmaxImplementation::TwoPass {
- main_pipeline,
- final_pipeline,
+ main_kernel,
+ final_kernel,
partial_results_buffer,
- } => self.encode_argmax_two_pass(
- main_pipeline,
- final_pipeline,
- last_logits_buffer,
- partial_results_buffer,
- sampled_tokens_buffer,
- batch_size,
- vocab_size,
- compute_encoder,
- ),
+ } => {
+ main_kernel.encode(
+ last_logits_buffer,
+ partial_results_buffer,
+ batch_size as u32,
+ vocab_size as u32,
+ compute_encoder,
+ );
+ final_kernel.encode(
+ partial_results_buffer,
+ sampled_tokens_buffer,
+ batch_size as u32,
+ vocab_size as u32,
+ compute_encoder,
+ );
+ },
}
- }
-
- pub fn encode_bitmask(
- &self,
- logits_buffer: &MTLBuffer,
- bitmask_buffer: &MTLBuffer,
- bitmask_offset: usize,
- processed_logits_buffer: &MTLBuffer,
- batch_size: u32,
- vocab_size: u32,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(&self.bitmask_pipeline);
-
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(
- 1,
- Some(bitmask_buffer),
- bitmask_offset as u64,
- );
- compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
- compute_encoder.set_bytes(
- 3,
- size_of::<u32>() as u64,
- &vocab_size as *const u32 as *const std::ffi::c_void,
- );
-
- let elements_in_group = BLOCK_SIZE * ELEMENTWISE_GRAIN_SIZE;
- let groups = (vocab_size + (elements_in_group as u32 - 1))
- / elements_in_group as u32;
-
- compute_encoder.dispatch_thread_groups(
- MTLSize::new(groups as u64, batch_size as u64, 1),
- MTLSize::new(BLOCK_SIZE as u64, 1, 1),
- );
-
- Ok(())
- }
-
- pub fn encode_temperature(
- &self,
- logits_buffer: &MTLBuffer,
- processed_logits_buffer: &MTLBuffer,
- batch_size: u32,
- vocab_size: u32,
- temperature: f32,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(&self.temperature_pipeline);
-
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(processed_logits_buffer), 0);
- compute_encoder.set_bytes(
- 2,
- size_of::<u32>() as u64,
- &vocab_size as *const u32 as *const std::ffi::c_void,
- );
- compute_encoder.set_bytes(
- 3,
- size_of::<f32>() as u64,
- &temperature as *const f32 as *const std::ffi::c_void,
- );
-
- let elements_in_group = BLOCK_SIZE * ELEMENTWISE_GRAIN_SIZE;
- let groups = (vocab_size + (elements_in_group as u32 - 1))
- / elements_in_group as u32;
-
- compute_encoder.dispatch_thread_groups(
- MTLSize::new(groups as u64, batch_size as u64, 1),
- MTLSize::new(BLOCK_SIZE as u64, 1, 1),
- );
-
- Ok(())
- }
-
- pub fn encode_topk(
- &self,
- logits_buffer: &MTLBuffer,
- processed_logits_buffer: &MTLBuffer,
- batch_size: u32,
- vocab_size: u32,
- top_k: u32,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(&self.topk_pipeline);
-
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(processed_logits_buffer), 0);
- compute_encoder.set_bytes(
- 2,
- size_of::<u32>() as u64,
- &vocab_size as *const u32 as *const std::ffi::c_void,
- );
- compute_encoder.set_bytes(
- 3,
- size_of::<u32>() as u64,
- &top_k as *const u32 as *const std::ffi::c_void,
- );
-
- compute_encoder.dispatch_thread_groups(
- MTLSize::new(batch_size as u64, 1, 1),
- MTLSize::new(BLOCK_SIZE as u64, 1, 1),
- );
-
- Ok(())
- }
-
- pub fn encode_topp(
- &self,
- logits_buffer: &MTLBuffer,
- processed_logits_buffer: &MTLBuffer,
- batch_size: u32,
- vocab_size: u32,
- top_p: f32,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(&self.topp_pipeline);
-
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(processed_logits_buffer), 0);
- compute_encoder.set_bytes(
- 2,
- size_of::<u32>() as u64,
- &vocab_size as *const u32 as *const std::ffi::c_void,
- );
- compute_encoder.set_bytes(
- 3,
- size_of::<f32>() as u64,
- &top_p as *const f32 as *const std::ffi::c_void,
- );
-
- compute_encoder.dispatch_thread_groups(
- MTLSize::new(batch_size as u64, 1, 1),
- MTLSize::new(BLOCK_SIZE as u64, 1, 1),
- );
-
- Ok(())
- }
-
- pub fn encode_minp(
- &self,
- logits_buffer: &MTLBuffer,
- processed_logits_buffer: &MTLBuffer,
- batch_size: u32,
- vocab_size: u32,
- min_p: f32,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(&self.minp_pipeline);
-
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(processed_logits_buffer), 0);
- compute_encoder.set_bytes(
- 2,
- size_of::<u32>() as u64,
- &vocab_size as *const u32 as *const std::ffi::c_void,
- );
- compute_encoder.set_bytes(
- 3,
- size_of::<f32>() as u64,
- &min_p as *const f32 as *const std::ffi::c_void,
- );
-
- compute_encoder.dispatch_thread_groups(
- MTLSize::new(batch_size as u64, 1, 1),
- MTLSize::new(BLOCK_SIZE as u64, 1, 1),
- );
-
- Ok(())
- }
-
- pub fn encode_gumbel(
- &self,
- logits_buffer: &MTLBuffer,
- seeds_buffer: &MTLBuffer,
- seeds_offset: usize,
- processed_logits_buffer: &MTLBuffer,
- batch_size: u32,
- vocab_size: u32,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(&self.gumbel_pipeline);
-
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(seeds_buffer), seeds_offset as u64);
- compute_encoder.set_buffer(2, Some(processed_logits_buffer), 0);
- compute_encoder.set_bytes(
- 3,
- size_of::<u32>() as u64,
- &vocab_size as *const u32 as *const std::ffi::c_void,
- );
-
- let elements_in_group = BLOCK_SIZE * ELEMENTWISE_GRAIN_SIZE;
- let groups = (vocab_size + (elements_in_group as u32 - 1))
- / elements_in_group as u32;
-
- compute_encoder.dispatch_thread_groups(
- MTLSize::new(groups as u64, batch_size as u64, 1),
- MTLSize::new(BLOCK_SIZE as u64, 1, 1),
- );
-
- Ok(())
- }
-
- pub fn encode_argmax_single_pass(
- &self,
- pipeline: &MTLComputePipelineState,
- logits_buffer: &MTLBuffer,
- sampled_tokens_buffer: &MTLBuffer,
- batch_size: usize,
- vocab_size: usize,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- compute_encoder.set_compute_pipeline_state(pipeline);
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(sampled_tokens_buffer), 0);
- compute_encoder.set_bytes(
- 2,
- size_of::<u32>() as u64,
- &(batch_size as u32) as *const u32 as *const std::ffi::c_void,
- );
- compute_encoder.set_bytes(
- 3,
- size_of::<u32>() as u64,
- &(vocab_size as u32) as *const u32 as *const std::ffi::c_void,
- );
-
- let threadgroups_per_grid = MTLSize::new(batch_size as u64, 1, 1);
- let threads_per_threadgroup = MTLSize::new(BLOCK_SIZE as u64, 1, 1);
- compute_encoder.dispatch_thread_groups(
- threadgroups_per_grid,
- threads_per_threadgroup,
- );
-
- Ok(())
- }
-
- pub fn encode_argmax_two_pass(
- &self,
- main_pipeline: &MTLComputePipelineState,
- final_pipeline: &MTLComputePipelineState,
- logits_buffer: &MTLBuffer,
- partial_results_buffer: &MTLBuffer,
- sampled_tokens_buffer: &MTLBuffer,
- batch_size: usize,
- vocab_size: usize,
- compute_encoder: &ComputeCommandEncoderRef,
- ) -> Result<(), SamplingError> {
- let elements_per_group = BLOCK_SIZE * TWOPASS_ARGMAX_GRAIN_SIZE;
- let vocab_groups_per_batch =
- (vocab_size + elements_per_group - 1) / elements_per_group;
-
- // Main pass
- compute_encoder.set_compute_pipeline_state(main_pipeline);
- compute_encoder.set_buffer(0, Some(logits_buffer), 0);
- compute_encoder.set_buffer(1, Some(partial_results_buffer), 0);
- compute_encoder.set_bytes(
- 2,
- size_of::<u32>() as u64,
- &(batch_size as u32) as *const u32 as *const std::ffi::c_void,
- );
- compute_encoder.set_bytes(
- 3,
- size_of::<u32>() as u64,
- &(vocab_size as u32) as *const u32 as *const std::ffi::c_void,
- );
-
- let threadgroups_per_grid =
- MTLSize::new(batch_size as u64, vocab_groups_per_batch as u64, 1);
- let threads_per_threadgroup = MTLSize::new(BLOCK_SIZE as u64, 1, 1);
- compute_encoder.dispatch_thread_groups(
- threadgroups_per_grid,
- threads_per_threadgroup,
- );
-
- // Final pass - two-pass argmax always has exactly 2 stages
- compute_encoder.set_compute_pipeline_state(final_pipeline);
- compute_encoder.set_buffer(0, Some(partial_results_buffer), 0);
- compute_encoder.set_buffer(1, Some(sampled_tokens_buffer), 0);
- compute_encoder.set_bytes(
- 2,
- size_of::<u32>() as u64,
- &(batch_size as u32) as *const u32 as *const std::ffi::c_void,
- );
- compute_encoder.set_bytes(
- 3,
- size_of::<u32>() as u64,
- &(vocab_size as u32) as *const u32 as *const std::ffi::c_void,
- );
-
- let final_threadgroups = MTLSize::new(batch_size as u64, 1, 1);
- compute_encoder.dispatch_thread_groups(
- final_threadgroups,
- threads_per_threadgroup,
- );
Ok(())
}
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
index 027813031..a462c4655 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/temperature.metal
@@ -1,18 +1,18 @@
#include <metal_stdlib>
#include "../definitions.metal"
-#define GRAIN_SIZE 64
#define BLOCK_SIZE 1024
+#define GRAIN_SIZE 64
-template <typename T>
-void batched_temperature(
- device const T* logits_data,
+SPECIALIZE(T, float, half, bfloat) KERNEL(Temperature) (
+ device const T* logits,
device T* processed_logits,
+ constant uint& batch_size,
constant uint& vocab_size,
constant float& temperature,
- uint group_idx,
- uint batch_idx,
- uint thread_idx
+ uint group_idx GROUPS(vocab_size.div_ceil(BLOCK_SIZE * GRAIN_SIZE)),
+ uint batch_idx GROUPS(batch_size),
+ uint thread_idx THREADS(BLOCK_SIZE)
) {
uint base_idx =
batch_idx * vocab_size + group_idx * BLOCK_SIZE * GRAIN_SIZE + thread_idx;
@@ -22,64 +22,7 @@ void batched_temperature(
for (uint i = 0; i < GRAIN_SIZE; i++) {
uint global_idx = base_idx + i * BLOCK_SIZE;
if (global_idx < batch_end) {
- processed_logits[global_idx] =
- T(float(logits_data[global_idx]) / temperature);
+ processed_logits[global_idx] = T(float(logits[global_idx]) / temperature);
}
}
}
-
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device T* processed_logits [[buffer(1)]], \
- constant uint& vocab_size [[buffer(2)]], \
- constant float& temperature [[buffer(3)]], \
- uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
- uint3 thread_idx [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- processed_logits, \
- vocab_size, \
- temperature, \
- threadgroup_idx.x, \
- threadgroup_idx.y, \
- thread_idx.x)
-
-#define generateTemperatureKernel( \
- functionName, \
- scalarType, \
- outerArgs, \
- innerArgs \
-) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##scalarType outerArgs { \
- functionName innerArgs; \
- }
-
-#define generateTemperatureKernels(functionName) \
- generateTemperatureKernel( \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateTemperatureKernel( \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateTemperatureKernel( \
- functionName, \
- half, \
- outerArguments(half), \
- innerArguments \
- );
-
-generateTemperatureKernels(batched_temperature)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateTemperatureKernel
-#undef generateTemperatureKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
index dff9f782c..9b5c856af 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_k.metal
@@ -5,15 +5,15 @@
#define BLOCK_SIZE_IN_SIMDS (BLOCK_SIZE / 32)
#define MAX_ITERS 16
-template <typename T>
-void batched_topk(
- device const T* logits_data,
+SPECIALIZE(T, float, half, bfloat) KERNEL(TopK) (
+ device const T* logits,
device T* processed_logits,
threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
+ constant uint& batch_size,
constant uint& vocab_size,
constant uint& top_k,
- uint batch_idx,
- uint thread_idx
+ uint batch_idx GROUPS(batch_size),
+ uint thread_idx THREADS(BLOCK_SIZE)
) {
uint batch_start = batch_idx * vocab_size;
@@ -22,7 +22,7 @@ void batched_topk(
float local_min = INFINITY;
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
+ float logit_value = float(logits[batch_start + i]);
local_max = fmax(local_max, logit_value);
local_min = select(
local_min,
@@ -50,7 +50,7 @@ void batched_topk(
uint local_num_above_threshold = 0;
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
+ float logit_value = float(logits[batch_start + i]);
local_num_above_threshold += select(0, 1, logit_value >= threshold);
}
uint num_above_threshold = threadgroup_cooperative_reduce_sum<BLOCK_SIZE>(
@@ -72,58 +72,11 @@ void batched_topk(
T t_threshold = T(threshold);
-// We know the threshold, just mask everything below it
+ // We know the threshold, just mask everything below it
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- T logit_value = logits_data[batch_start + i];
+ T logit_value = logits[batch_start + i];
processed_logits[batch_start + i] =
select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
-
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device T* processed_logits [[buffer(1)]], \
- constant uint& vocab_size [[buffer(2)]], \
- constant uint& top_k [[buffer(3)]], \
- uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
- uint3 thread_idx [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- processed_logits, \
- shared_reduce_buffer, \
- vocab_size, \
- top_k, \
- threadgroup_idx.x, \
- thread_idx.x)
-
-#define generateTopkKernel(functionName, scalarType, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
- functionName innerArgs; \
- }
-
-#define generateTopkKernels(functionName) \
- generateTopkKernel( \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateTopkKernel( \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateTopkKernel(functionName, half, outerArguments(half), innerArguments);
-
-generateTopkKernels(batched_topk)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateTopkKernel
-#undef generateTopkKernels
diff --git a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
index f16125107..cb0b8654b 100644
--- a/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
+++ b/crates/uzu/src/backends/metal/kernel/sampling/top_p.metal
@@ -6,15 +6,15 @@
#define MAX_ITERS 16
-template <typename T>
-void batched_topp(
- device const T* logits_data,
+SPECIALIZE(T, float, half, bfloat) KERNEL(TopP) (
+ device const T* logits,
device T* processed_logits,
threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS],
+ constant uint& batch_size,
constant uint& vocab_size,
constant float& top_p,
- uint batch_idx,
- uint thread_idx
+ uint batch_idx GROUPS(batch_size),
+ uint thread_idx THREADS(BLOCK_SIZE)
) {
uint batch_start = batch_idx * vocab_size;
@@ -23,7 +23,7 @@ void batched_topp(
float local_min = INFINITY;
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
+ float logit_value = float(logits[batch_start + i]);
local_max = fmax(local_max, logit_value);
local_min = select(
local_min,
@@ -46,7 +46,7 @@ void batched_topp(
float local_sum = 0.0f;
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
+ float logit_value = float(logits[batch_start + i]);
local_sum += select(
0.0,
fast::exp(logit_value - max_logit),
@@ -72,7 +72,7 @@ void batched_topp(
float local_min_above_threshold = INFINITY;
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- float logit_value = float(logits_data[batch_start + i]);
+ float logit_value = float(logits[batch_start + i]);
float logit_mass = fast::exp(logit_value - max_logit);
local_sum_above_threshold +=
select(0.0, logit_mass, logit_value >= threshold);
@@ -108,58 +108,11 @@ void batched_topp(
T t_threshold = T(threshold);
-// We know the threshold, just mask everything below it
+ // We know the threshold, just mask everything below it
#pragma unroll(4)
for (uint i = thread_idx; i < vocab_size; i += BLOCK_SIZE) {
- T logit_value = logits_data[batch_start + i];
+ T logit_value = logits[batch_start + i];
processed_logits[batch_start + i] =
select(T(-INFINITY), logit_value, logit_value >= t_threshold);
}
}
-
-#define outerArguments(T) \
- (device const T* logits_data [[buffer(0)]], \
- device T* processed_logits [[buffer(1)]], \
- constant uint& vocab_size [[buffer(2)]], \
- constant float& top_p [[buffer(3)]], \
- uint3 threadgroup_idx [[threadgroup_position_in_grid]], \
- uint3 thread_idx [[thread_position_in_threadgroup]])
-
-#define innerArguments \
- (logits_data, \
- processed_logits, \
- shared_reduce_buffer, \
- vocab_size, \
- top_p, \
- threadgroup_idx.x, \
- thread_idx.x)
-
-#define generateToppKernel(functionName, scalarType, outerArgs, innerArgs) \
- [[max_total_threads_per_threadgroup( \
- 1024 \
- )]] kernel void functionName##_##scalarType outerArgs { \
- threadgroup float shared_reduce_buffer[BLOCK_SIZE_IN_SIMDS]; \
- functionName innerArgs; \
- }
-
-#define generateToppKernels(functionName) \
- generateToppKernel( \
- functionName, \
- float, \
- outerArguments(float), \
- innerArguments \
- ); \
- generateToppKernel( \
- functionName, \
- bfloat, \
- outerArguments(bfloat), \
- innerArguments \
- ); \
- generateToppKernel(functionName, half, outerArguments(half), innerArguments);
-
-generateToppKernels(batched_topp)
-
-#undef outerArguments
-#undef innerArguments
-#undef generateToppKernel
-#undef generateToppKernels