Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Gpu execution time is always present (#328)

uuuvn12968bf06710
diff --git a/Cargo.lock b/Cargo.lock
index 9bcac21d7..50259b37e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -386,16 +386,16 @@ checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"
[[package]]
name = "blake3"
-version = "1.8.3"
+version = "1.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"
+checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e"
dependencies = [
"arrayref",
"arrayvec",
"cc",
"cfg-if",
"constant_time_eq",
- "cpufeatures 0.2.17",
+ "cpufeatures 0.3.0",
]
[[package]]
@@ -483,7 +483,7 @@ checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799"
dependencies = [
"clap",
"heck",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"log",
"proc-macro2",
"quote",
@@ -496,9 +496,9 @@ dependencies = [
[[package]]
name = "cc"
-version = "1.2.57"
+version = "1.2.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
+checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283"
dependencies = [
"find-msvc-tools",
"shlex",
@@ -645,9 +645,9 @@ dependencies = [
[[package]]
name = "cmake"
-version = "0.1.57"
+version = "0.1.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
dependencies = [
"cc",
]
@@ -762,7 +762,7 @@ dependencies = [
"cookie",
"document-features",
"idna",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"log",
"serde",
"serde_derive",
@@ -971,7 +971,7 @@ checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e"
dependencies = [
"cc",
"codespan-reporting",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"proc-macro2",
"quote",
"scratch",
@@ -985,7 +985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "035b6c61a944483e8a4b2ad4fb8b13830d63491bd004943716ad16d85dcc64bc"
dependencies = [
"codespan-reporting",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -999,7 +999,7 @@ checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328"
dependencies = [
"clap",
"codespan-reporting",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -1017,7 +1017,7 @@ version = "1.0.194"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -1320,9 +1320,9 @@ dependencies = [
[[package]]
name = "fastrand"
-version = "2.3.0"
+version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"
[[package]]
name = "figment"
@@ -1571,7 +1571,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http 0.2.12",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"slab",
"tokio",
"tokio-util",
@@ -1590,7 +1590,7 @@ dependencies = [
"futures-core",
"futures-sink",
"http 1.4.0",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"slab",
"tokio",
"tokio-util",
@@ -1775,9 +1775,9 @@ dependencies = [
[[package]]
name = "hyper"
-version = "1.8.1"
+version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
+checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
dependencies = [
"atomic-waker",
"bytes",
@@ -1789,7 +1789,6 @@ dependencies = [
"httparse",
"itoa",
"pin-project-lite",
- "pin-utils",
"smallvec",
"tokio",
"want",
@@ -1802,7 +1801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"
dependencies = [
"http 1.4.0",
- "hyper 1.8.1",
+ "hyper 1.9.0",
"hyper-util",
"rustls",
"rustls-pki-types",
@@ -1819,7 +1818,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [
"bytes",
"http-body-util",
- "hyper 1.8.1",
+ "hyper 1.9.0",
"hyper-util",
"native-tls",
"tokio",
@@ -1839,7 +1838,7 @@ dependencies = [
"futures-util",
"http 1.4.0",
"http-body 1.0.1",
- "hyper 1.8.1",
+ "hyper 1.9.0",
"ipnet",
"libc",
"percent-encoding",
@@ -1854,12 +1853,13 @@ dependencies = [
[[package]]
name = "icu_collections"
-version = "2.1.1"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
+checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
dependencies = [
"displaydoc",
"potential_utf",
+ "utf8_iter",
"yoke",
"zerofrom",
"zerovec",
@@ -1867,9 +1867,9 @@ dependencies = [
[[package]]
name = "icu_locale_core"
-version = "2.1.1"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
+checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
dependencies = [
"displaydoc",
"litemap",
@@ -1880,9 +1880,9 @@ dependencies = [
[[package]]
name = "icu_normalizer"
-version = "2.1.1"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
+checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
dependencies = [
"icu_collections",
"icu_normalizer_data",
@@ -1894,15 +1894,15 @@ dependencies = [
[[package]]
name = "icu_normalizer_data"
-version = "2.1.1"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
+checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
[[package]]
name = "icu_properties"
-version = "2.1.2"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"
+checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
dependencies = [
"icu_collections",
"icu_locale_core",
@@ -1914,15 +1914,15 @@ dependencies = [
[[package]]
name = "icu_properties_data"
-version = "2.1.2"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"
+checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
[[package]]
name = "icu_provider"
-version = "2.1.1"
+version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
+checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
dependencies = [
"displaydoc",
"icu_locale_core",
@@ -1979,9 +1979,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.13.0"
+version = "2.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
+checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
@@ -2036,9 +2036,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iri-string"
-version = "0.7.11"
+version = "0.7.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb"
+checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20"
dependencies = [
"memchr",
"serde",
@@ -2114,10 +2114,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "js-sys"
-version = "0.3.91"
+version = "0.3.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"
+checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9"
dependencies = [
+ "cfg-if",
+ "futures-util",
"once_cell",
"wasm-bindgen",
]
@@ -2136,9 +2138,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"
[[package]]
name = "libc"
-version = "0.2.183"
+version = "0.2.184"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
+checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
[[package]]
name = "libloading"
@@ -2182,9 +2184,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "litemap"
-version = "0.8.1"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
+checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
[[package]]
name = "litrs"
@@ -2349,9 +2351,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "minijinja"
-version = "2.18.0"
+version = "2.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "328251e58ad8e415be6198888fc207502727dc77945806421ab34f35bf012e7d"
+checksum = "805bfd7352166bae857ee569628b52bcd85a1cecf7810861ebceb1686b72b75d"
dependencies = [
"memo-map",
"serde",
@@ -2359,9 +2361,9 @@ dependencies = [
[[package]]
name = "minijinja-contrib"
-version = "2.18.0"
+version = "2.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c6302e47d2b51f9fc978268ff7f5a014de5caa2ad48440309fd10ee711480d7"
+checksum = "45092d80391870622fcf3bd82f5d2af18f99533ea60debb4bc9db0c76f0e809a"
dependencies = [
"minijinja",
"serde",
@@ -2385,9 +2387,9 @@ dependencies = [
[[package]]
name = "mio"
-version = "1.1.1"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
dependencies = [
"libc",
"log",
@@ -2428,9 +2430,9 @@ dependencies = [
[[package]]
name = "mtl-rs"
-version = "0.1.8"
+version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2db417bbf074d7ecc1d54494680c7712892110d882fe409f32a8ddd8832f5023"
+checksum = "e851d258c741f92ea3e92824afcfa888bbe814ac23d2825a9352baa5ce4f8ce0"
dependencies = [
"bitflags",
"block2",
@@ -2859,12 +2861,6 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
-[[package]]
-name = "pin-utils"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
[[package]]
name = "pkg-config"
version = "0.3.32"
@@ -2916,9 +2912,9 @@ dependencies = [
[[package]]
name = "potential_utf"
-version = "0.1.4"
+version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
+checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
dependencies = [
"zerovec",
]
@@ -3302,7 +3298,7 @@ dependencies = [
"http 1.4.0",
"http-body 1.0.1",
"http-body-util",
- "hyper 1.8.1",
+ "hyper 1.9.0",
"hyper-rustls",
"hyper-tls",
"hyper-util",
@@ -3386,7 +3382,7 @@ dependencies = [
"either",
"figment",
"futures",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"log",
"memchr",
"multer",
@@ -3418,7 +3414,7 @@ checksum = "575d32d7ec1a9770108c879fc7c47815a80073f96ca07ff9525a94fcede1dd46"
dependencies = [
"devise",
"glob",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"proc-macro2",
"quote",
"rocket_http",
@@ -3438,7 +3434,7 @@ dependencies = [
"futures",
"http 0.2.12",
"hyper 0.14.32",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"log",
"memchr",
"pear",
@@ -3456,9 +3452,9 @@ dependencies = [
[[package]]
name = "rustc-hash"
-version = "2.1.1"
+version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
[[package]]
name = "rustc_version"
@@ -3627,9 +3623,9 @@ dependencies = [
[[package]]
name = "semver"
-version = "1.0.27"
+version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
[[package]]
name = "serde"
@@ -3707,9 +3703,9 @@ dependencies = [
[[package]]
name = "serde_spanned"
-version = "1.1.0"
+version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98"
+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
dependencies = [
"serde_core",
]
@@ -3785,9 +3781,9 @@ dependencies = [
[[package]]
name = "simd-adler32"
-version = "0.3.8"
+version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
[[package]]
name = "slab"
@@ -4110,9 +4106,9 @@ dependencies = [
[[package]]
name = "tinystr"
-version = "0.8.2"
+version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
+checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
dependencies = [
"displaydoc",
"zerovec",
@@ -4164,9 +4160,9 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.50.0"
+version = "1.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
+checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd"
dependencies = [
"bytes",
"libc",
@@ -4181,9 +4177,9 @@ dependencies = [
[[package]]
name = "tokio-macros"
-version = "2.6.1"
+version = "2.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
dependencies = [
"proc-macro2",
"quote",
@@ -4252,9 +4248,9 @@ version = "0.9.12+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"serde_core",
- "serde_spanned 1.1.0",
+ "serde_spanned 1.1.1",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"toml_writer",
@@ -4285,7 +4281,7 @@ version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"serde",
"serde_spanned 0.6.9",
"toml_datetime 0.6.11",
@@ -4295,11 +4291,11 @@ dependencies = [
[[package]]
name = "toml_parser"
-version = "1.1.0+spec-1.1.0"
+version = "1.1.2+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011"
+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
dependencies = [
- "winnow 1.0.0",
+ "winnow 1.0.1",
]
[[package]]
@@ -4310,9 +4306,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "toml_writer"
-version = "1.1.0+spec-1.1.0"
+version = "1.1.1+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed"
+checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"
[[package]]
name = "tower"
@@ -4588,9 +4584,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "uuid"
-version = "1.22.0"
+version = "1.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"
+checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"
dependencies = [
"getrandom 0.4.2",
"js-sys",
@@ -4726,9 +4722,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen"
-version = "0.2.114"
+version = "0.2.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
+checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0"
dependencies = [
"cfg-if",
"once_cell",
@@ -4739,23 +4735,19 @@ dependencies = [
[[package]]
name = "wasm-bindgen-futures"
-version = "0.4.64"
+version = "0.4.67"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8"
+checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e"
dependencies = [
- "cfg-if",
- "futures-util",
"js-sys",
- "once_cell",
"wasm-bindgen",
- "web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
-version = "0.2.114"
+version = "0.2.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
+checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -4763,9 +4755,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
-version = "0.2.114"
+version = "0.2.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
+checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2"
dependencies = [
"bumpalo",
"proc-macro2",
@@ -4776,9 +4768,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
-version = "0.2.114"
+version = "0.2.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
+checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b"
dependencies = [
"unicode-ident",
]
@@ -4800,7 +4792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"
dependencies = [
"anyhow",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"wasm-encoder",
"wasmparser",
]
@@ -4826,15 +4818,15 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"
dependencies = [
"bitflags",
"hashbrown 0.15.5",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"semver",
]
[[package]]
name = "web-sys"
-version = "0.3.91"
+version = "0.3.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"
+checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a"
dependencies = [
"js-sys",
"wasm-bindgen",
@@ -5179,9 +5171,9 @@ dependencies = [
[[package]]
name = "winnow"
-version = "1.0.0"
+version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8"
+checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"
[[package]]
name = "wit-bindgen"
@@ -5211,7 +5203,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"
dependencies = [
"anyhow",
"heck",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"prettyplease",
"syn 2.0.117",
"wasm-metadata",
@@ -5242,7 +5234,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"
dependencies = [
"anyhow",
"bitflags",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"log",
"serde",
"serde_derive",
@@ -5261,7 +5253,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"
dependencies = [
"anyhow",
"id-arena",
- "indexmap 2.13.0",
+ "indexmap 2.13.1",
"log",
"semver",
"serde",
@@ -5273,9 +5265,9 @@ dependencies = [
[[package]]
name = "writeable"
-version = "0.6.2"
+version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
+checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
[[package]]
name = "xgrammar-rs"
@@ -5310,9 +5302,9 @@ dependencies = [
[[package]]
name = "yoke"
-version = "0.8.1"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
+checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
dependencies = [
"stable_deref_trait",
"yoke-derive",
@@ -5321,9 +5313,9 @@ dependencies = [
[[package]]
name = "yoke-derive"
-version = "0.8.1"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
+checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
dependencies = [
"proc-macro2",
"quote",
@@ -5333,18 +5325,18 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.8.47"
+version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
+checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.47"
+version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
+checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"
dependencies = [
"proc-macro2",
"quote",
@@ -5353,18 +5345,18 @@ dependencies = [
[[package]]
name = "zerofrom"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
+checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
dependencies = [
"zerofrom-derive",
]
[[package]]
name = "zerofrom-derive"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
+checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
dependencies = [
"proc-macro2",
"quote",
@@ -5380,9 +5372,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
[[package]]
name = "zerotrie"
-version = "0.2.3"
+version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
+checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
dependencies = [
"displaydoc",
"yoke",
@@ -5391,9 +5383,9 @@ dependencies = [
[[package]]
name = "zerovec"
-version = "0.11.5"
+version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
+checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
dependencies = [
"yoke",
"zerofrom",
@@ -5402,9 +5394,9 @@ dependencies = [
[[package]]
name = "zerovec-derive"
-version = "0.11.2"
+version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
+checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
dependencies = [
"proc-macro2",
"quote",
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index 20a7b2331..e93754450 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -137,5 +137,5 @@ pub trait CommandBufferPending {
pub trait CommandBufferCompleted {
type CommandBuffer: CommandBuffer<Completed = Self>;
- fn gpu_execution_time(&self) -> Option<Duration>;
+ fn gpu_execution_time(&self) -> Duration;
}
diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rs
index 3a0e74009..efd780d05 100644
--- a/crates/uzu/src/backends/common/encoder.rs
+++ b/crates/uzu/src/backends/common/encoder.rs
@@ -170,7 +170,7 @@ pub struct Completed<B: Backend> {
}
impl<B: Backend> Completed<B> {
- pub fn gpu_execution_time(&self) -> Option<Duration> {
+ pub fn gpu_execution_time(&self) -> Duration {
self.command_buffer.gpu_execution_time()
}
}
diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rs
index 4fed33635..7d2d8abcb 100644
--- a/crates/uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu/src/backends/cpu/command_buffer.rs
@@ -212,7 +212,7 @@ pub struct CpuCommandBufferCompleted {
impl CommandBufferCompleted for CpuCommandBufferCompleted {
type CommandBuffer = CpuCommandBuffer;
- fn gpu_execution_time(&self) -> Option<Duration> {
- Some(self.gpu_execution_time)
+ fn gpu_execution_time(&self) -> Duration {
+ self.gpu_execution_time
}
}
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index b2f294885..d8ac4af5d 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -232,10 +232,10 @@ pub struct MetalCommandBufferCompleted {
impl CommandBufferCompleted for MetalCommandBufferCompleted {
type CommandBuffer = MetalCommandBuffer;
- fn gpu_execution_time(&self) -> Option<Duration> {
- match (self.command_buffer.gpu_start_time(), self.command_buffer.gpu_end_time()) {
- (Some(start), Some(end)) if end > start => Some(Duration::from_secs_f64(end - start)),
- _ => None,
- }
+ fn gpu_execution_time(&self) -> Duration {
+ // They're always present, https://developer.apple.com/documentation/metal/mtlcommandbuffer/gpustarttime?language=objc
+ let start = self.command_buffer.gpu_start_time();
+ let end = self.command_buffer.gpu_end_time();
+ Duration::from_secs_f64(end - start)
}
}
diff --git a/crates/uzu/tests/performance/matmul/bench.rs b/crates/uzu/tests/performance/matmul/bench.rs
index ec8481f44..23c7c2b4e 100644
--- a/crates/uzu/tests/performance/matmul/bench.rs
+++ b/crates/uzu/tests/performance/matmul/bench.rs
@@ -63,7 +63,7 @@ fn encode_and_run(
let completed = encoder.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;
- completed.gpu_execution_time().map(|duration| duration.as_secs_f64() * 1000.0).ok_or(BenchError::GpuTimestamps)
+ Ok(completed.gpu_execution_time().as_secs_f64() * 1000.0)
}
fn run_benchmark(
diff --git a/crates/uzu/tests/performance/matmul/error.rs b/crates/uzu/tests/performance/matmul/error.rs
index dfd690cfe..b06aa8fd7 100644
--- a/crates/uzu/tests/performance/matmul/error.rs
+++ b/crates/uzu/tests/performance/matmul/error.rs
@@ -4,8 +4,6 @@ use thiserror::Error;
pub enum BenchError {
#[error("failed to create command buffer")]
CommandBuffer,
- #[error("GPU timestamps unavailable")]
- GpuTimestamps,
#[error("kernel creation failed: {0}")]
Kernel(String),
#[error("buffer allocation failed")]
diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rs
index 3f08b7e08..f1031a1f1 100644
--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs
@@ -1047,29 +1047,16 @@ fn perf_two_pass_attention() {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
- Some(gpu_time_ms) => {
- println!(
- "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",
- num_heads,
- seq_len - suffix_length,
- suffix_length,
- head_dim,
- gpu_time_ms,
- host_elapsed_ms
- );
- },
- None => {
- println!(
- "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): Host-side={:.2} ms (GPU timing unavailable)",
- num_heads,
- seq_len - suffix_length,
- suffix_length,
- head_dim,
- host_elapsed_ms
- );
- },
- }
+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;
+ println!(
+ "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",
+ num_heads,
+ seq_len - suffix_length,
+ suffix_length,
+ head_dim,
+ gpu_time_ms,
+ host_elapsed_ms
+ );
// ---- Sanity check ----
let output_ptr = output_buffer.contents().as_ptr() as *const f32;
diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
index 4d4fd78aa..723749022 100644
--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
@@ -192,20 +192,11 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
- Some(gpu_time_ms) => {
- println!(
- "Argmax sampling perf (batch={}, vocab={}, strategy={:?}): GPU={:.2} ms, Host-side={:.2} ms",
- BATCH, VOCAB, strategy, gpu_time_ms, host_elapsed_ms
- );
- },
- None => {
- println!(
- "Argmax sampling perf (batch={}, vocab={}, strategy={:?}): Host-side={:.2} ms (GPU timing unavailable)",
- BATCH, VOCAB, strategy, host_elapsed_ms
- );
- },
- }
+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;
+ println!(
+ "Argmax sampling perf (batch={}, vocab={}, strategy={:?}): GPU={:.2} ms, Host-side={:.2} ms",
+ BATCH, VOCAB, strategy, gpu_time_ms, host_elapsed_ms
+ );
// Ensure the kernel produced *some* output (sanity).
let ptr = output_buf.contents().as_ptr() as *const u32;
@@ -548,20 +539,11 @@ fn perf_categorical_128k_vocab() {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
- Some(gpu_time_ms) => {
- println!(
- "Categorical sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",
- BATCH, VOCAB, gpu_time_ms, host_elapsed_ms
- );
- },
- None => {
- println!(
- "Categorical sampling perf (batch={}, vocab={}): Host-side={:.2} ms (GPU timing unavailable)",
- BATCH, VOCAB, host_elapsed_ms
- );
- },
- }
+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;
+ println!(
+ "Categorical sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",
+ BATCH, VOCAB, gpu_time_ms, host_elapsed_ms
+ );
// Sanity check
let ptr = output_buf.contents().as_ptr() as *const u32;
diff --git a/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
index d2fb5eff9..30f89f124 100644
--- a/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
+++ b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs
@@ -1,6 +1,9 @@
#![cfg(metal_backend)]
-use std::{ops::{Deref, DerefMut}, rc::Rc};
+use std::{
+ ops::{Deref, DerefMut},
+ rc::Rc,
+};
use criterion::{BenchmarkId, Criterion, Throughput};
use half::bf16;
@@ -41,18 +44,10 @@ fn get_test_data(
ab_scale: f32,
accumulate: bool,
) -> Input {
- let left: Vec<bf16> = (0..batch_dim * input_dim)
- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6))
- .collect();
- let right: Vec<bf16> = (0..output_dim * input_dim)
- .map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8))
- .collect();
+ let left: Vec<bf16> = (0..batch_dim * input_dim).map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6)).collect();
+ let right: Vec<bf16> = (0..output_dim * input_dim).map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8)).collect();
let destination_prefill = if accumulate {
- Some(
- (0..batch_dim * output_dim)
- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09))
- .collect(),
- )
+ Some((0..batch_dim * output_dim).map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09)).collect())
} else {
None
};
@@ -168,14 +163,11 @@ fn get_mpp_output(
}
fn create_metal_context_and_matmul_kernel()
- -> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel)
-{
+-> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel) {
let metal_context = MetalContext::new().expect("Metal context");
- let matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
- &metal_context,
- bf16::data_type(),
- )
- .expect("MatmulKernel");
+ let matmul_kernel =
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())
+ .expect("MatmulKernel");
(metal_context, matmul_kernel)
}
@@ -249,12 +241,8 @@ fn test_bf16_scale_and_accumulate() {
// Benchmarks
-const BENCHMARK_SHAPES: &[(usize, usize, usize)] = &[
- (128, 2048, 8192),
- (128, 4096, 14336),
- (256, 4096, 4096),
- (512, 8192, 2048),
-];
+const BENCHMARK_SHAPES: &[(usize, usize, usize)] =
+ &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];
#[uzu_bench]
fn bench_gemm_mpp(criterion: &mut Criterion) {
@@ -263,21 +251,16 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {
return;
}
- let mut matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(
- &metal_context,
- bf16::data_type(),
- )
- .expect("MatmulKernel");
+ let mut matmul_kernel =
+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())
+ .expect("MatmulKernel");
- let mut benchmark_group = criterion.benchmark_group(
- format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()),
- );
+ let mut benchmark_group =
+ criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()));
for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {
- let left_array =
- metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");
- let right_array =
- metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");
+ let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");
+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");
let destination_array =
metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");
@@ -316,13 +299,7 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {
);
}
- encoder
- .end_encoding()
- .submit()
- .wait_until_completed()
- .unwrap()
- .gpu_execution_time()
- .unwrap()
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
})
},
);
diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
index 035dbfeb9..34ab9a9db 100644
--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs
@@ -282,25 +282,31 @@ macro_rules! qmv_fast_test {
($name:ident, gs=$gs:expr, bits=$bits:expr, zp=$zp:expr, mlx=$mlx:expr) => {
#[uzu_test]
fn $name() {
- for_each_float_type!(|F| { test_basic::<F>($gs, $bits, $zp, $mlx); })
+ for_each_float_type!(|F| {
+ test_basic::<F>($gs, $bits, $zp, $mlx);
+ })
}
};
}
-qmv_fast_test!(test_gs32_4bit_zp, gs=32, bits=4, zp=true, mlx=false);
-qmv_fast_test!(test_gs64_4bit_zp, gs=64, bits=4, zp=true, mlx=false);
-qmv_fast_test!(test_gs128_4bit_zp, gs=128, bits=4, zp=true, mlx=false);
-qmv_fast_test!(test_gs32_8bit_zp, gs=32, bits=8, zp=true, mlx=false);
-qmv_fast_test!(test_gs64_8bit_zp, gs=64, bits=8, zp=true, mlx=false);
-qmv_fast_test!(test_gs128_8bit_zp, gs=128, bits=8, zp=true, mlx=false);
-qmv_fast_test!(test_gs32_4bit_mlx, gs=32, bits=4, zp=false, mlx=true);
-qmv_fast_test!(test_gs64_4bit_mlx, gs=64, bits=4, zp=false, mlx=true);
-qmv_fast_test!(test_gs128_4bit_mlx, gs=128, bits=4, zp=false, mlx=true);
-qmv_fast_test!(test_gs32_8bit_mlx, gs=32, bits=8, zp=false, mlx=true);
-qmv_fast_test!(test_gs64_8bit_mlx, gs=64, bits=8, zp=false, mlx=true);
-qmv_fast_test!(test_gs128_8bit_mlx, gs=128, bits=8, zp=false, mlx=true);
-
-fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(rng: &mut R, range: std::ops::Range<T>, len: usize) -> Box<[T]> {
+qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, zp = true, mlx = false);
+qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, zp = true, mlx = false);
+qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, zp = true, mlx = false);
+qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, zp = true, mlx = false);
+qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, zp = true, mlx = false);
+qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, zp = true, mlx = false);
+qmv_fast_test!(test_gs32_4bit_mlx, gs = 32, bits = 4, zp = false, mlx = true);
+qmv_fast_test!(test_gs64_4bit_mlx, gs = 64, bits = 4, zp = false, mlx = true);
+qmv_fast_test!(test_gs128_4bit_mlx, gs = 128, bits = 4, zp = false, mlx = true);
+qmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, zp = false, mlx = true);
+qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, zp = false, mlx = true);
+qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, zp = false, mlx = true);
+
+fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(
+ rng: &mut R,
+ range: std::ops::Range<T>,
+ len: usize,
+) -> Box<[T]> {
(0..len).map(|_| rng.random_range(range.clone())).collect()
}
@@ -314,29 +320,63 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
use_mlx_quant: bool,
) {
let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));
- let block_size: usize = if bits == 4 { 512 } else { 256 };
+ let block_size: usize = if bits == 4 {
+ 512
+ } else {
+ 256
+ };
for (m, n, k) in iproduct!([1, 2, 3, 4], [2048, 4096, 14336], [2048, 4096, 8192, 14336]) {
- if n % 8 != 0 || k % block_size != 0 { continue; }
+ if n % 8 != 0 || k % block_size != 0 {
+ continue;
+ }
let num_groups = k.div_ceil(group_size as usize);
let mut rng = SmallRng::seed_from_u64(42);
let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(
- context, T::data_type(), group_size, bits, use_zero_points, use_mlx_quant,
- ).unwrap();
-
- let w_buf = alloc_buffer_with_data::<B, u32>(context, &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32));
- let scales_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());
- let x_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());
+ context,
+ T::data_type(),
+ group_size,
+ bits,
+ use_zero_points,
+ use_mlx_quant,
+ )
+ .unwrap();
+
+ let w_buf = alloc_buffer_with_data::<B, u32>(
+ context,
+ &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),
+ );
+ let scales_buf = alloc_buffer_with_data::<B, T>(
+ context,
+ &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)
+ .iter()
+ .map(|&v| T::from(v).unwrap())
+ .collect::<Vec<_>>(),
+ );
+ let x_buf = alloc_buffer_with_data::<B, T>(
+ context,
+ &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>(),
+ );
let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();
let zp_buf = use_zero_points.then(|| {
- let zp_stride = if bits == 4 { (num_groups + 1) / 2 } else { num_groups };
+ let zp_stride = if bits == 4 {
+ (num_groups + 1) / 2
+ } else {
+ num_groups
+ };
alloc_buffer_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))
});
let bias_buf = use_mlx_quant.then(|| {
- alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>())
+ alloc_buffer_with_data::<B, T>(
+ context,
+ &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)
+ .iter()
+ .map(|&v| T::from(v).unwrap())
+ .collect::<Vec<_>>(),
+ )
});
group.throughput(Throughput::Elements((m * n * k) as u64));
@@ -344,9 +384,20 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(
b.iter_custom(|n_iters| {
let mut encoder = Encoder::<B>::new(context).unwrap();
for _ in 0..n_iters {
- kernel.encode(&w_buf, &scales_buf, zp_buf.as_ref(), bias_buf.as_ref(), &x_buf, &mut y_buf, k as u32, n as u32, m as u32, &mut encoder);
+ kernel.encode(
+ &w_buf,
+ &scales_buf,
+ zp_buf.as_ref(),
+ bias_buf.as_ref(),
+ &x_buf,
+ &mut y_buf,
+ k as u32,
+ n as u32,
+ m as u32,
+ &mut encoder,
+ );
}
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
})
});
}
diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
index 11e06e464..11a648738 100644
--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs
+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs
@@ -121,7 +121,7 @@ fn get_output<
AccumT: ArrayElement + Float,
>(
input: &Input<InputT, ScaleT, OutputT>
-) -> (Vec<OutputT>, f64, Option<Duration>) {
+) -> (Vec<OutputT>, f64, Duration) {
let context = B::Context::new().expect("Failed to create Context");
let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(
&context,
@@ -461,7 +461,7 @@ fn bench_rms_norm(c: &mut Criterion) {
);
}
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
})
});
}
diff --git a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs
index d8b17834a..12afa5a66 100644
--- a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs
@@ -180,7 +180,7 @@ fn bench_argmax(c: &mut Criterion) {
);
}
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
})
});
@@ -205,7 +205,7 @@ fn bench_argmax(c: &mut Criterion) {
);
}
- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()
+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()
})
});
}
diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
index deaf6745d..30890f2f7 100644
--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
@@ -489,27 +489,15 @@ fn perf_topp_128k_vocab() {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
- Some(gpu_time_ms) => {
- println!(
- "Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",
- BATCH,
- VOCAB,
- std::any::type_name::<B>(),
- gpu_time_ms,
- host_elapsed_ms
- );
- },
- None => {
- println!(
- "Top-p sampling perf (batch={}, vocab={}, backend={}): Host-side={:.2} ms (GPU timing unavailable)",
- BATCH,
- VOCAB,
- std::any::type_name::<B>(),
- host_elapsed_ms
- );
- },
- }
+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;
+ println!(
+ "Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",
+ BATCH,
+ VOCAB,
+ std::any::type_name::<B>(),
+ gpu_time_ms,
+ host_elapsed_ms
+ );
let sample_ids: &[u32] = output_array.as_slice();
for &tok in sample_ids {