A high-performance inference engine for AI models
Gpu execution time is always present (#328)
uuuvn12968bf06710
diff --git a/Cargo.lock b/Cargo.lockindex 9bcac21d7..50259b37e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -386,16 +386,16 @@ checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af"[[package]]name = "blake3"-version = "1.8.3"+version = "1.8.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d"+checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e"dependencies = ["arrayref","arrayvec","cc","cfg-if","constant_time_eq",- "cpufeatures 0.2.17",+ "cpufeatures 0.3.0",][[package]]@@ -483,7 +483,7 @@ checksum = "befbfd072a8e81c02f8c507aefce431fe5e7d051f83d48a23ffc9b9fe5a11799"dependencies = ["clap","heck",- "indexmap 2.13.0",+ "indexmap 2.13.1","log","proc-macro2","quote",@@ -496,9 +496,9 @@ dependencies = [[[package]]name = "cc"-version = "1.2.57"+version = "1.2.59"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"+checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283"dependencies = ["find-msvc-tools","shlex",@@ -645,9 +645,9 @@ dependencies = [[[package]]name = "cmake"-version = "0.1.57"+version = "0.1.58"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d"+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"dependencies = ["cc",]@@ -762,7 +762,7 @@ dependencies = ["cookie","document-features","idna",- "indexmap 2.13.0",+ "indexmap 2.13.1","log","serde","serde_derive",@@ -971,7 +971,7 @@ checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e"dependencies = ["cc","codespan-reporting",- "indexmap 2.13.0",+ "indexmap 2.13.1","proc-macro2","quote","scratch",@@ -985,7 +985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "035b6c61a944483e8a4b2ad4fb8b13830d63491bd004943716ad16d85dcc64bc"dependencies = ["codespan-reporting",- "indexmap 2.13.0",+ "indexmap 2.13.1","proc-macro2","quote","syn 2.0.117",@@ -999,7 +999,7 @@ checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328"dependencies = ["clap","codespan-reporting",- "indexmap 2.13.0",+ "indexmap 2.13.1","proc-macro2","quote","syn 2.0.117",@@ -1017,7 +1017,7 @@ version = "1.0.194"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf"dependencies = [- "indexmap 2.13.0",+ "indexmap 2.13.1","proc-macro2","quote","syn 2.0.117",@@ -1320,9 +1320,9 @@ dependencies = [[[package]]name = "fastrand"-version = "2.3.0"+version = "2.4.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"+checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6"[[package]]name = "figment"@@ -1571,7 +1571,7 @@ dependencies = ["futures-sink","futures-util","http 0.2.12",- "indexmap 2.13.0",+ "indexmap 2.13.1","slab","tokio","tokio-util",@@ -1590,7 +1590,7 @@ dependencies = ["futures-core","futures-sink","http 1.4.0",- "indexmap 2.13.0",+ "indexmap 2.13.1","slab","tokio","tokio-util",@@ -1775,9 +1775,9 @@ dependencies = [[[package]]name = "hyper"-version = "1.8.1"+version = "1.9.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"+checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"dependencies = ["atomic-waker","bytes",@@ -1789,7 +1789,6 @@ dependencies = ["httparse","itoa","pin-project-lite",- "pin-utils","smallvec","tokio","want",@@ -1802,7 +1801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58"dependencies = ["http 1.4.0",- "hyper 1.8.1",+ "hyper 1.9.0","hyper-util","rustls","rustls-pki-types",@@ -1819,7 +1818,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"dependencies = ["bytes","http-body-util",- "hyper 1.8.1",+ "hyper 1.9.0","hyper-util","native-tls","tokio",@@ -1839,7 +1838,7 @@ dependencies = ["futures-util","http 1.4.0","http-body 1.0.1",- "hyper 1.8.1",+ "hyper 1.9.0","ipnet","libc","percent-encoding",@@ -1854,12 +1853,13 @@ dependencies = [[[package]]name = "icu_collections"-version = "2.1.1"+version = "2.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"+checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"dependencies = ["displaydoc","potential_utf",+ "utf8_iter","yoke","zerofrom","zerovec",@@ -1867,9 +1867,9 @@ dependencies = [[[package]]name = "icu_locale_core"-version = "2.1.1"+version = "2.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"+checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"dependencies = ["displaydoc","litemap",@@ -1880,9 +1880,9 @@ dependencies = [[[package]]name = "icu_normalizer"-version = "2.1.1"+version = "2.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"+checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"dependencies = ["icu_collections","icu_normalizer_data",@@ -1894,15 +1894,15 @@ dependencies = [[[package]]name = "icu_normalizer_data"-version = "2.1.1"+version = "2.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"+checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"[[package]]name = "icu_properties"-version = "2.1.2"+version = "2.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec"+checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"dependencies = ["icu_collections","icu_locale_core",@@ -1914,15 +1914,15 @@ dependencies = [[[package]]name = "icu_properties_data"-version = "2.1.2"+version = "2.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af"+checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"[[package]]name = "icu_provider"-version = "2.1.1"+version = "2.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"+checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"dependencies = ["displaydoc","icu_locale_core",@@ -1979,9 +1979,9 @@ dependencies = [[[package]]name = "indexmap"-version = "2.13.0"+version = "2.13.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"+checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff"dependencies = ["equivalent","hashbrown 0.16.1",@@ -2036,9 +2036,9 @@ checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"[[package]]name = "iri-string"-version = "0.7.11"+version = "0.7.12"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb"+checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20"dependencies = ["memchr","serde",@@ -2114,10 +2114,12 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"[[package]]name = "js-sys"-version = "0.3.91"+version = "0.3.94"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c"+checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9"dependencies = [+ "cfg-if",+ "futures-util","once_cell","wasm-bindgen",]@@ -2136,9 +2138,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2"[[package]]name = "libc"-version = "0.2.183"+version = "0.2.184"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"+checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"[[package]]name = "libloading"@@ -2182,9 +2184,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"[[package]]name = "litemap"-version = "0.8.1"+version = "0.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"+checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"[[package]]name = "litrs"@@ -2349,9 +2351,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"[[package]]name = "minijinja"-version = "2.18.0"+version = "2.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "328251e58ad8e415be6198888fc207502727dc77945806421ab34f35bf012e7d"+checksum = "805bfd7352166bae857ee569628b52bcd85a1cecf7810861ebceb1686b72b75d"dependencies = ["memo-map","serde",@@ -2359,9 +2361,9 @@ dependencies = [[[package]]name = "minijinja-contrib"-version = "2.18.0"+version = "2.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8c6302e47d2b51f9fc978268ff7f5a014de5caa2ad48440309fd10ee711480d7"+checksum = "45092d80391870622fcf3bd82f5d2af18f99533ea60debb4bc9db0c76f0e809a"dependencies = ["minijinja","serde",@@ -2385,9 +2387,9 @@ dependencies = [[[package]]name = "mio"-version = "1.1.1"+version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"dependencies = ["libc","log",@@ -2428,9 +2430,9 @@ dependencies = [[[package]]name = "mtl-rs"-version = "0.1.8"+version = "0.1.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2db417bbf074d7ecc1d54494680c7712892110d882fe409f32a8ddd8832f5023"+checksum = "e851d258c741f92ea3e92824afcfa888bbe814ac23d2825a9352baa5ce4f8ce0"dependencies = ["bitflags","block2",@@ -2859,12 +2861,6 @@ version = "0.2.17"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"-[[package]]-name = "pin-utils"-version = "0.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"-[[package]]name = "pkg-config"version = "0.3.32"@@ -2916,9 +2912,9 @@ dependencies = [[[package]]name = "potential_utf"-version = "0.1.4"+version = "0.1.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"+checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"dependencies = ["zerovec",]@@ -3302,7 +3298,7 @@ dependencies = ["http 1.4.0","http-body 1.0.1","http-body-util",- "hyper 1.8.1",+ "hyper 1.9.0","hyper-rustls","hyper-tls","hyper-util",@@ -3386,7 +3382,7 @@ dependencies = ["either","figment","futures",- "indexmap 2.13.0",+ "indexmap 2.13.1","log","memchr","multer",@@ -3418,7 +3414,7 @@ checksum = "575d32d7ec1a9770108c879fc7c47815a80073f96ca07ff9525a94fcede1dd46"dependencies = ["devise","glob",- "indexmap 2.13.0",+ "indexmap 2.13.1","proc-macro2","quote","rocket_http",@@ -3438,7 +3434,7 @@ dependencies = ["futures","http 0.2.12","hyper 0.14.32",- "indexmap 2.13.0",+ "indexmap 2.13.1","log","memchr","pear",@@ -3456,9 +3452,9 @@ dependencies = [[[package]]name = "rustc-hash"-version = "2.1.1"+version = "2.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"[[package]]name = "rustc_version"@@ -3627,9 +3623,9 @@ dependencies = [[[package]]name = "semver"-version = "1.0.27"+version = "1.0.28"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"[[package]]name = "serde"@@ -3707,9 +3703,9 @@ dependencies = [[[package]]name = "serde_spanned"-version = "1.1.0"+version = "1.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "876ac351060d4f882bb1032b6369eb0aef79ad9df1ea8bc404874d8cc3d0cd98"+checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"dependencies = ["serde_core",]@@ -3785,9 +3781,9 @@ dependencies = [[[package]]name = "simd-adler32"-version = "0.3.8"+version = "0.3.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"+checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"[[package]]name = "slab"@@ -4110,9 +4106,9 @@ dependencies = [[[package]]name = "tinystr"-version = "0.8.2"+version = "0.8.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"+checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"dependencies = ["displaydoc","zerovec",@@ -4164,9 +4160,9 @@ dependencies = [[[package]]name = "tokio"-version = "1.50.0"+version = "1.51.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"+checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd"dependencies = ["bytes","libc",@@ -4181,9 +4177,9 @@ dependencies = [[[package]]name = "tokio-macros"-version = "2.6.1"+version = "2.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"dependencies = ["proc-macro2","quote",@@ -4252,9 +4248,9 @@ version = "0.9.12+spec-1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"dependencies = [- "indexmap 2.13.0",+ "indexmap 2.13.1","serde_core",- "serde_spanned 1.1.0",+ "serde_spanned 1.1.1","toml_datetime 0.7.5+spec-1.1.0","toml_parser","toml_writer",@@ -4285,7 +4281,7 @@ version = "0.22.27"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"dependencies = [- "indexmap 2.13.0",+ "indexmap 2.13.1","serde","serde_spanned 0.6.9","toml_datetime 0.6.11",@@ -4295,11 +4291,11 @@ dependencies = [[[package]]name = "toml_parser"-version = "1.1.0+spec-1.1.0"+version = "1.1.2+spec-1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011"+checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"dependencies = [- "winnow 1.0.0",+ "winnow 1.0.1",][[package]]@@ -4310,9 +4306,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"[[package]]name = "toml_writer"-version = "1.1.0+spec-1.1.0"+version = "1.1.1+spec-1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d282ade6016312faf3e41e57ebbba0c073e4056dab1232ab1cb624199648f8ed"+checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"[[package]]name = "tower"@@ -4588,9 +4584,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"[[package]]name = "uuid"-version = "1.22.0"+version = "1.23.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37"+checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9"dependencies = ["getrandom 0.4.2","js-sys",@@ -4726,9 +4722,9 @@ dependencies = [[[package]]name = "wasm-bindgen"-version = "0.2.114"+version = "0.2.117"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"+checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0"dependencies = ["cfg-if","once_cell",@@ -4739,23 +4735,19 @@ dependencies = [[[package]]name = "wasm-bindgen-futures"-version = "0.4.64"+version = "0.4.67"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e9c5522b3a28661442748e09d40924dfb9ca614b21c00d3fd135720e48b67db8"+checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e"dependencies = [- "cfg-if",- "futures-util","js-sys",- "once_cell","wasm-bindgen",- "web-sys",][[package]]name = "wasm-bindgen-macro"-version = "0.2.114"+version = "0.2.117"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"+checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be"dependencies = ["quote","wasm-bindgen-macro-support",@@ -4763,9 +4755,9 @@ dependencies = [[[package]]name = "wasm-bindgen-macro-support"-version = "0.2.114"+version = "0.2.117"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"+checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2"dependencies = ["bumpalo","proc-macro2",@@ -4776,9 +4768,9 @@ dependencies = [[[package]]name = "wasm-bindgen-shared"-version = "0.2.114"+version = "0.2.117"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"+checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b"dependencies = ["unicode-ident",]@@ -4800,7 +4792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909"dependencies = ["anyhow",- "indexmap 2.13.0",+ "indexmap 2.13.1","wasm-encoder","wasmparser",]@@ -4826,15 +4818,15 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe"dependencies = ["bitflags","hashbrown 0.15.5",- "indexmap 2.13.0",+ "indexmap 2.13.1","semver",][[package]]name = "web-sys"-version = "0.3.91"+version = "0.3.94"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "854ba17bb104abfb26ba36da9729addc7ce7f06f5c0f90f3c391f8461cca21f9"+checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a"dependencies = ["js-sys","wasm-bindgen",@@ -5179,9 +5171,9 @@ dependencies = [[[package]]name = "winnow"-version = "1.0.0"+version = "1.0.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8"+checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5"[[package]]name = "wit-bindgen"@@ -5211,7 +5203,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21"dependencies = ["anyhow","heck",- "indexmap 2.13.0",+ "indexmap 2.13.1","prettyplease","syn 2.0.117","wasm-metadata",@@ -5242,7 +5234,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2"dependencies = ["anyhow","bitflags",- "indexmap 2.13.0",+ "indexmap 2.13.1","log","serde","serde_derive",@@ -5261,7 +5253,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736"dependencies = ["anyhow","id-arena",- "indexmap 2.13.0",+ "indexmap 2.13.1","log","semver","serde",@@ -5273,9 +5265,9 @@ dependencies = [[[package]]name = "writeable"-version = "0.6.2"+version = "0.6.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"+checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"[[package]]name = "xgrammar-rs"@@ -5310,9 +5302,9 @@ dependencies = [[[package]]name = "yoke"-version = "0.8.1"+version = "0.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"+checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"dependencies = ["stable_deref_trait","yoke-derive",@@ -5321,9 +5313,9 @@ dependencies = [[[package]]name = "yoke-derive"-version = "0.8.1"+version = "0.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"+checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"dependencies = ["proc-macro2","quote",@@ -5333,18 +5325,18 @@ dependencies = [[[package]]name = "zerocopy"-version = "0.8.47"+version = "0.8.48"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"+checksum = "eed437bf9d6692032087e337407a86f04cd8d6a16a37199ed57949d415bd68e9"dependencies = ["zerocopy-derive",][[package]]name = "zerocopy-derive"-version = "0.8.47"+version = "0.8.48"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"+checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4"dependencies = ["proc-macro2","quote",@@ -5353,18 +5345,18 @@ dependencies = [[[package]]name = "zerofrom"-version = "0.1.6"+version = "0.1.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"+checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"dependencies = ["zerofrom-derive",][[package]]name = "zerofrom-derive"-version = "0.1.6"+version = "0.1.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"+checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"dependencies = ["proc-macro2","quote",@@ -5380,9 +5372,9 @@ checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"[[package]]name = "zerotrie"-version = "0.2.3"+version = "0.2.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"+checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"dependencies = ["displaydoc","yoke",@@ -5391,9 +5383,9 @@ dependencies = [[[package]]name = "zerovec"-version = "0.11.5"+version = "0.11.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"+checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"dependencies = ["yoke","zerofrom",@@ -5402,9 +5394,9 @@ dependencies = [[[package]]name = "zerovec-derive"-version = "0.11.2"+version = "0.11.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"+checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"dependencies = ["proc-macro2","quote",diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex 20a7b2331..e93754450 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -137,5 +137,5 @@ pub trait CommandBufferPending {pub trait CommandBufferCompleted {type CommandBuffer: CommandBuffer<Completed = Self>;- fn gpu_execution_time(&self) -> Option<Duration>;+ fn gpu_execution_time(&self) -> Duration;}diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rsindex 3a0e74009..efd780d05 100644--- a/crates/uzu/src/backends/common/encoder.rs+++ b/crates/uzu/src/backends/common/encoder.rs@@ -170,7 +170,7 @@ pub struct Completed<B: Backend> {}impl<B: Backend> Completed<B> {- pub fn gpu_execution_time(&self) -> Option<Duration> {+ pub fn gpu_execution_time(&self) -> Duration {self.command_buffer.gpu_execution_time()}}diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rsindex 4fed33635..7d2d8abcb 100644--- a/crates/uzu/src/backends/cpu/command_buffer.rs+++ b/crates/uzu/src/backends/cpu/command_buffer.rs@@ -212,7 +212,7 @@ pub struct CpuCommandBufferCompleted {impl CommandBufferCompleted for CpuCommandBufferCompleted {type CommandBuffer = CpuCommandBuffer;- fn gpu_execution_time(&self) -> Option<Duration> {- Some(self.gpu_execution_time)+ fn gpu_execution_time(&self) -> Duration {+ self.gpu_execution_time}}diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex b2f294885..d8ac4af5d 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -232,10 +232,10 @@ pub struct MetalCommandBufferCompleted {impl CommandBufferCompleted for MetalCommandBufferCompleted {type CommandBuffer = MetalCommandBuffer;- fn gpu_execution_time(&self) -> Option<Duration> {- match (self.command_buffer.gpu_start_time(), self.command_buffer.gpu_end_time()) {- (Some(start), Some(end)) if end > start => Some(Duration::from_secs_f64(end - start)),- _ => None,- }+ fn gpu_execution_time(&self) -> Duration {+ // They're always present, https://developer.apple.com/documentation/metal/mtlcommandbuffer/gpustarttime?language=objc+ let start = self.command_buffer.gpu_start_time();+ let end = self.command_buffer.gpu_end_time();+ Duration::from_secs_f64(end - start)}}diff --git a/crates/uzu/tests/performance/matmul/bench.rs b/crates/uzu/tests/performance/matmul/bench.rsindex ec8481f44..23c7c2b4e 100644--- a/crates/uzu/tests/performance/matmul/bench.rs+++ b/crates/uzu/tests/performance/matmul/bench.rs@@ -63,7 +63,7 @@ fn encode_and_run(let completed = encoder.end_encoding().submit().wait_until_completed().map_err(|_| BenchError::CommandBuffer)?;- completed.gpu_execution_time().map(|duration| duration.as_secs_f64() * 1000.0).ok_or(BenchError::GpuTimestamps)+ Ok(completed.gpu_execution_time().as_secs_f64() * 1000.0)}fn run_benchmark(diff --git a/crates/uzu/tests/performance/matmul/error.rs b/crates/uzu/tests/performance/matmul/error.rsindex dfd690cfe..b06aa8fd7 100644--- a/crates/uzu/tests/performance/matmul/error.rs+++ b/crates/uzu/tests/performance/matmul/error.rs@@ -4,8 +4,6 @@ use thiserror::Error;pub enum BenchError {#[error("failed to create command buffer")]CommandBuffer,- #[error("GPU timestamps unavailable")]- GpuTimestamps,#[error("kernel creation failed: {0}")]Kernel(String),#[error("buffer allocation failed")]diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rsindex 3f08b7e08..f1031a1f1 100644--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs@@ -1047,29 +1047,16 @@ fn perf_two_pass_attention() {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {- Some(gpu_time_ms) => {- println!(- "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",- num_heads,- seq_len - suffix_length,- suffix_length,- head_dim,- gpu_time_ms,- host_elapsed_ms- );- },- None => {- println!(- "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): Host-side={:.2} ms (GPU timing unavailable)",- num_heads,- seq_len - suffix_length,- suffix_length,- head_dim,- host_elapsed_ms- );- },- }+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;+ println!(+ "Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",+ num_heads,+ seq_len - suffix_length,+ suffix_length,+ head_dim,+ gpu_time_ms,+ host_elapsed_ms+ );// ---- Sanity check ----let output_ptr = output_buffer.contents().as_ptr() as *const f32;diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rsindex 4d4fd78aa..723749022 100644--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs@@ -192,20 +192,11 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {- Some(gpu_time_ms) => {- println!(- "Argmax sampling perf (batch={}, vocab={}, strategy={:?}): GPU={:.2} ms, Host-side={:.2} ms",- BATCH, VOCAB, strategy, gpu_time_ms, host_elapsed_ms- );- },- None => {- println!(- "Argmax sampling perf (batch={}, vocab={}, strategy={:?}): Host-side={:.2} ms (GPU timing unavailable)",- BATCH, VOCAB, strategy, host_elapsed_ms- );- },- }+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;+ println!(+ "Argmax sampling perf (batch={}, vocab={}, strategy={:?}): GPU={:.2} ms, Host-side={:.2} ms",+ BATCH, VOCAB, strategy, gpu_time_ms, host_elapsed_ms+ );// Ensure the kernel produced *some* output (sanity).let ptr = output_buf.contents().as_ptr() as *const u32;@@ -548,20 +539,11 @@ fn perf_categorical_128k_vocab() {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {- Some(gpu_time_ms) => {- println!(- "Categorical sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",- BATCH, VOCAB, gpu_time_ms, host_elapsed_ms- );- },- None => {- println!(- "Categorical sampling perf (batch={}, vocab={}): Host-side={:.2} ms (GPU timing unavailable)",- BATCH, VOCAB, host_elapsed_ms- );- },- }+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;+ println!(+ "Categorical sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",+ BATCH, VOCAB, gpu_time_ms, host_elapsed_ms+ );// Sanity checklet ptr = output_buf.contents().as_ptr() as *const u32;diff --git a/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rsindex d2fb5eff9..30f89f124 100644--- a/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs+++ b/crates/uzu/tests/unit/kernel/matmul/gemm_mpp_test.rs@@ -1,6 +1,9 @@#![cfg(metal_backend)]-use std::{ops::{Deref, DerefMut}, rc::Rc};+use std::{+ ops::{Deref, DerefMut},+ rc::Rc,+};use criterion::{BenchmarkId, Criterion, Throughput};use half::bf16;@@ -41,18 +44,10 @@ fn get_test_data(ab_scale: f32,accumulate: bool,) -> Input {- let left: Vec<bf16> = (0..batch_dim * input_dim)- .map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6))- .collect();- let right: Vec<bf16> = (0..output_dim * input_dim)- .map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8))- .collect();+ let left: Vec<bf16> = (0..batch_dim * input_dim).map(|i| bf16::from_f32(((i % 13) as f32) * 0.1 - 0.6)).collect();+ let right: Vec<bf16> = (0..output_dim * input_dim).map(|i| bf16::from_f32(((i % 17) as f32) * 0.1 - 0.8)).collect();let destination_prefill = if accumulate {- Some(- (0..batch_dim * output_dim)- .map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09))- .collect(),- )+ Some((0..batch_dim * output_dim).map(|i| bf16::from_f32(((i % 7) as f32) * 0.03 - 0.09)).collect())} else {None};@@ -168,14 +163,11 @@ fn get_mpp_output(}fn create_metal_context_and_matmul_kernel()- -> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel)-{+-> (Rc<MetalContext>, <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel) {let metal_context = MetalContext::new().expect("Metal context");- let matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(- &metal_context,- bf16::data_type(),- )- .expect("MatmulKernel");+ let matmul_kernel =+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())+ .expect("MatmulKernel");(metal_context, matmul_kernel)}@@ -249,12 +241,8 @@ fn test_bf16_scale_and_accumulate() {// Benchmarks-const BENCHMARK_SHAPES: &[(usize, usize, usize)] = &[- (128, 2048, 8192),- (128, 4096, 14336),- (256, 4096, 4096),- (512, 8192, 2048),-];+const BENCHMARK_SHAPES: &[(usize, usize, usize)] =+ &[(128, 2048, 8192), (128, 4096, 14336), (256, 4096, 4096), (512, 8192, 2048)];#[uzu_bench]fn bench_gemm_mpp(criterion: &mut Criterion) {@@ -263,21 +251,16 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {return;}- let mut matmul_kernel = <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(- &metal_context,- bf16::data_type(),- )- .expect("MatmulKernel");+ let mut matmul_kernel =+ <<Metal as Backend>::Kernels as ManualKernels>::MatmulKernel::new(&metal_context, bf16::data_type())+ .expect("MatmulKernel");- let mut benchmark_group = criterion.benchmark_group(- format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()),- );+ let mut benchmark_group =+ criterion.benchmark_group(format!("{}/Kernel/Matmul/GEMM_MPP", type_short_name::<Metal>()));for &(batch_dim, input_dim, output_dim) in BENCHMARK_SHAPES {- let left_array =- metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");- let right_array =- metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");+ let left_array = metal_context.create_array_uninitialized(&[batch_dim, input_dim], bf16::data_type(), "");+ let right_array = metal_context.create_array_uninitialized(&[output_dim, input_dim], bf16::data_type(), "");let destination_array =metal_context.create_array_uninitialized(&[batch_dim, output_dim], bf16::data_type(), "");@@ -316,13 +299,7 @@ fn bench_gemm_mpp(criterion: &mut Criterion) {);}- encoder- .end_encoding()- .submit()- .wait_until_completed()- .unwrap()- .gpu_execution_time()- .unwrap()+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()})},);diff --git a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rsindex 035dbfeb9..34ab9a9db 100644--- a/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs+++ b/crates/uzu/tests/unit/kernel/quant_matmul/qmv_fast_test.rs@@ -282,25 +282,31 @@ macro_rules! qmv_fast_test {($name:ident, gs=$gs:expr, bits=$bits:expr, zp=$zp:expr, mlx=$mlx:expr) => {#[uzu_test]fn $name() {- for_each_float_type!(|F| { test_basic::<F>($gs, $bits, $zp, $mlx); })+ for_each_float_type!(|F| {+ test_basic::<F>($gs, $bits, $zp, $mlx);+ })}};}-qmv_fast_test!(test_gs32_4bit_zp, gs=32, bits=4, zp=true, mlx=false);-qmv_fast_test!(test_gs64_4bit_zp, gs=64, bits=4, zp=true, mlx=false);-qmv_fast_test!(test_gs128_4bit_zp, gs=128, bits=4, zp=true, mlx=false);-qmv_fast_test!(test_gs32_8bit_zp, gs=32, bits=8, zp=true, mlx=false);-qmv_fast_test!(test_gs64_8bit_zp, gs=64, bits=8, zp=true, mlx=false);-qmv_fast_test!(test_gs128_8bit_zp, gs=128, bits=8, zp=true, mlx=false);-qmv_fast_test!(test_gs32_4bit_mlx, gs=32, bits=4, zp=false, mlx=true);-qmv_fast_test!(test_gs64_4bit_mlx, gs=64, bits=4, zp=false, mlx=true);-qmv_fast_test!(test_gs128_4bit_mlx, gs=128, bits=4, zp=false, mlx=true);-qmv_fast_test!(test_gs32_8bit_mlx, gs=32, bits=8, zp=false, mlx=true);-qmv_fast_test!(test_gs64_8bit_mlx, gs=64, bits=8, zp=false, mlx=true);-qmv_fast_test!(test_gs128_8bit_mlx, gs=128, bits=8, zp=false, mlx=true);--fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(rng: &mut R, range: std::ops::Range<T>, len: usize) -> Box<[T]> {+qmv_fast_test!(test_gs32_4bit_zp, gs = 32, bits = 4, zp = true, mlx = false);+qmv_fast_test!(test_gs64_4bit_zp, gs = 64, bits = 4, zp = true, mlx = false);+qmv_fast_test!(test_gs128_4bit_zp, gs = 128, bits = 4, zp = true, mlx = false);+qmv_fast_test!(test_gs32_8bit_zp, gs = 32, bits = 8, zp = true, mlx = false);+qmv_fast_test!(test_gs64_8bit_zp, gs = 64, bits = 8, zp = true, mlx = false);+qmv_fast_test!(test_gs128_8bit_zp, gs = 128, bits = 8, zp = true, mlx = false);+qmv_fast_test!(test_gs32_4bit_mlx, gs = 32, bits = 4, zp = false, mlx = true);+qmv_fast_test!(test_gs64_4bit_mlx, gs = 64, bits = 4, zp = false, mlx = true);+qmv_fast_test!(test_gs128_4bit_mlx, gs = 128, bits = 4, zp = false, mlx = true);+qmv_fast_test!(test_gs32_8bit_mlx, gs = 32, bits = 8, zp = false, mlx = true);+qmv_fast_test!(test_gs64_8bit_mlx, gs = 64, bits = 8, zp = false, mlx = true);+qmv_fast_test!(test_gs128_8bit_mlx, gs = 128, bits = 8, zp = false, mlx = true);++fn gen_random<T: rand::distr::uniform::SampleUniform + PartialOrd + Copy, R: rand::Rng>(+ rng: &mut R,+ range: std::ops::Range<T>,+ len: usize,+) -> Box<[T]> {(0..len).map(|_| rng.random_range(range.clone())).collect()}@@ -314,29 +320,63 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(use_mlx_quant: bool,) {let mut group = c.benchmark_group(format!("{}/Kernel/QmvFast/{}", type_short_name::<B>(), label));- let block_size: usize = if bits == 4 { 512 } else { 256 };+ let block_size: usize = if bits == 4 {+ 512+ } else {+ 256+ };for (m, n, k) in iproduct!([1, 2, 3, 4], [2048, 4096, 14336], [2048, 4096, 8192, 14336]) {- if n % 8 != 0 || k % block_size != 0 { continue; }+ if n % 8 != 0 || k % block_size != 0 {+ continue;+ }let num_groups = k.div_ceil(group_size as usize);let mut rng = SmallRng::seed_from_u64(42);let kernel = <<B as Backend>::Kernels as Kernels>::QuantizedMatmulQmvFastKernel::new(- context, T::data_type(), group_size, bits, use_zero_points, use_mlx_quant,- ).unwrap();-- let w_buf = alloc_buffer_with_data::<B, u32>(context, &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32));- let scales_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());- let x_buf = alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>());+ context,+ T::data_type(),+ group_size,+ bits,+ use_zero_points,+ use_mlx_quant,+ )+ .unwrap();++ let w_buf = alloc_buffer_with_data::<B, u32>(+ context,+ &gen_random::<u32, _>(&mut rng, 0..u32::MAX, n * k * bits as usize / 32),+ );+ let scales_buf = alloc_buffer_with_data::<B, T>(+ context,+ &gen_random::<f32, _>(&mut rng, 0.01..1.0, n * num_groups)+ .iter()+ .map(|&v| T::from(v).unwrap())+ .collect::<Vec<_>>(),+ );+ let x_buf = alloc_buffer_with_data::<B, T>(+ context,+ &gen_random::<f32, _>(&mut rng, -1.0..1.0, m * k).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>(),+ );let mut y_buf = context.create_buffer(m * n * std::mem::size_of::<T>()).unwrap();let zp_buf = use_zero_points.then(|| {- let zp_stride = if bits == 4 { (num_groups + 1) / 2 } else { num_groups };+ let zp_stride = if bits == 4 {+ (num_groups + 1) / 2+ } else {+ num_groups+ };alloc_buffer_with_data::<B, u8>(context, &gen_random::<u8, _>(&mut rng, 0..u8::MAX, n * zp_stride))});let bias_buf = use_mlx_quant.then(|| {- alloc_buffer_with_data::<B, T>(context, &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups).iter().map(|&v| T::from(v).unwrap()).collect::<Vec<_>>())+ alloc_buffer_with_data::<B, T>(+ context,+ &gen_random::<f32, _>(&mut rng, -0.5..0.5, n * num_groups)+ .iter()+ .map(|&v| T::from(v).unwrap())+ .collect::<Vec<_>>(),+ )});group.throughput(Throughput::Elements((m * n * k) as u64));@@ -344,9 +384,20 @@ fn bench_qmv_fast_typed<B: Backend, T: ArrayElement + Float>(b.iter_custom(|n_iters| {let mut encoder = Encoder::<B>::new(context).unwrap();for _ in 0..n_iters {- kernel.encode(&w_buf, &scales_buf, zp_buf.as_ref(), bias_buf.as_ref(), &x_buf, &mut y_buf, k as u32, n as u32, m as u32, &mut encoder);+ kernel.encode(+ &w_buf,+ &scales_buf,+ zp_buf.as_ref(),+ bias_buf.as_ref(),+ &x_buf,+ &mut y_buf,+ k as u32,+ n as u32,+ m as u32,+ &mut encoder,+ );}- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()})});}diff --git a/crates/uzu/tests/unit/kernel/rms_norm_test.rs b/crates/uzu/tests/unit/kernel/rms_norm_test.rsindex 11e06e464..11a648738 100644--- a/crates/uzu/tests/unit/kernel/rms_norm_test.rs+++ b/crates/uzu/tests/unit/kernel/rms_norm_test.rs@@ -121,7 +121,7 @@ fn get_output<AccumT: ArrayElement + Float,>(input: &Input<InputT, ScaleT, OutputT>-) -> (Vec<OutputT>, f64, Option<Duration>) {+) -> (Vec<OutputT>, f64, Duration) {let context = B::Context::new().expect("Failed to create Context");let kernel = <<B as Backend>::Kernels as Kernels>::RMSNormKernel::new(&context,@@ -461,7 +461,7 @@ fn bench_rms_norm(c: &mut Criterion) {);}- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()})});}diff --git a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rsindex d8b17834a..12afa5a66 100644--- a/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs+++ b/crates/uzu/tests/unit/kernel/sampling/test_argmax.rs@@ -180,7 +180,7 @@ fn bench_argmax(c: &mut Criterion) {);}- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()})});@@ -205,7 +205,7 @@ fn bench_argmax(c: &mut Criterion) {);}- encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time().unwrap()+ encoder.end_encoding().submit().wait_until_completed().unwrap().gpu_execution_time()})});}diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rsindex deaf6745d..30890f2f7 100644--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs@@ -489,27 +489,15 @@ fn perf_topp_128k_vocab() {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {- Some(gpu_time_ms) => {- println!(- "Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",- BATCH,- VOCAB,- std::any::type_name::<B>(),- gpu_time_ms,- host_elapsed_ms- );- },- None => {- println!(- "Top-p sampling perf (batch={}, vocab={}, backend={}): Host-side={:.2} ms (GPU timing unavailable)",- BATCH,- VOCAB,- std::any::type_name::<B>(),- host_elapsed_ms- );- },- }+ let gpu_time_ms = completed.gpu_execution_time().as_secs_f64() * 1e3;+ println!(+ "Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",+ BATCH,+ VOCAB,+ std::any::type_name::<B>(),+ gpu_time_ms,+ host_elapsed_ms+ );let sample_ids: &[u32] = output_array.as_slice();for &tok in sample_ids {