Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Remove KernelDataType (#181)

uuuvna117bfd42684
diff --git a/crates/uzu/build/metal/bindgen.rs b/crates/uzu/build/metal/bindgen.rs
index 951e164b0..89dba2a53 100644
--- a/crates/uzu/build/metal/bindgen.rs
+++ b/crates/uzu/build/metal/bindgen.rs
@@ -34,7 +34,7 @@ pub fn bindgen(
(
variant_names.iter().map(|name| quote! { #[allow(non_snake_case)] #name: crate::DataType }).collect(),
- quote! { &format!(#kernel_format, #entry_name #(, KernelDataType::from(#variant_names).function_name_suffix())*) },
+ quote! { &format!(#kernel_format, #entry_name #(, #variant_names.metal_type())*) },
)
} else {
(Vec::new(), quote! { #entry_name })
diff --git a/crates/uzu/build/metal/compiler.rs b/crates/uzu/build/metal/compiler.rs
index 793eafac9..e32620491 100644
--- a/crates/uzu/build/metal/compiler.rs
+++ b/crates/uzu/build/metal/compiler.rs
@@ -286,7 +286,7 @@ impl MetalCompiler {
use crate::backends::metal::{
ComputeEncoderSetValue,
FunctionConstantValuesSetValue,
- KernelDataType,
+ data_type::MetalDataTypeExt,
MTLContext,
MTLError,
MTLFunctionConstantValues,
diff --git a/crates/uzu/src/backends/metal/data_type.rs b/crates/uzu/src/backends/metal/data_type.rs
new file mode 100644
index 000000000..cf91d58c7
--- /dev/null
+++ b/crates/uzu/src/backends/metal/data_type.rs
@@ -0,0 +1,16 @@
+use crate::DataType;
+
+pub trait MetalDataTypeExt {
+ fn metal_type(&self) -> &'static str;
+}
+
+impl MetalDataTypeExt for DataType {
+ fn metal_type(&self) -> &'static str {
+ match self {
+ DataType::F16 => "half",
+ DataType::BF16 => "bfloat",
+ DataType::F32 => "float",
+ _ => panic!("Unsupported data type: {0:?}", self),
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/metal/encodable_block/attention.rs b/crates/uzu/src/backends/metal/encodable_block/attention.rs
index 31eb40078..b37c35821 100644
--- a/crates/uzu/src/backends/metal/encodable_block/attention.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/attention.rs
@@ -2,9 +2,9 @@
use super::{EncodableBlock, Metal};
use crate::{
+ DataType,
backends::metal::{
- KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject,
- Retained,
+ MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, ProtocolObject, Retained,
kernel::attention::{
AttentionError, AttentionGemmArguments, AttentionKernel, AttentionKernelVariant,
AttentionSinglePassArguments, AttentionTwoPassArguments, KVCacheUpdateArguments,
@@ -42,7 +42,7 @@ pub struct Attention {
impl Attention {
pub fn new(
context: &MTLContext,
- data_type: KernelDataType,
+ data_type: DataType,
layer_index: usize,
attention_scale: Option<f32>,
has_sinks: bool,
diff --git a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
index d3eddbcdf..d548afdb6 100644
--- a/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/classifier_layer.rs
@@ -9,7 +9,7 @@ use crate::{
DataType,
backends::metal::{
MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, Metal, ProtocolObject, Retained,
- compilation_parameters::CompilationConfig, kernel::KernelDataType,
+ compilation_parameters::CompilationConfig,
},
config::TransformerLayerConfig,
encodable_block::EncodingParameters,
@@ -56,7 +56,6 @@ impl ClassifierLayer {
let attention_config =
layer_config.mixer_config.as_attention().expect("Classifier layers must use attention");
let intermediate_data_type: DataType = attention_config.qkv_projection_config.activation_precision().into();
- let kernel_data_type: KernelDataType = intermediate_data_type.into();
let copy_main_to_shortcut_mixer: Box<dyn EncodableBlock<Metal>> = Box::new(
TensorCopy::<Metal>::new(
@@ -208,7 +207,7 @@ impl ClassifierLayer {
let attention: Box<dyn EncodableBlock<Metal>> = Box::new(
Attention::new(
ctx,
- kernel_data_type,
+ intermediate_data_type,
layer_index,
attention_scale,
attention_config.has_sinks,
diff --git a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
index a8c1a96fc..8955a3296 100644
--- a/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/layer/executables.rs
@@ -17,7 +17,7 @@ use crate::{
DataType, DecoderLayerConfig,
backends::metal::{
MTLCommandBuffer, MTLComputeCommandEncoder, MTLContext, Metal, ProtocolObject, Retained,
- compilation_parameters::CompilationConfig, kernel::KernelDataType,
+ compilation_parameters::CompilationConfig,
},
config::{DecoderLayerType, MixerConfig},
encodable_block::EncodingParameters,
@@ -62,8 +62,6 @@ impl LayerExecutables {
MixerConfig::Mamba(mamba) => mamba.in_projection_config.activation_precision().into(),
MixerConfig::ShortConv(short_conv) => short_conv.in_projection_config.activation_precision().into(),
};
- let kernel_data_type: KernelDataType = intermediate_data_type.into();
-
let copy_main_to_shortcut: Box<dyn EncodableBlock<Metal>> = Box::new(
TensorCopy::<Metal>::new(
ctx,
@@ -136,7 +134,7 @@ impl LayerExecutables {
let attention = Box::new(
Attention::new(
ctx,
- kernel_data_type,
+ intermediate_data_type,
layer_index,
attention_scale,
attention_config.has_sinks,
diff --git a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
index 0472d34d1..578ab69e8 100644
--- a/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/mamba_mixer.rs
@@ -8,8 +8,8 @@ use crate::{
backends::{
common::kernel::{SSDUpdateKernel, SplitInProjKernel},
metal::{
- KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray,
- ProtocolObject, Retained,
+ MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
+ Retained,
compilation_parameters::CompilationConfig,
kernel::{
dsl::{SSDUpdateMetalKernel, SplitInProjMetalKernel},
@@ -65,7 +65,6 @@ impl MambaMixer {
let conv_tree = resolve_subtree(&split_tree, &["conv", "conv1d"]);
let data_type: DataType = mamba_config.in_projection_config.activation_precision().into();
- let kernel_data_type: KernelDataType = data_type.into();
let in_projection = transformer_layer::linear_block(
&mamba_config.in_projection_config,
@@ -100,14 +99,12 @@ impl MambaMixer {
let gate_bias = split_tree.leaf("gate_bias").unwrap().clone();
let skip_connection_weight = split_tree.leaf("skip_connection_weight").unwrap().clone();
- let split_inproj = SplitInProjMetalKernel::new(mtl_context, kernel_data_type.into())
- .expect("Failed to create split in-projection kernel");
- let conv_scan = Conv1dScanKernel::new(mtl_context, kernel_data_type, &mamba_config.activation)
+ let split_inproj =
+ SplitInProjMetalKernel::new(mtl_context, data_type).expect("Failed to create split in-projection kernel");
+ let conv_scan = Conv1dScanKernel::new(mtl_context, data_type, &mamba_config.activation)
.expect("Failed to create conv scan kernel");
- let ssd_prefill =
- SSDPrefillKernels::new(mtl_context, kernel_data_type).expect("Failed to create SSD prefill kernel");
- let ssd_update = SSDUpdateMetalKernel::new(mtl_context, kernel_data_type.into())
- .expect("Failed to create SSD decode kernel");
+ let ssd_prefill = SSDPrefillKernels::new(mtl_context, data_type).expect("Failed to create SSD prefill kernel");
+ let ssd_update = SSDUpdateMetalKernel::new(mtl_context, data_type).expect("Failed to create SSD decode kernel");
let prefill_mode = resolve_prefill_mode_from_env();
Self {
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
index 9baeff4ff..8580b9226 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -11,8 +11,8 @@ use crate::{
backends::{
common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel},
metal::{
- KernelDataType, MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
- MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject, Retained,
+ MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder,
+ MTLContext, MetalArray, ProtocolObject, Retained,
kernel::{
MoeGatherKernels,
dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
@@ -38,7 +38,7 @@ enum RouterBlock {
pub struct MoeBlock {
router: RouterBlock,
- router_data_type: KernelDataType,
+ router_data_type: DataType,
router_renorm: bool,
router_topk_kernel: MoeRouterTopKKernel,
counts_offsets_kernel: MoeCountsOffsetsFusedMetalKernel,
@@ -50,7 +50,7 @@ pub struct MoeBlock {
moe_config: MixtureOfExpertsConfig,
model_dim: usize,
hidden_dim: usize,
- data_type: KernelDataType,
+ data_type: DataType,
shared_weights: SharedMoeWeights,
}
@@ -62,11 +62,9 @@ impl MoeBlock {
hidden_dim: usize,
parameter_tree: &ParameterTree<MTLContext>,
) -> Result<Self, crate::backends::metal::MTLError> {
- let activation_data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();
- let data_type: KernelDataType = activation_data_type.into();
+ let data_type: DataType = moe_config.expert_config.linear_config.activation_precision().into();
let router_data_type: DataType = moe_config.router_config.activation_precision().into();
- let router_kernel_data_type: KernelDataType = router_data_type.into();
let router_renorm = matches!(moe_config.routing_function, RoutingFunctionConfig::SoftmaxRouting);
@@ -121,7 +119,7 @@ impl MoeBlock {
let experts_two_pass_prefill_kernel = MoeExpertsTwoPassPrefillKernel::new(context).map_err(|e| {
crate::backends::metal::MTLError::Generic(format!("Experts two-pass prefill kernel error: {:?}", e))
})?;
- let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type.into())
+ let finalize_kernel = MoeFinalizeMetalKernel::new(context, data_type)
.map_err(|e| crate::backends::metal::MTLError::Generic(format!("Finalize kernel error: {:?}", e)))?;
let experts_tree = parameter_tree
@@ -165,7 +163,7 @@ impl MoeBlock {
Ok(Self {
router,
- router_data_type: router_kernel_data_type,
+ router_data_type,
router_renorm,
router_topk_kernel,
counts_offsets_kernel,
@@ -260,11 +258,6 @@ impl EncodableBlock<Metal> for MoeBlock {
let k_tile = 128;
// Clear internal MoE buffers
- let dtype_size = match self.data_type {
- KernelDataType::BFloat16 | KernelDataType::Float16 => 2,
- KernelDataType::Float32 => 4,
- };
-
let blit_encoder = root.new_blit_command_encoder().expect("Failed to create blit command encoder");
if suffix_length > 0 && k > 0 {
@@ -281,19 +274,19 @@ impl EncodableBlock<Metal> for MoeBlock {
}
// Clear hidden buffer
- let hidden_bytes = suffix_length * k * self.hidden_dim * dtype_size;
+ let hidden_bytes = suffix_length * k * self.hidden_dim * self.data_type.size_in_bytes();
if hidden_bytes > 0 {
blit_encoder.fill_buffer_range_value(&hidden_buf, 0..hidden_bytes, 0);
}
// Clear y_partial buffer
- let y_partial_bytes = suffix_length * k * self.model_dim * dtype_size;
+ let y_partial_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
if y_partial_bytes > 0 {
blit_encoder.fill_buffer_range_value(&y_partial_buf, 0..y_partial_bytes, 0);
}
// Clear x_perm buffer
- let x_perm_bytes = suffix_length * k * self.model_dim * dtype_size;
+ let x_perm_bytes = suffix_length * k * self.model_dim * self.data_type.size_in_bytes();
if x_perm_bytes > 0 {
blit_encoder.fill_buffer_range_value(&x_perm_buf, 0..x_perm_bytes, 0);
}
diff --git a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
index c5e484e51..5dca379e9 100644
--- a/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/short_conv_mixer.rs
@@ -6,8 +6,8 @@ use crate::{
backends::{
common::Context,
metal::{
- KernelDataType, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray,
- ProtocolObject, Retained,
+ MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLContext, MetalArray, ProtocolObject,
+ Retained,
compilation_parameters::CompilationConfig,
kernel::short_conv::{
ShortConvDecodeArguments, ShortConvKernel, ShortConvPackArguments, ShortConvPrefillArguments,
@@ -63,7 +63,6 @@ impl ShortConvMixer {
let conv_tree = resolve_subtree(&mixer_tree, &["conv"]);
let data_type: DataType = short_conv_config.in_projection_config.activation_precision().into();
- let kernel_data_type: KernelDataType = data_type.into();
let in_projection = transformer_layer::linear_block(
&short_conv_config.in_projection_config,
@@ -97,7 +96,7 @@ impl ShortConvMixer {
};
let short_conv_kernel =
- ShortConvKernel::new(mtl_context, kernel_data_type).expect("Failed to create short conv kernel");
+ ShortConvKernel::new(mtl_context, data_type).expect("Failed to create short conv kernel");
Self {
layer_index,
diff --git a/crates/uzu/src/backends/metal/kernel/attention/mod.rs b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
index 8a91a5f3c..51a48d9d4 100644
--- a/crates/uzu/src/backends/metal/kernel/attention/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/attention/mod.rs
@@ -3,11 +3,15 @@ use std::{collections::HashMap, mem::size_of, ptr::NonNull};
use objc2::rc::Retained;
use thiserror::Error;
-use crate::backends::{
- common::gpu_types::{AttnMaskParams, AttnParams},
- metal::{
- FunctionConstantValuesSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject,
+use crate::{
+ DataType,
+ backends::{
+ common::gpu_types::{AttnMaskParams, AttnParams},
+ metal::{
+ FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
+ MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, data_type::MetalDataTypeExt,
+ kernel::moe::dtype_suffix,
+ },
},
};
@@ -27,7 +31,7 @@ pub struct AttentionKernelPipelines {
}
pub struct AttentionKernel {
- data_type: KernelDataType,
+ data_type: DataType,
pipelines: AttentionKernelPipelines,
}
@@ -145,9 +149,9 @@ fn make_function_constants(
impl AttentionKernel {
pub fn new(
context: &MTLContext,
- data_type: KernelDataType,
+ data_type: DataType,
) -> Result<Self, AttentionError> {
- let data_suffix = data_type.function_name_suffix();
+ let data_suffix = data_type.metal_type();
let supported_head_dims = [64, 128, 256];
let mut single_pass = HashMap::new();
@@ -579,11 +583,7 @@ impl AttentionKernel {
// Kernel name matches gemm_attention.metal instantiations:
// attention_gemm_{f16|bf16|f32}_{head_dim}_bk{bk}
- let type_name = match self.data_type {
- KernelDataType::Float16 => "f16",
- KernelDataType::BFloat16 => "bf16",
- KernelDataType::Float32 => "f32",
- };
+ let type_name = dtype_suffix(self.data_type);
let function_name = format!("attention_gemm_{}_{}_bk{}", type_name, args.head_dim, bk);
diff --git a/crates/uzu/src/backends/metal/kernel/data_type.rs b/crates/uzu/src/backends/metal/kernel/data_type.rs
deleted file mode 100644
index 1237a4185..000000000
--- a/crates/uzu/src/backends/metal/kernel/data_type.rs
+++ /dev/null
@@ -1,39 +0,0 @@
-use crate::DataType;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum KernelDataType {
- Float32,
- Float16,
- BFloat16,
-}
-
-impl KernelDataType {
- pub fn function_name_suffix(&self) -> &'static str {
- match self {
- KernelDataType::Float32 => "float",
- KernelDataType::Float16 => "half",
- KernelDataType::BFloat16 => "bfloat",
- }
- }
-}
-
-impl From<KernelDataType> for DataType {
- fn from(val: KernelDataType) -> Self {
- match val {
- KernelDataType::BFloat16 => DataType::BF16,
- KernelDataType::Float16 => DataType::F16,
- KernelDataType::Float32 => DataType::F32,
- }
- }
-}
-
-impl From<DataType> for KernelDataType {
- fn from(dtype: DataType) -> Self {
- match dtype {
- DataType::BF16 => KernelDataType::BFloat16,
- DataType::F16 => KernelDataType::Float16,
- DataType::F32 => KernelDataType::Float32,
- _ => panic!("Unsupported data type: {0:?}", dtype),
- }
- }
-}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index 202e9b4d7..f46e41e24 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,5 +1,4 @@
pub mod attention;
-mod data_type;
use crate::backends::common::kernel::matmul::MatmulKernels;
pub mod dsl {
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
@@ -22,7 +21,6 @@ pub use attention::{
AttentionError, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments, AttentionTwoPassArguments,
KVCacheUpdateArguments,
};
-pub use data_type::KernelDataType;
pub use matmul::{MatmulArguments, MatmulKernel};
pub use moe::{
MoeBlockBasesArguments, MoeExpertsArguments, MoeExpertsError, MoeExpertsTwoPassArguments,
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts.rs b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
index 7de2161bf..fc5d21e19 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts.rs
@@ -1,13 +1,16 @@
use std::{mem::size_of, ptr::NonNull};
use super::{dtype_index, dtype_suffix};
-use crate::backends::metal::{
- FunctionConstantValuesSetValue, KernelDataType, MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer,
- MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError,
- MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
- kernel::moe::tiles_map::{
- MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,
- MoeTileScanArguments,
+use crate::{
+ DataType,
+ backends::metal::{
+ FunctionConstantValuesSetValue, MTLBlitCommandEncoderExt, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
+ MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues, MTLSize,
+ ProtocolObject, Retained,
+ kernel::moe::tiles_map::{
+ MoeTileCountsArguments, MoeTileDispatchArguments, MoeTileMapBuildArguments, MoeTileMapKernels,
+ MoeTileScanArguments,
+ },
},
};
@@ -50,7 +53,7 @@ pub struct MoeExpertsArguments<'a> {
pub up_clip_min: f32,
pub up_clip_max: f32,
pub silu_alpha: f32,
- pub data_type: KernelDataType,
+ pub data_type: DataType,
}
pub struct MoeExpertsTwoPassPrefillKernel {
@@ -86,12 +89,12 @@ pub struct MoeExpertsTwoPassArguments<'a> {
pub up_clip_min: f32,
pub up_clip_max: f32,
pub silu_alpha: f32,
- pub data_type: KernelDataType,
+ pub data_type: DataType,
}
impl MoeExpertsTwoPassPrefillKernel {
pub fn new(ctx: &MTLContext) -> Result<Self, MoeExpertsError> {
- let dtypes = [KernelDataType::Float16, KernelDataType::BFloat16, KernelDataType::Float32];
+ let dtypes = [DataType::F16, DataType::BF16, DataType::F32];
let mut pass_a_indirect = vec![Vec::with_capacity(dtypes.len()); 4];
for gate in 0u32..4u32 {
for dtype in &dtypes {
@@ -130,11 +133,7 @@ impl MoeExpertsTwoPassPrefillKernel {
}
let gate_idx = (args.gating_code.min(3)) as usize;
let dtype_idx = dtype_index(args.data_type);
- let dtype_size = match args.data_type {
- KernelDataType::BFloat16 | KernelDataType::Float16 => 2,
- KernelDataType::Float32 => 4,
- };
- let hidden_bytes = args.total_rows * args.d_ff * dtype_size;
+ let hidden_bytes = args.total_rows * args.d_ff * args.data_type.size_in_bytes();
let blit_encoder = command_buffer.new_blit_command_encoder().expect("Failed to create blit command encoder");
blit_encoder.fill_buffer_range_value(args.hidden_buffer, 0..hidden_bytes, 0);
blit_encoder.end_encoding();
diff --git a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs b/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
index 5ebb1ec6b..586833efd 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/experts_single.rs
@@ -1,15 +1,18 @@
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder};
use objc2::{__framework_prelude::ProtocolObject, Message, rc::Retained};
-use crate::backends::{
- common::kernel::{MoeExpertsDecodeSinglePassAKernel as _, MoeExpertsDecodeSinglePassBKernel as _},
- metal::{
- KernelDataType, MTLContext, MTLError,
- kernel::dsl::{MoeExpertsDecodeSinglePassAMetalKernel, MoeExpertsDecodeSinglePassBMetalKernel},
+use crate::{
+ DataType,
+ backends::{
+ common::kernel::{MoeExpertsDecodeSinglePassAKernel as _, MoeExpertsDecodeSinglePassBKernel as _},
+ metal::{
+ MTLContext, MTLError,
+ kernel::dsl::{MoeExpertsDecodeSinglePassAMetalKernel, MoeExpertsDecodeSinglePassBMetalKernel},
+ },
},
};
-static DTYPES: [KernelDataType; 3] = [KernelDataType::Float16, KernelDataType::BFloat16, KernelDataType::Float32];
+static DTYPES: [DataType; 3] = [DataType::F16, DataType::BF16, DataType::F32];
/// Arguments for single-token MoE decode (T=1 optimized path)
#[derive(Debug)]
@@ -51,7 +54,7 @@ pub struct MoeExpertsSingleDecodeArguments<'a> {
/// Up clipping max
pub up_clip_max: f32,
/// Data type
- pub data_type: KernelDataType,
+ pub data_type: DataType,
}
pub struct MoeExpertsSingleDecodeKernels {
diff --git a/crates/uzu/src/backends/metal/kernel/moe/gather.rs b/crates/uzu/src/backends/metal/kernel/moe/gather.rs
index 977e7d674..eebffe8d9 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/gather.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/gather.rs
@@ -5,8 +5,7 @@ use crate::{
backends::{
common::kernel::{MoeGatherXPerm1DKernel, MoeGatherXPerm2DKernel},
metal::{
- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError, ProtocolObject,
- Retained,
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext, MTLError, ProtocolObject, Retained,
kernel::dsl::{MoeGatherXPerm1DMetalKernel, MoeGatherXPerm2DMetalKernel},
},
},
@@ -41,12 +40,12 @@ impl MoeGatherKernels {
pub fn encode(
&self,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- dtype: KernelDataType,
+ dtype: DataType,
args: &MoeGatherArguments,
) {
let encoder = command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
match dtype {
- KernelDataType::Float32 => self.f32.encode(
+ DataType::F32 => self.f32.encode(
&args.x_buffer.retain(),
&args.bucketed_ids_buffer.retain(),
&args.x_perm_buffer.retain(),
@@ -56,7 +55,7 @@ impl MoeGatherKernels {
args.k as u32,
&encoder,
),
- KernelDataType::Float16 => self.f16.encode(
+ DataType::F16 => self.f16.encode(
&args.x_buffer.retain(),
&args.bucketed_ids_buffer.retain(),
&args.x_perm_buffer.retain(),
@@ -66,7 +65,7 @@ impl MoeGatherKernels {
args.k as u32,
&encoder,
),
- KernelDataType::BFloat16 => self.bf16.encode(
+ DataType::BF16 => self.bf16.encode(
&args.x_buffer.retain(),
&args.bucketed_ids_buffer.retain(),
&args.x_perm_buffer.retain(),
@@ -76,6 +75,7 @@ impl MoeGatherKernels {
args.k as u32,
&encoder,
),
+ _ => panic!("Unsupported data type: {:?}", dtype),
}
encoder.end_encoding();
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/mod.rs b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
index 5981a3a92..d868ace3a 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/mod.rs
@@ -1,4 +1,4 @@
-use crate::backends::metal::KernelDataType;
+use crate::DataType;
// Submodules
mod experts;
@@ -25,18 +25,20 @@ pub use tiles_pass_a::{
};
// Common utility functions
-pub(crate) fn dtype_suffix(dtype: KernelDataType) -> &'static str {
+pub(crate) fn dtype_suffix(dtype: DataType) -> &'static str {
match dtype {
- KernelDataType::Float16 => "f16",
- KernelDataType::BFloat16 => "bf16",
- KernelDataType::Float32 => "f32",
+ DataType::F16 => "f16",
+ DataType::BF16 => "bf16",
+ DataType::F32 => "f32",
+ _ => panic!("Unsupported data type: {:?}", dtype),
}
}
-pub(crate) fn dtype_index(dtype: KernelDataType) -> usize {
+pub(crate) fn dtype_index(dtype: DataType) -> usize {
match dtype {
- KernelDataType::Float16 => 0,
- KernelDataType::BFloat16 => 1,
- KernelDataType::Float32 => 2,
+ DataType::F16 => 0,
+ DataType::BF16 => 1,
+ DataType::F32 => 2,
+ _ => panic!("Unsupported data type: {:?}", dtype),
}
}
diff --git a/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs b/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs
index 8c30769fd..adfa009aa 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/router_topk.rs
@@ -1,6 +1,9 @@
-use crate::backends::metal::{
- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLError, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
+ MTLError, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
+ },
};
const THREADS_PER_THREADGROUP: usize = 256;
@@ -55,7 +58,7 @@ impl MoeRouterTopKKernel {
pub fn encode(
&self,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
- dtype: KernelDataType,
+ dtype: DataType,
args: MoeRouterTopKArguments,
) -> Result<(), MoeRouterTopKError> {
if args.t == 0 || args.e == 0 || args.k == 0 {
@@ -81,9 +84,10 @@ impl MoeRouterTopKKernel {
let compute_encoder =
command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
match dtype {
- KernelDataType::Float16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f16),
- KernelDataType::Float32 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f32),
- KernelDataType::BFloat16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_bf16),
+ DataType::F16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f16),
+ DataType::F32 => compute_encoder.set_compute_pipeline_state(&self.pipeline_f32),
+ DataType::BF16 => compute_encoder.set_compute_pipeline_state(&self.pipeline_bf16),
+ _ => panic!("Unsupported data type: {:?}", dtype),
}
compute_encoder.set_buffer(Some(args.input_buffer), 0, 0);
diff --git a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs b/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
index bc55f413a..9de76dcf6 100644
--- a/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
+++ b/crates/uzu/src/backends/metal/kernel/moe/scatter.rs
@@ -1,6 +1,9 @@
-use crate::backends::metal::{
- KernelDataType, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
+use crate::{
+ DataType,
+ backends::metal::{
+ MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
+ MTLSize, ProtocolObject, Retained, metal_extensions::ComputeEncoderSetValue,
+ },
};
// ---- Scatter Buckets Kernels ----
@@ -110,21 +113,22 @@ impl MoeScatterKernels {
&self,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
args: MoeScatterArguments,
- dtype: KernelDataType,
+ dtype: DataType,
) -> Result<(), MoeScatterError> {
let compute_encoder =
command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
// Select pipeline based on dtype
match dtype {
- KernelDataType::Float16 => {
+ DataType::F16 => {
compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f16);
},
- KernelDataType::Float32 => {
+ DataType::F32 => {
compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_f32);
},
- KernelDataType::BFloat16 => {
+ DataType::BF16 => {
compute_encoder.set_compute_pipeline_state(&self.pipeline_scatter_bf16);
},
+ _ => panic!("Unsupported data type: {:?}", dtype),
}
compute_encoder.set_buffer(Some(args.topk_ids_buffer), 0, 0);
compute_encoder.set_buffer(Some(args.topk_probs_buffer), 0, 1);
@@ -157,14 +161,15 @@ impl MoeScatterKernels {
&self,
command_buffer: &Retained<ProtocolObject<dyn MTLCommandBuffer>>,
args: MoeScatterWithMapArguments,
- dtype: KernelDataType,
+ dtype: DataType,
) -> Result<(), MoeScatterError> {
let compute_encoder =
command_buffer.new_compute_command_encoder().expect("Failed to create compute command encoder");
let pipeline = match dtype {
- KernelDataType::Float16 => &self.pipeline_scatter_map_f16,
- KernelDataType::Float32 => &self.pipeline_scatter_map_f32,
- KernelDataType::BFloat16 => &self.pipeline_scatter_map_bf16,
+ DataType::F16 => &self.pipeline_scatter_map_f16,
+ DataType::F32 => &self.pipeline_scatter_map_f32,
+ DataType::BF16 => &self.pipeline_scatter_map_bf16,
+ _ => panic!("Unsupported data type: {:?}", dtype),
};
compute_encoder.set_compute_pipeline_state(pipeline);
let base = &args.base;
diff --git a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
index fb63af9c1..e0ee7dc4e 100644
--- a/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/short_conv/mod.rs
@@ -1,6 +1,10 @@
-use crate::backends::metal::{
- ComputeEncoderSetValue, FunctionConstantValuesSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder,
- MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
+use crate::{
+ DataType,
+ backends::metal::{
+ ComputeEncoderSetValue, FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder,
+ MTLComputePipelineState, MTLContext, MTLError, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
+ data_type::MetalDataTypeExt,
+ },
};
#[derive(Debug, thiserror::Error)]
@@ -9,10 +13,6 @@ pub enum ShortConvKernelError {
MetalError(#[from] MTLError),
}
-fn fn_suffix(dt: KernelDataType) -> &'static str {
- dt.function_name_suffix()
-}
-
fn make_function_constants(has_bias: bool) -> Retained<MTLFunctionConstantValues> {
let function_constants = MTLFunctionConstantValues::new();
function_constants.set_value(&has_bias, 0);
@@ -95,12 +95,12 @@ pub struct ShortConvTrieArguments<'a> {
impl ShortConvKernel {
pub fn new(
context: &MTLContext,
- data_type: KernelDataType,
+ data_type: DataType,
) -> Result<Self, ShortConvKernelError> {
- let pack_name = format!("short_conv_pack_kernel_{}", fn_suffix(data_type));
- let prefill_name = format!("short_conv_prefill_kernel_{}", fn_suffix(data_type));
- let decode_name = format!("short_conv_decode_kernel_{}", fn_suffix(data_type));
- let trie_name = format!("short_conv_trie_kernel_{}", fn_suffix(data_type));
+ let pack_name = format!("short_conv_pack_kernel_{}", data_type.metal_type());
+ let prefill_name = format!("short_conv_prefill_kernel_{}", data_type.metal_type());
+ let decode_name = format!("short_conv_decode_kernel_{}", data_type.metal_type());
+ let trie_name = format!("short_conv_trie_kernel_{}", data_type.metal_type());
let pack_pipeline =
context.compute_pipeline_state(&pack_name, None).map_err(ShortConvKernelError::MetalError)?;
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
index ff0427c0a..6056644ba 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d_scan.rs
@@ -1,8 +1,9 @@
-use super::{SSMKernelError, fn_suffix};
+use super::SSMKernelError;
use crate::{
+ DataType,
backends::metal::{
- FunctionConstantValuesSetValue, KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState,
- MTLContext, MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained,
+ FunctionConstantValuesSetValue, MTLBuffer, MTLComputeCommandEncoder, MTLComputePipelineState, MTLContext,
+ MTLFunctionConstantValues, MTLSize, ProtocolObject, Retained, data_type::MetalDataTypeExt,
metal_extensions::ComputeEncoderSetValue,
},
config::Activation,
@@ -93,12 +94,12 @@ pub struct Conv1dDecodeArguments<'a> {
impl Conv1dScanKernel {
pub fn new(
context: &MTLContext,
- data_type: KernelDataType,
+ data_type: DataType,
activation: &Activation,
) -> Result<Self, SSMKernelError> {
- let fn_name = format!("conv1d_scan_kernel_{}", fn_suffix(data_type));
- let pack_name = format!("conv1d_pack_prefix_kernel_{}", fn_suffix(data_type));
- let decode_name = format!("conv1d_decode_kernel_{}", fn_suffix(data_type));
+ let fn_name = format!("conv1d_scan_kernel_{}", data_type.metal_type());
+ let pack_name = format!("conv1d_pack_prefix_kernel_{}", data_type.metal_type());
+ let decode_name = format!("conv1d_decode_kernel_{}", data_type.metal_type());
let activation_id = activation_to_int(activation);
let pipeline_no_bias = {
let function_constants = make_function_constants(activation, false);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
index e21c5cad0..c64f8af63 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/mod.rs
@@ -1,4 +1,4 @@
-use crate::backends::metal::{KernelDataType, MTLError};
+use crate::backends::metal::MTLError;
#[derive(Debug, thiserror::Error)]
pub enum SSMKernelError {
@@ -8,10 +8,6 @@ pub enum SSMKernelError {
FunctionNotFound(String),
}
-fn fn_suffix(dt: KernelDataType) -> &'static str {
- dt.function_name_suffix()
-}
-
pub mod conv1d_scan;
pub mod ssd_prefill;
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
index 6fc06c806..ae7993b92 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.rs
@@ -1,10 +1,13 @@
use objc2::Message;
-use crate::backends::{
- common::kernel::{SSDPrefill64Kernel as _, SSDPrefillKernel as _, SSDPrefillSequentialKernel as _},
- metal::{
- KernelDataType, MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
- kernel::dsl::{SSDPrefill64MetalKernel, SSDPrefillMetalKernel, SSDPrefillSequentialMetalKernel},
+use crate::{
+ DataType,
+ backends::{
+ common::kernel::{SSDPrefill64Kernel as _, SSDPrefillKernel as _, SSDPrefillSequentialKernel as _},
+ metal::{
+ MTLBuffer, MTLComputeCommandEncoder, MTLContext, MTLError, ProtocolObject,
+ kernel::dsl::{SSDPrefill64MetalKernel, SSDPrefillMetalKernel, SSDPrefillSequentialMetalKernel},
+ },
},
};
@@ -43,11 +46,11 @@ pub struct SSDPrefillKernels {
impl SSDPrefillKernels {
pub fn new(
context: &MTLContext,
- data_type: KernelDataType,
+ data_type: DataType,
) -> Result<Self, MTLError> {
- let single = SSDPrefillMetalKernel::new(context, data_type.into())?;
- let single_64 = SSDPrefill64MetalKernel::new(context, data_type.into())?;
- let sequential = SSDPrefillSequentialMetalKernel::new(context, data_type.into())?;
+ let single = SSDPrefillMetalKernel::new(context, data_type)?;
+ let single_64 = SSDPrefill64MetalKernel::new(context, data_type)?;
+ let sequential = SSDPrefillSequentialMetalKernel::new(context, data_type)?;
Ok(Self {
single,
single_64,
diff --git a/crates/uzu/src/backends/metal/language_model_generator_context.rs b/crates/uzu/src/backends/metal/language_model_generator_context.rs
index 73b1fe021..6fcdafb97 100644
--- a/crates/uzu/src/backends/metal/language_model_generator_context.rs
+++ b/crates/uzu/src/backends/metal/language_model_generator_context.rs
@@ -6,7 +6,7 @@ use std::{
rc::Rc,
};
-use super::{Decoder, KVCacheUpdate, KernelDataType, MTLContext, Metal, compilation_parameters::CompilationConfig};
+use super::{Decoder, KVCacheUpdate, MTLContext, Metal, compilation_parameters::CompilationConfig};
use crate::{
DataType,
backends::{
@@ -206,7 +206,6 @@ impl LanguageModelGeneratorContext {
)));
let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();
- let kernel_data_type: KernelDataType = intermediate_data_type.into();
let kv_cache_update = Box::new(
KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)
.map_err(|_| Error::UnableToCreateMetalContext)?,
@@ -224,7 +223,7 @@ impl LanguageModelGeneratorContext {
// Create mask update kernel if model has attention layers
let mask_update = if decoder_config.has_attention_layers() {
Some(
- MaskUpdateMetalKernel::new(&context, kernel_data_type.into())
+ MaskUpdateMetalKernel::new(&context, intermediate_data_type)
.map_err(|_| Error::UnableToCreateMetalContext)?,
)
} else {
diff --git a/crates/uzu/src/backends/metal/mod.rs b/crates/uzu/src/backends/metal/mod.rs
index 93c1c5306..5aed4dd6f 100644
--- a/crates/uzu/src/backends/metal/mod.rs
+++ b/crates/uzu/src/backends/metal/mod.rs
@@ -6,6 +6,7 @@ mod command_buffer;
pub mod compilation_parameters;
mod context;
mod copy_encoder;
+mod data_type;
pub mod encodable_block;
pub mod error;
pub mod kernel;
@@ -17,7 +18,7 @@ pub use classifier_context::ClassifierContext;
pub use context::{DeviceArchitecture, DeviceClass, DeviceGeneration, MTLContext};
pub use encodable_block::Decoder;
pub use error::MTLError;
-pub use kernel::{KernelDataType, dsl::MetalKernels};
+pub use kernel::dsl::MetalKernels;
pub use language_model_generator_context::LanguageModelGeneratorContext;
pub use metal::prelude::*;
pub use metal_extensions::{ComputeEncoderSetValue, FunctionConstantValuesSetValue};
diff --git a/crates/uzu/tests/attention_kernel_test.rs b/crates/uzu/tests/attention_kernel_test.rs
index d8383a721..c2a0cfb02 100644
--- a/crates/uzu/tests/attention_kernel_test.rs
+++ b/crates/uzu/tests/attention_kernel_test.rs
@@ -7,15 +7,18 @@ use std::mem::size_of;
use bytemuck;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
use ndarray::{Array3, Array4, s};
-use uzu::backends::{
- common::Context,
- metal::{
- KernelDataType, MTLContext, ProtocolObject, Retained,
- kernel::attention::{
- AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,
- AttentionTwoPassArguments,
+use uzu::{
+ DataType,
+ backends::{
+ common::Context,
+ metal::{
+ MTLContext, ProtocolObject, Retained,
+ kernel::attention::{
+ AttentionGemmArguments, AttentionKernel, AttentionKernelVariant, AttentionSinglePassArguments,
+ AttentionTwoPassArguments,
+ },
+ metal_extensions::CommandBufferTimingExt,
},
- metal_extensions::CommandBufferTimingExt,
},
};
@@ -553,7 +556,7 @@ fn test_single_pass_attention_basic() {
println!("Reference output range (no mask): [{}, {}]", ref_min, ref_max);
println!("Reference sample values: {:?}", &reference_output.slice(s![0, 0, 0, 0..4]).to_vec());
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
@@ -602,7 +605,7 @@ fn test_gemm_attention_basic() {
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 123);
- let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");
+ let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
@@ -630,7 +633,7 @@ fn test_gemm_attention_f32_head_dim_128() {
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 456);
- let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");
+ let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");
let reference_output = reference_attention(&queries, &keys, &values, None, None, scale);
@@ -659,7 +662,7 @@ fn test_matrix_attention_matches_vector_and_cpu_seq256() {
let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 2026);
- let kernel = AttentionKernel::new(&context, KernelDataType::Float32).expect("Failed to create AttentionKernel");
+ let kernel = AttentionKernel::new(&context, DataType::F32).expect("Failed to create AttentionKernel");
// CPU reference (uses the same additive mask)
let reference_output = reference_attention(&queries, &keys, &values, Some(&mask), None, scale);
@@ -714,7 +717,7 @@ fn test_single_pass_attention_with_mask() {
let (queries, keys, values, mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
@@ -766,7 +769,7 @@ fn test_single_pass_attention_with_sinks() {
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 - (num_heads as f32 / 2.0)) * 0.25).collect();
println!("Using sinks: {:?}", sinks);
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
@@ -810,7 +813,7 @@ fn test_single_pass_attention_with_sinks_long_sequence() {
let sinks: Vec<f32> = (0..num_heads).map(|h| (h as f32 * 0.1) - 0.15).collect();
println!("Long sequence sinks: {:?}", sinks);
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
@@ -856,7 +859,7 @@ fn test_single_pass_attention_gqa() {
let (queries, keys, values, _mask) = create_test_data(batch_size, num_heads, num_kv_heads, seq_len, head_dim, 42);
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
@@ -982,7 +985,7 @@ fn run_two_pass_attention(
fn test_two_pass_attention() {
let context = MTLContext::new().expect("Failed to create MTLContext");
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
@@ -1028,7 +1031,7 @@ fn test_two_pass_attention() {
fn test_two_pass_attention_gqa() {
let context = MTLContext::new().expect("Failed to create MTLContext");
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
@@ -1076,7 +1079,7 @@ fn perf_two_pass_attention() {
let context = MTLContext::new().expect("Failed to create MTLContext");
- let kernel = match AttentionKernel::new(&context, KernelDataType::Float32) {
+ let kernel = match AttentionKernel::new(&context, DataType::F32) {
Ok(k) => k,
Err(e) => {
panic!("Failed to create AttentionKernel: {:?}", e);
diff --git a/crates/uzu/tests/moe/counts_offsets_fused_test.rs b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
index 342ef513c..66bd80d69 100644
--- a/crates/uzu/tests/moe/counts_offsets_fused_test.rs
+++ b/crates/uzu/tests/moe/counts_offsets_fused_test.rs
@@ -3,14 +3,16 @@
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::{
- common::kernel::MoeCountsOffsetsFusedKernel,
- metal::{
- MTLContext, ProtocolObject, Retained,
- kernel::{
- KernelDataType,
- dsl::MoeCountsOffsetsFusedMetalKernel,
- moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},
+use uzu::{
+ DataType,
+ backends::{
+ common::kernel::MoeCountsOffsetsFusedKernel,
+ metal::{
+ MTLContext, ProtocolObject, Retained,
+ kernel::{
+ dsl::MoeCountsOffsetsFusedMetalKernel,
+ moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},
+ },
},
},
};
@@ -89,7 +91,7 @@ fn gen_topk_ids_from_logits(
k,
renorm: true,
};
- router_topk.encode(&cb, KernelDataType::BFloat16, args).expect("encode router_topk");
+ router_topk.encode(&cb, DataType::BF16, args).expect("encode router_topk");
cb.commit();
cb.wait_until_completed();
diff --git a/crates/uzu/tests/moe/moe_block_e2e_test.rs b/crates/uzu/tests/moe/moe_block_e2e_test.rs
index cc5df1800..175fca611 100644
--- a/crates/uzu/tests/moe/moe_block_e2e_test.rs
+++ b/crates/uzu/tests/moe/moe_block_e2e_test.rs
@@ -1,16 +1,19 @@
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::{
- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel},
- metal::{
- MTLContext,
- kernel::{
- KernelDataType, MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,
- dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
- moe::{
- MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,
- MoeRouterTopKArguments, MoeRouterTopKKernel,
+use uzu::{
+ DataType,
+ backends::{
+ common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel},
+ metal::{
+ MTLContext,
+ kernel::{
+ MoeBlockBasesArguments, MoeScatterKernels, MoeScatterWithMapArguments,
+ dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
+ moe::{
+ MoeExpertsTwoPassArguments, MoeExpertsTwoPassPrefillKernel, MoeGatherArguments, MoeGatherKernels,
+ MoeRouterTopKArguments, MoeRouterTopKKernel,
+ },
},
},
},
@@ -351,7 +354,7 @@ fn run_moe_parity_test_internal(
router_topk
.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
MoeRouterTopKArguments {
input_buffer: &x_buf,
weight_buffer: &router_w_buf,
@@ -407,14 +410,14 @@ fn run_moe_parity_test_internal(
},
tok2row_buffer: &tok2row_buf,
},
- KernelDataType::BFloat16,
+ DataType::BF16,
)
.expect("scatter");
let gather = MoeGatherKernels::new(&ctx).expect("gather");
gather.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
&MoeGatherArguments {
x_buffer: &x_buf,
bucketed_ids_buffer: &bucketed_ids_buf,
@@ -462,12 +465,12 @@ fn run_moe_parity_test_internal(
up_clip_min: up_clip.0,
up_clip_max: up_clip.1,
silu_alpha,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
},
)
.expect("experts encode");
- let finalize = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize");
+ let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
let encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
&tok2row_buf,
diff --git a/crates/uzu/tests/moe/moe_experts_perf_test.rs b/crates/uzu/tests/moe/moe_experts_perf_test.rs
index 1a684516f..1178cbf0a 100644
--- a/crates/uzu/tests/moe/moe_experts_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_perf_test.rs
@@ -3,11 +3,14 @@ use std::time::Instant;
use half::bf16;
use metal::{MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::metal::{
- KernelDataType, MTLContext,
- kernel::moe::{
- MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
- MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::moe::{
+ MoeExpertsSingleDecodeArguments, MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments,
+ MoeExpertsTwoPassDecodeKernels, MoeExpertsTwoPassPrefillKernel,
+ },
},
};
@@ -125,7 +128,7 @@ fn run_decode_case(
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
silu_alpha: 1.702,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
};
for _ in 0..warmup {
@@ -258,7 +261,7 @@ fn run_two_pass_prefill_case(
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
silu_alpha: 1.702,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
};
for _ in 0..warmup {
@@ -367,7 +370,7 @@ fn run_fused_single_token_case(
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
};
for _ in 0..warmup {
@@ -530,7 +533,7 @@ fn run_indirect_decode_timed(
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
silu_alpha: 1.0,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
};
for _ in 0..warmup {
@@ -611,7 +614,7 @@ fn run_fused_decode_timed(
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
};
for _ in 0..warmup {
diff --git a/crates/uzu/tests/moe/moe_experts_test.rs b/crates/uzu/tests/moe/moe_experts_test.rs
index a7a0e3892..663d0ed4b 100644
--- a/crates/uzu/tests/moe/moe_experts_test.rs
+++ b/crates/uzu/tests/moe/moe_experts_test.rs
@@ -11,9 +11,9 @@
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::metal::{
- KernelDataType,
- kernel::moe::{
+use uzu::{
+ DataType,
+ backends::metal::kernel::moe::{
MoeExpertsSingleDecodeKernels, MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels,
MoeExpertsTwoPassPrefillKernel,
},
@@ -446,7 +446,7 @@ fn test_two_pass_decode_correctness() {
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
silu_alpha,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
},
);
@@ -586,7 +586,7 @@ fn test_two_pass_decode_multi_token() {
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
silu_alpha,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
},
);
cb.commit();
@@ -689,7 +689,7 @@ fn test_two_pass_prefill_correctness() {
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
silu_alpha,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
},
)
.expect("encode");
@@ -815,7 +815,7 @@ fn test_fused_single_token_decode() {
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
},
);
@@ -938,7 +938,7 @@ fn test_fused_single_token_k4() {
gate_clip_max: f32::INFINITY,
up_clip_min: f32::NEG_INFINITY,
up_clip_max: f32::INFINITY,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
},
);
cb.commit();
diff --git a/crates/uzu/tests/moe/moe_finalize_test.rs b/crates/uzu/tests/moe/moe_finalize_test.rs
index 225fa4565..c74e621e6 100644
--- a/crates/uzu/tests/moe/moe_finalize_test.rs
+++ b/crates/uzu/tests/moe/moe_finalize_test.rs
@@ -3,9 +3,9 @@
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::{
- common::kernel::MoeFinalizeKernel,
- metal::{KernelDataType, kernel::dsl::MoeFinalizeMetalKernel},
+use uzu::{
+ DataType,
+ backends::{common::kernel::MoeFinalizeKernel, metal::kernel::dsl::MoeFinalizeMetalKernel},
};
use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};
@@ -85,7 +85,7 @@ fn test_finalize_correctness() {
let y_out_buf = alloc_buffer::<bf16>(&ctx, t * d_model);
// Execute finalize kernel
- let finalize = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize kernel");
+ let finalize = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize kernel");
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let encoder = cb.new_compute_command_encoder().expect("encoder");
finalize.encode(
diff --git a/crates/uzu/tests/moe/moe_gather_test.rs b/crates/uzu/tests/moe/moe_gather_test.rs
index a568bc3ee..e9e1bc743 100644
--- a/crates/uzu/tests/moe/moe_gather_test.rs
+++ b/crates/uzu/tests/moe/moe_gather_test.rs
@@ -3,9 +3,9 @@
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::metal::kernel::{
- KernelDataType,
- moe::{MoeGatherArguments, MoeGatherKernels},
+use uzu::{
+ DataType,
+ backends::metal::kernel::moe::{MoeGatherArguments, MoeGatherKernels},
};
use super::test_utils::{alloc_buffer, alloc_buffer_with_data, assert_bf16_close, create_ctx};
@@ -78,7 +78,7 @@ fn test_gather_correctness() {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
gather.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
&MoeGatherArguments {
x_buffer: &x_buf,
bucketed_ids_buffer: &ids_buf,
diff --git a/crates/uzu/tests/moe/moe_perf_test.rs b/crates/uzu/tests/moe/moe_perf_test.rs
index 6766e7eda..5759b538e 100644
--- a/crates/uzu/tests/moe/moe_perf_test.rs
+++ b/crates/uzu/tests/moe/moe_perf_test.rs
@@ -3,11 +3,11 @@ use std::time::Instant;
use half::bf16;
use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::{
- common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel},
- metal::{
- KernelDataType,
- kernel::{
+use uzu::{
+ DataType,
+ backends::{
+ common::kernel::{MoeCountsOffsetsFusedKernel, MoeFinalizeKernel},
+ metal::kernel::{
dsl::{MoeCountsOffsetsFusedMetalKernel, MoeFinalizeMetalKernel},
moe::{
MoeExpertsTwoPassArguments, MoeExpertsTwoPassDecodeKernels, MoeGatherArguments, MoeGatherKernels,
@@ -123,7 +123,7 @@ fn test_moe_e2e_decode_perf() {
router_topk
.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
MoeRouterTopKArguments {
input_buffer: &x_buf,
weight_buffer: &router_w_buf,
@@ -186,7 +186,7 @@ fn test_moe_e2e_prefill_perf() {
router_topk
.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
MoeRouterTopKArguments {
input_buffer: &x_buf,
weight_buffer: &router_w_buf,
@@ -301,7 +301,7 @@ fn test_moe_pipeline_breakdown_decode() {
let scatter_kernel = MoeScatterKernels::new(&ctx).expect("scatter");
let gather_kernel = MoeGatherKernels::new(&ctx).expect("gather");
let experts_kernel = MoeExpertsTwoPassDecodeKernels::new(&ctx).expect("experts two-pass decode");
- let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).expect("finalize");
+ let finalize_kernel = MoeFinalizeMetalKernel::new(&ctx, DataType::BF16).expect("finalize");
let router_topk_fused_kernel = MoeRouterTopKKernel::new(&ctx).expect("router+topk fused");
// Testing: Router + TopK + Counts+Offsets (FUSED)
@@ -310,7 +310,7 @@ fn test_moe_pipeline_breakdown_decode() {
router_topk_fused_kernel
.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
MoeRouterTopKArguments {
input_buffer: &x_buf,
weight_buffer: &router_w_buf,
@@ -382,7 +382,7 @@ fn test_moe_pipeline_breakdown_decode() {
},
tok2row_buffer: &tok2row_buf,
},
- KernelDataType::BFloat16,
+ DataType::BF16,
)
.expect("scatter");
cb.commit();
@@ -393,7 +393,7 @@ fn test_moe_pipeline_breakdown_decode() {
let cb = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
gather_kernel.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
&MoeGatherArguments {
x_buffer: &x_buf,
bucketed_ids_buffer: &bucketed_ids_buf,
@@ -438,7 +438,7 @@ fn test_moe_pipeline_breakdown_decode() {
up_clip_min: -19.0,
up_clip_max: 21.0,
silu_alpha: 1.702,
- data_type: KernelDataType::BFloat16,
+ data_type: DataType::BF16,
},
);
cb.commit();
diff --git a/crates/uzu/tests/moe/moe_router_topk_test.rs b/crates/uzu/tests/moe/moe_router_topk_test.rs
index 7e5a11ead..55f0841f7 100644
--- a/crates/uzu/tests/moe/moe_router_topk_test.rs
+++ b/crates/uzu/tests/moe/moe_router_topk_test.rs
@@ -3,9 +3,12 @@
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::metal::{
- KernelDataType, MTLContext,
- kernel::moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},
+use uzu::{
+ DataType,
+ backends::metal::{
+ MTLContext,
+ kernel::moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},
+ },
};
use super::test_utils::{alloc_buffer, alloc_buffer_with_data, create_ctx};
@@ -167,7 +170,7 @@ fn run_router_topk_once(
k,
renorm,
};
- kernel.encode(&cb, KernelDataType::BFloat16, args).expect("encode fused router+topk");
+ kernel.encode(&cb, DataType::BF16, args).expect("encode fused router+topk");
cb.commit();
cb.wait_until_completed();
diff --git a/crates/uzu/tests/moe/moe_scatter_test.rs b/crates/uzu/tests/moe/moe_scatter_test.rs
index c68f7fb68..857f0a781 100644
--- a/crates/uzu/tests/moe/moe_scatter_test.rs
+++ b/crates/uzu/tests/moe/moe_scatter_test.rs
@@ -3,12 +3,15 @@
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
use rand::{RngExt, SeedableRng, rngs::StdRng};
-use uzu::backends::{
- common::kernel::MoeCountsOffsetsFusedKernel,
- metal::kernel::{
- KernelDataType, MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,
- dsl::MoeCountsOffsetsFusedMetalKernel,
- moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},
+use uzu::{
+ DataType,
+ backends::{
+ common::kernel::MoeCountsOffsetsFusedKernel,
+ metal::kernel::{
+ MoeBlockBasesArguments, MoeScatterArguments, MoeScatterKernels,
+ dsl::MoeCountsOffsetsFusedMetalKernel,
+ moe::{MoeRouterTopKArguments, MoeRouterTopKKernel},
+ },
},
};
@@ -82,7 +85,7 @@ fn test_scatter_buckets_parity() {
router_topk
.encode(
&cb,
- KernelDataType::BFloat16,
+ DataType::BF16,
MoeRouterTopKArguments {
input_buffer: &input_buf,
weight_buffer: &weight_buf,
@@ -175,7 +178,7 @@ fn test_scatter_buckets_parity() {
num_blocks,
num_tiles,
},
- KernelDataType::BFloat16,
+ DataType::BF16,
)
.expect("encode scatter");
cb.commit();
diff --git a/crates/uzu/tests/sampling_test.rs b/crates/uzu/tests/sampling_test.rs
index 14a27695b..a186cb98c 100644
--- a/crates/uzu/tests/sampling_test.rs
+++ b/crates/uzu/tests/sampling_test.rs
@@ -14,7 +14,7 @@ use uzu::{
},
},
metal::{
- KernelDataType, MTLContext, Metal,
+ MTLContext, Metal,
kernel::dsl::{
GumbelMetalKernel, MinPMetalKernel, TemperatureMetalKernel, TopKMetalKernel, TopPMetalKernel,
},
@@ -1012,8 +1012,7 @@ fn test_temperature_gpu_cpu_match() {
const RTOL: f32 = 1e-6;
const ATOL: f32 = 1e-6;
- let kernel = TemperatureMetalKernel::new(&context, KernelDataType::Float32.into())
- .expect("Failed to create temperature kernel");
+ let kernel = TemperatureMetalKernel::new(&context, DataType::F32).expect("Failed to create temperature kernel");
let logits: Vec<f32> = (0..BATCH * VOCAB).map(|i| ((i * 37 % 1000) as f32 - 500.0) * 0.01).collect();
@@ -1074,7 +1073,7 @@ fn test_topk_gpu_cpu_match() {
const VOCAB: usize = 1024;
const TOPK: u32 = 16;
- let kernel = TopKMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create topk kernel");
+ let kernel = TopKMetalKernel::new(&context, DataType::F32).expect("Failed to create topk kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1138,7 +1137,7 @@ fn test_topp_gpu_cpu_match() {
const VOCAB: usize = 1024;
const TOPP: f32 = 0.9;
- let kernel = TopPMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create topp kernel");
+ let kernel = TopPMetalKernel::new(&context, DataType::F32).expect("Failed to create topp kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1207,7 +1206,7 @@ fn test_minp_gpu_cpu_match() {
const VOCAB: usize = 1024;
const MINP: f32 = 0.1;
- let kernel = MinPMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create minp kernel");
+ let kernel = MinPMetalKernel::new(&context, DataType::F32).expect("Failed to create minp kernel");
let mut rng = StdRng::seed_from_u64(42);
let mut logits = vec![0.0f32; BATCH * VOCAB];
@@ -1391,8 +1390,7 @@ fn test_gumbel_gpu_cpu_match() {
const RTOL: f32 = 0.01;
const ATOL: f32 = 1e-6;
- let kernel =
- GumbelMetalKernel::new(&context, KernelDataType::Float32.into()).expect("Failed to create gumbel kernel");
+ let kernel = GumbelMetalKernel::new(&context, DataType::F32).expect("Failed to create gumbel kernel");
let logits = vec![0.0f32; BATCH * VOCAB];
let seeds: Vec<u64> = (0_u64..BATCH as u64).collect();
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index 75d36df52..a4bd00717 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -7,10 +7,11 @@ use metal::{
MTLBlitCommandEncoder, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions,
};
use uzu::{
+ DataType,
backends::{
common::Context,
metal::{
- KernelDataType, MTLBuffer, MTLContext, ProtocolObject,
+ MTLBuffer, MTLContext, ProtocolObject,
kernel::ssm::{
SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode,
conv1d_scan::{Conv1dScanArguments, Conv1dScanKernel},
@@ -362,7 +363,7 @@ fn assert_deterministic_for_mode(mode: SSDPrefillMode) {
eprintln!("Skipping SSD prefill determinism test: no Metal device");
return;
};
- let kernel = SSDPrefillKernels::new(&ctx, KernelDataType::Float32).unwrap();
+ let kernel = SSDPrefillKernels::new(&ctx, DataType::F32).unwrap();
let fixture = SSDPrefillFixture::new();
let (y_a, state_a, _) = run_prefill_kernel_mode(&ctx, &kernel, &fixture, mode);
@@ -377,7 +378,7 @@ fn assert_matches_cpu_reference(mode: SSDPrefillMode) {
eprintln!("Skipping SSD prefill reference test: no Metal device");
return;
};
- let kernel = SSDPrefillKernels::new(&ctx, KernelDataType::Float32).unwrap();
+ let kernel = SSDPrefillKernels::new(&ctx, DataType::F32).unwrap();
let fixture = SSDPrefillFixture::new();
let (y_ref, state_ref) = ssd_prefill_cpu_reference(
@@ -464,7 +465,7 @@ fn conv1d_scan_is_deterministic() {
return;
};
let activation = Activation::Identity;
- let kernel = Conv1dScanKernel::new(&ctx, KernelDataType::Float32, &activation).unwrap();
+ let kernel = Conv1dScanKernel::new(&ctx, DataType::F32, &activation).unwrap();
let suffix_len = 192usize;
let channels = 8usize;
diff --git a/crates/uzu/tests/ssm_update_test.rs b/crates/uzu/tests/ssm_update_test.rs
index 67fa85419..115fed290 100644
--- a/crates/uzu/tests/ssm_update_test.rs
+++ b/crates/uzu/tests/ssm_update_test.rs
@@ -3,9 +3,12 @@
use bytemuck;
use half::bf16;
use metal::{MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue, MTLDeviceExt, MTLResourceOptions};
-use uzu::backends::{
- common::{Context, kernel::SSDUpdateKernel},
- metal::{KernelDataType, MTLContext, kernel::dsl::SSDUpdateMetalKernel},
+use uzu::{
+ DataType,
+ backends::{
+ common::{Context, kernel::SSDUpdateKernel},
+ metal::{MTLContext, kernel::dsl::SSDUpdateMetalKernel},
+ },
};
#[allow(dead_code)]
@@ -235,7 +238,7 @@ fn ssd_update_with_z_bf16() {
.new_buffer(bsz * h * dh * n * std::mem::size_of::<bf16>(), MTLResourceOptions::STORAGE_MODE_SHARED)
.expect("Failed to create buffer");
- let kernel = SSDUpdateMetalKernel::new(&ctx, KernelDataType::BFloat16.into()).unwrap();
+ let kernel = SSDUpdateMetalKernel::new(&ctx, DataType::BF16).unwrap();
let cb_ref = ctx.command_queue.command_buffer().expect("Failed to create command buffer");
let cb = cb_ref.to_owned();
let enc = cb.new_compute_command_encoder().expect("Failed to create compute encoder");