A high-performance inference engine for AI models
update for new moe config
eugene38edeae5ace4
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rsindex 50cc11aff..4f56924ed 100644--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs@@ -1,3 +1,5 @@+use std::mem::size_of;+use thiserror::Error;use crate::{@@ -36,6 +38,10 @@ pub enum KVCacheUpdateError<B: Backend> {}impl<B: Backend> KVCacheUpdate<B> {+ // Metal's set_bytes supports up to 4KB per bound value.+ // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc+ const MAX_INLINE_BYTES: usize = 4096;+pub fn new(context: &B::Context,data_type: DataType,@@ -78,11 +84,7 @@ impl<B: Backend> KVCacheUpdate<B> {if swaps.len() > self.max_sequence_length {return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);}-- // For small swap counts, use set_bytes to avoid shared buffer race in async mode.- // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.- let use_inline_bytes = swaps.len() <= 32;- assert!(use_inline_bytes, "non-inline is not supported yet (and is broken anyways due to a data race)");+ let max_inline_swaps = (Self::MAX_INLINE_BYTES / size_of::<Swap>()).max(1);for layer_data in in_place_data {if layer_data.key_shape != layer_data.value_shape {@@ -91,16 +93,19 @@ impl<B: Backend> KVCacheUpdate<B> {let [num_heads, max_sequence_length, head_dim] = layer_data.key_shape;- self.kernel.encode(- &layer_data.key_buffer,- &layer_data.value_buffer,- &swaps,- swaps.len() as u32,- num_heads as u32,- max_sequence_length as u32,- head_dim as u32,- encoder,- );+ // non-inline is not supported yet (and is broken anyways due to a data race)+ for swaps_chunk in swaps.chunks(max_inline_swaps) {+ self.kernel.encode(+ &layer_data.key_buffer,+ &layer_data.value_buffer,+ swaps_chunk,+ swaps_chunk.len() as u32,+ num_heads as u32,+ max_sequence_length as u32,+ head_dim as u32,+ encoder,+ );+ }}Ok(())diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rsindex 8580b9226..d18725db8 100644--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs@@ -251,8 +251,8 @@ impl EncodableBlock<Metal> for MoeBlock {let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());debug_assert!(array_iter.next().is_none());- let e = self.moe_config.mixture_size;- let k = self.moe_config.num_experts_per_token;+ let e = self.moe_config.num_routed_experts;+ let k = self.moe_config.num_active_routed_experts;let root = command_buffer;let k_tile = 128;diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rsindex 5ca9af14f..7e279ec78 100644--- a/crates/uzu/src/config/mlp.rs+++ b/crates/uzu/src/config/mlp.rs@@ -33,8 +33,8 @@ fn default_activation_to_gate() -> bool {#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct MixtureOfExpertsConfig {- pub mixture_size: usize,- pub num_experts_per_token: usize,+ pub num_routed_experts: usize,+ pub num_active_routed_experts: usize,pub routing_function: RoutingFunctionConfig,pub router_config: LinearConfig,pub router_has_biases: bool,@@ -113,8 +113,8 @@ mod tests {let config_str = r#"{"type": "MixtureOfExpertsConfig",- "mixture_size": 32,- "num_experts_per_token": 4,+ "num_routed_experts": 32,+ "num_active_routed_experts": 4,"routing_function": {"type": "SoftmaxRouting"},"router_config": {"type": "FullPrecisionLinearConfig", "precision": "bfloat16"},"router_has_biases": true,@@ -138,8 +138,73 @@ mod tests {"#;let ground_truth_config = MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {- mixture_size: 32,- num_experts_per_token: 4,+ num_routed_experts: 32,+ num_active_routed_experts: 4,+ routing_function: RoutingFunctionConfig::SoftmaxRouting,+ router_config: LinearConfig::FullPrecision {+ precision: ConfigDataType::BFloat16,+ },+ router_has_biases: true,+ expert_config: MoeExpertConfig {+ linear_config: LinearConfig::QLoRA {+ quantization: QuantizationConfig {+ group_size: 32,+ weight_quantization_mode: QuantizationMode::UInt4,+ activation_quantization_mode: Some(QuantizationMode::Int8),+ activation_precision: ConfigDataType::BFloat16,+ },+ lora_rank: 16,+ lora_scale: 2.0,+ },+ activation: Activation::SiLU {+ alpha: 1.0,+ },+ has_up_biases: true,+ has_down_biases: true,+ gate_clipping: [None, Some(7.0)],+ up_clipping: [-6.0, 8.0],+ },+ });++ let deserialized_config: MLPConfig = from_str(config_str).unwrap();+ assert_eq!(deserialized_config, ground_truth_config);+ }++ #[test]+ fn test_moe_mlp_config_with_additional_fields() {+ let config_str = r#"+ {+ "type": "MixtureOfExpertsConfig",+ "num_routed_experts": 32,+ "num_active_routed_experts": 4,+ "routing_function": {"type": "SoftmaxRouting"},+ "router_config": {"type": "FullPrecisionLinearConfig", "precision": "bfloat16"},+ "router_has_biases": true,+ "num_shared_experts": 0,+ "expert_hidden_dim": 2880,+ "gate_config": null,+ "expert_config": {+ "linear_config": {+ "type": "QLoRALinearConfig",+ "group_size": 32,+ "weight_quantization_mode": "uint4",+ "activation_quantization_mode": "int8",+ "activation_precision": "bfloat16",+ "lora_rank": 16,+ "lora_scale": 2.0+ },+ "activation": {"type": "SiLU"},+ "has_up_biases": true,+ "has_down_biases": true,+ "gate_clipping": [null, 7.0],+ "up_clipping": [-6.0, 8.0]+ }+ }+ "#;++ let ground_truth_config = MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {+ num_routed_experts: 32,+ num_active_routed_experts: 4,routing_function: RoutingFunctionConfig::SoftmaxRouting,router_config: LinearConfig::FullPrecision {precision: ConfigDataType::BFloat16,diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rsindex e19a04dd9..f41e16520 100644--- a/crates/uzu/src/forward_pass/scratch_buffers.rs+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs@@ -91,10 +91,10 @@ impl<B: Backend> ScratchBuffers<B> {MLPConfig::MixtureOfExperts(moe) => Some(moe),_ => None,};- let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_experts_per_token);+ let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_active_routed_experts);let moe_scatter_entries = moe.map(|moe| {let num_blocks = ((max_suffix_len + 255) / 256).max(1);- let num_tiles = ((moe.mixture_size + 512 - 1) / 512).max(1);+ let num_tiles = ((moe.num_routed_experts + 512 - 1) / 512).max(1);num_blocks * num_tiles * 512});@@ -160,20 +160,20 @@ impl<B: Backend> ScratchBuffers<B> {moe_topk_ids: moe.map(|moe| {alloc(- &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_experts_per_token),+ &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_active_routed_experts),DataType::U32,"moe_topk_ids",)}),moe_topk_probs: moe.map(|moe| {alloc(- &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_experts_per_token),+ &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_active_routed_experts),act_ty,"moe_topk_probs",)}),moe_offsets: moe- .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_offsets")),+ .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_offsets")),moe_sumk: moe.map(|_| alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")),moe_bucketed_token_ids: moe_max_routed.map(|max_routed| {alloc(&model_shape.moe_bucketed_token_ids_shape(max_routed), DataType::U32, "moe_bucketed_token_ids")@@ -185,7 +185,7 @@ impl<B: Backend> ScratchBuffers<B> {.map(|max_routed| alloc(&model_shape.moe_x_perm_shape(max_routed), DataType::F16, "moe_x_perm")),moe_tok2row: moe.map(|moe| {alloc(- &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_experts_per_token),+ &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_active_routed_experts),DataType::I32,"moe_tok2row",)@@ -197,9 +197,13 @@ impl<B: Backend> ScratchBuffers<B> {moe_two_pass_row_expert_map: moe_max_routed.map(|max_routed| alloc(&[max_routed], DataType::U32, "moe_two_pass_row_expert_map")),moe_tile_counts: moe- .map(|moe| alloc(&model_shape.moe_counts_shape(moe.mixture_size), DataType::U32, "moe_tile_counts")),+ .map(|moe| {+ alloc(&model_shape.moe_counts_shape(moe.num_routed_experts), DataType::U32, "moe_tile_counts")+ }),moe_tile_offsets: moe- .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_tile_offsets")),+ .map(|moe| {+ alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_tile_offsets")+ }),moe_tile_map: moe_max_routed.map(|max_routed| alloc(&model_shape.moe_tile_map_shape(max_routed), DataType::U32, "moe_tile_map")),moe_total_tiles: moe.map(|_| alloc(&model_shape.moe_total_tiles_shape(), DataType::U32, "moe_total_tiles")),diff --git a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rsindex 47cdde3b4..be01197d7 100644--- a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs+++ b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs@@ -49,10 +49,10 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {MLPConfig::MixtureOfExperts(moe) => Some(moe),_ => None,};- let max_routed = moe.map(|moe| suffix_length * moe.num_experts_per_token);+ let max_routed = moe.map(|moe| suffix_length * moe.num_active_routed_experts);let num_blocks = suffix_length.div_ceil(256).max(1);- let scatter_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1) * 512);- let block_alloc_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1));+ let scatter_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1) * 512);+ let block_alloc_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1));Self {ssm_inproj: scratch@@ -102,13 +102,15 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {.map(|(buf, shape)| buf.view(&shape)),moe_topk_ids: moe.zip(scratch.moe_topk_ids.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_experts_per_token))),+ .map(|(moe, buf)| {+ buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_active_routed_experts))+ }),moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(|(moe, buf)| {- buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_experts_per_token))+ buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_active_routed_experts))}),moe_offsets: moe.zip(scratch.moe_offsets.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),moe_sumk: moe.and(scratch.moe_sumk.as_ref()).map(|buf| buf.view(&model_shape.moe_sumk_shape())),moe_bucketed_token_ids: max_routed.zip(scratch.moe_bucketed_token_ids.as_ref())@@ -121,7 +123,9 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {.map(|(max_routed, buf)| buf.view(&model_shape.moe_x_perm_shape(max_routed))),moe_tok2row: moe.zip(scratch.moe_tok2row.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_experts_per_token))),+ .map(|(moe, buf)| {+ buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_active_routed_experts))+ }),moe_y_partial: max_routed.zip(scratch.moe_y_partial.as_ref()).map(|(max_routed, buf)| buf.view(&model_shape.moe_y_partial_shape(max_routed))),@@ -133,10 +137,10 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {.map(|(max_routed, buf)| buf.view(&[max_routed])),moe_tile_counts: moe.zip(scratch.moe_tile_counts.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.mixture_size))),+ .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.num_routed_experts))),moe_tile_offsets: moe.zip(scratch.moe_tile_offsets.as_ref())- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),moe_tile_map: max_routed.zip(scratch.moe_tile_map.as_ref()).map(|(max_routed, buf)| buf.view(&model_shape.moe_tile_map_shape(max_routed))),