Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

update for new moe config

eugene38edeae5ace4
diff --git a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
index 50cc11aff..4f56924ed 100644
--- a/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
+++ b/crates/uzu/src/backends/common/kernel/kv_cache_update.rs
@@ -1,3 +1,5 @@
+use std::mem::size_of;
+
use thiserror::Error;
use crate::{
@@ -36,6 +38,10 @@ pub enum KVCacheUpdateError<B: Backend> {
}
impl<B: Backend> KVCacheUpdate<B> {
+ // Metal's set_bytes supports up to 4KB per bound value.
+ // https://developer.apple.com/documentation/metal/mtlcomputecommandencoder/setbytes(_:length:index:)?language=objc
+ const MAX_INLINE_BYTES: usize = 4096;
+
pub fn new(
context: &B::Context,
data_type: DataType,
@@ -78,11 +84,7 @@ impl<B: Backend> KVCacheUpdate<B> {
if swaps.len() > self.max_sequence_length {
return Err(KVCacheUpdateError::MaxSequenceLengthExceeded);
}
-
- // For small swap counts, use set_bytes to avoid shared buffer race in async mode.
- // Metal's set_bytes limit is 4KB; 32 swaps = 256 bytes, well under limit.
- let use_inline_bytes = swaps.len() <= 32;
- assert!(use_inline_bytes, "non-inline is not supported yet (and is broken anyways due to a data race)");
+ let max_inline_swaps = (Self::MAX_INLINE_BYTES / size_of::<Swap>()).max(1);
for layer_data in in_place_data {
if layer_data.key_shape != layer_data.value_shape {
@@ -91,16 +93,19 @@ impl<B: Backend> KVCacheUpdate<B> {
let [num_heads, max_sequence_length, head_dim] = layer_data.key_shape;
- self.kernel.encode(
- &layer_data.key_buffer,
- &layer_data.value_buffer,
- &swaps,
- swaps.len() as u32,
- num_heads as u32,
- max_sequence_length as u32,
- head_dim as u32,
- encoder,
- );
+ // non-inline is not supported yet (and is broken anyways due to a data race)
+ for swaps_chunk in swaps.chunks(max_inline_swaps) {
+ self.kernel.encode(
+ &layer_data.key_buffer,
+ &layer_data.value_buffer,
+ swaps_chunk,
+ swaps_chunk.len() as u32,
+ num_heads as u32,
+ max_sequence_length as u32,
+ head_dim as u32,
+ encoder,
+ );
+ }
}
Ok(())
diff --git a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
index 8580b9226..d18725db8 100644
--- a/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
+++ b/crates/uzu/src/backends/metal/encodable_block/moe_block/moe_block.rs
@@ -251,8 +251,8 @@ impl EncodableBlock<Metal> for MoeBlock {
let row_expert_map_buf = clone_buffer(array_iter.next().unwrap());
debug_assert!(array_iter.next().is_none());
- let e = self.moe_config.mixture_size;
- let k = self.moe_config.num_experts_per_token;
+ let e = self.moe_config.num_routed_experts;
+ let k = self.moe_config.num_active_routed_experts;
let root = command_buffer;
let k_tile = 128;
diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rs
index 5ca9af14f..7e279ec78 100644
--- a/crates/uzu/src/config/mlp.rs
+++ b/crates/uzu/src/config/mlp.rs
@@ -33,8 +33,8 @@ fn default_activation_to_gate() -> bool {
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct MixtureOfExpertsConfig {
- pub mixture_size: usize,
- pub num_experts_per_token: usize,
+ pub num_routed_experts: usize,
+ pub num_active_routed_experts: usize,
pub routing_function: RoutingFunctionConfig,
pub router_config: LinearConfig,
pub router_has_biases: bool,
@@ -113,8 +113,8 @@ mod tests {
let config_str = r#"
{
"type": "MixtureOfExpertsConfig",
- "mixture_size": 32,
- "num_experts_per_token": 4,
+ "num_routed_experts": 32,
+ "num_active_routed_experts": 4,
"routing_function": {"type": "SoftmaxRouting"},
"router_config": {"type": "FullPrecisionLinearConfig", "precision": "bfloat16"},
"router_has_biases": true,
@@ -138,8 +138,73 @@ mod tests {
"#;
let ground_truth_config = MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {
- mixture_size: 32,
- num_experts_per_token: 4,
+ num_routed_experts: 32,
+ num_active_routed_experts: 4,
+ routing_function: RoutingFunctionConfig::SoftmaxRouting,
+ router_config: LinearConfig::FullPrecision {
+ precision: ConfigDataType::BFloat16,
+ },
+ router_has_biases: true,
+ expert_config: MoeExpertConfig {
+ linear_config: LinearConfig::QLoRA {
+ quantization: QuantizationConfig {
+ group_size: 32,
+ weight_quantization_mode: QuantizationMode::UInt4,
+ activation_quantization_mode: Some(QuantizationMode::Int8),
+ activation_precision: ConfigDataType::BFloat16,
+ },
+ lora_rank: 16,
+ lora_scale: 2.0,
+ },
+ activation: Activation::SiLU {
+ alpha: 1.0,
+ },
+ has_up_biases: true,
+ has_down_biases: true,
+ gate_clipping: [None, Some(7.0)],
+ up_clipping: [-6.0, 8.0],
+ },
+ });
+
+ let deserialized_config: MLPConfig = from_str(config_str).unwrap();
+ assert_eq!(deserialized_config, ground_truth_config);
+ }
+
+ #[test]
+ fn test_moe_mlp_config_with_additional_fields() {
+ let config_str = r#"
+ {
+ "type": "MixtureOfExpertsConfig",
+ "num_routed_experts": 32,
+ "num_active_routed_experts": 4,
+ "routing_function": {"type": "SoftmaxRouting"},
+ "router_config": {"type": "FullPrecisionLinearConfig", "precision": "bfloat16"},
+ "router_has_biases": true,
+ "num_shared_experts": 0,
+ "expert_hidden_dim": 2880,
+ "gate_config": null,
+ "expert_config": {
+ "linear_config": {
+ "type": "QLoRALinearConfig",
+ "group_size": 32,
+ "weight_quantization_mode": "uint4",
+ "activation_quantization_mode": "int8",
+ "activation_precision": "bfloat16",
+ "lora_rank": 16,
+ "lora_scale": 2.0
+ },
+ "activation": {"type": "SiLU"},
+ "has_up_biases": true,
+ "has_down_biases": true,
+ "gate_clipping": [null, 7.0],
+ "up_clipping": [-6.0, 8.0]
+ }
+ }
+ "#;
+
+ let ground_truth_config = MLPConfig::MixtureOfExperts(MixtureOfExpertsConfig {
+ num_routed_experts: 32,
+ num_active_routed_experts: 4,
routing_function: RoutingFunctionConfig::SoftmaxRouting,
router_config: LinearConfig::FullPrecision {
precision: ConfigDataType::BFloat16,
diff --git a/crates/uzu/src/forward_pass/scratch_buffers.rs b/crates/uzu/src/forward_pass/scratch_buffers.rs
index e19a04dd9..f41e16520 100644
--- a/crates/uzu/src/forward_pass/scratch_buffers.rs
+++ b/crates/uzu/src/forward_pass/scratch_buffers.rs
@@ -91,10 +91,10 @@ impl<B: Backend> ScratchBuffers<B> {
MLPConfig::MixtureOfExperts(moe) => Some(moe),
_ => None,
};
- let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_experts_per_token);
+ let moe_max_routed = moe.map(|moe| max_suffix_len * moe.num_active_routed_experts);
let moe_scatter_entries = moe.map(|moe| {
let num_blocks = ((max_suffix_len + 255) / 256).max(1);
- let num_tiles = ((moe.mixture_size + 512 - 1) / 512).max(1);
+ let num_tiles = ((moe.num_routed_experts + 512 - 1) / 512).max(1);
num_blocks * num_tiles * 512
});
@@ -160,20 +160,20 @@ impl<B: Backend> ScratchBuffers<B> {
moe_topk_ids: moe.map(|moe| {
alloc(
- &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_experts_per_token),
+ &model_shape.moe_topk_ids_shape(max_suffix_len, moe.num_active_routed_experts),
DataType::U32,
"moe_topk_ids",
)
}),
moe_topk_probs: moe.map(|moe| {
alloc(
- &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_experts_per_token),
+ &model_shape.moe_topk_probs_shape(max_suffix_len, moe.num_active_routed_experts),
act_ty,
"moe_topk_probs",
)
}),
moe_offsets: moe
- .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_offsets")),
+ .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_offsets")),
moe_sumk: moe.map(|_| alloc(&model_shape.moe_sumk_shape(), DataType::U32, "moe_sumk")),
moe_bucketed_token_ids: moe_max_routed.map(|max_routed| {
alloc(&model_shape.moe_bucketed_token_ids_shape(max_routed), DataType::U32, "moe_bucketed_token_ids")
@@ -185,7 +185,7 @@ impl<B: Backend> ScratchBuffers<B> {
.map(|max_routed| alloc(&model_shape.moe_x_perm_shape(max_routed), DataType::F16, "moe_x_perm")),
moe_tok2row: moe.map(|moe| {
alloc(
- &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_experts_per_token),
+ &model_shape.moe_tok2row_shape(max_suffix_len, moe.num_active_routed_experts),
DataType::I32,
"moe_tok2row",
)
@@ -197,9 +197,13 @@ impl<B: Backend> ScratchBuffers<B> {
moe_two_pass_row_expert_map: moe_max_routed
.map(|max_routed| alloc(&[max_routed], DataType::U32, "moe_two_pass_row_expert_map")),
moe_tile_counts: moe
- .map(|moe| alloc(&model_shape.moe_counts_shape(moe.mixture_size), DataType::U32, "moe_tile_counts")),
+ .map(|moe| {
+ alloc(&model_shape.moe_counts_shape(moe.num_routed_experts), DataType::U32, "moe_tile_counts")
+ }),
moe_tile_offsets: moe
- .map(|moe| alloc(&model_shape.moe_offsets_shape(moe.mixture_size), DataType::U32, "moe_tile_offsets")),
+ .map(|moe| {
+ alloc(&model_shape.moe_offsets_shape(moe.num_routed_experts), DataType::U32, "moe_tile_offsets")
+ }),
moe_tile_map: moe_max_routed
.map(|max_routed| alloc(&model_shape.moe_tile_map_shape(max_routed), DataType::U32, "moe_tile_map")),
moe_total_tiles: moe.map(|_| alloc(&model_shape.moe_total_tiles_shape(), DataType::U32, "moe_total_tiles")),
diff --git a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
index 47cdde3b4..be01197d7 100644
--- a/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
+++ b/crates/uzu/src/forward_pass/state/language_model_generator_aux_buffers.rs
@@ -49,10 +49,10 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
MLPConfig::MixtureOfExperts(moe) => Some(moe),
_ => None,
};
- let max_routed = moe.map(|moe| suffix_length * moe.num_experts_per_token);
+ let max_routed = moe.map(|moe| suffix_length * moe.num_active_routed_experts);
let num_blocks = suffix_length.div_ceil(256).max(1);
- let scatter_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1) * 512);
- let block_alloc_entries = moe.map(|moe| num_blocks * moe.mixture_size.div_ceil(512).max(1));
+ let scatter_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1) * 512);
+ let block_alloc_entries = moe.map(|moe| num_blocks * moe.num_routed_experts.div_ceil(512).max(1));
Self {
ssm_inproj: scratch
@@ -102,13 +102,15 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
.map(|(buf, shape)| buf.view(&shape)),
moe_topk_ids: moe
.zip(scratch.moe_topk_ids.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_experts_per_token))),
+ .map(|(moe, buf)| {
+ buf.view(&model_shape.moe_topk_ids_shape(suffix_length, moe.num_active_routed_experts))
+ }),
moe_topk_probs: moe.zip(scratch.moe_topk_probs.as_ref()).map(|(moe, buf)| {
- buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_experts_per_token))
+ buf.view(&model_shape.moe_topk_probs_shape(suffix_length, moe.num_active_routed_experts))
}),
moe_offsets: moe
.zip(scratch.moe_offsets.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),
+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),
moe_sumk: moe.and(scratch.moe_sumk.as_ref()).map(|buf| buf.view(&model_shape.moe_sumk_shape())),
moe_bucketed_token_ids: max_routed
.zip(scratch.moe_bucketed_token_ids.as_ref())
@@ -121,7 +123,9 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
.map(|(max_routed, buf)| buf.view(&model_shape.moe_x_perm_shape(max_routed))),
moe_tok2row: moe
.zip(scratch.moe_tok2row.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_experts_per_token))),
+ .map(|(moe, buf)| {
+ buf.view(&model_shape.moe_tok2row_shape(suffix_length, moe.num_active_routed_experts))
+ }),
moe_y_partial: max_routed
.zip(scratch.moe_y_partial.as_ref())
.map(|(max_routed, buf)| buf.view(&model_shape.moe_y_partial_shape(max_routed))),
@@ -133,10 +137,10 @@ impl<B: Backend> LanguageModelGeneratorAuxBuffers<B> {
.map(|(max_routed, buf)| buf.view(&[max_routed])),
moe_tile_counts: moe
.zip(scratch.moe_tile_counts.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.mixture_size))),
+ .map(|(moe, buf)| buf.view(&model_shape.moe_counts_shape(moe.num_routed_experts))),
moe_tile_offsets: moe
.zip(scratch.moe_tile_offsets.as_ref())
- .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.mixture_size))),
+ .map(|(moe, buf)| buf.view(&model_shape.moe_offsets_shape(moe.num_routed_experts))),
moe_tile_map: max_routed
.zip(scratch.moe_tile_map.as_ref())
.map(|(max_routed, buf)| buf.view(&model_shape.moe_tile_map_shape(max_routed))),