A high-performance inference engine for AI models
.clone() cleanup (#214)
Aleksandr Golokoza74c96a06738
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rsindex dc3cc50b4..610c9c27c 100644--- a/crates/uzu/src/encodable_block/attention.rs+++ b/crates/uzu/src/encodable_block/attention.rs@@ -367,8 +367,6 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {Some(k) => k,None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),};- let mask_buffer_opt = attention_bias_buffer.as_ref();- let sinks_buffer_opt = sinks_buffer.as_ref();kernel.encode(queries_buffer,&key_cache_buffer,@@ -381,11 +379,11 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {v_head_stride as u32,v_seq_stride as u32,scale,- mask_buffer_opt,+ attention_bias_buffer.as_ref(),mask_kv_seq_stride_opt,mask_q_seq_stride_opt,mask_head_stride_opt,- sinks_buffer_opt,+ sinks_buffer.as_ref(),num_heads as u32,suffix_length as u32,&compute_encoder,@@ -400,9 +398,6 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {Some(k) => k,None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),};-- let mask_buffer_opt = attention_bias_buffer.as_ref().map(|b| b);- let sinks_buffer_opt = sinks_buffer.as_ref().map(|b| b);kernel_pass1.encode(queries_buffer,&key_cache_buffer,@@ -419,11 +414,11 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {scale,num_heads as u32,suffix_length as u32,- mask_buffer_opt,+ attention_bias_buffer.as_ref(),mask_kv_seq_stride_opt,mask_q_seq_stride_opt,mask_head_stride_opt,- sinks_buffer_opt,+ sinks_buffer.as_ref(),&compute_encoder,);kernel_pass2.encode(diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rsindex bed8112f2..dfca55db8 100644--- a/crates/uzu/src/encodable_block/layer/executables.rs+++ b/crates/uzu/src/encodable_block/layer/executables.rs@@ -82,7 +82,7 @@ wherelet mixer = match &layer_config.mixer_config {MixerConfig::Attention(attention_config) => {- let rope_block = rope.clone().expect("RoPE encoder missing for attention layer");+ let rope_block = rope.expect("RoPE encoder missing for attention layer");let qkv_projection = linear_block(&attention_config.qkv_projection_config,diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex 3807e1dc5..9b70fe786 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -175,29 +175,31 @@ impl<B: Backend> MambaMixer<B> {ArrayId::SsmZ(self.layer_index),ArrayId::SsmDt(self.layer_index),]);- let in_proj = arrays[0].borrow_mut();- let conv_inputs = arrays[1].borrow_mut();- let gate = arrays[2].borrow_mut();- let dt = arrays[3].borrow_mut();-- let input_buf = in_proj.buffer().clone();- let conv_buf = conv_inputs.buffer().clone();- let gate_buf = gate.buffer().clone();- let dt_buf = dt.buffer().clone();- let gate_bias = self.gate_bias.clone();- let bias_buf = gate_bias.buffer().clone();+ let in_proj = arrays[0].borrow();+ let input_buf = in_proj.buffer();++ let conv_inputs = arrays[1].borrow();+ let conv_buf = conv_inputs.buffer();++ let gate = arrays[2].borrow();+ let gate_buf = gate.buffer();++ let dt = arrays[3].borrow();+ let dt_buf = dt.buffer();++ let bias_buf = self.gate_bias.buffer();let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let num_heads = self.config.num_heads;let total_dim = conv_dim + inner_dim + num_heads;self.split_inproj.encode(- &input_buf,- &conv_buf,- &gate_buf,- &dt_buf,- &bias_buf,+ input_buf,+ conv_buf,+ gate_buf,+ dt_buf,+ bias_buf,suffix_length as u32,total_dim as u32,conv_dim as u32,@@ -220,39 +222,37 @@ impl<B: Backend> MambaMixer<B> {ArrayId::SsmB(self.layer_index),ArrayId::SsmC(self.layer_index),]);- let conv_inputs = arrays[0].borrow_mut();- let conv_state = arrays[1].borrow_mut();- let x_arr = arrays[2].borrow_mut();- let b_arr = arrays[3].borrow_mut();- let c_arr = arrays[4].borrow_mut();-- let input_buf = conv_inputs.buffer().clone();- let state_buf = conv_state.buffer().clone();- let x_buf = x_arr.buffer().clone();- let b_buf = b_arr.buffer().clone();- let c_buf = c_arr.buffer().clone();-- let weight_storage = self.conv_weight.clone();- let weight_buf = weight_storage.buffer().clone();+ let conv_inputs = arrays[0].borrow();+ let conv_state = arrays[1].borrow();+ let x_arr = arrays[2].borrow();+ let b_arr = arrays[3].borrow();+ let c_arr = arrays[4].borrow();++ let input_buf = conv_inputs.buffer();+ let state_buf = conv_state.buffer();+ let x_buf = x_arr.buffer();+ let b_buf = b_arr.buffer();+ let c_buf = c_arr.buffer();++ let weight_buf = self.conv_weight.buffer();let bias_buf = self.conv_bias.as_ref().map(|arr| arr.buffer());let conv_dim = self.config.conv_dim();let inner_dim = self.config.inner_dim();let proj_dim = self.config.num_groups * self.config.state_dim;let state_stride = self.config.kernel_size.saturating_sub(1);- drop(conv_state);if suffix_length == 1 {if conv_dim > 0 && self.config.kernel_size > 0 {self.conv_decode.encode(- &input_buf,- &weight_buf,+ input_buf,+ weight_buf,bias_buf,- &state_buf,- &x_buf,- &b_buf,- &c_buf,- &state_buf,+ state_buf,+ x_buf,+ b_buf,+ c_buf,+ state_buf,self.config.kernel_size as u32,conv_dim as u32,state_stride as u32,@@ -266,13 +266,13 @@ impl<B: Backend> MambaMixer<B> {} else {let padded_buf = if state_stride > 0 {let array = state.conv_padded_buffer().expect("Missing conv padded buffer");- let borrow = array.borrow_mut();+ let borrow = array.borrow();let buf = borrow.buffer().clone();drop(borrow);self.conv_pack.encode(- &state_buf,- &input_buf,+ state_buf,+ input_buf,&buf,state_stride as u32,conv_dim as u32,@@ -287,15 +287,15 @@ impl<B: Backend> MambaMixer<B> {};if conv_dim > 0 && self.config.kernel_size > 0 {- let conv_source = padded_buf.as_ref().unwrap_or(&input_buf).clone();+ let conv_source = padded_buf.as_ref().unwrap_or(&input_buf);self.conv_scan.encode(- &conv_source,- &weight_buf,+ conv_source,+ weight_buf,bias_buf,- &x_buf,- &b_buf,- &c_buf,- &state_buf,+ x_buf,+ b_buf,+ c_buf,+ state_buf,suffix_length as u32,self.config.kernel_size as u32,conv_dim as u32,@@ -324,42 +324,40 @@ impl<B: Backend> MambaMixer<B> {ArrayId::SsmState(self.layer_index),ArrayId::AttentionOutput,]);+let x = base_arrays[0].borrow();- let x_buf = x.buffer().clone();- drop(x);+ let x_buf = x.buffer();+let b = base_arrays[1].borrow();- let b_buf = b.buffer().clone();- drop(b);+ let b_buf = b.buffer();+let c = base_arrays[2].borrow();- let c_buf = c.buffer().clone();- drop(c);+ let c_buf = c.buffer();+let dt = base_arrays[3].borrow();- let dt_buf = dt.buffer().clone();- drop(dt);+ let dt_buf = dt.buffer();+let z = base_arrays[4].borrow();- let z_buf = z.buffer().clone();- drop(z);+ let z_buf = z.buffer();+let state_buf = base_arrays[5].borrow();- let state_raw = state_buf.buffer().clone();- drop(state_buf);- let out = base_arrays[6].borrow();- let out_buf = out.buffer().clone();- drop(out);+ let state_raw = state_buf.buffer();- let skip_weights = self.skip_connection_weight.clone();- let skip = skip_weights.buffer().clone();+ let out = base_arrays[6].borrow();+ let out_buf = out.buffer();+ let skip = self.skip_connection_weight.buffer();self.ssd_prefill.encode(encoder,SSDPrefillArguments {- x: &x_buf,- dt: &dt_buf,- b: &b_buf,- c: &c_buf,- d: &skip,- z: &z_buf,- state: &state_raw,- y: &out_buf,+ x: x_buf,+ dt: dt_buf,+ b: b_buf,+ c: c_buf,+ d: skip,+ z: z_buf,+ state: state_raw,+ y: out_buf,suffix_len: suffix_length,group_size: (self.config.num_heads / self.config.num_groups) as i32,state_size: self.config.state_dim as i32,@@ -390,27 +388,26 @@ impl<B: Backend> MambaMixer<B> {ArrayId::AttentionOutput,]);let x = arrays[0].borrow();- let x_buf = x.buffer().clone();- drop(x);+ let x_buf = x.buffer();+let b = arrays[1].borrow();- let b_buf = b.buffer().clone();- drop(b);+ let b_buf = b.buffer();+let c = arrays[2].borrow();- let c_buf = c.buffer().clone();- drop(c);+ let c_buf = c.buffer();+let dt = arrays[3].borrow();- let dt_buf = dt.buffer().clone();- drop(dt);+ let dt_buf = dt.buffer();+let z = arrays[4].borrow();- let z_buf = z.buffer().clone();- drop(z);+ let z_buf = z.buffer();+let state_arr = arrays[5].borrow();- let state_buf = state_arr.buffer().clone();- drop(state_arr);+ let state_buf = state_arr.buffer();+let y = arrays[6].borrow();- let y_buf = y.buffer().clone();- let skip_weights = self.skip_connection_weight.clone();- let skip_buf = skip_weights.buffer().clone();+ let y_buf = y.buffer();+ let skip_buf = self.skip_connection_weight.buffer();let h = self.config.num_heads as u32;let g = self.config.num_groups as u32;@@ -425,15 +422,15 @@ impl<B: Backend> MambaMixer<B> {let state_size = n as i32;self.ssd_update.encode(- &x_buf,- &dt_buf,- &b_buf,- &c_buf,- &skip_buf,- &z_buf,- &state_buf,- &y_buf,- &state_buf,+ x_buf,+ dt_buf,+ b_buf,+ c_buf,+ skip_buf,+ z_buf,+ state_buf,+ y_buf,+ state_buf,group_size as u32,state_size as u32,x_strides.as_slice(),diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rsindex c7641dd1f..97c45f455 100644--- a/crates/uzu/src/encodable_block/quantized_linear.rs+++ b/crates/uzu/src/encodable_block/quantized_linear.rs@@ -139,8 +139,8 @@ impl<B: Backend> QuantizedLinear<B> {&& deq_biases_shape == [output_dim, k_g]){return Err(QuantizedLinearError::InvalidMlxShapes {- weights: weights_shape.clone().into_boxed_slice(),- scales: scales_shape.clone().into_boxed_slice(),+ weights: weights_shape.into_boxed_slice(),+ scales: scales_shape.into_boxed_slice(),deq_biases: deq_biases_shape.into_boxed_slice(),packing_divisor,});@@ -164,8 +164,8 @@ impl<B: Backend> QuantizedLinear<B> {&& zero_points_shape == [output_dim, expected_zero_points_entries]){return Err(QuantizedLinearError::InvalidAwqShapes {- weights: weights_shape.clone().into_boxed_slice(),- scales: scales_shape.clone().into_boxed_slice(),+ weights: weights_shape.into_boxed_slice(),+ scales: scales_shape.into_boxed_slice(),zero_points: zero_points_shape.into_boxed_slice(),packing_divisor,packing_minus_one: packing_divisor - 1,diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rsindex f5f1bf1ab..c8f9da72d 100644--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs@@ -229,12 +229,11 @@ impl<B: Backend> ShortConvMixer<B> {let conv_state = arrays[1].borrow_mut();let out = arrays[2].borrow_mut();- let in_proj_buf = in_proj.buffer().clone();- let state_buf = conv_state.buffer().clone();- let out_buf = out.buffer().clone();+ let in_proj_buf = in_proj.buffer();+ let state_buf = conv_state.buffer();+ let out_buf = out.buffer();- let conv_weight = self.conv_weight.clone();- let weight_buf = conv_weight.buffer().clone();+ let weight_buf = self.conv_weight.buffer();let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());let kernel_size = self.config.kernel_size;@@ -247,8 +246,8 @@ impl<B: Backend> ShortConvMixer<B> {let padded_size = padded_rows * self.model_dim * element_size;let padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");self.short_conv_pack.encode(- &state_buf,- &in_proj_buf,+ state_buf,+ in_proj_buf,&padded_buf,state_stride as u32,suffix_length as u32,@@ -259,11 +258,11 @@ impl<B: Backend> ShortConvMixer<B> {self.short_conv_prefill.encode(&padded_buf,- &in_proj_buf,- &weight_buf,+ in_proj_buf,+ weight_buf,bias_buf,- &out_buf,- &state_buf,+ out_buf,+ state_buf,suffix_length as u32,kernel_size as u32,self.model_dim as u32 * 3,