Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

.clone() cleanup (#214)

Aleksandr Golokoza74c96a06738
diff --git a/crates/uzu/src/encodable_block/attention.rs b/crates/uzu/src/encodable_block/attention.rs
index dc3cc50b4..610c9c27c 100644
--- a/crates/uzu/src/encodable_block/attention.rs
+++ b/crates/uzu/src/encodable_block/attention.rs
@@ -367,8 +367,6 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
Some(k) => k,
None => panic!("Can not find AttentionSinglePassKernel for key {:?}", kernel_key),
};
- let mask_buffer_opt = attention_bias_buffer.as_ref();
- let sinks_buffer_opt = sinks_buffer.as_ref();
kernel.encode(
queries_buffer,
&key_cache_buffer,
@@ -381,11 +379,11 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
v_head_stride as u32,
v_seq_stride as u32,
scale,
- mask_buffer_opt,
+ attention_bias_buffer.as_ref(),
mask_kv_seq_stride_opt,
mask_q_seq_stride_opt,
mask_head_stride_opt,
- sinks_buffer_opt,
+ sinks_buffer.as_ref(),
num_heads as u32,
suffix_length as u32,
&compute_encoder,
@@ -400,9 +398,6 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
Some(k) => k,
None => panic!("Can not find AttentionTwoPass2Kernel for key {:?}", kernel_key),
};
-
- let mask_buffer_opt = attention_bias_buffer.as_ref().map(|b| b);
- let sinks_buffer_opt = sinks_buffer.as_ref().map(|b| b);
kernel_pass1.encode(
queries_buffer,
&key_cache_buffer,
@@ -419,11 +414,11 @@ impl<B: Backend> EncodableBlock<B> for Attention<B> {
scale,
num_heads as u32,
suffix_length as u32,
- mask_buffer_opt,
+ attention_bias_buffer.as_ref(),
mask_kv_seq_stride_opt,
mask_q_seq_stride_opt,
mask_head_stride_opt,
- sinks_buffer_opt,
+ sinks_buffer.as_ref(),
&compute_encoder,
);
kernel_pass2.encode(
diff --git a/crates/uzu/src/encodable_block/layer/executables.rs b/crates/uzu/src/encodable_block/layer/executables.rs
index bed8112f2..dfca55db8 100644
--- a/crates/uzu/src/encodable_block/layer/executables.rs
+++ b/crates/uzu/src/encodable_block/layer/executables.rs
@@ -82,7 +82,7 @@ where
let mixer = match &layer_config.mixer_config {
MixerConfig::Attention(attention_config) => {
- let rope_block = rope.clone().expect("RoPE encoder missing for attention layer");
+ let rope_block = rope.expect("RoPE encoder missing for attention layer");
let qkv_projection = linear_block(
&attention_config.qkv_projection_config,
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index 3807e1dc5..9b70fe786 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -175,29 +175,31 @@ impl<B: Backend> MambaMixer<B> {
ArrayId::SsmZ(self.layer_index),
ArrayId::SsmDt(self.layer_index),
]);
- let in_proj = arrays[0].borrow_mut();
- let conv_inputs = arrays[1].borrow_mut();
- let gate = arrays[2].borrow_mut();
- let dt = arrays[3].borrow_mut();
-
- let input_buf = in_proj.buffer().clone();
- let conv_buf = conv_inputs.buffer().clone();
- let gate_buf = gate.buffer().clone();
- let dt_buf = dt.buffer().clone();
- let gate_bias = self.gate_bias.clone();
- let bias_buf = gate_bias.buffer().clone();
+ let in_proj = arrays[0].borrow();
+ let input_buf = in_proj.buffer();
+
+ let conv_inputs = arrays[1].borrow();
+ let conv_buf = conv_inputs.buffer();
+
+ let gate = arrays[2].borrow();
+ let gate_buf = gate.buffer();
+
+ let dt = arrays[3].borrow();
+ let dt_buf = dt.buffer();
+
+ let bias_buf = self.gate_bias.buffer();
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let num_heads = self.config.num_heads;
let total_dim = conv_dim + inner_dim + num_heads;
self.split_inproj.encode(
- &input_buf,
- &conv_buf,
- &gate_buf,
- &dt_buf,
- &bias_buf,
+ input_buf,
+ conv_buf,
+ gate_buf,
+ dt_buf,
+ bias_buf,
suffix_length as u32,
total_dim as u32,
conv_dim as u32,
@@ -220,39 +222,37 @@ impl<B: Backend> MambaMixer<B> {
ArrayId::SsmB(self.layer_index),
ArrayId::SsmC(self.layer_index),
]);
- let conv_inputs = arrays[0].borrow_mut();
- let conv_state = arrays[1].borrow_mut();
- let x_arr = arrays[2].borrow_mut();
- let b_arr = arrays[3].borrow_mut();
- let c_arr = arrays[4].borrow_mut();
-
- let input_buf = conv_inputs.buffer().clone();
- let state_buf = conv_state.buffer().clone();
- let x_buf = x_arr.buffer().clone();
- let b_buf = b_arr.buffer().clone();
- let c_buf = c_arr.buffer().clone();
-
- let weight_storage = self.conv_weight.clone();
- let weight_buf = weight_storage.buffer().clone();
+ let conv_inputs = arrays[0].borrow();
+ let conv_state = arrays[1].borrow();
+ let x_arr = arrays[2].borrow();
+ let b_arr = arrays[3].borrow();
+ let c_arr = arrays[4].borrow();
+
+ let input_buf = conv_inputs.buffer();
+ let state_buf = conv_state.buffer();
+ let x_buf = x_arr.buffer();
+ let b_buf = b_arr.buffer();
+ let c_buf = c_arr.buffer();
+
+ let weight_buf = self.conv_weight.buffer();
let bias_buf = self.conv_bias.as_ref().map(|arr| arr.buffer());
let conv_dim = self.config.conv_dim();
let inner_dim = self.config.inner_dim();
let proj_dim = self.config.num_groups * self.config.state_dim;
let state_stride = self.config.kernel_size.saturating_sub(1);
- drop(conv_state);
if suffix_length == 1 {
if conv_dim > 0 && self.config.kernel_size > 0 {
self.conv_decode.encode(
- &input_buf,
- &weight_buf,
+ input_buf,
+ weight_buf,
bias_buf,
- &state_buf,
- &x_buf,
- &b_buf,
- &c_buf,
- &state_buf,
+ state_buf,
+ x_buf,
+ b_buf,
+ c_buf,
+ state_buf,
self.config.kernel_size as u32,
conv_dim as u32,
state_stride as u32,
@@ -266,13 +266,13 @@ impl<B: Backend> MambaMixer<B> {
} else {
let padded_buf = if state_stride > 0 {
let array = state.conv_padded_buffer().expect("Missing conv padded buffer");
- let borrow = array.borrow_mut();
+ let borrow = array.borrow();
let buf = borrow.buffer().clone();
drop(borrow);
self.conv_pack.encode(
- &state_buf,
- &input_buf,
+ state_buf,
+ input_buf,
&buf,
state_stride as u32,
conv_dim as u32,
@@ -287,15 +287,15 @@ impl<B: Backend> MambaMixer<B> {
};
if conv_dim > 0 && self.config.kernel_size > 0 {
- let conv_source = padded_buf.as_ref().unwrap_or(&input_buf).clone();
+ let conv_source = padded_buf.as_ref().unwrap_or(&input_buf);
self.conv_scan.encode(
- &conv_source,
- &weight_buf,
+ conv_source,
+ weight_buf,
bias_buf,
- &x_buf,
- &b_buf,
- &c_buf,
- &state_buf,
+ x_buf,
+ b_buf,
+ c_buf,
+ state_buf,
suffix_length as u32,
self.config.kernel_size as u32,
conv_dim as u32,
@@ -324,42 +324,40 @@ impl<B: Backend> MambaMixer<B> {
ArrayId::SsmState(self.layer_index),
ArrayId::AttentionOutput,
]);
+
let x = base_arrays[0].borrow();
- let x_buf = x.buffer().clone();
- drop(x);
+ let x_buf = x.buffer();
+
let b = base_arrays[1].borrow();
- let b_buf = b.buffer().clone();
- drop(b);
+ let b_buf = b.buffer();
+
let c = base_arrays[2].borrow();
- let c_buf = c.buffer().clone();
- drop(c);
+ let c_buf = c.buffer();
+
let dt = base_arrays[3].borrow();
- let dt_buf = dt.buffer().clone();
- drop(dt);
+ let dt_buf = dt.buffer();
+
let z = base_arrays[4].borrow();
- let z_buf = z.buffer().clone();
- drop(z);
+ let z_buf = z.buffer();
+
let state_buf = base_arrays[5].borrow();
- let state_raw = state_buf.buffer().clone();
- drop(state_buf);
- let out = base_arrays[6].borrow();
- let out_buf = out.buffer().clone();
- drop(out);
+ let state_raw = state_buf.buffer();
- let skip_weights = self.skip_connection_weight.clone();
- let skip = skip_weights.buffer().clone();
+ let out = base_arrays[6].borrow();
+ let out_buf = out.buffer();
+ let skip = self.skip_connection_weight.buffer();
self.ssd_prefill.encode(
encoder,
SSDPrefillArguments {
- x: &x_buf,
- dt: &dt_buf,
- b: &b_buf,
- c: &c_buf,
- d: &skip,
- z: &z_buf,
- state: &state_raw,
- y: &out_buf,
+ x: x_buf,
+ dt: dt_buf,
+ b: b_buf,
+ c: c_buf,
+ d: skip,
+ z: z_buf,
+ state: state_raw,
+ y: out_buf,
suffix_len: suffix_length,
group_size: (self.config.num_heads / self.config.num_groups) as i32,
state_size: self.config.state_dim as i32,
@@ -390,27 +388,26 @@ impl<B: Backend> MambaMixer<B> {
ArrayId::AttentionOutput,
]);
let x = arrays[0].borrow();
- let x_buf = x.buffer().clone();
- drop(x);
+ let x_buf = x.buffer();
+
let b = arrays[1].borrow();
- let b_buf = b.buffer().clone();
- drop(b);
+ let b_buf = b.buffer();
+
let c = arrays[2].borrow();
- let c_buf = c.buffer().clone();
- drop(c);
+ let c_buf = c.buffer();
+
let dt = arrays[3].borrow();
- let dt_buf = dt.buffer().clone();
- drop(dt);
+ let dt_buf = dt.buffer();
+
let z = arrays[4].borrow();
- let z_buf = z.buffer().clone();
- drop(z);
+ let z_buf = z.buffer();
+
let state_arr = arrays[5].borrow();
- let state_buf = state_arr.buffer().clone();
- drop(state_arr);
+ let state_buf = state_arr.buffer();
+
let y = arrays[6].borrow();
- let y_buf = y.buffer().clone();
- let skip_weights = self.skip_connection_weight.clone();
- let skip_buf = skip_weights.buffer().clone();
+ let y_buf = y.buffer();
+ let skip_buf = self.skip_connection_weight.buffer();
let h = self.config.num_heads as u32;
let g = self.config.num_groups as u32;
@@ -425,15 +422,15 @@ impl<B: Backend> MambaMixer<B> {
let state_size = n as i32;
self.ssd_update.encode(
- &x_buf,
- &dt_buf,
- &b_buf,
- &c_buf,
- &skip_buf,
- &z_buf,
- &state_buf,
- &y_buf,
- &state_buf,
+ x_buf,
+ dt_buf,
+ b_buf,
+ c_buf,
+ skip_buf,
+ z_buf,
+ state_buf,
+ y_buf,
+ state_buf,
group_size as u32,
state_size as u32,
x_strides.as_slice(),
diff --git a/crates/uzu/src/encodable_block/quantized_linear.rs b/crates/uzu/src/encodable_block/quantized_linear.rs
index c7641dd1f..97c45f455 100644
--- a/crates/uzu/src/encodable_block/quantized_linear.rs
+++ b/crates/uzu/src/encodable_block/quantized_linear.rs
@@ -139,8 +139,8 @@ impl<B: Backend> QuantizedLinear<B> {
&& deq_biases_shape == [output_dim, k_g])
{
return Err(QuantizedLinearError::InvalidMlxShapes {
- weights: weights_shape.clone().into_boxed_slice(),
- scales: scales_shape.clone().into_boxed_slice(),
+ weights: weights_shape.into_boxed_slice(),
+ scales: scales_shape.into_boxed_slice(),
deq_biases: deq_biases_shape.into_boxed_slice(),
packing_divisor,
});
@@ -164,8 +164,8 @@ impl<B: Backend> QuantizedLinear<B> {
&& zero_points_shape == [output_dim, expected_zero_points_entries])
{
return Err(QuantizedLinearError::InvalidAwqShapes {
- weights: weights_shape.clone().into_boxed_slice(),
- scales: scales_shape.clone().into_boxed_slice(),
+ weights: weights_shape.into_boxed_slice(),
+ scales: scales_shape.into_boxed_slice(),
zero_points: zero_points_shape.into_boxed_slice(),
packing_divisor,
packing_minus_one: packing_divisor - 1,
diff --git a/crates/uzu/src/encodable_block/short_conv_mixer.rs b/crates/uzu/src/encodable_block/short_conv_mixer.rs
index f5f1bf1ab..c8f9da72d 100644
--- a/crates/uzu/src/encodable_block/short_conv_mixer.rs
+++ b/crates/uzu/src/encodable_block/short_conv_mixer.rs
@@ -229,12 +229,11 @@ impl<B: Backend> ShortConvMixer<B> {
let conv_state = arrays[1].borrow_mut();
let out = arrays[2].borrow_mut();
- let in_proj_buf = in_proj.buffer().clone();
- let state_buf = conv_state.buffer().clone();
- let out_buf = out.buffer().clone();
+ let in_proj_buf = in_proj.buffer();
+ let state_buf = conv_state.buffer();
+ let out_buf = out.buffer();
- let conv_weight = self.conv_weight.clone();
- let weight_buf = conv_weight.buffer().clone();
+ let weight_buf = self.conv_weight.buffer();
let bias_buf = self.conv_bias.as_ref().map(|b| b.buffer());
let kernel_size = self.config.kernel_size;
@@ -247,8 +246,8 @@ impl<B: Backend> ShortConvMixer<B> {
let padded_size = padded_rows * self.model_dim * element_size;
let padded_buf = state.context().create_buffer(padded_size).expect("Failed to create padded buffer");
self.short_conv_pack.encode(
- &state_buf,
- &in_proj_buf,
+ state_buf,
+ in_proj_buf,
&padded_buf,
state_stride as u32,
suffix_length as u32,
@@ -259,11 +258,11 @@ impl<B: Backend> ShortConvMixer<B> {
self.short_conv_prefill.encode(
&padded_buf,
- &in_proj_buf,
- &weight_buf,
+ in_proj_buf,
+ weight_buf,
bias_buf,
- &out_buf,
- &state_buf,
+ out_buf,
+ state_buf,
suffix_length as u32,
kernel_size as u32,
self.model_dim as u32 * 3,