Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fix attention bias fill

LuckyIYI23f0856ea57c
diff --git a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
index 069f2e452..7cb3cfdfd 100644
--- a/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/cache_layers.rs
@@ -266,17 +266,7 @@ impl CacheLayers {
) {
for layer in self.data.iter() {
if let CacheLayer::Transformer(layer) = layer {
- let key: Option<usize> = match &layer.state {
- KVCacheLayerState::Full {
- ..
- } => None,
- KVCacheLayerState::Windowed {
- window_length,
- ..
- } => Some(*window_length),
- };
-
- if let Some(array) = dst.get_mut(&key) {
+ if let Some(array) = dst.get_mut(&layer.window_length()) {
layer.fill_attention_bias(
array,
suffix_token_positions,
@@ -289,6 +279,28 @@ impl CacheLayers {
}
}
+ pub fn fill_attention_bias_scratch(
+ &self,
+ dst: &HashMap<Option<usize>, RefCell<MetalArray>>,
+ suffix_token_positions: &[usize],
+ suffix_length: usize,
+ context: &MTLContext,
+ ) {
+ for layer in self.data.iter() {
+ if let CacheLayer::Transformer(layer) = layer {
+ if let Some(cell) = dst.get(&layer.window_length()) {
+ layer.fill_attention_bias(
+ &mut cell.borrow_mut(),
+ suffix_token_positions,
+ suffix_length,
+ context,
+ None,
+ );
+ }
+ }
+ }
+ }
+
pub fn update_after_acceptance(
&mut self,
accepted_suffix_indices: &[usize],
diff --git a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
index 86f5747ed..970c00c01 100644
--- a/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
+++ b/crates/uzu/src/backends/metal/forward_pass/state/mod.rs
@@ -13,7 +13,9 @@ pub use array_id::ArrayId;
pub use common_aux_buffers::CommonAuxBuffers;
pub use hash_map_id::HashMapId;
pub use language_model_generator_aux_buffers::LanguageModelGeneratorAuxBuffers;
-pub use mode::{ClassifierModeState, ForwardPassMode, LanguageModelGeneratorModeState};
+pub use mode::{
+ ClassifierModeState, ForwardPassMode, LanguageModelGeneratorModeState,
+};
pub use rope_buffers::RopeBuffers;
pub use rope_type::RopeType;
pub use shared_buffers::{MoeExpertWeights, SharedBuffers};
@@ -24,8 +26,9 @@ use super::{ModelShape, ScratchBuffers, cache_layers::CacheLayers};
use crate::{
Array, DataType, DecoderConfig, DeviceContext,
backends::metal::{
- MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder, MTLContext,
- MTLDeviceExt, MTLResourceOptions, MetalArray, ProtocolObject, Retained,
+ MTLBlitCommandEncoder, MTLBuffer, MTLCommandBuffer, MTLCommandEncoder,
+ MTLContext, MTLDeviceExt, MTLResourceOptions, MetalArray,
+ ProtocolObject, Retained,
},
session::parameter::SamplingMethod,
};
@@ -110,8 +113,11 @@ impl ForwardPassState {
is_prefilling: bool,
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
skip_token_ids_copy: bool,
- skip_attention_bias_fill: bool,
- async_positions: Option<(&Retained<ProtocolObject<dyn MTLBuffer>>, usize)>,
+ should_fill_attention_bias: bool,
+ async_positions: Option<(
+ &Retained<ProtocolObject<dyn MTLBuffer>>,
+ usize,
+ )>,
async_seeds: Option<(&Retained<ProtocolObject<dyn MTLBuffer>>, usize)>,
) -> Self {
let suffix_length = token_ids.len();
@@ -222,7 +228,7 @@ impl ForwardPassState {
act_dtype,
token_positions,
external_bias_fn,
- skip_attention_bias_fill,
+ should_fill_attention_bias,
);
// Common aux buffers
@@ -282,7 +288,7 @@ impl ForwardPassState {
act_dtype: DataType,
token_positions: &[usize],
external_bias_fn: Option<&dyn Fn(usize, usize) -> bool>,
- skip_fill: bool,
+ should_fill_attention_bias: bool,
) -> HashMap<Option<usize>, ArrayCell> {
let cache_ref = cache_layers.borrow();
let mut attention_bias_map: HashMap<Option<usize>, MetalArray> =
@@ -309,7 +315,7 @@ impl ForwardPassState {
// Use cache_layers' fill_attention_bias which properly handles
// both causal masking and sliding window constraints
// Skip fill for async decode passes after the first one (bias already set)
- if !skip_fill {
+ if should_fill_attention_bias {
cache_layers.borrow().fill_attention_bias(
&mut attention_bias_map,
token_positions,
diff --git a/crates/uzu/src/language_model/language_model_generator.rs b/crates/uzu/src/language_model/language_model_generator.rs
index 838560bf0..70c815fdd 100644
--- a/crates/uzu/src/language_model/language_model_generator.rs
+++ b/crates/uzu/src/language_model/language_model_generator.rs
@@ -1,4 +1,6 @@
-use std::{collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant};
+use std::{
+ collections::HashMap, iter::repeat_n, path::Path, sync::Arc, time::Instant,
+};
use itertools::{Either, Itertools, izip};
@@ -13,11 +15,11 @@ use super::{
use crate::{
Array,
backends::metal::{
- MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt, MTLCommandBufferHandler,
- MTLCommandEncoder, MTLCommandQueue,
+ MTLBuffer, MTLCommandBuffer, MTLCommandBufferExt,
+ MTLCommandBufferHandler, MTLCommandEncoder, MTLCommandQueue,
forward_pass::{
- AttentionBiasUpdate, EncodableBlock, EncodingParameters, ForwardPassState,
- INVALID_POSITION,
+ AttentionBiasUpdate, EncodableBlock, EncodingParameters,
+ ForwardPassState, INVALID_POSITION,
},
},
session::{
@@ -234,7 +236,7 @@ impl LanguageModelGenerator {
false,
self.allow_pre_encode(),
sampling_method,
- self.skip_attention_bias_fill(),
+ self.should_fill_attention_bias(),
);
if should_capture {
@@ -401,7 +403,7 @@ impl LanguageModelGenerator {
false,
self.allow_pre_encode(),
sampling_method,
- self.skip_attention_bias_fill(),
+ self.should_fill_attention_bias(),
);
let sampled_tokens = self.sample(&mut state)?;
@@ -427,24 +429,20 @@ impl LanguageModelGenerator {
tokens_to_generate: usize,
) {
let prefill_count = self.tokens.len();
+ let first_decode_position = prefill_count.saturating_sub(1);
- // Initialize attention bias buffers to zero for async mode
- // This ensures unwritten columns (beyond what pass 0 fills) are 0 (attend)
- // rather than garbage (which could be -inf and mask incorrectly)
- for (_, mask_buffer) in
- &self.context.scratch_buffers.attention_window_size_to_bias
- {
- unsafe {
- let ptr =
- mask_buffer.borrow().backend_buffer().contents().as_ptr()
- as *mut u8;
- std::ptr::write_bytes(
- ptr,
- 0,
- mask_buffer.borrow().backend_buffer().length() as usize,
- );
- }
- }
+ // Fill attention bias from KV cache state for the first decode position.
+ // Windowed buffers get the correct mask; GPU patches maintain it for subsequent passes.
+ // Full-attention buffers also get filled (masking beyond-prefix positions).
+ self.context
+ .cache_layers
+ .borrow()
+ .fill_attention_bias_scratch(
+ &self.context.scratch_buffers.attention_window_size_to_bias,
+ &[first_decode_position],
+ 1,
+ &self.context.mtl_context,
+ );
self.context
.async_buffers
@@ -509,9 +507,7 @@ impl LanguageModelGenerator {
let async_positions = Some((&async_positions_buffer, pass_idx));
let async_seeds = Some((&async_seeds_buffer, pass_idx));
- let skip_attention_bias_fill =
- pass_idx > 0 && self.skip_attention_bias_fill();
-
+ let should_fill_attention_bias = false; // we fill it once in prepare
let skip_token_ids_copy = pass_idx > 0;
let is_first_decode = !is_continuation;
@@ -540,7 +536,7 @@ impl LanguageModelGenerator {
task.is_prefilling,
None,
skip_token_ids_copy,
- skip_attention_bias_fill,
+ should_fill_attention_bias,
async_positions,
async_seeds,
);
@@ -575,13 +571,10 @@ impl LanguageModelGenerator {
let sampling_output = state
.sampling_output()
.expect("sampling_output must exist after sampling encode");
- let sampling_output_buffer = sampling_output.borrow().mtl_buffer_cloned();
- let token_ids_buffer = self
- .context
- .scratch_buffers
- .token_ids
- .borrow()
- .mtl_buffer_cloned();
+ let sampling_output_buffer =
+ sampling_output.borrow().mtl_buffer_cloned();
+ let token_ids_buffer =
+ self.context.scratch_buffers.token_ids.borrow().mtl_buffer_cloned();
let encoder = root_command_buffer
.new_compute_command_encoder()
@@ -898,14 +891,14 @@ impl LanguageModelGenerator {
}
}
- fn skip_attention_bias_fill(&self) -> bool {
+ fn should_fill_attention_bias(&self) -> bool {
let sliding_window_sizes =
self.context.model_shape.sliding_window_length_per_layer.clone();
let has_sliding_window =
sliding_window_sizes.iter().any(|size| size.is_some());
let has_speculative_suffix =
self.decoding_config.generate_suffix_length() > 1;
- let should_skip = !has_sliding_window && !has_speculative_suffix;
- return should_skip;
+
+ has_sliding_window || has_speculative_suffix
}
}