A high-performance inference engine for AI models
Lalamo 0.11.0 (#454)
uuuvn0db89d2fe2cc
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rsindex 6408898ba..514497050 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs@@ -86,7 +86,8 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(expected_out_dim: usize,expected_in_dim: usize,) -> AudioResult<StructuredAudioPointwiseConv<B>> {- let weight = tree.leaf("weights")?.validate(&[expected_out_dim, expected_in_dim], data_type)?.read_array()?;+ let weight =+ tree.leaf("weights.weights")?.validate(&[expected_out_dim, expected_in_dim], data_type)?.read_array()?;let bias = tree.leaf("biases")?.validate(&[expected_out_dim], data_type)?.read_array()?;Ok(StructuredAudioPointwiseConv {weight,@@ -128,7 +129,7 @@ pub(super) fn read_convnext_layer<B: Backend>(dim: usize,) -> AudioResult<StructuredAudioConvNeXt<B>> {let hidden_dim = dim.checked_mul(4).ok_or(AudioError::Runtime("ConvNeXt hidden dimension overflow".to_string()))?;- let depthwise_conv = read_conv1d_layer::<B>(&tree.subtree("dwconv")?, data_type, dim, dim, 7, 1, dim)?;+ let depthwise_conv = read_conv1d_layer::<B>(&tree.subtree("depthwise_conv")?, data_type, dim, dim, 7, 1, dim)?;let norm = read_norm_layer::<B>(context,&tree.subtree("norm")?,@@ -138,8 +139,8 @@ pub(super) fn read_convnext_layer<B: Backend>(norm_config.subtract_mean,norm_config.has_biases,)?;- let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, hidden_dim, dim)?;- let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, dim, hidden_dim)?;+ let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pointwise_conv_step1")?, data_type, hidden_dim, dim)?;+ let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pointwise_conv_step2")?, data_type, dim, hidden_dim)?;Ok(StructuredAudioConvNeXt {activation_type,depthwise_conv,diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rsindex 34a20a188..d558dd20d 100644--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -106,7 +106,8 @@ impl StructuredAudioCodecGraph {let root_loader_view = loader.tree();let transformer_subtree_name = "audio_decoder.quantizer.post_module";- let max_sequence_length = decoder_config.transformer_config.context_length.max(required_sequence_length.max(1));+ let max_sequence_length =+ decoder_config.transformer_config.max_sequence_length().unwrap_or(0).max(required_sequence_length.max(1));let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);{let transformer_tree = root_loader_view@@ -177,11 +178,11 @@ impl StructuredAudioCodecGraph {let semantic_tree = quantizer_tree.subtree("semantic_quantizer")?.subtree("quantizers")?.subtree("0")?;let codebook_dim = self.config.codebook_dim;let semantic_codebook = semantic_tree- .leaf("codebook.weights")?+ .leaf("codebook.embedding.weights")?.validate(&[self.semantic_codebook_size, codebook_dim], data_type)?.read_array()?;let semantic_out_proj = semantic_tree- .leaf("out_proj.weights")?+ .leaf("out_proj.weights.weights")?.validate(&[self.input_dim, codebook_dim], data_type)?.read_array()?;let semantic_out_bias =@@ -200,11 +201,11 @@ impl StructuredAudioCodecGraph {for index in 0..residual_quantizers {let quantizer_tree = residual_root.subtree(&index.to_string())?;let codebook = quantizer_tree- .leaf("codebook.weights")?+ .leaf("codebook.embedding.weights")?.validate(&[self.codebook_size, codebook_dim], data_type)?.read_array()?;let out_proj = quantizer_tree- .leaf("out_proj.weights")?+ .leaf("out_proj.weights.weights")?.validate(&[self.input_dim, codebook_dim], data_type)?.read_array()?;let out_bias =diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rsindex 49b9da09f..ef12fe9b9 100644--- a/crates/backend-uzu/src/config/rope/mod.rs+++ b/crates/backend-uzu/src/config/rope/mod.rs@@ -14,6 +14,5 @@ pub mod yarn_rope;pub struct RoPEConfig {pub base: f32,pub max_sequence_length: usize,- pub head_dim: Option<usize>,- pub partial_rotary_dim: Option<usize>,+ pub head_dim: usize,}diff --git a/crates/backend-uzu/src/config/transformer.rs b/crates/backend-uzu/src/config/transformer.rsindex b968c2f93..8f3a794c1 100644--- a/crates/backend-uzu/src/config/transformer.rs+++ b/crates/backend-uzu/src/config/transformer.rs@@ -8,5 +8,15 @@ pub struct TransformerConfig {pub output_norm_config: NormalizationConfig,pub model_dim: usize,pub hidden_dim: usize,- pub context_length: usize,+}++impl TransformerConfig {+ pub fn max_sequence_length(&self) -> Option<usize> {+ self.layer_configs+ .iter()+ .filter_map(|layer_config| {+ layer_config.rope_config.as_ref().map(|rope_config| *rope_config.max_sequence_length())+ })+ .max()+ }}diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rsindex 0907943b0..2730adb3f 100644--- a/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs+++ b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs@@ -1,3 +1,4 @@+use monostate::MustBe;use proc_macros::uzu_config;use crate::config::{@@ -51,7 +52,7 @@ pub struct DownsampleResidualVectorQuantizeConfig {pub struct ResidualUnitConfig {pub snake_config: Snake1dConfig,pub conv_config: CausalConv1dConfig,- pub causal: bool,+ pub causal: MustBe!(true),}#[uzu_config]@@ -59,7 +60,7 @@ pub struct DACDecoderBlockConfig {pub snake_config: Snake1dConfig,pub trans_conv_config: CausalTransposeConv1dConfig,pub res_unit_config: ResidualUnitConfig,- pub causal: bool,+ pub causal: MustBe!(true),}#[uzu_config]@@ -67,5 +68,5 @@ pub struct DACDecoderConfig {pub conv_config: CausalConv1dConfig,pub snake_config: Snake1dConfig,pub decoder_block_config: DACDecoderBlockConfig,- pub causal: bool,+ pub causal: MustBe!(true),}diff --git a/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs b/crates/backend-uzu/src/config/weight_matrix/int_spec.rssimilarity index 90%rename from crates/backend-uzu/src/config/weight_matrix/awq_spec.rsrename to crates/backend-uzu/src/config/weight_matrix/int_spec.rsindex 914c4eded..500478697 100644--- a/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs+++ b/crates/backend-uzu/src/config/weight_matrix/int_spec.rs@@ -3,7 +3,7 @@ use proc_macros::uzu_config;use crate::config::weight_matrix::Layout;#[uzu_config(super::WeightMatrixSpec)]-pub struct AWQSpec {+pub struct IntSpec {pub bits: u32,pub group_size: usize,pub is_symmetric: bool,diff --git a/crates/backend-uzu/src/config/weight_matrix/mod.rs b/crates/backend-uzu/src/config/weight_matrix/mod.rsindex bd87da016..f02036b09 100644--- a/crates/backend-uzu/src/config/weight_matrix/mod.rs+++ b/crates/backend-uzu/src/config/weight_matrix/mod.rs@@ -1,8 +1,8 @@use proc_macros::{uzu_config, uzu_config_abstract};-pub mod awq_spec;pub mod full_precision_spec;pub mod hybrid_spec;+pub mod int_spec;pub mod low_rank_spec;pub mod mlx_spec;@@ -17,7 +17,7 @@ pub enum Layout {full_precision_spec::FullPrecisionSpec,low_rank_spec::LowRankSpec,hybrid_spec::HybridSpec,- awq_spec::AWQSpec,+ int_spec::IntSpec,mlx_spec::MLXSpec)]pub struct WeightMatrixSpec;diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rsindex 223ddc0d2..83089a518 100644--- a/crates/backend-uzu/src/encodable_block/decoder.rs+++ b/crates/backend-uzu/src/encodable_block/decoder.rs@@ -91,27 +91,25 @@ impl<B: Backend> Decoder<B> {})}- /// Used by models whose token lookup weights and logits readout weights+ /// Used by TTS models whose token lookup weights live outside the+ /// standard decoder subtree. The TTS runtime owns its separate readout.#[cfg(metal_backend)]- pub fn new_with_embedding_and_readout_subtrees(+ pub fn new_with_embedding_subtree(context: &B::Context,decoder_config: &DecoderConfig,root_weight_loader: &ParameterTree<B>,transformer_subtree: &str,embedding_subtree: &str,- readout_subtree: &str,model_shape: &ModelShape,) -> Result<Self, DecoderError<B>> {let embedding_weight_loader = root_weight_loader.subtree(embedding_subtree)?;- let readout_weight_loader = root_weight_loader.subtree(readout_subtree)?;- let (embed, readout_input_hadamard_factors) = Embedding::new_with_lookup_and_readout_trees(+ let (embed, readout_input_hadamard_factors) = Embedding::new(context,decoder_config.vocab_size as u32,decoder_config.transformer_config.model_dim as u32,&decoder_config.embedding_config,&embedding_weight_loader,- &readout_weight_loader,model_shape,)?;diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rsindex 8c4fe862c..405cf7728 100644--- a/crates/backend-uzu/src/encodable_block/embedding.rs+++ b/crates/backend-uzu/src/encodable_block/embedding.rs@@ -17,9 +17,9 @@ use crate::{embedding::AnyEmbeddingConfig,weight_matrix::{AnyWeightMatrixSpec, Layout,- awq_spec::AWQSpec,full_precision_spec::FullPrecisionSpec,hybrid_spec::{HybridSpec, IncoherenceProcessingMode},+ int_spec::IntSpec,mlx_spec::MLXSpec,},},@@ -116,30 +116,10 @@ impl<B: Backend> Embedding<B> {config: &AnyEmbeddingConfig,parameter_tree: &ParameterTree<B>,model_shape: &ModelShape,- ) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {- Self::new_with_lookup_and_readout_trees(- context,- vocab_size,- model_dim,- config,- parameter_tree,- parameter_tree,- model_shape,- )- }-- pub fn new_with_lookup_and_readout_trees(- context: &B::Context,- vocab_size: u32,- model_dim: u32,- config: &AnyEmbeddingConfig,- lookup_tree: &ParameterTree<B>,- readout_tree: &ParameterTree<B>,- model_shape: &ModelShape,) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {let (tying, readout_input_hadamard_factors) = match config {AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {- let embedding_tree = lookup_tree.subtree("embedding")?;+ let embedding_tree = parameter_tree.subtree("embedding")?;let embedding_spec = embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;let (ty, readout_input_hadamard_factors) = match embedding_spec {@@ -176,7 +156,7 @@ impl<B: Backend> Embedding<B> {None,)},- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {let (embedding_quantization_mode, group_size, quantization_method) =input_quantization_from_spec(spec)?;let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(@@ -295,7 +275,7 @@ impl<B: Backend> Embedding<B> {)},AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {- let input_embedding_tree = lookup_tree.subtree("input_embedding")?;+ let input_embedding_tree = parameter_tree.subtree("input_embedding")?;let input_embedding_spec = input_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;let input_ty = match input_embedding_spec {@@ -319,7 +299,7 @@ impl<B: Backend> Embedding<B> {lookup,}},- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {let (embedding_quantization_mode, group_size, quantization_method) =input_quantization_from_spec(spec)?;let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(@@ -354,7 +334,7 @@ impl<B: Backend> Embedding<B> {spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),};- let output_embedding_tree = readout_tree.subtree("output_embedding")?;+ let output_embedding_tree = parameter_tree.subtree("output_embedding")?;let output_embedding_spec = output_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;let output_ty = match output_embedding_spec {@@ -381,7 +361,7 @@ impl<B: Backend> Embedding<B> {readout,}},- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {let (embedding_quantization_mode, group_size, quantization_method) =output_quantization_from_spec(spec)?;let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(@@ -660,14 +640,14 @@ fn input_quantization_from_spec<B: Backend>(layout: Layout::InputOutput,..}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),- AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ AnyWeightMatrixSpec::IntSpec(IntSpec {bits,group_size,is_symmetric: false,layout: Layout::InputOutput,..}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),- AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ AnyWeightMatrixSpec::IntSpec(IntSpec {bits,group_size,is_symmetric: true,@@ -688,14 +668,14 @@ fn output_quantization_from_spec<B: Backend>(layout: Layout::OutputInput,..}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),- AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ AnyWeightMatrixSpec::IntSpec(IntSpec {bits,group_size,is_symmetric: false,layout: Layout::OutputInput,..}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),- AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ AnyWeightMatrixSpec::IntSpec(IntSpec {bits,group_size,is_symmetric: true,diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rsindex d851fc9d7..edd14a0ea 100644--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs@@ -14,7 +14,7 @@ use crate::{matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},},},- config::weight_matrix::{AnyWeightMatrixSpec, Layout, awq_spec::AWQSpec, mlx_spec::MLXSpec},+ config::weight_matrix::{AnyWeightMatrixSpec, Layout, int_spec::IntSpec, mlx_spec::MLXSpec},parameters::{ParameterLoaderError, ParameterTree},};@@ -114,14 +114,14 @@ impl<B: Backend> LinearMatmul<B> {layout: Layout::OutputInput,..}) => (bits, group_size, QuantizationMethod::ScaleBias),- AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ AnyWeightMatrixSpec::IntSpec(IntSpec {bits,group_size,is_symmetric: false,layout: Layout::OutputInput,..}) => (bits, group_size, QuantizationMethod::ScaleZeroPoint),- AnyWeightMatrixSpec::AWQSpec(AWQSpec {+ AnyWeightMatrixSpec::IntSpec(IntSpec {bits,group_size,is_symmetric: true,diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rsindex 6b044aabf..ec772901f 100644--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs@@ -125,7 +125,7 @@ impl<B: Backend> dyn Linear<B> {&weights_tree,)?))},- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {let block = LinearMatmul::quantized(context,spec,@@ -178,7 +178,7 @@ impl<B: Backend> dyn Linear<B> {let weights_tree = parameter_tree.subtree("weights")?.subtree("quantized")?;let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;match spec {- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {Ok(Box::new(LinearMatmul::quantized(context,spec,diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rsindex 57b50f9d9..e5ed8867d 100644--- a/crates/backend-uzu/src/forward_pass/model_shape.rs+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs@@ -10,7 +10,6 @@ pub struct ModelShape {vocabulary_size: usize,model_dim: usize,- context_length: usize,num_groups: usize,pub num_layers: usize,@@ -42,7 +41,6 @@ impl ModelShape {rope_data_type: DataType::F32,vocabulary_size: decoder_config.vocab_size,model_dim: tf.model_dim,- context_length: tf.context_length,num_groups,num_layers,layer_mixers,@@ -94,8 +92,4 @@ impl ModelShape {) -> [usize; 2] {[suffix_length, self.vocabulary_size]}-- pub fn context_length(&self) -> usize {- self.context_length- }}diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rsindex de58f52a8..4799ea6fa 100644--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs+++ b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs@@ -31,13 +31,13 @@ impl<B: Backend> SharedBuffers<B> {.layer_configs.iter().map(|layer_config| {- let Some(attention_config) = layer_config.mixer_config.as_attention() else {+ if layer_config.mixer_config.as_attention().is_none() {return LayerRopeKind::NoKernel;- };+ }let Some(rope_config) = &layer_config.rope_config else {return LayerRopeKind::NoKernel;};- let head_dim = rope_config.head_dim().unwrap_or(attention_config.head_dim);+ let head_dim = *rope_config.head_dim();let index = configs.iter().position(|(existing_config, existing_head_dim)| {diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rsindex 2c9028212..79419832e 100644--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs@@ -198,7 +198,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {decoder_config: &DecoderConfig,decoding_config: &DecodingConfig,) -> usize {- let model_length = decoder_config.transformer_config.context_length;+ let model_length = decoder_config.transformer_config.max_sequence_length().unwrap_or(65536);let proposed_value = match decoding_config.context_length {ContextLength::Default => {if context.sparse_buffers_supported() {diff --git a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs b/crates/backend-uzu/src/session/parameter/prefill_step_size.rsindex 027c909a0..1d9caffb8 100644--- a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs+++ b/crates/backend-uzu/src/session/parameter/prefill_step_size.rs@@ -31,7 +31,8 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {config: &LanguageModelConfig,) -> usize {let default_limit: usize = 4096;- let model_context_length = config.decoder_config.transformer_config.context_length;+ let model_context_length =+ config.decoder_config.transformer_config.max_sequence_length().unwrap_or(default_limit);let minimal_sliding_window_size = config.decoder_configdiff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rsindex cbb0280ed..d6198f064 100644--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs@@ -3,7 +3,7 @@ use crate::{array::{Array, ArrayContextExt},backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},config::decoder::DecoderConfig,- encodable_block::{DecoderArguments, DecoderDecodeInput, SamplingInputs},+ encodable_block::{DecoderArguments, DecoderDecodeInput, LayerArguments, Linear, SamplingInputs},forward_pass::token_inputs::TokenInputs,session::types::TtsModelConfigError,};@@ -11,6 +11,7 @@ use crate::{struct TokenDecoderLoadedModel<B: Backend> {shared_buffers: Rc<SharedBuffers<B>>,executables: Decoder<B>,+ readout: Box<dyn Linear<B>>,sampler: GpuSampling<B>,token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,}@@ -32,16 +33,26 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {root_loader_view,transformer_subtree,)?;- let executables = Decoder::new_with_embedding_and_readout_subtrees(+ let executables = Decoder::new_with_embedding_subtree(context.as_ref(),&decoder_config,root_loader_view,transformer_subtree,embedding_subtree,- readout_subtree,model_shape,).map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;+ let readout_tree =+ root_loader_view.subtree(readout_subtree).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;+ let readout = <dyn Linear<B>>::new(+ decoder_config.transformer_config.model_dim,+ [decoder_config.vocab_size],+ false,+ context.as_ref(),+ model_shape.data_type,+ &readout_tree,+ )+ .map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;let sampler = GpuSampling::new(context.as_ref(), model_shape.data_type).map_err(unable_to_create_context)?;let token_copy_sampled =<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;@@ -49,6 +60,7 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {Ok(Self {shared_buffers,executables,+ readout,sampler,token_copy_sampled,})@@ -63,6 +75,7 @@ struct TokenDecoderContext<B: Backend> {decoder_config: Rc<DecoderConfig>,runtime_config: TextDecoderRuntimeConfig,executables: Decoder<B>,+ readout: Box<dyn Linear<B>>,sampler: GpuSampling<B>,kv_cache_update: KVCacheUpdate<B>,token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,@@ -94,8 +107,11 @@ impl<B: Backend> TokenDecoderContext<B> {let root_loader_view = loader.tree();let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::F32);- let max_suffix_length =- text_decoder_prefill_step_size(runtime_config, decoder_config.transformer_config.context_length).max(32);+ let max_suffix_length = text_decoder_prefill_step_size(+ runtime_config,+ decoder_config.transformer_config.max_sequence_length().unwrap_or(usize::MAX),+ )+ .max(32);let max_prefix_length = max_suffix_length;let loaded_model = TokenDecoderLoadedModel::<B>::load(&context,@@ -126,6 +142,7 @@ impl<B: Backend> TokenDecoderContext<B> {decoder_config,runtime_config: runtime_config.clone(),executables: loaded_model.executables,+ readout: loaded_model.readout,sampler: loaded_model.sampler,kv_cache_update,token_copy_sampled: loaded_model.token_copy_sampled,@@ -331,7 +348,12 @@ impl<B: Backend> TokenDecoderRunner<B> {&mut self,max_prefix_length: usize,) -> Result<(), Error> {- let max_prefix_length = max_prefix_length.max(1).min(self.ctx.decoder_config.transformer_config.context_length);+ let max_prefix_length = self+ .ctx+ .decoder_config+ .transformer_config+ .max_sequence_length()+ .map_or(max_prefix_length.max(1), |model_length| max_prefix_length.max(1).min(model_length));let max_suffix_length = text_decoder_prefill_step_size(&self.ctx.runtime_config, max_prefix_length).max(32);if self.ctx.current_max_prefix_length == max_prefix_length {return Ok(());@@ -554,7 +576,6 @@ impl<B: Backend> TokenDecoderRunner<B> {pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,pending_sampling_outputs: &mut Vec<Allocation<B>>,) -> Result<(), Error> {- let cache_layers_rc = self.ctx.cache_layers.clone();let token_bitmask =vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit)).map(Box::<[u32]>::from);let mut next_token_ids = first_token_ids;@@ -570,19 +591,15 @@ impl<B: Backend> TokenDecoderRunner<B> {let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");{- let mut logits = {- let mut cache_layers = cache_layers_rc.borrow_mut();- cache_layers.prepare_for_forward_pass(&self.ctx.context, 1);- self.ctx- .executables- .encode_decode(- self.decoder_arguments(&token_inputs, 1, 0, 1, &mut *cache_layers),- DecoderDecodeInput::TokenIds(token_inputs.token_ids()),- None,- encoder,- )- .map_err(|err| Error::EncodeFailed(Box::new(err)))?- };+ let mut logits = self.encode_decode_with_fishaudio_readout_on(+ encoder,+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),+ token_inputs,+ 1,+ 0,+ 1,+ false,+ )?;let mut sampling_output = self.create_sampling_output(1);self.encode_sampling_on(encoder,@@ -690,6 +707,75 @@ impl<B: Backend> TokenDecoderRunner<B> {self.ctx.context.create_array_uninitialized(&[sampling_length], crate::DataType::U32).into_allocation()}+ fn encode_decode_with_fishaudio_readout_on<'input>(+ &mut self,+ encoder: &mut Encoder<B>,+ input: DecoderDecodeInput<'input, B>,+ token_inputs: &TokenInputs<B>,+ batch_dim: usize,+ sampling_start: usize,+ sampling_length: usize,+ capture_hidden: bool,+ ) -> Result<Allocation<B>, Error> {+ let mut main = match input {+ DecoderDecodeInput::TokenIds(token_ids) => self+ .ctx+ .executables+ .embed+ .encode_lookup(token_ids, batch_dim, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?,+ DecoderDecodeInput::Embeddings(main) => main,+ };+ let mut shortcut = encoder+ .allocate_scratch(main.as_buffer_range_ref().range().len())+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;++ {+ let mut cache_layers = self.ctx.cache_layers.borrow_mut();+ cache_layers.prepare_for_forward_pass(&self.ctx.context, batch_dim);+ for layer in self.ctx.executables.layers.iter() {+ main = layer+ .encode(+ LayerArguments {+ batch_dim,+ token_positions: token_inputs.token_positions(),+ token_parents: token_inputs.token_parents(),+ token_subtrie_ranges: token_inputs.token_subtrie_ranges(),+ rope_buffers: self.ctx.shared_buffers.rope_buffers_for_layer(layer.layer_index),+ sampling_start,+ sampling_length,+ cache_access: Some(cache_layers.cache_for_layer(layer.layer_index)),+ #[cfg(feature = "tracing")]+ trace: None,+ },+ main,+ &mut shortcut,+ encoder,+ )+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ }+ }++ let output_norm = self+ .ctx+ .executables+ .norm+ .encode(&main, sampling_start, sampling_length, Some(&mut shortcut), encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ if capture_hidden {+ let row_size_bytes = shortcut.as_buffer_range_ref().range().len() / batch_dim;+ let input_offset = (batch_dim - 1) * row_size_bytes;+ encoder.encode_copy(+ &shortcut,+ input_offset..input_offset + row_size_bytes,+ &mut self.single_hidden_capture,+ ..,+ );+ }++ self.ctx.readout.encode(output_norm, sampling_length, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))+ }+fn encode_single_forward_pass_on(&mut self,encoder: &mut Encoder<B>,@@ -793,27 +879,15 @@ impl<B: Backend> TokenDecoderRunner<B> {self.encode_override_first_row_from_device_on(encoder, &mut main, &self.single_override_embedding)?;},}- let mut logits = {- let mut cache_layers = self.ctx.cache_layers.borrow_mut();- cache_layers.prepare_for_forward_pass(&self.ctx.context, token_count);- let decoder_arguments = DecoderArguments {- token_positions: token_inputs.token_positions(),- token_parents: token_inputs.token_parents(),- token_subtrie_ranges: token_inputs.token_subtrie_ranges(),- shared_buffers: self.ctx.shared_buffers.as_ref(),- cache_layers: Some(&mut *cache_layers),- batch_dim: token_count,- sampling_start,- sampling_length,- #[cfg(feature = "tracing")]- trace: None,- };- let hidden_capture = capture_hidden.then_some(&mut self.single_hidden_capture);- self.ctx- .executables- .encode_decode(decoder_arguments, DecoderDecodeInput::Embeddings(main), hidden_capture, encoder)- .map_err(|err| Error::EncodeFailed(Box::new(err)))?- };+ let mut logits = self.encode_decode_with_fishaudio_readout_on(+ encoder,+ DecoderDecodeInput::Embeddings(main),+ token_inputs,+ token_count,+ sampling_start,+ sampling_length,+ capture_hidden,+ )?;let mut sampling_output = self.create_sampling_output(sampling_length);self.encode_sampling_on(encoder,diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rsindex 8888ef48b..0bea2c31a 100644--- a/crates/backend-uzu/src/utils/version.rs+++ b/crates/backend-uzu/src/utils/version.rs@@ -1,2 +1,2 @@pub const VERSION: &str = env!("CARGO_PKG_VERSION");-pub const TOOLCHAIN_VERSION: &str = "0.10.0";+pub const TOOLCHAIN_VERSION: &str = "0.11.0";