Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Lalamo 0.11.0 (#454)

uuuvn0db89d2fe2cc
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
index 6408898ba..514497050 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/build.rs
@@ -86,7 +86,8 @@ pub(super) fn read_pointwise_conv_layer<B: Backend>(
expected_out_dim: usize,
expected_in_dim: usize,
) -> AudioResult<StructuredAudioPointwiseConv<B>> {
- let weight = tree.leaf("weights")?.validate(&[expected_out_dim, expected_in_dim], data_type)?.read_array()?;
+ let weight =
+ tree.leaf("weights.weights")?.validate(&[expected_out_dim, expected_in_dim], data_type)?.read_array()?;
let bias = tree.leaf("biases")?.validate(&[expected_out_dim], data_type)?.read_array()?;
Ok(StructuredAudioPointwiseConv {
weight,
@@ -128,7 +129,7 @@ pub(super) fn read_convnext_layer<B: Backend>(
dim: usize,
) -> AudioResult<StructuredAudioConvNeXt<B>> {
let hidden_dim = dim.checked_mul(4).ok_or(AudioError::Runtime("ConvNeXt hidden dimension overflow".to_string()))?;
- let depthwise_conv = read_conv1d_layer::<B>(&tree.subtree("dwconv")?, data_type, dim, dim, 7, 1, dim)?;
+ let depthwise_conv = read_conv1d_layer::<B>(&tree.subtree("depthwise_conv")?, data_type, dim, dim, 7, 1, dim)?;
let norm = read_norm_layer::<B>(
context,
&tree.subtree("norm")?,
@@ -138,8 +139,8 @@ pub(super) fn read_convnext_layer<B: Backend>(
norm_config.subtract_mean,
norm_config.has_biases,
)?;
- let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, hidden_dim, dim)?;
- let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, dim, hidden_dim)?;
+ let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pointwise_conv_step1")?, data_type, hidden_dim, dim)?;
+ let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pointwise_conv_step2")?, data_type, dim, hidden_dim)?;
Ok(StructuredAudioConvNeXt {
activation_type,
depthwise_conv,
diff --git a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
index 34a20a188..d558dd20d 100644
--- a/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
+++ b/crates/backend-uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -106,7 +106,8 @@ impl StructuredAudioCodecGraph {
let root_loader_view = loader.tree();
let transformer_subtree_name = "audio_decoder.quantizer.post_module";
- let max_sequence_length = decoder_config.transformer_config.context_length.max(required_sequence_length.max(1));
+ let max_sequence_length =
+ decoder_config.transformer_config.max_sequence_length().unwrap_or(0).max(required_sequence_length.max(1));
let mut shared_buffers = SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape);
{
let transformer_tree = root_loader_view
@@ -177,11 +178,11 @@ impl StructuredAudioCodecGraph {
let semantic_tree = quantizer_tree.subtree("semantic_quantizer")?.subtree("quantizers")?.subtree("0")?;
let codebook_dim = self.config.codebook_dim;
let semantic_codebook = semantic_tree
- .leaf("codebook.weights")?
+ .leaf("codebook.embedding.weights")?
.validate(&[self.semantic_codebook_size, codebook_dim], data_type)?
.read_array()?;
let semantic_out_proj = semantic_tree
- .leaf("out_proj.weights")?
+ .leaf("out_proj.weights.weights")?
.validate(&[self.input_dim, codebook_dim], data_type)?
.read_array()?;
let semantic_out_bias =
@@ -200,11 +201,11 @@ impl StructuredAudioCodecGraph {
for index in 0..residual_quantizers {
let quantizer_tree = residual_root.subtree(&index.to_string())?;
let codebook = quantizer_tree
- .leaf("codebook.weights")?
+ .leaf("codebook.embedding.weights")?
.validate(&[self.codebook_size, codebook_dim], data_type)?
.read_array()?;
let out_proj = quantizer_tree
- .leaf("out_proj.weights")?
+ .leaf("out_proj.weights.weights")?
.validate(&[self.input_dim, codebook_dim], data_type)?
.read_array()?;
let out_bias =
diff --git a/crates/backend-uzu/src/config/rope/mod.rs b/crates/backend-uzu/src/config/rope/mod.rs
index 49b9da09f..ef12fe9b9 100644
--- a/crates/backend-uzu/src/config/rope/mod.rs
+++ b/crates/backend-uzu/src/config/rope/mod.rs
@@ -14,6 +14,5 @@ pub mod yarn_rope;
pub struct RoPEConfig {
pub base: f32,
pub max_sequence_length: usize,
- pub head_dim: Option<usize>,
- pub partial_rotary_dim: Option<usize>,
+ pub head_dim: usize,
}
diff --git a/crates/backend-uzu/src/config/transformer.rs b/crates/backend-uzu/src/config/transformer.rs
index b968c2f93..8f3a794c1 100644
--- a/crates/backend-uzu/src/config/transformer.rs
+++ b/crates/backend-uzu/src/config/transformer.rs
@@ -8,5 +8,15 @@ pub struct TransformerConfig {
pub output_norm_config: NormalizationConfig,
pub model_dim: usize,
pub hidden_dim: usize,
- pub context_length: usize,
+}
+
+impl TransformerConfig {
+ pub fn max_sequence_length(&self) -> Option<usize> {
+ self.layer_configs
+ .iter()
+ .filter_map(|layer_config| {
+ layer_config.rope_config.as_ref().map(|rope_config| *rope_config.max_sequence_length())
+ })
+ .max()
+ }
}
diff --git a/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs
index 0907943b0..2730adb3f 100644
--- a/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs
+++ b/crates/backend-uzu/src/config/tts/audio_decoder/fish_audio_modules.rs
@@ -1,3 +1,4 @@
+use monostate::MustBe;
use proc_macros::uzu_config;
use crate::config::{
@@ -51,7 +52,7 @@ pub struct DownsampleResidualVectorQuantizeConfig {
pub struct ResidualUnitConfig {
pub snake_config: Snake1dConfig,
pub conv_config: CausalConv1dConfig,
- pub causal: bool,
+ pub causal: MustBe!(true),
}
#[uzu_config]
@@ -59,7 +60,7 @@ pub struct DACDecoderBlockConfig {
pub snake_config: Snake1dConfig,
pub trans_conv_config: CausalTransposeConv1dConfig,
pub res_unit_config: ResidualUnitConfig,
- pub causal: bool,
+ pub causal: MustBe!(true),
}
#[uzu_config]
@@ -67,5 +68,5 @@ pub struct DACDecoderConfig {
pub conv_config: CausalConv1dConfig,
pub snake_config: Snake1dConfig,
pub decoder_block_config: DACDecoderBlockConfig,
- pub causal: bool,
+ pub causal: MustBe!(true),
}
diff --git a/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs b/crates/backend-uzu/src/config/weight_matrix/int_spec.rs
similarity index 90%
rename from crates/backend-uzu/src/config/weight_matrix/awq_spec.rs
rename to crates/backend-uzu/src/config/weight_matrix/int_spec.rs
index 914c4eded..500478697 100644
--- a/crates/backend-uzu/src/config/weight_matrix/awq_spec.rs
+++ b/crates/backend-uzu/src/config/weight_matrix/int_spec.rs
@@ -3,7 +3,7 @@ use proc_macros::uzu_config;
use crate::config::weight_matrix::Layout;
#[uzu_config(super::WeightMatrixSpec)]
-pub struct AWQSpec {
+pub struct IntSpec {
pub bits: u32,
pub group_size: usize,
pub is_symmetric: bool,
diff --git a/crates/backend-uzu/src/config/weight_matrix/mod.rs b/crates/backend-uzu/src/config/weight_matrix/mod.rs
index bd87da016..f02036b09 100644
--- a/crates/backend-uzu/src/config/weight_matrix/mod.rs
+++ b/crates/backend-uzu/src/config/weight_matrix/mod.rs
@@ -1,8 +1,8 @@
use proc_macros::{uzu_config, uzu_config_abstract};
-pub mod awq_spec;
pub mod full_precision_spec;
pub mod hybrid_spec;
+pub mod int_spec;
pub mod low_rank_spec;
pub mod mlx_spec;
@@ -17,7 +17,7 @@ pub enum Layout {
full_precision_spec::FullPrecisionSpec,
low_rank_spec::LowRankSpec,
hybrid_spec::HybridSpec,
- awq_spec::AWQSpec,
+ int_spec::IntSpec,
mlx_spec::MLXSpec
)]
pub struct WeightMatrixSpec;
diff --git a/crates/backend-uzu/src/encodable_block/decoder.rs b/crates/backend-uzu/src/encodable_block/decoder.rs
index 223ddc0d2..83089a518 100644
--- a/crates/backend-uzu/src/encodable_block/decoder.rs
+++ b/crates/backend-uzu/src/encodable_block/decoder.rs
@@ -91,27 +91,25 @@ impl<B: Backend> Decoder<B> {
})
}
- /// Used by models whose token lookup weights and logits readout weights
+ /// Used by TTS models whose token lookup weights live outside the
+ /// standard decoder subtree. The TTS runtime owns its separate readout.
#[cfg(metal_backend)]
- pub fn new_with_embedding_and_readout_subtrees(
+ pub fn new_with_embedding_subtree(
context: &B::Context,
decoder_config: &DecoderConfig,
root_weight_loader: &ParameterTree<B>,
transformer_subtree: &str,
embedding_subtree: &str,
- readout_subtree: &str,
model_shape: &ModelShape,
) -> Result<Self, DecoderError<B>> {
let embedding_weight_loader = root_weight_loader.subtree(embedding_subtree)?;
- let readout_weight_loader = root_weight_loader.subtree(readout_subtree)?;
- let (embed, readout_input_hadamard_factors) = Embedding::new_with_lookup_and_readout_trees(
+ let (embed, readout_input_hadamard_factors) = Embedding::new(
context,
decoder_config.vocab_size as u32,
decoder_config.transformer_config.model_dim as u32,
&decoder_config.embedding_config,
&embedding_weight_loader,
- &readout_weight_loader,
model_shape,
)?;
diff --git a/crates/backend-uzu/src/encodable_block/embedding.rs b/crates/backend-uzu/src/encodable_block/embedding.rs
index 8c4fe862c..405cf7728 100644
--- a/crates/backend-uzu/src/encodable_block/embedding.rs
+++ b/crates/backend-uzu/src/encodable_block/embedding.rs
@@ -17,9 +17,9 @@ use crate::{
embedding::AnyEmbeddingConfig,
weight_matrix::{
AnyWeightMatrixSpec, Layout,
- awq_spec::AWQSpec,
full_precision_spec::FullPrecisionSpec,
hybrid_spec::{HybridSpec, IncoherenceProcessingMode},
+ int_spec::IntSpec,
mlx_spec::MLXSpec,
},
},
@@ -116,30 +116,10 @@ impl<B: Backend> Embedding<B> {
config: &AnyEmbeddingConfig,
parameter_tree: &ParameterTree<B>,
model_shape: &ModelShape,
- ) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {
- Self::new_with_lookup_and_readout_trees(
- context,
- vocab_size,
- model_dim,
- config,
- parameter_tree,
- parameter_tree,
- model_shape,
- )
- }
-
- pub fn new_with_lookup_and_readout_trees(
- context: &B::Context,
- vocab_size: u32,
- model_dim: u32,
- config: &AnyEmbeddingConfig,
- lookup_tree: &ParameterTree<B>,
- readout_tree: &ParameterTree<B>,
- model_shape: &ModelShape,
) -> Result<(Self, Option<Allocation<B>>), EmbeddingError<B>> {
let (tying, readout_input_hadamard_factors) = match config {
AnyEmbeddingConfig::TiedEmbeddingConfig(_) => {
- let embedding_tree = lookup_tree.subtree("embedding")?;
+ let embedding_tree = parameter_tree.subtree("embedding")?;
let embedding_spec = embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
let (ty, readout_input_hadamard_factors) = match embedding_spec {
@@ -176,7 +156,7 @@ impl<B: Backend> Embedding<B> {
None,
)
},
- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {
let (embedding_quantization_mode, group_size, quantization_method) =
input_quantization_from_spec(spec)?;
let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
@@ -295,7 +275,7 @@ impl<B: Backend> Embedding<B> {
)
},
AnyEmbeddingConfig::UntiedEmbeddingConfig(_) => {
- let input_embedding_tree = lookup_tree.subtree("input_embedding")?;
+ let input_embedding_tree = parameter_tree.subtree("input_embedding")?;
let input_embedding_spec = input_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
let input_ty = match input_embedding_spec {
@@ -319,7 +299,7 @@ impl<B: Backend> Embedding<B> {
lookup,
}
},
- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {
let (embedding_quantization_mode, group_size, quantization_method) =
input_quantization_from_spec(spec)?;
let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
@@ -354,7 +334,7 @@ impl<B: Backend> Embedding<B> {
spec => return Err(EmbeddingError::UnsupportedConfiguration(format!("{spec:?}"))),
};
- let output_embedding_tree = readout_tree.subtree("output_embedding")?;
+ let output_embedding_tree = parameter_tree.subtree("output_embedding")?;
let output_embedding_spec = output_embedding_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
let output_ty = match output_embedding_spec {
@@ -381,7 +361,7 @@ impl<B: Backend> Embedding<B> {
readout,
}
},
- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {
let (embedding_quantization_mode, group_size, quantization_method) =
output_quantization_from_spec(spec)?;
let (weights, scales, zero_points_or_biases) = load_quantized_embedding_parts(
@@ -660,14 +640,14 @@ fn input_quantization_from_spec<B: Backend>(
layout: Layout::InputOutput,
..
}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),
- AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ AnyWeightMatrixSpec::IntSpec(IntSpec {
bits,
group_size,
is_symmetric: false,
layout: Layout::InputOutput,
..
}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),
- AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ AnyWeightMatrixSpec::IntSpec(IntSpec {
bits,
group_size,
is_symmetric: true,
@@ -688,14 +668,14 @@ fn output_quantization_from_spec<B: Backend>(
layout: Layout::OutputInput,
..
}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleBias),
- AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ AnyWeightMatrixSpec::IntSpec(IntSpec {
bits,
group_size,
is_symmetric: false,
layout: Layout::OutputInput,
..
}) => quantization_mode(bits, group_size, QuantizationMethod::ScaleZeroPoint),
- AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ AnyWeightMatrixSpec::IntSpec(IntSpec {
bits,
group_size,
is_symmetric: true,
diff --git a/crates/backend-uzu/src/encodable_block/linear/matmul.rs b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
index d851fc9d7..edd14a0ea 100644
--- a/crates/backend-uzu/src/encodable_block/linear/matmul.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/matmul.rs
@@ -14,7 +14,7 @@ use crate::{
matmul::{MatmulArguments, MatmulB, MatmulDOps, MatmulKernel, MatmulQuantCombo},
},
},
- config::weight_matrix::{AnyWeightMatrixSpec, Layout, awq_spec::AWQSpec, mlx_spec::MLXSpec},
+ config::weight_matrix::{AnyWeightMatrixSpec, Layout, int_spec::IntSpec, mlx_spec::MLXSpec},
parameters::{ParameterLoaderError, ParameterTree},
};
@@ -114,14 +114,14 @@ impl<B: Backend> LinearMatmul<B> {
layout: Layout::OutputInput,
..
}) => (bits, group_size, QuantizationMethod::ScaleBias),
- AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ AnyWeightMatrixSpec::IntSpec(IntSpec {
bits,
group_size,
is_symmetric: false,
layout: Layout::OutputInput,
..
}) => (bits, group_size, QuantizationMethod::ScaleZeroPoint),
- AnyWeightMatrixSpec::AWQSpec(AWQSpec {
+ AnyWeightMatrixSpec::IntSpec(IntSpec {
bits,
group_size,
is_symmetric: true,
diff --git a/crates/backend-uzu/src/encodable_block/linear/mod.rs b/crates/backend-uzu/src/encodable_block/linear/mod.rs
index 6b044aabf..ec772901f 100644
--- a/crates/backend-uzu/src/encodable_block/linear/mod.rs
+++ b/crates/backend-uzu/src/encodable_block/linear/mod.rs
@@ -125,7 +125,7 @@ impl<B: Backend> dyn Linear<B> {
&weights_tree,
)?))
},
- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {
let block = LinearMatmul::quantized(
context,
spec,
@@ -178,7 +178,7 @@ impl<B: Backend> dyn Linear<B> {
let weights_tree = parameter_tree.subtree("weights")?.subtree("quantized")?;
let spec = weights_tree.metadata::<AnyWeightMatrixSpec>("spec")?;
match spec {
- spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::AWQSpec(_)) => {
+ spec @ (AnyWeightMatrixSpec::MLXSpec(_) | AnyWeightMatrixSpec::IntSpec(_)) => {
Ok(Box::new(LinearMatmul::quantized(
context,
spec,
diff --git a/crates/backend-uzu/src/forward_pass/model_shape.rs b/crates/backend-uzu/src/forward_pass/model_shape.rs
index 57b50f9d9..e5ed8867d 100644
--- a/crates/backend-uzu/src/forward_pass/model_shape.rs
+++ b/crates/backend-uzu/src/forward_pass/model_shape.rs
@@ -10,7 +10,6 @@ pub struct ModelShape {
vocabulary_size: usize,
model_dim: usize,
- context_length: usize,
num_groups: usize,
pub num_layers: usize,
@@ -42,7 +41,6 @@ impl ModelShape {
rope_data_type: DataType::F32,
vocabulary_size: decoder_config.vocab_size,
model_dim: tf.model_dim,
- context_length: tf.context_length,
num_groups,
num_layers,
layer_mixers,
@@ -94,8 +92,4 @@ impl ModelShape {
) -> [usize; 2] {
[suffix_length, self.vocabulary_size]
}
-
- pub fn context_length(&self) -> usize {
- self.context_length
- }
}
diff --git a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
index de58f52a8..4799ea6fa 100644
--- a/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
+++ b/crates/backend-uzu/src/forward_pass/state/shared_buffers.rs
@@ -31,13 +31,13 @@ impl<B: Backend> SharedBuffers<B> {
.layer_configs
.iter()
.map(|layer_config| {
- let Some(attention_config) = layer_config.mixer_config.as_attention() else {
+ if layer_config.mixer_config.as_attention().is_none() {
return LayerRopeKind::NoKernel;
- };
+ }
let Some(rope_config) = &layer_config.rope_config else {
return LayerRopeKind::NoKernel;
};
- let head_dim = rope_config.head_dim().unwrap_or(attention_config.head_dim);
+ let head_dim = *rope_config.head_dim();
let index = configs
.iter()
.position(|(existing_config, existing_head_dim)| {
diff --git a/crates/backend-uzu/src/language_model/language_model_generator_context.rs b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
index 2c9028212..79419832e 100644
--- a/crates/backend-uzu/src/language_model/language_model_generator_context.rs
+++ b/crates/backend-uzu/src/language_model/language_model_generator_context.rs
@@ -198,7 +198,7 @@ impl<B: Backend> LanguageModelGeneratorContext<B> {
decoder_config: &DecoderConfig,
decoding_config: &DecodingConfig,
) -> usize {
- let model_length = decoder_config.transformer_config.context_length;
+ let model_length = decoder_config.transformer_config.max_sequence_length().unwrap_or(65536);
let proposed_value = match decoding_config.context_length {
ContextLength::Default => {
if context.sparse_buffers_supported() {
diff --git a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs b/crates/backend-uzu/src/session/parameter/prefill_step_size.rs
index 027c909a0..1d9caffb8 100644
--- a/crates/backend-uzu/src/session/parameter/prefill_step_size.rs
+++ b/crates/backend-uzu/src/session/parameter/prefill_step_size.rs
@@ -31,7 +31,8 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
config: &LanguageModelConfig,
) -> usize {
let default_limit: usize = 4096;
- let model_context_length = config.decoder_config.transformer_config.context_length;
+ let model_context_length =
+ config.decoder_config.transformer_config.max_sequence_length().unwrap_or(default_limit);
let minimal_sliding_window_size = config
.decoder_config
diff --git a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
index cbb0280ed..d6198f064 100644
--- a/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
+++ b/crates/backend-uzu/src/session/tts_session/decoder_runtime.rs
@@ -3,7 +3,7 @@ use crate::{
array::{Array, ArrayContextExt},
backends::common::{Allocation, AsBufferRangeRef, DenseBuffer},
config::decoder::DecoderConfig,
- encodable_block::{DecoderArguments, DecoderDecodeInput, SamplingInputs},
+ encodable_block::{DecoderArguments, DecoderDecodeInput, LayerArguments, Linear, SamplingInputs},
forward_pass::token_inputs::TokenInputs,
session::types::TtsModelConfigError,
};
@@ -11,6 +11,7 @@ use crate::{
struct TokenDecoderLoadedModel<B: Backend> {
shared_buffers: Rc<SharedBuffers<B>>,
executables: Decoder<B>,
+ readout: Box<dyn Linear<B>>,
sampler: GpuSampling<B>,
token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
}
@@ -32,16 +33,26 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
root_loader_view,
transformer_subtree,
)?;
- let executables = Decoder::new_with_embedding_and_readout_subtrees(
+ let executables = Decoder::new_with_embedding_subtree(
context.as_ref(),
&decoder_config,
root_loader_view,
transformer_subtree,
embedding_subtree,
- readout_subtree,
model_shape,
)
.map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;
+ let readout_tree =
+ root_loader_view.subtree(readout_subtree).map_err(|error| Error::UnableToLoadWeights(Box::new(error)))?;
+ let readout = <dyn Linear<B>>::new(
+ decoder_config.transformer_config.model_dim,
+ [decoder_config.vocab_size],
+ false,
+ context.as_ref(),
+ model_shape.data_type,
+ &readout_tree,
+ )
+ .map_err(|error| Error::UnableToCreateDecoder(Box::new(error)))?;
let sampler = GpuSampling::new(context.as_ref(), model_shape.data_type).map_err(unable_to_create_context)?;
let token_copy_sampled =
<B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;
@@ -49,6 +60,7 @@ impl<B: Backend> TokenDecoderLoadedModel<B> {
Ok(Self {
shared_buffers,
executables,
+ readout,
sampler,
token_copy_sampled,
})
@@ -63,6 +75,7 @@ struct TokenDecoderContext<B: Backend> {
decoder_config: Rc<DecoderConfig>,
runtime_config: TextDecoderRuntimeConfig,
executables: Decoder<B>,
+ readout: Box<dyn Linear<B>>,
sampler: GpuSampling<B>,
kv_cache_update: KVCacheUpdate<B>,
token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
@@ -94,8 +107,11 @@ impl<B: Backend> TokenDecoderContext<B> {
let root_loader_view = loader.tree();
let model_shape = ModelShape::from_decoder_config(&decoder_config, DataType::F32);
- let max_suffix_length =
- text_decoder_prefill_step_size(runtime_config, decoder_config.transformer_config.context_length).max(32);
+ let max_suffix_length = text_decoder_prefill_step_size(
+ runtime_config,
+ decoder_config.transformer_config.max_sequence_length().unwrap_or(usize::MAX),
+ )
+ .max(32);
let max_prefix_length = max_suffix_length;
let loaded_model = TokenDecoderLoadedModel::<B>::load(
&context,
@@ -126,6 +142,7 @@ impl<B: Backend> TokenDecoderContext<B> {
decoder_config,
runtime_config: runtime_config.clone(),
executables: loaded_model.executables,
+ readout: loaded_model.readout,
sampler: loaded_model.sampler,
kv_cache_update,
token_copy_sampled: loaded_model.token_copy_sampled,
@@ -331,7 +348,12 @@ impl<B: Backend> TokenDecoderRunner<B> {
&mut self,
max_prefix_length: usize,
) -> Result<(), Error> {
- let max_prefix_length = max_prefix_length.max(1).min(self.ctx.decoder_config.transformer_config.context_length);
+ let max_prefix_length = self
+ .ctx
+ .decoder_config
+ .transformer_config
+ .max_sequence_length()
+ .map_or(max_prefix_length.max(1), |model_length| max_prefix_length.max(1).min(model_length));
let max_suffix_length = text_decoder_prefill_step_size(&self.ctx.runtime_config, max_prefix_length).max(32);
if self.ctx.current_max_prefix_length == max_prefix_length {
return Ok(());
@@ -554,7 +576,6 @@ impl<B: Backend> TokenDecoderRunner<B> {
pending_sampling_inputs: &mut Vec<SamplingInputs<B>>,
pending_sampling_outputs: &mut Vec<Allocation<B>>,
) -> Result<(), Error> {
- let cache_layers_rc = self.ctx.cache_layers.clone();
let token_bitmask =
vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit)).map(Box::<[u32]>::from);
let mut next_token_ids = first_token_ids;
@@ -570,19 +591,15 @@ impl<B: Backend> TokenDecoderRunner<B> {
let token_inputs = pending_token_inputs.last().expect("token inputs must be pending");
{
- let mut logits = {
- let mut cache_layers = cache_layers_rc.borrow_mut();
- cache_layers.prepare_for_forward_pass(&self.ctx.context, 1);
- self.ctx
- .executables
- .encode_decode(
- self.decoder_arguments(&token_inputs, 1, 0, 1, &mut *cache_layers),
- DecoderDecodeInput::TokenIds(token_inputs.token_ids()),
- None,
- encoder,
- )
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?
- };
+ let mut logits = self.encode_decode_with_fishaudio_readout_on(
+ encoder,
+ DecoderDecodeInput::TokenIds(token_inputs.token_ids()),
+ token_inputs,
+ 1,
+ 0,
+ 1,
+ false,
+ )?;
let mut sampling_output = self.create_sampling_output(1);
self.encode_sampling_on(
encoder,
@@ -690,6 +707,75 @@ impl<B: Backend> TokenDecoderRunner<B> {
self.ctx.context.create_array_uninitialized(&[sampling_length], crate::DataType::U32).into_allocation()
}
+ fn encode_decode_with_fishaudio_readout_on<'input>(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ input: DecoderDecodeInput<'input, B>,
+ token_inputs: &TokenInputs<B>,
+ batch_dim: usize,
+ sampling_start: usize,
+ sampling_length: usize,
+ capture_hidden: bool,
+ ) -> Result<Allocation<B>, Error> {
+ let mut main = match input {
+ DecoderDecodeInput::TokenIds(token_ids) => self
+ .ctx
+ .executables
+ .embed
+ .encode_lookup(token_ids, batch_dim, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?,
+ DecoderDecodeInput::Embeddings(main) => main,
+ };
+ let mut shortcut = encoder
+ .allocate_scratch(main.as_buffer_range_ref().range().len())
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+
+ {
+ let mut cache_layers = self.ctx.cache_layers.borrow_mut();
+ cache_layers.prepare_for_forward_pass(&self.ctx.context, batch_dim);
+ for layer in self.ctx.executables.layers.iter() {
+ main = layer
+ .encode(
+ LayerArguments {
+ batch_dim,
+ token_positions: token_inputs.token_positions(),
+ token_parents: token_inputs.token_parents(),
+ token_subtrie_ranges: token_inputs.token_subtrie_ranges(),
+ rope_buffers: self.ctx.shared_buffers.rope_buffers_for_layer(layer.layer_index),
+ sampling_start,
+ sampling_length,
+ cache_access: Some(cache_layers.cache_for_layer(layer.layer_index)),
+ #[cfg(feature = "tracing")]
+ trace: None,
+ },
+ main,
+ &mut shortcut,
+ encoder,
+ )
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ }
+ }
+
+ let output_norm = self
+ .ctx
+ .executables
+ .norm
+ .encode(&main, sampling_start, sampling_length, Some(&mut shortcut), encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ if capture_hidden {
+ let row_size_bytes = shortcut.as_buffer_range_ref().range().len() / batch_dim;
+ let input_offset = (batch_dim - 1) * row_size_bytes;
+ encoder.encode_copy(
+ &shortcut,
+ input_offset..input_offset + row_size_bytes,
+ &mut self.single_hidden_capture,
+ ..,
+ );
+ }
+
+ self.ctx.readout.encode(output_norm, sampling_length, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))
+ }
+
fn encode_single_forward_pass_on(
&mut self,
encoder: &mut Encoder<B>,
@@ -793,27 +879,15 @@ impl<B: Backend> TokenDecoderRunner<B> {
self.encode_override_first_row_from_device_on(encoder, &mut main, &self.single_override_embedding)?;
},
}
- let mut logits = {
- let mut cache_layers = self.ctx.cache_layers.borrow_mut();
- cache_layers.prepare_for_forward_pass(&self.ctx.context, token_count);
- let decoder_arguments = DecoderArguments {
- token_positions: token_inputs.token_positions(),
- token_parents: token_inputs.token_parents(),
- token_subtrie_ranges: token_inputs.token_subtrie_ranges(),
- shared_buffers: self.ctx.shared_buffers.as_ref(),
- cache_layers: Some(&mut *cache_layers),
- batch_dim: token_count,
- sampling_start,
- sampling_length,
- #[cfg(feature = "tracing")]
- trace: None,
- };
- let hidden_capture = capture_hidden.then_some(&mut self.single_hidden_capture);
- self.ctx
- .executables
- .encode_decode(decoder_arguments, DecoderDecodeInput::Embeddings(main), hidden_capture, encoder)
- .map_err(|err| Error::EncodeFailed(Box::new(err)))?
- };
+ let mut logits = self.encode_decode_with_fishaudio_readout_on(
+ encoder,
+ DecoderDecodeInput::Embeddings(main),
+ token_inputs,
+ token_count,
+ sampling_start,
+ sampling_length,
+ capture_hidden,
+ )?;
let mut sampling_output = self.create_sampling_output(sampling_length);
self.encode_sampling_on(
encoder,
diff --git a/crates/backend-uzu/src/utils/version.rs b/crates/backend-uzu/src/utils/version.rs
index 8888ef48b..0bea2c31a 100644
--- a/crates/backend-uzu/src/utils/version.rs
+++ b/crates/backend-uzu/src/utils/version.rs
@@ -1,2 +1,2 @@
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
-pub const TOOLCHAIN_VERSION: &str = "0.10.0";
+pub const TOOLCHAIN_VERSION: &str = "0.11.0";