Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Fish Audio TTS (#266)

LuckyIYIb7a964e6b4fc
diff --git a/crates/uzu/src/audio/codec.rs b/crates/uzu/src/audio/codec.rs
index e973653ed..231fc1735 100644
--- a/crates/uzu/src/audio/codec.rs
+++ b/crates/uzu/src/audio/codec.rs
@@ -1,6 +1,6 @@
use super::{AudioPcmBatch, AudioResult, AudioTokenGrid};
-pub trait AudioCodecRuntime: Send + Sync {
+pub trait AudioCodecRuntime {
fn encode(
&self,
pcm: &AudioPcmBatch,
diff --git a/crates/uzu/src/audio/context.rs b/crates/uzu/src/audio/context.rs
new file mode 100644
index 000000000..e85c62d66
--- /dev/null
+++ b/crates/uzu/src/audio/context.rs
@@ -0,0 +1,56 @@
+#![cfg(feature = "audio-runtime")]
+
+use std::{path::Path, rc::Rc};
+
+use super::{AudioResult, NanoCodecFsqRuntime};
+use crate::{backends::common::Backend, config::TtsConfig};
+
+#[derive(Clone)]
+pub struct AudioGenerationContext<B: Backend> {
+ runtime: Rc<NanoCodecFsqRuntime<B>>,
+ codec_cardinality: usize,
+ semantic_codec_cardinality: Option<usize>,
+ num_codebooks: usize,
+ sample_rate: u32,
+}
+
+impl<B: Backend> AudioGenerationContext<B> {
+ pub fn from_tts_config_and_model_path(
+ tts_config: &TtsConfig,
+ model_path: &Path,
+ ) -> AudioResult<Self> {
+ Self::with_runtime(NanoCodecFsqRuntime::from_tts_config_and_model_path(tts_config, model_path)?)
+ }
+
+ fn with_runtime(runtime: NanoCodecFsqRuntime<B>) -> AudioResult<Self> {
+ let codec_cardinality = usize::try_from(runtime.config().codec_cardinality())
+ .map_err(|_| super::AudioError::Runtime("audio codec cardinality exceeds usize".to_string()))?;
+ Ok(Self {
+ codec_cardinality,
+ semantic_codec_cardinality: runtime.config().semantic_codec_cardinality(),
+ num_codebooks: runtime.config().num_groups(),
+ sample_rate: runtime.config().sample_rate(),
+ runtime: Rc::new(runtime),
+ })
+ }
+
+ pub fn runtime(&self) -> &NanoCodecFsqRuntime<B> {
+ self.runtime.as_ref()
+ }
+
+ pub fn codec_cardinality(&self) -> usize {
+ self.codec_cardinality
+ }
+
+ pub fn semantic_codec_cardinality(&self) -> Option<usize> {
+ self.semantic_codec_cardinality
+ }
+
+ pub fn num_codebooks(&self) -> usize {
+ self.num_codebooks
+ }
+
+ pub fn sample_rate(&self) -> u32 {
+ self.sample_rate
+ }
+}
diff --git a/crates/uzu/src/audio/mod.rs b/crates/uzu/src/audio/mod.rs
index 9b68e6d6d..68052a2e0 100644
--- a/crates/uzu/src/audio/mod.rs
+++ b/crates/uzu/src/audio/mod.rs
@@ -1,5 +1,12 @@
mod codec;
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+mod context;
+pub mod nanocodec;
mod types;
pub use codec::AudioCodecRuntime;
-pub use types::{AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid, AudioTokenPacking};
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub use context::AudioGenerationContext;
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub use nanocodec::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};
+pub use types::{AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid};
diff --git a/crates/uzu/src/audio/nanocodec/mod.rs b/crates/uzu/src/audio/nanocodec/mod.rs
new file mode 100644
index 000000000..f4ee66705
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/mod.rs
@@ -0,0 +1,5 @@
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub mod runtime;
+
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub use runtime::{AudioDecodeStepStats, AudioDecodeStreamState, NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};
diff --git a/crates/uzu/src/audio/nanocodec/runtime.rs b/crates/uzu/src/audio/nanocodec/runtime.rs
new file mode 100644
index 000000000..a7b340ce6
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime.rs
@@ -0,0 +1,809 @@
+use std::{
+ cell::{Cell, RefCell},
+ collections::{BTreeMap, HashMap},
+ fs::File,
+ path::Path,
+ rc::Rc,
+};
+
+use serde::Deserialize;
+
+use crate::{
+ DataType,
+ array::{Array, ArrayContextExt, size_for_shape},
+ audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid},
+ backends::common::{
+ Backend, Buffer, Context, Encoder, Kernels, Pending,
+ kernel::{
+ ActivationKernel, AudioAddKernel, AudioCausalConv1dGroupedKernel, AudioCausalConv1dGroupedResidualKernel,
+ AudioCausalConv1dKernel, AudioCausalConvTranspose1dCausalPadKernel, AudioConv1dKernel,
+ AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel, AudioNormNcsKernel,
+ AudioQuantizerDecodeKernel,
+ },
+ },
+ config::{
+ ConfigDataType, DescriptAudioCodecConfig, DescriptAudioConvNeXtNormConfig, EmbeddingConfig,
+ EmbeddingConfigCommon, InnerModelConfig, TtsAudioDecoderConfig, TtsConfig,
+ },
+ encodable_block::{Decoder, EncodingParameters, LayerExecutables, RMSNorm},
+ forward_pass::{
+ model_shape::ModelShape,
+ scratch_buffers::ScratchBuffers,
+ state::{ArrayId, ForwardPassState, SharedBuffers},
+ },
+ parameters::ParameterLoader,
+};
+
+mod loaders;
+mod profile;
+mod stream;
+mod structured;
+mod support;
+
+use loaders::load_audio_runtime_from_tts_config;
+pub(crate) use profile::PendingStreamPcmChunk;
+pub use stream::{AudioDecodeStepStats, AudioDecodeStreamState};
+use stream::{extract_delta_from_padded_with_offset_snapshot, pack_pcm_to_padded, unpack_padded_to_pcm};
+use structured::{StructuredAudioCodecGraph, StructuredAudioRuntimeResources};
+use support::{DecodedPaddedAudio, checked_product, convert_lengths_to_i32, usize_to_i32};
+
+fn default_eps() -> f32 {
+ 1e-3
+}
+
+fn compute_dim_base_index(num_levels: &[i32]) -> AudioResult<Box<[i32]>> {
+ if num_levels.is_empty() {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+
+ let mut out = vec![0_i32; num_levels.len()];
+ let mut base = 1_i32;
+
+ for (index, &levels) in num_levels.iter().enumerate() {
+ if levels <= 1 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ out[index] = base;
+ base = base.checked_mul(levels).ok_or(AudioError::Runtime("dim_base_index overflow".to_string()))?;
+ }
+
+ Ok(out.into_boxed_slice())
+}
+
+#[derive(Debug, Clone, Deserialize)]
+#[serde(deny_unknown_fields)]
+struct RuntimeConfigJson {
+ sample_rate: u32,
+ num_groups: usize,
+ num_levels_per_group: Vec<i32>,
+ #[serde(default = "default_eps")]
+ eps: f32,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct NanoCodecFsqRuntimeConfig {
+ sample_rate: u32,
+ num_groups: usize,
+ num_levels_per_group: Box<[i32]>,
+ dim_base_index: Box<[i32]>,
+ codebook_dim_per_group: usize,
+ channels: usize,
+ codec_cardinality: u32,
+ eps: f32,
+ structured_decoder: Option<StructuredAudioCodecGraph>,
+}
+
+impl NanoCodecFsqRuntimeConfig {
+ pub fn from_tts_config_and_model_path(
+ tts_config: &TtsConfig,
+ model_path: &Path,
+ ) -> AudioResult<Self> {
+ let (runtime_config, decoder) = load_audio_runtime_from_tts_config(tts_config, model_path)?;
+ let mut runtime = Self::from_runtime_config_json(runtime_config)?;
+ runtime.structured_decoder = Some(decoder);
+ Ok(runtime)
+ }
+
+ fn from_runtime_config_json(parsed: RuntimeConfigJson) -> AudioResult<Self> {
+ Self::new(parsed.sample_rate, parsed.num_groups, parsed.num_levels_per_group.into_boxed_slice(), parsed.eps)
+ }
+
+ pub fn new(
+ sample_rate: u32,
+ num_groups: usize,
+ num_levels_per_group: Box<[i32]>,
+ eps: f32,
+ ) -> AudioResult<Self> {
+ if sample_rate == 0 {
+ return Err(AudioError::InvalidSampleRate);
+ }
+ if num_groups == 0 || num_levels_per_group.is_empty() {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ if !eps.is_finite() || !(0.0..1.0).contains(&eps) {
+ return Err(AudioError::Runtime("eps must be finite and satisfy 0.0 <= eps < 1.0".to_string()));
+ }
+ for &level in num_levels_per_group.iter() {
+ if level <= 1 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ }
+
+ let codebook_dim_per_group = num_levels_per_group.len();
+ let channels = num_groups
+ .checked_mul(codebook_dim_per_group)
+ .ok_or(AudioError::Runtime("num_groups * codebook_dim_per_group overflow".to_string()))?;
+ let dim_base_index = compute_dim_base_index(&num_levels_per_group)?;
+
+ let mut codec_cardinality_u64 = 1_u64;
+ for &level in num_levels_per_group.iter() {
+ codec_cardinality_u64 = codec_cardinality_u64
+ .checked_mul(level as u64)
+ .ok_or(AudioError::Runtime("codec cardinality overflow".to_string()))?;
+ }
+
+ if codec_cardinality_u64 > u32::MAX as u64 {
+ return Err(AudioError::Runtime("codec cardinality exceeds u32 range".to_string()));
+ }
+
+ let codec_cardinality = codec_cardinality_u64 as u32;
+ if codec_cardinality_u64 > (i32::MAX as u64 + 1) {
+ return Err(AudioError::Runtime("codec cardinality exceeds i32 token kernel range".to_string()));
+ }
+
+ Ok(Self {
+ sample_rate,
+ num_groups,
+ num_levels_per_group,
+ dim_base_index,
+ codebook_dim_per_group,
+ channels,
+ codec_cardinality,
+ eps,
+ structured_decoder: None,
+ })
+ }
+
+ pub fn sample_rate(&self) -> u32 {
+ self.sample_rate
+ }
+
+ pub fn num_groups(&self) -> usize {
+ self.num_groups
+ }
+
+ pub fn num_levels_per_group(&self) -> &[i32] {
+ &self.num_levels_per_group
+ }
+
+ pub fn dim_base_index(&self) -> &[i32] {
+ &self.dim_base_index
+ }
+
+ pub fn codebook_dim_per_group(&self) -> usize {
+ self.codebook_dim_per_group
+ }
+
+ pub fn channels(&self) -> usize {
+ self.channels
+ }
+
+ pub fn codec_cardinality(&self) -> u32 {
+ self.codec_cardinality
+ }
+
+ pub fn semantic_codec_cardinality(&self) -> Option<usize> {
+ self.structured_decoder.as_ref().map(|decoder| decoder.semantic_codebook_size)
+ }
+
+ pub fn eps(&self) -> f32 {
+ self.eps
+ }
+
+ fn structured_decoder(&self) -> Option<&StructuredAudioCodecGraph> {
+ self.structured_decoder.as_ref()
+ }
+}
+
+pub struct NanoCodecFsqRuntime<B: Backend> {
+ config: NanoCodecFsqRuntimeConfig,
+ structured_resources: RefCell<Option<Rc<StructuredAudioRuntimeResources<B>>>>,
+}
+
+impl<B: Backend> Clone for NanoCodecFsqRuntime<B> {
+ fn clone(&self) -> Self {
+ Self {
+ config: self.config.clone(),
+ structured_resources: self.structured_resources.clone(),
+ }
+ }
+}
+
+impl<B: Backend> std::fmt::Debug for NanoCodecFsqRuntime<B> {
+ fn fmt(
+ &self,
+ f: &mut std::fmt::Formatter<'_>,
+ ) -> std::fmt::Result {
+ f.debug_struct("NanoCodecFsqRuntime").field("config", &self.config).finish()
+ }
+}
+
+impl<B: Backend> NanoCodecFsqRuntime<B> {
+ pub fn new(config: NanoCodecFsqRuntimeConfig) -> Self {
+ Self {
+ config,
+ structured_resources: RefCell::new(None),
+ }
+ }
+
+ pub fn from_tts_config_and_model_path(
+ tts_config: &TtsConfig,
+ model_path: &Path,
+ ) -> AudioResult<Self> {
+ Ok(Self::new(NanoCodecFsqRuntimeConfig::from_tts_config_and_model_path(tts_config, model_path)?))
+ }
+
+ pub fn config(&self) -> &NanoCodecFsqRuntimeConfig {
+ &self.config
+ }
+
+ fn validate_fishaudio_token_delta(
+ &self,
+ tokens: &AudioTokenGrid,
+ fishaudio: &StructuredAudioCodecGraph,
+ ) -> AudioResult<()> {
+ let semantic_cardinality = u32::try_from(fishaudio.semantic_codebook_size).map_err(|_| {
+ AudioError::Runtime("FishAudio semantic codebook cardinality exceeds u32 range".to_string())
+ })?;
+ let residual_cardinality = u32::try_from(fishaudio.codebook_size).map_err(|_| {
+ AudioError::Runtime("FishAudio residual codebook cardinality exceeds u32 range".to_string())
+ })?;
+ let frames = tokens.frames();
+ for batch in 0..tokens.batch_size() {
+ for codebook in 0..tokens.codebooks() {
+ let cardinality = if codebook == 0 {
+ semantic_cardinality
+ } else {
+ residual_cardinality
+ };
+ for frame in 0..frames {
+ let token = tokens.get(batch, codebook, frame);
+ if token >= cardinality {
+ return Err(AudioError::InvalidCodecToken {
+ token,
+ cardinality,
+ });
+ }
+ }
+ }
+ }
+ Ok(())
+ }
+
+ fn fsq_decode_padded(
+ &self,
+ tokens: &AudioTokenGrid,
+ ) -> AudioResult<DecodedPaddedAudio> {
+ if tokens.codebooks() != self.config.num_groups() {
+ return Err(AudioError::Runtime(format!(
+ "token codebook mismatch: expected {}, got {}",
+ self.config.num_groups(),
+ tokens.codebooks()
+ )));
+ }
+
+ let batch_size = tokens.batch_size();
+ let frames = tokens.frames();
+ let lengths_usize = tokens.lengths().to_vec();
+ let lengths_i32 = convert_lengths_to_i32(&lengths_usize, frames)?;
+
+ if batch_size == 0 || frames == 0 {
+ return Ok(DecodedPaddedAudio {
+ samples: Vec::new(),
+ channels: self.config.channels(),
+ frames: 0,
+ lengths: lengths_usize,
+ });
+ }
+
+ let mut tokens_i32 = vec![0_i32; tokens.tokens().len()];
+ for (index, &token) in tokens.tokens().iter().enumerate() {
+ if token >= self.config.codec_cardinality() {
+ return Err(AudioError::InvalidCodecToken {
+ token,
+ cardinality: self.config.codec_cardinality(),
+ });
+ }
+ if token > i32::MAX as u32 {
+ return Err(AudioError::Runtime(format!("token at index {index} exceeds i32 kernel range: {token}")));
+ }
+ tokens_i32[index] = token as i32;
+ }
+
+ let context = Self::create_context()?;
+ let kernel = <B::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize fsq decode kernel: {err}")))?;
+
+ let mut tokens_array = context.create_array(
+ &[batch_size, self.config.num_groups(), frames],
+ DataType::I32,
+ "nanocodec_fsq_decode_tokens",
+ );
+ tokens_array.as_slice_mut::<i32>().copy_from_slice(&tokens_i32);
+
+ let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");
+ lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
+
+ let output = context.create_array(
+ &[batch_size, self.config.channels(), frames],
+ DataType::F32,
+ "nanocodec_fsq_decode_output",
+ );
+
+ let mut encoder = Encoder::new(context.as_ref())
+ .map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;
+
+ let num_groups_i32 = usize_to_i32(self.config.num_groups(), "num_groups")?;
+ let frames_i32 = usize_to_i32(frames, "frames")?;
+ let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;
+ let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;
+ {
+ let tokens_buffer = tokens_array.buffer();
+ let tokens_buffer = tokens_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+
+ kernel.encode(
+ &*tokens_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ num_groups_i32,
+ frames_i32,
+ codebook_dim_i32,
+ self.config.num_levels_per_group(),
+ self.config.dim_base_index(),
+ batch_size_i32,
+ &mut encoder,
+ );
+ }
+
+ encoder
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ decode command buffer: {err}")))?;
+
+ Ok(DecodedPaddedAudio {
+ samples: output.as_slice::<f32>().to_vec(),
+ channels: self.config.channels(),
+ frames,
+ lengths: lengths_usize,
+ })
+ }
+
+ fn fsq_encode(
+ &self,
+ pcm: &AudioPcmBatch,
+ ) -> AudioResult<AudioTokenGrid> {
+ if pcm.sample_rate() != self.config.sample_rate() {
+ return Err(AudioError::Runtime(format!(
+ "pcm sample-rate mismatch: expected {}, got {}",
+ self.config.sample_rate(),
+ pcm.sample_rate()
+ )));
+ }
+
+ let (padded_input, lengths_usize, lengths_i32, frames) = pack_pcm_to_padded(pcm, self.config.channels())?;
+ let batch_size = pcm.batch_size();
+
+ if batch_size == 0 || frames == 0 {
+ let empty_tokens = Vec::<u32>::new().into_boxed_slice();
+ let grid = AudioTokenGrid::new(
+ empty_tokens,
+ batch_size,
+ self.config.num_groups(),
+ frames,
+ lengths_usize.into_boxed_slice(),
+ )?;
+ return Ok(grid);
+ }
+
+ let context = Self::create_context()?;
+ let kernel = <B::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize fsq encode kernel: {err}")))?;
+
+ let mut input = context.create_array(
+ &[batch_size, self.config.channels(), frames],
+ DataType::F32,
+ "nanocodec_fsq_encode_input",
+ );
+ input.as_slice_mut::<f32>().copy_from_slice(&padded_input);
+
+ let mut lengths = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");
+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
+
+ let tokens = context.create_array(
+ &[batch_size, self.config.num_groups(), frames],
+ DataType::I32,
+ "nanocodec_fsq_encode_tokens",
+ );
+
+ let mut encoder = Encoder::new(context.as_ref())
+ .map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;
+
+ let num_groups_i32 = usize_to_i32(self.config.num_groups(), "num_groups")?;
+ let frames_i32 = usize_to_i32(frames, "frames")?;
+ let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;
+ let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;
+ {
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let tokens_buffer = tokens.buffer();
+ let mut tokens_buffer = tokens_buffer.borrow_mut();
+ let lengths_buffer = lengths.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+
+ kernel.encode(
+ &*input_buffer,
+ &mut *tokens_buffer,
+ &*lengths_buffer,
+ num_groups_i32,
+ frames_i32,
+ codebook_dim_i32,
+ self.config.num_levels_per_group(),
+ self.config.dim_base_index(),
+ self.config.eps(),
+ batch_size_i32,
+ &mut encoder,
+ );
+ }
+
+ encoder
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ encode command buffer: {err}")))?;
+
+ let mut tokens_u32 = vec![0_u32; tokens.num_elements()];
+ for (index, &token) in tokens.as_slice::<i32>().iter().enumerate() {
+ if token < 0 {
+ return Err(AudioError::Runtime(format!(
+ "fsq encode returned negative token at index {index}: {token}"
+ )));
+ }
+
+ let token_u32 = token as u32;
+ if token_u32 >= self.config.codec_cardinality() {
+ return Err(AudioError::InvalidCodecToken {
+ token: token_u32,
+ cardinality: self.config.codec_cardinality(),
+ });
+ }
+ tokens_u32[index] = token_u32;
+ }
+
+ AudioTokenGrid::new(
+ tokens_u32.into_boxed_slice(),
+ batch_size,
+ self.config.num_groups(),
+ frames,
+ lengths_usize.into_boxed_slice(),
+ )
+ }
+
+ pub fn begin_decode_stream(
+ &self,
+ batch_size: usize,
+ codebooks: usize,
+ ) -> AudioResult<AudioDecodeStreamState> {
+ self.begin_decode_stream_with_options(batch_size, codebooks, 256)
+ }
+
+ pub fn begin_decode_stream_with_options(
+ &self,
+ batch_size: usize,
+ codebooks: usize,
+ max_workspace_frames: usize,
+ ) -> AudioResult<AudioDecodeStreamState> {
+ if codebooks != self.config.num_groups() {
+ return Err(AudioError::Runtime(format!(
+ "stream codebook mismatch: expected {}, got {}",
+ self.config.num_groups(),
+ codebooks
+ )));
+ }
+ AudioDecodeStreamState::new(batch_size, codebooks, max_workspace_frames)
+ }
+
+ pub(crate) fn decoded_padded_to_pcm_batch(
+ &self,
+ decoded: &DecodedPaddedAudio,
+ ) -> AudioResult<AudioPcmBatch> {
+ let samples = unpack_padded_to_pcm(
+ &decoded.samples,
+ decoded.lengths.len(),
+ decoded.channels,
+ decoded.frames,
+ &decoded.lengths,
+ )?;
+ AudioPcmBatch::new(
+ samples.into_boxed_slice(),
+ self.config.sample_rate(),
+ decoded.channels,
+ decoded.lengths.clone().into_boxed_slice(),
+ )
+ }
+
+ pub fn end_decode_stream(
+ &self,
+ _state: AudioDecodeStreamState,
+ ) -> AudioResult<()> {
+ Ok(())
+ }
+
+ fn create_context() -> AudioResult<Rc<B::Context>> {
+ B::Context::new().map_err(|err| AudioError::Runtime(format!("failed to create audio context: {err}")))
+ }
+
+ fn structured_resources(&self) -> AudioResult<Rc<StructuredAudioRuntimeResources<B>>> {
+ if let Some(existing) = self.structured_resources.borrow().as_ref() {
+ return Ok(existing.clone());
+ }
+ let context = Self::create_context()?;
+ let created = Rc::new(StructuredAudioRuntimeResources::new(context));
+ *self.structured_resources.borrow_mut() = Some(created.clone());
+ Ok(created)
+ }
+
+ fn decode_structured_stream_delta(
+ &self,
+ state: &mut AudioDecodeStreamState,
+ fishaudio: &StructuredAudioCodecGraph,
+ input_frames: usize,
+ ) -> AudioResult<DecodedPaddedAudio> {
+ if state.total_frames() == 0 {
+ state.record_last_step_stats(input_frames, 0, 0);
+ return Ok(DecodedPaddedAudio {
+ samples: Vec::new(),
+ channels: 1,
+ frames: 0,
+ lengths: vec![0usize; state.batch_size],
+ });
+ }
+
+ let resources = self.structured_resources()?;
+ let Some(context_frames) = fishaudio.streaming_decode_context_frames(resources.as_ref())? else {
+ let full_grid = state.to_full_grid()?;
+ let full_padded = self.decode_padded(&full_grid)?;
+ state.record_last_step_stats(input_frames, 0, state.total_frames());
+ return state.extract_delta_from_padded_with_offset(&full_padded, 0, fishaudio.upsample_factor);
+ };
+ let mut window_start = state.total_frames();
+ for &emitted in &state.emitted_semantic_lengths {
+ window_start = window_start.min(emitted.saturating_sub(context_frames));
+ }
+ let window_end = state.total_frames();
+ let batch_size = state.batch_size;
+ let codebooks = state.codebooks;
+ let (window_tokens, window_lengths, window_frames) = state.flatten_window(window_start, window_end)?;
+ let decoded_window = fishaudio.decode_padded(
+ resources.as_ref(),
+ window_tokens,
+ window_lengths,
+ batch_size,
+ codebooks,
+ window_frames,
+ )?;
+ let audio_offset_frames = window_start
+ .checked_mul(fishaudio.upsample_factor)
+ .ok_or(AudioError::Runtime("stream audio offset overflow".to_string()))?;
+ state.record_last_step_stats(input_frames, window_start, window_frames);
+ state.extract_delta_from_padded_with_offset(&decoded_window, audio_offset_frames, fishaudio.upsample_factor)
+ }
+
+ pub(crate) fn decode_padded(
+ &self,
+ tokens: &AudioTokenGrid,
+ ) -> AudioResult<DecodedPaddedAudio> {
+ if tokens.codebooks() != self.config.num_groups() {
+ return Err(AudioError::Runtime(format!(
+ "token codebook mismatch: expected {}, got {}",
+ self.config.num_groups(),
+ tokens.codebooks()
+ )));
+ }
+
+ let batch_size = tokens.batch_size();
+ let frames = tokens.frames();
+ let lengths_usize = tokens.lengths().to_vec();
+
+ if batch_size == 0 || frames == 0 {
+ let channels = if self.config.structured_decoder().is_some() {
+ 1
+ } else {
+ self.config.channels()
+ };
+ let out_lengths = if let Some(decoder) = self.config.structured_decoder() {
+ lengths_usize
+ .iter()
+ .map(|&length| {
+ length
+ .checked_mul(decoder.upsample_factor)
+ .ok_or(AudioError::Runtime("decoder length scaling overflow".to_string()))
+ })
+ .collect::<AudioResult<Vec<_>>>()?
+ } else {
+ lengths_usize.clone()
+ };
+ return Ok(DecodedPaddedAudio {
+ samples: Vec::new(),
+ channels,
+ frames: out_lengths.iter().copied().max().unwrap_or(0),
+ lengths: out_lengths,
+ });
+ }
+
+ if let Some(fishaudio) = self.config.structured_decoder() {
+ self.validate_fishaudio_token_delta(tokens, fishaudio)?;
+ let resources = self.structured_resources()?;
+ let decoded = fishaudio.decode_padded(
+ resources.as_ref(),
+ tokens.tokens(),
+ &lengths_usize,
+ batch_size,
+ tokens.codebooks(),
+ frames,
+ )?;
+ return Ok(decoded);
+ }
+
+ self.fsq_decode_padded(tokens)
+ }
+
+ pub(crate) fn decode_stream_step(
+ &self,
+ state: &mut AudioDecodeStreamState,
+ new_tokens: &AudioTokenGrid,
+ _is_final: bool,
+ ) -> AudioResult<DecodedPaddedAudio> {
+ if new_tokens.codebooks() != state.codebooks {
+ return Err(AudioError::Runtime(format!(
+ "stream delta codebook mismatch: expected {}, got {}",
+ state.codebooks,
+ new_tokens.codebooks()
+ )));
+ }
+ if new_tokens.batch_size() != state.batch_size {
+ return Err(AudioError::Runtime(format!(
+ "stream delta batch mismatch: expected {}, got {}",
+ state.batch_size,
+ new_tokens.batch_size()
+ )));
+ }
+ if new_tokens.frames() == 0 {
+ state.record_last_step_stats(0, state.total_frames(), 0);
+ return Ok(DecodedPaddedAudio {
+ samples: Vec::new(),
+ channels: if self.config.structured_decoder().is_some() {
+ 1
+ } else {
+ self.config.channels()
+ },
+ frames: 0,
+ lengths: vec![0; state.batch_size],
+ });
+ }
+
+ if let Some(fishaudio) = self.config.structured_decoder() {
+ self.validate_fishaudio_token_delta(new_tokens, fishaudio)?;
+ state.append_delta(new_tokens)?;
+ return self.decode_structured_stream_delta(state, fishaudio, new_tokens.frames());
+ }
+
+ state.append_delta(new_tokens)?;
+ let full_grid = state.to_full_grid()?;
+ let full_padded = self.fsq_decode_padded(&full_grid)?;
+ state.record_last_step_stats(new_tokens.frames(), 0, state.total_frames());
+ state.extract_delta_from_padded_with_offset(&full_padded, 0, 1)
+ }
+
+ pub(crate) fn submit_decode_stream_step(
+ &self,
+ state: &mut AudioDecodeStreamState,
+ new_tokens: &AudioTokenGrid,
+ is_final: bool,
+ ) -> AudioResult<Option<PendingStreamPcmChunk<B>>> {
+ if is_final {
+ return Ok(None);
+ }
+ if new_tokens.codebooks() != state.codebooks {
+ return Err(AudioError::Runtime(format!(
+ "stream delta codebook mismatch: expected {}, got {}",
+ state.codebooks,
+ new_tokens.codebooks()
+ )));
+ }
+ if new_tokens.batch_size() != state.batch_size {
+ return Err(AudioError::Runtime(format!(
+ "stream delta batch mismatch: expected {}, got {}",
+ state.batch_size,
+ new_tokens.batch_size()
+ )));
+ }
+ if new_tokens.frames() == 0 {
+ return Ok(None);
+ }
+
+ let Some(fishaudio) = self.config.structured_decoder() else {
+ return Ok(None);
+ };
+ let resources = self.structured_resources()?;
+ let Some(context_frames) = fishaudio.streaming_decode_context_frames(resources.as_ref())? else {
+ return Ok(None);
+ };
+
+ self.validate_fishaudio_token_delta(new_tokens, fishaudio)?;
+ state.append_delta(new_tokens)?;
+
+ let mut window_start = state.total_frames();
+ for &emitted in &state.emitted_semantic_lengths {
+ window_start = window_start.min(emitted.saturating_sub(context_frames));
+ }
+ let window_end = state.total_frames();
+ let batch_size = state.batch_size;
+ let codebooks = state.codebooks;
+ let previous_audio_lengths = state.emitted_audio_lengths.clone().into_boxed_slice();
+ let semantic_lengths = state.semantic_lengths.clone().into_boxed_slice();
+ let (window_tokens, window_lengths, window_frames) = state.flatten_window(window_start, window_end)?;
+ let submitted = fishaudio.submit_decode_padded(
+ resources.as_ref(),
+ window_tokens,
+ window_lengths,
+ batch_size,
+ codebooks,
+ window_frames,
+ )?;
+ // The scratch buffers inside the workspace were encoded into the
+ // submitted command buffer. Reset the workspace so the next decode
+ // pass allocates fresh buffers instead of reusing ones the in-flight
+ // command buffer may still be reading.
+ resources.reset_for_pending();
+ let audio_offset_frames = window_start
+ .checked_mul(fishaudio.upsample_factor)
+ .ok_or(AudioError::Runtime("stream audio offset overflow".to_string()))?;
+ state.record_last_step_stats(new_tokens.frames(), window_start, window_frames);
+ state.mark_submitted_audio_window(&semantic_lengths, fishaudio.upsample_factor)?;
+
+ Ok(Some(PendingStreamPcmChunk {
+ runtime: self.clone(),
+ submitted,
+ previous_audio_lengths,
+ semantic_lengths,
+ audio_offset_frames,
+ upsample_factor: fishaudio.upsample_factor,
+ step_stats: state.last_step_stats(),
+ }))
+ }
+}
+
+impl<B: Backend> AudioCodecRuntime for NanoCodecFsqRuntime<B> {
+ fn encode(
+ &self,
+ pcm: &AudioPcmBatch,
+ ) -> AudioResult<AudioTokenGrid> {
+ if self.config.structured_decoder().is_some() {
+ return Err(AudioError::Runtime("encode is not supported when a decoder graph is configured".to_string()));
+ }
+ self.fsq_encode(pcm)
+ }
+
+ fn decode(
+ &self,
+ tokens: &AudioTokenGrid,
+ ) -> AudioResult<AudioPcmBatch> {
+ let decoded = self.decode_padded(tokens)?;
+ self.decoded_padded_to_pcm_batch(&decoded)
+ }
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/loaders.rs b/crates/uzu/src/audio/nanocodec/runtime/loaders.rs
new file mode 100644
index 000000000..a81ea9359
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/loaders.rs
@@ -0,0 +1,93 @@
+use super::*;
+
+pub(super) fn resolve_descript_audio_codec_vocoder_data_type(
+ top_level_precision: Option<ConfigDataType>,
+ config: &DescriptAudioCodecConfig,
+) -> AudioResult<DataType> {
+ let mut resolved_precision: Option<ConfigDataType> = None;
+ for (field_name, precision) in [
+ ("tts_config.activation_precision", top_level_precision),
+ ("tts_config.audio_decoder_config.precision", Some(config.precision)),
+ ("tts_config.audio_decoder_config.quantizer_config.precision", config.quantizer_config.precision),
+ ] {
+ if let Some(precision) = precision {
+ if let Some(existing) = resolved_precision {
+ if existing != precision {
+ return Err(AudioError::Runtime(format!(
+ "conflicting DescriptAudioCodec precision in Lalamo export: {field_name}={precision:?} conflicts with {existing:?}"
+ )));
+ }
+ } else {
+ resolved_precision = Some(precision);
+ }
+ }
+ }
+
+ let precision = resolved_precision.ok_or(AudioError::Runtime(
+ "missing DescriptAudioCodec precision in Lalamo export; expected one of tts_config.activation_precision, \
+tts_config.audio_decoder_config.precision, or tts_config.audio_decoder_config.quantizer_config.precision"
+ .to_string(),
+ ))?;
+ let data_type: DataType = precision.into();
+ if !matches!(data_type, DataType::F32 | DataType::F16 | DataType::BF16) {
+ return Err(AudioError::Runtime(format!(
+ "unsupported DescriptAudioCodec vocoder precision in Lalamo export: {precision:?} (expected float32/float16/bfloat16)"
+ )));
+ }
+ Ok(data_type)
+}
+
+pub(super) fn load_audio_runtime_from_tts_config(
+ tts_config: &TtsConfig,
+ model_path: &Path,
+) -> AudioResult<(RuntimeConfigJson, StructuredAudioCodecGraph)> {
+ let cfg = match &tts_config.audio_decoder_config {
+ TtsAudioDecoderConfig::DescriptAudioCodecConfig {
+ config,
+ } => config,
+ };
+ let fishaudio_weights = model_path.join("model.safetensors");
+ if !fishaudio_weights.is_file() {
+ return Err(AudioError::Runtime(format!(
+ "missing exported FishAudio decoder weights '{}'",
+ fishaudio_weights.display()
+ )));
+ }
+
+ let vocoder_data_type = resolve_descript_audio_codec_vocoder_data_type(tts_config.activation_precision, cfg)?;
+
+ let total_codebooks = cfg
+ .n_codebooks
+ .checked_add(1)
+ .ok_or(AudioError::Runtime("FishAudio codebook count overflow while building runtime config".to_string()))?;
+ let codebook_size_i32 = i32::try_from(cfg.codebook_size)
+ .map_err(|_| AudioError::Runtime("FishAudio codebook_size exceeds i32 kernel range".to_string()))?;
+ if codebook_size_i32 <= 1 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+
+ let upsample_factor = cfg
+ .downsample_factor
+ .iter()
+ .chain(cfg.decoder_rates.iter())
+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))
+ .ok_or(AudioError::Runtime("FishAudio upsample factor overflow".to_string()))?;
+
+ let runtime = RuntimeConfigJson {
+ sample_rate: cfg.samplerate,
+ num_groups: total_codebooks,
+ num_levels_per_group: vec![codebook_size_i32],
+ eps: default_eps(),
+ };
+ let decoder = StructuredAudioCodecGraph {
+ config: cfg.clone(),
+ weights_path: fishaudio_weights.display().to_string(),
+ codebook_size: cfg.codebook_size,
+ semantic_codebook_size: cfg.semantic_codebook_size,
+ input_dim: cfg.input_dim,
+ total_codebooks,
+ upsample_factor,
+ vocoder_data_type,
+ };
+ Ok((runtime, decoder))
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/profile.rs b/crates/uzu/src/audio/nanocodec/runtime/profile.rs
new file mode 100644
index 000000000..8c19f5df3
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/profile.rs
@@ -0,0 +1,75 @@
+use std::sync::mpsc::Receiver;
+
+use super::*;
+
+pub struct SubmittedDecodedPaddedAudio<B: Backend> {
+ pub(in crate::audio::nanocodec::runtime) output: Array<B>,
+ pub(in crate::audio::nanocodec::runtime) channels: usize,
+ pub(in crate::audio::nanocodec::runtime) frames: usize,
+ pub(in crate::audio::nanocodec::runtime) lengths: Vec<usize>,
+ pub(in crate::audio::nanocodec::runtime) final_command_buffer: Option<Pending<B>>,
+ pub(in crate::audio::nanocodec::runtime) completion_notification: Option<Receiver<()>>,
+}
+
+impl<B: Backend> SubmittedDecodedPaddedAudio<B> {
+ pub(crate) fn is_complete(&self) -> bool {
+ use std::sync::mpsc::TryRecvError;
+
+ self.completion_notification.as_ref().is_none_or(|notification| match notification.try_recv() {
+ Ok(()) | Err(TryRecvError::Disconnected) => true,
+ Err(TryRecvError::Empty) => false,
+ })
+ }
+
+ pub(in crate::audio::nanocodec::runtime) fn resolve(mut self) -> AudioResult<DecodedPaddedAudio> {
+ if let Some(command_buffer) = self.final_command_buffer.take() {
+ command_buffer.wait_until_completed().map_err(|err| {
+ AudioError::Runtime(format!("failed to wait for FishAudio decoder command buffer: {err}"))
+ })?;
+ }
+ let samples: Vec<f32> = match self.output.data_type() {
+ DataType::F32 => self.output.as_slice::<f32>().to_vec(),
+ DataType::F16 => self.output.as_slice::<half::f16>().iter().map(|&v| f32::from(v)).collect(),
+ DataType::BF16 => self.output.as_slice::<half::bf16>().iter().map(|&v| f32::from(v)).collect(),
+ dt => return Err(AudioError::Runtime(format!("unsupported vocoder output dtype: {dt:?}"))),
+ };
+ Ok(DecodedPaddedAudio {
+ samples,
+ channels: self.channels,
+ frames: self.frames,
+ lengths: self.lengths,
+ })
+ }
+}
+
+pub(crate) struct PendingStreamPcmChunk<B: Backend> {
+ pub(in crate::audio::nanocodec::runtime) runtime: NanoCodecFsqRuntime<B>,
+ pub(in crate::audio::nanocodec::runtime) submitted: SubmittedDecodedPaddedAudio<B>,
+ pub(in crate::audio::nanocodec::runtime) previous_audio_lengths: Box<[usize]>,
+ pub(in crate::audio::nanocodec::runtime) semantic_lengths: Box<[usize]>,
+ pub(in crate::audio::nanocodec::runtime) audio_offset_frames: usize,
+ pub(in crate::audio::nanocodec::runtime) upsample_factor: usize,
+ pub(in crate::audio::nanocodec::runtime) step_stats: AudioDecodeStepStats,
+}
+
+impl<B: Backend> PendingStreamPcmChunk<B> {
+ pub(crate) fn step_stats(&self) -> AudioDecodeStepStats {
+ self.step_stats
+ }
+
+ pub(crate) fn is_complete(&self) -> bool {
+ self.submitted.is_complete()
+ }
+
+ pub(crate) fn resolve(self) -> AudioResult<AudioPcmBatch> {
+ let decoded_window = self.submitted.resolve()?;
+ let delta = extract_delta_from_padded_with_offset_snapshot(
+ &decoded_window,
+ &self.previous_audio_lengths,
+ &self.semantic_lengths,
+ self.audio_offset_frames,
+ self.upsample_factor,
+ )?;
+ self.runtime.decoded_padded_to_pcm_batch(&delta)
+ }
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/stream.rs b/crates/uzu/src/audio/nanocodec/runtime/stream.rs
new file mode 100644
index 000000000..330eb0804
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/stream.rs
@@ -0,0 +1,481 @@
+use super::*;
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub struct AudioDecodeStepStats {
+ pub input_frames: usize,
+ pub total_semantic_frames: usize,
+ pub decoded_window_start_frame: usize,
+ pub decoded_window_frames: usize,
+}
+
+/// Ring-buffer backed stream state for incremental audio decoding.
+///
+/// Token rows are stored in fixed-capacity circular buffers so that evicting
+/// old frames is O(1) (just advance the head pointer) instead of O(N) via
+/// `Vec::drain`.
+#[derive(Debug, Clone)]
+pub struct AudioDecodeStreamState {
+ pub(super) batch_size: usize,
+ pub(super) codebooks: usize,
+ retained_capacity: usize,
+ stored_frame_start: usize,
+ stored_frames: usize,
+ total_frames_generated: usize,
+ row_tokens: Vec<Vec<u32>>,
+ flattened_tokens: Vec<u32>,
+ window_lengths: Vec<usize>,
+ pub(super) semantic_lengths: Vec<usize>,
+ pub(super) emitted_semantic_lengths: Vec<usize>,
+ pub(super) emitted_audio_lengths: Vec<usize>,
+ last_step_stats: AudioDecodeStepStats,
+}
+
+impl AudioDecodeStreamState {
+ pub(super) fn new(
+ batch_size: usize,
+ codebooks: usize,
+ max_workspace_frames: usize,
+ ) -> AudioResult<Self> {
+ if batch_size == 0 {
+ return Err(AudioError::Runtime("stream state batch_size must be > 0".to_string()));
+ }
+ if codebooks == 0 {
+ return Err(AudioError::Runtime("stream state codebooks must be > 0".to_string()));
+ }
+ if max_workspace_frames == 0 {
+ return Err(AudioError::Runtime("stream state max_workspace_frames must be > 0".to_string()));
+ }
+
+ let row_count = batch_size
+ .checked_mul(codebooks)
+ .ok_or(AudioError::Runtime("stream state row count overflow".to_string()))?;
+ // Double the workspace for ring capacity so we have room for context
+ // overlap without immediate eviction.
+ let retained_capacity = max_workspace_frames
+ .checked_mul(2)
+ .ok_or(AudioError::Runtime("stream retained capacity overflow".to_string()))?;
+ let mut row_tokens = Vec::with_capacity(row_count);
+ for _ in 0..row_count {
+ row_tokens.push(vec![0u32; retained_capacity]);
+ }
+
+ Ok(Self {
+ batch_size,
+ codebooks,
+ retained_capacity,
+ stored_frame_start: 0,
+ stored_frames: 0,
+ total_frames_generated: 0,
+ row_tokens,
+ flattened_tokens: Vec::with_capacity(
+ row_count
+ .checked_mul(max_workspace_frames)
+ .ok_or(AudioError::Runtime("stream token capacity overflow".to_string()))?,
+ ),
+ window_lengths: Vec::with_capacity(batch_size),
+ semantic_lengths: vec![0; batch_size],
+ emitted_semantic_lengths: vec![0; batch_size],
+ emitted_audio_lengths: vec![0; batch_size],
+ last_step_stats: AudioDecodeStepStats::default(),
+ })
+ }
+
+ pub(super) fn total_frames(&self) -> usize {
+ self.total_frames_generated
+ }
+
+ pub fn last_step_stats(&self) -> AudioDecodeStepStats {
+ self.last_step_stats
+ }
+
+ /// Global frame index one past the last retained frame.
+ fn stored_frame_end(&self) -> usize {
+ self.stored_frame_start.saturating_add(self.stored_frames)
+ }
+
+ /// Physical ring slot for a global frame index.
+ #[inline]
+ fn slot(
+ &self,
+ global_frame: usize,
+ ) -> usize {
+ global_frame % self.retained_capacity
+ }
+
+ pub(super) fn append_delta(
+ &mut self,
+ delta_tokens: &AudioTokenGrid,
+ ) -> AudioResult<()> {
+ if delta_tokens.batch_size() != self.batch_size {
+ return Err(AudioError::Runtime(format!(
+ "stream delta batch mismatch: expected {}, got {}",
+ self.batch_size,
+ delta_tokens.batch_size()
+ )));
+ }
+ if delta_tokens.codebooks() != self.codebooks {
+ return Err(AudioError::Runtime(format!(
+ "stream delta codebook mismatch: expected {}, got {}",
+ self.codebooks,
+ delta_tokens.codebooks()
+ )));
+ }
+ if delta_tokens.frames() == 0 {
+ return Ok(());
+ }
+
+ let delta_frames = delta_tokens.frames();
+ let tokens = delta_tokens.tokens();
+ let delta_lengths = delta_tokens.lengths();
+
+ for frame_offset in 0..delta_frames {
+ let global_frame = self.stored_frame_start + self.stored_frames;
+ // Evict oldest frame if ring is full.
+ if self.stored_frames == self.retained_capacity {
+ self.stored_frame_start += 1;
+ self.stored_frames -= 1;
+ }
+ let physical = self.slot(global_frame);
+
+ for batch in 0..self.batch_size {
+ for codebook in 0..self.codebooks {
+ let row_index = batch * self.codebooks + codebook;
+ let src_index = row_index * delta_frames + frame_offset;
+ self.row_tokens[row_index][physical] = tokens[src_index];
+ }
+ }
+ self.stored_frames += 1;
+ }
+ self.total_frames_generated = self.total_frames_generated.saturating_add(delta_frames);
+
+ for (length, &delta_len) in self.semantic_lengths.iter_mut().zip(delta_lengths.iter()) {
+ *length = length
+ .checked_add(delta_len)
+ .ok_or(AudioError::Runtime("stream semantic length overflow".to_string()))?;
+ }
+
+ Ok(())
+ }
+
+ pub(super) fn to_full_grid(&self) -> AudioResult<AudioTokenGrid> {
+ if self.stored_frame_start != 0 {
+ return Err(AudioError::Runtime(format!(
+ "full-grid decode requires retained prefix, but {} frames were evicted",
+ self.stored_frame_start
+ )));
+ }
+ let total_frames = self.stored_frames;
+
+ let expected_tokens = self
+ .batch_size
+ .checked_mul(self.codebooks)
+ .and_then(|value| value.checked_mul(total_frames))
+ .ok_or(AudioError::Runtime("stream token count overflow".to_string()))?;
+ let mut tokens = Vec::with_capacity(expected_tokens);
+
+ for row in &self.row_tokens {
+ // When stored_frame_start == 0 and no wrap has occurred, frames
+ // 0..total_frames map to physical slots 0..total_frames.
+ for frame in 0..total_frames {
+ let physical = self.slot(frame);
+ tokens.push(row[physical]);
+ }
+ }
+ if tokens.len() != expected_tokens {
+ return Err(AudioError::Runtime("stream flattened token count mismatch".to_string()));
+ }
+
+ AudioTokenGrid::new(
+ tokens.into_boxed_slice(),
+ self.batch_size,
+ self.codebooks,
+ total_frames,
+ self.semantic_lengths.clone().into_boxed_slice(),
+ )
+ }
+
+ pub(super) fn flatten_window(
+ &mut self,
+ start_frame: usize,
+ end_frame: usize,
+ ) -> AudioResult<(&[u32], &[usize], usize)> {
+ if end_frame < start_frame {
+ return Err(AudioError::Runtime(format!(
+ "invalid stream window: start_frame={start_frame}, end_frame={end_frame}"
+ )));
+ }
+ let total_frames = self.total_frames();
+ if end_frame > total_frames {
+ return Err(AudioError::Runtime(format!(
+ "stream window end exceeds token frames: end={end_frame}, total={total_frames}"
+ )));
+ }
+ if start_frame < self.stored_frame_start || end_frame > self.stored_frame_end() {
+ return Err(AudioError::Runtime(format!(
+ "stream window [{start_frame}, {end_frame}) exceeds retained workspace [{}, {})",
+ self.stored_frame_start,
+ self.stored_frame_end()
+ )));
+ }
+
+ let window_frames = end_frame.saturating_sub(start_frame);
+ let row_count = self
+ .batch_size
+ .checked_mul(self.codebooks)
+ .ok_or(AudioError::Runtime("stream row count overflow".to_string()))?;
+ let required_capacity = row_count
+ .checked_mul(window_frames)
+ .ok_or(AudioError::Runtime("stream window token capacity overflow".to_string()))?;
+
+ // Grow the scratch buffer if needed (only happens once in practice).
+ if self.flattened_tokens.capacity() < required_capacity {
+ self.flattened_tokens.reserve(required_capacity - self.flattened_tokens.capacity());
+ }
+ self.flattened_tokens.clear();
+
+ // Read from ring in logical order via modulo indexing.
+ for row in &self.row_tokens {
+ for global_frame in start_frame..end_frame {
+ let physical = self.slot(global_frame);
+ self.flattened_tokens.push(row[physical]);
+ }
+ }
+
+ self.window_lengths.clear();
+ for &length in &self.semantic_lengths {
+ self.window_lengths.push(length.saturating_sub(start_frame));
+ }
+
+ Ok((&self.flattened_tokens, &self.window_lengths, window_frames))
+ }
+
+ pub(super) fn extract_delta_from_padded_with_offset(
+ &mut self,
+ full_padded: &DecodedPaddedAudio,
+ audio_offset_frames: usize,
+ upsample_factor: usize,
+ ) -> AudioResult<DecodedPaddedAudio> {
+ let delta = extract_delta_from_padded_with_offset_snapshot(
+ full_padded,
+ &self.emitted_audio_lengths,
+ &self.semantic_lengths,
+ audio_offset_frames,
+ upsample_factor,
+ )?;
+ for index in 0..self.semantic_lengths.len() {
+ let semantic_length = self.semantic_lengths[index];
+ let full_audio_length = semantic_length
+ .checked_mul(upsample_factor)
+ .ok_or(AudioError::Runtime("stream audio length overflow".to_string()))?;
+ self.emitted_audio_lengths[index] = full_audio_length;
+ self.emitted_semantic_lengths[index] = semantic_length;
+ }
+ Ok(delta)
+ }
+
+ pub(super) fn mark_submitted_audio_window(
+ &mut self,
+ semantic_lengths: &[usize],
+ upsample_factor: usize,
+ ) -> AudioResult<()> {
+ if semantic_lengths.len() != self.batch_size {
+ return Err(AudioError::Runtime(format!(
+ "stream semantic lengths mismatch: expected {}, got {}",
+ self.batch_size,
+ semantic_lengths.len()
+ )));
+ }
+ for (index, &semantic_length) in semantic_lengths.iter().enumerate() {
+ let full_audio_length = semantic_length
+ .checked_mul(upsample_factor)
+ .ok_or(AudioError::Runtime("stream audio length overflow".to_string()))?;
+ self.emitted_audio_lengths[index] = full_audio_length;
+ self.emitted_semantic_lengths[index] = semantic_length;
+ }
+ Ok(())
+ }
+
+ pub(super) fn record_last_step_stats(
+ &mut self,
+ input_frames: usize,
+ decoded_window_start_frame: usize,
+ decoded_window_frames: usize,
+ ) {
+ self.last_step_stats = AudioDecodeStepStats {
+ input_frames,
+ total_semantic_frames: self.total_frames_generated,
+ decoded_window_start_frame,
+ decoded_window_frames,
+ };
+ }
+}
+
+pub(super) fn extract_delta_from_padded_with_offset_snapshot(
+ full_padded: &DecodedPaddedAudio,
+ previous_audio_lengths: &[usize],
+ semantic_lengths: &[usize],
+ audio_offset_frames: usize,
+ upsample_factor: usize,
+) -> AudioResult<DecodedPaddedAudio> {
+ if full_padded.lengths.len() != semantic_lengths.len() || previous_audio_lengths.len() != semantic_lengths.len() {
+ return Err(AudioError::Runtime(format!(
+ "stream decoded batch mismatch: full_lengths={}, previous_lengths={}, semantic_lengths={}",
+ full_padded.lengths.len(),
+ previous_audio_lengths.len(),
+ semantic_lengths.len()
+ )));
+ }
+ if upsample_factor == 0 {
+ return Err(AudioError::Runtime("stream upsample_factor must be > 0".to_string()));
+ }
+
+ let batch_size = semantic_lengths.len();
+ let channels = full_padded.channels;
+ let decoded_frames = full_padded.frames;
+ let mut delta_lengths = vec![0usize; batch_size];
+ let mut max_delta_frames = 0usize;
+ let mut local_delta_ranges = vec![(0usize, 0usize); batch_size];
+
+ for batch in 0..batch_size {
+ let full_audio_length = semantic_lengths[batch]
+ .checked_mul(upsample_factor)
+ .ok_or(AudioError::Runtime("stream audio length overflow".to_string()))?;
+ let previous_audio_length = previous_audio_lengths[batch].min(full_audio_length);
+ let decoded_batch_length = full_padded.lengths[batch];
+ let target_end_local = full_audio_length.saturating_sub(audio_offset_frames).min(decoded_batch_length);
+ let target_start_local = previous_audio_length.saturating_sub(audio_offset_frames).min(target_end_local);
+ let delta = target_end_local.saturating_sub(target_start_local);
+ local_delta_ranges[batch] = (target_start_local, target_end_local);
+ delta_lengths[batch] = delta;
+ max_delta_frames = max_delta_frames.max(delta);
+ }
+
+ let padded_len = checked_product(&[batch_size, channels, max_delta_frames])?;
+ let mut delta_padded = vec![0.0_f32; padded_len];
+ for (batch, &(start_local, end_local)) in local_delta_ranges.iter().enumerate() {
+ let delta_len = end_local.saturating_sub(start_local);
+ if delta_len == 0 {
+ continue;
+ }
+ for channel in 0..channels {
+ let src_start = (batch * channels + channel)
+ .checked_mul(decoded_frames)
+ .and_then(|value| value.checked_add(start_local))
+ .ok_or(AudioError::Runtime("stream padded source offset overflow".to_string()))?;
+ let src_end = src_start
+ .checked_add(delta_len)
+ .ok_or(AudioError::Runtime("stream padded source offset overflow".to_string()))?;
+ let dst_start = (batch * channels + channel)
+ .checked_mul(max_delta_frames)
+ .ok_or(AudioError::Runtime("stream padded destination offset overflow".to_string()))?;
+ let dst_end = dst_start
+ .checked_add(delta_len)
+ .ok_or(AudioError::Runtime("stream padded destination offset overflow".to_string()))?;
+ delta_padded[dst_start..dst_end].copy_from_slice(&full_padded.samples[src_start..src_end]);
+ }
+ }
+
+ Ok(DecodedPaddedAudio {
+ samples: delta_padded,
+ channels,
+ frames: max_delta_frames,
+ lengths: delta_lengths,
+ })
+}
+
+pub(super) fn pack_pcm_to_padded(
+ pcm: &AudioPcmBatch,
+ expected_channels: usize,
+) -> AudioResult<(Vec<f32>, Vec<usize>, Vec<i32>, usize)> {
+ if pcm.channels() != expected_channels {
+ return Err(AudioError::Runtime(format!(
+ "pcm channel mismatch: expected {expected_channels}, got {}",
+ pcm.channels()
+ )));
+ }
+
+ let lengths = pcm.lengths().to_vec();
+ let frames = lengths.iter().copied().max().unwrap_or(0);
+ let lengths_i32 = convert_lengths_to_i32(&lengths, frames)?;
+ let padded_len = checked_product(&[pcm.batch_size(), expected_channels, frames])?;
+ let mut padded = vec![0.0_f32; padded_len];
+
+ let mut src_offset = 0usize;
+ let samples = pcm.samples();
+ for (batch, &frame_count) in lengths.iter().enumerate() {
+ let sample_count = frame_count
+ .checked_mul(expected_channels)
+ .ok_or(AudioError::Runtime("pcm frame-count overflow".to_string()))?;
+ let src_end =
+ src_offset.checked_add(sample_count).ok_or(AudioError::Runtime("pcm indexing overflow".to_string()))?;
+ if src_end > samples.len() {
+ return Err(AudioError::Runtime("pcm indexing out of bounds".to_string()));
+ }
+
+ for frame in 0..frame_count {
+ let src_frame_base = src_offset + frame * expected_channels;
+ for channel in 0..expected_channels {
+ let dst_index = (batch * expected_channels + channel) * frames + frame;
+ padded[dst_index] = samples[src_frame_base + channel];
+ }
+ }
+
+ src_offset = src_end;
+ }
+
+ Ok((padded, lengths, lengths_i32, frames))
+}
+
+pub(super) fn unpack_padded_to_pcm(
+ padded: &[f32],
+ batch_size: usize,
+ channels: usize,
+ frames: usize,
+ lengths: &[usize],
+) -> AudioResult<Vec<f32>> {
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+
+ let expected_padded = checked_product(&[batch_size, channels, frames])?;
+ if padded.len() != expected_padded {
+ return Err(AudioError::InvalidPcmShape {
+ expected_samples: expected_padded,
+ actual_samples: padded.len(),
+ });
+ }
+
+ let total_frames = lengths.iter().sum::<usize>();
+ let packed_len = checked_product(&[total_frames, channels])?;
+ let mut packed = vec![0.0_f32; packed_len];
+ let mut dst_offset = 0usize;
+
+ for (batch, &frame_count) in lengths.iter().enumerate() {
+ if frame_count > frames {
+ return Err(AudioError::InvalidTokenLengthValue {
+ length: frame_count,
+ frames,
+ });
+ }
+
+ for frame in 0..frame_count {
+ for channel in 0..channels {
+ let src_index = (batch * channels + channel) * frames + frame;
+ let dst_index = dst_offset + frame * channels + channel;
+ packed[dst_index] = padded[src_index];
+ }
+ }
+ dst_offset = dst_offset
+ .checked_add(
+ frame_count
+ .checked_mul(channels)
+ .ok_or(AudioError::Runtime("pcm destination indexing overflow".to_string()))?,
+ )
+ .ok_or(AudioError::Runtime("pcm destination indexing overflow".to_string()))?;
+ }
+
+ Ok(packed)
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs
new file mode 100644
index 000000000..034414d0c
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs
@@ -0,0 +1,358 @@
+use super::*;
+
+pub(super) type BackendParameterTree<'loader, B> = crate::parameters::ParameterTree<'loader, <B as Backend>::Context>;
+
+pub(super) fn read_float_array<B: Backend, const RANK: usize>(
+ tree: &BackendParameterTree<'_, B>,
+ name: &str,
+ expected_data_type: DataType,
+) -> AudioResult<([usize; RANK], Array<B>)> {
+ let array = tree.leaf_array(name)?;
+ if array.data_type() != expected_data_type {
+ return Err(AudioError::Runtime(format!(
+ "tensor '{name}' dtype mismatch: expected {expected_data_type:?}, got {:?}",
+ array.data_type()
+ )));
+ }
+ if array.shape().len() != RANK {
+ return Err(AudioError::Runtime(format!(
+ "expected rank-{RANK} tensor for '{name}', got shape {:?}",
+ array.shape()
+ )));
+ }
+ let mut dims = [0usize; RANK];
+ dims.copy_from_slice(array.shape());
+ Ok((dims, array))
+}
+
+pub(super) fn read_float_vector_exact<B: Backend>(
+ tree: &BackendParameterTree<'_, B>,
+ name: &str,
+ expected_len: usize,
+ expected_data_type: DataType,
+) -> AudioResult<Array<B>> {
+ let (shape, array) = read_float_array::<B, 1>(tree, name, expected_data_type)?;
+ if shape[0] != expected_len {
+ return Err(AudioError::Runtime(format!(
+ "tensor '{name}' shape mismatch: expected [{expected_len}], got {:?}",
+ shape
+ )));
+ }
+ Ok(array)
+}
+
+pub(super) fn read_float_matrix_exact<B: Backend>(
+ tree: &BackendParameterTree<'_, B>,
+ name: &str,
+ expected_rows: usize,
+ expected_cols: usize,
+ expected_data_type: DataType,
+) -> AudioResult<Array<B>> {
+ let (shape, array) = read_float_array::<B, 2>(tree, name, expected_data_type)?;
+ if shape != [expected_rows, expected_cols] {
+ return Err(AudioError::Runtime(format!(
+ "tensor '{name}' shape mismatch: expected [{expected_rows}, {expected_cols}], got {:?}",
+ shape
+ )));
+ }
+ Ok(array)
+}
+
+pub(super) fn outer_axis_view<B: Backend>(
+ array: &Array<B>,
+ index: usize,
+ slice_shape: &[usize],
+) -> AudioResult<Array<B>> {
+ let slice_bytes = size_for_shape(slice_shape, array.data_type());
+ let offset = index
+ .checked_mul(slice_bytes)
+ .and_then(|value| value.checked_add(array.offset()))
+ .ok_or(AudioError::Runtime("array outer-axis view offset overflow".to_string()))?;
+ Ok(unsafe { Array::from_parts(array.buffer(), offset, slice_shape, array.data_type()) })
+}
+
+pub(super) fn read_conv1d_layer<B: Backend>(
+ tree: &BackendParameterTree<'_, B>,
+ data_type: DataType,
+ dilation: usize,
+ groups: usize,
+) -> AudioResult<StructuredAudioConv1d<B>> {
+ let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;
+ let bias = read_float_vector_exact::<B>(tree, "biases", shape[0], data_type)?;
+ if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ if shape[0] % groups != 0 {
+ return Err(AudioError::Runtime(format!(
+ "invalid grouped conv weights for 'weights': out_channels {} not divisible by groups {groups}",
+ shape[0]
+ )));
+ }
+ Ok(StructuredAudioConv1d {
+ weight,
+ bias,
+ cin: shape[1].checked_mul(groups).ok_or(AudioError::Runtime("conv input channel overflow".to_string()))?,
+ cout: shape[0],
+ kernel_size: shape[2],
+ dilation,
+ groups,
+ })
+}
+
+pub(super) fn read_conv_transpose1d_layer<B: Backend>(
+ tree: &BackendParameterTree<'_, B>,
+ data_type: DataType,
+ stride: usize,
+ groups: usize,
+) -> AudioResult<StructuredAudioConvTranspose1d<B>> {
+ if stride == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;
+ if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ let export_cout = shape[0];
+ let export_cin =
+ shape[1].checked_mul(groups).ok_or(AudioError::Runtime("transpose conv input channel overflow".to_string()))?;
+ if export_cout % groups != 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ let bias = read_float_vector_exact::<B>(tree, "biases", export_cout, data_type)?;
+ let expected_weight_shape = [export_cout, export_cin / groups, shape[2]];
+ if shape != expected_weight_shape {
+ return Err(AudioError::Runtime(format!(
+ "transpose conv export-layout weight shape mismatch: expected {:?}, got {:?}",
+ expected_weight_shape, shape
+ )));
+ }
+
+ Ok(StructuredAudioConvTranspose1d {
+ weight,
+ bias,
+ cin: export_cin,
+ cout: export_cout,
+ kernel_size: shape[2],
+ stride,
+ groups,
+ })
+}
+
+pub(super) fn read_pointwise_conv_layer<B: Backend>(
+ tree: &BackendParameterTree<'_, B>,
+ data_type: DataType,
+ expected_out_dim: usize,
+ expected_in_dim: usize,
+) -> AudioResult<StructuredAudioPointwiseConv<B>> {
+ let weight = read_float_matrix_exact::<B>(tree, "weights", expected_out_dim, expected_in_dim, data_type)?;
+ let bias = read_float_vector_exact::<B>(tree, "biases", expected_out_dim, data_type)?;
+ Ok(StructuredAudioPointwiseConv {
+ weight: weight.view(&[expected_out_dim, expected_in_dim, 1]),
+ bias,
+ cin: expected_in_dim,
+ cout: expected_out_dim,
+ })
+}
+
+pub(super) fn read_norm_layer<B: Backend>(
+ context: &Rc<B::Context>,
+ tree: &BackendParameterTree<'_, B>,
+ data_type: DataType,
+ channels: usize,
+ epsilon: f32,
+ subtract_mean: bool,
+ use_bias: bool,
+ label_prefix: &str,
+) -> AudioResult<StructuredAudioNorm<B>> {
+ let scales = read_float_vector_exact::<B>(tree, "scales", channels, data_type)?;
+ let bias = if use_bias {
+ read_float_vector_exact::<B>(tree, "biases", channels, data_type)?
+ } else {
+ context.create_array(&[channels], data_type, &format!("{label_prefix}_bias"))
+ };
+ Ok(StructuredAudioNorm {
+ scales,
+ bias,
+ epsilon,
+ subtract_mean,
+ })
+}
+
+pub(super) fn read_convnext_layer<B: Backend>(
+ context: &Rc<B::Context>,
+ tree: &BackendParameterTree<'_, B>,
+ data_type: DataType,
+ norm_config: &DescriptAudioConvNeXtNormConfig,
+) -> AudioResult<StructuredAudioConvNeXt<B>> {
+ let depthwise_tree = tree.subtree("dwconv")?;
+ let (depthwise_shape, depthwise_weight) = read_float_array::<B, 3>(&depthwise_tree, "weights", data_type)?;
+ if depthwise_shape[1] != 1 {
+ return Err(AudioError::Runtime(format!(
+ "ConvNeXt depthwise weight in_channels_per_group must be 1 at 'dwconv.weights', got {}",
+ depthwise_shape[1]
+ )));
+ }
+ let depthwise_conv = StructuredAudioConv1d {
+ weight: depthwise_weight,
+ bias: read_float_vector_exact::<B>(&depthwise_tree, "biases", depthwise_shape[0], data_type)?,
+ cin: depthwise_shape[0],
+ cout: depthwise_shape[0],
+ kernel_size: depthwise_shape[2],
+ dilation: 1,
+ groups: depthwise_shape[0],
+ };
+ if depthwise_conv.bias.shape()[0] != depthwise_conv.cout {
+ return Err(AudioError::Runtime(format!(
+ "ConvNeXt depthwise bias mismatch at 'dwconv.biases': expected {}, got {}",
+ depthwise_conv.cout,
+ depthwise_conv.bias.shape()[0]
+ )));
+ }
+ if depthwise_conv.cout == 0 || depthwise_conv.kernel_size == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ if depthwise_conv.cin != depthwise_conv.cout {
+ return Err(AudioError::Runtime(format!(
+ "ConvNeXt depthwise conv expects cin==cout, got {} vs {} at 'dwconv.weights'",
+ depthwise_conv.cin, depthwise_conv.cout
+ )));
+ }
+ let channels = depthwise_conv.cout;
+ let norm = read_norm_layer::<B>(
+ context,
+ &tree.subtree("norm")?,
+ data_type,
+ channels,
+ norm_config.epsilon,
+ norm_config.subtract_mean,
+ norm_config.use_bias,
+ "structured_audio_convnext_norm",
+ )?;
+ let (pwconv1_shape, _) = read_float_array::<B, 2>(&tree.subtree("pwconv1")?, "weights", data_type)?;
+ if pwconv1_shape[1] != channels {
+ return Err(AudioError::Runtime(format!(
+ "ConvNeXt pwconv1 input mismatch at 'pwconv1.weights': expected {}, got {}",
+ channels, pwconv1_shape[1]
+ )));
+ }
+ let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, pwconv1_shape[0], channels)?;
+ let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, channels, pwconv1_shape[0])?;
+ Ok(StructuredAudioConvNeXt {
+ depthwise_conv,
+ norm,
+ pwconv1,
+ pwconv2,
+ })
+}
+
+pub(super) fn read_residual_unit_layer<B: Backend>(
+ tree: &BackendParameterTree<'_, B>,
+ data_type: DataType,
+ dilation: usize,
+) -> AudioResult<StructuredAudioResidualUnit<B>> {
+ let snake1_alpha = {
+ let snake1_tree = tree.subtree("snake1")?;
+ let (shape, alpha) = read_float_array::<B, 1>(&snake1_tree, "alpha", data_type)?;
+ if shape[0] == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ alpha
+ };
+ let conv1 = read_conv1d_layer::<B>(&tree.subtree("conv1")?, data_type, dilation, 1)?;
+ let snake2_alpha = {
+ let snake2_tree = tree.subtree("snake2")?;
+ let (shape, alpha) = read_float_array::<B, 1>(&snake2_tree, "alpha", data_type)?;
+ if shape[0] != conv1.cout {
+ return Err(AudioError::Runtime(format!(
+ "residual snake2 alpha mismatch: expected {}, got {}",
+ conv1.cout, shape[0]
+ )));
+ }
+ alpha
+ };
+ if snake1_alpha.shape()[0] != conv1.cin {
+ return Err(AudioError::Runtime(format!(
+ "residual snake1 alpha mismatch: expected {}, got {}",
+ conv1.cin,
+ snake1_alpha.shape()[0]
+ )));
+ }
+ let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, 1, 1)?;
+ Ok(StructuredAudioResidualUnit {
+ snake1_alpha,
+ conv1,
+ snake2_alpha,
+ conv2,
+ })
+}
+
+pub(super) fn build_vocoder_graph_from_tree<B: Backend>(
+ context: &Rc<B::Context>,
+ root: &BackendParameterTree<'_, B>,
+ config: &DescriptAudioCodecConfig,
+ data_type: DataType,
+) -> AudioResult<StructuredAudioDecoderGraph<B>> {
+ let audio_decoder_tree = root.subtree("audio_decoder")?;
+ let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;
+ let decoder_tree = audio_decoder_tree.subtree("decoder")?;
+
+ let first_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("first_conv")?, data_type, 1, 1)?;
+ let final_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("final_conv")?, data_type, 1, 1)?;
+ let final_snake_alpha =
+ read_float_vector_exact::<B>(&decoder_tree.subtree("final_snake")?, "alpha", final_conv.cin, data_type)?;
+
+ let mut upsample_blocks = Vec::with_capacity(config.downsample_factor.len());
+ for (index, &stride) in config.downsample_factor.iter().rev().enumerate() {
+ let block_tree = quantizer_tree.subtree("upsampler")?.subtree("blocks")?.subtree(&index.to_string())?;
+ let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;
+ let convnext = read_convnext_layer::<B>(
+ context,
+ &block_tree.subtree("convnext")?,
+ data_type,
+ &config.quantizer_config.upsampler_config.block_configs[index].convnext_config.norm_config,
+ )?;
+ if convnext.depthwise_conv.cin != trans_conv.cout {
+ return Err(AudioError::Runtime(format!(
+ "structured audio upsampler convnext channel mismatch at block {index}: trans_conv out {} vs convnext in {}",
+ trans_conv.cout, convnext.depthwise_conv.cin
+ )));
+ }
+ upsample_blocks.push((trans_conv, convnext));
+ }
+
+ let mut decoder_blocks = Vec::with_capacity(config.decoder_rates.len());
+ for (index, &stride) in config.decoder_rates.iter().enumerate() {
+ let block_tree = decoder_tree.subtree("decoder_blocks")?.subtree(&index.to_string())?;
+ let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;
+ let snake_alpha =
+ read_float_vector_exact::<B>(&block_tree.subtree("snake")?, "alpha", trans_conv.cin, data_type)?;
+ let channels = trans_conv.cout;
+ let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1)?;
+ let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3)?;
+ let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9)?;
+ if snake_alpha.shape()[0] != trans_conv.cin
+ || res_unit1.conv1.cin != channels
+ || res_unit2.conv1.cin != channels
+ || res_unit3.conv1.cin != channels
+ {
+ return Err(AudioError::Runtime(format!(
+ "structured audio decoder block {index} channel mismatch in exported weights"
+ )));
+ }
+ decoder_blocks.push(StructuredAudioDecoderBlock {
+ snake_alpha,
+ trans_conv,
+ res_unit1,
+ res_unit2,
+ res_unit3,
+ });
+ }
+
+ Ok(StructuredAudioDecoderGraph {
+ first_conv,
+ upsample_blocks,
+ decoder_blocks,
+ final_snake_alpha,
+ final_conv,
+ })
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs
new file mode 100644
index 000000000..f07975c82
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs
@@ -0,0 +1,206 @@
+use super::*;
+
+impl StructuredAudioCodecGraph {
+ fn conv1d_input_context<B: Backend>(layer: &StructuredAudioConv1d<B>) -> AudioResult<usize> {
+ layer
+ .kernel_size
+ .checked_sub(1)
+ .and_then(|value| value.checked_mul(layer.dilation))
+ .ok_or(AudioError::Runtime("conv1d streaming context overflow".to_string()))
+ }
+
+ fn convtranspose_input_context<B: Backend>(
+ output_context: usize,
+ layer: &StructuredAudioConvTranspose1d<B>,
+ ) -> AudioResult<usize> {
+ let kernel_minus_one = layer.kernel_size.saturating_sub(1);
+ let numerator = output_context
+ .checked_add(kernel_minus_one)
+ .ok_or(AudioError::Runtime("transpose streaming context overflow".to_string()))?;
+ checked_div_ceil(numerator, layer.stride)
+ }
+
+ fn residual_unit_input_context<B: Backend>(
+ output_context: usize,
+ layer: &StructuredAudioResidualUnit<B>,
+ ) -> AudioResult<usize> {
+ let conv2 = Self::conv1d_input_context(&layer.conv2)?;
+ let conv1 = Self::conv1d_input_context(&layer.conv1)?;
+ output_context
+ .checked_add(conv2)
+ .and_then(|value| value.checked_add(conv1))
+ .ok_or(AudioError::Runtime("residual-unit streaming context overflow".to_string()))
+ }
+
+ fn convnext_input_context<B: Backend>(
+ output_context: usize,
+ layer: &StructuredAudioConvNeXt<B>,
+ ) -> AudioResult<usize> {
+ output_context
+ .checked_add(Self::conv1d_input_context(&layer.depthwise_conv)?)
+ .ok_or(AudioError::Runtime("convnext streaming context overflow".to_string()))
+ }
+
+ fn decoder_block_input_context<B: Backend>(
+ output_context: usize,
+ layer: &StructuredAudioDecoderBlock<B>,
+ ) -> AudioResult<usize> {
+ let after_res3 = Self::residual_unit_input_context(output_context, &layer.res_unit3)?;
+ let after_res2 = Self::residual_unit_input_context(after_res3, &layer.res_unit2)?;
+ let after_res1 = Self::residual_unit_input_context(after_res2, &layer.res_unit1)?;
+ Self::convtranspose_input_context(after_res1, &layer.trans_conv)
+ }
+
+ fn streaming_vocoder_context_frames<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ ) -> AudioResult<usize> {
+ let vocoder = self.vocoder_graph(resources)?;
+ let mut required = Self::conv1d_input_context(&vocoder.final_conv)?;
+
+ for block in vocoder.decoder_blocks.iter().rev() {
+ required = Self::decoder_block_input_context(required, block)?;
+ }
+
+ required = required
+ .checked_add(Self::conv1d_input_context(&vocoder.first_conv)?)
+ .ok_or(AudioError::Runtime("decoder-first streaming context overflow".to_string()))?;
+
+ for (trans_conv, convnext) in vocoder.upsample_blocks.iter().rev() {
+ required = Self::convnext_input_context(required, convnext)?;
+ required = Self::convtranspose_input_context(required, trans_conv)?;
+ }
+
+ Ok(required)
+ }
+
+ fn post_module_streaming_context_frames(&self) -> Option<usize> {
+ let mut context = 0usize;
+ for layer in &self.config.quantizer_config.post_module_config.layer_configs {
+ let window = layer.mixer_config.sliding_window_size()?;
+ context = context.max(window.saturating_sub(1));
+ }
+ Some(context)
+ }
+
+ pub(in crate::audio::nanocodec::runtime) fn streaming_decode_context_frames<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ ) -> AudioResult<Option<usize>> {
+ let vocoder_context = self.streaming_vocoder_context_frames(resources)?;
+ let Some(post_module_context) = self.post_module_streaming_context_frames() else {
+ return Ok(None);
+ };
+ Ok(Some(vocoder_context.max(post_module_context)))
+ }
+
+ pub(super) fn decode_quantizer_to_nsc_array_enqueued<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ context: &Rc<B::Context>,
+ encoder: &mut Encoder<B>,
+ tokens: &[u32],
+ lengths: &[usize],
+ batch_size: usize,
+ codebooks: usize,
+ frames: usize,
+ ) -> AudioResult<Array<B>> {
+ if codebooks != self.total_codebooks {
+ return Err(AudioError::Runtime(format!(
+ "structured audio codebook mismatch: expected {}, got {codebooks}",
+ self.total_codebooks
+ )));
+ }
+ let expected_tokens = checked_product(&[batch_size, codebooks, frames])?;
+ if tokens.len() != expected_tokens {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens,
+ actual_tokens: tokens.len(),
+ });
+ }
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+ for &length in lengths {
+ if length > frames {
+ return Err(AudioError::InvalidTokenLengthValue {
+ length,
+ frames,
+ });
+ }
+ }
+ let quantizer_resources = self.quantizer_resources(resources)?;
+ if quantizer_resources.residual_quantizers + 1 != codebooks {
+ return Err(AudioError::Runtime(format!(
+ "structured audio residual quantizer count mismatch: expected {}, got {}",
+ codebooks.saturating_sub(1),
+ quantizer_resources.residual_quantizers
+ )));
+ }
+
+ let lengths_i32 = convert_lengths_to_i32(lengths, frames)?;
+
+ let mut tokens_array = resources.token_staging(batch_size * codebooks * frames);
+ tokens_array.as_slice_mut::<u32>()[..tokens.len()].copy_from_slice(tokens);
+ let mut lengths_array = resources.length_staging(batch_size);
+ lengths_array.as_slice_mut::<i32>()[..lengths_i32.len()].copy_from_slice(&lengths_i32);
+
+ let output = context.create_array(
+ &[batch_size, frames, self.input_dim],
+ quantizer_resources.data_type,
+ "structured_audio_quantizer_output_nsc",
+ );
+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+ let codebooks_i32 = usize_to_i32(codebooks, "codebooks")?;
+ let frames_i32 = usize_to_i32(frames, "frames")?;
+ let input_dim_i32 = usize_to_i32(self.input_dim, "input_dim")?;
+ let codebook_dim_i32 = usize_to_i32(quantizer_resources.codebook_dim, "codebook_dim")?;
+ let residual_quantizers_i32 = usize_to_i32(quantizer_resources.residual_quantizers, "residual_quantizers")?;
+ let semantic_cardinality_i32 = usize_to_i32(quantizer_resources.semantic_cardinality, "semantic_cardinality")?;
+ let residual_cardinality_i32 = usize_to_i32(quantizer_resources.residual_cardinality, "residual_cardinality")?;
+
+ let tokens_buffer = tokens_array.buffer();
+ let tokens_buffer = tokens_buffer.borrow();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+ let semantic_codebook_buffer = quantizer_resources.semantic_codebook.buffer();
+ let semantic_codebook_buffer = semantic_codebook_buffer.borrow();
+ let semantic_out_proj_buffer = quantizer_resources.semantic_out_proj.buffer();
+ let semantic_out_proj_buffer = semantic_out_proj_buffer.borrow();
+ let semantic_out_bias_buffer = quantizer_resources.semantic_out_bias.buffer();
+ let semantic_out_bias_buffer = semantic_out_bias_buffer.borrow();
+ let residual_codebooks_buffer = quantizer_resources.residual_codebooks.buffer();
+ let residual_codebooks_buffer = residual_codebooks_buffer.borrow();
+ let residual_out_proj_buffer = quantizer_resources.residual_out_proj.buffer();
+ let residual_out_proj_buffer = residual_out_proj_buffer.borrow();
+ let residual_out_bias_buffer = quantizer_resources.residual_out_bias.buffer();
+ let residual_out_bias_buffer = residual_out_bias_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ quantizer_resources.kernel.encode(
+ &*tokens_buffer,
+ &*lengths_buffer,
+ &*semantic_codebook_buffer,
+ &*semantic_out_proj_buffer,
+ &*semantic_out_bias_buffer,
+ &*residual_codebooks_buffer,
+ &*residual_out_proj_buffer,
+ &*residual_out_bias_buffer,
+ &mut *output_buffer,
+ batch_i32,
+ codebooks_i32,
+ frames_i32,
+ input_dim_i32,
+ codebook_dim_i32,
+ residual_quantizers_i32,
+ semantic_cardinality_i32,
+ residual_cardinality_i32,
+ encoder,
+ );
+
+ Ok(output)
+ }
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs
new file mode 100644
index 000000000..494ab745b
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs
@@ -0,0 +1,386 @@
+use std::sync::mpsc::channel;
+
+use super::*;
+use crate::audio::nanocodec::runtime::profile::SubmittedDecodedPaddedAudio;
+
+impl StructuredAudioCodecGraph {
+ fn run_residual_unit_enqueued<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ unit: &StructuredAudioResidualUnit<B>,
+ lengths: &[i32],
+ lengths_array: &Array<B>,
+ batch_size: usize,
+ channels: usize,
+ seq_len: usize,
+ kernels: &StructuredAudioKernelCache<B>,
+ ) -> AudioResult<Array<B>> {
+ let ws = &resources.decode_workspace;
+ let ctx = resources.context();
+ let data_type = input.data_type();
+
+ let residual = input.clone();
+ let x = snake1d_enqueue(
+ encoder,
+ input,
+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake1"),
+ &unit.snake1_alpha,
+ batch_size,
+ channels,
+ seq_len,
+ kernels,
+ )?;
+ let x = causal_conv1d_grouped_enqueue(
+ encoder,
+ &x,
+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv1"),
+ &unit.conv1,
+ SequenceLayout::Ncs,
+ lengths,
+ lengths_array,
+ batch_size,
+ seq_len,
+ kernels,
+ )?;
+ let x = snake1d_enqueue(
+ encoder,
+ &x,
+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake2"),
+ &unit.snake2_alpha,
+ batch_size,
+ channels,
+ seq_len,
+ kernels,
+ )?;
+ causal_conv1d_grouped_residual_enqueue(
+ encoder,
+ &x,
+ &residual,
+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv2"),
+ &unit.conv2,
+ lengths,
+ lengths_array,
+ batch_size,
+ seq_len,
+ kernels,
+ )
+ }
+
+ pub(in crate::audio::nanocodec::runtime) fn submit_decode_padded<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ tokens: &[u32],
+ lengths: &[usize],
+ batch_size: usize,
+ codebooks: usize,
+ frames: usize,
+ ) -> AudioResult<SubmittedDecodedPaddedAudio<B>> {
+ if batch_size == 0 || frames == 0 {
+ let out_lengths = lengths
+ .iter()
+ .map(|&length| {
+ length
+ .checked_mul(self.upsample_factor)
+ .ok_or(AudioError::Runtime("structured audio length scaling overflow".to_string()))
+ })
+ .collect::<AudioResult<Vec<_>>>()?;
+ let context = resources.context().clone();
+ return Ok(SubmittedDecodedPaddedAudio {
+ output: context.create_array(&[0], DataType::F32, "structured_audio_empty_decode_output"),
+ channels: 1,
+ frames: out_lengths.iter().copied().max().unwrap_or(0),
+ lengths: out_lengths,
+ final_command_buffer: None,
+ completion_notification: None,
+ });
+ }
+
+ let mut lengths_i32 = lengths
+ .iter()
+ .map(|&length| {
+ i32::try_from(length)
+ .map_err(|_| AudioError::Runtime("structured audio length exceeds i32 range".to_string()))
+ })
+ .collect::<AudioResult<Vec<_>>>()?;
+ let context = resources.context().clone();
+ let mut encoder = Encoder::new(context.as_ref()).map_err(|err| {
+ AudioError::Runtime(format!("failed to create structured audio decode command buffer: {err}"))
+ })?;
+
+ let ws = &resources.decode_workspace;
+
+ let mut x;
+ let mut x_layout = SequenceLayout::Nsc;
+ let quantized_nsc = self.decode_quantizer_to_nsc_array_enqueued(
+ resources,
+ &context,
+ &mut encoder,
+ tokens,
+ lengths,
+ batch_size,
+ codebooks,
+ frames,
+ )?;
+ let _ = codebooks;
+ x = self.apply_post_module_enqueued(
+ resources,
+ &context,
+ &mut encoder,
+ &quantized_nsc,
+ lengths,
+ batch_size,
+ frames,
+ )?;
+
+ let vocoder_graph = self.vocoder_graph(resources)?;
+ let kernels = resources.kernels(self.vocoder_data_type)?;
+ let mut current_channels = self.input_dim;
+ let mut current_frames = frames;
+ let mut next_lengths_i32 = vec![0_i32; lengths_i32.len()];
+ let mut lengths_array = ws.lengths_array(&context, 0, lengths_i32.len());
+ lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);
+ let mut next_lengths_array = ws.lengths_array(&context, 1, lengths_i32.len());
+
+ let data_type = x.data_type();
+
+ for (block_index, (trans_conv, convnext)) in vocoder_graph.upsample_blocks.iter().enumerate() {
+ if trans_conv.cin != current_channels {
+ return Err(AudioError::Runtime(format!(
+ "structured audio upsampler input channel mismatch: expected {}, got {}",
+ trans_conv.cin, current_channels
+ )));
+ }
+ let next_frames = current_frames
+ .checked_mul(trans_conv.stride)
+ .ok_or(AudioError::Runtime("structured audio upsampler frame overflow".to_string()))?;
+ scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, trans_conv.stride)?;
+ next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);
+
+ x = causal_conv_transpose1d_causal_pad_enqueue(
+ &mut encoder,
+ &x,
+ ws.next_scratch(&context, &[batch_size, trans_conv.cout, next_frames], data_type, "up_tconv"),
+ trans_conv,
+ &next_lengths_i32,
+ batch_size,
+ current_frames,
+ next_frames,
+ x_layout,
+ &next_lengths_array,
+ &kernels,
+ )
+ .map_err(|err| {
+ AudioError::Runtime(format!(
+ "structured audio upsample block {block_index} transpose_conv failed: {err} (x_len={}, batch_size={}, cin={}, seq_len_in={}, seq_len_out={})",
+ x.num_elements(),
+ batch_size,
+ trans_conv.cin,
+ current_frames,
+ next_frames
+ ))
+ })?;
+ x = self
+ .apply_convnext_ncs_enqueued(
+ resources,
+ &mut encoder,
+ &x,
+ convnext,
+ &next_lengths_i32,
+ &next_lengths_array,
+ batch_size,
+ trans_conv.cout,
+ next_frames,
+ &kernels,
+ )
+ .map_err(|err| {
+ AudioError::Runtime(format!("structured audio upsample block {block_index} convnext failed: {err}"))
+ })?;
+
+ x_layout = SequenceLayout::Ncs;
+ current_frames = next_frames;
+ current_channels = trans_conv.cout;
+ std::mem::swap(&mut lengths_i32, &mut next_lengths_i32);
+ std::mem::swap(&mut lengths_array, &mut next_lengths_array);
+ }
+
+ if vocoder_graph.first_conv.cin != current_channels {
+ return Err(AudioError::Runtime(format!(
+ "structured audio decoder input channels mismatch: expected {}, got {}",
+ vocoder_graph.first_conv.cin, current_channels
+ )));
+ }
+ x = causal_conv1d_grouped_enqueue(
+ &mut encoder,
+ &x,
+ ws.next_scratch(
+ &context,
+ &[batch_size, vocoder_graph.first_conv.cout, current_frames],
+ data_type,
+ "dec_first_conv",
+ ),
+ &vocoder_graph.first_conv,
+ x_layout,
+ &lengths_i32,
+ &lengths_array,
+ batch_size,
+ current_frames,
+ &kernels,
+ )?;
+ current_channels = vocoder_graph.first_conv.cout;
+
+ for block in vocoder_graph.decoder_blocks.iter() {
+ if block.trans_conv.cin != current_channels {
+ return Err(AudioError::Runtime(format!(
+ "structured audio decoder block input mismatch: expected {}, got {}",
+ block.trans_conv.cin, current_channels
+ )));
+ }
+ x = snake1d_enqueue(
+ &mut encoder,
+ &x,
+ ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "dec_snake"),
+ &block.snake_alpha,
+ batch_size,
+ current_channels,
+ current_frames,
+ &kernels,
+ )?;
+
+ let next_frames = current_frames
+ .checked_mul(block.trans_conv.stride)
+ .ok_or(AudioError::Runtime("structured audio decoder frame overflow".to_string()))?;
+ scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, block.trans_conv.stride)?;
+ next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);
+
+ x = causal_conv_transpose1d_causal_pad_enqueue(
+ &mut encoder,
+ &x,
+ ws.next_scratch(&context, &[batch_size, block.trans_conv.cout, next_frames], data_type, "dec_tconv"),
+ &block.trans_conv,
+ &next_lengths_i32,
+ batch_size,
+ current_frames,
+ next_frames,
+ SequenceLayout::Ncs,
+ &next_lengths_array,
+ &kernels,
+ )?;
+
+ current_frames = next_frames;
+ current_channels = block.trans_conv.cout;
+ std::mem::swap(&mut lengths_i32, &mut next_lengths_i32);
+ std::mem::swap(&mut lengths_array, &mut next_lengths_array);
+
+ x = self.run_residual_unit_enqueued(
+ resources,
+ &mut encoder,
+ &x,
+ &block.res_unit1,
+ &lengths_i32,
+ &lengths_array,
+ batch_size,
+ current_channels,
+ current_frames,
+ &kernels,
+ )?;
+ x = self.run_residual_unit_enqueued(
+ resources,
+ &mut encoder,
+ &x,
+ &block.res_unit2,
+ &lengths_i32,
+ &lengths_array,
+ batch_size,
+ current_channels,
+ current_frames,
+ &kernels,
+ )?;
+ x = self.run_residual_unit_enqueued(
+ resources,
+ &mut encoder,
+ &x,
+ &block.res_unit3,
+ &lengths_i32,
+ &lengths_array,
+ batch_size,
+ current_channels,
+ current_frames,
+ &kernels,
+ )?;
+ }
+
+ x = snake1d_enqueue(
+ &mut encoder,
+ &x,
+ ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "final_snake"),
+ &vocoder_graph.final_snake_alpha,
+ batch_size,
+ current_channels,
+ current_frames,
+ &kernels,
+ )?;
+ x = causal_conv1d_grouped_enqueue(
+ &mut encoder,
+ &x,
+ ws.next_scratch(
+ &context,
+ &[batch_size, vocoder_graph.final_conv.cout, current_frames],
+ data_type,
+ "final_conv",
+ ),
+ &vocoder_graph.final_conv,
+ SequenceLayout::Ncs,
+ &lengths_i32,
+ &lengths_array,
+ batch_size,
+ current_frames,
+ &kernels,
+ )?;
+ // The final output is returned and held by the caller, so allocate a
+ // fresh array instead of using a scratch buffer.
+ x = tanh_enqueue(
+ &mut encoder,
+ &x,
+ context.create_array(x.shape(), data_type, "structured_audio_tanh_output"),
+ &kernels,
+ )?;
+
+ let (completion_sender, completion_notification) = channel();
+ encoder.add_completion_handler({
+ move |_| {
+ let _ = completion_sender.send(());
+ }
+ });
+ let final_command_buffer = Some(encoder.end_encoding().submit());
+ let out_lengths = lengths_i32
+ .into_iter()
+ .map(|length| {
+ usize::try_from(length).map_err(|_| {
+ AudioError::Runtime("structured audio decoder produced invalid negative length".to_string())
+ })
+ })
+ .collect::<AudioResult<Vec<_>>>()?;
+ Ok(SubmittedDecodedPaddedAudio {
+ output: x,
+ channels: vocoder_graph.final_conv.cout,
+ frames: current_frames,
+ lengths: out_lengths,
+ final_command_buffer,
+ completion_notification: Some(completion_notification),
+ })
+ }
+
+ pub(in crate::audio::nanocodec::runtime) fn decode_padded<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ tokens: &[u32],
+ lengths: &[usize],
+ batch_size: usize,
+ codebooks: usize,
+ frames: usize,
+ ) -> AudioResult<DecodedPaddedAudio> {
+ self.submit_decode_padded(resources, tokens, lengths, batch_size, codebooks, frames)?.resolve()
+ }
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/enqueue.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
new file mode 100644
index 000000000..5f0051723
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/enqueue.rs
@@ -0,0 +1,565 @@
+use super::*;
+use crate::backends::common::gpu_types::ActivationType;
+
+pub(super) fn snake1d_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ output: Array<B>,
+ alpha: &Array<B>,
+ batch_size: usize,
+ channels: usize,
+ seq_len: usize,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ let expected_input = checked_product(&[batch_size, channels, seq_len])?;
+ if input.num_elements() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.num_elements(),
+ });
+ }
+ if alpha.shape() != [channels] {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: channels,
+ actual_tokens: alpha.num_elements(),
+ });
+ }
+
+ let data_type = input.data_type();
+ if alpha.data_type() != data_type {
+ return Err(AudioError::Runtime(format!(
+ "snake alpha dtype mismatch: expected {:?}, got {:?}",
+ data_type,
+ alpha.data_type()
+ )));
+ }
+
+ let channels_i32 = usize_to_i32(channels, "channels")?;
+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let alpha_buffer = alpha.buffer();
+ let alpha_buffer = alpha_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ kernels.half_snake.encode(
+ &*input_buffer,
+ &*alpha_buffer,
+ &mut *output_buffer,
+ channels_i32,
+ seq_len_i32,
+ channels_i32,
+ 0.0,
+ 1e-9,
+ batch_i32,
+ encoder,
+ );
+
+ Ok(output)
+}
+
+fn validate_grouped_conv_params<B: Backend>(
+ input: &Array<B>,
+ layer: &StructuredAudioConv1d<B>,
+ lengths: &[i32],
+ batch_size: usize,
+ seq_len: usize,
+) -> AudioResult<()> {
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+ if layer.groups == 0 || layer.cin % layer.groups != 0 || layer.cout % layer.groups != 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];
+ if layer.weight.shape() != expected_weight_shape {
+ return Err(AudioError::Runtime(format!(
+ "causal conv1d weight shape mismatch: expected {:?}, got {:?}",
+ expected_weight_shape,
+ layer.weight.shape()
+ )));
+ }
+ if layer.bias.shape() != [layer.cout] {
+ return Err(AudioError::Runtime(format!(
+ "causal conv1d bias shape mismatch: expected [{}], got {:?}",
+ layer.cout,
+ layer.bias.shape()
+ )));
+ }
+ let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;
+ if input.num_elements() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.num_elements(),
+ });
+ }
+ let data_type = input.data_type();
+ if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {
+ return Err(AudioError::Runtime("causal conv1d dtype mismatch".to_string()));
+ }
+ Ok(())
+}
+
+pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ output: Array<B>,
+ layer: &StructuredAudioConv1d<B>,
+ input_layout: SequenceLayout,
+ lengths: &[i32],
+ lengths_array: &Array<B>,
+ batch_size: usize,
+ seq_len: usize,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;
+
+ let cin_i32 = usize_to_i32(layer.cin, "cin")?;
+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;
+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
+ let kernel_size_i32 = usize_to_i32(layer.kernel_size, "kernel_size")?;
+ let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;
+ let input_layout_i32 = input_layout.as_i32();
+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+ if layer.groups == 1 {
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let weight_buffer = layer.weight.buffer();
+ let weight_buffer = weight_buffer.borrow();
+ let bias_buffer = layer.bias.buffer();
+ let bias_buffer = bias_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+ kernels.causal_conv1d.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin_i32,
+ cout_i32,
+ seq_len_i32,
+ kernel_size_i32,
+ dilation_i32,
+ input_layout_i32,
+ batch_i32,
+ encoder,
+ );
+ } else {
+ let groups_i32 = usize_to_i32(layer.groups, "groups")?;
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let weight_buffer = layer.weight.buffer();
+ let weight_buffer = weight_buffer.borrow();
+ let bias_buffer = layer.bias.buffer();
+ let bias_buffer = bias_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+ kernels.causal_conv1d_grouped.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin_i32,
+ cout_i32,
+ seq_len_i32,
+ kernel_size_i32,
+ dilation_i32,
+ groups_i32,
+ input_layout_i32,
+ batch_i32,
+ encoder,
+ );
+ }
+
+ Ok(output)
+}
+
+pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ residual: &Array<B>,
+ output: Array<B>,
+ layer: &StructuredAudioConv1d<B>,
+ lengths: &[i32],
+ lengths_array: &Array<B>,
+ batch_size: usize,
+ seq_len: usize,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;
+ let expected_residual = checked_product(&[batch_size, layer.cout, seq_len])?;
+ if residual.num_elements() != expected_residual {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_residual,
+ actual_tokens: residual.num_elements(),
+ });
+ }
+ let cin_i32 = usize_to_i32(layer.cin, "cin")?;
+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;
+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
+ let kernel_size_i32 = usize_to_i32(layer.kernel_size, "kernel_size")?;
+ let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;
+ let groups_i32 = usize_to_i32(layer.groups, "groups")?;
+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+ let data_type = input.data_type();
+ if residual.data_type() != data_type {
+ return Err(AudioError::Runtime("causal conv1d residual dtype mismatch".to_string()));
+ }
+
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let residual_buffer = residual.buffer();
+ let residual_buffer = residual_buffer.borrow();
+ let weight_buffer = layer.weight.buffer();
+ let weight_buffer = weight_buffer.borrow();
+ let bias_buffer = layer.bias.buffer();
+ let bias_buffer = bias_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+ kernels.causal_conv1d_grouped_residual.encode(
+ &*input_buffer,
+ &*residual_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin_i32,
+ cout_i32,
+ seq_len_i32,
+ kernel_size_i32,
+ dilation_i32,
+ groups_i32,
+ batch_i32,
+ encoder,
+ );
+
+ Ok(output)
+}
+
+pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ output: Array<B>,
+ layer: &StructuredAudioConvTranspose1d<B>,
+ lengths: &[i32],
+ batch_size: usize,
+ seq_len_in: usize,
+ seq_len_out: usize,
+ input_layout: SequenceLayout,
+ lengths_array: &Array<B>,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+ let expected_input = checked_product(&[batch_size, layer.cin, seq_len_in])?;
+ if input.num_elements() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.num_elements(),
+ });
+ }
+ if layer.groups == 0 || layer.cin % layer.groups != 0 || layer.cout % layer.groups != 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];
+ if layer.weight.shape() != expected_weight_shape {
+ return Err(AudioError::Runtime(format!(
+ "causal transpose weight shape mismatch: expected {:?}, got {:?}",
+ expected_weight_shape,
+ layer.weight.shape()
+ )));
+ }
+ if layer.bias.shape() != [layer.cout] {
+ return Err(AudioError::Runtime(format!(
+ "causal transpose bias shape mismatch: expected [{}], got {:?}",
+ layer.cout,
+ layer.bias.shape()
+ )));
+ }
+
+ let data_type = input.data_type();
+ if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {
+ return Err(AudioError::Runtime("causal transpose dtype mismatch".to_string()));
+ }
+
+ let cin_i32 = usize_to_i32(layer.cin, "cin")?;
+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;
+ let seq_in_i32 = usize_to_i32(seq_len_in, "seq_len_in")?;
+ let seq_out_i32 = usize_to_i32(seq_len_out, "seq_len_out")?;
+ let kernel_size_i32 = usize_to_i32(layer.kernel_size, "kernel_size")?;
+ let stride_i32 = usize_to_i32(layer.stride, "stride")?;
+ let groups_i32 = usize_to_i32(layer.groups, "groups")?;
+ let input_layout_i32 = input_layout.as_i32();
+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let weight_buffer = layer.weight.buffer();
+ let weight_buffer = weight_buffer.borrow();
+ let bias_buffer = layer.bias.buffer();
+ let bias_buffer = bias_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+ kernels.causal_conv_transpose1d_causal_pad.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin_i32,
+ cout_i32,
+ seq_in_i32,
+ seq_out_i32,
+ kernel_size_i32,
+ stride_i32,
+ groups_i32,
+ input_layout_i32,
+ batch_i32,
+ encoder,
+ );
+
+ Ok(output)
+}
+
+pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ output: Array<B>,
+ layer: &StructuredAudioPointwiseConv<B>,
+ lengths: &[i32],
+ lengths_array: &Array<B>,
+ batch_size: usize,
+ seq_len: usize,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+ let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;
+ if input.num_elements() != expected_input {
+ return Err(AudioError::Runtime(format!(
+ "pointwise conv input shape mismatch: expected {expected_input} elements ([batch={batch_size}, cin={}, seq_len={seq_len}]), got {}",
+ layer.cin,
+ input.num_elements()
+ )));
+ }
+ if layer.weight.shape() != [layer.cout, layer.cin, 1] {
+ return Err(AudioError::Runtime(format!(
+ "pointwise conv weight shape mismatch: expected [{}, {}, 1], got {:?}",
+ layer.cout,
+ layer.cin,
+ layer.weight.shape()
+ )));
+ }
+ if layer.bias.shape() != [layer.cout] {
+ return Err(AudioError::Runtime(format!(
+ "pointwise conv bias shape mismatch: expected [{}], got {:?}",
+ layer.cout,
+ layer.bias.shape()
+ )));
+ }
+
+ let data_type = input.data_type();
+ if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {
+ return Err(AudioError::Runtime("pointwise conv dtype mismatch".to_string()));
+ }
+
+ let cin_i32 = usize_to_i32(layer.cin, "cin")?;
+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;
+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let weight_buffer = layer.weight.buffer();
+ let weight_buffer = weight_buffer.borrow();
+ let bias_buffer = layer.bias.buffer();
+ let bias_buffer = bias_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+ kernels.conv1d.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin_i32,
+ cout_i32,
+ seq_len_i32,
+ seq_len_i32,
+ 1,
+ 1,
+ 1,
+ 0,
+ 0,
+ batch_i32,
+ encoder,
+ );
+
+ Ok(output)
+}
+
+pub(super) fn norm_ncs_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ output: Array<B>,
+ norm: &StructuredAudioNorm<B>,
+ lengths: &[i32],
+ lengths_array: &Array<B>,
+ batch_size: usize,
+ channels: usize,
+ seq_len: usize,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+ if norm.scales.shape() != [channels] {
+ return Err(AudioError::Runtime(format!(
+ "norm scale shape mismatch: expected [{channels}], got {:?}",
+ norm.scales.shape()
+ )));
+ }
+ if norm.bias.shape() != [channels] {
+ return Err(AudioError::Runtime(format!(
+ "norm bias shape mismatch: expected [{channels}], got {:?}",
+ norm.bias.shape()
+ )));
+ }
+
+ let expected_input = checked_product(&[batch_size, channels, seq_len])?;
+ if input.num_elements() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.num_elements(),
+ });
+ }
+
+ let data_type = input.data_type();
+ if norm.scales.data_type() != data_type || norm.bias.data_type() != data_type {
+ return Err(AudioError::Runtime("norm dtype mismatch".to_string()));
+ }
+
+ let channels_i32 = usize_to_i32(channels, "channels")?;
+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;
+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;
+ let subtract_mean = if norm.subtract_mean {
+ 1
+ } else {
+ 0
+ };
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let scales_buffer = norm.scales.buffer();
+ let scales_buffer = scales_buffer.borrow();
+ let bias_buffer = norm.bias.buffer();
+ let bias_buffer = bias_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ let lengths_buffer = lengths_array.buffer();
+ let lengths_buffer = lengths_buffer.borrow();
+ kernels.norm_ncs.encode(
+ &*input_buffer,
+ &*scales_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ channels_i32,
+ seq_len_i32,
+ norm.epsilon,
+ subtract_mean,
+ batch_i32,
+ encoder,
+ );
+
+ Ok(output)
+}
+
+pub(super) fn gelu_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ output: Array<B>,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ let n_u32 = u32::try_from(input.num_elements())
+ .map_err(|_| AudioError::Runtime("gelu element count exceeds u32 range".to_string()))?;
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::GELU, encoder);
+ Ok(output)
+}
+
+pub(super) fn add_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ a: &Array<B>,
+ b: &Array<B>,
+ output: Array<B>,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ if a.num_elements() != b.num_elements() {
+ return Err(AudioError::Runtime(format!(
+ "elementwise add shape mismatch: {} vs {}",
+ a.num_elements(),
+ b.num_elements()
+ )));
+ }
+ if a.data_type() != b.data_type() {
+ return Err(AudioError::Runtime(format!(
+ "elementwise add dtype mismatch: {:?} vs {:?}",
+ a.data_type(),
+ b.data_type()
+ )));
+ }
+ let n_i32 = usize_to_i32(a.num_elements(), "n")?;
+ let a_buffer = a.buffer();
+ let a_buffer = a_buffer.borrow();
+ let b_buffer = b.buffer();
+ let b_buffer = b_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ kernels.add.encode(&*a_buffer, &*b_buffer, &mut *output_buffer, n_i32, encoder);
+ Ok(output)
+}
+
+pub(super) fn tanh_enqueue<B: Backend>(
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ output: Array<B>,
+ kernels: &StructuredAudioKernelCache<B>,
+) -> AudioResult<Array<B>> {
+ let n_u32 = u32::try_from(input.num_elements())
+ .map_err(|_| AudioError::Runtime("tanh element count exceeds u32 range".to_string()))?;
+ let input_buffer = input.buffer();
+ let input_buffer = input_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::TANH, encoder);
+ Ok(output)
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
new file mode 100644
index 000000000..a30f99533
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs
@@ -0,0 +1,257 @@
+use super::*;
+
+pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {
+ match data_type {
+ DataType::F16 => 1,
+ DataType::BF16 => 2,
+ _ => 0,
+ }
+}
+
+pub(super) struct StructuredAudioPostModuleRuntime<B: Backend> {
+ pub(super) context: Rc<B::Context>,
+ pub(super) model_shape: ModelShape,
+ pub(super) scratch_buffers: ScratchBuffers<B>,
+ pub(super) shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
+ pub(super) layers: Box<[LayerExecutables<B>]>,
+ pub(super) output_norm: RMSNorm<B>,
+ pub(super) max_sequence_length: usize,
+}
+
+pub(super) struct StructuredAudioKernelCache<B: Backend> {
+ pub(super) half_snake: <B::Kernels as Kernels>::AudioHalfSnakeKernel,
+ pub(super) causal_conv1d: <B::Kernels as Kernels>::AudioCausalConv1dKernel,
+ pub(super) causal_conv1d_grouped: <B::Kernels as Kernels>::AudioCausalConv1dGroupedKernel,
+ pub(super) causal_conv1d_grouped_residual: <B::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel,
+ pub(super) causal_conv_transpose1d_causal_pad: <B::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel,
+ pub(super) conv1d: <B::Kernels as Kernels>::AudioConv1dKernel,
+ pub(super) norm_ncs: <B::Kernels as Kernels>::AudioNormNcsKernel,
+ pub(super) activation: <B::Kernels as Kernels>::ActivationKernel,
+ pub(super) add: <B::Kernels as Kernels>::AudioAddKernel,
+}
+
+pub(super) struct FishAudioQuantizerResources<B: Backend> {
+ pub(super) data_type: DataType,
+ pub(super) codebook_dim: usize,
+ pub(super) residual_quantizers: usize,
+ pub(super) semantic_cardinality: usize,
+ pub(super) residual_cardinality: usize,
+ pub(super) kernel: <B::Kernels as Kernels>::AudioQuantizerDecodeKernel,
+ pub(super) semantic_codebook: Array<B>,
+ pub(super) semantic_out_proj: Array<B>,
+ pub(super) semantic_out_bias: Array<B>,
+ pub(super) residual_codebooks: Array<B>,
+ pub(super) residual_out_proj: Array<B>,
+ pub(super) residual_out_bias: Array<B>,
+}
+
+/// Ping-pong scratch pool for the sequential decode pipeline.
+///
+/// At any point in the pipeline at most two data buffers are live (the current
+/// input and the current output). `next_scratch` alternates between two slots,
+/// re-using a buffer when its `Rc` strong-count shows it is exclusively owned
+/// by the pool and it is large enough. When a residual connection keeps an
+/// extra reference alive, a fresh buffer is allocated transparently.
+pub(super) struct DecodeWorkspace<B: Backend> {
+ slots: [RefCell<Option<Rc<RefCell<B::Buffer>>>>; 2],
+ toggle: Cell<bool>,
+ /// Pair of reusable I32 arrays for per-stage sequence lengths. Cached to
+ /// avoid allocating new GPU buffers on every decode call.
+ lengths_pair: [RefCell<Option<Array<B>>>; 2],
+}
+
+impl<B: Backend> DecodeWorkspace<B> {
+ fn new() -> Self {
+ Self {
+ slots: [RefCell::new(None), RefCell::new(None)],
+ toggle: Cell::new(false),
+ lengths_pair: [RefCell::new(None), RefCell::new(None)],
+ }
+ }
+
+ /// Return a reusable I32 array of at least `min_elements` entries for
+ /// sequence length data. `index` must be 0 or 1 (ping-pong pair).
+ pub(super) fn lengths_array(
+ &self,
+ context: &B::Context,
+ index: usize,
+ min_elements: usize,
+ ) -> Array<B> {
+ debug_assert!(index < 2);
+ let mut slot = self.lengths_pair[index].borrow_mut();
+ if let Some(existing) = slot.as_ref() {
+ if existing.num_elements() >= min_elements {
+ return existing.clone();
+ }
+ }
+ let label = if index == 0 {
+ "structured_audio_lengths_a"
+ } else {
+ "structured_audio_lengths_b"
+ };
+ let array = context.create_array(&[min_elements], DataType::I32, label);
+ *slot = Some(array.clone());
+ array
+ }
+
+ /// Return a scratch [`Array`] backed by the next slot in the ping-pong
+ /// pair. The underlying GPU buffer is reused when possible.
+ pub(super) fn next_scratch(
+ &self,
+ context: &B::Context,
+ shape: &[usize],
+ data_type: DataType,
+ label: &str,
+ ) -> Array<B> {
+ let idx = usize::from(self.toggle.get());
+ self.toggle.set(!self.toggle.get());
+
+ let needed = size_for_shape(shape, data_type);
+ let mut slot = self.slots[idx].borrow_mut();
+
+ // Try to reuse the existing buffer if nobody else holds a reference
+ // and it is large enough.
+ if let Some(buf_rc) = slot.as_ref() {
+ if Rc::strong_count(buf_rc) == 1 && buf_rc.borrow().length() >= needed {
+ let array = unsafe { Array::from_parts(buf_rc.clone(), 0, shape, data_type) };
+ return array;
+ }
+ }
+
+ // Allocate a fresh buffer and cache it.
+ let mut buffer = context.create_buffer(needed).expect("Failed to create scratch buffer");
+ buffer.set_label(Some(label));
+ let buf_rc = Rc::new(RefCell::new(buffer));
+ *slot = Some(buf_rc.clone());
+ unsafe { Array::from_parts(buf_rc, 0, shape, data_type) }
+ }
+
+ /// Clear both scratch slots, forcing the next call to allocate fresh
+ /// buffers. This must be called after submitting a command buffer that
+ /// references scratch intermediates so that the in-flight GPU work is
+ /// not corrupted by a subsequent decode pass reusing the same buffers.
+ pub(super) fn reset(&self) {
+ *self.slots[0].borrow_mut() = None;
+ *self.slots[1].borrow_mut() = None;
+ self.toggle.set(false);
+ *self.lengths_pair[0].borrow_mut() = None;
+ *self.lengths_pair[1].borrow_mut() = None;
+ }
+}
+
+pub(in crate::audio::nanocodec::runtime) struct StructuredAudioRuntimeResources<B: Backend> {
+ context: Rc<B::Context>,
+ pub(super) kernels_by_dtype: RefCell<HashMap<u8, Rc<StructuredAudioKernelCache<B>>>>,
+ pub(super) post_module_runtime: RefCell<Option<Rc<StructuredAudioPostModuleRuntime<B>>>>,
+ pub(super) vocoder_graph: RefCell<Option<Rc<StructuredAudioDecoderGraph<B>>>>,
+ pub(super) quantizer_resources: RefCell<Option<Rc<FishAudioQuantizerResources<B>>>>,
+ token_staging: RefCell<Option<Array<B>>>,
+ length_staging: RefCell<Option<Array<B>>>,
+ pub(super) decode_workspace: DecodeWorkspace<B>,
+}
+
+impl<B: Backend> StructuredAudioRuntimeResources<B> {
+ pub(in crate::audio::nanocodec::runtime) fn new(context: Rc<B::Context>) -> Self {
+ Self {
+ context,
+ kernels_by_dtype: RefCell::new(HashMap::new()),
+ post_module_runtime: RefCell::new(None),
+ vocoder_graph: RefCell::new(None),
+ quantizer_resources: RefCell::new(None),
+ token_staging: RefCell::new(None),
+ length_staging: RefCell::new(None),
+ decode_workspace: DecodeWorkspace::new(),
+ }
+ }
+
+ pub(super) fn context(&self) -> &Rc<B::Context> {
+ &self.context
+ }
+
+ /// Reset the decode workspace and staging arrays so that subsequent
+ /// decode passes allocate fresh buffers. Call this after submitting a
+ /// command buffer whose encoded work references the current buffers.
+ pub(in crate::audio::nanocodec::runtime) fn reset_for_pending(&self) {
+ self.decode_workspace.reset();
+ *self.token_staging.borrow_mut() = None;
+ *self.length_staging.borrow_mut() = None;
+ }
+
+ pub(super) fn kernels(
+ &self,
+ data_type: DataType,
+ ) -> AudioResult<Rc<StructuredAudioKernelCache<B>>> {
+ let key = structured_audio_dtype_key(data_type);
+ if let Some(existing) = self.kernels_by_dtype.borrow().get(&key) {
+ return Ok(existing.clone());
+ }
+ let created = Rc::new(build_structured_audio_kernels(&self.context, data_type)?);
+ self.kernels_by_dtype.borrow_mut().insert(key, created.clone());
+ Ok(created)
+ }
+
+ pub(super) fn token_staging(
+ &self,
+ min_elements: usize,
+ ) -> Array<B> {
+ let mut slot = self.token_staging.borrow_mut();
+ if let Some(existing) = slot.as_ref() {
+ if existing.num_elements() >= min_elements {
+ return existing.clone();
+ }
+ }
+ let array = self.context.create_array(&[min_elements], DataType::U32, "quantizer_token_staging");
+ *slot = Some(array.clone());
+ array
+ }
+
+ pub(super) fn length_staging(
+ &self,
+ min_elements: usize,
+ ) -> Array<B> {
+ let mut slot = self.length_staging.borrow_mut();
+ if let Some(existing) = slot.as_ref() {
+ if existing.num_elements() >= min_elements {
+ return existing.clone();
+ }
+ }
+ let array = self.context.create_array(&[min_elements], DataType::I32, "quantizer_length_staging");
+ *slot = Some(array.clone());
+ array
+ }
+}
+
+pub(super) fn build_structured_audio_kernels<B: Backend>(
+ context: &Rc<B::Context>,
+ data_type: DataType,
+) -> AudioResult<StructuredAudioKernelCache<B>> {
+ Ok(StructuredAudioKernelCache {
+ half_snake: <B::Kernels as Kernels>::AudioHalfSnakeKernel::new(context.as_ref(), data_type)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize snake1d kernel: {err}")))?,
+ causal_conv1d: <B::Kernels as Kernels>::AudioCausalConv1dKernel::new(context.as_ref(), data_type)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize causal conv1d kernel: {err}")))?,
+ causal_conv1d_grouped: <B::Kernels as Kernels>::AudioCausalConv1dGroupedKernel::new(
+ context.as_ref(),
+ data_type,
+ )
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize grouped causal conv1d kernel: {err}")))?,
+ causal_conv1d_grouped_residual: <B::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel::new(
+ context.as_ref(),
+ data_type,
+ )
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize grouped residual conv1d kernel: {err}")))?,
+ causal_conv_transpose1d_causal_pad: <B::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(
+ context.as_ref(),
+ data_type,
+ )
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize causal transpose-conv kernel: {err}")))?,
+ conv1d: <B::Kernels as Kernels>::AudioConv1dKernel::new(context.as_ref(), data_type)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize conv1d kernel: {err}")))?,
+ norm_ncs: <B::Kernels as Kernels>::AudioNormNcsKernel::new(context.as_ref(), data_type)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize norm kernel: {err}")))?,
+ activation: <B::Kernels as Kernels>::ActivationKernel::new(context.as_ref(), data_type, false)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize activation kernel: {err}")))?,
+ add: <B::Kernels as Kernels>::AudioAddKernel::new(context.as_ref(), data_type)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize add kernel: {err}")))?,
+ })
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/mod.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/mod.rs
new file mode 100644
index 000000000..1d204b8fd
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/mod.rs
@@ -0,0 +1,20 @@
+use super::{
+ support::{checked_div_ceil, scale_lengths_i32_in_place},
+ *,
+};
+
+mod build;
+mod context;
+mod decode;
+mod enqueue;
+mod fishaudio;
+mod post_module;
+mod shared;
+
+use build::*;
+use enqueue::*;
+// Re-export to parent (runtime.rs)
+pub(in crate::audio::nanocodec::runtime) use fishaudio::StructuredAudioRuntimeResources;
+use fishaudio::*;
+pub(super) use shared::StructuredAudioCodecGraph;
+use shared::*;
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
new file mode 100644
index 000000000..054db8d82
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs
@@ -0,0 +1,532 @@
+use super::*;
+
+impl StructuredAudioCodecGraph {
+ pub(super) fn apply_convnext_ncs_enqueued<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ encoder: &mut Encoder<B>,
+ input: &Array<B>,
+ layer: &StructuredAudioConvNeXt<B>,
+ lengths: &[i32],
+ lengths_array: &Array<B>,
+ batch_size: usize,
+ channels: usize,
+ seq_len: usize,
+ kernels: &StructuredAudioKernelCache<B>,
+ ) -> AudioResult<Array<B>> {
+ let ws = &resources.decode_workspace;
+ let ctx = resources.context();
+ let data_type = input.data_type();
+
+ let residual = input.clone();
+ let x = causal_conv1d_grouped_enqueue(
+ encoder,
+ input,
+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_dw"),
+ &layer.depthwise_conv,
+ SequenceLayout::Ncs,
+ lengths,
+ lengths_array,
+ batch_size,
+ seq_len,
+ kernels,
+ )?;
+ let x = norm_ncs_enqueue(
+ encoder,
+ &x,
+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_norm"),
+ &layer.norm,
+ lengths,
+ lengths_array,
+ batch_size,
+ channels,
+ seq_len,
+ kernels,
+ )?;
+ let x = conv1d_pointwise_ncs_enqueue(
+ encoder,
+ &x,
+ ws.next_scratch(ctx, &[batch_size, layer.pwconv1.cout, seq_len], data_type, "cnxt_pw1"),
+ &layer.pwconv1,
+ lengths,
+ lengths_array,
+ batch_size,
+ seq_len,
+ kernels,
+ )?;
+ let x = gelu_enqueue(encoder, &x, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_gelu"), kernels)?;
+ let x = conv1d_pointwise_ncs_enqueue(
+ encoder,
+ &x,
+ ws.next_scratch(ctx, &[batch_size, layer.pwconv2.cout, seq_len], data_type, "cnxt_pw2"),
+ &layer.pwconv2,
+ lengths,
+ lengths_array,
+ batch_size,
+ seq_len,
+ kernels,
+ )?;
+ add_enqueue(encoder, &x, &residual, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_add"), kernels)
+ }
+
+ pub(super) fn build_post_module_runtime<B: Backend>(
+ &self,
+ context: Rc<B::Context>,
+ required_sequence_length: usize,
+ ) -> AudioResult<StructuredAudioPostModuleRuntime<B>> {
+ let inner_model_config = InnerModelConfig {
+ embedding_config: EmbeddingConfig::Untied {
+ common: EmbeddingConfigCommon {
+ input_scale: None,
+ logit_soft_cap: None,
+ },
+ precision: self.vocoder_data_type.into(),
+ },
+ transformer_config: self.config.quantizer_config.post_module_config.clone(),
+ vocab_size: 1,
+ };
+ let decoder_config = Rc::new(inner_model_config.to_decoder_config().map_err(|_| {
+ AudioError::Runtime("failed to build structured audio post_module decoder config".to_string())
+ })?);
+ let model_shape = ModelShape::from_decoder_config(&decoder_config);
+
+ let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {
+ AudioError::Runtime(format!("failed to open post_module weights '{}': {err}", self.weights_path))
+ })?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|err| {
+ AudioError::Runtime(format!("failed to load post_module weights '{}': {err}", self.weights_path))
+ })?;
+ let root_loader_view = loader.tree();
+ let transformer_subtree_name = "audio_decoder.quantizer.post_module";
+ root_loader_view
+ .subtree(transformer_subtree_name)
+ .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
+
+ let max_sequence_length = decoder_config.context_length.max(required_sequence_length.max(1));
+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));
+ {
+ let transformer_tree = root_loader_view
+ .subtree(transformer_subtree_name)
+ .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;
+ let mut shared_buffers = shared_buffers.borrow_mut();
+ if let Some(global_rope) = &mut shared_buffers.global_rope {
+ global_rope.update_data(&transformer_tree, "global_rope");
+ }
+ if let Some(local_rope) = &mut shared_buffers.local_rope {
+ local_rope.update_data(&transformer_tree, "local_rope");
+ }
+ }
+ let scratch_buffers = ScratchBuffers::new(
+ context.as_ref(),
+ &decoder_config,
+ &model_shape,
+ max_sequence_length,
+ max_sequence_length,
+ );
+ let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(
+ context.clone(),
+ decoder_config,
+ &root_loader_view,
+ transformer_subtree_name,
+ );
+
+ Ok(StructuredAudioPostModuleRuntime {
+ context,
+ model_shape,
+ scratch_buffers,
+ shared_buffers,
+ layers,
+ output_norm,
+ max_sequence_length,
+ })
+ }
+
+ pub(super) fn post_module_runtime<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ required_sequence_length: usize,
+ ) -> AudioResult<Rc<StructuredAudioPostModuleRuntime<B>>> {
+ {
+ let cached = resources.post_module_runtime.borrow();
+ if let Some(runtime) = cached.as_ref() {
+ if runtime.max_sequence_length >= required_sequence_length.max(1) {
+ return Ok(runtime.clone());
+ }
+ }
+ }
+ let runtime = Rc::new(self.build_post_module_runtime(resources.context().clone(), required_sequence_length)?);
+ *resources.post_module_runtime.borrow_mut() = Some(runtime.clone());
+ Ok(runtime)
+ }
+
+ pub(super) fn build_quantizer_resources<B: Backend>(
+ &self,
+ context: &Rc<B::Context>,
+ ) -> AudioResult<FishAudioQuantizerResources<B>> {
+ if self.semantic_codebook_size == 0 || self.codebook_size == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ let data_type = self.vocoder_data_type;
+ let kernel = <B::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(context.as_ref(), data_type)
+ .map_err(|err| AudioError::Runtime(format!("failed to initialize quantizer decode kernel: {err}")))?;
+ let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {
+ AudioError::Runtime(format!("failed to open structured audio weights '{}': {err}", self.weights_path))
+ })?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|err| {
+ AudioError::Runtime(format!("failed to load structured audio weights '{}': {err}", self.weights_path))
+ })?;
+ let root = loader.tree();
+ let audio_decoder_tree = root.subtree("audio_decoder")?;
+ let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;
+ let semantic_tree = quantizer_tree.subtree("semantic_quantizer")?.subtree("quantizers")?.subtree("0")?;
+ let semantic_codebook = read_float_matrix_exact::<B>(
+ &semantic_tree.subtree("codebook")?,
+ "weights",
+ self.semantic_codebook_size,
+ self.config.codebook_dim,
+ data_type,
+ )?;
+ let codebook_dim = semantic_codebook.shape()[1];
+ let semantic_out_proj = read_float_matrix_exact::<B>(
+ &semantic_tree.subtree("out_proj")?,
+ "weights",
+ self.input_dim,
+ codebook_dim,
+ data_type,
+ )?;
+ let semantic_out_bias =
+ read_float_vector_exact::<B>(&semantic_tree.subtree("out_proj")?, "biases", self.input_dim, data_type)?;
+
+ let residual_quantizers = self.config.n_codebooks;
+ let residual_count_for_shape = residual_quantizers.max(1);
+ let residual_codebook_rows_for_shape = self.codebook_size.max(1);
+ let residual_codebooks = context.create_array(
+ &[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim],
+ data_type,
+ "structured_audio_quantizer_residual_codebooks",
+ );
+ let residual_out_proj = context.create_array(
+ &[residual_count_for_shape, self.input_dim, codebook_dim],
+ data_type,
+ "structured_audio_quantizer_residual_out_proj",
+ );
+ let residual_out_bias = context.create_array(
+ &[residual_count_for_shape, self.input_dim],
+ data_type,
+ "structured_audio_quantizer_residual_out_bias",
+ );
+
+ let residual_root = quantizer_tree.subtree("quantizer")?.subtree("quantizers")?;
+ for index in 0..residual_quantizers {
+ let quantizer_tree = residual_root.subtree(&index.to_string())?;
+ let codebook = read_float_matrix_exact::<B>(
+ &quantizer_tree.subtree("codebook")?,
+ "weights",
+ self.codebook_size,
+ codebook_dim,
+ data_type,
+ )?;
+ let out_proj = read_float_matrix_exact::<B>(
+ &quantizer_tree.subtree("out_proj")?,
+ "weights",
+ self.input_dim,
+ codebook_dim,
+ data_type,
+ )?;
+ let out_bias = read_float_vector_exact::<B>(
+ &quantizer_tree.subtree("out_proj")?,
+ "biases",
+ self.input_dim,
+ data_type,
+ )?;
+
+ let mut dst_codebook = outer_axis_view(&residual_codebooks, index, &[self.codebook_size, codebook_dim])?;
+ dst_codebook.copy_from_array(&codebook);
+ let mut dst_out_proj = outer_axis_view(&residual_out_proj, index, &[self.input_dim, codebook_dim])?;
+ dst_out_proj.copy_from_array(&out_proj);
+ let mut dst_out_bias = outer_axis_view(&residual_out_bias, index, &[self.input_dim])?;
+ dst_out_bias.copy_from_array(&out_bias);
+ }
+
+ Ok(FishAudioQuantizerResources {
+ data_type,
+ codebook_dim,
+ residual_quantizers,
+ semantic_cardinality: self.semantic_codebook_size,
+ residual_cardinality: self.codebook_size,
+ kernel,
+ semantic_codebook,
+ semantic_out_proj,
+ semantic_out_bias,
+ residual_codebooks,
+ residual_out_proj,
+ residual_out_bias,
+ })
+ }
+
+ pub(super) fn quantizer_resources<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ ) -> AudioResult<Rc<FishAudioQuantizerResources<B>>> {
+ if let Some(existing) = resources.quantizer_resources.borrow().as_ref() {
+ return Ok(existing.clone());
+ }
+ let created = Rc::new(self.build_quantizer_resources(resources.context())?);
+ *resources.quantizer_resources.borrow_mut() = Some(created.clone());
+ Ok(created)
+ }
+
+ pub(super) fn build_vocoder_graph<B: Backend>(
+ &self,
+ context: &Rc<B::Context>,
+ ) -> AudioResult<StructuredAudioDecoderGraph<B>> {
+ let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {
+ AudioError::Runtime(format!("failed to open structured audio weights '{}': {err}", self.weights_path))
+ })?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|err| {
+ AudioError::Runtime(format!("failed to load structured audio weights '{}': {err}", self.weights_path))
+ })?;
+ let root = loader.tree();
+ build_vocoder_graph_from_tree::<B>(context, &root, &self.config, self.vocoder_data_type)
+ }
+
+ pub(super) fn vocoder_graph<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ ) -> AudioResult<Rc<StructuredAudioDecoderGraph<B>>> {
+ if let Some(existing) = resources.vocoder_graph.borrow().as_ref() {
+ return Ok(existing.clone());
+ }
+ let created = Rc::new(self.build_vocoder_graph(resources.context())?);
+ *resources.vocoder_graph.borrow_mut() = Some(created.clone());
+ Ok(created)
+ }
+
+ pub(super) fn encode_post_module_layers<B: Backend>(
+ runtime: &StructuredAudioPostModuleRuntime<B>,
+ state: &mut ForwardPassState<B>,
+ encoder: &mut Encoder<B>,
+ ) -> AudioResult<()> {
+ let encoding_parameters = EncodingParameters::new();
+ for layer in runtime.layers.iter() {
+ layer
+ .encode(state, &encoding_parameters, encoder)
+ .map_err(|err| AudioError::Runtime(format!("post_module layer encode failed: {err}")))?;
+ }
+ runtime
+ .output_norm
+ .encode(state, encoder)
+ .map_err(|err| AudioError::Runtime(format!("post_module output norm encode failed: {err}")))?;
+ Ok(())
+ }
+
+ pub(super) fn apply_post_module_single_batch_enqueued<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ encoder: &mut Encoder<B>,
+ latent_nsc: &Array<B>,
+ frames: usize,
+ ) -> AudioResult<Array<B>> {
+ if frames == 0 {
+ return Ok(latent_nsc.clone());
+ }
+
+ let expected_elements = checked_product(&[frames, self.input_dim])?;
+ if latent_nsc.num_elements() != expected_elements {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_elements,
+ actual_tokens: latent_nsc.num_elements(),
+ });
+ }
+
+ let runtime = self.post_module_runtime(resources, frames.max(1))?;
+ let token_ids = vec![0_u64; frames];
+ let token_positions = (0..frames).collect::<Vec<_>>();
+ let mut state = ForwardPassState::new_classifier(
+ runtime.context.clone(),
+ &runtime.model_shape,
+ &runtime.scratch_buffers,
+ runtime.shared_buffers.clone(),
+ &token_ids,
+ &token_positions,
+ false,
+ 1,
+ );
+
+ let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let main_output = {
+ if main.shape() != [frames, self.input_dim] {
+ return Err(AudioError::Runtime(format!(
+ "post_module main shape mismatch: expected [{frames}, {}], got {:?}",
+ self.input_dim,
+ main.shape()
+ )));
+ }
+ if main.data_type() != latent_nsc.data_type() {
+ return Err(AudioError::Runtime(format!(
+ "post_module dtype mismatch: main={:?}, latent={:?}",
+ main.data_type(),
+ latent_nsc.data_type()
+ )));
+ }
+ main.clone()
+ };
+
+ let copy_bytes = latent_nsc
+ .num_elements()
+ .checked_mul(latent_nsc.data_type().size_in_bytes())
+ .ok_or(AudioError::Runtime("post_module copy size overflow".to_string()))?;
+ {
+ let latent_buffer = latent_nsc.buffer();
+ let main_output_buffer = main_output.buffer();
+ let latent_buffer = latent_buffer.borrow();
+ let mut main_output_buffer = main_output_buffer.borrow_mut();
+ encoder.encode_copy(
+ &latent_buffer,
+ latent_nsc.offset()..latent_nsc.offset() + copy_bytes,
+ &mut main_output_buffer,
+ main_output.offset()..main_output.offset() + copy_bytes,
+ );
+ }
+ Self::encode_post_module_layers(&runtime, &mut state, encoder)?;
+
+ Ok(main_output)
+ }
+
+ pub(super) fn apply_post_module_enqueued<B: Backend>(
+ &self,
+ resources: &StructuredAudioRuntimeResources<B>,
+ context: &Rc<B::Context>,
+ encoder: &mut Encoder<B>,
+ latent_nsc: &Array<B>,
+ lengths: &[usize],
+ batch_size: usize,
+ frames: usize,
+ ) -> AudioResult<Array<B>> {
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+ let expected_elements = checked_product(&[batch_size, frames, self.input_dim])?;
+ if latent_nsc.num_elements() != expected_elements {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_elements,
+ actual_tokens: latent_nsc.num_elements(),
+ });
+ }
+ if batch_size == 1 && lengths.first().copied() == Some(frames) {
+ return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);
+ }
+
+ let output = context.create_array(
+ &[batch_size, frames, self.input_dim],
+ latent_nsc.data_type(),
+ "structured_audio_post_module_output_nsc",
+ );
+ let mut batch_indices_by_length = BTreeMap::<usize, Vec<usize>>::new();
+ for (batch_index, &active_len) in lengths.iter().enumerate() {
+ if active_len == 0 {
+ continue;
+ }
+ if active_len > frames {
+ return Err(AudioError::InvalidTokenLengthValue {
+ length: active_len,
+ frames,
+ });
+ }
+ batch_indices_by_length.entry(active_len).or_default().push(batch_index);
+ }
+
+ if batch_indices_by_length.is_empty() {
+ let mut output = output;
+ output.copy_from_array(latent_nsc);
+ return Ok(output);
+ }
+
+ let full_copy_bytes = latent_nsc.size();
+ let mut copied_output_prefix = false;
+ for (active_len, batch_indices) in batch_indices_by_length {
+ let runtime = self.post_module_runtime(resources, active_len.max(1))?;
+ let token_ids = vec![0_u64; active_len];
+ let token_positions = (0..active_len).collect::<Vec<_>>();
+ let mut state = ForwardPassState::new_classifier(
+ runtime.context.clone(),
+ &runtime.model_shape,
+ &runtime.scratch_buffers,
+ runtime.shared_buffers.clone(),
+ &token_ids,
+ &token_positions,
+ false,
+ 1,
+ );
+
+ let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let main_output = {
+ if main.shape() != [active_len, self.input_dim] {
+ return Err(AudioError::Runtime(format!(
+ "post_module main shape mismatch: expected [{active_len}, {}], got {:?}",
+ self.input_dim,
+ main.shape()
+ )));
+ }
+ if main.data_type() != latent_nsc.data_type() {
+ return Err(AudioError::Runtime(format!(
+ "post_module dtype mismatch: main={:?}, latent={:?}",
+ main.data_type(),
+ latent_nsc.data_type()
+ )));
+ }
+ main.clone()
+ };
+
+ for &batch_index in &batch_indices {
+ if !copied_output_prefix {
+ {
+ let latent_buffer = latent_nsc.buffer();
+ let output_buffer = output.buffer();
+ let latent_buffer = latent_buffer.borrow();
+ let mut output_buffer = output_buffer.borrow_mut();
+ encoder.encode_copy(
+ &latent_buffer,
+ latent_nsc.offset()..latent_nsc.offset() + full_copy_bytes,
+ &mut output_buffer,
+ output.offset()..output.offset() + full_copy_bytes,
+ );
+ }
+ copied_output_prefix = true;
+ }
+ let source = array_batch_view(latent_nsc, batch_index, frames, self.input_dim, active_len)?;
+ {
+ let source_buffer = source.buffer();
+ let main_output_buffer = main_output.buffer();
+ let source_buffer = source_buffer.borrow();
+ let mut main_output_buffer = main_output_buffer.borrow_mut();
+ encoder.encode_copy(
+ &source_buffer,
+ source.offset()..source.offset() + source.size(),
+ &mut main_output_buffer,
+ main_output.offset()..main_output.offset() + source.size(),
+ );
+ }
+ Self::encode_post_module_layers(&runtime, &mut state, encoder)?;
+ let destination = array_batch_view(&output, batch_index, frames, self.input_dim, active_len)?;
+ {
+ let main_output_buffer = main_output.buffer();
+ let destination_buffer = destination.buffer();
+ let main_output_buffer = main_output_buffer.borrow();
+ let mut destination_buffer = destination_buffer.borrow_mut();
+ encoder.encode_copy(
+ &main_output_buffer,
+ main_output.offset()..main_output.offset() + destination.size(),
+ &mut destination_buffer,
+ destination.offset()..destination.offset() + destination.size(),
+ );
+ }
+ }
+ }
+
+ Ok(output)
+ }
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/shared.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/shared.rs
new file mode 100644
index 000000000..249ee8573
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/shared.rs
@@ -0,0 +1,118 @@
+use super::*;
+
+pub(super) fn array_batch_view<B: Backend>(
+ array: &Array<B>,
+ batch_index: usize,
+ frames: usize,
+ channels: usize,
+ active_frames: usize,
+) -> AudioResult<Array<B>> {
+ let batch_stride_bytes = size_for_shape(&[frames, channels], array.data_type());
+ let batch_offset = batch_index
+ .checked_mul(batch_stride_bytes)
+ .and_then(|value| value.checked_add(array.offset()))
+ .ok_or(AudioError::Runtime("array batch view offset overflow".to_string()))?;
+ if active_frames > frames {
+ return Err(AudioError::Runtime("array batch view active_frames exceeds frames".to_string()));
+ }
+ Ok(unsafe { Array::from_parts(array.buffer(), batch_offset, &[active_frames, channels], array.data_type()) })
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum SequenceLayout {
+ Ncs,
+ Nsc,
+}
+
+impl SequenceLayout {
+ pub(super) fn as_i32(self) -> i32 {
+ match self {
+ Self::Ncs => 0,
+ Self::Nsc => 1,
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct StructuredAudioCodecGraph {
+ pub(in crate::audio::nanocodec::runtime) config: DescriptAudioCodecConfig,
+ pub(in crate::audio::nanocodec::runtime) weights_path: String,
+ pub(in crate::audio::nanocodec::runtime) codebook_size: usize,
+ pub(in crate::audio::nanocodec::runtime) semantic_codebook_size: usize,
+ pub(in crate::audio::nanocodec::runtime) input_dim: usize,
+ pub(in crate::audio::nanocodec::runtime) total_codebooks: usize,
+ pub(in crate::audio::nanocodec::runtime) upsample_factor: usize,
+ pub(in crate::audio::nanocodec::runtime) vocoder_data_type: DataType,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioConv1d<B: Backend> {
+ pub(super) weight: Array<B>,
+ pub(super) bias: Array<B>,
+ pub(super) cin: usize,
+ pub(super) cout: usize,
+ pub(super) kernel_size: usize,
+ pub(super) dilation: usize,
+ pub(super) groups: usize,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioConvTranspose1d<B: Backend> {
+ pub(super) weight: Array<B>,
+ pub(super) bias: Array<B>,
+ pub(super) cin: usize,
+ pub(super) cout: usize,
+ pub(super) kernel_size: usize,
+ pub(super) stride: usize,
+ pub(super) groups: usize,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioPointwiseConv<B: Backend> {
+ pub(super) weight: Array<B>,
+ pub(super) bias: Array<B>,
+ pub(super) cin: usize,
+ pub(super) cout: usize,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioNorm<B: Backend> {
+ pub(super) scales: Array<B>,
+ pub(super) bias: Array<B>,
+ pub(super) epsilon: f32,
+ pub(super) subtract_mean: bool,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioConvNeXt<B: Backend> {
+ pub(super) depthwise_conv: StructuredAudioConv1d<B>,
+ pub(super) norm: StructuredAudioNorm<B>,
+ pub(super) pwconv1: StructuredAudioPointwiseConv<B>,
+ pub(super) pwconv2: StructuredAudioPointwiseConv<B>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioResidualUnit<B: Backend> {
+ pub(super) snake1_alpha: Array<B>,
+ pub(super) conv1: StructuredAudioConv1d<B>,
+ pub(super) snake2_alpha: Array<B>,
+ pub(super) conv2: StructuredAudioConv1d<B>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioDecoderBlock<B: Backend> {
+ pub(super) snake_alpha: Array<B>,
+ pub(super) trans_conv: StructuredAudioConvTranspose1d<B>,
+ pub(super) res_unit1: StructuredAudioResidualUnit<B>,
+ pub(super) res_unit2: StructuredAudioResidualUnit<B>,
+ pub(super) res_unit3: StructuredAudioResidualUnit<B>,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct StructuredAudioDecoderGraph<B: Backend> {
+ pub(super) first_conv: StructuredAudioConv1d<B>,
+ pub(super) upsample_blocks: Vec<(StructuredAudioConvTranspose1d<B>, StructuredAudioConvNeXt<B>)>,
+ pub(super) decoder_blocks: Vec<StructuredAudioDecoderBlock<B>>,
+ pub(super) final_snake_alpha: Array<B>,
+ pub(super) final_conv: StructuredAudioConv1d<B>,
+}
diff --git a/crates/uzu/src/audio/nanocodec/runtime/support.rs b/crates/uzu/src/audio/nanocodec/runtime/support.rs
new file mode 100644
index 000000000..08e8198c6
--- /dev/null
+++ b/crates/uzu/src/audio/nanocodec/runtime/support.rs
@@ -0,0 +1,82 @@
+use super::*;
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct DecodedPaddedAudio {
+ pub samples: Vec<f32>,
+ pub channels: usize,
+ pub frames: usize,
+ pub lengths: Vec<usize>,
+}
+
+pub(in crate::audio::nanocodec::runtime) fn checked_product(values: &[usize]) -> AudioResult<usize> {
+ values
+ .iter()
+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))
+ .ok_or(AudioError::Runtime("dimension product overflow".to_string()))
+}
+
+pub(in crate::audio::nanocodec::runtime) fn usize_to_i32(
+ value: usize,
+ name: &str,
+) -> AudioResult<i32> {
+ i32::try_from(value).map_err(|_| AudioError::Runtime(format!("{name} exceeds i32 range")))
+}
+
+pub(in crate::audio::nanocodec::runtime) fn convert_lengths_to_i32(
+ lengths: &[usize],
+ frames: usize,
+) -> AudioResult<Vec<i32>> {
+ let mut out = Vec::with_capacity(lengths.len());
+ for &length in lengths {
+ if length > frames {
+ return Err(AudioError::InvalidTokenLengthValue {
+ length,
+ frames,
+ });
+ }
+ out.push(usize_to_i32(length, "length")?);
+ }
+ Ok(out)
+}
+
+pub(in crate::audio::nanocodec::runtime) fn checked_mul_i32(
+ value: i32,
+ mul: usize,
+) -> AudioResult<i32> {
+ i32::try_from(mul)
+ .map_err(|_| AudioError::Runtime("length scaling factor exceeds i32 range".to_string()))?
+ .checked_mul(value)
+ .ok_or(AudioError::Runtime("scaled length overflow".to_string()))
+}
+
+pub(in crate::audio::nanocodec::runtime) fn checked_div_ceil(
+ numerator: usize,
+ denominator: usize,
+) -> AudioResult<usize> {
+ if denominator == 0 {
+ return Err(AudioError::Runtime("division by zero".to_string()));
+ }
+ let addend = denominator.saturating_sub(1);
+ numerator
+ .checked_add(addend)
+ .ok_or(AudioError::Runtime("ceil-division overflow".to_string()))
+ .map(|value| value / denominator)
+}
+
+pub(in crate::audio::nanocodec::runtime) fn scale_lengths_i32_in_place(
+ source: &[i32],
+ destination: &mut [i32],
+ factor: usize,
+) -> AudioResult<()> {
+ if destination.len() != source.len() {
+ return Err(AudioError::Runtime(format!(
+ "scaled length buffer mismatch: expected {}, got {}",
+ source.len(),
+ destination.len()
+ )));
+ }
+ for (dst, &src) in destination.iter_mut().zip(source.iter()) {
+ *dst = checked_mul_i32(src, factor)?;
+ }
+ Ok(())
+}
diff --git a/crates/uzu/src/audio/types.rs b/crates/uzu/src/audio/types.rs
index be933455d..a06ef2049 100644
--- a/crates/uzu/src/audio/types.rs
+++ b/crates/uzu/src/audio/types.rs
@@ -1,5 +1,7 @@
use std::num::{NonZeroU32, NonZeroUsize};
+use crate::{backends::common::Backend, parameters::ParameterLoaderError};
+
#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)]
pub enum AudioError {
#[error("sample_rate must be > 0")]
@@ -47,11 +49,10 @@ pub enum AudioError {
pub type AudioResult<T> = Result<T, AudioError>;
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
-pub enum AudioTokenPacking {
- #[default]
- FrameMajor,
- CodebookMajor,
+impl<B: Backend> From<ParameterLoaderError<B>> for AudioError {
+ fn from(value: ParameterLoaderError<B>) -> Self {
+ Self::Runtime(value.to_string())
+ }
}
#[derive(Debug, Clone, PartialEq)]
@@ -123,7 +124,6 @@ pub struct AudioTokenGrid {
codebooks: usize,
frames: usize,
lengths: Box<[usize]>,
- packing: AudioTokenPacking,
}
impl AudioTokenGrid {
@@ -133,7 +133,6 @@ impl AudioTokenGrid {
codebooks: usize,
frames: usize,
lengths: Box<[usize]>,
- packing: AudioTokenPacking,
) -> AudioResult<Self> {
if codebooks == 0 {
return Err(AudioError::InvalidTokenCardinality);
@@ -173,7 +172,6 @@ impl AudioTokenGrid {
codebooks,
frames,
lengths,
- packing,
})
}
@@ -193,16 +191,12 @@ impl AudioTokenGrid {
&self.lengths
}
- pub fn packing(&self) -> AudioTokenPacking {
- self.packing
- }
-
pub fn tokens(&self) -> &[u32] {
&self.tokens
}
- pub fn into_parts(self) -> (Box<[u32]>, usize, usize, usize, Box<[usize]>, AudioTokenPacking) {
- (self.tokens, self.batch_size, self.codebooks, self.frames, self.lengths, self.packing)
+ pub fn into_parts(self) -> (Box<[u32]>, usize, usize, usize, Box<[usize]>) {
+ (self.tokens, self.batch_size, self.codebooks, self.frames, self.lengths)
}
pub fn get(
@@ -211,73 +205,18 @@ impl AudioTokenGrid {
codebook: usize,
frame: usize,
) -> u32 {
- let index =
- Self::token_index(self.packing, self.batch_size, self.codebooks, self.frames, batch, codebook, frame);
+ let index = Self::token_index(self.codebooks, self.frames, batch, codebook, frame);
self.tokens[index]
}
- pub fn to_packing(
- &self,
- packing: AudioTokenPacking,
- ) -> Self {
- if packing == self.packing {
- return self.clone();
- }
-
- let mut tokens = vec![0u32; self.tokens.len()];
- for batch in 0..self.batch_size {
- for frame in 0..self.frames {
- for codebook in 0..self.codebooks {
- let src_idx = Self::token_index(
- self.packing,
- self.batch_size,
- self.codebooks,
- self.frames,
- batch,
- codebook,
- frame,
- );
- let dst_idx = Self::token_index(
- packing,
- self.batch_size,
- self.codebooks,
- self.frames,
- batch,
- codebook,
- frame,
- );
- tokens[dst_idx] = self.tokens[src_idx];
- }
- }
- }
-
- Self {
- tokens: tokens.into_boxed_slice(),
- batch_size: self.batch_size,
- codebooks: self.codebooks,
- frames: self.frames,
- lengths: self.lengths.clone(),
- packing,
- }
- }
-
fn token_index(
- packing: AudioTokenPacking,
- batch_size: usize,
codebooks: usize,
frames: usize,
batch: usize,
codebook: usize,
frame: usize,
) -> usize {
- debug_assert!(batch < batch_size);
- debug_assert!(codebook < codebooks);
- debug_assert!(frame < frames);
-
- match packing {
- AudioTokenPacking::FrameMajor => ((batch * frames + frame) * codebooks) + codebook,
- AudioTokenPacking::CodebookMajor => ((batch * codebooks + codebook) * frames) + frame,
- }
+ ((batch * codebooks + codebook) * frames) + frame
}
}
diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rs
index 523aff8cf..7abf67225 100644
--- a/crates/uzu/src/backends/common/command_buffer.rs
+++ b/crates/uzu/src/backends/common/command_buffer.rs
@@ -73,8 +73,9 @@ pub trait CommandBufferEncoding {
fn encode_copy(
&mut self,
src: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,
+ src_range: Range<usize>,
dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,
- size: usize,
+ dst_range: Range<usize>,
);
fn encode_fill(
@@ -135,6 +136,5 @@ pub trait CommandBufferPending {
pub trait CommandBufferCompleted {
type CommandBuffer: CommandBuffer<Completed = Self>;
- /// Returns GPU execution time.
fn gpu_execution_time(&self) -> Option<Duration>;
}
diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rs
index 7c0136060..7717599d9 100644
--- a/crates/uzu/src/backends/common/encoder.rs
+++ b/crates/uzu/src/backends/common/encoder.rs
@@ -59,20 +59,23 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {
pub fn encode_copy(
&mut self,
src: &B::Buffer,
+ src_range: Range<usize>,
dst: &mut B::Buffer,
- size: usize,
+ dst_range: Range<usize>,
) {
+ let size = src_range.end - src_range.start;
+ debug_assert_eq!(size, dst_range.end - dst_range.start);
self.access(&[
Access {
- range: src.gpu_address_subrange(0..size),
+ range: src.gpu_address_subrange(src_range.clone()),
flags: AccessFlags::copy_read(),
},
Access {
- range: dst.gpu_address_subrange(0..size),
+ range: dst.gpu_address_subrange(dst_range.clone()),
flags: AccessFlags::copy_write(),
},
]);
- self.command_buffer.encode_copy(src, dst, size);
+ self.command_buffer.encode_copy(src, src_range, dst, dst_range);
}
pub fn encode_fill(
diff --git a/crates/uzu/src/backends/common/gpu_types/activation_type.rs b/crates/uzu/src/backends/common/gpu_types/activation_type.rs
index 939965237..09ebe4657 100644
--- a/crates/uzu/src/backends/common/gpu_types/activation_type.rs
+++ b/crates/uzu/src/backends/common/gpu_types/activation_type.rs
@@ -8,6 +8,7 @@ const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)
pub enum ActivationType {
SILU,
GELU,
+ TANH,
IDENTITY,
}
@@ -19,6 +20,7 @@ impl ActivationType {
match self {
ActivationType::SILU => silu(x),
ActivationType::GELU => gelu(x),
+ ActivationType::TANH => tanh_activation(x),
ActivationType::IDENTITY => x,
}
}
@@ -36,3 +38,8 @@ fn gelu<T: Float>(x: T) -> T {
let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());
T::from(y_float).unwrap()
}
+
+fn tanh_activation<T: Float>(x: T) -> T {
+ let x_float = x.to_f32().unwrap();
+ T::from(x_float.tanh()).unwrap()
+}
diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rs
index f782acf4a..b5a284288 100644
--- a/crates/uzu/src/backends/common/kernel/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/mod.rs
@@ -3,7 +3,6 @@
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
pub mod attention;
-pub mod audio;
pub mod kv_cache_update;
pub mod matmul;
pub mod mlp_gate_act_mul;
diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rs
index 400faa501..491431e61 100644
--- a/crates/uzu/src/backends/common/kernel/sampling.rs
+++ b/crates/uzu/src/backends/common/kernel/sampling.rs
@@ -16,7 +16,7 @@ use crate::{
TemperatureKernel, TopKKernel, TopPKernel,
},
},
- session::parameter::SamplingMethod,
+ session::parameter::{SamplingMethod, SamplingProcessingOrder},
};
#[derive(Debug, Clone, Copy, PartialEq)]
@@ -177,9 +177,12 @@ impl<B: Backend> SamplingKernel<B> {
top_k,
top_p,
min_p,
+ processing_order,
} = sampling_method
{
- if let Some(temperature) = temperature {
+ if let Some(temperature) = temperature
+ && processing_order == SamplingProcessingOrder::TemperatureThenFilters
+ {
self.temperature.encode(
None::<&B::Buffer>,
logits_buffer.deref_mut(),
@@ -200,7 +203,6 @@ impl<B: Backend> SamplingKernel<B> {
encoder,
);
}
-
if let Some(top_p) = top_p {
self.topp.encode(
None::<&B::Buffer>,
@@ -211,7 +213,6 @@ impl<B: Backend> SamplingKernel<B> {
encoder,
);
}
-
if let Some(min_p) = min_p {
self.minp.encode(
None::<&B::Buffer>,
@@ -223,6 +224,19 @@ impl<B: Backend> SamplingKernel<B> {
);
}
+ if let Some(temperature) = temperature
+ && processing_order == SamplingProcessingOrder::FiltersThenTemperature
+ {
+ self.temperature.encode(
+ None::<&B::Buffer>,
+ logits_buffer.deref_mut(),
+ batch_size as u32,
+ vocab_size as u32,
+ temperature,
+ encoder,
+ );
+ }
+
self.gumbel.encode(
None::<&B::Buffer>,
(seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),
diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rs
index 8ce9063bd..086877080 100644
--- a/crates/uzu/src/backends/cpu/command_buffer.rs
+++ b/crates/uzu/src/backends/cpu/command_buffer.rs
@@ -59,13 +59,19 @@ impl CommandBufferEncoding for CpuCommandBuffer {
fn encode_copy(
&mut self,
src: &UnsafeCell<Box<[u8]>>,
+ src_range: std::ops::Range<usize>,
dst: &mut UnsafeCell<Box<[u8]>>,
- size: usize,
+ dst_range: std::ops::Range<usize>,
) {
- let src = unsafe { &*src.get() }.as_ptr();
- let dst = dst.get_mut().as_mut_ptr();
+ let size = src_range.end - src_range.start;
+ assert_eq!(size, dst_range.end - dst_range.start);
+ assert!(unsafe { &*src.get() }.len() >= src_range.end);
+ assert!(dst.get_mut().len() >= dst_range.end);
+
+ let src_ptr = unsafe { (&*src.get()).as_ptr().add(src_range.start) };
+ let dst_ptr = unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) };
self.push_command(Box::new(move || unsafe {
- std::ptr::copy(src, dst, size);
+ std::ptr::copy(src_ptr, dst_ptr, size);
}))
}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/add.rs b/crates/uzu/src/backends/cpu/kernel/audio/add.rs
new file mode 100644
index 000000000..7e711eb22
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/add.rs
@@ -0,0 +1,16 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioAdd)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_add<T: ArrayElement + Float>(
+ #[allow(unused)] a: *const T,
+ #[allow(unused)] b: *const T,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] n: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/audio.rs b/crates/uzu/src/backends/cpu/kernel/audio/audio.rs
deleted file mode 100644
index 81a2fd57a..000000000
--- a/crates/uzu/src/backends/cpu/kernel/audio/audio.rs
+++ /dev/null
@@ -1,167 +0,0 @@
-use dsl::kernel;
-use half::{bf16, f16};
-use num_traits::Float;
-
-use crate::ArrayElement;
-
-#[kernel(AudioFsqDecode)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_fsq_decode<T: ArrayElement + Float>(
- #[allow(unused)] tokens: *const i32,
- #[allow(unused)] out: *mut T,
- #[allow(unused)] lengths: *const i32,
- #[allow(unused)] num_groups: i32,
- #[allow(unused)] seq_len: i32,
- #[allow(unused)] codebook_dim: i32,
- #[allow(unused)] num_levels: &[i32],
- #[allow(unused)] batch_size: i32,
-) {
- todo!()
-}
-
-#[kernel(AudioLeakyRelu)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_leaky_relu<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] n: i32,
- #[allow(unused)] negative_slope: f32,
-) {
- todo!()
-}
-
-#[kernel(AudioTanh)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_tanh<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] n: i32,
-) {
- todo!()
-}
-
-#[kernel(AudioAdd)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_add<T: ArrayElement + Float>(
- #[allow(unused)] a: *const T,
- #[allow(unused)] b: *const T,
- #[allow(unused)] out: *mut T,
- #[allow(unused)] n: i32,
-) {
- todo!()
-}
-
-#[kernel(AudioScale)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_scale<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] n: i32,
- #[allow(unused)] scale: f32,
-) {
- todo!()
-}
-
-#[kernel(AudioCausalConv1d)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_causal_conv1d<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] weight: *const T,
- #[allow(unused)] bias: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] lengths: *const i32,
- #[allow(unused)] cin: i32,
- #[allow(unused)] cout: i32,
- #[allow(unused)] seq_len: i32,
- #[allow(unused)] kernel_size: i32,
- #[allow(unused)] dilation: i32,
- #[allow(unused)] batch_size: i32,
-) {
- todo!()
-}
-
-#[kernel(AudioCausalConvTranspose1d)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_causal_conv_transpose1d<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] weight: *const T,
- #[allow(unused)] bias: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] lengths: *const i32,
- #[allow(unused)] cin: i32,
- #[allow(unused)] cout: i32,
- #[allow(unused)] seq_len_in: i32,
- #[allow(unused)] seq_len_out: i32,
- #[allow(unused)] stride: i32,
- #[allow(unused)] groups: i32,
- #[allow(unused)] batch_size: i32,
-) {
- todo!()
-}
-
-#[kernel(AudioHalfSnake)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_half_snake<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] alpha: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] channels: i32,
- #[allow(unused)] seq_len: i32,
- #[allow(unused)] snake_channels: i32,
- #[allow(unused)] negative_slope: f32,
- #[allow(unused)] eps: f32,
- #[allow(unused)] batch_size: i32,
-) {
- todo!()
-}
-
-#[kernel(AudioClamp)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_clamp<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] n: i32,
- #[allow(unused)] min_value: f32,
- #[allow(unused)] max_value: f32,
-) {
- todo!()
-}
-
-#[kernel(AudioConv1d)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_conv1d<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] weight: *const T,
- #[allow(unused)] bias: *const T,
- #[allow(unused)] output: *mut T,
- #[allow(unused)] lengths: *const i32,
- #[allow(unused)] cin: i32,
- #[allow(unused)] cout: i32,
- #[allow(unused)] seq_len_in: i32,
- #[allow(unused)] seq_len_out: i32,
- #[allow(unused)] kernel_size: i32,
- #[allow(unused)] stride: i32,
- #[allow(unused)] dilation: i32,
- #[allow(unused)] padding: i32,
- #[allow(unused)] pad_mode: i32,
- #[allow(unused)] batch_size: i32,
-) {
- todo!()
-}
-
-#[kernel(AudioFsqEncode)]
-#[variants(T, f32, f16, bf16)]
-pub fn audio_fsq_encode<T: ArrayElement + Float>(
- #[allow(unused)] input: *const T,
- #[allow(unused)] tokens: *mut i32,
- #[allow(unused)] lengths: *const i32,
- #[allow(unused)] num_groups: i32,
- #[allow(unused)] seq_len: i32,
- #[allow(unused)] codebook_dim: i32,
- #[allow(unused)] num_levels: &[i32],
- #[allow(unused)] dim_base_index: &[i32],
- #[allow(unused)] eps: f32,
- #[allow(unused)] batch_size: i32,
-) {
- todo!()
-}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d.rs
new file mode 100644
index 000000000..a7725f357
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d.rs
@@ -0,0 +1,24 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioCausalConv1d)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_causal_conv1d<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] kernel_size: i32,
+ #[allow(unused)] dilation: i32,
+ #[allow(unused)] input_layout: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped.rs
new file mode 100644
index 000000000..26c6c521f
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped.rs
@@ -0,0 +1,25 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioCausalConv1dGrouped)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_causal_conv1d_grouped<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] kernel_size: i32,
+ #[allow(unused)] dilation: i32,
+ #[allow(unused)] groups: i32,
+ #[allow(unused)] input_layout: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped_residual.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped_residual.rs
new file mode 100644
index 000000000..cc247a183
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped_residual.rs
@@ -0,0 +1,25 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioCausalConv1dGroupedResidual)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_causal_conv1d_grouped_residual<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] residual: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] kernel_size: i32,
+ #[allow(unused)] dilation: i32,
+ #[allow(unused)] groups: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d.rs
new file mode 100644
index 000000000..e143dc71b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d.rs
@@ -0,0 +1,24 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioCausalConvTranspose1d)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_causal_conv_transpose1d<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len_in: i32,
+ #[allow(unused)] seq_len_out: i32,
+ #[allow(unused)] stride: i32,
+ #[allow(unused)] groups: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d_causal_pad.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d_causal_pad.rs
new file mode 100644
index 000000000..d9f33326b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d_causal_pad.rs
@@ -0,0 +1,26 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioCausalConvTranspose1dCausalPad)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_causal_conv_transpose1d_causal_pad<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len_in: i32,
+ #[allow(unused)] seq_len_out: i32,
+ #[allow(unused)] kernel_size: i32,
+ #[allow(unused)] stride: i32,
+ #[allow(unused)] groups: i32,
+ #[allow(unused)] input_layout: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/audio/conv1d.rs
new file mode 100644
index 000000000..5096e3994
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/conv1d.rs
@@ -0,0 +1,27 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioConv1d)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_conv1d<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] weight: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] cin: i32,
+ #[allow(unused)] cout: i32,
+ #[allow(unused)] seq_len_in: i32,
+ #[allow(unused)] seq_len_out: i32,
+ #[allow(unused)] kernel_size: i32,
+ #[allow(unused)] stride: i32,
+ #[allow(unused)] dilation: i32,
+ #[allow(unused)] padding: i32,
+ #[allow(unused)] pad_mode: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/fsq_decode.rs b/crates/uzu/src/backends/cpu/kernel/audio/fsq_decode.rs
new file mode 100644
index 000000000..8da21f019
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/fsq_decode.rs
@@ -0,0 +1,21 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioFsqDecode)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_fsq_decode<T: ArrayElement + Float>(
+ #[allow(unused)] tokens: *const i32,
+ #[allow(unused)] out: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] num_groups: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] codebook_dim: i32,
+ #[allow(unused)] num_levels: &[i32],
+ #[allow(unused)] dim_base_index: &[i32],
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/fsq_encode.rs b/crates/uzu/src/backends/cpu/kernel/audio/fsq_encode.rs
new file mode 100644
index 000000000..580cabc38
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/fsq_encode.rs
@@ -0,0 +1,22 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioFsqEncode)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_fsq_encode<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] tokens: *mut i32,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] num_groups: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] codebook_dim: i32,
+ #[allow(unused)] num_levels: &[i32],
+ #[allow(unused)] dim_base_index: &[i32],
+ #[allow(unused)] eps: f32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/half_snake.rs b/crates/uzu/src/backends/cpu/kernel/audio/half_snake.rs
new file mode 100644
index 000000000..d3aaeace0
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/half_snake.rs
@@ -0,0 +1,21 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioHalfSnake)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_half_snake<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] alpha: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] channels: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] snake_channels: i32,
+ #[allow(unused)] negative_slope: f32,
+ #[allow(unused)] eps: f32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/mod.rs b/crates/uzu/src/backends/cpu/kernel/audio/mod.rs
index fa3e4e468..cd7577077 100644
--- a/crates/uzu/src/backends/cpu/kernel/audio/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/audio/mod.rs
@@ -1 +1,12 @@
-pub mod audio;
+pub mod add;
+pub mod causal_conv1d;
+pub mod causal_conv1d_grouped;
+pub mod causal_conv1d_grouped_residual;
+pub mod causal_conv_transpose1d;
+pub mod causal_conv_transpose1d_causal_pad;
+pub mod conv1d;
+pub mod fsq_decode;
+pub mod fsq_encode;
+pub mod half_snake;
+pub mod norm_ncs;
+pub mod quantizer_decode;
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/norm_ncs.rs b/crates/uzu/src/backends/cpu/kernel/audio/norm_ncs.rs
new file mode 100644
index 000000000..9830f981b
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/norm_ncs.rs
@@ -0,0 +1,22 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioNormNcs)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_norm_ncs<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] scales: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] channels: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] epsilon: f32,
+ #[allow(unused)] subtract_mean: i32,
+ #[allow(unused)] batch_size: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/audio/quantizer_decode.rs b/crates/uzu/src/backends/cpu/kernel/audio/quantizer_decode.rs
new file mode 100644
index 000000000..7bb16000e
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/audio/quantizer_decode.rs
@@ -0,0 +1,29 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(AudioQuantizerDecode)]
+#[variants(T, f32, f16, bf16)]
+pub fn audio_quantizer_decode<T: ArrayElement + Float>(
+ #[allow(unused)] tokens: *const u32,
+ #[allow(unused)] lengths: *const i32,
+ #[allow(unused)] semantic_codebook: *const T,
+ #[allow(unused)] semantic_out_proj: *const T,
+ #[allow(unused)] semantic_out_bias: *const T,
+ #[allow(unused)] residual_codebooks: *const T,
+ #[allow(unused)] residual_out_proj: *const T,
+ #[allow(unused)] residual_out_bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] batch_size: i32,
+ #[allow(unused)] total_codebooks: i32,
+ #[allow(unused)] seq_len: i32,
+ #[allow(unused)] input_dim: i32,
+ #[allow(unused)] codebook_dim: i32,
+ #[allow(unused)] residual_quantizers: i32,
+ #[allow(unused)] semantic_cardinality: i32,
+ #[allow(unused)] residual_cardinality: i32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs
new file mode 100644
index 000000000..eccc8c7f1
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs
@@ -0,0 +1,19 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(EmbeddingRowsSum)]
+#[variants(T, f32, f16, bf16)]
+pub fn embedding_rows_sum<T: ArrayElement + Float>(
+ #[allow(unused)] token_indices: *const u32,
+ #[allow(unused)] weights: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] num_rows: u32,
+ #[allow(unused)] total_rows: u32,
+ #[allow(unused)] model_dim: u32,
+ #[allow(unused)] codebook_stride: u32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs
index 93f7cf1f0..556c0cfb7 100644
--- a/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs
@@ -1,2 +1,3 @@
+pub mod embedding_rows_sum;
pub mod full_precision_embedding;
pub mod quant_embedding;
diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rs
index 287488d31..2d9020b1f 100644
--- a/crates/uzu/src/backends/cpu/kernel/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs
@@ -18,6 +18,7 @@ mod short_conv;
mod sigmoid;
mod ssm;
mod tensor_add_bias;
+mod tensor_add_scale;
mod tensor_add_swap;
mod tensor_copy;
mod token_copy;
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/mod.rs
new file mode 100644
index 000000000..287ef450d
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/mod.rs
@@ -0,0 +1 @@
+pub mod tensor_add_scale;
diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
new file mode 100644
index 000000000..e0d2289b5
--- /dev/null
+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs
@@ -0,0 +1,18 @@
+use dsl::kernel;
+use half::{bf16, f16};
+use num_traits::Float;
+
+use crate::ArrayElement;
+
+#[kernel(TensorAddScale)]
+#[variants(T, f32, f16, bf16)]
+pub fn tensor_add_scale<T: ArrayElement + Float>(
+ #[allow(unused)] input: *const T,
+ #[allow(unused)] bias: *const T,
+ #[allow(unused)] output: *mut T,
+ #[allow(unused)] num_cols: u32,
+ #[allow(unused)] length: u32,
+ #[allow(unused)] scale: f32,
+) {
+ todo!()
+}
diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rs
index 894b5eb96..aa3de6135 100644
--- a/crates/uzu/src/backends/metal/command_buffer.rs
+++ b/crates/uzu/src/backends/metal/command_buffer.rs
@@ -121,12 +121,15 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {
fn encode_copy(
&mut self,
src: &Retained<ProtocolObject<dyn MTLBuffer>>,
+ src_range: std::ops::Range<usize>,
dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,
- size: usize,
+ dst_range: std::ops::Range<usize>,
) {
- assert!(src.length() >= size && dst.length() >= size);
+ let size = src_range.end - src_range.start;
+ assert_eq!(size, dst_range.end - dst_range.start);
+ assert!(src.length() >= src_range.end && dst.length() >= dst_range.end);
- self.ensure_blit().copy_buffer_to_buffer(src, 0, dst, 0, size);
+ self.ensure_blit().copy_buffer_to_buffer(src, src_range.start, dst, dst_range.start, size);
}
fn encode_fill(
@@ -231,7 +234,7 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {
fn gpu_execution_time(&self) -> Option<Duration> {
match (self.command_buffer.gpu_start_time(), self.command_buffer.gpu_end_time()) {
- (Some(start), Some(end)) => Some(Duration::from_secs_f64(end - start)),
+ (Some(start), Some(end)) if end > start => Some(Duration::from_secs_f64(end - start)),
_ => None,
}
}
diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.h
index 741264acd..290f091ea 100644
--- a/crates/uzu/src/backends/metal/generated/activation_type.h
+++ b/crates/uzu/src/backends/metal/generated/activation_type.h
@@ -17,6 +17,7 @@ namespace activation_type {
enum ActivationType : uint32_t {
SILU,
GELU,
+ TANH,
IDENTITY,
};
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.h
index 68ee975d5..b8c053e85 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h
+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h
@@ -21,6 +21,18 @@ inline T activate_gelu(T x) {
return T(yf);
}
+template <typename T>
+inline T activate_leaky_relu(T x, float alpha) {
+ const float xf = float(x);
+ const float yf = (xf >= 0.0f) ? xf : (alpha * xf);
+ return static_cast<T>(yf);
+}
+
+template <typename T>
+inline T activate_tanh(T x) {
+ return static_cast<T>(metal::tanh(float(x)));
+}
+
template <typename T>
inline T activate(T x, ActivationType type) {
switch (type) {
@@ -28,6 +40,8 @@ inline T activate(T x, ActivationType type) {
return activate_silu(x);
case GELU:
return activate_gelu(x);
+ case TANH:
+ return activate_tanh(x);
case IDENTITY:
return x;
}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/add.metal b/crates/uzu/src/backends/metal/kernel/audio/add.metal
new file mode 100644
index 000000000..545920cd5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/add.metal
@@ -0,0 +1,30 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+template <typename T>
+void add_tensors(
+ device const T* a,
+ device const T* b,
+ device T* out,
+ const constant int& n,
+ const uint tid
+) {
+ if ((int)tid >= n) {
+ return;
+ }
+ out[tid] = a[tid] + b[tid];
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioAdd)(
+ device const T* a,
+ device const T* b,
+ device T* out,
+ const constant int& n,
+ uint tid AXIS(n, 256)
+) {
+ add_tensors<T>(a, b, out, n, tid);
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metal
deleted file mode 100644
index e95cb27f0..000000000
--- a/crates/uzu/src/backends/metal/kernel/audio/audio.metal
+++ /dev/null
@@ -1,738 +0,0 @@
-#include <metal_stdlib>
-#include "../common/dsl.h"
-
-using namespace metal;
-
-// === FSQ (Finite Scalar Quantization) decode ===
-//
-// Implements NeMo's FiniteScalarQuantizer.decode() applied per group:
-// - tokens are stored as [B, G, T] int32
-// - output codes are stored as [B, G*D, T] where D = codebook_dim_per_group
-//
-// NeMo reference:
-// codes_nonnegative = (indices // dim_base_index) % num_levels
-// dequantized = (codes_nonnegative - (num_levels//2)) / (num_levels//2)
-//
-// Here dim_base_index is derived from num_levels:
-// base[0]=1, base[d]=prod_{k<d} num_levels[k]
-
-template <typename T>
-void fsq_decode(
- device const int* tokens,
- device T* out,
- device const int* lengths,
- const constant int& num_groups,
- const constant int& seq_len,
- const constant int& codebook_dim,
- const constant int* num_levels,
- const uint3 gid
-) {
- const uint t = gid.x;
- const uint g = gid.y;
- const uint b = gid.z;
-
- if (t >= (uint)seq_len || g >= (uint)num_groups) {
- return;
- }
-
- const int len_b = lengths ? lengths[b] : seq_len;
- if ((int)t >= len_b) {
- // Zero all D codes for this token when masked.
- for (int d = 0; d < codebook_dim; ++d) {
- const uint out_c = g * (uint)codebook_dim + (uint)d;
- const uint out_idx =
- (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;
- out[out_idx] = (T)0;
- }
- return;
- }
-
- const uint token_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;
- const int token = tokens[token_idx];
-
- // Compute dim bases (small D, do it per-thread).
- int base = 1;
- for (int d = 0; d < codebook_dim; ++d) {
- const int levels = num_levels[d];
- const int scale = levels / 2; // integer division (matches PyTorch // 2)
- const int offset = scale;
-
- // Euclidean modulo to keep result in [0, levels) even if token is negative
- // (tokens are expected nonnegative, but keep this robust).
- const int div = token / base;
- int code_nonneg = div % levels;
- if (code_nonneg < 0) {
- code_nonneg += levels;
- }
- const float code = ((float)(code_nonneg - offset)) / (float)scale;
-
- const uint out_c = g * (uint)codebook_dim + (uint)d;
- const uint out_idx =
- (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;
- out[out_idx] = (T)code;
-
- base *= levels;
- }
-}
-
-// === Elementwise ops for codec blocks ===
-
-template <typename T>
-void audio_codec_leaky_relu(
- device const T* input,
- device T* output,
- const constant int& n,
- const constant float& negative_slope,
- const uint tid
-) {
- if ((int)tid >= n) {
- return;
- }
- const float x = float(input[tid]);
- const float y = (x >= 0.0f) ? x : (negative_slope * x);
- output[tid] = (T)y;
-}
-
-template <typename T>
-void audio_codec_tanh(
- device const T* input,
- device T* output,
- const constant int& n,
- const uint tid
-) {
- if ((int)tid >= n) {
- return;
- }
- const float x = float(input[tid]);
- output[tid] = (T)tanh(x);
-}
-
-template <typename T>
-void audio_codec_add(
- device const T* a,
- device const T* b,
- device T* out,
- const constant int& n,
- const uint tid
-) {
- if ((int)tid >= n) {
- return;
- }
- out[tid] = a[tid] + b[tid];
-}
-
-template <typename T>
-void audio_codec_scale(
- device const T* input,
- device T* output,
- const constant int& n,
- const constant float& scale,
- const uint tid
-) {
- if ((int)tid >= n) {
- return;
- }
- const float x = float(input[tid]);
- output[tid] = (T)(x * scale);
-}
-
-// === Causal Conv1d (stride=1, groups=1) ===
-//
-// Matches NeMo's CausalConv1dNorm forward for the decoder path where
-// stride=1:
-// - left padding by padding_total = (kernel_size_eff - 1) where
-// kernel_size_eff=(K-1)*dilation+1
-// - no right padding (extra_padding=0 for stride=1)
-// - output length equals input length
-template <typename T>
-void audio_codec_causal_conv1d(
- device const T* input, // [B, Cin, T]
- device const T* weight, // [Cout, Cin, K]
- device const T* bias, // [Cout]
- device T* output, // [B, Cout, T]
- device const int* lengths, // [B]
- const constant int& cin,
- const constant int& cout,
- const constant int& seq_len,
- const constant int& kernel_size,
- const constant int& dilation,
- const uint3 gid
-) {
- const uint t = gid.x;
- const uint oc = gid.y;
- const uint b = gid.z;
-
- if (t >= (uint)seq_len || oc >= (uint)cout) {
- return;
- }
-
- const int len_b = lengths ? lengths[b] : seq_len;
- const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len + t;
- if ((int)t >= len_b) {
- output[out_idx] = (T)0;
- return;
- }
-
- const int pad = (kernel_size - 1) * dilation;
- float acc = float(bias[oc]);
-
- for (int ic = 0; ic < cin; ++ic) {
- const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;
- const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;
- for (int k = 0; k < kernel_size; ++k) {
- const int x_t = (int)t + k * dilation - pad;
- if (x_t < 0) {
- continue;
- }
- // For stride=1 and causal padding, x_t never exceeds len_b for valid t,
- // but for safety clamp to seq_len.
- if (x_t >= seq_len) {
- continue;
- }
- acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);
- }
- }
-
- output[out_idx] = (T)acc;
-}
-
-// === Causal ConvTranspose1d upsample (kernel_size = 2*stride) ===
-//
-// Matches NeMo's CausalConvTranspose1dNorm forward for trim_right_ratio=1:
-// - ConvTranspose1d with padding=0, output_padding=0, dilation=1
-// - kernel_size = 2 * stride
-// - unpad: trim_right = kernel_size - stride (= stride), trim_left = 0
-// Resulting output length is input_len * stride.
-template <typename T>
-void audio_codec_causal_conv_transpose1d(
- device const T* input, // [B, Cin, Tin]
- device const T* weight, // [Cin, Cout_per_group, 2*stride]
- device const T* bias, // [Cout]
- device T* output, // [B, Cout, Tout]
- device const int* lengths, // [B] (output lengths)
- const constant int& cin,
- const constant int& cout,
- const constant int& seq_len_in,
- const constant int& seq_len_out,
- const constant int& stride,
- const constant int& groups,
- const uint3 gid
-) {
- const uint t_out = gid.x;
- const uint oc = gid.y;
- const uint b = gid.z;
-
- if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
- return;
- }
-
- const int len_b = lengths ? lengths[b] : seq_len_out;
- const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;
- if ((int)t_out >= len_b) {
- output[out_idx] = (T)0;
- return;
- }
-
- const int cout_per_group = cout / groups;
- const int cin_per_group = cin / groups;
- const int group_idx = (int)oc / cout_per_group;
- const int oc_in_group = (int)oc - group_idx * cout_per_group;
-
- const int q = (int)t_out / stride;
- const int r = (int)t_out - q * stride;
-
- float acc = float(bias[oc]);
-
- const uint in_base_b = b * (uint)cin * (uint)seq_len_in;
- const uint w_base_oc = (uint)oc_in_group * (uint)(2 * stride);
-
- const int ic_begin = group_idx * cin_per_group;
- const int ic_end = ic_begin + cin_per_group;
-
- for (int ic = ic_begin; ic < ic_end; ++ic) {
- if (q >= seq_len_in) {
- continue;
- }
- const uint in_base = in_base_b + (uint)ic * (uint)seq_len_in;
-
- // Weight layout: [Cin, Cout_per_group, K]
- const uint w_base =
- ((uint)ic * (uint)cout_per_group) * (uint)(2 * stride) + w_base_oc;
-
- // Contribution from input[q] with k=r
- acc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);
-
- // Contribution from input[q-1] with k=stride+r
- if (q > 0) {
- acc += float(weight[w_base + (uint)(stride + r)]) *
- float(input[in_base + (uint)(q - 1)]);
- }
- }
-
- output[out_idx] = (T)acc;
-}
-
-// === Snake / HalfSnake / Clamp (NeMo common.parts.utils) ===
-//
-// snake(x, alpha, eps) = x + (alpha + eps)^-1 * sin(alpha * x)^2
-// HalfSnake applies snake to the first half channels and LeakyReLU to the
-// rest.
-
-template <typename T>
-void audio_codec_half_snake(
- device const T* input, // [B, C, T]
- device const T* alpha, // [1, C_snake, 1] (contiguous, index by channel)
- device T* output, // [B, C, T]
- const constant int& channels,
- const constant int& seq_len,
- const constant int& snake_channels,
- const constant float& negative_slope,
- const constant float& eps,
- const uint3 gid
-) {
- const uint t = gid.x;
- const uint c = gid.y;
- const uint b = gid.z;
-
- if (t >= (uint)seq_len || c >= (uint)channels) {
- return;
- }
-
- const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;
- const float x = float(input[idx]);
-
- if ((int)c < snake_channels) {
- const float a = float(alpha[c]);
- const float ax = a * x;
- const float s = sin(ax);
- const float y = x + (1.0f / (a + eps)) * (s * s);
- output[idx] = (T)y;
- } else {
- const float y = (x >= 0.0f) ? x : (negative_slope * x);
- output[idx] = (T)y;
- }
-}
-
-template <typename T>
-void audio_codec_clamp(
- device const T* input,
- device T* output,
- const constant int& n,
- const constant float& min_value,
- const constant float& max_value,
- const uint tid
-) {
- if ((int)tid >= n) {
- return;
- }
- const float x = float(input[tid]);
- const float y = clamp(x, min_value, max_value);
- output[tid] = (T)y;
-}
-
-// === Non-causal Conv1d with padding + stride (for encoder) ===
-//
-// Matches torch.nn.Conv1d with explicit padding and padding_mode:
-// out[t_out] = bias + sum_{ic,k} w[oc,ic,k] * x[ic, t_out*stride - padding
-// + k*dilation]
-// - pad_mode=0: zeros
-// - pad_mode=1: replicate (clamp index to [0, seq_len_in-1])
-template <typename T>
-void audio_codec_conv1d(
- device const T* input, // [B, Cin, Tin]
- device const T* weight, // [Cout, Cin, K]
- device const T* bias, // [Cout]
- device T* output, // [B, Cout, Tout]
- device const int* lengths, // [B] (Tout lengths)
- const constant int& cin,
- const constant int& cout,
- const constant int& seq_len_in,
- const constant int& seq_len_out,
- const constant int& kernel_size,
- const constant int& stride,
- const constant int& dilation,
- const constant int& padding,
- const constant int& pad_mode,
- const uint3 gid
-) {
- const uint t_out = gid.x;
- const uint oc = gid.y;
- const uint b = gid.z;
-
- if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
- return;
- }
-
- const int len_b = lengths ? lengths[b] : seq_len_out;
- const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;
- if ((int)t_out >= len_b) {
- output[out_idx] = (T)0;
- return;
- }
-
- const int base = (int)t_out * stride - padding;
- float acc = float(bias[oc]);
-
- for (int ic = 0; ic < cin; ++ic) {
- const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;
- const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;
- for (int k = 0; k < kernel_size; ++k) {
- int x_t = base + k * dilation;
- if (pad_mode == 0) {
- if (x_t < 0 || x_t >= seq_len_in) {
- continue;
- }
- } else {
- x_t = clamp(x_t, 0, seq_len_in - 1);
- }
- acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);
- }
- }
-
- output[out_idx] = (T)acc;
-}
-
-// === FSQ (Finite Scalar Quantization) encode ===
-//
-// Implements GroupFiniteScalarQuantizer.encode() for inference:
-// - input: [B, G*D, T] float
-// - output: [B, G, T] int32
-//
-// NeMo reference (FiniteScalarQuantizer):
-// output_scale = (num_levels - 1) / 2 * (1 - eps)
-// output_offset = 0.5 if even else 0
-// input_shift = tan(output_offset / output_scale)
-// compressed = output_scale * tanh(inputs + input_shift) - output_offset
-// rounded = round(compressed) # half away from zero
-// code_nonneg = rounded + (num_levels//2)
-// token = sum_d code_nonneg[d] * dim_base_index[d]
-
-// Match torch.round() semantics (ties to even).
-inline float round_ties_to_even(const float x) {
- const float f = floor(x);
- const float frac = x - f;
- if (frac < 0.5f) {
- return f;
- }
- if (frac > 0.5f) {
- return f + 1.0f;
- }
- const int fi = (int)f; // f is integral
- return ((fi & 1) != 0) ? (f + 1.0f) : f;
-}
-
-template <typename T>
-void fsq_encode(
- device const T* input, // [B, G*D, T]
- device int* tokens, // [B, G, T]
- device const int* lengths, // [B]
- const constant int& num_groups,
- const constant int& seq_len,
- const constant int& codebook_dim,
- const constant int* num_levels,
- const constant int* dim_base_index,
- const constant float& eps,
- const uint3 gid
-) {
- const uint t = gid.x;
- const uint g = gid.y;
- const uint b = gid.z;
-
- if (t >= (uint)seq_len || g >= (uint)num_groups) {
- return;
- }
-
- const int len_b = lengths ? lengths[b] : seq_len;
- const uint out_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;
- if ((int)t >= len_b) {
- tokens[out_idx] = 0;
- return;
- }
-
- const uint in_base =
- (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *
- (uint)seq_len +
- t;
-
- int token = 0;
- for (int d = 0; d < codebook_dim; ++d) {
- const int levels = num_levels[d];
- const int scale_i = levels / 2;
- const float output_scale = ((float)(levels - 1)) * 0.5f * (1.0f - eps);
- const float output_offset = (levels % 2 == 0) ? 0.5f : 0.0f;
- const float input_shift = tan(output_offset / output_scale);
-
- const float x = float(input[in_base + (uint)d * (uint)seq_len]);
- const float compressed =
- output_scale * tanh(x + input_shift) - output_offset;
- const float rounded = round_ties_to_even(compressed);
-
- int code_nonneg = (int)rounded + scale_i;
- code_nonneg = clamp(code_nonneg, 0, levels - 1);
- token += code_nonneg * dim_base_index[d];
- }
-
- tokens[out_idx] = token;
-}
-
-// === DSL kernel wrappers ===
-//
-// Keep the scalar helper implementations above and expose a DSL-annotated
-// surface so audio kernels are integrated through the same generated wrapper
-// path as other kernels.
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioFsqDecode)(
- device const int* tokens,
- device T* out,
- device const int* lengths,
- const constant int& num_groups,
- const constant int& seq_len,
- const constant int& codebook_dim,
- const constant int* num_levels,
- const constant int& batch_size,
- uint t AXIS(seq_len, 32),
- uint g AXIS(num_groups, 1),
- uint b AXIS(batch_size, 1)
-) {
- fsq_decode<T>(
- tokens,
- out,
- lengths,
- num_groups,
- seq_len,
- codebook_dim,
- num_levels,
- uint3(t, g, b)
- );
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioLeakyRelu)(
- device const T* input,
- device T* output,
- const constant int& n,
- const constant float& negative_slope,
- uint tid AXIS(n, 256)
-) {
- audio_codec_leaky_relu<T>(input, output, n, negative_slope, tid);
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioTanh)(
- device const T* input,
- device T* output,
- const constant int& n,
- uint tid AXIS(n, 256)
-) {
- audio_codec_tanh<T>(input, output, n, tid);
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioAdd)(
- device const T* a,
- device const T* b,
- device T* out,
- const constant int& n,
- uint tid AXIS(n, 256)
-) {
- audio_codec_add<T>(a, b, out, n, tid);
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioScale)(
- device const T* input,
- device T* output,
- const constant int& n,
- const constant float& scale,
- uint tid AXIS(n, 256)
-) {
- audio_codec_scale<T>(input, output, n, scale, tid);
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioCausalConv1d)(
- device const T* input,
- device const T* weight,
- device const T* bias,
- device T* output,
- device const int* lengths,
- const constant int& cin,
- const constant int& cout,
- const constant int& seq_len,
- const constant int& kernel_size,
- const constant int& dilation,
- const constant int& batch_size,
- uint t AXIS(seq_len, 32),
- uint oc AXIS(cout, 1),
- uint b AXIS(batch_size, 1)
-) {
- audio_codec_causal_conv1d<T>(
- input,
- weight,
- bias,
- output,
- lengths,
- cin,
- cout,
- seq_len,
- kernel_size,
- dilation,
- uint3(t, oc, b)
- );
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioCausalConvTranspose1d)(
- device const T* input,
- device const T* weight,
- device const T* bias,
- device T* output,
- device const int* lengths,
- const constant int& cin,
- const constant int& cout,
- const constant int& seq_len_in,
- const constant int& seq_len_out,
- const constant int& stride,
- const constant int& groups,
- const constant int& batch_size,
- uint t_out AXIS(seq_len_out, 32),
- uint oc AXIS(cout, 1),
- uint b AXIS(batch_size, 1)
-) {
- audio_codec_causal_conv_transpose1d<T>(
- input,
- weight,
- bias,
- output,
- lengths,
- cin,
- cout,
- seq_len_in,
- seq_len_out,
- stride,
- groups,
- uint3(t_out, oc, b)
- );
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioHalfSnake)(
- device const T* input,
- device const T* alpha,
- device T* output,
- const constant int& channels,
- const constant int& seq_len,
- const constant int& snake_channels,
- const constant float& negative_slope,
- const constant float& eps,
- const constant int& batch_size,
- uint t AXIS(seq_len, 32),
- uint c AXIS(channels, 1),
- uint b AXIS(batch_size, 1)
-) {
- audio_codec_half_snake<T>(
- input,
- alpha,
- output,
- channels,
- seq_len,
- snake_channels,
- negative_slope,
- eps,
- uint3(t, c, b)
- );
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioClamp)(
- device const T* input,
- device T* output,
- const constant int& n,
- const constant float& min_value,
- const constant float& max_value,
- uint tid AXIS(n, 256)
-) {
- audio_codec_clamp<T>(input, output, n, min_value, max_value, tid);
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioConv1d)(
- device const T* input,
- device const T* weight,
- device const T* bias,
- device T* output,
- device const int* lengths,
- const constant int& cin,
- const constant int& cout,
- const constant int& seq_len_in,
- const constant int& seq_len_out,
- const constant int& kernel_size,
- const constant int& stride,
- const constant int& dilation,
- const constant int& padding,
- const constant int& pad_mode,
- const constant int& batch_size,
- uint t_out AXIS(seq_len_out, 32),
- uint oc AXIS(cout, 1),
- uint b AXIS(batch_size, 1)
-) {
- audio_codec_conv1d<T>(
- input,
- weight,
- bias,
- output,
- lengths,
- cin,
- cout,
- seq_len_in,
- seq_len_out,
- kernel_size,
- stride,
- dilation,
- padding,
- pad_mode,
- uint3(t_out, oc, b)
- );
-}
-
-template <typename T>
-VARIANTS(T, float, half, bfloat)
-PUBLIC KERNEL(AudioFsqEncode)(
- device const T* input,
- device int* tokens,
- device const int* lengths,
- const constant int& num_groups,
- const constant int& seq_len,
- const constant int& codebook_dim,
- const constant int* num_levels,
- const constant int* dim_base_index,
- const constant float& eps,
- const constant int& batch_size,
- uint t AXIS(seq_len, 32),
- uint g AXIS(num_groups, 1),
- uint b AXIS(batch_size, 1)
-) {
- fsq_encode<T>(
- input,
- tokens,
- lengths,
- num_groups,
- seq_len,
- codebook_dim,
- num_levels,
- dim_base_index,
- eps,
- uint3(t, g, b)
- );
-}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d.metal
new file mode 100644
index 000000000..cfcd6e82f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d.metal
@@ -0,0 +1,168 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+constant uint AUDIO_TIME_TILE = 4;
+constant int AUDIO_LAYOUT_NCS = 0;
+constant int AUDIO_LAYOUT_NSC = 1;
+
+template <typename T>
+void causal_conv1d(
+ device const T* input, // [B, Cin, T] when NCS, [B, T, Cin] when NSC
+ device const T* weight, // [Cout, Cin, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, T]
+ device const int* lengths, // [B]
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& input_layout,
+ const uint3 gid
+) {
+ const uint t = gid.x * AUDIO_TIME_TILE;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || oc >= (uint)cout) {
+ return;
+ }
+
+ if (input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;
+ int lane_count = seq_len - (int)t;
+ if (lane_count > (int)AUDIO_TIME_TILE) {
+ lane_count = (int)AUDIO_TIME_TILE;
+ }
+ if (lane_count <= 0) {
+ return;
+ }
+ int valid_count = len_b - (int)t;
+ if (valid_count > lane_count) {
+ valid_count = lane_count;
+ }
+ if (valid_count < 0) {
+ valid_count = 0;
+ }
+
+ const int pad = (kernel_size - 1) * dilation;
+ const float bias_value = float(bias[oc]);
+ float acc0 = bias_value;
+ float acc1 = bias_value;
+ float acc2 = bias_value;
+ float acc3 = bias_value;
+
+ if (valid_count > 0) {
+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;
+ for (int ic = 0; ic < cin; ++ic) {
+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;
+ const uint x_base_ncs = (b * (uint)cin + (uint)ic) * (uint)seq_len;
+ for (int k = 0; k < kernel_size; ++k) {
+ const float w = float(weight[w_base + (uint)k]);
+ const int x_t = (int)t + k * dilation - pad;
+ if (full_tile && x_t >= 0 && x_t + 3 < seq_len) {
+ if (input_layout == AUDIO_LAYOUT_NCS) {
+ const uint x_idx = x_base_ncs + (uint)x_t;
+ acc0 += w * float(input[x_idx]);
+ acc1 += w * float(input[x_idx + 1]);
+ acc2 += w * float(input[x_idx + 2]);
+ acc3 += w * float(input[x_idx + 3]);
+ } else {
+ const uint x_idx0 =
+ (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
+ acc0 += w * float(input[x_idx0]);
+ acc1 += w * float(input[x_idx0 + (uint)cin]);
+ acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);
+ acc3 += w * float(input[x_idx0 + (uint)(3 * cin)]);
+ }
+ continue;
+ }
+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {
+ const uint x_idx =
+ (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)x_t)
+ : ((b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic);
+ acc0 += w * float(input[x_idx]);
+ }
+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {
+ const uint x_idx =
+ (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)(x_t + 1))
+ : ((b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin +
+ (uint)ic);
+ acc1 += w * float(input[x_idx]);
+ }
+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {
+ const uint x_idx =
+ (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)(x_t + 2))
+ : ((b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin +
+ (uint)ic);
+ acc2 += w * float(input[x_idx]);
+ }
+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {
+ const uint x_idx =
+ (input_layout == AUDIO_LAYOUT_NCS)
+ ? (x_base_ncs + (uint)(x_t + 3))
+ : ((b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin +
+ (uint)ic);
+ acc3 += w * float(input[x_idx]);
+ }
+ }
+ }
+ }
+
+ if (lane_count > 0) {
+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;
+ }
+ if (lane_count > 1) {
+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;
+ }
+ if (lane_count > 2) {
+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;
+ }
+ if (lane_count > 3) {
+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;
+ }
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioCausalConv1d)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& input_layout,
+ const constant int& batch_size,
+ uint t AXIS((seq_len + 3) / 4, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ causal_conv1d<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len,
+ kernel_size,
+ dilation,
+ input_layout,
+ uint3(t, oc, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal
new file mode 100644
index 000000000..8b6840366
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal
@@ -0,0 +1,361 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+constant uint AUDIO_TIME_TILE = 8;
+constant int AUDIO_LAYOUT_NCS = 0;
+constant int AUDIO_LAYOUT_NSC = 1;
+
+template <typename T>
+void causal_conv1d_grouped_ncs(
+ device const T* input, // [B, Cin, T]
+ device const T* weight, // [Cout, Cin/groups, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, T]
+ device const int* lengths, // [B]
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& groups,
+ const uint3 gid
+) {
+ const uint t = gid.x * AUDIO_TIME_TILE;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || oc >= (uint)cout) {
+ return;
+ }
+
+ if (groups <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;
+ int lane_count = seq_len - (int)t;
+ if (lane_count > (int)AUDIO_TIME_TILE) {
+ lane_count = (int)AUDIO_TIME_TILE;
+ }
+ if (lane_count <= 0) {
+ return;
+ }
+ int valid_count = len_b - (int)t;
+ if (valid_count > lane_count) {
+ valid_count = lane_count;
+ }
+ if (valid_count < 0) {
+ valid_count = 0;
+ }
+
+ const int cin_per_group = cin / groups;
+ const int cout_per_group = cout / groups;
+ const int group_idx = (int)oc / cout_per_group;
+ const int oc_in_group = (int)oc - group_idx * cout_per_group;
+ const int in_begin = group_idx * cin_per_group;
+
+ const int pad = (kernel_size - 1) * dilation;
+ const float bias_value = float(bias[oc]);
+ float acc0 = bias_value;
+ float acc1 = bias_value;
+ float acc2 = bias_value;
+ float acc3 = bias_value;
+ float acc4 = bias_value;
+ float acc5 = bias_value;
+ float acc6 = bias_value;
+ float acc7 = bias_value;
+
+ if (valid_count > 0) {
+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;
+ for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {
+ const uint w_base =
+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +
+ (uint)ic_local) *
+ (uint)kernel_size;
+ const int ic = in_begin + ic_local;
+ const uint x_base_ncs = (b * (uint)cin + (uint)ic) * (uint)seq_len;
+
+ for (int k = 0; k < kernel_size; ++k) {
+ const float w = float(weight[w_base + (uint)k]);
+ const int x_t = (int)t + k * dilation - pad;
+ if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {
+ // Fast path: all 8 lanes in bounds, use vectorized loads
+ const uint x_idx = x_base_ncs + (uint)x_t;
+ vec<T, 4> v0 = *(device const vec<T, 4>*)(input + x_idx);
+ vec<T, 4> v1 = *(device const vec<T, 4>*)(input + x_idx + 4);
+ acc0 += w * float(v0.x);
+ acc1 += w * float(v0.y);
+ acc2 += w * float(v0.z);
+ acc3 += w * float(v0.w);
+ acc4 += w * float(v1.x);
+ acc5 += w * float(v1.y);
+ acc6 += w * float(v1.z);
+ acc7 += w * float(v1.w);
+ continue;
+ }
+ // Slow path: per-lane boundary checks
+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {
+ acc0 += w * float(input[x_base_ncs + (uint)x_t]);
+ }
+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {
+ acc1 += w * float(input[x_base_ncs + (uint)(x_t + 1)]);
+ }
+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {
+ acc2 += w * float(input[x_base_ncs + (uint)(x_t + 2)]);
+ }
+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {
+ acc3 += w * float(input[x_base_ncs + (uint)(x_t + 3)]);
+ }
+ if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {
+ acc4 += w * float(input[x_base_ncs + (uint)(x_t + 4)]);
+ }
+ if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {
+ acc5 += w * float(input[x_base_ncs + (uint)(x_t + 5)]);
+ }
+ if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {
+ acc6 += w * float(input[x_base_ncs + (uint)(x_t + 6)]);
+ }
+ if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {
+ acc7 += w * float(input[x_base_ncs + (uint)(x_t + 7)]);
+ }
+ }
+ }
+ }
+
+ if (lane_count > 0) {
+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;
+ }
+ if (lane_count > 1) {
+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;
+ }
+ if (lane_count > 2) {
+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;
+ }
+ if (lane_count > 3) {
+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;
+ }
+ if (lane_count > 4) {
+ output[out_base + 4] = valid_count > 4 ? (T)acc4 : (T)0;
+ }
+ if (lane_count > 5) {
+ output[out_base + 5] = valid_count > 5 ? (T)acc5 : (T)0;
+ }
+ if (lane_count > 6) {
+ output[out_base + 6] = valid_count > 6 ? (T)acc6 : (T)0;
+ }
+ if (lane_count > 7) {
+ output[out_base + 7] = valid_count > 7 ? (T)acc7 : (T)0;
+ }
+}
+
+template <typename T>
+void causal_conv1d_grouped_nsc(
+ device const T* input, // [B, T, Cin]
+ device const T* weight, // [Cout, Cin/groups, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, T]
+ device const int* lengths, // [B]
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& groups,
+ const uint3 gid
+) {
+ const uint t = gid.x * AUDIO_TIME_TILE;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || oc >= (uint)cout) {
+ return;
+ }
+
+ if (groups <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;
+ int lane_count = seq_len - (int)t;
+ if (lane_count > (int)AUDIO_TIME_TILE) {
+ lane_count = (int)AUDIO_TIME_TILE;
+ }
+ if (lane_count <= 0) {
+ return;
+ }
+ int valid_count = len_b - (int)t;
+ if (valid_count > lane_count) {
+ valid_count = lane_count;
+ }
+ if (valid_count < 0) {
+ valid_count = 0;
+ }
+
+ const int cin_per_group = cin / groups;
+ const int cout_per_group = cout / groups;
+ const int group_idx = (int)oc / cout_per_group;
+ const int oc_in_group = (int)oc - group_idx * cout_per_group;
+ const int in_begin = group_idx * cin_per_group;
+
+ const int pad = (kernel_size - 1) * dilation;
+ const float bias_value = float(bias[oc]);
+ float acc0 = bias_value;
+ float acc1 = bias_value;
+ float acc2 = bias_value;
+ float acc3 = bias_value;
+ float acc4 = bias_value;
+ float acc5 = bias_value;
+ float acc6 = bias_value;
+ float acc7 = bias_value;
+
+ if (valid_count > 0) {
+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;
+ for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {
+ const int ic = in_begin + ic_local;
+ const uint w_base =
+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +
+ (uint)ic_local) *
+ (uint)kernel_size;
+
+ for (int k = 0; k < kernel_size; ++k) {
+ const float w = float(weight[w_base + (uint)k]);
+ const int x_t = (int)t + k * dilation - pad;
+ if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {
+ const uint x_idx0 =
+ (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
+ acc0 += w * float(input[x_idx0]);
+ acc1 += w * float(input[x_idx0 + (uint)cin]);
+ acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);
+ acc3 += w * float(input[x_idx0 + (uint)(3 * cin)]);
+ acc4 += w * float(input[x_idx0 + (uint)(4 * cin)]);
+ acc5 += w * float(input[x_idx0 + (uint)(5 * cin)]);
+ acc6 += w * float(input[x_idx0 + (uint)(6 * cin)]);
+ acc7 += w * float(input[x_idx0 + (uint)(7 * cin)]);
+ continue;
+ }
+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;
+ acc0 += w * float(input[x_idx]);
+ }
+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic;
+ acc1 += w * float(input[x_idx]);
+ }
+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic;
+ acc2 += w * float(input[x_idx]);
+ }
+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic;
+ acc3 += w * float(input[x_idx]);
+ }
+ if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)(x_t + 4)) * (uint)cin + (uint)ic;
+ acc4 += w * float(input[x_idx]);
+ }
+ if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)(x_t + 5)) * (uint)cin + (uint)ic;
+ acc5 += w * float(input[x_idx]);
+ }
+ if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)(x_t + 6)) * (uint)cin + (uint)ic;
+ acc6 += w * float(input[x_idx]);
+ }
+ if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {
+ const uint x_idx =
+ (b * (uint)seq_len + (uint)(x_t + 7)) * (uint)cin + (uint)ic;
+ acc7 += w * float(input[x_idx]);
+ }
+ }
+ }
+ }
+
+ if (lane_count > 0) {
+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;
+ }
+ if (lane_count > 1) {
+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;
+ }
+ if (lane_count > 2) {
+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;
+ }
+ if (lane_count > 3) {
+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;
+ }
+ if (lane_count > 4) {
+ output[out_base + 4] = valid_count > 4 ? (T)acc4 : (T)0;
+ }
+ if (lane_count > 5) {
+ output[out_base + 5] = valid_count > 5 ? (T)acc5 : (T)0;
+ }
+ if (lane_count > 6) {
+ output[out_base + 6] = valid_count > 6 ? (T)acc6 : (T)0;
+ }
+ if (lane_count > 7) {
+ output[out_base + 7] = valid_count > 7 ? (T)acc7 : (T)0;
+ }
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioCausalConv1dGrouped)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& groups,
+ const constant int& input_layout,
+ const constant int& batch_size,
+ uint t AXIS((seq_len + 7) / 8, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ if (input_layout == AUDIO_LAYOUT_NCS) {
+ causal_conv1d_grouped_ncs<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len,
+ kernel_size,
+ dilation,
+ groups,
+ uint3(t, oc, b)
+ );
+ } else {
+ causal_conv1d_grouped_nsc<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len,
+ kernel_size,
+ dilation,
+ groups,
+ uint3(t, oc, b)
+ );
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal
new file mode 100644
index 000000000..bfc694c9f
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal
@@ -0,0 +1,195 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+constant uint AUDIO_TIME_TILE = 8;
+
+template <typename T>
+void causal_conv1d_grouped_residual(
+ device const T* input, // [B, Cin, T]
+ device const T* residual, // [B, Cout, T]
+ device const T* weight, // [Cout, Cin/groups, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, T]
+ device const int* lengths, // [B]
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& groups,
+ const uint3 gid
+) {
+ const uint t = gid.x * AUDIO_TIME_TILE;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || oc >= (uint)cout) {
+ return;
+ }
+
+ if (groups <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;
+ int lane_count = seq_len - (int)t;
+ if (lane_count > (int)AUDIO_TIME_TILE) {
+ lane_count = (int)AUDIO_TIME_TILE;
+ }
+ if (lane_count <= 0) {
+ return;
+ }
+ int valid_count = len_b - (int)t;
+ if (valid_count > lane_count) {
+ valid_count = lane_count;
+ }
+ if (valid_count < 0) {
+ valid_count = 0;
+ }
+
+ const float bias_value = float(bias[oc]);
+ const float residual0 = lane_count > 0 ? float(residual[out_base]) : 0.0f;
+ const float residual1 = lane_count > 1 ? float(residual[out_base + 1]) : 0.0f;
+ const float residual2 = lane_count > 2 ? float(residual[out_base + 2]) : 0.0f;
+ const float residual3 = lane_count > 3 ? float(residual[out_base + 3]) : 0.0f;
+ const float residual4 = lane_count > 4 ? float(residual[out_base + 4]) : 0.0f;
+ const float residual5 = lane_count > 5 ? float(residual[out_base + 5]) : 0.0f;
+ const float residual6 = lane_count > 6 ? float(residual[out_base + 6]) : 0.0f;
+ const float residual7 = lane_count > 7 ? float(residual[out_base + 7]) : 0.0f;
+ float acc0 = bias_value + residual0;
+ float acc1 = bias_value + residual1;
+ float acc2 = bias_value + residual2;
+ float acc3 = bias_value + residual3;
+ float acc4 = bias_value + residual4;
+ float acc5 = bias_value + residual5;
+ float acc6 = bias_value + residual6;
+ float acc7 = bias_value + residual7;
+
+ const int cin_per_group = cin / groups;
+ const int cout_per_group = cout / groups;
+ const int group_idx = (int)oc / cout_per_group;
+ const int oc_in_group = (int)oc - group_idx * cout_per_group;
+ const int in_begin = group_idx * cin_per_group;
+
+ const int pad = (kernel_size - 1) * dilation;
+ if (valid_count > 0) {
+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;
+ for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {
+ const int ic = in_begin + ic_local;
+ const uint w_base =
+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group +
+ (uint)ic_local) *
+ (uint)kernel_size;
+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;
+
+ for (int k = 0; k < kernel_size; ++k) {
+ const float w = float(weight[w_base + (uint)k]);
+ const int x_t = (int)t + k * dilation - pad;
+ if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {
+ // Fast path: all 8 lanes in bounds, use vectorized loads
+ const uint x_idx = x_base + (uint)x_t;
+ vec<T, 4> v0 = *(device const vec<T, 4>*)(input + x_idx);
+ vec<T, 4> v1 = *(device const vec<T, 4>*)(input + x_idx + 4);
+ acc0 += w * float(v0.x);
+ acc1 += w * float(v0.y);
+ acc2 += w * float(v0.z);
+ acc3 += w * float(v0.w);
+ acc4 += w * float(v1.x);
+ acc5 += w * float(v1.y);
+ acc6 += w * float(v1.z);
+ acc7 += w * float(v1.w);
+ continue;
+ }
+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {
+ acc0 += w * float(input[x_base + (uint)x_t]);
+ }
+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {
+ acc1 += w * float(input[x_base + (uint)(x_t + 1)]);
+ }
+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {
+ acc2 += w * float(input[x_base + (uint)(x_t + 2)]);
+ }
+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {
+ acc3 += w * float(input[x_base + (uint)(x_t + 3)]);
+ }
+ if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {
+ acc4 += w * float(input[x_base + (uint)(x_t + 4)]);
+ }
+ if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {
+ acc5 += w * float(input[x_base + (uint)(x_t + 5)]);
+ }
+ if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {
+ acc6 += w * float(input[x_base + (uint)(x_t + 6)]);
+ }
+ if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {
+ acc7 += w * float(input[x_base + (uint)(x_t + 7)]);
+ }
+ }
+ }
+ }
+
+ if (lane_count > 0) {
+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)residual0;
+ }
+ if (lane_count > 1) {
+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)residual1;
+ }
+ if (lane_count > 2) {
+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)residual2;
+ }
+ if (lane_count > 3) {
+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)residual3;
+ }
+ if (lane_count > 4) {
+ output[out_base + 4] = valid_count > 4 ? (T)acc4 : (T)residual4;
+ }
+ if (lane_count > 5) {
+ output[out_base + 5] = valid_count > 5 ? (T)acc5 : (T)residual5;
+ }
+ if (lane_count > 6) {
+ output[out_base + 6] = valid_count > 6 ? (T)acc6 : (T)residual6;
+ }
+ if (lane_count > 7) {
+ output[out_base + 7] = valid_count > 7 ? (T)acc7 : (T)residual7;
+ }
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioCausalConv1dGroupedResidual)(
+ device const T* input,
+ device const T* residual,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& groups,
+ const constant int& batch_size,
+ uint t AXIS((seq_len + 7) / 8, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ causal_conv1d_grouped_residual<T>(
+ input,
+ residual,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len,
+ kernel_size,
+ dilation,
+ groups,
+ uint3(t, oc, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal
new file mode 100644
index 000000000..016716aaa
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal
@@ -0,0 +1,106 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+template <typename T>
+void causal_conv_transpose1d(
+ device const T* input, // [B, Cin, Tin]
+ device const T* weight, // [Cout, Cin_per_group, 2*stride]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, Tout]
+ device const int* lengths, // [B] (output lengths)
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& stride,
+ const constant int& groups,
+ const uint3 gid
+) {
+ const uint t_out = gid.x;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len_out;
+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;
+ if ((int)t_out >= len_b) {
+ output[out_idx] = (T)0;
+ return;
+ }
+
+ const int cout_per_group = cout / groups;
+ const int cin_per_group = cin / groups;
+ const int group_idx = (int)oc / cout_per_group;
+
+ const int q = (int)t_out / stride;
+ const int r = (int)t_out - q * stride;
+
+ float acc = float(bias[oc]);
+
+ const uint in_base_b = b * (uint)cin * (uint)seq_len_in;
+ const int ic_begin = group_idx * cin_per_group;
+ const int ic_end = ic_begin + cin_per_group;
+
+ for (int ic = ic_begin; ic < ic_end; ++ic) {
+ if (q >= seq_len_in) {
+ continue;
+ }
+ const uint in_base = in_base_b + (uint)ic * (uint)seq_len_in;
+
+ // Weight layout: [Cout, Cin_per_group, K]
+ const uint ic_local = (uint)(ic - ic_begin);
+ const uint w_base =
+ ((uint)oc * (uint)cin_per_group + ic_local) * (uint)(2 * stride);
+
+ // Contribution from input[q] with k=r
+ acc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);
+
+ // Contribution from input[q-1] with k=stride+r
+ if (q > 0) {
+ acc += float(weight[w_base + (uint)(stride + r)]) *
+ float(input[in_base + (uint)(q - 1)]);
+ }
+ }
+
+ output[out_idx] = (T)acc;
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioCausalConvTranspose1d)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& stride,
+ const constant int& groups,
+ const constant int& batch_size,
+ uint t_out AXIS(seq_len_out, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ causal_conv_transpose1d<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ stride,
+ groups,
+ uint3(t_out, oc, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal
new file mode 100644
index 000000000..46d3bed81
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal
@@ -0,0 +1,479 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+constant uint AUDIO_TIME_TILE = 4;
+
+// Input layout for causal_conv_transpose1d_causal_pad:
+// 0 = NCS ([B, Cin, Tin]), 1 = NSC ([B, Tin, Cin]).
+constant int AUDIO_LAYOUT_NCS = 0;
+constant int AUDIO_LAYOUT_NSC = 1;
+
+// === Causal ConvTranspose1d (causal-pad semantics, arbitrary kernel_size) ===
+//
+// This matches causal_conv_transpose1d_causal_pad_reference():
+// - Expand input by stride with zeros inserted between samples.
+// - Left pad by (kernel_size - 1), no right pad.
+// - Convolve with provided kernel along expanded timeline.
+template <typename T>
+void causal_conv_transpose1d_causal_pad(
+ device const T* input, // [B, Cin, Tin] when NCS, [B, Tin, Cin] when NSC
+ device const T* weight, // [Cout, Cin_per_group, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, Tout]
+ device const int* lengths, // [B]
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& kernel_size,
+ const constant int& stride,
+ const constant int& groups,
+ const constant int& input_layout,
+ const uint3 gid
+) {
+ const uint t_out = gid.x * AUDIO_TIME_TILE;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
+ return;
+ }
+ if ((input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) ||
+ groups <= 0 || stride <= 0 || kernel_size <= 0 || (cin % groups) != 0 ||
+ (cout % groups) != 0) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len_out;
+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;
+ int lane_count = seq_len_out - (int)t_out;
+ if (lane_count > (int)AUDIO_TIME_TILE) {
+ lane_count = (int)AUDIO_TIME_TILE;
+ }
+ if (lane_count <= 0) {
+ return;
+ }
+ int valid_count = len_b - (int)t_out;
+ if (valid_count > lane_count) {
+ valid_count = lane_count;
+ }
+ if (valid_count < 0) {
+ valid_count = 0;
+ }
+
+ const int cout_per_group = cout / groups;
+ const int cin_per_group = cin / groups;
+ const int group_idx = (int)oc / cout_per_group;
+ const int ic_begin = group_idx * cin_per_group;
+ const int ic_end = ic_begin + cin_per_group;
+
+ const int seq_len_expanded =
+ (seq_len_in > 0) ? (((seq_len_in - 1) * stride) + 1) : 0;
+ const int left_pad = kernel_size - 1;
+ const bool fast_two_tap = (kernel_size == (stride * 2));
+
+ const float bias_value = float(bias[oc]);
+ float acc0 = bias_value;
+ float acc1 = bias_value;
+ float acc2 = bias_value;
+ float acc3 = bias_value;
+
+ // Process each lane that is valid (within sequence length)
+ if (valid_count > 0) {
+ // Precompute q and r for lane 0; consecutive lanes have t_out+1, t_out+2,
+ // t_out+3.
+ const int t0 = (int)t_out;
+ const int q0 = t0 / stride;
+ const int r0 = t0 % stride;
+
+ if (input_layout == AUDIO_LAYOUT_NCS) {
+ for (int ic = ic_begin; ic < ic_end; ++ic) {
+ const uint in_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;
+ const uint ic_local = (uint)(ic - ic_begin);
+ const uint w_base =
+ ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;
+ if (fast_two_tap) {
+ // When stride >= AUDIO_TIME_TILE (common for decoder blocks with
+ // strides 4,8), all 4 lanes share the same q value. This means
+ // they all read from the same input positions (q and q-1), just
+ // with different weight indices. We load each input value once
+ // and broadcast it across lanes.
+ if (stride >= (int)AUDIO_TIME_TILE) {
+ // All lanes have same q since t_out+3 < t_out+stride means
+ // (t_out+3)/stride == t_out/stride.
+ const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in)
+ ? float(input[in_base + (uint)(q0 - 1)])
+ : 0.0f;
+ const float in_q0 = (q0 >= 0 && q0 < seq_len_in)
+ ? float(input[in_base + (uint)q0])
+ : 0.0f;
+ // Lane 0: r = r0
+ if (valid_count > 0) {
+ acc0 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);
+ acc0 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
+ }
+ // Lane 1: r = r0+1
+ if (valid_count > 1) {
+ const int r1 = r0 + 1;
+ acc1 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);
+ acc1 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ }
+ // Lane 2: r = r0+2
+ if (valid_count > 2) {
+ const int r2 = r0 + 2;
+ acc2 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);
+ acc2 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ }
+ // Lane 3: r = r0+3
+ if (valid_count > 3) {
+ const int r3 = r0 + 3;
+ acc3 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);
+ acc3 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ }
+ } else {
+ // stride < 4 (e.g., stride=2 for upsampler blocks).
+ // Lanes may cross stride boundaries (different q values).
+ if (valid_count > 0) {
+ const int k_lo = (stride - 1) - r0;
+ const int k_hi = (kernel_size - 1) - r0;
+ if (q0 > 0 && q0 - 1 < seq_len_in) {
+ acc0 += float(input[in_base + (uint)(q0 - 1)]) *
+ float(weight[w_base + (uint)k_lo]);
+ }
+ if (q0 >= 0 && q0 < seq_len_in) {
+ acc0 += float(input[in_base + (uint)q0]) *
+ float(weight[w_base + (uint)k_hi]);
+ }
+ }
+ if (valid_count > 1) {
+ const int q1 = (t0 + 1) / stride;
+ const int r1 = (t0 + 1) % stride;
+ if (q1 > 0 && q1 - 1 < seq_len_in) {
+ acc1 += float(input[in_base + (uint)(q1 - 1)]) *
+ float(weight[w_base + (uint)((stride - 1) - r1)]);
+ }
+ if (q1 >= 0 && q1 < seq_len_in) {
+ acc1 += float(input[in_base + (uint)q1]) *
+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ }
+ }
+ if (valid_count > 2) {
+ const int q2 = (t0 + 2) / stride;
+ const int r2 = (t0 + 2) % stride;
+ if (q2 > 0 && q2 - 1 < seq_len_in) {
+ acc2 += float(input[in_base + (uint)(q2 - 1)]) *
+ float(weight[w_base + (uint)((stride - 1) - r2)]);
+ }
+ if (q2 >= 0 && q2 < seq_len_in) {
+ acc2 += float(input[in_base + (uint)q2]) *
+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ }
+ }
+ if (valid_count > 3) {
+ const int q3 = (t0 + 3) / stride;
+ const int r3 = (t0 + 3) % stride;
+ if (q3 > 0 && q3 - 1 < seq_len_in) {
+ acc3 += float(input[in_base + (uint)(q3 - 1)]) *
+ float(weight[w_base + (uint)((stride - 1) - r3)]);
+ }
+ if (q3 >= 0 && q3 < seq_len_in) {
+ acc3 += float(input[in_base + (uint)q3]) *
+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ }
+ }
+ }
+ } else {
+ for (int k = 0; k < kernel_size; ++k) {
+ const float w = float(weight[w_base + (uint)k]);
+ // Lane 0
+ if (valid_count > 0) {
+ const int expanded_time0 = (int)t_out + k - left_pad;
+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&
+ (expanded_time0 % stride) == 0) {
+ const int src_time0 = expanded_time0 / stride;
+ if (src_time0 >= 0 && src_time0 < seq_len_in) {
+ acc0 += float(input[in_base + (uint)src_time0]) * w;
+ }
+ }
+ }
+ // Lane 1
+ if (valid_count > 1) {
+ const int expanded_time1 = (int)t_out + 1 + k - left_pad;
+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&
+ (expanded_time1 % stride) == 0) {
+ const int src_time1 = expanded_time1 / stride;
+ if (src_time1 >= 0 && src_time1 < seq_len_in) {
+ acc1 += float(input[in_base + (uint)src_time1]) * w;
+ }
+ }
+ }
+ // Lane 2
+ if (valid_count > 2) {
+ const int expanded_time2 = (int)t_out + 2 + k - left_pad;
+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&
+ (expanded_time2 % stride) == 0) {
+ const int src_time2 = expanded_time2 / stride;
+ if (src_time2 >= 0 && src_time2 < seq_len_in) {
+ acc2 += float(input[in_base + (uint)src_time2]) * w;
+ }
+ }
+ }
+ // Lane 3
+ if (valid_count > 3) {
+ const int expanded_time3 = (int)t_out + 3 + k - left_pad;
+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&
+ (expanded_time3 % stride) == 0) {
+ const int src_time3 = expanded_time3 / stride;
+ if (src_time3 >= 0 && src_time3 < seq_len_in) {
+ acc3 += float(input[in_base + (uint)src_time3]) * w;
+ }
+ }
+ }
+ }
+ }
+ }
+ } else {
+ // NSC layout
+ for (int ic = ic_begin; ic < ic_end; ++ic) {
+ const uint ic_local = (uint)(ic - ic_begin);
+ const uint w_base =
+ ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;
+ if (fast_two_tap) {
+ if (stride >= (int)AUDIO_TIME_TILE) {
+ // All lanes share q0; load input once per (q-1, q) pair.
+ const float in_qm1 =
+ (q0 > 0 && q0 - 1 < seq_len_in)
+ ? float(input
+ [(b * (uint)seq_len_in + (uint)(q0 - 1)) *
+ (uint)cin +
+ (uint)ic])
+ : 0.0f;
+ const float in_q0 =
+ (q0 >= 0 && q0 < seq_len_in)
+ ? float(input
+ [(b * (uint)seq_len_in + (uint)q0) * (uint)cin +
+ (uint)ic])
+ : 0.0f;
+ if (valid_count > 0) {
+ acc0 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);
+ acc0 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
+ }
+ if (valid_count > 1) {
+ const int r1 = r0 + 1;
+ acc1 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);
+ acc1 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ }
+ if (valid_count > 2) {
+ const int r2 = r0 + 2;
+ acc2 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);
+ acc2 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ }
+ if (valid_count > 3) {
+ const int r3 = r0 + 3;
+ acc3 +=
+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);
+ acc3 += in_q0 *
+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ }
+ } else {
+ // stride < 4: lanes may have different q values.
+ if (valid_count > 0) {
+ if (q0 > 0 && q0 - 1 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin +
+ (uint)ic;
+ acc0 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((stride - 1) - r0)]);
+ }
+ if (q0 >= 0 && q0 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic;
+ acc0 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((kernel_size - 1) - r0)]);
+ }
+ }
+ if (valid_count > 1) {
+ const int q1 = (t0 + 1) / stride;
+ const int r1 = (t0 + 1) % stride;
+ if (q1 > 0 && q1 - 1 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)(q1 - 1)) * (uint)cin +
+ (uint)ic;
+ acc1 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((stride - 1) - r1)]);
+ }
+ if (q1 >= 0 && q1 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)q1) * (uint)cin + (uint)ic;
+ acc1 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);
+ }
+ }
+ if (valid_count > 2) {
+ const int q2 = (t0 + 2) / stride;
+ const int r2 = (t0 + 2) % stride;
+ if (q2 > 0 && q2 - 1 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)(q2 - 1)) * (uint)cin +
+ (uint)ic;
+ acc2 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((stride - 1) - r2)]);
+ }
+ if (q2 >= 0 && q2 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)q2) * (uint)cin + (uint)ic;
+ acc2 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);
+ }
+ }
+ if (valid_count > 3) {
+ const int q3 = (t0 + 3) / stride;
+ const int r3 = (t0 + 3) % stride;
+ if (q3 > 0 && q3 - 1 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)(q3 - 1)) * (uint)cin +
+ (uint)ic;
+ acc3 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((stride - 1) - r3)]);
+ }
+ if (q3 >= 0 && q3 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)q3) * (uint)cin + (uint)ic;
+ acc3 += float(input[in_idx]) *
+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);
+ }
+ }
+ }
+ } else {
+ for (int k = 0; k < kernel_size; ++k) {
+ const float w = float(weight[w_base + (uint)k]);
+ // Lane 0
+ if (valid_count > 0) {
+ const int expanded_time0 = (int)t_out + k - left_pad;
+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&
+ (expanded_time0 % stride) == 0) {
+ const int src_time0 = expanded_time0 / stride;
+ if (src_time0 >= 0 && src_time0 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)src_time0) * (uint)cin +
+ (uint)ic;
+ acc0 += float(input[in_idx]) * w;
+ }
+ }
+ }
+ // Lane 1
+ if (valid_count > 1) {
+ const int expanded_time1 = (int)t_out + 1 + k - left_pad;
+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&
+ (expanded_time1 % stride) == 0) {
+ const int src_time1 = expanded_time1 / stride;
+ if (src_time1 >= 0 && src_time1 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)src_time1) * (uint)cin +
+ (uint)ic;
+ acc1 += float(input[in_idx]) * w;
+ }
+ }
+ }
+ // Lane 2
+ if (valid_count > 2) {
+ const int expanded_time2 = (int)t_out + 2 + k - left_pad;
+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&
+ (expanded_time2 % stride) == 0) {
+ const int src_time2 = expanded_time2 / stride;
+ if (src_time2 >= 0 && src_time2 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)src_time2) * (uint)cin +
+ (uint)ic;
+ acc2 += float(input[in_idx]) * w;
+ }
+ }
+ }
+ // Lane 3
+ if (valid_count > 3) {
+ const int expanded_time3 = (int)t_out + 3 + k - left_pad;
+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&
+ (expanded_time3 % stride) == 0) {
+ const int src_time3 = expanded_time3 / stride;
+ if (src_time3 >= 0 && src_time3 < seq_len_in) {
+ const uint in_idx =
+ (b * (uint)seq_len_in + (uint)src_time3) * (uint)cin +
+ (uint)ic;
+ acc3 += float(input[in_idx]) * w;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (lane_count > 0) {
+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;
+ }
+ if (lane_count > 1) {
+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;
+ }
+ if (lane_count > 2) {
+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;
+ }
+ if (lane_count > 3) {
+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;
+ }
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioCausalConvTranspose1dCausalPad)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& kernel_size,
+ const constant int& stride,
+ const constant int& groups,
+ const constant int& input_layout,
+ const constant int& batch_size,
+ uint t_out AXIS((seq_len_out + 3) / 4, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ causal_conv_transpose1d_causal_pad<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ kernel_size,
+ stride,
+ groups,
+ input_layout,
+ uint3(t_out, oc, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/conv1d.metal b/crates/uzu/src/backends/metal/kernel/audio/conv1d.metal
new file mode 100644
index 000000000..33e61d901
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/conv1d.metal
@@ -0,0 +1,100 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+template <typename T>
+void conv1d(
+ device const T* input, // [B, Cin, Tin]
+ device const T* weight, // [Cout, Cin, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, Tout]
+ device const int* lengths, // [B] (Tout lengths)
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& kernel_size,
+ const constant int& stride,
+ const constant int& dilation,
+ const constant int& padding,
+ const constant int& pad_mode,
+ const uint3 gid
+) {
+ const uint t_out = gid.x;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len_out;
+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;
+ if ((int)t_out >= len_b) {
+ output[out_idx] = (T)0;
+ return;
+ }
+
+ const int base = (int)t_out * stride - padding;
+ float acc = float(bias[oc]);
+
+ for (int ic = 0; ic < cin; ++ic) {
+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;
+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;
+ for (int k = 0; k < kernel_size; ++k) {
+ int x_t = base + k * dilation;
+ if (pad_mode == 0) {
+ if (x_t < 0 || x_t >= seq_len_in) {
+ continue;
+ }
+ } else {
+ x_t = clamp(x_t, 0, seq_len_in - 1);
+ }
+ acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);
+ }
+ }
+
+ output[out_idx] = (T)acc;
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioConv1d)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& kernel_size,
+ const constant int& stride,
+ const constant int& dilation,
+ const constant int& padding,
+ const constant int& pad_mode,
+ const constant int& batch_size,
+ uint t_out AXIS(seq_len_out, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ conv1d<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ kernel_size,
+ stride,
+ dilation,
+ padding,
+ pad_mode,
+ uint3(t_out, oc, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/fsq_decode.metal b/crates/uzu/src/backends/metal/kernel/audio/fsq_decode.metal
new file mode 100644
index 000000000..95e3fb2db
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/fsq_decode.metal
@@ -0,0 +1,83 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+template <typename T>
+void fsq_decode(
+ device const int* tokens,
+ device T* out,
+ device const int* lengths,
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const constant int* dim_base_index,
+ const uint3 gid
+) {
+ const uint t = gid.x;
+ const uint gd = gid.y;
+ const uint b = gid.z;
+ const uint channels = (uint)(num_groups * codebook_dim);
+
+ if (t >= (uint)seq_len || gd >= channels) {
+ return;
+ }
+
+ const uint g = gd / (uint)codebook_dim;
+ const uint d = gd % (uint)codebook_dim;
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_idx = (b * channels + gd) * (uint)seq_len + t;
+ if ((int)t >= len_b) {
+ out[out_idx] = (T)0;
+ return;
+ }
+
+ const uint token_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;
+ const int token = tokens[token_idx];
+
+ const int levels = num_levels[d];
+ const int scale = levels / 2; // integer division (matches PyTorch // 2)
+ const int offset = scale;
+ const int base = dim_base_index[d];
+
+ // Euclidean modulo to keep result in [0, levels) even if token is negative
+ // (tokens are expected nonnegative, but keep this robust).
+ const int div = token / base;
+ int code_nonneg = div % levels;
+ if (code_nonneg < 0) {
+ code_nonneg += levels;
+ }
+ const float code = ((float)(code_nonneg - offset)) / (float)scale;
+ out[out_idx] = (T)code;
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioFsqDecode)(
+ device const int* tokens,
+ device T* out,
+ device const int* lengths,
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const constant int* dim_base_index,
+ const constant int& batch_size,
+ uint t AXIS(seq_len, 32),
+ uint gd AXIS(num_groups * codebook_dim, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ fsq_decode<T>(
+ tokens,
+ out,
+ lengths,
+ num_groups,
+ seq_len,
+ codebook_dim,
+ num_levels,
+ dim_base_index,
+ uint3(t, gd, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/fsq_encode.metal b/crates/uzu/src/backends/metal/kernel/audio/fsq_encode.metal
new file mode 100644
index 000000000..34320fcae
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/fsq_encode.metal
@@ -0,0 +1,102 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+inline float round_ties_to_even(const float x) {
+ const float f = floor(x);
+ const float frac = x - f;
+ if (frac < 0.5f) {
+ return f;
+ }
+ if (frac > 0.5f) {
+ return f + 1.0f;
+ }
+ const int fi = (int)f; // f is integral
+ return ((fi & 1) != 0) ? (f + 1.0f) : f;
+}
+
+template <typename T>
+void fsq_encode(
+ device const T* input, // [B, G*D, T]
+ device int* tokens, // [B, G, T]
+ device const int* lengths, // [B]
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const constant int* dim_base_index,
+ const constant float& eps,
+ const uint3 gid
+) {
+ const uint t = gid.x;
+ const uint g = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || g >= (uint)num_groups) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;
+ if ((int)t >= len_b) {
+ tokens[out_idx] = 0;
+ return;
+ }
+
+ const uint in_base =
+ (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *
+ (uint)seq_len +
+ t;
+
+ int token = 0;
+ for (int d = 0; d < codebook_dim; ++d) {
+ const int levels = num_levels[d];
+ const int scale_i = levels / 2;
+ const float output_scale = ((float)(levels - 1)) * 0.5f * (1.0f - eps);
+ const float output_offset = (levels % 2 == 0) ? 0.5f : 0.0f;
+ const float input_shift = tan(output_offset / output_scale);
+
+ const float x = float(input[in_base + (uint)d * (uint)seq_len]);
+ const float compressed =
+ output_scale * tanh(x + input_shift) - output_offset;
+ const float rounded = round_ties_to_even(compressed);
+
+ int code_nonneg = (int)rounded + scale_i;
+ code_nonneg = clamp(code_nonneg, 0, levels - 1);
+ token += code_nonneg * dim_base_index[d];
+ }
+
+ tokens[out_idx] = token;
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioFsqEncode)(
+ device const T* input,
+ device int* tokens,
+ device const int* lengths,
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const constant int* dim_base_index,
+ const constant float& eps,
+ const constant int& batch_size,
+ uint t AXIS(seq_len, 32),
+ uint g AXIS(num_groups, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ fsq_encode<T>(
+ input,
+ tokens,
+ lengths,
+ num_groups,
+ seq_len,
+ codebook_dim,
+ num_levels,
+ dim_base_index,
+ eps,
+ uint3(t, g, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/half_snake.metal b/crates/uzu/src/backends/metal/kernel/audio/half_snake.metal
new file mode 100644
index 000000000..84e866842
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/half_snake.metal
@@ -0,0 +1,68 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+#include "../activation/activations.h"
+
+using namespace metal;
+
+template <typename T>
+void half_snake(
+ device const T* input, // [B, C, T]
+ device const T* alpha, // [1, C_snake, 1] (contiguous, index by channel)
+ device T* output, // [B, C, T]
+ const constant int& channels,
+ const constant int& seq_len,
+ const constant int& snake_channels,
+ const constant float& negative_slope,
+ const constant float& eps,
+ const uint3 gid
+) {
+ const uint t = gid.x;
+ const uint c = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || c >= (uint)channels) {
+ return;
+ }
+
+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;
+ const float x = float(input[idx]);
+
+ if ((int)c < snake_channels) {
+ const float a = float(alpha[c]);
+ const float ax = a * x;
+ const float s = sin(ax);
+ const float y = x + (1.0f / (a + eps)) * (s * s);
+ output[idx] = (T)y;
+ } else {
+ output[idx] = activate_leaky_relu((T)x, negative_slope);
+ }
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioHalfSnake)(
+ device const T* input,
+ device const T* alpha,
+ device T* output,
+ const constant int& channels,
+ const constant int& seq_len,
+ const constant int& snake_channels,
+ const constant float& negative_slope,
+ const constant float& eps,
+ const constant int& batch_size,
+ uint t AXIS(seq_len, 32),
+ uint c AXIS(channels, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ half_snake<T>(
+ input,
+ alpha,
+ output,
+ channels,
+ seq_len,
+ snake_channels,
+ negative_slope,
+ eps,
+ uint3(t, c, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal
new file mode 100644
index 000000000..46d59eb11
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal
@@ -0,0 +1,123 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+#include "../common/thread_context.h"
+#include "../common/threadgroup_reduce.h"
+
+using namespace metal;
+
+#define AUDIO_NORM_NCS_BLOCK_SIZE 256
+#define AUDIO_NORM_NCS_MAX_SIMDS 32
+
+template <typename T>
+void norm_ncs(
+ device const T* input, // [B, C, T]
+ device const T* scales, // [C]
+ device const T* bias, // [C]
+ device T* output, // [B, C, T]
+ device const int* lengths, // [B]
+ const constant int& channels,
+ const constant int& seq_len,
+ const constant float& epsilon,
+ const constant int& subtract_mean,
+ threadgroup float* shared_mean,
+ threadgroup float* shared_variance,
+ const thread ThreadContext& thread_context,
+ const uint b,
+ const uint t,
+ const uint lid
+) {
+ if (t >= (uint)seq_len || channels <= 0) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+
+ if ((int)t >= len_b) {
+ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {
+ const uint out_idx = (b * (uint)channels + c) * (uint)seq_len + t;
+ output[out_idx] = (T)0;
+ }
+ return;
+ }
+
+ float partial_sum = 0.0f;
+ if (subtract_mean != 0) {
+ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {
+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;
+ partial_sum += float(input[idx]);
+ }
+ }
+
+ const float sum =
+ threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(
+ partial_sum,
+ shared_mean,
+ (ushort)lid,
+ thread_context
+ );
+ const float mean = (subtract_mean != 0) ? (sum / (float)channels) : 0.0f;
+
+ float partial_variance = 0.0f;
+ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {
+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;
+ const float x = float(input[idx]);
+ const float centered = (subtract_mean != 0) ? (x - mean) : x;
+ partial_variance += centered * centered;
+ }
+
+ const float variance_sum =
+ threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(
+ partial_variance,
+ shared_variance,
+ (ushort)lid,
+ thread_context
+ );
+ const float inv_std = rsqrt(variance_sum / (float)channels + epsilon);
+
+ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {
+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;
+ const float x = float(input[idx]);
+ const float centered = (subtract_mean != 0) ? (x - mean) : x;
+ const float y = centered * inv_std * float(scales[c]) + float(bias[c]);
+ output[idx] = (T)y;
+ }
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioNormNcs)(
+ device const T* input,
+ device const T* scales,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& channels,
+ const constant int& seq_len,
+ const constant float& epsilon,
+ const constant int& subtract_mean,
+ const constant int& batch_size,
+ threadgroup float shared_mean[AUDIO_NORM_NCS_MAX_SIMDS],
+ threadgroup float shared_variance[AUDIO_NORM_NCS_MAX_SIMDS],
+ const ThreadContext thread_context,
+ const uint b GROUPS(batch_size),
+ const uint t GROUPS(seq_len),
+ const uint lid THREADS(AUDIO_NORM_NCS_BLOCK_SIZE)
+) {
+ norm_ncs<T>(
+ input,
+ scales,
+ bias,
+ output,
+ lengths,
+ channels,
+ seq_len,
+ epsilon,
+ subtract_mean,
+ shared_mean,
+ shared_variance,
+ thread_context,
+ b,
+ t,
+ lid
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/audio/quantizer_decode.metal b/crates/uzu/src/backends/metal/kernel/audio/quantizer_decode.metal
new file mode 100644
index 000000000..ec147ca1d
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/quantizer_decode.metal
@@ -0,0 +1,120 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+using namespace metal;
+
+template <typename T>
+void quantizer_decode(
+ device const uint* tokens, // [B, K, T]
+ device const int* lengths, // [B]
+ device const T* semantic_codebook, // [semantic_cardinality, codebook_dim]
+ device const T* semantic_out_proj, // [input_dim, codebook_dim]
+ device const T* semantic_out_bias, // [input_dim]
+ device const T*
+ residual_codebooks, // [R, residual_cardinality, codebook_dim]
+ device const T* residual_out_proj, // [R, input_dim, codebook_dim]
+ device const T* residual_out_bias, // [R, input_dim]
+ device T* output, // [B, T, input_dim]
+ const constant int& batch_size,
+ const constant int& total_codebooks,
+ const constant int& seq_len,
+ const constant int& input_dim,
+ const constant int& codebook_dim,
+ const constant int& residual_quantizers,
+ const constant int& semantic_cardinality,
+ const constant int& residual_cardinality,
+ const uint3 gid
+) {
+ const uint d = gid.x;
+ const uint t = gid.y;
+ const uint b = gid.z;
+
+ if (d >= (uint)input_dim || t >= (uint)seq_len || b >= (uint)batch_size) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_idx = (b * (uint)seq_len + t) * (uint)input_dim + d;
+ if ((int)t >= len_b) {
+ output[out_idx] = (T)0;
+ return;
+ }
+
+ const uint token_base = (b * (uint)total_codebooks) * (uint)seq_len + t;
+ uint semantic_token = tokens[token_base];
+ semantic_token = min(semantic_token, (uint)(semantic_cardinality - 1));
+
+ float acc = float(semantic_out_bias[d]);
+ const uint semantic_code_base = (uint)semantic_token * (uint)codebook_dim;
+ const uint semantic_proj_base = d * (uint)codebook_dim;
+ for (uint k = 0; k < (uint)codebook_dim; ++k) {
+ acc += float(semantic_out_proj[semantic_proj_base + k]) *
+ float(semantic_codebook[semantic_code_base + k]);
+ }
+
+ for (int r = 0; r < residual_quantizers; ++r) {
+ const uint token_idx = token_base + (uint)(r + 1) * (uint)seq_len;
+ uint residual_token = tokens[token_idx];
+ residual_token = min(residual_token, (uint)(residual_cardinality - 1));
+
+ const uint bias_idx = (uint)r * (uint)input_dim + d;
+ acc += float(residual_out_bias[bias_idx]);
+
+ const uint code_base =
+ ((uint)r * (uint)residual_cardinality + (uint)residual_token) *
+ (uint)codebook_dim;
+ const uint proj_base = ((uint)r * (uint)input_dim + d) * (uint)codebook_dim;
+ for (uint k = 0; k < (uint)codebook_dim; ++k) {
+ acc += float(residual_out_proj[proj_base + k]) *
+ float(residual_codebooks[code_base + k]);
+ }
+ }
+
+ output[out_idx] = (T)acc;
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(AudioQuantizerDecode)(
+ device const uint* tokens,
+ device const int* lengths,
+ device const T* semantic_codebook,
+ device const T* semantic_out_proj,
+ device const T* semantic_out_bias,
+ device const T* residual_codebooks,
+ device const T* residual_out_proj,
+ device const T* residual_out_bias,
+ device T* output,
+ const constant int& batch_size,
+ const constant int& total_codebooks,
+ const constant int& seq_len,
+ const constant int& input_dim,
+ const constant int& codebook_dim,
+ const constant int& residual_quantizers,
+ const constant int& semantic_cardinality,
+ const constant int& residual_cardinality,
+ uint d AXIS(input_dim, 32),
+ uint t AXIS(seq_len, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ quantizer_decode<T>(
+ tokens,
+ lengths,
+ semantic_codebook,
+ semantic_out_proj,
+ semantic_out_bias,
+ residual_codebooks,
+ residual_out_proj,
+ residual_out_bias,
+ output,
+ batch_size,
+ total_codebooks,
+ seq_len,
+ input_dim,
+ codebook_dim,
+ residual_quantizers,
+ semantic_cardinality,
+ residual_cardinality,
+ uint3(d, t, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/embedding_rows_sum.metal b/crates/uzu/src/backends/metal/kernel/embedding/embedding_rows_sum.metal
new file mode 100644
index 000000000..fa08f9da0
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/embedding/embedding_rows_sum.metal
@@ -0,0 +1,27 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+#define BLOCK_SIZE 256
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(EmbeddingRowsSum)(
+ const device uint32_t* token_indices, // [num_rows]
+ const device T* weights, // [total_rows, model_dim]
+ device T* output, // [model_dim]
+ constant uint32_t& num_rows,
+ constant uint32_t& total_rows,
+ constant uint32_t& model_dim,
+ constant uint32_t& codebook_stride,
+ uint dim_idx AXIS(model_dim, BLOCK_SIZE)
+) {
+ float sum = 0.0f;
+ for (uint row_idx = 0; row_idx < num_rows; row_idx++) {
+ uint token = token_indices[row_idx];
+ uint row = row_idx * codebook_stride + token;
+ if (row < total_rows) {
+ sum += float(weights[row * model_dim + dim_idx]);
+ }
+ }
+ output[dim_idx] = T(sum);
+}
diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal
new file mode 100644
index 000000000..45afc5789
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal
@@ -0,0 +1,18 @@
+#include <metal_stdlib>
+#include "../common/dsl.h"
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+PUBLIC KERNEL(TensorAddScale)(
+ const device T* input,
+ const device T* bias,
+ device T* output,
+ constant uint& num_cols,
+ constant uint& length,
+ constant float& scale,
+ const uint position AXIS(length, 32)
+) {
+ uint col = position % num_cols;
+ float value = (float(input[position]) + float(bias[col])) * scale;
+ output[position] = T(value);
+}
diff --git a/crates/uzu/src/config/mod.rs b/crates/uzu/src/config/mod.rs
index c44e73c80..61c4abf13 100644
--- a/crates/uzu/src/config/mod.rs
+++ b/crates/uzu/src/config/mod.rs
@@ -20,6 +20,7 @@ mod rope;
mod short_conv;
mod transformer;
mod transformer_layer;
+mod tts_model;
pub use attention::*;
pub use classifier::*;
@@ -43,3 +44,4 @@ pub use rope::*;
pub use short_conv::*;
pub use transformer::*;
pub use transformer_layer::*;
+pub use tts_model::*;
diff --git a/crates/uzu/src/config/model_config.rs b/crates/uzu/src/config/model_config.rs
index 90587d598..51673d313 100644
--- a/crates/uzu/src/config/model_config.rs
+++ b/crates/uzu/src/config/model_config.rs
@@ -1,12 +1,13 @@
use serde::{Deserialize, Serialize};
-use super::{ClassifierModelConfig, LanguageModelConfig};
+use super::{ClassifierModelConfig, LanguageModelConfig, TtsModelConfig};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
#[serde(untagged)]
pub enum ModelConfig {
LanguageModel(LanguageModelConfig),
Classifier(ClassifierModelConfig),
+ Tts(TtsModelConfig),
}
impl ModelConfig {
@@ -24,6 +25,13 @@ impl ModelConfig {
}
}
+ pub fn as_tts(&self) -> Option<&TtsModelConfig> {
+ match self {
+ ModelConfig::Tts(config) => Some(config),
+ _ => None,
+ }
+ }
+
pub fn is_language_model(&self) -> bool {
matches!(self, ModelConfig::LanguageModel(_))
}
@@ -31,4 +39,8 @@ impl ModelConfig {
pub fn is_classifier(&self) -> bool {
matches!(self, ModelConfig::Classifier(_))
}
+
+ pub fn is_tts(&self) -> bool {
+ matches!(self, ModelConfig::Tts(_))
+ }
}
diff --git a/crates/uzu/src/config/model_type.rs b/crates/uzu/src/config/model_type.rs
index e5dc7d691..1e27bf44d 100644
--- a/crates/uzu/src/config/model_type.rs
+++ b/crates/uzu/src/config/model_type.rs
@@ -5,4 +5,5 @@ use serde::{Deserialize, Serialize};
pub enum ModelType {
LanguageModel,
ClassifierModel,
+ TtsModel,
}
diff --git a/crates/uzu/src/config/tts_model.rs b/crates/uzu/src/config/tts_model.rs
new file mode 100644
index 000000000..23e407898
--- /dev/null
+++ b/crates/uzu/src/config/tts_model.rs
@@ -0,0 +1,551 @@
+use std::collections::BTreeMap;
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+use std::path::Path;
+
+use serde::{Deserialize, Serialize};
+
+use super::{EmbeddingConfig, EmbeddingConfigCommon, LinearConfig, TransformerConfig};
+use crate::ConfigDataType;
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct TtsMessageProcessorConfig {
+ pub prompt_template: String,
+ #[serde(default = "default_drop_initial_newline")]
+ pub drop_initial_newline: bool,
+ #[serde(default = "default_system_role_name")]
+ pub system_role_name: String,
+ #[serde(default = "default_user_role_name")]
+ pub user_role_name: String,
+ #[serde(default = "default_assistant_role_name")]
+ pub assistant_role_name: String,
+ #[serde(default = "default_tts_message_fields")]
+ pub default_message_fields: BTreeMap<String, String>,
+}
+
+fn default_drop_initial_newline() -> bool {
+ true
+}
+
+fn default_system_role_name() -> String {
+ String::from("system")
+}
+
+fn default_user_role_name() -> String {
+ String::from("user")
+}
+
+fn default_assistant_role_name() -> String {
+ String::from("assistant")
+}
+
+fn default_tts_message_fields() -> BTreeMap<String, String> {
+ BTreeMap::new()
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct TtsConfig {
+ pub text_decoder_config: TtsTextDecoderConfig,
+ pub audio_decoder_config: TtsAudioDecoderConfig,
+ #[serde(default)]
+ pub activation_precision: Option<ConfigDataType>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(tag = "type")]
+pub enum TtsTextDecoderConfig {
+ FishAudioTextDecoderConfig {
+ #[serde(flatten)]
+ config: FishAudioTextDecoderConfig,
+ },
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct FishAudioTextDecoderConfig {
+ pub slow_embeddings_config: FishAudioEmbeddingConfig,
+ pub slow_model_config: TransformerConfig,
+ pub slow_readout_config: FishAudioLinearConfig,
+ pub fast_embeddings_config: FishAudioEmbeddingConfig,
+ pub fast_model_config: TransformerConfig,
+ pub fast_readout_config: FishAudioLinearConfig,
+ pub codebook_embeddings_config: FishAudioEmbeddingConfig,
+ pub fast_model_projection_config: Option<FishAudioLinearConfig>,
+ pub semantic_token_begin_id: i64,
+ pub semantic_token_end_id: i64,
+ pub im_end_token_id: i64,
+ pub codebook_size: usize,
+ pub vocab_size: usize,
+ pub slow_model_dim: usize,
+ pub fast_model_dim: usize,
+ pub num_codebooks: usize,
+ pub max_seq_len: usize,
+ pub scale_codebook_embeddings: bool,
+ #[serde(default)]
+ pub precision: Option<ConfigDataType>,
+ pub short_logits_size: usize,
+ pub repeat_window_size: usize,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(untagged)]
+pub enum FishAudioLinearConfig {
+ Tagged(LinearConfig),
+ UntaggedFullPrecision {
+ #[serde(rename = "precision")]
+ precision: ConfigDataType,
+ },
+}
+
+impl FishAudioLinearConfig {
+ pub fn is_full_precision(&self) -> bool {
+ matches!(
+ self,
+ FishAudioLinearConfig::Tagged(LinearConfig::FullPrecision { .. })
+ | FishAudioLinearConfig::UntaggedFullPrecision { .. }
+ )
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(untagged)]
+pub enum FishAudioEmbeddingConfig {
+ Tagged(EmbeddingConfig),
+ UntaggedFullPrecision {
+ input_scale: Option<f32>,
+ logit_soft_cap: Option<f32>,
+ precision: ConfigDataType,
+ },
+}
+
+impl FishAudioEmbeddingConfig {
+ pub fn to_embedding_config(&self) -> EmbeddingConfig {
+ match self {
+ FishAudioEmbeddingConfig::Tagged(config) => config.clone(),
+ FishAudioEmbeddingConfig::UntaggedFullPrecision {
+ input_scale,
+ logit_soft_cap,
+ precision,
+ } => EmbeddingConfig::Tied {
+ common: EmbeddingConfigCommon {
+ input_scale: *input_scale,
+ logit_soft_cap: *logit_soft_cap,
+ },
+ precision: *precision,
+ },
+ }
+ }
+
+ #[cfg(all(feature = "audio-runtime", metal_backend))]
+ pub(crate) fn to_text_decoder_embedding_config(&self) -> EmbeddingConfig {
+ match self {
+ FishAudioEmbeddingConfig::Tagged(config) => config.clone(),
+ FishAudioEmbeddingConfig::UntaggedFullPrecision {
+ input_scale,
+ logit_soft_cap,
+ precision,
+ } => EmbeddingConfig::Untied {
+ common: EmbeddingConfigCommon {
+ input_scale: *input_scale,
+ logit_soft_cap: *logit_soft_cap,
+ },
+ precision: *precision,
+ },
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(tag = "type")]
+pub enum TtsAudioDecoderConfig {
+ DescriptAudioCodecConfig {
+ #[serde(flatten)]
+ config: DescriptAudioCodecConfig,
+ },
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct DescriptAudioCodecConfig {
+ pub precision: ConfigDataType,
+ pub quantizer_config: DescriptAudioQuantizerConfig,
+ pub decoder_config: DescriptAudioDacDecoderConfig,
+ pub samplerate: u32,
+ pub encoder_dim: usize,
+ pub encoder_rates: Vec<usize>,
+ pub decoder_dim: usize,
+ pub decoder_rates: Vec<usize>,
+ pub input_dim: usize,
+ pub n_codebooks: usize,
+ pub codebook_dim: usize,
+ pub downsample_factor: Vec<usize>,
+ pub codebook_size: usize,
+ pub semantic_codebook_size: usize,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct DescriptAudioDacDecoderConfig {
+ pub precision: ConfigDataType,
+ pub conv_config: DescriptAudioCausalConv1dConfig,
+ pub snake_config: DescriptAudioSnake1dConfig,
+ pub decoder_block_config: DescriptAudioDacDecoderBlockConfig,
+ pub causal: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct DescriptAudioDacDecoderBlockConfig {
+ pub precision: ConfigDataType,
+ pub snake_config: DescriptAudioSnake1dConfig,
+ pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,
+ pub res_unit_config: DescriptAudioResidualUnitConfig,
+ pub causal: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct DescriptAudioResidualUnitConfig {
+ pub precision: ConfigDataType,
+ pub snake_config: DescriptAudioSnake1dConfig,
+ pub conv_config: DescriptAudioCausalConv1dConfig,
+ pub causal: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct DescriptAudioCausalConv1dConfig {
+ pub precision: ConfigDataType,
+ pub has_biases: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct DescriptAudioCausalTransposeConv1dConfig {
+ pub precision: ConfigDataType,
+ pub has_biases: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(deny_unknown_fields)]
+pub struct DescriptAudioSnake1dConfig {
+ pub precision: ConfigDataType,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct DescriptAudioQuantizerConfig {
+ #[serde(default)]
+ pub precision: Option<ConfigDataType>,
+ pub post_module_config: TransformerConfig,
+ pub upsampler_config: DescriptAudioUpsamplerConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct DescriptAudioUpsamplerConfig {
+ pub block_configs: Vec<DescriptAudioUpsamplingBlockConfig>,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct DescriptAudioUpsamplingBlockConfig {
+ pub convnext_config: DescriptAudioConvNeXtConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct DescriptAudioConvNeXtConfig {
+ pub norm_config: DescriptAudioConvNeXtNormConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct DescriptAudioConvNeXtNormConfig {
+ pub epsilon: f32,
+ #[serde(default)]
+ pub subtract_mean: bool,
+ #[serde(default)]
+ pub use_bias: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+pub struct TtsModelConfig {
+ pub tts_config: TtsConfig,
+ pub message_processor_config: TtsMessageProcessorConfig,
+}
+
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+impl TtsModelConfig {
+ pub fn create_audio_generation_context_with_model_path<B: crate::backends::common::Backend>(
+ &self,
+ model_path: &Path,
+ ) -> crate::audio::AudioResult<crate::audio::AudioGenerationContext<B>> {
+ crate::audio::AudioGenerationContext::from_tts_config_and_model_path(&self.tts_config, model_path)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::config::{ModelConfig, ModelMetadata, ModelType};
+
+ fn descript_audio_decoder_config_json() -> serde_json::Value {
+ serde_json::json!({
+ "precision": "float16",
+ "conv_config": {
+ "precision": "float16",
+ "has_biases": true
+ },
+ "snake_config": {
+ "precision": "float16"
+ },
+ "decoder_block_config": {
+ "precision": "float16",
+ "snake_config": {
+ "precision": "float16"
+ },
+ "trans_conv_config": {
+ "precision": "float16",
+ "has_biases": true
+ },
+ "res_unit_config": {
+ "precision": "float16",
+ "snake_config": {
+ "precision": "float16"
+ },
+ "conv_config": {
+ "precision": "float16",
+ "has_biases": true
+ },
+ "causal": true
+ },
+ "causal": true
+ },
+ "causal": true
+ })
+ }
+
+ fn fishaudio_text_decoder_config_json() -> serde_json::Value {
+ let norm_config = serde_json::json!({
+ "scale_precision": "float32",
+ "accumulation_precision": "float32",
+ "epsilon": 1e-5,
+ "scale_offset": null,
+ "upcast_mode": "full_layer",
+ "subtract_mean": true
+ });
+ let transformer_config = serde_json::json!({
+ "global_rope_config": null,
+ "local_rope_config": null,
+ "layer_configs": [],
+ "output_norm_config": norm_config,
+ "model_dim": 256,
+ "hidden_dim": 256,
+ "context_length": 512
+ });
+ serde_json::json!({
+ "type": "FishAudioTextDecoderConfig",
+ "slow_embeddings_config": { "precision": "float16", "input_scale": null, "logit_soft_cap": null },
+ "slow_model_config": transformer_config,
+ "slow_readout_config": { "precision": "float16" },
+ "fast_embeddings_config": { "precision": "float16", "input_scale": null, "logit_soft_cap": null },
+ "fast_model_config": transformer_config,
+ "fast_readout_config": { "precision": "float16" },
+ "codebook_embeddings_config": { "precision": "float16", "input_scale": null, "logit_soft_cap": null },
+ "fast_model_projection_config": null,
+ "semantic_token_begin_id": 100,
+ "semantic_token_end_id": 1123,
+ "im_end_token_id": 4,
+ "codebook_size": 1024,
+ "vocab_size": 2048,
+ "slow_model_dim": 256,
+ "fast_model_dim": 256,
+ "num_codebooks": 8,
+ "max_seq_len": 4096,
+ "scale_codebook_embeddings": false,
+ "short_logits_size": 1124,
+ "repeat_window_size": 16
+ })
+ }
+
+ #[test]
+ fn metadata_with_tts_model_type_parses() {
+ let config_json = serde_json::json!({
+ "toolchain_version": "0.0.0-test",
+ "vendor": "NVIDIA",
+ "family": "nanocodec",
+ "name": "nemo-nano-codec-22khz-1.78kbps-12.5fps",
+ "size": "0.1B",
+ "quantization": null,
+ "repo": "nvidia/nemo-nano-codec-22khz-1.78kbps-12.5fps",
+ "use_cases": [],
+ "model_type": "tts_model",
+ "model_config": {
+ "tts_config": {
+ "text_decoder_config": fishaudio_text_decoder_config_json(),
+ "audio_decoder_config": {
+ "type": "DescriptAudioCodecConfig",
+ "precision": "float16",
+ "samplerate": 44100,
+ "encoder_dim": 1024,
+ "encoder_rates": [2, 4, 8, 8],
+ "decoder_dim": 1536,
+ "n_codebooks": 9,
+ "codebook_dim": 8,
+ "codebook_size": 1024,
+ "semantic_codebook_size": 4096,
+ "input_dim": 1024,
+ "downsample_factor": [2, 2],
+ "decoder_rates": [8, 8, 4, 2],
+ "decoder_config": descript_audio_decoder_config_json(),
+ "quantizer_config": {
+ "post_module_config": {
+ "global_rope_config": null,
+ "local_rope_config": null,
+ "layer_configs": [],
+ "output_norm_config": {
+ "scale_precision": "float32",
+ "accumulation_precision": "float32",
+ "epsilon": 1e-5,
+ "scale_offset": null,
+ "upcast_mode": "full_layer",
+ "subtract_mean": true
+ },
+ "model_dim": 1024,
+ "hidden_dim": 1024,
+ "context_length": 1
+ },
+ "upsampler_config": {
+ "block_configs": []
+ }
+ }
+ },
+ },
+ "message_processor_config": {
+ "prompt_template": "{% for message in messages %}{{message.content}}{% endfor %}",
+ "drop_initial_newline": true
+ }
+ }
+ });
+
+ let metadata: ModelMetadata = serde_json::from_value(config_json).expect("metadata parse");
+ assert_eq!(metadata.model_type, ModelType::TtsModel);
+
+ let tts_config = metadata.model_config.as_tts().expect("tts config");
+ assert_eq!(
+ tts_config.message_processor_config.prompt_template,
+ "{% for message in messages %}{{message.content}}{% endfor %}"
+ );
+ assert!(tts_config.message_processor_config.drop_initial_newline);
+ assert!(tts_config.message_processor_config.default_message_fields.is_empty());
+ }
+
+ #[test]
+ fn tts_model_config_helpers_work() {
+ let model_config_json = serde_json::json!({
+ "tts_config": {
+ "text_decoder_config": fishaudio_text_decoder_config_json(),
+ "audio_decoder_config": {
+ "type": "DescriptAudioCodecConfig",
+ "precision": "float16",
+ "samplerate": 44100,
+ "encoder_dim": 1024,
+ "encoder_rates": [2, 4, 8, 8],
+ "decoder_dim": 1536,
+ "n_codebooks": 9,
+ "codebook_dim": 8,
+ "codebook_size": 1024,
+ "semantic_codebook_size": 4096,
+ "input_dim": 1024,
+ "downsample_factor": [2, 2],
+ "decoder_rates": [8, 8, 4, 2],
+ "decoder_config": descript_audio_decoder_config_json(),
+ "quantizer_config": {
+ "post_module_config": {
+ "global_rope_config": null,
+ "local_rope_config": null,
+ "layer_configs": [],
+ "output_norm_config": {
+ "scale_precision": "float32",
+ "accumulation_precision": "float32",
+ "epsilon": 1e-5,
+ "scale_offset": null,
+ "upcast_mode": "full_layer",
+ "subtract_mean": true
+ },
+ "model_dim": 1024,
+ "hidden_dim": 1024,
+ "context_length": 1
+ },
+ "upsampler_config": {
+ "block_configs": []
+ }
+ }
+ },
+ },
+ "message_processor_config": {
+ "prompt_template": "{{messages[0].content}}"
+ }
+ });
+
+ let model_config: ModelConfig = serde_json::from_value(model_config_json).expect("model config parse");
+ assert!(model_config.is_tts());
+ assert!(model_config.as_tts().is_some());
+ assert!(!model_config.is_classifier());
+ assert!(!model_config.is_language_model());
+
+ let tts = model_config.as_tts().expect("tts variant");
+ assert_eq!(tts.message_processor_config.prompt_template, "{{messages[0].content}}");
+ assert!(tts.message_processor_config.drop_initial_newline);
+ assert!(tts.message_processor_config.default_message_fields.is_empty());
+ }
+
+ #[test]
+ fn tts_model_config_rejects_untyped_decoder_config() {
+ let model_config_json = serde_json::json!({
+ "tts_config": {
+ "text_decoder_config": fishaudio_text_decoder_config_json(),
+ "audio_decoder_config": {
+ "type": "DescriptAudioCodecConfig",
+ "precision": "float16",
+ "samplerate": 44100,
+ "encoder_dim": 1024,
+ "encoder_rates": [2, 4, 8, 8],
+ "decoder_dim": 1536,
+ "n_codebooks": 9,
+ "codebook_dim": 8,
+ "codebook_size": 1024,
+ "semantic_codebook_size": 4096,
+ "input_dim": 1024,
+ "downsample_factor": [2, 2],
+ "decoder_rates": [8, 8, 4, 2],
+ "decoder_config": {},
+ "quantizer_config": {
+ "post_module_config": {
+ "global_rope_config": null,
+ "local_rope_config": null,
+ "layer_configs": [],
+ "output_norm_config": {
+ "scale_precision": "float32",
+ "accumulation_precision": "float32",
+ "epsilon": 1e-5,
+ "scale_offset": null,
+ "upcast_mode": "full_layer",
+ "subtract_mean": true
+ },
+ "model_dim": 1024,
+ "hidden_dim": 1024,
+ "context_length": 1
+ },
+ "upsampler_config": {
+ "block_configs": []
+ }
+ }
+ },
+ },
+ "message_processor_config": {
+ "prompt_template": "{{messages[0].content}}"
+ }
+ });
+
+ let error =
+ serde_json::from_value::<ModelConfig>(model_config_json).expect_err("decoder_config should be typed");
+ let error_text = error.to_string();
+ assert!(
+ error_text.contains("did not match any variant") || error_text.contains("conv_config"),
+ "unexpected error: {error}"
+ );
+ }
+}
diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rs
index b2d4f700e..305d89a84 100644
--- a/crates/uzu/src/encodable_block/decoder.rs
+++ b/crates/uzu/src/encodable_block/decoder.rs
@@ -37,17 +37,62 @@ impl<B: Backend> Decoder<B> {
decoder_config: Rc<DecoderConfig>,
root_weight_loader: &ParameterTree<B::Context>,
) -> Self {
- let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");
+ Self::new_with_embedding_and_readout_subtrees(
+ context,
+ decoder_config,
+ root_weight_loader,
+ "transformer",
+ "embedding",
+ "embedding",
+ )
+ }
+
+ /// Used by models whose token lookup weights and logits readout weights
+ pub fn new_with_embedding_and_readout_subtrees(
+ context: Rc<B::Context>,
+ decoder_config: Rc<DecoderConfig>,
+ root_weight_loader: &ParameterTree<B::Context>,
+ transformer_subtree: &str,
+ embedding_subtree: &str,
+ readout_subtree: &str,
+ ) -> Self {
+ let embedding_weight_loader =
+ root_weight_loader.subtree(embedding_subtree).expect("Failed to get embedding subtree");
+ let readout_weight_loader = root_weight_loader.subtree(readout_subtree).expect("Failed to get readout subtree");
- let embed = Embedding::new(
+ let embed = Embedding::new_with_lookup_and_readout_trees(
context.as_ref(),
decoder_config.vocab_size as u32,
decoder_config.model_dim as u32,
&decoder_config.embedding_config,
- &root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree"),
+ &embedding_weight_loader,
+ &readout_weight_loader,
)
.expect("Failed to create embedding");
+ let (layers, norm) = Self::build_transformer_layers_and_norm(
+ context,
+ decoder_config.clone(),
+ root_weight_loader,
+ transformer_subtree,
+ );
+
+ Self {
+ embed,
+ layers,
+ norm,
+ }
+ }
+
+ pub(crate) fn build_transformer_layers_and_norm(
+ context: Rc<B::Context>,
+ decoder_config: Rc<DecoderConfig>,
+ root_weight_loader: &ParameterTree<B::Context>,
+ transformer_subtree: &str,
+ ) -> (Box<[LayerExecutables<B>]>, RMSNorm<B>) {
+ let decoder_weight_loader =
+ root_weight_loader.subtree(transformer_subtree).expect("transformer subtree not found");
+
let attention_data_type = Self::attention_data_type(&decoder_config);
let norm_reference_layer =
decoder_config.layer_configs.as_ref().map(|configs| &configs[0]).unwrap_or(&decoder_config.layer_config);
@@ -134,11 +179,7 @@ impl<B: Backend> Decoder<B> {
.map(RMSNorm::with_sampling_range)
.expect("Failed to create output RMS norm kernel");
- Self {
- embed,
- layers: layers.into_boxed_slice(),
- norm: norm_block,
- }
+ (layers.into_boxed_slice(), norm_block)
}
fn create_rope_block(
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index cf281bbe5..92c68ed77 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -102,6 +102,33 @@ pub struct Embedding<B: Backend> {
model_dim: u32,
}
+struct UntiedWeightKeys {
+ lookup_weights: &'static str,
+ lookup_scales: &'static str,
+ lookup_biases: &'static str,
+ readout_weights: &'static str,
+ readout_scales: &'static str,
+ readout_biases: &'static str,
+}
+
+const SHARED_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {
+ lookup_weights: "input_weights",
+ lookup_scales: "input_scales",
+ lookup_biases: "input_biases",
+ readout_weights: "output_weights",
+ readout_scales: "output_scales",
+ readout_biases: "output_biases",
+};
+
+const SPLIT_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {
+ lookup_weights: "weights",
+ lookup_scales: "scales",
+ lookup_biases: "biases",
+ readout_weights: "weights",
+ readout_scales: "scales",
+ readout_biases: "biases",
+};
+
fn validate_tensor<'file, 'context, 'leaf, B: Backend>(
weights_leaf: &ParameterLeaf<'file, 'context, 'leaf, B::Context>,
expected_shape: [usize; 2],
@@ -129,6 +156,45 @@ impl<B: Backend> Embedding<B> {
model_dim: u32,
config: &EmbeddingConfig,
parameter_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, EmbeddingError<B>> {
+ Self::new_with_weight_trees(
+ context,
+ vocab_size,
+ model_dim,
+ config,
+ parameter_tree,
+ parameter_tree,
+ &SHARED_TREE_UNTIED_KEYS,
+ )
+ }
+
+ pub(crate) fn new_with_lookup_and_readout_trees(
+ context: &B::Context,
+ vocab_size: u32,
+ model_dim: u32,
+ config: &EmbeddingConfig,
+ lookup_tree: &ParameterTree<B::Context>,
+ readout_tree: &ParameterTree<B::Context>,
+ ) -> Result<Self, EmbeddingError<B>> {
+ Self::new_with_weight_trees(
+ context,
+ vocab_size,
+ model_dim,
+ config,
+ lookup_tree,
+ readout_tree,
+ &SPLIT_TREE_UNTIED_KEYS,
+ )
+ }
+
+ fn new_with_weight_trees(
+ context: &B::Context,
+ vocab_size: u32,
+ model_dim: u32,
+ config: &EmbeddingConfig,
+ lookup_tree: &ParameterTree<B::Context>,
+ readout_tree: &ParameterTree<B::Context>,
+ untied_keys: &UntiedWeightKeys,
) -> Result<Self, EmbeddingError<B>> {
let common = config.common();
@@ -139,9 +205,8 @@ impl<B: Backend> Embedding<B> {
} => {
let data_type = (*precision).into();
- let weights_leaf = parameter_tree.leaf("weights")?;
+ let weights_leaf = lookup_tree.leaf("weights")?;
validate_tensor(&weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
-
let weights = weights_leaf.read_buffer()?;
let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)
@@ -162,9 +227,9 @@ impl<B: Backend> Embedding<B> {
} => {
let data_type = (*precision).into();
- let input_weights_leaf = parameter_tree.leaf("input_weights")?;
+ let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;
validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
- let output_weights_leaf = parameter_tree.leaf("output_weights")?;
+ let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;
validate_tensor(&output_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
let input_weights = input_weights_leaf.read_buffer()?;
@@ -199,15 +264,15 @@ impl<B: Backend> Embedding<B> {
let num_groups = (model_dim as usize).div_ceil(*group_size);
- let weights_leaf = parameter_tree.leaf("weights")?;
+ let weights_leaf = lookup_tree.leaf("weights")?;
validate_tensor(
&weights_leaf,
[vocab_size as usize, model_dim as usize / packing_divisor],
storage_data_type,
)?;
- let scales_leaf = parameter_tree.leaf("scales")?;
+ let scales_leaf = lookup_tree.leaf("scales")?;
validate_tensor(&scales_leaf, [vocab_size as usize, num_groups], data_type)?;
- let biases_leaf = parameter_tree.leaf("biases")?;
+ let biases_leaf = lookup_tree.leaf("biases")?;
validate_tensor(&biases_leaf, [vocab_size as usize, num_groups], data_type)?;
let weights = weights_leaf.read_buffer()?;
@@ -264,26 +329,26 @@ impl<B: Backend> Embedding<B> {
let num_groups = (model_dim as usize).div_ceil(*group_size);
- let input_weights_leaf = parameter_tree.leaf("input_weights")?;
+ let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;
validate_tensor(
&input_weights_leaf,
[vocab_size as usize, model_dim as usize / packing_divisor],
storage_data_type,
)?;
- let input_scales_leaf = parameter_tree.leaf("input_scales")?;
+ let input_scales_leaf = lookup_tree.leaf(untied_keys.lookup_scales)?;
validate_tensor(&input_scales_leaf, [vocab_size as usize, num_groups], data_type)?;
- let input_biases_leaf = parameter_tree.leaf("input_biases")?;
+ let input_biases_leaf = lookup_tree.leaf(untied_keys.lookup_biases)?;
validate_tensor(&input_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
- let output_weights_leaf = parameter_tree.leaf("output_weights")?;
+ let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;
validate_tensor(
&output_weights_leaf,
[vocab_size as usize, model_dim as usize / packing_divisor],
storage_data_type,
)?;
- let output_scales_leaf = parameter_tree.leaf("output_scales")?;
+ let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;
validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;
- let output_biases_leaf = parameter_tree.leaf("output_biases")?;
+ let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;
validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
let input_weights = input_weights_leaf.read_buffer()?;
@@ -349,18 +414,18 @@ impl<B: Backend> Embedding<B> {
let num_groups = (model_dim as usize).div_ceil(*group_size);
- let input_weights_leaf = parameter_tree.leaf("input_weights")?;
+ let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;
validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;
- let output_weights_leaf = parameter_tree.leaf("output_weights")?;
+ let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;
validate_tensor(
&output_weights_leaf,
[vocab_size as usize, model_dim as usize / packing_divisor],
storage_data_type,
)?;
- let output_scales_leaf = parameter_tree.leaf("output_scales")?;
+ let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;
validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;
- let output_biases_leaf = parameter_tree.leaf("output_biases")?;
+ let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;
validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;
let input_weights = input_weights_leaf.read_buffer()?;
diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
index c3bb078ea..6ab607cd7 100644
--- a/crates/uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -183,6 +183,9 @@ impl<B: Backend> MoeBlock<B> {
ActivationType::SILU {
..
} => 2,
+ ActivationType::TANH => {
+ panic!("Tanh activation is not supported for MoE kernels")
+ },
ActivationType::IDENTITY => {
panic!("Identity activation is not supported for MoE kernels")
},
diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rs
index d1a6ae5b6..f2b84ea6f 100644
--- a/crates/uzu/src/forward_pass/state/state.rs
+++ b/crates/uzu/src/forward_pass/state/state.rs
@@ -757,6 +757,11 @@ impl<B: Backend> ForwardPassState<B> {
let copy_size_bytes = destination_array.size();
debug_assert_eq!(destination_array.size(), source_ref.size());
- encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);
+ encoder.encode_copy(
+ src_buf_rc.borrow().deref(),
+ 0..copy_size_bytes,
+ dst_buf_rc.borrow_mut().deref_mut(),
+ 0..copy_size_bytes,
+ );
}
}
diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rs
index 23cda3918..fdb352ea3 100644
--- a/crates/uzu/src/lib.rs
+++ b/crates/uzu/src/lib.rs
@@ -18,6 +18,8 @@ pub mod prelude;
pub mod session;
pub use array::{Array, ArrayContextExt};
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub use audio::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};
pub use config::ConfigDataType;
pub use data_type::{ArrayElement, DataType};
pub use language_model::gumbel::{gumbel_float, revidx};
diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rs
index 612fa1549..2f33b9151 100644
--- a/crates/uzu/src/prelude.rs
+++ b/crates/uzu/src/prelude.rs
@@ -9,9 +9,13 @@
// Session Parameters
// Session Types
// Speculators
+#[cfg(metal_backend)]
+pub use crate::backends::metal::MetalContext;
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub use crate::session::TtsSession;
pub use crate::{
VERSION,
- audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid, AudioTokenPacking},
+ audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid},
parameters::{HeaderLoadingError, ParameterLeaf, ParameterLoaderError, ParameterTree},
session::{
ChatSession, ClassificationSession,
diff --git a/crates/uzu/src/session/config/mod.rs b/crates/uzu/src/session/config/mod.rs
index 50dfc113c..d87a0831c 100644
--- a/crates/uzu/src/session/config/mod.rs
+++ b/crates/uzu/src/session/config/mod.rs
@@ -2,8 +2,14 @@ mod decoding_config;
mod grammar_config;
mod run_config;
mod speculator_config;
+mod tts_run_config;
+mod tts_session_options;
pub use decoding_config::DecodingConfig;
pub use grammar_config::GrammarConfig;
pub use run_config::RunConfig;
pub use speculator_config::SpeculatorConfig;
+pub use tts_run_config::{TtsChunkPolicy, TtsNonStreamingMode, TtsPerformanceConfig, TtsRunConfig, TtsRunConfigError};
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub use tts_session_options::TtsSessionOptions;
+pub use tts_session_options::{TextDecoderRuntimeConfig, TextSamplingConfig};
diff --git a/crates/uzu/src/session/config/tts_run_config.rs b/crates/uzu/src/session/config/tts_run_config.rs
new file mode 100644
index 000000000..16453a895
--- /dev/null
+++ b/crates/uzu/src/session/config/tts_run_config.rs
@@ -0,0 +1,222 @@
+use thiserror::Error;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TtsChunkPolicy {
+ Fixed,
+ Adaptive,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TtsNonStreamingMode {
+ FullDecode,
+ ChunkedIfNeeded,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct TtsRunConfig {
+ pub streaming_enabled: bool,
+ pub target_emit_latency_ms: u32,
+ pub initial_chunk_frames: usize,
+ pub min_chunk_frames: usize,
+ pub max_chunk_frames: usize,
+ pub max_stream_workspace_frames: usize,
+ pub max_semantic_frames: usize,
+ pub chunk_policy: TtsChunkPolicy,
+ pub non_streaming_mode: TtsNonStreamingMode,
+ pub default_seed: u64,
+ pub chunk_ema_alpha: f64,
+ pub chunk_hysteresis_fraction: f64,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
+pub enum TtsRunConfigError {
+ #[error("initial_chunk_frames must be greater than zero")]
+ InitialChunkFramesMustBePositive,
+ #[error("min_chunk_frames must be greater than zero")]
+ MinChunkFramesMustBePositive,
+ #[error("max_chunk_frames must be greater than or equal to min_chunk_frames")]
+ MaxChunkFramesBelowMinChunkFrames,
+ #[error("max_stream_workspace_frames must be greater than zero")]
+ MaxStreamWorkspaceFramesMustBePositive,
+ #[error("target_emit_latency_ms must be greater than zero")]
+ TargetEmitLatencyMustBePositive,
+ #[error("max_semantic_frames must be greater than zero")]
+ MaxSemanticFramesMustBePositive,
+ #[error("max_stream_workspace_frames must be greater than or equal to max_chunk_frames")]
+ MaxStreamWorkspaceFramesBelowMaxChunkFrames,
+}
+
+impl TtsRunConfig {
+ pub fn validate(&self) -> Result<(), TtsRunConfigError> {
+ if self.initial_chunk_frames == 0 {
+ return Err(TtsRunConfigError::InitialChunkFramesMustBePositive);
+ }
+ if self.min_chunk_frames == 0 {
+ return Err(TtsRunConfigError::MinChunkFramesMustBePositive);
+ }
+ if self.max_chunk_frames < self.min_chunk_frames {
+ return Err(TtsRunConfigError::MaxChunkFramesBelowMinChunkFrames);
+ }
+ if self.max_stream_workspace_frames == 0 {
+ return Err(TtsRunConfigError::MaxStreamWorkspaceFramesMustBePositive);
+ }
+ if self.target_emit_latency_ms == 0 {
+ return Err(TtsRunConfigError::TargetEmitLatencyMustBePositive);
+ }
+ if self.max_semantic_frames == 0 {
+ return Err(TtsRunConfigError::MaxSemanticFramesMustBePositive);
+ }
+ Ok(())
+ }
+
+ pub fn validate_stream_decode(&self) -> Result<(), TtsRunConfigError> {
+ self.validate()?;
+ if self.max_stream_workspace_frames < self.max_chunk_frames {
+ return Err(TtsRunConfigError::MaxStreamWorkspaceFramesBelowMaxChunkFrames);
+ }
+ Ok(())
+ }
+
+ pub fn fixed_chunk_frames(chunk_frames: usize) -> Self {
+ Self {
+ chunk_policy: TtsChunkPolicy::Fixed,
+ initial_chunk_frames: chunk_frames.max(1),
+ min_chunk_frames: chunk_frames.max(1),
+ max_chunk_frames: chunk_frames.max(1),
+ ..Self::default()
+ }
+ }
+}
+
+impl Default for TtsRunConfig {
+ fn default() -> Self {
+ Self {
+ streaming_enabled: true,
+ target_emit_latency_ms: 80,
+ initial_chunk_frames: 1,
+ min_chunk_frames: 16,
+ max_chunk_frames: 256,
+ max_stream_workspace_frames: 768,
+ max_semantic_frames: 768,
+ chunk_policy: TtsChunkPolicy::Adaptive,
+ non_streaming_mode: TtsNonStreamingMode::ChunkedIfNeeded,
+ default_seed: 123,
+ chunk_ema_alpha: 0.2,
+ chunk_hysteresis_fraction: 0.25,
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct TtsPerformanceConfig {
+ pub streaming: TtsRunConfig,
+ pub non_streaming: TtsRunConfig,
+ pub non_streaming_chunked_threshold_frames: usize,
+}
+
+impl Default for TtsPerformanceConfig {
+ fn default() -> Self {
+ let streaming = TtsRunConfig::default();
+ let non_streaming = TtsRunConfig {
+ streaming_enabled: false,
+ chunk_policy: TtsChunkPolicy::Fixed,
+ non_streaming_mode: TtsNonStreamingMode::ChunkedIfNeeded,
+ initial_chunk_frames: 128,
+ min_chunk_frames: 128,
+ max_chunk_frames: 128,
+ ..TtsRunConfig::default()
+ };
+ Self {
+ streaming,
+ non_streaming,
+ non_streaming_chunked_threshold_frames: 4096,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{TtsChunkPolicy, TtsNonStreamingMode, TtsPerformanceConfig, TtsRunConfig};
+
+ #[test]
+ fn default_run_config_matches_production_defaults() {
+ let config = TtsRunConfig::default();
+ assert!(config.streaming_enabled);
+ assert_eq!(config.target_emit_latency_ms, 80);
+ assert_eq!(config.initial_chunk_frames, 1);
+ assert_eq!(config.min_chunk_frames, 16);
+ assert_eq!(config.max_chunk_frames, 256);
+ assert_eq!(config.max_stream_workspace_frames, 768);
+ assert_eq!(config.max_semantic_frames, 768);
+ assert_eq!(config.chunk_policy, TtsChunkPolicy::Adaptive);
+ assert_eq!(config.non_streaming_mode, TtsNonStreamingMode::ChunkedIfNeeded);
+ assert_eq!(config.default_seed, 123);
+ assert!((config.chunk_ema_alpha - 0.2).abs() < f64::EPSILON);
+ assert!((config.chunk_hysteresis_fraction - 0.25).abs() < f64::EPSILON);
+ }
+
+ #[test]
+ fn fixed_chunk_constructor_sets_bounds() {
+ let config = TtsRunConfig::fixed_chunk_frames(64);
+ assert_eq!(config.chunk_policy, TtsChunkPolicy::Fixed);
+ assert_eq!(config.initial_chunk_frames, 64);
+ assert_eq!(config.min_chunk_frames, 64);
+ assert_eq!(config.max_chunk_frames, 64);
+ }
+
+ #[test]
+ fn run_config_validation_rejects_invalid_bounds() {
+ let invalid = TtsRunConfig {
+ min_chunk_frames: 32,
+ max_chunk_frames: 16,
+ ..TtsRunConfig::default()
+ };
+ assert!(invalid.validate().is_err());
+ }
+
+ #[test]
+ fn run_config_validation_rejects_zero_initial_chunk() {
+ let invalid = TtsRunConfig {
+ initial_chunk_frames: 0,
+ ..TtsRunConfig::default()
+ };
+ assert!(invalid.validate().is_err());
+ }
+
+ #[test]
+ fn run_config_validation_rejects_zero_semantic_cap() {
+ let invalid = TtsRunConfig {
+ max_semantic_frames: 0,
+ ..TtsRunConfig::default()
+ };
+ assert!(invalid.validate().is_err());
+ }
+
+ #[test]
+ fn run_config_validation_rejects_zero_workspace_frames() {
+ let invalid = TtsRunConfig {
+ max_stream_workspace_frames: 0,
+ ..TtsRunConfig::default()
+ };
+ assert!(invalid.validate().is_err());
+ }
+
+ #[test]
+ fn run_config_validation_rejects_workspace_smaller_than_max_chunk() {
+ let invalid = TtsRunConfig {
+ max_chunk_frames: 256,
+ max_stream_workspace_frames: 128,
+ ..TtsRunConfig::default()
+ };
+ assert!(invalid.validate_stream_decode().is_err());
+ }
+
+ #[test]
+ fn performance_config_defaults_are_consistent() {
+ let config = TtsPerformanceConfig::default();
+ assert_eq!(config.streaming.chunk_policy, TtsChunkPolicy::Adaptive);
+ assert_eq!(config.non_streaming.chunk_policy, TtsChunkPolicy::Fixed);
+ assert_eq!(config.non_streaming.min_chunk_frames, config.non_streaming.max_chunk_frames);
+ assert_eq!(config.non_streaming.non_streaming_mode, TtsNonStreamingMode::ChunkedIfNeeded);
+ }
+}
diff --git a/crates/uzu/src/session/config/tts_session_options.rs b/crates/uzu/src/session/config/tts_session_options.rs
new file mode 100644
index 000000000..5a3e72e0d
--- /dev/null
+++ b/crates/uzu/src/session/config/tts_session_options.rs
@@ -0,0 +1,41 @@
+#[derive(Debug, Clone, Copy, PartialEq)]
+pub struct TextSamplingConfig {
+ pub temperature: f32,
+ pub top_p: f32,
+}
+
+impl Default for TextSamplingConfig {
+ fn default() -> Self {
+ Self {
+ temperature: 0.8008,
+ top_p: 0.8008,
+ }
+ }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct TextDecoderRuntimeConfig {
+ pub sampling: TextSamplingConfig,
+ pub min_frames_before_im_end: usize,
+ pub max_new_tokens_override: Option<usize>,
+ pub force_semantic_sampling_mask: Option<bool>,
+ pub prefill_step_size: usize,
+}
+
+impl Default for TextDecoderRuntimeConfig {
+ fn default() -> Self {
+ Self {
+ sampling: TextSamplingConfig::default(),
+ min_frames_before_im_end: 8,
+ max_new_tokens_override: None,
+ force_semantic_sampling_mask: None,
+ prefill_step_size: 128,
+ }
+ }
+}
+
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+#[derive(Debug, Clone, PartialEq, Default)]
+pub struct TtsSessionOptions {
+ pub text_decoder: TextDecoderRuntimeConfig,
+}
diff --git a/crates/uzu/src/session/mod.rs b/crates/uzu/src/session/mod.rs
index 8633ea0d9..0ff8b0af5 100644
--- a/crates/uzu/src/session/mod.rs
+++ b/crates/uzu/src/session/mod.rs
@@ -3,7 +3,11 @@ pub mod classification_session;
pub mod config;
pub mod helpers;
pub mod parameter;
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+mod tts_session;
pub mod types;
pub use chat_session::{ChatSession, ChatSession as Session};
pub use classification_session::ClassificationSession;
+#[cfg(all(feature = "audio-runtime", metal_backend))]
+pub use tts_session::TtsSession;
diff --git a/crates/uzu/src/session/parameter/mod.rs b/crates/uzu/src/session/parameter/mod.rs
index 6470f6549..ba653526b 100644
--- a/crates/uzu/src/session/parameter/mod.rs
+++ b/crates/uzu/src/session/parameter/mod.rs
@@ -10,5 +10,5 @@ pub use context_length::ContextLength;
pub use context_mode::ContextMode;
pub use prefill_step_size::PrefillStepSize;
pub use resolvable_value::{ConfigResolvableValue, ResolvableValue};
-pub use sampling_policy::{SamplingMethod, SamplingPolicy};
+pub use sampling_policy::{SamplingMethod, SamplingPolicy, SamplingProcessingOrder};
pub use sampling_seed::SamplingSeed;
diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rs
index 94d8d5b73..e60c6e1e3 100644
--- a/crates/uzu/src/session/parameter/sampling_policy.rs
+++ b/crates/uzu/src/session/parameter/sampling_policy.rs
@@ -1,5 +1,11 @@
use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub enum SamplingProcessingOrder {
+ TemperatureThenFilters,
+ FiltersThenTemperature,
+}
+
#[derive(Debug, Copy, Clone, PartialEq)]
pub enum SamplingMethod {
Greedy,
@@ -8,6 +14,7 @@ pub enum SamplingMethod {
top_k: Option<u32>,
top_p: Option<f32>,
min_p: Option<f32>,
+ processing_order: SamplingProcessingOrder,
},
}
@@ -43,6 +50,7 @@ impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod> for SamplingPoli
top_k: generation_config.top_k,
top_p: generation_config.top_p,
min_p: generation_config.min_p,
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
SamplingPolicy::Custom {
value,
diff --git a/crates/uzu/src/session/tts_session.rs b/crates/uzu/src/session/tts_session.rs
new file mode 100644
index 000000000..040cefdd3
--- /dev/null
+++ b/crates/uzu/src/session/tts_session.rs
@@ -0,0 +1,236 @@
+#![cfg(all(feature = "audio-runtime", metal_backend))]
+
+mod audio_backend;
+mod backend_factory;
+mod decoder_runtime;
+mod decoder_support;
+mod fishaudio;
+mod generator;
+
+use std::{
+ cell::RefCell,
+ collections::HashMap,
+ fs::File,
+ path::{Path, PathBuf},
+ rc::Rc,
+ time::Instant,
+};
+
+use backend_factory::load_tts_runtime;
+use decoder_runtime::*;
+use decoder_support::*;
+use minijinja::{Environment, context};
+use rand::{RngExt, SeedableRng, rngs::StdRng};
+use tokenizers::Tokenizer;
+
+use crate::{
+ DataType,
+ array::ArrayContextExt,
+ audio::{
+ AudioCodecRuntime, AudioGenerationContext, AudioPcmBatch, AudioTokenGrid,
+ nanocodec::{AudioDecodeStepStats, AudioDecodeStreamState},
+ },
+ backends::common::{
+ Backend, Context as BackendContext, Encoder, Kernels,
+ kernel::{
+ EmbeddingRowsSumKernel, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,
+ TokenCopyToResultsKernel,
+ kv_cache_update::KVCacheUpdate,
+ matmul::{MatmulArguments, MatmulKernel, MatmulKernels},
+ },
+ },
+ config::{InnerModelConfig, ModelMetadata, TtsMessageProcessorConfig},
+ encodable_block::{Decoder, EncodingParameters, Sampling as GpuSampling},
+ forward_pass::{
+ cache_layers::CacheLayers,
+ model_shape::ModelShape,
+ scratch_buffers::ScratchBuffers,
+ state::{ArrayId, ForwardPassState, SharedBuffers},
+ },
+ parameters::ParameterLoader,
+ session::{
+ config::{TextDecoderRuntimeConfig, TextSamplingConfig, TtsChunkPolicy, TtsRunConfig, TtsSessionOptions},
+ parameter::{ConfigResolvableValue, SamplingMethod, SamplingProcessingOrder},
+ types::{Error, Input},
+ },
+};
+
+fn unable_to_create_context<E: std::error::Error + 'static>(err: E) -> Error {
+ Error::UnableToCreateContext(Box::new(err))
+}
+
+#[derive(Debug, Clone, Default, PartialEq)]
+pub struct TtsExecutionStats {
+ pub semantic_decode_seconds: f64,
+ pub audio_decode_seconds: f64,
+ pub callback_seconds: f64,
+ pub first_chunk_seconds: f64,
+ pub command_buffers_submitted: usize,
+ pub host_waits: usize,
+ pub semantic_frames: usize,
+ pub first_chunk_frames: usize,
+ pub emitted_chunks: usize,
+ pub audio_decode_calls: usize,
+ pub audio_input_frames: usize,
+ pub audio_decoded_window_frames: usize,
+ pub audio_max_decoded_window_frames: usize,
+}
+
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub(super) struct RunnerInstrumentation {
+ pub(super) command_buffers_submitted: usize,
+ pub(super) host_waits: usize,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub(super) struct AdaptiveChunkController {
+ pub(super) ema_ms_per_frame: Option<f64>,
+ pub(super) current_chunk_frames: usize,
+}
+
+pub struct TtsSession<B: Backend> {
+ tokenizer: Tokenizer,
+ audio: AudioGenerationContext<B>,
+ audio_decoder: Box<dyn AudioDecoderBackend>,
+ message_processor_config: TtsMessageProcessorConfig,
+ text_decoder: Box<dyn SemanticDecoderBackend>,
+ last_execution_stats: Option<TtsExecutionStats>,
+}
+
+trait SemanticDecoderBackend {
+ fn generate_semantic_tokens(
+ &mut self,
+ text_tokens: &[u64],
+ codec_cardinality: usize,
+ seed: u64,
+ max_semantic_frames: usize,
+ ) -> Result<AudioTokenGrid, Error>;
+
+ fn generate_semantic_tokens_with_callback(
+ &mut self,
+ text_tokens: &[u64],
+ codec_cardinality: usize,
+ seed: u64,
+ max_semantic_frames: usize,
+ on_frame: &mut dyn FnMut(&[u32]) -> Result<(), Error>,
+ ) -> Result<AudioTokenGrid, Error>;
+
+ fn take_instrumentation(&mut self) -> RunnerInstrumentation;
+}
+
+trait AudioDecoderBackend {
+ fn codec_cardinality(&self) -> usize;
+
+ fn num_codebooks(&self) -> usize;
+
+ fn sample_rate(&self) -> u32;
+
+ fn decode(
+ &self,
+ tokens: &AudioTokenGrid,
+ ) -> Result<AudioPcmBatch, Error>;
+
+ fn begin_stream(
+ &self,
+ batch_size: usize,
+ codebooks: usize,
+ max_workspace_frames: usize,
+ ) -> Result<Box<dyn AudioDecoderStreamBackend>, Error>;
+}
+
+trait AudioDecoderStreamBackend {
+ fn decode_step(
+ &mut self,
+ new_tokens: &AudioTokenGrid,
+ is_final: bool,
+ ) -> Result<AudioPcmBatch, Error>;
+
+ fn decode_step_pending(
+ &mut self,
+ new_tokens: &AudioTokenGrid,
+ is_final: bool,
+ ) -> Result<Box<dyn PendingAudioChunkBackend>, Error> {
+ let pcm = self.decode_step(new_tokens, is_final)?;
+ Ok(Box::new(ImmediatePendingAudioChunk {
+ pcm: Some(pcm),
+ step_stats: self.last_step_stats(),
+ }))
+ }
+
+ fn last_step_stats(&self) -> Option<AudioDecodeStepStats>;
+
+ fn finish(self: Box<Self>) -> Result<(), Error>;
+}
+
+pub(super) trait PendingAudioChunkBackend {
+ fn step_stats(&self) -> Option<AudioDecodeStepStats>;
+
+ fn try_resolve(&mut self) -> Result<Option<AudioPcmBatch>, Error>;
+
+ fn resolve(self: Box<Self>) -> Result<AudioPcmBatch, Error>;
+
+ fn resolve_with_decode_duration(self: Box<Self>) -> Result<(AudioPcmBatch, std::time::Duration), Error> {
+ let start = Instant::now();
+ let pcm = self.resolve()?;
+ Ok((pcm, start.elapsed()))
+ }
+
+ fn try_resolve_with_decode_duration(&mut self) -> Result<Option<(AudioPcmBatch, std::time::Duration)>, Error> {
+ let start = Instant::now();
+ let Some(pcm) = self.try_resolve()? else {
+ return Ok(None);
+ };
+ Ok(Some((pcm, start.elapsed())))
+ }
+}
+
+struct ImmediatePendingAudioChunk {
+ pcm: Option<AudioPcmBatch>,
+ step_stats: Option<AudioDecodeStepStats>,
+}
+
+impl PendingAudioChunkBackend for ImmediatePendingAudioChunk {
+ fn step_stats(&self) -> Option<AudioDecodeStepStats> {
+ self.step_stats
+ }
+
+ fn try_resolve(&mut self) -> Result<Option<AudioPcmBatch>, Error> {
+ Ok(Some(self.pcm.take().ok_or(Error::GenerateFailed)?))
+ }
+
+ fn resolve(mut self: Box<Self>) -> Result<AudioPcmBatch, Error> {
+ self.pcm.take().ok_or(Error::GenerateFailed)
+ }
+}
+
+struct NanoCodecPendingAudioChunk<B: Backend> {
+ inner: Option<crate::audio::nanocodec::runtime::PendingStreamPcmChunk<B>>,
+}
+
+impl<B: Backend> PendingAudioChunkBackend for NanoCodecPendingAudioChunk<B> {
+ fn step_stats(&self) -> Option<AudioDecodeStepStats> {
+ self.inner.as_ref().map(|pending| pending.step_stats())
+ }
+
+ fn try_resolve(&mut self) -> Result<Option<AudioPcmBatch>, Error> {
+ let ready = self.inner.as_ref().is_none_or(|pending| pending.is_complete());
+ if !ready {
+ return Ok(None);
+ }
+
+ let pending = self.inner.take().ok_or(Error::GenerateFailed)?;
+ pending.resolve().map(Some).map_err(Error::from)
+ }
+
+ fn resolve(mut self: Box<Self>) -> Result<AudioPcmBatch, Error> {
+ let pending = self.inner.take().ok_or(Error::GenerateFailed)?;
+ pending.resolve().map_err(Error::from)
+ }
+}
+
+pub(super) struct PendingStreamingChunk {
+ pub(super) submission_decode_duration: std::time::Duration,
+ pub(super) ready_frames: usize,
+ pub(super) next_chunk_frames: usize,
+ pub(super) chunk: Box<dyn PendingAudioChunkBackend>,
+}
diff --git a/crates/uzu/src/session/tts_session/audio_backend.rs b/crates/uzu/src/session/tts_session/audio_backend.rs
new file mode 100644
index 000000000..5e3ea6663
--- /dev/null
+++ b/crates/uzu/src/session/tts_session/audio_backend.rs
@@ -0,0 +1,102 @@
+use super::*;
+
+pub(super) struct NanoCodecAudioDecoderBackend<B: Backend> {
+ audio: AudioGenerationContext<B>,
+ codec_cardinality: usize,
+}
+
+pub(super) struct NanoCodecAudioDecoderStream<B: Backend> {
+ audio: AudioGenerationContext<B>,
+ state: Option<AudioDecodeStreamState>,
+}
+
+impl<B: Backend> NanoCodecAudioDecoderBackend<B> {
+ pub(super) fn new(audio: AudioGenerationContext<B>) -> Self {
+ let codec_cardinality = audio.codec_cardinality();
+ Self {
+ audio,
+ codec_cardinality,
+ }
+ }
+}
+
+impl<B: Backend + Send + Sync> AudioDecoderBackend for NanoCodecAudioDecoderBackend<B> {
+ fn codec_cardinality(&self) -> usize {
+ self.codec_cardinality
+ }
+
+ fn num_codebooks(&self) -> usize {
+ self.audio.runtime().config().num_groups()
+ }
+
+ fn sample_rate(&self) -> u32 {
+ self.audio.sample_rate()
+ }
+
+ fn decode(
+ &self,
+ tokens: &AudioTokenGrid,
+ ) -> Result<AudioPcmBatch, Error> {
+ self.audio.runtime().decode(tokens).map_err(Error::from)
+ }
+
+ fn begin_stream(
+ &self,
+ batch_size: usize,
+ codebooks: usize,
+ max_workspace_frames: usize,
+ ) -> Result<Box<dyn AudioDecoderStreamBackend>, Error> {
+ let state = self
+ .audio
+ .runtime()
+ .begin_decode_stream_with_options(batch_size, codebooks, max_workspace_frames)
+ .map_err(Error::from)?;
+ Ok(Box::new(NanoCodecAudioDecoderStream {
+ audio: self.audio.clone(),
+ state: Some(state),
+ }))
+ }
+}
+
+impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {
+ fn decode_step(
+ &mut self,
+ new_tokens: &AudioTokenGrid,
+ is_final: bool,
+ ) -> Result<AudioPcmBatch, Error> {
+ let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;
+ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;
+ self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)
+ }
+
+ fn decode_step_pending(
+ &mut self,
+ new_tokens: &AudioTokenGrid,
+ is_final: bool,
+ ) -> Result<Box<dyn PendingAudioChunkBackend>, Error> {
+ let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;
+ if let Some(pending) =
+ self.audio.runtime().submit_decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?
+ {
+ return Ok(Box::new(NanoCodecPendingAudioChunk {
+ inner: Some(pending),
+ }));
+ }
+
+ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;
+ let pcm = self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)?;
+ Ok(Box::new(ImmediatePendingAudioChunk {
+ pcm: Some(pcm),
+ step_stats: Some(state.last_step_stats()),
+ }))
+ }
+
+ fn last_step_stats(&self) -> Option<AudioDecodeStepStats> {
+ self.state.as_ref().map(AudioDecodeStreamState::last_step_stats)
+ }
+
+ fn finish(mut self: Box<Self>) -> Result<(), Error> {
+ let state = self.state.take().ok_or(Error::GenerateFailed)?;
+ self.audio.runtime().end_decode_stream(state).map_err(Error::from)
+ }
+}
diff --git a/crates/uzu/src/session/tts_session/backend_factory.rs b/crates/uzu/src/session/tts_session/backend_factory.rs
new file mode 100644
index 000000000..82b2f11fe
--- /dev/null
+++ b/crates/uzu/src/session/tts_session/backend_factory.rs
@@ -0,0 +1,63 @@
+use super::*;
+use crate::{
+ config::{ModelType, TtsModelConfig, TtsTextDecoderConfig},
+ session::types::TtsModelConfigError,
+};
+
+pub(super) struct LoadedTtsRuntime<B: Backend> {
+ pub(super) audio: AudioGenerationContext<B>,
+ pub(super) audio_decoder: Box<dyn AudioDecoderBackend>,
+ pub(super) text_decoder: Box<dyn SemanticDecoderBackend>,
+ pub(super) message_processor_config: TtsMessageProcessorConfig,
+}
+
+pub(super) fn build_audio_decoder_backend<B: Backend + Send + Sync>(
+ audio: &AudioGenerationContext<B>
+) -> Result<Box<dyn AudioDecoderBackend>, Error> {
+ Ok(Box::new(audio_backend::NanoCodecAudioDecoderBackend::new(audio.clone())))
+}
+
+pub(super) fn load_tts_runtime<B: Backend + Send + Sync>(
+ model_path: &Path,
+ model_metadata: &ModelMetadata,
+ options: &TtsSessionOptions,
+) -> Result<LoadedTtsRuntime<B>, Error> {
+ if model_metadata.model_type != ModelType::TtsModel {
+ return Err(TtsModelConfigError::ModelTypeMismatch {
+ expected: ModelType::TtsModel,
+ actual: model_metadata.model_type.clone(),
+ }
+ .into());
+ }
+
+ let tts_model_config =
+ model_metadata.model_config.as_tts().ok_or(TtsModelConfigError::MissingTtsModelConfig)?.clone();
+ let audio: AudioGenerationContext<B> =
+ tts_model_config.create_audio_generation_context_with_model_path(model_path)?;
+ let text_decoder = build_text_decoder_backend(&tts_model_config, &audio, model_path, options)?;
+ let audio_decoder = build_audio_decoder_backend(&audio)?;
+ let message_processor_config = tts_model_config.message_processor_config.clone();
+
+ Ok(LoadedTtsRuntime {
+ audio,
+ audio_decoder,
+ text_decoder,
+ message_processor_config,
+ })
+}
+
+pub(super) fn build_text_decoder_backend<B: Backend>(
+ tts_model_config: &TtsModelConfig,
+ audio: &AudioGenerationContext<B>,
+ model_path: &Path,
+ options: &TtsSessionOptions,
+) -> Result<Box<dyn SemanticDecoderBackend>, Error> {
+ match &tts_model_config.tts_config.text_decoder_config {
+ TtsTextDecoderConfig::FishAudioTextDecoderConfig {
+ config,
+ } => {
+ fishaudio::validate_fishaudio_message_processor_config(&tts_model_config.message_processor_config)?;
+ fishaudio::build_fishaudio_text_decoder_runtime(config, audio, model_path, &options.text_decoder)
+ },
+ }
+}
diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rs
new file mode 100644
index 000000000..a236b2207
--- /dev/null
+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs
@@ -0,0 +1,1168 @@
+use super::{decoder_support::*, *};
+use crate::{array::Array, backends::common::Buffer, session::types::TtsModelConfigError};
+
+struct TokenDecoderLoadedModel<B: Backend> {
+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
+ scratch_buffers: ScratchBuffers<B>,
+ executables: Decoder<B>,
+ sampler: GpuSampling<B>,
+ token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
+ token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,
+}
+
+impl<B: Backend> TokenDecoderLoadedModel<B> {
+ fn load(
+ context: &Rc<B::Context>,
+ model_path: &Path,
+ decoder_config: &Rc<crate::config::DecoderConfig>,
+ model_shape: &ModelShape,
+ transformer_subtree: &str,
+ embedding_subtree: &str,
+ readout_subtree: &str,
+ max_prefix_length: usize,
+ max_suffix_length: usize,
+ ) -> Result<Self, Error> {
+ let weights_path = model_path.join("model.safetensors");
+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
+ let root_loader_view = loader.tree();
+
+ let shared_buffers = TokenDecoderContext::<B>::build_shared_buffers(
+ context,
+ decoder_config,
+ model_shape,
+ &root_loader_view,
+ transformer_subtree,
+ )?;
+ let scratch_buffers =
+ ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_prefix_length, max_suffix_length);
+ let executables = Decoder::new_with_embedding_and_readout_subtrees(
+ context.clone(),
+ decoder_config.clone(),
+ &root_loader_view,
+ transformer_subtree,
+ embedding_subtree,
+ readout_subtree,
+ );
+ let logits_data_type = scratch_buffers.logits.data_type();
+ let sampler =
+ GpuSampling::new(context.as_ref(), logits_data_type, max_suffix_length, decoder_config.vocab_size)
+ .map_err(unable_to_create_context)?;
+ let token_copy_sampled =
+ <B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;
+ let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(context.as_ref())
+ .map_err(unable_to_create_context)?;
+
+ Ok(Self {
+ shared_buffers,
+ scratch_buffers,
+ executables,
+ sampler,
+ token_copy_sampled,
+ token_copy_results,
+ })
+ }
+}
+
+struct TokenDecoderContext<B: Backend> {
+ context: Rc<B::Context>,
+ cache_layers: Rc<RefCell<CacheLayers<B>>>,
+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,
+ scratch_buffers: ScratchBuffers<B>,
+ model_shape: ModelShape,
+ decoder_config: Rc<crate::config::DecoderConfig>,
+ executables: Decoder<B>,
+ sampler: GpuSampling<B>,
+ kv_cache_update: KVCacheUpdate<B>,
+ token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,
+ token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,
+ async_chain_positions: Rc<RefCell<B::Buffer>>,
+ async_chain_seeds: Rc<RefCell<B::Buffer>>,
+ async_chain_results: Rc<RefCell<B::Buffer>>,
+ async_chain_capacity: usize,
+}
+
+impl<B: Backend> TokenDecoderContext<B> {
+ pub(super) fn context(&self) -> &Rc<B::Context> {
+ &self.context
+ }
+
+ fn new(
+ context: Rc<B::Context>,
+ model_path: &Path,
+ decoder_config: Rc<crate::config::DecoderConfig>,
+ transformer_subtree: &str,
+ embedding_subtree: &str,
+ readout_subtree: &str,
+ runtime_config: &TextDecoderRuntimeConfig,
+ ) -> Result<(Self, DataType, bool), Error> {
+ let model_shape = ModelShape::from_decoder_config(&decoder_config);
+ let max_prefix_length = decoder_config.context_length;
+ let max_suffix_length = text_decoder_prefill_step_size(runtime_config, decoder_config.context_length).max(32);
+ let should_fill_attention_bias =
+ model_shape.sliding_window_length_per_layer.iter().any(|value| value.is_some());
+ let activation_data_type = model_shape.activation_data_type();
+ let loaded_model = TokenDecoderLoadedModel::<B>::load(
+ &context,
+ model_path,
+ &decoder_config,
+ &model_shape,
+ transformer_subtree,
+ embedding_subtree,
+ readout_subtree,
+ max_prefix_length,
+ max_suffix_length,
+ )?;
+ let async_chain_capacity = max_suffix_length.max(1);
+ let (async_chain_positions, async_chain_seeds, async_chain_results) =
+ Self::build_async_chain_buffers(&context, async_chain_capacity)?;
+ let cache_layers = Rc::new(RefCell::new(CacheLayers::new(
+ context.as_ref(),
+ &model_shape,
+ max_prefix_length,
+ max_suffix_length,
+ )));
+ let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();
+ let kv_cache_update = KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)
+ .map_err(unable_to_create_context)?;
+
+ Ok((
+ Self {
+ context,
+ cache_layers,
+ shared_buffers: loaded_model.shared_buffers,
+ scratch_buffers: loaded_model.scratch_buffers,
+ model_shape,
+ decoder_config,
+ executables: loaded_model.executables,
+ sampler: loaded_model.sampler,
+ kv_cache_update,
+ token_copy_sampled: loaded_model.token_copy_sampled,
+ token_copy_results: loaded_model.token_copy_results,
+ async_chain_positions,
+ async_chain_seeds,
+ async_chain_results,
+ async_chain_capacity,
+ },
+ activation_data_type,
+ should_fill_attention_bias,
+ ))
+ }
+
+ fn build_shared_buffers(
+ context: &Rc<B::Context>,
+ decoder_config: &Rc<crate::config::DecoderConfig>,
+ model_shape: &ModelShape,
+ root_loader_view: &crate::parameters::ParameterTree<B::Context>,
+ transformer_subtree: &str,
+ ) -> Result<Rc<RefCell<SharedBuffers<B>>>, Error> {
+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), decoder_config, model_shape)));
+ let transformer_tree = root_loader_view.subtree(transformer_subtree).map_err(|_| Error::UnableToLoadWeights)?;
+ let mut shared_buffers_borrow = shared_buffers.borrow_mut();
+ if let Some(global_rope) = &mut shared_buffers_borrow.global_rope {
+ global_rope.update_data(&transformer_tree, "global_rope");
+ }
+ if let Some(local_rope) = &mut shared_buffers_borrow.local_rope {
+ local_rope.update_data(&transformer_tree, "local_rope");
+ }
+ drop(shared_buffers_borrow);
+ Ok(shared_buffers)
+ }
+
+ fn build_async_chain_buffers(
+ context: &Rc<B::Context>,
+ async_chain_capacity: usize,
+ ) -> Result<(Rc<RefCell<B::Buffer>>, Rc<RefCell<B::Buffer>>, Rc<RefCell<B::Buffer>>), Error> {
+ let positions = Rc::new(RefCell::new(
+ context
+ .create_buffer(async_chain_capacity * std::mem::size_of::<i32>())
+ .map_err(unable_to_create_context)?,
+ ));
+ let seeds = Rc::new(RefCell::new(
+ context
+ .create_buffer(async_chain_capacity * std::mem::size_of::<u64>())
+ .map_err(unable_to_create_context)?,
+ ));
+ let results = Rc::new(RefCell::new(
+ context
+ .create_buffer(async_chain_capacity * std::mem::size_of::<u32>())
+ .map_err(unable_to_create_context)?,
+ ));
+ Ok((positions, seeds, results))
+ }
+}
+
+pub(super) struct TokenDecoderRunner<B: Backend> {
+ ctx: TokenDecoderContext<B>,
+ pub(super) single_hidden_capture: Array<B>,
+ pub(super) single_override_embedding: Array<B>,
+ tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,
+ tensor_add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,
+ single_token_vocab_masks: HashMap<usize, Box<[u32]>>,
+ two_token_vocab_masks: HashMap<usize, Box<[u32]>>,
+ should_fill_attention_bias: bool,
+ next_position: usize,
+ instrumentation: RunnerInstrumentation,
+}
+
+impl<B: Backend> TokenDecoderRunner<B> {
+ pub(super) fn context(&self) -> &Rc<B::Context> {
+ self.ctx.context()
+ }
+
+ pub(super) fn new_with_context(
+ context: Rc<B::Context>,
+ model_path: &Path,
+ decoder_config: Rc<crate::config::DecoderConfig>,
+ transformer_subtree: &str,
+ embedding_subtree: &str,
+ readout_subtree: &str,
+ runtime_config: &TextDecoderRuntimeConfig,
+ ) -> Result<Self, Error> {
+ let (ctx, activation_data_type, should_fill_attention_bias) = TokenDecoderContext::new(
+ context,
+ model_path,
+ decoder_config,
+ transformer_subtree,
+ embedding_subtree,
+ readout_subtree,
+ runtime_config,
+ )?;
+ let context: Rc<B::Context> = Rc::clone(ctx.context());
+ let model_dim = ctx.decoder_config.model_dim;
+ let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(context.as_ref(), activation_data_type)
+ .map_err(unable_to_create_context)?;
+ let tensor_add_scale =
+ <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type)
+ .map_err(unable_to_create_context)?;
+ let single_hidden_capture =
+ context.create_array(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");
+ let single_override_embedding =
+ context.create_array(&[1, model_dim], activation_data_type, "tts_single_override_embedding");
+
+ Ok(Self {
+ ctx,
+ single_hidden_capture,
+ single_override_embedding,
+ tensor_copy,
+ tensor_add_scale,
+ single_token_vocab_masks: HashMap::new(),
+ two_token_vocab_masks: HashMap::new(),
+ should_fill_attention_bias,
+ next_position: 0,
+ instrumentation: RunnerInstrumentation::default(),
+ })
+ }
+
+ pub(super) fn reset(&mut self) {
+ self.ctx.cache_layers.borrow_mut().clear();
+ self.next_position = 0;
+ }
+
+ pub(super) fn prefill_without_sampling(
+ &mut self,
+ token_ids: &[u64],
+ ) -> Result<(), Error> {
+ if token_ids.is_empty() {
+ return Ok(());
+ }
+
+ objc2::rc::autoreleasepool(|_| {
+ let token_count = token_ids.len();
+
+ let positions: Vec<usize> = (self.next_position..self.next_position + token_count).collect();
+ let token_seeds = vec![0_u64; token_count];
+
+ let mut state = ForwardPassState::new_llm(
+ self.ctx.context.clone(),
+ &self.ctx.decoder_config,
+ &self.ctx.model_shape,
+ &self.ctx.scratch_buffers,
+ self.ctx.cache_layers.clone(),
+ self.ctx.shared_buffers.clone(),
+ token_ids,
+ &positions,
+ None, // no bitmask
+ &token_seeds,
+ token_count,
+ 0, // sampling_start: irrelevant
+ 0, // sampling_length: no sampling
+ false,
+ None,
+ false,
+ self.should_fill_attention_bias,
+ None,
+ None,
+ );
+
+ let encoding_parameters = EncodingParameters::new();
+
+ let context = Rc::clone(&self.ctx.context);
+ let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
+
+ self.ctx
+ .executables
+ .embed
+ .encode_lookup(&mut state, &mut encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ for layer in self.ctx.executables.layers.iter() {
+ layer
+ .encode(&mut state, &encoding_parameters, &mut encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ }
+ self.encode_cache_acceptance_update_on(&mut encoder, token_count);
+
+ self.submit_and_wait_command_buffer(encoder)?;
+ self.register_positions_and_advance(token_count);
+ Ok(())
+ })
+ }
+
+ pub(super) fn decode_next_token_with_hidden_capture(
+ &mut self,
+ token_ids: &[u64],
+ embedding_injection: EmbeddingInjection,
+ sampling: &mut TextSamplingState,
+ precomputed_token_bitmask: Option<&[u32]>,
+ ) -> Result<u64, Error> {
+ self.decode_next_step(token_ids, embedding_injection, None, sampling, precomputed_token_bitmask, true, None)
+ }
+
+ /// Encode the first fast token plus followup passes onto the given
+ /// command buffer, without submitting or reading results. The caller
+ /// must submit the command buffer later (potentially after encoding
+ /// more work from another runner).
+ ///
+ /// Returns `total_count = 1 + followup_count`. After the eventual
+ /// submit+wait the results live in `async_chain_results[0..total_count]`.
+ pub(super) fn encode_first_and_followup_tokens_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ initial_token_ids: &[u64],
+ initial_embedding_injection: EmbeddingInjection,
+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ followup_count: usize,
+ vocab_limit: Option<usize>,
+ sampling: &mut TextSamplingState,
+ ) -> Result<usize, Error> {
+ let total_count =
+ 1_usize.checked_add(followup_count).ok_or(TtsModelConfigError::AsyncChainFollowupCountOverflow)?;
+ if total_count > self.ctx.async_chain_capacity {
+ return Err(TtsModelConfigError::AsyncChainTokenCountExceedsCapacity {
+ total_count,
+ capacity: self.ctx.async_chain_capacity,
+ }
+ .into());
+ }
+
+ let vocab_mask_limit = if let Some(limit_raw) = vocab_limit {
+ let limit = limit_raw.min(self.ctx.decoder_config.vocab_size);
+ if limit == 0 || limit >= self.ctx.decoder_config.vocab_size {
+ None
+ } else {
+ self.prepare_two_token_vocab_mask(limit)?;
+ self.prepare_single_token_vocab_mask(limit)?;
+ Some(limit)
+ }
+ } else {
+ None
+ };
+
+ let initial_token_count = initial_token_ids.len();
+ if initial_token_count == 0 {
+ return Err(Error::GenerateFailed);
+ }
+
+ // Resolve the bitmask into a local copy so that it does not borrow
+ // from `self` when we later call `encode_single_forward_pass`.
+ let initial_token_bitmask_owned: Option<Box<[u32]>> = if let Some(limit) = vocab_mask_limit {
+ if initial_token_count == 1 {
+ self.get_single_token_vocab_mask(limit).map(|m| m.into())
+ } else if initial_token_count == 2 {
+ self.get_two_token_vocab_mask(limit).map(|m| m.into())
+ } else {
+ return Err(TtsModelConfigError::UnsupportedInitialTokenCountForVocabMask {
+ initial_token_count,
+ }
+ .into());
+ }
+ } else {
+ None
+ };
+ let initial_token_bitmask: Option<&[u32]> = initial_token_bitmask_owned.as_deref();
+
+ // Consume the initial seed FIRST to preserve RNG ordering with the
+ // old two-submit path (initial token consumed its seed before followups).
+ let initial_seed = if !matches!(sampling.method(), SamplingMethod::Greedy) {
+ Some(sampling.next_seed())
+ } else {
+ None
+ };
+
+ // Pre-fill followup positions and seeds into the async chain buffers.
+ let followup_base_position = self.next_position + initial_token_ids.len();
+ {
+ let positions_borrow = self.ctx.async_chain_positions.borrow();
+ let positions_ptr = positions_borrow.cpu_ptr().as_ptr() as *mut i32;
+ for pass in 0..followup_count {
+ unsafe {
+ *positions_ptr.add(pass) = (followup_base_position + pass) as i32;
+ }
+ }
+ }
+ {
+ let seeds_borrow = self.ctx.async_chain_seeds.borrow();
+ let seeds_ptr = seeds_borrow.cpu_ptr().as_ptr() as *mut u64;
+ if !matches!(sampling.method(), SamplingMethod::Greedy) {
+ for pass in 0..followup_count {
+ unsafe {
+ *seeds_ptr.add(pass) = sampling.next_seed();
+ }
+ }
+ } else {
+ for pass in 0..followup_count {
+ unsafe {
+ *seeds_ptr.add(pass) = 0;
+ }
+ }
+ }
+ }
+
+ // Encode the initial forward pass (no new CB, no submit).
+ let state = self.encode_single_forward_pass_on(
+ encoder,
+ initial_token_ids,
+ initial_embedding_injection,
+ sampling,
+ initial_token_bitmask,
+ false, // capture_hidden
+ pre_injection_encode,
+ initial_seed,
+ )?;
+
+ // Copy the sampled token for chaining and into results[0].
+ {
+ let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;
+ let sampling_output_buffer = sampling_output.buffer();
+ let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();
+
+ let sampling_output_buffer = sampling_output_buffer.borrow();
+ let mut token_ids_buffer = token_ids_buffer.borrow_mut();
+ if followup_count > 0 {
+ self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);
+ }
+ let mut results_buffer = self.ctx.async_chain_results.borrow_mut();
+ self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);
+ }
+
+ self.encode_cache_acceptance_update_on(encoder, initial_token_count);
+ self.register_positions_and_advance(initial_token_count);
+
+ if followup_count > 0 {
+ self.encode_followup_passes_on(encoder, followup_count, 0, vocab_mask_limit, sampling.method(), 1)?;
+ }
+
+ self.next_position = followup_base_position.saturating_add(followup_count);
+ // Do NOT submit -- the caller will submit after encoding more work.
+ Ok(total_count)
+ }
+
+ /// Encode `followup_count` autoregressive passes onto the current command
+ /// buffer (which must already be open). `first_token` is the embedding
+ /// input for the first pass; subsequent passes read from the
+ /// `token_copy_sampled` output of the preceding pass.
+ ///
+ /// `results_offset_slots` is the slot index in `async_chain_results` where
+ /// the first pass's result should be written (0 for standalone, 1+ for
+ /// merged paths).
+ fn encode_followup_passes_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ followup_count: usize,
+ first_token: u64,
+ vocab_mask_limit: Option<usize>,
+ sampling_method: SamplingMethod,
+ results_offset_slots: usize,
+ ) -> Result<(), Error> {
+ let encoding_parameters = EncodingParameters::new();
+ let context_rc = self.ctx.context.clone();
+ let cache_layers_rc = self.ctx.cache_layers.clone();
+ let shared_buffers_rc = self.ctx.shared_buffers.clone();
+ let positions_rc = self.ctx.async_chain_positions.clone();
+ let seeds_rc = self.ctx.async_chain_seeds.clone();
+ let token_bitmask = vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit));
+
+ for pass in 0..followup_count {
+ let results_slot = results_offset_slots + pass;
+ let token_ids = [if pass == 0 {
+ first_token
+ } else {
+ 0
+ }];
+ let mut state = ForwardPassState::new_llm(
+ context_rc.clone(),
+ &self.ctx.decoder_config,
+ &self.ctx.model_shape,
+ &self.ctx.scratch_buffers,
+ cache_layers_rc.clone(),
+ shared_buffers_rc.clone(),
+ &token_ids,
+ &[self.next_position + pass],
+ token_bitmask,
+ &[0],
+ 1,
+ 0,
+ 1,
+ false,
+ None,
+ pass > 0 || results_offset_slots > 0,
+ self.should_fill_attention_bias,
+ Some((positions_rc.clone(), pass)),
+ Some((seeds_rc.clone(), pass)),
+ );
+ if let Some(method) = state.sampling_method_mut() {
+ *method = Some(sampling_method);
+ }
+
+ {
+ self.ctx
+ .executables
+ .embed
+ .encode_lookup(&mut state, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ for layer in self.ctx.executables.layers.iter() {
+ layer
+ .encode(&mut state, &encoding_parameters, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ }
+ self.ctx
+ .executables
+ .norm
+ .encode(&mut state, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ self.ctx
+ .executables
+ .embed
+ .encode_readout(&mut state, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ }
+
+ let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;
+ let sampling_output_buffer = sampling_output.buffer();
+ let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();
+
+ {
+ let sampling_output_buffer = sampling_output_buffer.borrow();
+ let mut token_ids_buffer = token_ids_buffer.borrow_mut();
+ if pass + 1 < followup_count {
+ self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);
+ }
+ let results_offset = results_slot * std::mem::size_of::<u32>();
+ let mut results_buffer = self.ctx.async_chain_results.borrow_mut();
+ self.ctx.token_copy_results.encode(
+ &*sampling_output_buffer,
+ (&mut *results_buffer, results_offset),
+ encoder,
+ );
+ }
+
+ self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);
+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&[self.next_position + pass]);
+ }
+ Ok(())
+ }
+
+ pub(super) fn prepare_single_token_vocab_mask(
+ &mut self,
+ vocab_limit: usize,
+ ) -> Result<(), Error> {
+ let limit = vocab_limit.min(self.ctx.decoder_config.vocab_size);
+ if limit == 0 || limit >= self.ctx.decoder_config.vocab_size {
+ return Ok(());
+ }
+ if self.single_token_vocab_masks.contains_key(&limit) {
+ return Ok(());
+ }
+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);
+ let mut mask = vec![0_u32; row_words];
+ for token_index in 0..limit {
+ let word = token_index / 32;
+ let bit = token_index % 32;
+ mask[word] |= 1_u32 << bit;
+ }
+ self.single_token_vocab_masks.insert(limit, mask.into_boxed_slice());
+ Ok(())
+ }
+
+ pub(super) fn prepare_two_token_vocab_mask(
+ &mut self,
+ vocab_limit: usize,
+ ) -> Result<(), Error> {
+ let limit = vocab_limit.min(self.ctx.decoder_config.vocab_size);
+ if limit == 0 || limit >= self.ctx.decoder_config.vocab_size {
+ return Ok(());
+ }
+ if self.two_token_vocab_masks.contains_key(&limit) {
+ return Ok(());
+ }
+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);
+ let mut mask = vec![
+ 0_u32;
+ row_words.checked_mul(2).ok_or(TtsModelConfigError::TwoTokenVocabMaskSizeOverflow {
+ row_words
+ })?
+ ];
+ for token_index in 0..limit {
+ let word = token_index / 32;
+ let bit = token_index % 32;
+ mask[row_words + word] |= 1_u32 << bit;
+ }
+ self.two_token_vocab_masks.insert(limit, mask.into_boxed_slice());
+ Ok(())
+ }
+
+ fn get_single_token_vocab_mask(
+ &self,
+ vocab_limit: usize,
+ ) -> Option<&[u32]> {
+ self.single_token_vocab_masks.get(&vocab_limit).map(|mask| mask.as_ref())
+ }
+
+ fn get_two_token_vocab_mask(
+ &self,
+ vocab_limit: usize,
+ ) -> Option<&[u32]> {
+ self.two_token_vocab_masks.get(&vocab_limit).map(|mask| mask.as_ref())
+ }
+
+ fn encode_single_forward_pass_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ token_ids: &[u64],
+ embedding_injection: EmbeddingInjection,
+ sampling: &mut TextSamplingState,
+ token_bitmask: Option<&[u32]>,
+ capture_hidden: bool,
+ mut pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ preconsumed_seed: Option<u64>,
+ ) -> Result<ForwardPassState<B>, Error> {
+ if token_ids.is_empty() {
+ return Err(Error::GenerateFailed);
+ }
+
+ let token_count = token_ids.len();
+ let sampling_start = token_count - 1;
+ let sampling_length = 1usize;
+
+ let mut single_position = [0_usize; 1];
+ let mut two_positions = [0_usize; 2];
+ let positions_storage;
+ let positions: &[usize] = if token_count == 1 {
+ single_position[0] = self.next_position;
+ &single_position
+ } else if token_count == 2 {
+ two_positions[0] = self.next_position;
+ two_positions[1] = self.next_position + 1;
+ &two_positions
+ } else {
+ positions_storage = (self.next_position..self.next_position + token_count).collect::<Vec<_>>();
+ positions_storage.as_slice()
+ };
+
+ let mut single_seed = [0_u64; 1];
+ let mut two_seeds = [0_u64; 2];
+ let mut token_seeds_storage;
+ let token_seeds: &mut [u64] = if token_count == 1 {
+ &mut single_seed
+ } else if token_count == 2 {
+ &mut two_seeds
+ } else {
+ token_seeds_storage = vec![0_u64; token_count];
+ token_seeds_storage.as_mut_slice()
+ };
+ if !matches!(sampling.method(), SamplingMethod::Greedy) {
+ token_seeds[sampling_start] = preconsumed_seed.unwrap_or_else(|| sampling.next_seed());
+ }
+
+ if let Some(mask) = token_bitmask {
+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);
+ let expected_words =
+ token_count.checked_mul(row_words).ok_or(TtsModelConfigError::TokenBitmaskSizeOverflow)?;
+ if mask.len() != expected_words {
+ return Err(TtsModelConfigError::TokenBitmaskLengthMismatch {
+ actual_words: mask.len(),
+ expected_words,
+ token_count,
+ row_words,
+ }
+ .into());
+ }
+ }
+
+ if matches!(embedding_injection, EmbeddingInjection::OverrideFirstRowInternal) && capture_hidden {
+ return Err(Error::GenerateFailed);
+ }
+
+ let mut state = ForwardPassState::new_llm(
+ self.ctx.context.clone(),
+ &self.ctx.decoder_config,
+ &self.ctx.model_shape,
+ &self.ctx.scratch_buffers,
+ self.ctx.cache_layers.clone(),
+ self.ctx.shared_buffers.clone(),
+ token_ids,
+ positions,
+ token_bitmask,
+ token_seeds,
+ token_count,
+ sampling_start,
+ sampling_length,
+ false,
+ None,
+ false,
+ self.should_fill_attention_bias,
+ None,
+ None,
+ );
+ if let Some(method) = state.sampling_method_mut() {
+ *method = Some(sampling.method());
+ }
+
+ let encoding_parameters = EncodingParameters::new();
+ self.ctx
+ .executables
+ .embed
+ .encode_lookup(&mut state, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ if let Some(pre_encode) = pre_injection_encode.as_mut() {
+ pre_encode(self, &state, encoder)?;
+ }
+ match embedding_injection {
+ EmbeddingInjection::None => {},
+ EmbeddingInjection::AddPreloaded {
+ post_scale,
+ } => {
+ self.encode_add_scale_from_single_bias_on(encoder, &state, token_count, post_scale.unwrap_or(1.0))?;
+ },
+ EmbeddingInjection::OverrideFirstRowInternal => {
+ self.encode_override_first_row_from_device_on(encoder, &state, &self.single_override_embedding)?;
+ },
+ }
+ for layer in self.ctx.executables.layers.iter() {
+ layer
+ .encode(&mut state, &encoding_parameters, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ }
+ if capture_hidden {
+ self.encode_capture_last_hidden_into_single_buffer_on(encoder, &state, token_count)?;
+ }
+ self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ self.ctx
+ .executables
+ .embed
+ .encode_readout(&mut state, encoder)
+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+ self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;
+
+ Ok(state)
+ }
+
+ /// Encode the KV-cache acceptance update for `token_count` tokens onto
+ /// the current command buffer.
+ fn encode_cache_acceptance_update_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ token_count: usize,
+ ) {
+ let mut single_accepted = [0_usize; 1];
+ let two_accepted = [0_usize, 1_usize];
+ let accepted_suffix_indices_storage;
+ let accepted_suffix_indices: &[usize] = if token_count == 1 {
+ single_accepted[0] = 0;
+ &single_accepted
+ } else if token_count == 2 {
+ &two_accepted
+ } else {
+ accepted_suffix_indices_storage = (0..token_count).collect::<Vec<_>>();
+ accepted_suffix_indices_storage.as_slice()
+ };
+ self.ctx.cache_layers.borrow_mut().update_after_acceptance(
+ accepted_suffix_indices,
+ None,
+ encoder,
+ &self.ctx.kv_cache_update,
+ );
+ }
+
+ /// Register accepted token positions in the KV cache and advance
+ /// `next_position` by `token_count`.
+ fn register_positions_and_advance(
+ &mut self,
+ token_count: usize,
+ ) {
+ let positions_storage;
+ let positions: &[usize] = if token_count == 1 {
+ &[self.next_position]
+ } else if token_count == 2 {
+ positions_storage = [self.next_position, self.next_position + 1];
+ &positions_storage[..]
+ } else {
+ let v: Vec<usize> = (self.next_position..self.next_position + token_count).collect();
+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&v);
+ self.next_position = self.next_position.saturating_add(token_count);
+ return;
+ };
+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(positions);
+ self.next_position = self.next_position.saturating_add(token_count);
+ }
+
+ pub(super) fn decode_next_step(
+ &mut self,
+ token_ids: &[u64],
+ embedding_injection: EmbeddingInjection,
+ vocab_limit: Option<usize>,
+ sampling: &mut TextSamplingState,
+ precomputed_token_bitmask: Option<&[u32]>,
+ capture_hidden: bool,
+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ ) -> Result<u64, Error> {
+ objc2::rc::autoreleasepool(|_| {
+ if token_ids.is_empty() {
+ return Err(Error::GenerateFailed);
+ }
+
+ let token_count = token_ids.len();
+
+ // Resolve the token bitmask into a local copy so that it does not
+ // borrow from `self` when we later call `encode_single_forward_pass`.
+ let sampling_start = token_count - 1;
+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);
+ let token_bitmask_owned: Option<Box<[u32]>> = if let Some(mask) = precomputed_token_bitmask {
+ let expected_words =
+ token_count.checked_mul(row_words).ok_or(TtsModelConfigError::PrecomputedBitmaskSizeOverflow)?;
+ if mask.len() != expected_words {
+ return Err(TtsModelConfigError::PrecomputedBitmaskLengthMismatch {
+ actual_words: mask.len(),
+ expected_words,
+ }
+ .into());
+ }
+ Some(mask.into())
+ } else if let Some(limit_raw) = vocab_limit {
+ let limit = limit_raw.min(self.ctx.decoder_config.vocab_size);
+ if limit == 0 {
+ return Err(TtsModelConfigError::VocabLimitResolvedToZero.into());
+ }
+ if limit >= self.ctx.decoder_config.vocab_size {
+ None
+ } else if token_count == 1 {
+ if let Some(mask) = self.get_single_token_vocab_mask(limit) {
+ Some(mask.into())
+ } else {
+ let mut mask = vec![0_u32; row_words];
+ for token_index in 0..limit {
+ let word = token_index / 32;
+ let bit = token_index % 32;
+ mask[word] |= 1_u32 << bit;
+ }
+ Some(mask.into_boxed_slice())
+ }
+ } else if token_count == 2 {
+ if let Some(mask) = self.get_two_token_vocab_mask(limit) {
+ Some(mask.into())
+ } else {
+ let total_words = token_count.checked_mul(row_words).ok_or(
+ TtsModelConfigError::InlineBitmaskSizeOverflow {
+ token_count,
+ row_words,
+ },
+ )?;
+ let mut mask = vec![0_u32; total_words];
+ for token_index in 0..limit {
+ let word = token_index / 32;
+ let bit = token_index % 32;
+ mask[sampling_start * row_words + word] |= 1_u32 << bit;
+ }
+ Some(mask.into_boxed_slice())
+ }
+ } else {
+ // token_count > 2 with vocab_limit requires a precomputed bitmask;
+ // silently dropping the limit would produce incorrect sampling.
+ return Err(TtsModelConfigError::VocabLimitRequiresPrecomputedBitmask {
+ token_count,
+ }
+ .into());
+ }
+ } else {
+ None
+ };
+
+ let token_bitmask: Option<&[u32]> = token_bitmask_owned.as_deref();
+
+ let context = Rc::clone(&self.ctx.context);
+ let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
+
+ let state = self.encode_single_forward_pass_on(
+ &mut encoder,
+ token_ids,
+ embedding_injection,
+ sampling,
+ token_bitmask,
+ capture_hidden,
+ pre_injection_encode,
+ None,
+ )?;
+ self.encode_cache_acceptance_update_on(&mut encoder, token_count);
+
+ self.submit_and_wait_command_buffer(encoder)?;
+ let token = read_sampled_token_from_sampling_output(&state)?;
+ self.register_positions_and_advance(token_count);
+ Ok(token)
+ })
+ }
+
+ fn encode_capture_last_hidden_into_single_buffer_on(
+ &self,
+ encoder: &mut Encoder<B>,
+ state: &ForwardPassState<B>,
+ token_count: usize,
+ ) -> Result<(), Error> {
+ if token_count == 0 {
+ return Err(Error::GenerateFailed);
+ }
+ let model_dim = self.ctx.decoder_config.model_dim;
+ let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
+ model_dim,
+ })?;
+ let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let bytes_per_element = main.data_type().size_in_bytes();
+ let row_offset = (token_count - 1)
+ .checked_mul(model_dim)
+ .and_then(|value| value.checked_mul(bytes_per_element))
+ .ok_or(TtsModelConfigError::HiddenCaptureRowOffsetOverflow {
+ token_count,
+ model_dim,
+ })?;
+ let src_offset =
+ main.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;
+ let capture = &self.single_hidden_capture;
+ if capture.shape() != [1, model_dim] || capture.data_type() != main.data_type() {
+ return Err(TtsModelConfigError::HiddenCaptureTensorMismatch {
+ expected_shape: [1, model_dim].into(),
+ expected_data_type: main.data_type(),
+ actual_shape: capture.shape().into(),
+ actual_data_type: capture.data_type(),
+ }
+ .into());
+ }
+
+ let main_buffer = main.buffer();
+ let main_buffer = main_buffer.borrow();
+ let capture_buffer = capture.buffer();
+ let mut capture_buffer = capture_buffer.borrow_mut();
+ self.tensor_copy.encode((&*main_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);
+ Ok(())
+ }
+
+ fn encode_override_first_row_from_device_on(
+ &self,
+ encoder: &mut Encoder<B>,
+ state: &ForwardPassState<B>,
+ override_embedding: &Array<B>,
+ ) -> Result<(), Error> {
+ let model_dim = self.ctx.decoder_config.model_dim;
+ let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
+ model_dim,
+ })?;
+ let main = state.arrays(&[ArrayId::Main])[0].clone();
+ if override_embedding.shape() != [1, model_dim] || override_embedding.data_type() != main.data_type() {
+ return Err(TtsModelConfigError::OverrideEmbeddingTensorMismatch {
+ expected_shape: [1, model_dim].into(),
+ expected_data_type: main.data_type(),
+ actual_shape: override_embedding.shape().into(),
+ actual_data_type: override_embedding.data_type(),
+ }
+ .into());
+ }
+
+ let override_buffer = override_embedding.buffer();
+ let override_buffer = override_buffer.borrow();
+ let main_buffer = main.buffer();
+ let mut main_buffer = main_buffer.borrow_mut();
+ self.tensor_copy.encode(
+ (&*override_buffer, override_embedding.offset()),
+ (&mut *main_buffer, main.offset()),
+ model_dim_u32,
+ encoder,
+ );
+ Ok(())
+ }
+
+ fn encode_add_scale_from_single_bias_on(
+ &self,
+ encoder: &mut Encoder<B>,
+ state: &ForwardPassState<B>,
+ token_count: usize,
+ scale: f32,
+ ) -> Result<(), Error> {
+ if token_count == 0 {
+ return Err(Error::GenerateFailed);
+ }
+ let model_dim = self.ctx.decoder_config.model_dim;
+ let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {
+ model_dim,
+ })?;
+ let total_len = token_count.checked_mul(model_dim).ok_or(TtsModelConfigError::AddScaleTotalLengthOverflow {
+ token_count,
+ model_dim,
+ })?;
+ let total_len_u32 =
+ u32::try_from(total_len).map_err(|_| TtsModelConfigError::AddScaleTotalLengthExceedsU32 {
+ total_len,
+ })?;
+
+ let main = state.arrays(&[ArrayId::Main])[0].clone();
+ let bias = &self.single_override_embedding;
+ if bias.shape() != [1, model_dim] || bias.data_type() != main.data_type() {
+ return Err(TtsModelConfigError::AddScaleBiasTensorMismatch {
+ expected_shape: [1, model_dim].into(),
+ expected_data_type: main.data_type(),
+ actual_shape: bias.shape().into(),
+ actual_data_type: bias.data_type(),
+ }
+ .into());
+ }
+
+ let bias_buffer = bias.buffer();
+ let bias_buffer = bias_buffer.borrow();
+ let main_output_buffer = main.buffer();
+ let mut main_output_buffer = main_output_buffer.borrow_mut();
+ // TensorAddScale is elementwise, so in-place read/write aliasing is valid here.
+ let main_input_buffer: &B::Buffer = unsafe { &*(&*main_output_buffer as *const B::Buffer) };
+ self.tensor_add_scale.encode(
+ (main_input_buffer, main.offset()),
+ &*bias_buffer,
+ (&mut *main_output_buffer, main.offset()),
+ model_dim_u32,
+ total_len_u32,
+ scale,
+ encoder,
+ );
+ Ok(())
+ }
+
+ /// Encode a single decode step (embedding, transformer, readout, sampling,
+ /// and optionally hidden capture) onto the **current** command buffer.
+ /// Does NOT create a new command buffer, does NOT submit.
+ ///
+ /// The caller is responsible for submitting the command buffer and reading
+ /// the sampled token from `async_chain_results[0]` after the eventual
+ /// submit.
+ pub(super) fn encode_next_step_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ token_ids: &[u64],
+ embedding_injection: EmbeddingInjection,
+ sampling: &mut TextSamplingState,
+ precomputed_token_bitmask: Option<&[u32]>,
+ capture_hidden: bool,
+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,
+ ) -> Result<(), Error> {
+ let token_count = token_ids.len();
+
+ let state = self.encode_single_forward_pass_on(
+ encoder,
+ token_ids,
+ embedding_injection,
+ sampling,
+ precomputed_token_bitmask,
+ capture_hidden,
+ pre_injection_encode,
+ None,
+ )?;
+
+ // Copy sampled token to async_chain_results[0] so the caller can
+ // read it after the eventual submit.
+ {
+ let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;
+ let sampling_output_buffer = sampling_output.buffer();
+
+ let sampling_output_buffer = sampling_output_buffer.borrow();
+ let mut results_buffer = self.ctx.async_chain_results.borrow_mut();
+ self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);
+ }
+
+ self.encode_cache_acceptance_update_on(encoder, token_count);
+ // Do NOT submit -- the caller will submit.
+ self.register_positions_and_advance(token_count);
+ Ok(())
+ }
+
+ pub(super) fn submit_and_wait_command_buffer(
+ &mut self,
+ encoder: Encoder<B>,
+ ) -> Result<(), Error> {
+ self.instrumentation.command_buffers_submitted += 1;
+ encoder
+ .end_encoding()
+ .submit()
+ .wait_until_completed()
+ .map_err(|err| Error::CommandBufferFailed(Box::new(err)))?;
+ self.instrumentation.host_waits += 1;
+ Ok(())
+ }
+
+ pub(super) fn copy_async_chain_results_to_on(
+ &self,
+ encoder: &mut Encoder<B>,
+ src_slot: usize,
+ dst: (&mut B::Buffer, usize),
+ count: usize,
+ ) -> Result<(), Error> {
+ let src_offset = src_slot.checked_mul(std::mem::size_of::<u32>()).ok_or(
+ TtsModelConfigError::AsyncChainCopySourceOffsetOverflow {
+ src_slot,
+ },
+ )?;
+ let copy_size =
+ count.checked_mul(std::mem::size_of::<u32>()).ok_or(TtsModelConfigError::AsyncChainCopySizeOverflow {
+ count,
+ })?;
+ let results_buffer = self.ctx.async_chain_results.borrow();
+ encoder.encode_copy(&results_buffer, src_offset..src_offset + copy_size, dst.0, dst.1..dst.1 + copy_size);
+ Ok(())
+ }
+
+ pub(super) fn read_async_chain_result(
+ &self,
+ slot: usize,
+ ) -> Result<u32, Error> {
+ let results_buffer = self.ctx.async_chain_results.borrow();
+ let ptr = results_buffer.cpu_ptr().as_ptr() as *const u32;
+ let capacity = self.ctx.async_chain_capacity;
+ if slot >= capacity {
+ return Err(TtsModelConfigError::AsyncChainResultSlotOutOfBounds {
+ slot,
+ capacity,
+ }
+ .into());
+ }
+ Ok(unsafe { *ptr.add(slot) })
+ }
+
+ pub(super) fn take_instrumentation(&mut self) -> RunnerInstrumentation {
+ std::mem::take(&mut self.instrumentation)
+ }
+
+ pub(super) fn clear_instrumentation(&mut self) {
+ self.instrumentation = RunnerInstrumentation::default();
+ }
+}
+
+fn read_sampled_token_from_sampling_output<B: Backend>(state: &ForwardPassState<B>) -> Result<u64, Error> {
+ let output = state.sampling_output().ok_or(Error::SamplingFailed)?;
+ let tokens = output.as_slice::<u32>();
+ let token = tokens.first().copied().ok_or(Error::SamplingFailed)?;
+ Ok(u64::from(token))
+}
diff --git a/crates/uzu/src/session/tts_session/decoder_support.rs b/crates/uzu/src/session/tts_session/decoder_support.rs
new file mode 100644
index 000000000..773bf7e39
--- /dev/null
+++ b/crates/uzu/src/session/tts_session/decoder_support.rs
@@ -0,0 +1,598 @@
+use super::*;
+
+pub(super) type PreInjectionEncodeCallback<'a, B> =
+ dyn FnMut(&TokenDecoderRunner<B>, &ForwardPassState<B>, &mut Encoder<B>) -> Result<(), Error> + 'a;
+
+pub(super) enum EmbeddingInjection {
+ None,
+ AddPreloaded {
+ post_scale: Option<f32>,
+ },
+ OverrideFirstRowInternal,
+}
+
+pub(super) struct TextSamplingState {
+ rng: StdRng,
+ method: SamplingMethod,
+}
+
+impl TextSamplingState {
+ pub(super) fn from_config(
+ seed: u64,
+ config: &TextSamplingConfig,
+ ) -> Self {
+ Self::with_params(seed, config.temperature, config.top_p)
+ }
+
+ pub(super) fn with_params(
+ seed: u64,
+ temperature: f32,
+ top_p: f32,
+ ) -> Self {
+ let method = if temperature <= 0.0 || top_p <= 0.0 {
+ SamplingMethod::Greedy
+ } else {
+ SamplingMethod::Stochastic {
+ temperature: Some(temperature),
+ top_k: None,
+ top_p: Some(top_p),
+ min_p: None,
+ processing_order: SamplingProcessingOrder::FiltersThenTemperature,
+ }
+ };
+ Self {
+ rng: StdRng::seed_from_u64(seed),
+ method,
+ }
+ }
+
+ pub(super) fn method(&self) -> SamplingMethod {
+ self.method
+ }
+
+ pub(super) fn next_seed(&mut self) -> u64 {
+ self.rng.random::<u64>()
+ }
+}
+
+pub(super) fn text_decoder_prefill_step_size(
+ config: &TextDecoderRuntimeConfig,
+ context_length: usize,
+) -> usize {
+ config.prefill_step_size.min(context_length.max(1)).max(1)
+}
+
+pub(super) fn normalize_rendered_prompt(
+ mut rendered: String,
+ template: &str,
+ drop_initial_newline: bool,
+) -> String {
+ if drop_initial_newline && rendered.starts_with('\n') {
+ rendered.remove(0);
+ }
+ if template.ends_with('\n') && rendered.ends_with('\n') {
+ rendered.pop();
+ }
+ rendered
+}
+
+impl AdaptiveChunkController {
+ pub(super) fn new(config: &TtsRunConfig) -> Self {
+ Self {
+ ema_ms_per_frame: None,
+ current_chunk_frames: config.min_chunk_frames.max(1),
+ }
+ }
+
+ pub(super) fn target_frames(
+ &self,
+ config: &TtsRunConfig,
+ ) -> usize {
+ let min_frames = config.min_chunk_frames.max(1);
+ let max_frames = config.max_chunk_frames.max(min_frames);
+ match config.chunk_policy {
+ TtsChunkPolicy::Fixed => min_frames,
+ TtsChunkPolicy::Adaptive => {
+ let Some(ema_ms_per_frame) = self.ema_ms_per_frame else {
+ return min_frames;
+ };
+ let raw = (config.target_emit_latency_ms as f64 / ema_ms_per_frame).round();
+ let candidate = raw.max(min_frames as f64).min(max_frames as f64) as usize;
+ if self.current_chunk_frames == 0 {
+ return candidate;
+ }
+ if candidate <= self.current_chunk_frames {
+ return self.current_chunk_frames;
+ }
+ let change = ((candidate as f64 - self.current_chunk_frames as f64).abs()
+ / self.current_chunk_frames as f64)
+ .max(0.0);
+ if change < config.chunk_hysteresis_fraction {
+ self.current_chunk_frames
+ } else {
+ candidate
+ }
+ },
+ }
+ }
+
+ pub(super) fn observe(
+ &mut self,
+ config: &TtsRunConfig,
+ frames: usize,
+ decode_elapsed: std::time::Duration,
+ next_chunk_frames: usize,
+ ) {
+ if frames == 0 {
+ return;
+ }
+ let alpha = config.chunk_ema_alpha;
+ let ms_per_frame = (decode_elapsed.as_secs_f64() * 1000.0) / frames as f64;
+ self.ema_ms_per_frame = Some(match self.ema_ms_per_frame {
+ Some(previous) => previous * (1.0 - alpha) + ms_per_frame * alpha,
+ None => ms_per_frame,
+ });
+ self.current_chunk_frames = next_chunk_frames.max(1);
+ }
+
+ pub(super) fn adapt_up_for_realtime(
+ &mut self,
+ config: &TtsRunConfig,
+ generated_frames: usize,
+ sample_rate: u32,
+ decode_elapsed: std::time::Duration,
+ emitted_audio_frames: usize,
+ ) {
+ if generated_frames == 0 || emitted_audio_frames == 0 || sample_rate == 0 {
+ return;
+ }
+ let min_frames = config.min_chunk_frames.max(1);
+ let max_frames = config.max_chunk_frames.max(min_frames);
+ let decode_ms = decode_elapsed.as_secs_f64() * 1000.0;
+ let produced_audio_ms = (emitted_audio_frames as f64) * 1000.0 / f64::from(sample_rate);
+ if produced_audio_ms <= 0.0 {
+ return;
+ }
+ let realtime_ratio = decode_ms / produced_audio_ms;
+ if realtime_ratio <= 1.0 {
+ return;
+ }
+ let scaled = ((generated_frames as f64) * realtime_ratio * 1.1).ceil() as usize;
+ let clamped = scaled.clamp(min_frames, max_frames);
+ self.current_chunk_frames = self.current_chunk_frames.max(clamped);
+ }
+
+ pub(super) fn promote_to_max_chunk(
+ &mut self,
+ config: &TtsRunConfig,
+ ) {
+ let min_frames = config.min_chunk_frames.max(1);
+ let max_frames = config.max_chunk_frames.max(min_frames);
+ self.current_chunk_frames = max_frames;
+ }
+}
+
+pub(super) fn next_startup_target_frames(
+ current_target_frames: usize,
+ startup_cap_frames: usize,
+) -> usize {
+ let startup_cap_frames = startup_cap_frames.max(1);
+ current_target_frames.max(1).saturating_mul(2).min(startup_cap_frames)
+}
+
+pub(super) struct StreamingTokenAccumulator {
+ by_codebook: Vec<Vec<u32>>,
+}
+
+impl StreamingTokenAccumulator {
+ pub(super) fn new(num_codebooks: usize) -> Result<Self, Error> {
+ if num_codebooks == 0 {
+ return Err(Error::UnableToLoadConfig);
+ }
+ Ok(Self {
+ by_codebook: vec![Vec::new(); num_codebooks],
+ })
+ }
+
+ pub(super) fn push_frame(
+ &mut self,
+ frame_codes: &[u32],
+ ) -> Result<(), Error> {
+ if frame_codes.len() != self.by_codebook.len() {
+ return Err(Error::GenerateFailed);
+ }
+ for (codebook, &token) in self.by_codebook.iter_mut().zip(frame_codes.iter()) {
+ codebook.push(token);
+ }
+ Ok(())
+ }
+
+ pub(super) fn frames(&self) -> usize {
+ self.by_codebook.first().map_or(0, Vec::len)
+ }
+
+ pub(super) fn to_grid_range(
+ &self,
+ frame_start: usize,
+ frame_end: usize,
+ ) -> Result<AudioTokenGrid, Error> {
+ let frames = self.frames();
+ if frame_start > frame_end || frame_end > frames {
+ return Err(Error::GenerateFailed);
+ }
+ let range_frames = frame_end - frame_start;
+ let mut tokens = Vec::with_capacity(self.by_codebook.len() * range_frames);
+ for codebook in &self.by_codebook {
+ if codebook.len() != frames {
+ return Err(Error::GenerateFailed);
+ }
+ tokens.extend_from_slice(&codebook[frame_start..frame_end]);
+ }
+
+ AudioTokenGrid::new(
+ tokens.into_boxed_slice(),
+ 1,
+ self.by_codebook.len(),
+ range_frames,
+ vec![range_frames].into_boxed_slice(),
+ )
+ .map_err(Error::from)
+ }
+}
+
+pub(super) fn slice_grid_range(
+ grid: &AudioTokenGrid,
+ frame_start: usize,
+ frame_end: usize,
+) -> Result<AudioTokenGrid, Error> {
+ if frame_start > frame_end || frame_end > grid.frames() {
+ return Err(Error::GenerateFailed);
+ }
+ let range_frames = frame_end.saturating_sub(frame_start);
+ let batch_size = grid.batch_size();
+ let codebooks = grid.codebooks();
+ let frames = grid.frames();
+
+ let mut tokens = Vec::with_capacity(batch_size * codebooks * range_frames);
+ let row_stride = frames;
+ for batch in 0..batch_size {
+ for codebook in 0..codebooks {
+ let row_index = batch
+ .checked_mul(codebooks)
+ .and_then(|value| value.checked_add(codebook))
+ .ok_or(Error::GenerateFailed)?;
+ let row_start = row_index.checked_mul(row_stride).ok_or(Error::GenerateFailed)?;
+ let src_start = row_start.checked_add(frame_start).ok_or(Error::GenerateFailed)?;
+ let src_end = row_start.checked_add(frame_end).ok_or(Error::GenerateFailed)?;
+ tokens.extend_from_slice(&grid.tokens()[src_start..src_end]);
+ }
+ }
+
+ let mut lengths = Vec::with_capacity(batch_size);
+ for &length in grid.lengths() {
+ lengths.push(length.saturating_sub(frame_start).min(range_frames));
+ }
+
+ AudioTokenGrid::new(tokens.into_boxed_slice(), batch_size, codebooks, range_frames, lengths.into_boxed_slice())
+ .map_err(Error::from)
+}
+
+pub(super) fn accumulate_audio_decode_step_stats(
+ decode_calls: &mut usize,
+ input_frames: &mut usize,
+ decoded_window_frames: &mut usize,
+ max_decoded_window_frames: &mut usize,
+ step: Option<AudioDecodeStepStats>,
+) {
+ let Some(step) = step else {
+ return;
+ };
+ if step.input_frames == 0 && step.decoded_window_frames == 0 {
+ return;
+ }
+ *decode_calls = decode_calls.saturating_add(1);
+ *input_frames = input_frames.saturating_add(step.input_frames);
+ *decoded_window_frames = decoded_window_frames.saturating_add(step.decoded_window_frames);
+ *max_decoded_window_frames = (*max_decoded_window_frames).max(step.decoded_window_frames);
+}
+
+pub(super) struct StreamingSynthesisState<'a, F: FnMut(&AudioPcmBatch)> {
+ pub(super) on_chunk: &'a mut F,
+ pub(super) pending_chunk: Option<PendingStreamingChunk>,
+ pub(super) callback_seconds: f64,
+ pub(super) audio_decode_seconds_in_loop: f64,
+ pub(super) audio_decode_seconds: f64,
+ pub(super) output_samples: Vec<f32>,
+ pub(super) output_frames: usize,
+ pub(super) output_sample_rate: u32,
+ pub(super) output_channels: usize,
+ pub(super) emitted_chunks: usize,
+ pub(super) first_emit_pending: bool,
+ pub(super) first_chunk_seconds: Option<f64>,
+ pub(super) first_chunk_frames: usize,
+ pub(super) stream_start: Instant,
+ pub(super) startup_target_frames: usize,
+ pub(super) startup_cap_frames: usize,
+ pub(super) chunk_controller: AdaptiveChunkController,
+ pub(super) audio_decode_calls: usize,
+ pub(super) audio_input_frames: usize,
+ pub(super) audio_decoded_window_frames: usize,
+ pub(super) audio_max_decoded_window_frames: usize,
+}
+
+impl<'a, F: FnMut(&AudioPcmBatch)> StreamingSynthesisState<'a, F> {
+ pub(super) fn maybe_flush_pending(
+ &mut self,
+ force: bool,
+ count_in_loop: bool,
+ config: &TtsRunConfig,
+ ) -> Result<(), Error> {
+ let (
+ step_stats,
+ ready_frames,
+ next_chunk_frames,
+ submission_decode_duration,
+ partial_pcm,
+ resolve_decode_duration,
+ ) = if force {
+ let pending = self.pending_chunk.take().ok_or(Error::GenerateFailed)?;
+ let step_stats = pending.chunk.step_stats();
+ let ready_frames = pending.ready_frames;
+ let next_chunk_frames = pending.next_chunk_frames;
+ let submission_decode_duration = pending.submission_decode_duration;
+ let (partial_pcm, resolve_decode_duration) = pending.chunk.resolve_with_decode_duration()?;
+ (
+ step_stats,
+ ready_frames,
+ next_chunk_frames,
+ submission_decode_duration,
+ partial_pcm,
+ resolve_decode_duration,
+ )
+ } else {
+ let Some(pending) = self.pending_chunk.as_mut() else {
+ return Ok(());
+ };
+ let Some((partial_pcm, resolve_decode_duration)) = pending.chunk.try_resolve_with_decode_duration()? else {
+ return Ok(());
+ };
+ let step_stats = pending.chunk.step_stats();
+ let ready_frames = pending.ready_frames;
+ let next_chunk_frames = pending.next_chunk_frames;
+ let submission_decode_duration = pending.submission_decode_duration;
+ let _ = self.pending_chunk.take().ok_or(Error::GenerateFailed)?;
+ (
+ step_stats,
+ ready_frames,
+ next_chunk_frames,
+ submission_decode_duration,
+ partial_pcm,
+ resolve_decode_duration,
+ )
+ };
+
+ accumulate_audio_decode_step_stats(
+ &mut self.audio_decode_calls,
+ &mut self.audio_input_frames,
+ &mut self.audio_decoded_window_frames,
+ &mut self.audio_max_decoded_window_frames,
+ step_stats,
+ );
+ let decode_elapsed = submission_decode_duration.saturating_add(resolve_decode_duration);
+ if count_in_loop {
+ self.audio_decode_seconds_in_loop += decode_elapsed.as_secs_f64();
+ }
+ self.audio_decode_seconds += decode_elapsed.as_secs_f64();
+
+ let partial_sample_rate = partial_pcm.sample_rate();
+ let callback_start = Instant::now();
+ let emitted_frames = if partial_pcm.lengths().len() == 1 {
+ partial_pcm.lengths()[0]
+ } else {
+ return Err(Error::GenerateFailed);
+ };
+ if emitted_frames > 0 {
+ (self.on_chunk)(&partial_pcm);
+ }
+ self.callback_seconds += callback_start.elapsed().as_secs_f64();
+ self.chunk_controller.observe(config, ready_frames, decode_elapsed, next_chunk_frames);
+
+ if emitted_frames > 0 {
+ self.chunk_controller.adapt_up_for_realtime(
+ config,
+ ready_frames,
+ partial_sample_rate,
+ decode_elapsed,
+ emitted_frames,
+ );
+ self.output_samples.extend_from_slice(partial_pcm.samples());
+ self.output_frames = self.output_frames.saturating_add(emitted_frames);
+ self.output_sample_rate = partial_pcm.sample_rate();
+ self.output_channels = partial_pcm.channels();
+ self.emitted_chunks = self.emitted_chunks.saturating_add(1);
+ if self.first_emit_pending {
+ self.first_emit_pending = false;
+ self.first_chunk_seconds = Some(self.stream_start.elapsed().as_secs_f64());
+ self.first_chunk_frames = emitted_frames;
+ self.chunk_controller.promote_to_max_chunk(config);
+ }
+ } else if self.first_emit_pending {
+ self.startup_target_frames =
+ next_startup_target_frames(self.startup_target_frames, self.startup_cap_frames);
+ }
+
+ Ok(())
+ }
+}
+
+impl<B: Backend + Send + Sync> TtsSession<B> {
+ pub fn new(model_path: PathBuf) -> Result<Self, Error> {
+ Self::new_with_options(model_path, TtsSessionOptions::default())
+ }
+
+ pub fn new_with_options(
+ model_path: PathBuf,
+ options: TtsSessionOptions,
+ ) -> Result<Self, Error> {
+ if !model_path.exists() {
+ return Err(Error::ModelFolderNotFound);
+ }
+
+ let config_path = model_path.join("config.json");
+ if !config_path.exists() {
+ return Err(Error::UnableToLoadConfig);
+ }
+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;
+ let model_metadata: ModelMetadata =
+ serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;
+
+ Self::from_model_metadata_with_options(model_path, model_metadata, options)
+ }
+
+ pub fn last_execution_stats(&self) -> Option<TtsExecutionStats> {
+ self.last_execution_stats.clone()
+ }
+
+ pub fn sample_rate(&self) -> u32 {
+ self.audio.sample_rate()
+ }
+
+ fn from_model_metadata_with_options(
+ model_path: PathBuf,
+ model_metadata: ModelMetadata,
+ options: TtsSessionOptions,
+ ) -> Result<Self, Error> {
+ let tokenizer_path = model_path.join("tokenizer.json");
+ if !tokenizer_path.exists() {
+ return Err(Error::UnableToLoadTokenizer);
+ }
+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;
+
+ let loaded_runtime = load_tts_runtime(&model_path, &model_metadata, &options)?;
+
+ Ok(Self {
+ tokenizer,
+ audio: loaded_runtime.audio,
+ audio_decoder: loaded_runtime.audio_decoder,
+ message_processor_config: loaded_runtime.message_processor_config,
+ text_decoder: loaded_runtime.text_decoder,
+ last_execution_stats: None,
+ })
+ }
+
+ pub(super) fn render_prompt(
+ &self,
+ input: &Input,
+ ) -> Result<String, Error> {
+ let messages = input
+ .get_messages()
+ .into_iter()
+ .map(|message| message.resolve(&self.message_processor_config))
+ .collect::<Vec<_>>();
+
+ let template_name = "tts_prompt_template";
+ let mut environment = Environment::new();
+ environment
+ .add_template(template_name, self.message_processor_config.prompt_template.as_str())
+ .map_err(|_| Error::UnableToLoadPromptTemplate)?;
+ let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;
+
+ let result = template
+ .render(context!(
+ messages => messages
+ ))
+ .map_err(|_| Error::UnableToRenderPromptTemplate)?;
+
+ Ok(normalize_rendered_prompt(
+ result,
+ self.message_processor_config.prompt_template.as_str(),
+ self.message_processor_config.drop_initial_newline,
+ ))
+ }
+}
+
+pub(super) fn semantic_token_to_code(
+ semantic_token: u64,
+ semantic_begin: i64,
+ semantic_end: i64,
+ token_upper_bound: usize,
+) -> u32 {
+ if semantic_begin > semantic_end || token_upper_bound == 0 {
+ return 0;
+ }
+
+ let semantic = semantic_token as i64;
+ if semantic < semantic_begin || semantic > semantic_end {
+ return 0;
+ }
+
+ let relative = usize::try_from(semantic - semantic_begin).unwrap_or(0);
+ let clamped = relative.min(token_upper_bound.saturating_sub(1));
+ u32::try_from(clamped).unwrap_or(0)
+}
+
+pub(super) fn build_semantic_sampling_mask_row(
+ vocab_size: usize,
+ semantic_begin: i64,
+ semantic_end: i64,
+ im_end: i64,
+) -> Result<Box<[u32]>, Error> {
+ if vocab_size == 0 || semantic_begin > semantic_end {
+ return Err(Error::UnableToLoadConfig);
+ }
+
+ let max_token_id = i64::try_from(vocab_size.saturating_sub(1)).map_err(|_| Error::UnableToLoadConfig)?;
+ if semantic_begin < 0 || semantic_end < 0 || semantic_end > max_token_id || im_end < 0 || im_end > max_token_id {
+ return Err(Error::UnableToLoadConfig);
+ }
+
+ let row_words = vocab_size.div_ceil(32);
+ let mut mask = vec![0_u32; row_words];
+ for token_index in semantic_begin..=semantic_end {
+ let token = usize::try_from(token_index).map_err(|_| Error::UnableToLoadConfig)?;
+ let word = token / 32;
+ let bit = token % 32;
+ mask[word] |= 1_u32 << bit;
+ }
+ let im_end_token = usize::try_from(im_end).map_err(|_| Error::UnableToLoadConfig)?;
+ let word = im_end_token / 32;
+ let bit = im_end_token % 32;
+ mask[word] |= 1_u32 << bit;
+ Ok(mask.into_boxed_slice())
+}
+
+pub(super) fn clear_token_in_sampling_mask(
+ mask: &mut [u32],
+ token: i64,
+) -> Result<(), Error> {
+ if token < 0 {
+ return Err(Error::UnableToLoadConfig);
+ }
+ let token = usize::try_from(token).map_err(|_| Error::UnableToLoadConfig)?;
+ let word = token / 32;
+ let bit = token % 32;
+ if word >= mask.len() {
+ return Err(Error::UnableToLoadConfig);
+ }
+ mask[word] &= !(1_u32 << bit);
+ Ok(())
+}
+
+pub(super) fn expand_token_mask_for_sampling_row(
+ row_mask: &[u32],
+ token_count: usize,
+) -> Result<Box<[u32]>, Error> {
+ if token_count == 0 || row_mask.is_empty() {
+ return Err(Error::GenerateFailed);
+ }
+ if token_count == 1 {
+ return Ok(row_mask.to_vec().into_boxed_slice());
+ }
+
+ let row_words = row_mask.len();
+ let total_words = token_count.checked_mul(row_words).ok_or(Error::GenerateFailed)?;
+ let mut expanded = vec![u32::MAX; total_words];
+ let offset = (token_count - 1).checked_mul(row_words).ok_or(Error::GenerateFailed)?;
+ expanded[offset..offset + row_words].copy_from_slice(row_mask);
+ Ok(expanded.into_boxed_slice())
+}
diff --git a/crates/uzu/src/session/tts_session/fishaudio.rs b/crates/uzu/src/session/tts_session/fishaudio.rs
new file mode 100644
index 000000000..52afd9d61
--- /dev/null
+++ b/crates/uzu/src/session/tts_session/fishaudio.rs
@@ -0,0 +1,839 @@
+use std::{collections::BTreeSet, sync::LazyLock};
+
+use regex::Regex;
+
+use super::*;
+use crate::{
+ array::Array,
+ config::TtsMessageProcessorConfig,
+ session::types::{TtsModelConfigError, TtsPromptConfigError},
+};
+
+pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {
+ slow_runner: TokenDecoderRunner<B>,
+ fast_runner: TokenDecoderRunner<B>,
+ semantic_bridge: FishAudioSemanticBridge<B>,
+ runtime_config: TextDecoderRuntimeConfig,
+ semantic_token_begin_id: i64,
+ semantic_token_end_id: i64,
+ semantic_cardinality: usize,
+ im_end_token_id: i64,
+ codebook_size: usize,
+ num_codebooks: usize,
+ slow_model_dim: usize,
+ fast_model_dim: usize,
+ max_seq_len: usize,
+ scale_codebook_embeddings: bool,
+ fast_vocab_limit: usize,
+ apply_semantic_sampling_mask: bool,
+ semantic_sampling_mask_row: Box<[u32]>,
+ semantic_sampling_mask_without_im_end_row: Box<[u32]>,
+ current_codes_scratch: Vec<u32>,
+ instrumentation: RunnerInstrumentation,
+}
+
+struct FishAudioSemanticBridge<B: Backend> {
+ embedding_rows_sum: <B::Kernels as Kernels>::EmbeddingRowsSumKernel,
+ projection: <B::Kernels as MatmulKernels>::MatmulKernel,
+ tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,
+ codebook_embeddings: Array<B>,
+ codebook_token_indices: Array<B>,
+ projection_weights: Option<Array<B>>,
+}
+
+impl<B: Backend> FishAudioSemanticBridge<B> {
+ fn load(
+ context: &B::Context,
+ parameter_tree: &crate::parameters::ParameterTree<B::Context>,
+ config: &crate::config::FishAudioTextDecoderConfig,
+ data_type: DataType,
+ ) -> Result<Self, Error> {
+ let codebook_embeddings = load_float_tensor_array(
+ parameter_tree,
+ "text_decoder.codebook_embeddings.weights",
+ [
+ config.codebook_size.checked_mul(config.num_codebooks).ok_or(Error::UnableToLoadConfig)?,
+ config.slow_model_dim,
+ ],
+ data_type,
+ )?;
+ let fast_model_projection = if config.fast_model_projection_config.is_some() {
+ Some(load_float_tensor_array(
+ parameter_tree,
+ "text_decoder.fast_model_projection.weights",
+ [config.fast_model_dim, config.slow_model_dim],
+ data_type,
+ )?)
+ } else {
+ None
+ };
+
+ Self::new(
+ context,
+ data_type,
+ codebook_embeddings,
+ fast_model_projection,
+ config.num_codebooks,
+ config.codebook_size,
+ config.slow_model_dim,
+ config.fast_model_dim,
+ )
+ }
+
+ fn new(
+ context: &B::Context,
+ data_type: DataType,
+ codebook_embeddings: Array<B>,
+ fast_model_projection: Option<Array<B>>,
+ num_codebooks: usize,
+ codebook_size: usize,
+ slow_model_dim: usize,
+ fast_model_dim: usize,
+ ) -> Result<Self, Error> {
+ if codebook_embeddings.shape()
+ != [num_codebooks.checked_mul(codebook_size).ok_or(Error::UnableToLoadConfig)?, slow_model_dim]
+ || codebook_embeddings.data_type() != data_type
+ {
+ return Err(Error::UnableToLoadConfig);
+ }
+ if let Some(ref projection) = fast_model_projection
+ && (projection.shape() != [fast_model_dim, slow_model_dim] || projection.data_type() != data_type)
+ {
+ return Err(Error::UnableToLoadConfig);
+ }
+
+ let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type)
+ .map_err(unable_to_create_context)?;
+ let projection = <<B::Kernels as MatmulKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)
+ .map_err(unable_to_create_context)?;
+ let tensor_copy =
+ <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;
+ let codebook_token_indices =
+ context.create_array(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");
+
+ Ok(Self {
+ embedding_rows_sum,
+ projection,
+ tensor_copy,
+ codebook_embeddings,
+ codebook_token_indices,
+ projection_weights: fast_model_projection,
+ })
+ }
+}
+
+fn load_float_tensor_array<B: Backend>(
+ parameter_tree: &crate::parameters::ParameterTree<B::Context>,
+ key: &str,
+ expected_shape: [usize; 2],
+ target_data_type: DataType,
+) -> Result<Array<B>, Error> {
+ let array = parameter_tree.leaf_array(key).map_err(|_| Error::UnableToLoadWeights)?;
+ if array.shape() != expected_shape {
+ return Err(Error::UnableToLoadConfig);
+ }
+ if array.data_type() != target_data_type {
+ return Err(TtsModelConfigError::FishAudioTensorDataTypeMismatch {
+ key: key.into(),
+ expected: target_data_type,
+ actual: array.data_type(),
+ }
+ .into());
+ }
+ Ok(array)
+}
+
+fn validate_fishaudio_decoder_contract(
+ num_codebooks: usize,
+ codebook_size: usize,
+ max_seq_len: usize,
+ semantic_token_begin_id: i64,
+ semantic_token_end_id: i64,
+ audio_num_codebooks: usize,
+ audio_codec_cardinality: usize,
+ audio_semantic_cardinality: usize,
+) -> Result<usize, Error> {
+ if num_codebooks <= 1 || codebook_size == 0 || max_seq_len == 0 {
+ return Err(TtsModelConfigError::FishAudioDecoderCoreParametersInvalid {
+ num_codebooks,
+ codebook_size,
+ max_seq_len,
+ }
+ .into());
+ }
+ if num_codebooks != audio_num_codebooks {
+ return Err(TtsModelConfigError::FishAudioDecoderNumCodebooksMismatch {
+ decoder_num_codebooks: num_codebooks,
+ audio_num_codebooks,
+ }
+ .into());
+ }
+ if audio_codec_cardinality == 0 || audio_codec_cardinality > codebook_size {
+ return Err(TtsModelConfigError::FishAudioDecoderCodebookSizeTooSmall {
+ codebook_size,
+ audio_codec_cardinality,
+ }
+ .into());
+ }
+ if semantic_token_begin_id > semantic_token_end_id {
+ return Err(TtsModelConfigError::FishAudioSemanticTokenRangeInvalid {
+ begin: semantic_token_begin_id,
+ end: semantic_token_end_id,
+ }
+ .into());
+ }
+
+ let semantic_cardinality = usize::try_from(semantic_token_end_id - semantic_token_begin_id + 1).map_err(|_| {
+ TtsModelConfigError::FishAudioSemanticTokenRangeOverflow {
+ begin: semantic_token_begin_id,
+ end: semantic_token_end_id,
+ }
+ })?;
+ if semantic_cardinality == 0 || semantic_cardinality != audio_semantic_cardinality {
+ return Err(TtsModelConfigError::FishAudioSemanticCodecCardinalityMismatch {
+ semantic_cardinality,
+ audio_semantic_cardinality,
+ }
+ .into());
+ }
+
+ Ok(semantic_cardinality)
+}
+
+static MESSAGE_FIELD_REF_REGEX: LazyLock<Regex> =
+ LazyLock::new(|| Regex::new(r"message\.([A-Za-z_][A-Za-z0-9_]*)").expect("message field regex"));
+
+pub(super) fn validate_fishaudio_message_processor_config(config: &TtsMessageProcessorConfig) -> Result<(), Error> {
+ let builtin_fields =
+ ["role", "content", "reasoning_content", "speaker_id", "style"].into_iter().collect::<BTreeSet<_>>();
+ let referenced_fields = MESSAGE_FIELD_REF_REGEX
+ .captures_iter(config.prompt_template.as_str())
+ .filter_map(|capture| capture.get(1).map(|field| field.as_str().to_owned()))
+ .collect::<BTreeSet<_>>();
+
+ for field in referenced_fields {
+ if !builtin_fields.contains(field.as_str()) && !config.default_message_fields.contains_key(field.as_str()) {
+ return Err(TtsPromptConfigError::MissingDefaultMessageField {
+ field: field.into_boxed_str(),
+ }
+ .into());
+ }
+ }
+
+ Ok(())
+}
+
+pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(
+ config: &crate::config::FishAudioTextDecoderConfig,
+ audio: &AudioGenerationContext<B>,
+ model_path: &Path,
+ runtime_config: &TextDecoderRuntimeConfig,
+) -> Result<Box<dyn SemanticDecoderBackend>, Error> {
+ let audio_semantic_cardinality =
+ audio.runtime().config().semantic_codec_cardinality().ok_or(Error::UnableToLoadConfig)?;
+ let semantic_cardinality = validate_fishaudio_decoder_contract(
+ config.num_codebooks,
+ config.codebook_size,
+ config.max_seq_len,
+ config.semantic_token_begin_id,
+ config.semantic_token_end_id,
+ audio.num_codebooks(),
+ audio.codec_cardinality(),
+ audio_semantic_cardinality,
+ )?;
+ if !config.slow_readout_config.is_full_precision() {
+ return Err(Error::UnableToLoadConfig);
+ }
+ if !config.fast_readout_config.is_full_precision() {
+ return Err(Error::UnableToLoadConfig);
+ }
+ if config.slow_model_config.model_dim != config.slow_model_dim {
+ return Err(Error::UnableToLoadConfig);
+ }
+ if config.fast_model_config.model_dim != config.fast_model_dim {
+ return Err(Error::UnableToLoadConfig);
+ }
+ if config.short_logits_size == 0 {
+ return Err(Error::UnableToLoadConfig);
+ }
+ let semantic_sampling_mask_row = build_semantic_sampling_mask_row(
+ config.vocab_size,
+ config.semantic_token_begin_id,
+ config.semantic_token_end_id,
+ config.im_end_token_id,
+ )?;
+ let mut semantic_sampling_mask_without_im_end_row = semantic_sampling_mask_row.to_vec();
+ clear_token_in_sampling_mask(&mut semantic_sampling_mask_without_im_end_row, config.im_end_token_id)?;
+
+ let slow_transformer_config = config.slow_model_config.clone();
+ let fast_transformer_config = config.fast_model_config.clone();
+
+ let slow_inner_config = InnerModelConfig {
+ embedding_config: config.slow_embeddings_config.to_text_decoder_embedding_config(),
+ transformer_config: slow_transformer_config,
+ vocab_size: config.vocab_size,
+ };
+ let fast_inner_config = InnerModelConfig {
+ embedding_config: config.fast_embeddings_config.to_text_decoder_embedding_config(),
+ transformer_config: fast_transformer_config,
+ vocab_size: config.codebook_size,
+ };
+
+ let slow_decoder_config = Rc::new(slow_inner_config.to_decoder_config().map_err(|_| Error::UnableToLoadConfig)?);
+ let fast_decoder_config = Rc::new(fast_inner_config.to_decoder_config().map_err(|_| Error::UnableToLoadConfig)?);
+ let text_decoder_context = B::Context::new().map_err(unable_to_create_context)?;
+
+ let slow_runner = TokenDecoderRunner::new_with_context(
+ text_decoder_context.clone(),
+ model_path,
+ slow_decoder_config,
+ "text_decoder.transformer_slow",
+ "text_decoder.embeddings_slow",
+ "text_decoder.readout_slow",
+ runtime_config,
+ )?;
+
+ let fast_runner = TokenDecoderRunner::new_with_context(
+ text_decoder_context.clone(),
+ model_path,
+ fast_decoder_config,
+ "text_decoder.transformer_fast",
+ "text_decoder.embeddings_fast",
+ "text_decoder.readout_fast",
+ runtime_config,
+ )?;
+
+ let activation_data_type = slow_runner.single_hidden_capture.data_type();
+ let fast_data_type = fast_runner.single_override_embedding.data_type();
+ if fast_data_type != activation_data_type {
+ return Err(Error::UnableToLoadConfig);
+ }
+
+ let weights_path = model_path.join("model.safetensors");
+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;
+ let loader =
+ ParameterLoader::new(&weights_file, text_decoder_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;
+ let root_weights = loader.tree();
+
+ let apply_semantic_sampling_mask =
+ runtime_config.force_semantic_sampling_mask.unwrap_or(!matches!(activation_data_type, DataType::F32));
+
+ let semantic_bridge =
+ FishAudioSemanticBridge::load(text_decoder_context.as_ref(), &root_weights, config, activation_data_type)?;
+
+ Ok(Box::new(FishAudioTextDecoderRuntime::<B> {
+ slow_runner,
+ fast_runner,
+ semantic_bridge,
+ runtime_config: runtime_config.clone(),
+ semantic_token_begin_id: config.semantic_token_begin_id,
+ semantic_token_end_id: config.semantic_token_end_id,
+ semantic_cardinality,
+ im_end_token_id: config.im_end_token_id,
+ codebook_size: config.codebook_size,
+ num_codebooks: config.num_codebooks,
+ slow_model_dim: config.slow_model_dim,
+ fast_model_dim: config.fast_model_dim,
+ max_seq_len: config.max_seq_len,
+ scale_codebook_embeddings: config.scale_codebook_embeddings,
+ fast_vocab_limit: config.short_logits_size.min(config.codebook_size),
+ apply_semantic_sampling_mask,
+ semantic_sampling_mask_row,
+ semantic_sampling_mask_without_im_end_row: semantic_sampling_mask_without_im_end_row.into_boxed_slice(),
+ current_codes_scratch: vec![0_u32; config.num_codebooks],
+ instrumentation: RunnerInstrumentation::default(),
+ }))
+}
+
+impl<B: Backend> FishAudioTextDecoderRuntime<B> {
+ fn generate_semantic_tokens_internal(
+ &mut self,
+ text_tokens: &[u64],
+ codec_cardinality: usize,
+ seed: u64,
+ max_semantic_frames: usize,
+ mut on_frame: Option<&mut dyn FnMut(&[u32]) -> Result<(), Error>>,
+ ) -> Result<AudioTokenGrid, Error> {
+ if text_tokens.is_empty() {
+ return AudioTokenGrid::new(
+ Vec::new().into_boxed_slice(),
+ 1,
+ self.num_codebooks,
+ 0,
+ vec![0].into_boxed_slice(),
+ )
+ .map_err(Error::from);
+ }
+ if text_tokens.len() >= self.max_seq_len {
+ return Err(Error::GenerateFailed);
+ }
+
+ if codec_cardinality == 0 || codec_cardinality > self.codebook_size {
+ return Err(Error::UnableToLoadConfig);
+ }
+ let semantic_token_upper_bound = self.semantic_cardinality;
+ let residual_token_upper_bound = codec_cardinality;
+ if semantic_token_upper_bound == 0 || residual_token_upper_bound == 0 {
+ return Err(Error::UnableToLoadConfig);
+ }
+
+ self.reset_generation_state();
+ let mut sampling = TextSamplingState::from_config(seed, &self.runtime_config.sampling);
+ let mut current_semantic_token = self.decode_initial_semantic_token(text_tokens, &mut sampling)?;
+ let post_scale = if self.scale_codebook_embeddings {
+ Some(1.0 / ((self.num_codebooks + 1) as f32).sqrt())
+ } else {
+ None
+ };
+
+ let mut max_new_tokens = self.max_seq_len.saturating_sub(text_tokens.len());
+ max_new_tokens = max_new_tokens.min(max_semantic_frames.max(1));
+ if let Some(limit) = self.runtime_config.max_new_tokens_override {
+ max_new_tokens = max_new_tokens.min(limit);
+ }
+ let mut by_codebook =
+ (0..self.num_codebooks).map(|_| Vec::<u32>::with_capacity(max_new_tokens)).collect::<Vec<_>>();
+ if self.current_codes_scratch.len() != self.num_codebooks {
+ self.current_codes_scratch = vec![0_u32; self.num_codebooks];
+ }
+
+ self.prepare_fast_runner_masks(residual_token_upper_bound)?;
+
+ for _step in 0..max_new_tokens {
+ if current_semantic_token as i64 == self.im_end_token_id {
+ break;
+ }
+ let first_code = semantic_token_to_code(
+ current_semantic_token,
+ self.semantic_token_begin_id,
+ self.semantic_token_end_id,
+ semantic_token_upper_bound,
+ );
+ by_codebook[0].push(first_code);
+ self.current_codes_scratch[0] = first_code;
+
+ current_semantic_token = self.decode_frame_merged(
+ current_semantic_token,
+ first_code,
+ residual_token_upper_bound,
+ post_scale,
+ &mut sampling,
+ &mut by_codebook,
+ )?;
+
+ if let Some(callback) = on_frame.as_mut() {
+ callback(&self.current_codes_scratch)?;
+ }
+ }
+
+ self.instrumentation = self.slow_runner.take_instrumentation();
+ let fast = self.fast_runner.take_instrumentation();
+ self.instrumentation.command_buffers_submitted += fast.command_buffers_submitted;
+ self.instrumentation.host_waits += fast.host_waits;
+
+ self.finish_semantic_grid(&by_codebook)
+ }
+
+ fn reset_generation_state(&mut self) {
+ self.instrumentation = RunnerInstrumentation::default();
+ self.slow_runner.reset();
+ self.slow_runner.clear_instrumentation();
+ self.fast_runner.clear_instrumentation();
+ }
+
+ fn decode_initial_semantic_token(
+ &mut self,
+ text_tokens: &[u64],
+ sampling: &mut TextSamplingState,
+ ) -> Result<u64, Error> {
+ let slow_runner = &mut self.slow_runner;
+ let initial_sampling_row = if self.apply_semantic_sampling_mask {
+ if self.runtime_config.min_frames_before_im_end > 0 {
+ self.semantic_sampling_mask_without_im_end_row.as_ref()
+ } else {
+ self.semantic_sampling_mask_row.as_ref()
+ }
+ } else {
+ &[]
+ };
+ let prefill_step_size = text_decoder_prefill_step_size(&self.runtime_config, self.max_seq_len);
+ if text_tokens.len() > prefill_step_size {
+ for chunk in text_tokens[..text_tokens.len() - prefill_step_size].chunks(prefill_step_size) {
+ slow_runner.prefill_without_sampling(chunk)?;
+ }
+ }
+ let prefill_tail_start = text_tokens.len().saturating_sub(prefill_step_size);
+ let prefill_tail = &text_tokens[prefill_tail_start..];
+ let prefill_mask = if initial_sampling_row.is_empty() {
+ None
+ } else {
+ Some(expand_token_mask_for_sampling_row(initial_sampling_row, prefill_tail.len())?)
+ };
+ slow_runner.decode_next_token_with_hidden_capture(
+ prefill_tail,
+ EmbeddingInjection::None,
+ sampling,
+ prefill_mask.as_deref(),
+ )
+ }
+
+ fn prepare_fast_runner_masks(
+ &mut self,
+ residual_token_upper_bound: usize,
+ ) -> Result<(), Error> {
+ let fast_vocab_limit = self.fast_vocab_limit.min(residual_token_upper_bound);
+ if fast_vocab_limit == 0 {
+ return Err(Error::UnableToLoadConfig);
+ }
+ self.fast_runner.prepare_single_token_vocab_mask(fast_vocab_limit)?;
+ self.fast_runner.prepare_two_token_vocab_mask(fast_vocab_limit)?;
+ Ok(())
+ }
+
+ fn finish_semantic_grid(
+ &self,
+ by_codebook: &[Vec<u32>],
+ ) -> Result<AudioTokenGrid, Error> {
+ let frames = by_codebook.first().map_or(0, Vec::len);
+ let mut tokens = Vec::with_capacity(self.num_codebooks * frames);
+ for codebook_tokens in by_codebook {
+ if codebook_tokens.len() != frames {
+ return Err(Error::GenerateFailed);
+ }
+ tokens.extend_from_slice(codebook_tokens);
+ }
+ AudioTokenGrid::new(tokens.into_boxed_slice(), 1, self.num_codebooks, frames, vec![frames].into_boxed_slice())
+ .map_err(Error::from)
+ }
+
+ fn encode_project_slow_hidden_to_fast_on(
+ context: &B::Context,
+ semantic_bridge: &mut FishAudioSemanticBridge<B>,
+ slow_hidden_capture: &Array<B>,
+ output_embedding: &Array<B>,
+ slow_model_dim: usize,
+ fast_model_dim: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), Error> {
+ let model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;
+
+ if let Some(weights) = semantic_bridge.projection_weights.as_ref() {
+ let hidden = slow_hidden_capture;
+ let weights = weights;
+ let output = output_embedding;
+ if hidden.shape() != [1, slow_model_dim]
+ || output.shape() != [1, fast_model_dim]
+ || weights.shape() != [fast_model_dim, slow_model_dim]
+ {
+ return Err(Error::GenerateFailed);
+ }
+
+ let hidden_buffer = hidden.buffer();
+ let weights_buffer = weights.buffer();
+ let output_buffer = output.buffer();
+ let hidden_buffer = hidden_buffer.borrow();
+ let weights_buffer = weights_buffer.borrow();
+ let mut output_buffer = output_buffer.borrow_mut();
+ semantic_bridge.projection.encode(
+ context,
+ MatmulArguments {
+ a: &hidden_buffer,
+ a_offset: hidden.offset() as u64,
+ b: &weights_buffer,
+ d: &mut output_buffer,
+ bias: None,
+ batch: 1,
+ input_dim: slow_model_dim as i32,
+ output_dim: fast_model_dim as i32,
+ leading_dimension_a: slow_model_dim as i32,
+ leading_dimension_b: fast_model_dim as i32,
+ leading_dimension_d: fast_model_dim as i32,
+ transpose_b: false,
+ },
+ encoder,
+ );
+ return Ok(());
+ }
+
+ if slow_model_dim != fast_model_dim {
+ return Err(Error::UnableToLoadConfig);
+ }
+
+ let hidden = slow_hidden_capture;
+ let output = output_embedding;
+ if hidden.shape() != [1, slow_model_dim] || output.shape() != [1, fast_model_dim] {
+ return Err(Error::GenerateFailed);
+ }
+ let hidden_buffer = hidden.buffer();
+ let hidden_buffer = hidden_buffer.borrow();
+ let output_buffer = output.buffer();
+ let mut output_buffer = output_buffer.borrow_mut();
+ semantic_bridge.tensor_copy.encode(
+ (&*hidden_buffer, hidden.offset()),
+ (&mut *output_buffer, output.offset()),
+ model_dim_u32,
+ encoder,
+ );
+ Ok(())
+ }
+
+ /// Encode the codebook sum reading residual
+ /// codes directly from a GPU buffer instead of from CPU memory.
+ ///
+ /// `first_code` is written into slot 0 of the staging buffer from CPU.
+ /// Slots 1..num_codebooks are filled by a GPU blit from
+ /// `gpu_tokens_buffer` at the given byte offset.
+ fn encode_slow_codebook_sum_from_gpu_tokens_on(
+ semantic_bridge: &mut FishAudioSemanticBridge<B>,
+ slow_sum_embedding: &Array<B>,
+ first_code: u32,
+ source_runner: &TokenDecoderRunner<B>,
+ source_slot: usize,
+ num_codebooks: usize,
+ codebook_size: usize,
+ slow_model_dim: usize,
+ encoder: &mut Encoder<B>,
+ ) -> Result<(), Error> {
+ if num_codebooks == 0 {
+ return Err(Error::UnableToLoadConfig);
+ }
+
+ // Write first_code at position 0 from CPU.
+ {
+ let token_indices = &mut semantic_bridge.codebook_token_indices;
+ if token_indices.shape() != [num_codebooks] || token_indices.data_type() != DataType::U32 {
+ return Err(Error::GenerateFailed);
+ }
+ let indices_slice = token_indices.as_slice_mut::<u32>();
+ indices_slice[0] = first_code;
+ }
+
+ // Blit residual codes from GPU: async_chain_results[0..N-2] -> codebook_token_indices[1..N-1]
+ let residual_count = num_codebooks.saturating_sub(1);
+ if residual_count > 0 {
+ let dst_byte_offset = std::mem::size_of::<u32>(); // skip slot 0
+ let token_indices_buffer = semantic_bridge.codebook_token_indices.buffer();
+ let mut token_indices_buffer = token_indices_buffer.borrow_mut();
+ source_runner.copy_async_chain_results_to_on(
+ encoder,
+ source_slot,
+ (&mut *token_indices_buffer, semantic_bridge.codebook_token_indices.offset() + dst_byte_offset),
+ residual_count,
+ )?;
+ }
+
+ let total_vocab = num_codebooks.checked_mul(codebook_size).ok_or(Error::GenerateFailed)?;
+ let total_vocab_u32 = u32::try_from(total_vocab).map_err(|_| Error::GenerateFailed)?;
+ let num_codebooks_u32 = u32::try_from(num_codebooks).map_err(|_| Error::GenerateFailed)?;
+ let slow_model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;
+ let codebook_size_u32 = u32::try_from(codebook_size).map_err(|_| Error::GenerateFailed)?;
+
+ let codebook_token_indices = &semantic_bridge.codebook_token_indices;
+ let codebook_embeddings = &semantic_bridge.codebook_embeddings;
+ let slow_sum = slow_sum_embedding;
+ if codebook_token_indices.shape() != [num_codebooks]
+ || codebook_embeddings.shape() != [total_vocab, slow_model_dim]
+ || slow_sum.shape() != [1, slow_model_dim]
+ || codebook_embeddings.data_type() != slow_sum.data_type()
+ {
+ return Err(Error::GenerateFailed);
+ }
+
+ let codebook_token_indices_buffer = codebook_token_indices.buffer();
+ let codebook_token_indices_buffer = codebook_token_indices_buffer.borrow();
+ let codebook_embeddings_buffer = codebook_embeddings.buffer();
+ let codebook_embeddings_buffer = codebook_embeddings_buffer.borrow();
+ let slow_sum_buffer = slow_sum.buffer();
+ let mut slow_sum_buffer = slow_sum_buffer.borrow_mut();
+ semantic_bridge.embedding_rows_sum.encode(
+ (&*codebook_token_indices_buffer, codebook_token_indices.offset()),
+ (&*codebook_embeddings_buffer, codebook_embeddings.offset()),
+ (&mut *slow_sum_buffer, slow_sum.offset()),
+ num_codebooks_u32,
+ total_vocab_u32,
+ slow_model_dim_u32,
+ codebook_size_u32,
+ encoder,
+ );
+ Ok(())
+ }
+
+ /// Merged fast+slow frame: encodes all fast runner forward passes, the
+ /// codebook sum (reading GPU tokens directly), and the slow runner
+ /// forward pass into a **single** command buffer submission.
+ ///
+ /// Returns the next semantic token.
+ fn decode_frame_merged(
+ &mut self,
+ current_semantic_token: u64,
+ first_code: u32,
+ residual_token_upper_bound: usize,
+ post_scale: Option<f32>,
+ sampling: &mut TextSamplingState,
+ by_codebook: &mut [Vec<u32>],
+ ) -> Result<u64, Error> {
+ let fast_vocab_limit = self.fast_vocab_limit.min(residual_token_upper_bound);
+ let followup_count = self.num_codebooks.saturating_sub(2);
+
+ self.fast_runner.reset();
+
+ let context = Rc::clone(self.fast_runner.context());
+ let mut shared_encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;
+
+ let total_fast_count =
+ self.encode_fast_passes_on(&mut shared_encoder, first_code, fast_vocab_limit, followup_count, sampling)?;
+ self.encode_slow_pass_on(
+ &mut shared_encoder,
+ current_semantic_token,
+ first_code,
+ post_scale,
+ sampling,
+ by_codebook,
+ )?;
+ self.slow_runner.submit_and_wait_command_buffer(shared_encoder)?;
+
+ for pass in 0..total_fast_count {
+ let sampled = self.fast_runner.read_async_chain_result(pass)?;
+ let codebook_index = pass + 1;
+ let clamped = (sampled as usize).min(residual_token_upper_bound.saturating_sub(1));
+ let clamped = u32::try_from(clamped).map_err(|_| Error::GenerateFailed)?;
+ by_codebook[codebook_index].push(clamped);
+ self.current_codes_scratch[codebook_index] = clamped;
+ }
+
+ let slow_token = u64::from(self.slow_runner.read_async_chain_result(0)?);
+ Ok(slow_token)
+ }
+
+ /// Encode the fast runner's initial + followup passes onto the shared
+ /// command buffer.
+ fn encode_fast_passes_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ first_code: u32,
+ fast_vocab_limit: usize,
+ followup_count: usize,
+ sampling: &mut TextSamplingState,
+ ) -> Result<usize, Error> {
+ let (slow_runner, fast_runner, semantic_bridge) =
+ (&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);
+ let slow_hidden_capture = &slow_runner.single_hidden_capture;
+ let slow_model_dim = self.slow_model_dim;
+ let fast_model_dim = self.fast_model_dim;
+
+ let mut pre_projection =
+ |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {
+ Self::encode_project_slow_hidden_to_fast_on(
+ runner.context().as_ref(),
+ semantic_bridge,
+ slow_hidden_capture,
+ &runner.single_override_embedding,
+ slow_model_dim,
+ fast_model_dim,
+ encoder,
+ )
+ };
+
+ fast_runner.encode_first_and_followup_tokens_on(
+ encoder,
+ &[0, u64::from(first_code)],
+ EmbeddingInjection::OverrideFirstRowInternal,
+ Some(&mut pre_projection),
+ followup_count,
+ Some(fast_vocab_limit),
+ sampling,
+ )
+ }
+
+ /// Encode the slow runner's forward pass onto the shared command buffer.
+ fn encode_slow_pass_on(
+ &mut self,
+ encoder: &mut Encoder<B>,
+ current_semantic_token: u64,
+ first_code: u32,
+ post_scale: Option<f32>,
+ sampling: &mut TextSamplingState,
+ by_codebook: &[Vec<u32>],
+ ) -> Result<(), Error> {
+ let (slow_runner, fast_runner, semantic_bridge) =
+ (&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);
+ let num_codebooks = self.num_codebooks;
+ let codebook_size = self.codebook_size;
+ let slow_model_dim = self.slow_model_dim;
+
+ let sampled_frames = by_codebook[0].len();
+ let slow_sampling_mask = if self.apply_semantic_sampling_mask {
+ if sampled_frames < self.runtime_config.min_frames_before_im_end {
+ Some(self.semantic_sampling_mask_without_im_end_row.as_ref())
+ } else {
+ Some(self.semantic_sampling_mask_row.as_ref())
+ }
+ } else {
+ None
+ };
+
+ let mut pre_codebook_sum =
+ |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {
+ Self::encode_slow_codebook_sum_from_gpu_tokens_on(
+ semantic_bridge,
+ &runner.single_override_embedding,
+ first_code,
+ fast_runner,
+ 0,
+ num_codebooks,
+ codebook_size,
+ slow_model_dim,
+ encoder,
+ )
+ };
+
+ slow_runner.encode_next_step_on(
+ encoder,
+ &[current_semantic_token],
+ EmbeddingInjection::AddPreloaded {
+ post_scale,
+ },
+ sampling,
+ slow_sampling_mask,
+ true, // capture_hidden
+ Some(&mut pre_codebook_sum),
+ )
+ }
+
+ fn take_instrumentation(&mut self) -> RunnerInstrumentation {
+ std::mem::take(&mut self.instrumentation)
+ }
+}
+
+impl<B: Backend> SemanticDecoderBackend for FishAudioTextDecoderRuntime<B> {
+ fn generate_semantic_tokens(
+ &mut self,
+ text_tokens: &[u64],
+ codec_cardinality: usize,
+ seed: u64,
+ max_semantic_frames: usize,
+ ) -> Result<AudioTokenGrid, Error> {
+ self.generate_semantic_tokens_internal(text_tokens, codec_cardinality, seed, max_semantic_frames, None)
+ }
+
+ fn generate_semantic_tokens_with_callback(
+ &mut self,
+ text_tokens: &[u64],
+ codec_cardinality: usize,
+ seed: u64,
+ max_semantic_frames: usize,
+ on_frame: &mut dyn FnMut(&[u32]) -> Result<(), Error>,
+ ) -> Result<AudioTokenGrid, Error> {
+ self.generate_semantic_tokens_internal(
+ text_tokens,
+ codec_cardinality,
+ seed,
+ max_semantic_frames,
+ Some(on_frame),
+ )
+ }
+
+ fn take_instrumentation(&mut self) -> RunnerInstrumentation {
+ FishAudioTextDecoderRuntime::take_instrumentation(self)
+ }
+}
diff --git a/crates/uzu/src/session/tts_session/generator.rs b/crates/uzu/src/session/tts_session/generator.rs
new file mode 100644
index 000000000..690b21fe2
--- /dev/null
+++ b/crates/uzu/src/session/tts_session/generator.rs
@@ -0,0 +1,426 @@
+use super::*;
+
+impl<B: Backend + Send + Sync> TtsSession<B> {
+ pub fn synthesize(
+ &mut self,
+ input: Input,
+ ) -> Result<AudioPcmBatch, Error> {
+ self.synthesize_with_seed_and_config(input, TtsRunConfig::default().default_seed, &TtsRunConfig::default())
+ }
+
+ pub fn synthesize_with_seed(
+ &mut self,
+ input: Input,
+ seed: u64,
+ ) -> Result<AudioPcmBatch, Error> {
+ self.synthesize_with_seed_and_config(input, seed, &TtsRunConfig::default())
+ }
+
+ pub fn synthesize_with_config(
+ &mut self,
+ input: Input,
+ config: &TtsRunConfig,
+ ) -> Result<AudioPcmBatch, Error> {
+ self.synthesize_with_seed_and_config(input, config.default_seed, config)
+ }
+
+ pub fn synthesize_with_seed_and_config(
+ &mut self,
+ input: Input,
+ seed: u64,
+ config: &TtsRunConfig,
+ ) -> Result<AudioPcmBatch, Error> {
+ config.validate()?;
+
+ let prompt = self.render_prompt(&input)?;
+ let text_tokens: Vec<u64> = self
+ .tokenizer
+ .encode(prompt.as_str(), false)
+ .map_err(|_| Error::UnableToEncodeText)?
+ .get_ids()
+ .iter()
+ .map(|&token| token as u64)
+ .collect();
+
+ let semantic_start = Instant::now();
+ let semantic_tokens = self.generate_semantic_tokens(&text_tokens, seed, config.max_semantic_frames)?;
+ let semantic_decode_seconds = semantic_start.elapsed().as_secs_f64();
+ let instrumentation = self.take_text_decoder_instrumentation();
+
+ let audio_start = Instant::now();
+ let mut audio_decode_calls = 0usize;
+ let mut audio_input_frames = 0usize;
+ let mut audio_decoded_window_frames = 0usize;
+ let mut audio_max_decoded_window_frames = 0usize;
+ let pcm = match config.non_streaming_mode {
+ crate::session::config::TtsNonStreamingMode::FullDecode => {
+ audio_decode_calls = usize::from(semantic_tokens.frames() > 0);
+ audio_input_frames = semantic_tokens.frames();
+ audio_decoded_window_frames = semantic_tokens.frames();
+ audio_max_decoded_window_frames = semantic_tokens.frames();
+ self.audio_decoder.decode(&semantic_tokens)?
+ },
+ crate::session::config::TtsNonStreamingMode::ChunkedIfNeeded => {
+ if semantic_tokens.batch_size() != 1 {
+ return Err(Error::GenerateFailed);
+ }
+ config.validate_stream_decode()?;
+ let total_frames = semantic_tokens.frames();
+ let chunked_threshold = config.max_stream_workspace_frames.max(config.max_chunk_frames.max(1));
+ if total_frames <= chunked_threshold {
+ audio_decode_calls = usize::from(total_frames > 0);
+ audio_input_frames = total_frames;
+ audio_decoded_window_frames = total_frames;
+ audio_max_decoded_window_frames = total_frames;
+ self.audio_decoder.decode(&semantic_tokens)?
+ } else {
+ let chunk_frames = config.max_chunk_frames.max(config.min_chunk_frames.max(1));
+ let mut stream = self.audio_decoder.begin_stream(
+ semantic_tokens.batch_size(),
+ semantic_tokens.codebooks(),
+ config.max_stream_workspace_frames,
+ )?;
+
+ let mut all_samples = Vec::<f32>::with_capacity(total_frames.saturating_mul(512));
+ let mut accumulated_lengths = vec![0usize; semantic_tokens.batch_size()];
+ let mut sample_rate = self.audio_decoder.sample_rate();
+ let mut channels = 1usize;
+
+ let mut frame_start = 0usize;
+ while frame_start < total_frames {
+ let frame_end = (frame_start + chunk_frames).min(total_frames);
+ let delta_grid = slice_grid_range(&semantic_tokens, frame_start, frame_end)?;
+ let partial_pcm = stream.decode_step(&delta_grid, frame_end == total_frames)?;
+ accumulate_audio_decode_step_stats(
+ &mut audio_decode_calls,
+ &mut audio_input_frames,
+ &mut audio_decoded_window_frames,
+ &mut audio_max_decoded_window_frames,
+ stream.last_step_stats(),
+ );
+ if partial_pcm.lengths().len() != accumulated_lengths.len() {
+ return Err(Error::GenerateFailed);
+ }
+ for (acc, &len) in accumulated_lengths.iter_mut().zip(partial_pcm.lengths().iter()) {
+ *acc = acc.saturating_add(len);
+ }
+ if !partial_pcm.samples().is_empty() {
+ all_samples.extend_from_slice(partial_pcm.samples());
+ }
+ sample_rate = partial_pcm.sample_rate();
+ channels = partial_pcm.channels();
+ frame_start = frame_end;
+ }
+ stream.finish()?;
+ AudioPcmBatch::new(
+ all_samples.into_boxed_slice(),
+ sample_rate,
+ channels,
+ accumulated_lengths.into_boxed_slice(),
+ )
+ .map_err(Error::from)?
+ }
+ },
+ };
+ let audio_decode_seconds = audio_start.elapsed().as_secs_f64();
+
+ self.record_last_execution_stats(TtsExecutionStats {
+ semantic_decode_seconds,
+ audio_decode_seconds,
+ callback_seconds: 0.0,
+ first_chunk_seconds: 0.0,
+ command_buffers_submitted: instrumentation.command_buffers_submitted,
+ host_waits: instrumentation.host_waits,
+ semantic_frames: semantic_tokens.frames(),
+ first_chunk_frames: 0,
+ emitted_chunks: usize::from(config.streaming_enabled),
+ audio_decode_calls,
+ audio_input_frames,
+ audio_decoded_window_frames,
+ audio_max_decoded_window_frames,
+ });
+
+ Ok(pcm)
+ }
+
+ pub fn generate_semantic_tokens_with_seed(
+ &mut self,
+ input: Input,
+ seed: u64,
+ ) -> Result<AudioTokenGrid, Error> {
+ self.generate_semantic_tokens_with_seed_and_config(input, seed, &TtsRunConfig::default())
+ }
+
+ pub fn generate_semantic_tokens_with_seed_and_config(
+ &mut self,
+ input: Input,
+ seed: u64,
+ config: &TtsRunConfig,
+ ) -> Result<AudioTokenGrid, Error> {
+ config.validate()?;
+ let prompt = self.render_prompt(&input)?;
+ let text_tokens: Vec<u64> = self
+ .tokenizer
+ .encode(prompt.as_str(), false)
+ .map_err(|_| Error::UnableToEncodeText)?
+ .get_ids()
+ .iter()
+ .map(|&token| token as u64)
+ .collect();
+
+ self.generate_semantic_tokens(&text_tokens, seed, config.max_semantic_frames)
+ }
+
+ pub fn synthesize_streaming<F>(
+ &mut self,
+ input: Input,
+ chunk_frames: usize,
+ on_chunk: F,
+ ) -> Result<AudioPcmBatch, Error>
+ where
+ F: FnMut(&AudioPcmBatch),
+ {
+ let config = TtsRunConfig::default();
+ self.synthesize_streaming_with_seed(input, config.default_seed, chunk_frames, on_chunk)
+ }
+
+ pub fn synthesize_streaming_with_seed<F>(
+ &mut self,
+ input: Input,
+ seed: u64,
+ chunk_frames: usize,
+ on_chunk: F,
+ ) -> Result<AudioPcmBatch, Error>
+ where
+ F: FnMut(&AudioPcmBatch),
+ {
+ let config = TtsRunConfig::fixed_chunk_frames(chunk_frames);
+ self.synthesize_streaming_with_seed_and_config(input, seed, &config, on_chunk)
+ }
+
+ pub fn synthesize_streaming_with_config<F>(
+ &mut self,
+ input: Input,
+ config: &TtsRunConfig,
+ on_chunk: F,
+ ) -> Result<AudioPcmBatch, Error>
+ where
+ F: FnMut(&AudioPcmBatch),
+ {
+ self.synthesize_streaming_with_seed_and_config(input, config.default_seed, config, on_chunk)
+ }
+
+ pub fn synthesize_streaming_with_seed_and_config<F>(
+ &mut self,
+ input: Input,
+ seed: u64,
+ config: &TtsRunConfig,
+ mut on_chunk: F,
+ ) -> Result<AudioPcmBatch, Error>
+ where
+ F: FnMut(&AudioPcmBatch),
+ {
+ if !config.streaming_enabled {
+ let pcm = self.synthesize_with_seed_and_config(input, seed, config)?;
+ on_chunk(&pcm);
+ return Ok(pcm);
+ }
+ config.validate_stream_decode()?;
+
+ let prompt = self.render_prompt(&input)?;
+ let text_tokens: Vec<u64> = self
+ .tokenizer
+ .encode(prompt.as_str(), false)
+ .map_err(|_| Error::UnableToEncodeText)?
+ .get_ids()
+ .iter()
+ .map(|&token| token as u64)
+ .collect();
+
+ let mut streamed_tokens = StreamingTokenAccumulator::new(self.audio_decoder.num_codebooks())?;
+ let mut audio_stream = self.audio_decoder.begin_stream(
+ 1,
+ self.audio_decoder.num_codebooks(),
+ config.max_stream_workspace_frames,
+ )?;
+ let mut last_decoded_frames = 0usize;
+ let startup_cap_frames = config.max_chunk_frames.max(config.min_chunk_frames.max(1));
+ let initial_chunk_frames = config.initial_chunk_frames.max(1).min(startup_cap_frames);
+
+ let mut ss = StreamingSynthesisState {
+ on_chunk: &mut on_chunk,
+ pending_chunk: None,
+ callback_seconds: 0.0,
+ audio_decode_seconds_in_loop: 0.0,
+ audio_decode_seconds: 0.0,
+ output_samples: Vec::with_capacity(config.max_semantic_frames.saturating_mul(512)),
+ output_frames: 0,
+ output_sample_rate: self.audio_decoder.sample_rate(),
+ output_channels: 1,
+ emitted_chunks: 0,
+ first_emit_pending: true,
+ first_chunk_seconds: None,
+ first_chunk_frames: 0,
+ stream_start: Instant::now(),
+ startup_target_frames: initial_chunk_frames,
+ startup_cap_frames,
+ chunk_controller: AdaptiveChunkController::new(config),
+ audio_decode_calls: 0,
+ audio_input_frames: 0,
+ audio_decoded_window_frames: 0,
+ audio_max_decoded_window_frames: 0,
+ };
+
+ let semantic_start = Instant::now();
+ let semantic_tokens = self.generate_semantic_tokens_with_callback(
+ &text_tokens,
+ seed,
+ config.max_semantic_frames,
+ &mut |codes| {
+ ss.maybe_flush_pending(false, true, config)?;
+ streamed_tokens.push_frame(codes)?;
+ let ready_frames = streamed_tokens.frames().saturating_sub(last_decoded_frames);
+ let adaptive_target_frames = ss.chunk_controller.target_frames(config);
+ let target_frames = if ss.first_emit_pending {
+ ss.startup_target_frames
+ } else {
+ adaptive_target_frames
+ };
+ if ready_frames >= target_frames && ss.pending_chunk.is_none() {
+ let partial_grid = streamed_tokens.to_grid_range(last_decoded_frames, streamed_tokens.frames())?;
+ last_decoded_frames = streamed_tokens.frames();
+ let pending_next_chunk_frames = if ss.first_emit_pending {
+ adaptive_target_frames
+ } else {
+ target_frames
+ };
+ let decode_start = Instant::now();
+ let pending_audio_chunk = audio_stream.decode_step_pending(&partial_grid, false)?;
+ ss.pending_chunk = Some(PendingStreamingChunk {
+ submission_decode_duration: decode_start.elapsed(),
+ ready_frames,
+ next_chunk_frames: pending_next_chunk_frames,
+ chunk: pending_audio_chunk,
+ });
+ ss.maybe_flush_pending(false, true, config)?;
+ }
+ Ok(())
+ },
+ )?;
+ let semantic_loop_seconds = semantic_start.elapsed().as_secs_f64();
+
+ ss.maybe_flush_pending(true, false, config)?;
+
+ if last_decoded_frames < semantic_tokens.frames() {
+ let final_decode_start = Instant::now();
+ let final_delta_grid = streamed_tokens.to_grid_range(last_decoded_frames, semantic_tokens.frames())?;
+ let final_pcm = audio_stream.decode_step(&final_delta_grid, true)?;
+ accumulate_audio_decode_step_stats(
+ &mut ss.audio_decode_calls,
+ &mut ss.audio_input_frames,
+ &mut ss.audio_decoded_window_frames,
+ &mut ss.audio_max_decoded_window_frames,
+ audio_stream.last_step_stats(),
+ );
+ ss.audio_decode_seconds += final_decode_start.elapsed().as_secs_f64();
+ let callback_start = Instant::now();
+ let emitted_frames = if final_pcm.lengths().len() == 1 {
+ final_pcm.lengths()[0]
+ } else {
+ return Err(Error::GenerateFailed);
+ };
+ if emitted_frames > 0 {
+ (ss.on_chunk)(&final_pcm);
+ }
+ ss.callback_seconds += callback_start.elapsed().as_secs_f64();
+ if emitted_frames > 0 {
+ ss.output_samples.extend_from_slice(final_pcm.samples());
+ ss.output_frames = ss.output_frames.saturating_add(emitted_frames);
+ ss.output_sample_rate = final_pcm.sample_rate();
+ ss.output_channels = final_pcm.channels();
+ ss.emitted_chunks += 1;
+ if ss.first_emit_pending {
+ ss.first_chunk_seconds = Some(ss.stream_start.elapsed().as_secs_f64());
+ ss.first_chunk_frames = emitted_frames;
+ }
+ }
+ }
+ let semantic_decode_seconds =
+ (semantic_loop_seconds - ss.audio_decode_seconds_in_loop - ss.callback_seconds).max(0.0);
+ audio_stream.finish()?;
+
+ let stats = TtsExecutionStats {
+ semantic_decode_seconds,
+ audio_decode_seconds: ss.audio_decode_seconds,
+ callback_seconds: ss.callback_seconds,
+ first_chunk_seconds: ss.first_chunk_seconds.unwrap_or(0.0),
+ command_buffers_submitted: 0,
+ host_waits: 0,
+ semantic_frames: semantic_tokens.frames(),
+ first_chunk_frames: ss.first_chunk_frames,
+ emitted_chunks: ss.emitted_chunks,
+ audio_decode_calls: ss.audio_decode_calls,
+ audio_input_frames: ss.audio_input_frames,
+ audio_decoded_window_frames: ss.audio_decoded_window_frames,
+ audio_max_decoded_window_frames: ss.audio_max_decoded_window_frames,
+ };
+
+ let full_pcm = AudioPcmBatch::new(
+ ss.output_samples.into_boxed_slice(),
+ ss.output_sample_rate,
+ ss.output_channels,
+ vec![ss.output_frames].into_boxed_slice(),
+ )
+ .map_err(Error::from)?;
+
+ let instrumentation = self.take_text_decoder_instrumentation();
+ self.record_last_execution_stats(TtsExecutionStats {
+ command_buffers_submitted: instrumentation.command_buffers_submitted,
+ host_waits: instrumentation.host_waits,
+ ..stats
+ });
+
+ Ok(full_pcm)
+ }
+
+ fn generate_semantic_tokens(
+ &mut self,
+ text_tokens: &[u64],
+ seed: u64,
+ max_semantic_frames: usize,
+ ) -> Result<AudioTokenGrid, Error> {
+ let codec_cardinality = self.audio_decoder.codec_cardinality();
+ self.text_decoder.generate_semantic_tokens(text_tokens, codec_cardinality, seed, max_semantic_frames)
+ }
+
+ fn generate_semantic_tokens_with_callback<F>(
+ &mut self,
+ text_tokens: &[u64],
+ seed: u64,
+ max_semantic_frames: usize,
+ on_frame: &mut F,
+ ) -> Result<AudioTokenGrid, Error>
+ where
+ F: FnMut(&[u32]) -> Result<(), Error>,
+ {
+ let codec_cardinality = self.audio_decoder.codec_cardinality();
+ self.text_decoder.generate_semantic_tokens_with_callback(
+ text_tokens,
+ codec_cardinality,
+ seed,
+ max_semantic_frames,
+ on_frame,
+ )
+ }
+
+ fn take_text_decoder_instrumentation(&mut self) -> RunnerInstrumentation {
+ self.text_decoder.take_instrumentation()
+ }
+
+ fn record_last_execution_stats(
+ &mut self,
+ stats: TtsExecutionStats,
+ ) {
+ self.last_execution_stats.replace(stats);
+ }
+}
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
index 3e634ed4c..23191ee09 100644
--- a/crates/uzu/src/session/types/error.rs
+++ b/crates/uzu/src/session/types/error.rs
@@ -1,4 +1,172 @@
-use crate::classifier::ClassifierError;
+use crate::{DataType, classifier::ClassifierError, config::ModelType, session::config::TtsRunConfigError};
+
+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+pub enum TtsModelConfigError {
+ #[error("expected model_type={expected:?}, got {actual:?}")]
+ ModelTypeMismatch {
+ expected: ModelType,
+ actual: ModelType,
+ },
+ #[error("missing TTS model config")]
+ MissingTtsModelConfig,
+ #[error("FishAudio tensor {key} dtype mismatch: expected {expected:?}, got {actual:?}")]
+ FishAudioTensorDataTypeMismatch {
+ key: Box<str>,
+ expected: DataType,
+ actual: DataType,
+ },
+ #[error(
+ "FishAudio decoder requires num_codebooks > 1 plus positive codebook_size and max_seq_len, got num_codebooks={num_codebooks}, codebook_size={codebook_size}, max_seq_len={max_seq_len}"
+ )]
+ FishAudioDecoderCoreParametersInvalid {
+ num_codebooks: usize,
+ codebook_size: usize,
+ max_seq_len: usize,
+ },
+ #[error(
+ "FishAudio decoder num_codebooks={decoder_num_codebooks} does not match audio num_codebooks={audio_num_codebooks}"
+ )]
+ FishAudioDecoderNumCodebooksMismatch {
+ decoder_num_codebooks: usize,
+ audio_num_codebooks: usize,
+ },
+ #[error(
+ "FishAudio decoder codebook_size={codebook_size} must be at least audio residual codec cardinality={audio_codec_cardinality}"
+ )]
+ FishAudioDecoderCodebookSizeTooSmall {
+ codebook_size: usize,
+ audio_codec_cardinality: usize,
+ },
+ #[error("FishAudio semantic token range is invalid: begin={begin}, end={end}")]
+ FishAudioSemanticTokenRangeInvalid {
+ begin: i64,
+ end: i64,
+ },
+ #[error("FishAudio semantic token range overflow: begin={begin}, end={end}")]
+ FishAudioSemanticTokenRangeOverflow {
+ begin: i64,
+ end: i64,
+ },
+ #[error(
+ "FishAudio semantic codec cardinality={semantic_cardinality} does not match audio semantic codec cardinality={audio_semantic_cardinality}"
+ )]
+ FishAudioSemanticCodecCardinalityMismatch {
+ semantic_cardinality: usize,
+ audio_semantic_cardinality: usize,
+ },
+ #[error("followup_count overflow in async chain")]
+ AsyncChainFollowupCountOverflow,
+ #[error("async chain token count {total_count} exceeds capacity {capacity}")]
+ AsyncChainTokenCountExceedsCapacity {
+ total_count: usize,
+ capacity: usize,
+ },
+ #[error("vocab mask not supported for initial_token_count={initial_token_count} (expected 1 or 2)")]
+ UnsupportedInitialTokenCountForVocabMask {
+ initial_token_count: usize,
+ },
+ #[error("two-token vocab mask size overflow (row_words={row_words})")]
+ TwoTokenVocabMaskSizeOverflow {
+ row_words: usize,
+ },
+ #[error("token bitmask size overflow")]
+ TokenBitmaskSizeOverflow,
+ #[error(
+ "token bitmask length {actual_words}, expected {expected_words} (token_count={token_count}, row_words={row_words})"
+ )]
+ TokenBitmaskLengthMismatch {
+ actual_words: usize,
+ expected_words: usize,
+ token_count: usize,
+ row_words: usize,
+ },
+ #[error("precomputed bitmask size overflow")]
+ PrecomputedBitmaskSizeOverflow,
+ #[error("precomputed bitmask length {actual_words}, expected {expected_words}")]
+ PrecomputedBitmaskLengthMismatch {
+ actual_words: usize,
+ expected_words: usize,
+ },
+ #[error("vocab_limit resolved to 0")]
+ VocabLimitResolvedToZero,
+ #[error("inline bitmask size overflow (token_count={token_count}, row_words={row_words})")]
+ InlineBitmaskSizeOverflow {
+ token_count: usize,
+ row_words: usize,
+ },
+ #[error("vocab_limit with token_count={token_count} requires a precomputed bitmask")]
+ VocabLimitRequiresPrecomputedBitmask {
+ token_count: usize,
+ },
+ #[error("model_dim {model_dim} exceeds u32 range")]
+ ModelDimExceedsU32 {
+ model_dim: usize,
+ },
+ #[error("hidden capture row offset overflow (token_count={token_count}, model_dim={model_dim})")]
+ HiddenCaptureRowOffsetOverflow {
+ token_count: usize,
+ model_dim: usize,
+ },
+ #[error("hidden capture src offset overflow")]
+ HiddenCaptureSourceOffsetOverflow,
+ #[error(
+ "hidden capture shape/dtype mismatch: expected {expected_shape:?} @ {expected_data_type:?}, got {actual_shape:?} @ {actual_data_type:?}"
+ )]
+ HiddenCaptureTensorMismatch {
+ expected_shape: Box<[usize]>,
+ expected_data_type: DataType,
+ actual_shape: Box<[usize]>,
+ actual_data_type: DataType,
+ },
+ #[error(
+ "override embedding shape/dtype mismatch: expected {expected_shape:?} @ {expected_data_type:?}, got {actual_shape:?} @ {actual_data_type:?}"
+ )]
+ OverrideEmbeddingTensorMismatch {
+ expected_shape: Box<[usize]>,
+ expected_data_type: DataType,
+ actual_shape: Box<[usize]>,
+ actual_data_type: DataType,
+ },
+ #[error("add_scale total_len overflow (token_count={token_count}, model_dim={model_dim})")]
+ AddScaleTotalLengthOverflow {
+ token_count: usize,
+ model_dim: usize,
+ },
+ #[error("add_scale total_len {total_len} exceeds u32 range")]
+ AddScaleTotalLengthExceedsU32 {
+ total_len: usize,
+ },
+ #[error(
+ "add_scale bias shape/dtype mismatch: expected {expected_shape:?} @ {expected_data_type:?}, got {actual_shape:?} @ {actual_data_type:?}"
+ )]
+ AddScaleBiasTensorMismatch {
+ expected_shape: Box<[usize]>,
+ expected_data_type: DataType,
+ actual_shape: Box<[usize]>,
+ actual_data_type: DataType,
+ },
+ #[error("async chain copy src_offset overflow (src_slot={src_slot})")]
+ AsyncChainCopySourceOffsetOverflow {
+ src_slot: usize,
+ },
+ #[error("async chain copy size overflow (count={count})")]
+ AsyncChainCopySizeOverflow {
+ count: usize,
+ },
+ #[error("async chain result slot {slot} out of bounds (capacity={capacity})")]
+ AsyncChainResultSlotOutOfBounds {
+ slot: usize,
+ capacity: usize,
+ },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
+pub enum TtsPromptConfigError {
+ #[error("FishAudio prompt template references message.{field}, but default_message_fields does not provide it")]
+ MissingDefaultMessageField {
+ field: Box<str>,
+ },
+}
#[derive(Debug, thiserror::Error)]
pub enum Error {
@@ -12,6 +180,12 @@ pub enum Error {
UnableToCreateCommandBuffer(Box<dyn std::error::Error>),
#[error("Unable to load model configuration")]
UnableToLoadConfig,
+ #[error("Invalid TTS model config: {0}")]
+ InvalidTtsModelConfig(#[from] TtsModelConfigError),
+ #[error("Invalid TTS run config: {0}")]
+ InvalidTtsRunConfig(#[from] TtsRunConfigError),
+ #[error("Invalid TTS prompt config: {0}")]
+ InvalidTtsPromptConfig(#[from] TtsPromptConfigError),
#[error("Unable to load model weights")]
UnableToLoadWeights,
#[error("Unable to load tokenizer")]
@@ -58,4 +232,6 @@ pub enum Error {
CaptureFailed,
#[error("Classifier error: {0}")]
Classifier(#[from] ClassifierError),
+ #[error("Audio codec error: {0}")]
+ AudioCodec(#[from] crate::audio::AudioError),
}
diff --git a/crates/uzu/src/session/types/message.rs b/crates/uzu/src/session/types/message.rs
index f36961d3e..6f4afd26c 100644
--- a/crates/uzu/src/session/types/message.rs
+++ b/crates/uzu/src/session/types/message.rs
@@ -3,10 +3,13 @@ use std::collections::HashMap;
use serde::{Deserialize, Serialize};
use crate::{
- config::MessageProcessorConfig,
+ config::{MessageProcessorConfig, TtsMessageProcessorConfig},
session::{parameter::ConfigResolvableValue, types::Role},
};
+pub(crate) const DEFAULT_TTS_SPEAKER_ID: &str = "speaker:0";
+pub(crate) const DEFAULT_TTS_STYLE: &str = "interleave";
+
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct Message {
pub role: Role,
@@ -61,3 +64,25 @@ impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> for
result
}
}
+
+impl ConfigResolvableValue<TtsMessageProcessorConfig, HashMap<String, String>> for Message {
+ fn resolve(
+ &self,
+ config: &TtsMessageProcessorConfig,
+ ) -> HashMap<String, String> {
+ let role = match self.role {
+ Role::System => config.system_role_name.clone(),
+ Role::User => config.user_role_name.clone(),
+ Role::Assistant => config.assistant_role_name.clone(),
+ };
+ let mut result: HashMap<String, String> = config.default_message_fields.clone().into_iter().collect();
+ result.entry(String::from("speaker_id")).or_insert_with(|| String::from(DEFAULT_TTS_SPEAKER_ID));
+ result.entry(String::from("style")).or_insert_with(|| String::from(DEFAULT_TTS_STYLE));
+ result.insert(String::from("role"), role);
+ result.insert(String::from("content"), self.content.clone());
+ if let Some(reasoning_content) = self.reasoning_content.clone() {
+ result.insert(String::from("reasoning_content"), reasoning_content);
+ }
+ result
+ }
+}
diff --git a/crates/uzu/src/session/types/mod.rs b/crates/uzu/src/session/types/mod.rs
index c1dab1298..85e4668b4 100644
--- a/crates/uzu/src/session/types/mod.rs
+++ b/crates/uzu/src/session/types/mod.rs
@@ -5,7 +5,7 @@ mod output;
mod role;
mod stats;
mod text;
-pub use error::Error;
+pub use error::{Error, TtsModelConfigError, TtsPromptConfigError};
pub use input::Input;
pub use message::Message;
pub use output::{FinishReason, Output};
diff --git a/crates/uzu/tests/audio_kernel_perf_test.rs b/crates/uzu/tests/audio_kernel_perf_test.rs
new file mode 100644
index 000000000..034bf919c
--- /dev/null
+++ b/crates/uzu/tests/audio_kernel_perf_test.rs
@@ -0,0 +1,1582 @@
+#![cfg(all(feature = "audio-runtime", metal_backend))]
+
+use std::time::Instant;
+
+use uzu::{
+ Array, ArrayContextExt, DataType,
+ backends::{
+ common::{
+ Backend, Context, Encoder, Kernels,
+ gpu_types::ActivationType,
+ kernel::{
+ ActivationKernel, AudioAddKernel, AudioCausalConv1dGroupedKernel,
+ AudioCausalConv1dGroupedResidualKernel, AudioCausalConv1dKernel,
+ AudioCausalConvTranspose1dCausalPadKernel, AudioConv1dKernel, AudioHalfSnakeKernel, AudioNormNcsKernel,
+ AudioQuantizerDecodeKernel,
+ },
+ },
+ metal::Metal,
+ },
+};
+
+type Ctx = <Metal as Backend>::Context;
+
+const WARMUP: usize = 5;
+const ITERATIONS: usize = 20;
+
+fn measure<F: FnMut()>(
+ label: &str,
+ mut f: F,
+) -> f64 {
+ for _ in 0..WARMUP {
+ f();
+ }
+ let start = Instant::now();
+ for _ in 0..ITERATIONS {
+ f();
+ }
+ let elapsed = start.elapsed();
+ let mean_ms = elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64;
+ println!(" {label:<65} {mean_ms:>8.3} ms");
+ mean_ms
+}
+
+fn make_lengths(
+ context: &Ctx,
+ batch_size: usize,
+ value: i32,
+) -> Array<Metal> {
+ context.create_array_from(&[batch_size], &vec![value; batch_size], "lengths")
+}
+
+// ==========================================================================
+// FishAudio s1-mini vocoder configuration (from model config JSON)
+//
+// input_dim = 1024
+// decoder_dim = 1536
+// downsample_factor = [2, 2] (reversed for upsampler -> strides [2, 2])
+// decoder_rates = [8, 8, 4, 2]
+// n_codebooks = 9 (+ 1 semantic = 10 total)
+// codebook_dim = 8
+// codebook_size = 1024
+// semantic_codebook_size = 4096
+// samplerate = 44100
+// upsample_factor = 2*2 * 8*8*4*2 = 2048
+//
+// Channel progression (standard DAC with decoder_dim=1536):
+// Upsampler: 1024 -> 512 (s=2) -> 256 (s=2)
+// first_conv: 256 -> 1536 (k=7)
+// Decoder: 1536 -> 768 (s=8) -> 384 (s=8) -> 192 (s=4) -> 96 (s=2)
+// final_conv: 96 -> 1 (k=7)
+//
+// Residual units per decoder block: 3, with dilations [1, 3, 9], k=7
+// ConvNeXt depthwise: k=7, per-channel
+// Transpose conv kernel_size = stride * 2 (fast_two_tap path)
+//
+// For 10 seconds at 44100 Hz:
+// 441000 samples / 2048 upsample_factor = ~215 semantic frames
+// We use 215 frames as the starting point.
+// ==========================================================================
+
+const INPUT_DIM: usize = 1024;
+const DECODER_DIM: usize = 1536;
+const CODEBOOK_DIM: usize = 8;
+const TOTAL_CODEBOOKS: usize = 10;
+const RESIDUAL_QUANTIZERS: usize = 9;
+const SEMANTIC_CARDINALITY: usize = 4096;
+const RESIDUAL_CARDINALITY: usize = 1024;
+
+// Upsample block channel progression: input_dim -> input_dim/2 -> input_dim/4
+const UP_CHANNELS: [usize; 3] = [1024, 512, 256];
+const UP_STRIDES: [usize; 2] = [2, 2]; // downsample_factor reversed
+
+// Decoder block channel progression: decoder_dim -> decoder_dim/2 -> /4 -> /8 -> /16
+const DEC_CHANNELS: [usize; 5] = [1536, 768, 384, 192, 96];
+const DEC_STRIDES: [usize; 4] = [8, 8, 4, 2]; // decoder_rates
+
+const SEMANTIC_FRAMES_10S: usize = 215; // ~10s at 44100 Hz with upsample_factor=2048
+
+#[test]
+#[ignore] // Heavy benchmark (~240s); run explicitly with --ignored
+fn audio_kernel_perf() {
+ let context = <Metal as Backend>::Context::new().unwrap();
+ let dt = DataType::F16;
+ let batch_size: usize = 1;
+
+ let mut results: Vec<(String, f64)> = Vec::new();
+
+ println!();
+ println!(" FishAudio s1-mini vocoder kernel benchmark (10s audio, 44.1kHz, F16)");
+ println!(" =====================================================================");
+ println!();
+ println!(" {:<65} {:>8}", "Kernel", "Mean (ms)");
+ println!(" {:-<65} {:-<8}", "", "");
+
+ // -----------------------------------------------------------------------
+ // 1. AudioQuantizerDecodeKernel
+ // tokens: [B, K, T], output: [B, T, input_dim]
+ // -----------------------------------------------------------------------
+ {
+ let frames = SEMANTIC_FRAMES_10S;
+
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();
+
+ let tokens = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");
+ let lengths = make_lengths(&context, batch_size, frames as i32);
+ let semantic_codebook = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");
+ let semantic_out_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");
+ let semantic_out_bias = context.create_array(&[INPUT_DIM], dt, "perf_sem_bias");
+ let residual_codebooks =
+ context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "perf_res_cbs");
+ let residual_out_proj =
+ context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");
+ let residual_out_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");
+ let output = context.create_array(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");
+
+ let label = format!(
+ "QuantizerDecode [B={}, T={}, K={}, dim={}, cdim={}]",
+ batch_size, frames, TOTAL_CODEBOOKS, INPUT_DIM, CODEBOOK_DIM
+ );
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let tok = tokens.buffer();
+ let tok = tok.borrow();
+ let len = lengths.buffer();
+ let len = len.borrow();
+ let scb = semantic_codebook.buffer();
+ let scb = scb.borrow();
+ let sp = semantic_out_proj.buffer();
+ let sp = sp.borrow();
+ let sb = semantic_out_bias.buffer();
+ let sb = sb.borrow();
+ let rcb = residual_codebooks.buffer();
+ let rcb = rcb.borrow();
+ let rp = residual_out_proj.buffer();
+ let rp = rp.borrow();
+ let rb = residual_out_bias.buffer();
+ let rb = rb.borrow();
+ let out = output.buffer();
+ let mut out = out.borrow_mut();
+ kernel.encode(
+ &*tok,
+ &*len,
+ &*scb,
+ &*sp,
+ &*sb,
+ &*rcb,
+ &*rp,
+ &*rb,
+ &mut *out,
+ batch_size as i32,
+ TOTAL_CODEBOOKS as i32,
+ frames as i32,
+ INPUT_DIM as i32,
+ CODEBOOK_DIM as i32,
+ RESIDUAL_QUANTIZERS as i32,
+ SEMANTIC_CARDINALITY as i32,
+ RESIDUAL_CARDINALITY as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push(("QuantizerDecode".into(), ms));
+ }
+
+ // -----------------------------------------------------------------------
+ // 2. ConvTranspose1dCausalPad — all 6 stages where it appears
+ // Upsample: 1024->512 s=2, 512->256 s=2
+ // Decoder: 1536->768 s=8, 768->384 s=8, 384->192 s=4, 192->96 s=2
+ // -----------------------------------------------------------------------
+ {
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(&context, dt)
+ .unwrap();
+
+ // Upsample blocks (NSC input for first, NCS for rest)
+ let up_configs: [(usize, usize, usize, i32, &str); 2] = [
+ (UP_CHANNELS[0], UP_CHANNELS[1], UP_STRIDES[0], 1, "up0 NSC"), // 1024->512, s=2
+ (UP_CHANNELS[1], UP_CHANNELS[2], UP_STRIDES[1], 0, "up1 NCS"), // 512->256, s=2
+ ];
+ let mut frames = SEMANTIC_FRAMES_10S;
+ for (cin, cout, stride, layout, tag) in up_configs {
+ let seq_in = frames;
+ let seq_out = seq_in * stride;
+ let ksize = stride * 2;
+ let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");
+ let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");
+ let bias = context.create_array(&[cout], dt, "tconv_b");
+ let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");
+ let lengths = make_lengths(&context, batch_size, seq_out as i32);
+
+ let label = format!("TransConv {tag} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ kernel.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ cin as i32,
+ cout as i32,
+ seq_in as i32,
+ seq_out as i32,
+ ksize as i32,
+ stride as i32,
+ 1_i32,
+ layout,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("TransConv {tag}"), ms));
+ frames = seq_out;
+ }
+
+ // Decoder blocks (all NCS input)
+ frames = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1]; // 860
+ for block_idx in 0..4 {
+ let cin = DEC_CHANNELS[block_idx];
+ let cout = DEC_CHANNELS[block_idx + 1];
+ let stride = DEC_STRIDES[block_idx];
+ let seq_in = frames;
+ let seq_out = seq_in * stride;
+ let ksize = stride * 2;
+ let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");
+ let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");
+ let bias = context.create_array(&[cout], dt, "tconv_b");
+ let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");
+ let lengths = make_lengths(&context, batch_size, seq_out as i32);
+
+ let label = format!("TransConv dec{block_idx} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ kernel.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ cin as i32,
+ cout as i32,
+ seq_in as i32,
+ seq_out as i32,
+ ksize as i32,
+ stride as i32,
+ 1_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("TransConv dec{block_idx}"), ms));
+ frames = seq_out;
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // 3. CausalConv1d (groups=1) — first_conv, final_conv, and residual conv1
+ // first_conv: 256->1536, k=7
+ // residual conv1: same-channel, k=7, dilations [1,3,9]
+ // final_conv: 96->1, k=7
+ // -----------------------------------------------------------------------
+ {
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dKernel::new(&context, dt).unwrap();
+
+ // first_conv after upsampler: 256->1536, k=7
+ let fc_seq = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1]; // 860
+ {
+ let cin = UP_CHANNELS[2]; // 256
+ let cout = DECODER_DIM; // 1536
+ let ksize = 7;
+ let input = context.create_array(&[batch_size * cin * fc_seq], dt, "fc_in");
+ let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");
+ let bias = context.create_array(&[cout], dt, "fc_b");
+ let output = context.create_array(&[batch_size * cout * fc_seq], dt, "fc_out");
+ let lengths = make_lengths(&context, batch_size, fc_seq as i32);
+
+ let label = format!("CausalConv1d first_conv [{cin}->{cout}, k={ksize}, T={fc_seq}]");
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ kernel.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ cin as i32,
+ cout as i32,
+ fc_seq as i32,
+ ksize as i32,
+ 1_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push(("CausalConv1d first_conv".into(), ms));
+ }
+
+ // Residual conv1 at each decoder block resolution and dilation
+ let mut dec_frames = fc_seq;
+ for block_idx in 0..4 {
+ dec_frames *= DEC_STRIDES[block_idx];
+ let ch = DEC_CHANNELS[block_idx + 1];
+ let ksize = 7;
+
+ for &dilation in &[1, 3, 9] {
+ let label =
+ format!("CausalConv1d res dec{block_idx} [{ch}->{ch}, k={ksize}, d={dilation}, T={dec_frames}]");
+ let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");
+ let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");
+ let bias = context.create_array(&[ch], dt, "res_b");
+ let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");
+ let lengths = make_lengths(&context, batch_size, dec_frames as i32);
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ kernel.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ ch as i32,
+ ch as i32,
+ dec_frames as i32,
+ ksize as i32,
+ dilation as i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("CausalConv1d res dec{block_idx} d={dilation}"), ms));
+ }
+ }
+
+ // final_conv: 96->1, k=7
+ {
+ let cin = DEC_CHANNELS[4]; // 96
+ let cout = 1;
+ let ksize = 7;
+ let final_frames = SEMANTIC_FRAMES_10S * 2 * 2 * 8 * 8 * 4 * 2; // full resolution
+ let input = context.create_array(&[batch_size * cin * final_frames], dt, "fc_in");
+ let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");
+ let bias = context.create_array(&[cout], dt, "fc_b");
+ let output = context.create_array(&[batch_size * cout * final_frames], dt, "fc_out");
+ let lengths = make_lengths(&context, batch_size, final_frames as i32);
+
+ let label = format!("CausalConv1d final_conv [{cin}->{cout}, k={ksize}, T={final_frames}]");
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ kernel.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ cin as i32,
+ cout as i32,
+ final_frames as i32,
+ ksize as i32,
+ 1_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push(("CausalConv1d final_conv".into(), ms));
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // 4. CausalConv1dGroupedResidual — residual conv2 at each decoder stage
+ // Same dimensions as conv1 but with residual add, dilation=1
+ // -----------------------------------------------------------------------
+ {
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel::new(&context, dt)
+ .unwrap();
+
+ let mut dec_frames = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1];
+ for block_idx in 0..4 {
+ dec_frames *= DEC_STRIDES[block_idx];
+ let ch = DEC_CHANNELS[block_idx + 1];
+ let ksize = 7;
+
+ let label = format!("CausalConv1dResidual dec{block_idx} [{ch}->{ch}, k={ksize}, T={dec_frames}]");
+ let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");
+ let residual = context.create_array(&[batch_size * ch * dec_frames], dt, "res_res");
+ let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");
+ let bias = context.create_array(&[ch], dt, "res_b");
+ let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");
+ let lengths = make_lengths(&context, batch_size, dec_frames as i32);
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let r = residual.buffer();
+ let r = r.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ kernel.encode(
+ &*i,
+ &*r,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ ch as i32,
+ ch as i32,
+ dec_frames as i32,
+ ksize as i32,
+ 1_i32,
+ 1_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("CausalConv1dResidual dec{block_idx}"), ms));
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // 5. CausalConv1dGrouped (depthwise) — ConvNeXt depthwise at upsampler resolutions
+ // -----------------------------------------------------------------------
+ {
+ let kernel =
+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedKernel::new(&context, dt).unwrap();
+
+ let mut frames = SEMANTIC_FRAMES_10S;
+ for block_idx in 0..2 {
+ frames *= UP_STRIDES[block_idx];
+ let ch = UP_CHANNELS[block_idx + 1];
+ let ksize = 7;
+
+ let label = format!("CausalConv1dGrouped dw up{block_idx} [{ch}, g={ch}, k={ksize}, T={frames}]");
+ let input = context.create_array(&[batch_size * ch * frames], dt, "dw_in");
+ let weight = context.create_array(&[ch * 1 * ksize], dt, "dw_w");
+ let bias = context.create_array(&[ch], dt, "dw_b");
+ let output = context.create_array(&[batch_size * ch * frames], dt, "dw_out");
+ let lengths = make_lengths(&context, batch_size, frames as i32);
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ kernel.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ ch as i32,
+ ch as i32,
+ frames as i32,
+ ksize as i32,
+ 1_i32,
+ ch as i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("CausalConv1dGrouped dw up{block_idx}"), ms));
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // 6. HalfSnake — at each decoder stage resolution
+ // -----------------------------------------------------------------------
+ {
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, dt).unwrap();
+
+ // Before decoder blocks: snake on decoder_dim channels at upsampled frames
+ let fc_frames = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1];
+ let mut dec_frames = fc_frames;
+ for block_idx in 0..4 {
+ // Snake before trans_conv (at input resolution)
+ let ch_in = DEC_CHANNELS[block_idx];
+ {
+ let label = format!("HalfSnake pre-tconv dec{block_idx} [{ch_in}ch, T={dec_frames}]");
+ let input = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_in");
+ let alpha = context.create_array(&[ch_in], dt, "sn_a");
+ let output = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_out");
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let a = alpha.buffer();
+ let a = a.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ kernel.encode(
+ &*i,
+ &*a,
+ &mut *o,
+ ch_in as i32,
+ dec_frames as i32,
+ ch_in as i32,
+ 0.0_f32,
+ 1e-9_f32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("HalfSnake pre dec{block_idx}"), ms));
+ }
+
+ dec_frames *= DEC_STRIDES[block_idx];
+ // Snake in residual units (at output resolution)
+ let ch_out = DEC_CHANNELS[block_idx + 1];
+ {
+ let label = format!("HalfSnake res dec{block_idx} [{ch_out}ch, T={dec_frames}]");
+ let input = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_in");
+ let alpha = context.create_array(&[ch_out], dt, "sn_a");
+ let output = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_out");
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let a = alpha.buffer();
+ let a = a.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ kernel.encode(
+ &*i,
+ &*a,
+ &mut *o,
+ ch_out as i32,
+ dec_frames as i32,
+ ch_out as i32,
+ 0.0_f32,
+ 1e-9_f32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("HalfSnake res dec{block_idx}"), ms));
+ }
+ }
+ }
+
+ // -----------------------------------------------------------------------
+ // 7. NormNcs and Conv1d pointwise — ConvNeXt blocks in upsampler
+ // -----------------------------------------------------------------------
+ {
+ let k_norm = <<Metal as Backend>::Kernels as Kernels>::AudioNormNcsKernel::new(&context, dt).unwrap();
+ let k_conv1d = <<Metal as Backend>::Kernels as Kernels>::AudioConv1dKernel::new(&context, dt).unwrap();
+ let k_act = <<Metal as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, dt, false).unwrap();
+
+ let mut frames = SEMANTIC_FRAMES_10S;
+ for block_idx in 0..2 {
+ frames *= UP_STRIDES[block_idx];
+ let ch = UP_CHANNELS[block_idx + 1];
+
+ // Norm
+ {
+ let label = format!("NormNcs up{block_idx} [{ch}ch, T={frames}]");
+ let input = context.create_array(&[batch_size * ch * frames], dt, "norm_in");
+ let scales = context.create_array(&[ch], dt, "norm_s");
+ let bias = context.create_array(&[ch], dt, "norm_b");
+ let output = context.create_array(&[batch_size * ch * frames], dt, "norm_out");
+ let lengths = make_lengths(&context, batch_size, frames as i32);
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let s = scales.buffer();
+ let s = s.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ k_norm.encode(
+ &*i,
+ &*s,
+ &*b,
+ &mut *o,
+ &*l,
+ ch as i32,
+ frames as i32,
+ 1e-6_f32,
+ 1_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("NormNcs up{block_idx}"), ms));
+ }
+
+ // Pointwise conv1 (ch -> ch)
+ {
+ let label = format!("Conv1d pw up{block_idx} [{ch}->{ch}, k=1, T={frames}]");
+ let input = context.create_array(&[batch_size * ch * frames], dt, "pw_in");
+ let weight = context.create_array(&[ch * ch], dt, "pw_w");
+ let bias = context.create_array(&[ch], dt, "pw_b");
+ let output = context.create_array(&[batch_size * ch * frames], dt, "pw_out");
+ let lengths = make_lengths(&context, batch_size, frames as i32);
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let w = weight.buffer();
+ let w = w.borrow();
+ let b = bias.buffer();
+ let b = b.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ let l = lengths.buffer();
+ let l = l.borrow();
+ k_conv1d.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ ch as i32,
+ ch as i32,
+ frames as i32,
+ frames as i32,
+ 1_i32,
+ 1_i32,
+ 1_i32,
+ 0_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("Conv1d pw up{block_idx}"), ms));
+ }
+
+ // GELU
+ {
+ let n = ch * frames;
+ let label = format!("GELU up{block_idx} [n={n}]");
+ let input = context.create_array(&[n], dt, "gelu_in");
+ let output = context.create_array(&[n], dt, "gelu_out");
+
+ let ms = measure(&label, || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ {
+ let i = input.buffer();
+ let i = i.borrow();
+ let o = output.buffer();
+ let mut o = o.borrow_mut();
+ k_act.encode(Some(&*i), &mut *o, n as u32, ActivationType::GELU, &mut encoder);
+ }
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push((format!("GELU up{block_idx}"), ms));
+ }
+ }
+ }
+
+ // =======================================================================
+ // Full vocoder pipeline simulation — single command buffer
+ //
+ // Matches submit_decode_padded flow:
+ // quantizer -> 2 upsample blocks -> first_conv -> 4 decoder blocks -> final snake+conv+tanh
+ // =======================================================================
+ println!();
+ println!(" {:-<65} {:-<8}", "", "");
+ println!(" {:<65} {:>8}", "Full Pipeline Simulation (single command buffer)", "Mean (ms)");
+ println!(" {:-<65} {:-<8}", "", "");
+
+ // Build all kernels
+ let k_quant = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();
+ let k_tconv =
+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(&context, dt).unwrap();
+ let k_norm = <<Metal as Backend>::Kernels as Kernels>::AudioNormNcsKernel::new(&context, dt).unwrap();
+ let k_conv1d = <<Metal as Backend>::Kernels as Kernels>::AudioConv1dKernel::new(&context, dt).unwrap();
+ let k_act = <<Metal as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, dt, false).unwrap();
+ let k_add = <<Metal as Backend>::Kernels as Kernels>::AudioAddKernel::new(&context, dt).unwrap();
+ let k_snake = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, dt).unwrap();
+ let k_conv_grp =
+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedKernel::new(&context, dt).unwrap();
+ let k_conv1d_causal = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dKernel::new(&context, dt).unwrap();
+ let k_conv_res =
+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel::new(&context, dt).unwrap();
+
+ let frames = SEMANTIC_FRAMES_10S;
+
+ // Quantizer arrays
+ let p_q_tok = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");
+ let p_q_len = make_lengths(&context, batch_size, frames as i32);
+ let p_q_sem_cb = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");
+ let p_q_sem_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");
+ let p_q_sem_bias = context.create_array(&[INPUT_DIM], dt, "p_sem_bias");
+ let p_q_res_cbs =
+ context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");
+ let p_q_res_proj = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");
+ let p_q_res_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");
+
+ // Compute frame counts at each stage
+ let f_up0 = frames * UP_STRIDES[0]; // 430
+ let f_up1 = f_up0 * UP_STRIDES[1]; // 860
+ let f_dec0 = f_up1 * DEC_STRIDES[0]; // 6880
+ let f_dec1 = f_dec0 * DEC_STRIDES[1]; // 55040
+ let f_dec2 = f_dec1 * DEC_STRIDES[2]; // 220160
+ let f_dec3 = f_dec2 * DEC_STRIDES[3]; // 440320
+
+ // Pipeline scratch: large enough for the biggest intermediate
+ // Biggest is decoder_dim * f_up1 or ch * f_dec stages
+ let max_elems: usize = batch_size * DECODER_DIM * f_dec3.max(f_up1);
+ let scratch_a = context.create_array(&[max_elems], dt, "pipe_sa");
+ let scratch_b = context.create_array(&[max_elems], dt, "pipe_sb");
+ // Third scratch buffer for residual connections (avoids RefCell borrow conflict)
+ let scratch_r = context.create_array(&[max_elems], dt, "pipe_sr");
+
+ // Upsample block 0: 1024->512, stride=2
+ let u0_w = context.create_array(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");
+ let u0_b = context.create_array(&[UP_CHANNELS[1]], dt, "u0_b");
+ // ConvNeXt 0 (ch=512)
+ let c0_ch = UP_CHANNELS[1];
+ let c0_dw_w = context.create_array(&[c0_ch * 1 * 7], dt, "c0_dw_w");
+ let c0_dw_b = context.create_array(&[c0_ch], dt, "c0_dw_b");
+ let c0_ns = context.create_array(&[c0_ch], dt, "c0_ns");
+ let c0_nb = context.create_array(&[c0_ch], dt, "c0_nb");
+ let c0_p1_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p1_w");
+ let c0_p1_b = context.create_array(&[c0_ch], dt, "c0_p1_b");
+ let c0_p2_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p2_w");
+ let c0_p2_b = context.create_array(&[c0_ch], dt, "c0_p2_b");
+
+ // Upsample block 1: 512->256, stride=2
+ let u1_w = context.create_array(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");
+ let u1_b = context.create_array(&[UP_CHANNELS[2]], dt, "u1_b");
+ // ConvNeXt 1 (ch=256)
+ let c1_ch = UP_CHANNELS[2];
+ let c1_dw_w = context.create_array(&[c1_ch * 1 * 7], dt, "c1_dw_w");
+ let c1_dw_b = context.create_array(&[c1_ch], dt, "c1_dw_b");
+ let c1_ns = context.create_array(&[c1_ch], dt, "c1_ns");
+ let c1_nb = context.create_array(&[c1_ch], dt, "c1_nb");
+ let c1_p1_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p1_w");
+ let c1_p1_b = context.create_array(&[c1_ch], dt, "c1_p1_b");
+ let c1_p2_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p2_w");
+ let c1_p2_b = context.create_array(&[c1_ch], dt, "c1_p2_b");
+
+ // first_conv: 256->1536, k=7
+ let fc_w = context.create_array(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");
+ let fc_b = context.create_array(&[DECODER_DIM], dt, "fc_b");
+
+ // Decoder block weights for all 4 blocks
+ struct DecBlockWeights {
+ snake_a: Array<Metal>,
+ tconv_w: Array<Metal>,
+ tconv_b: Array<Metal>,
+ // 3 residual units, each with snake1, conv1, snake2, conv2
+ ru_s1: [Array<Metal>; 3],
+ ru_c1_w: [Array<Metal>; 3],
+ ru_c1_b: [Array<Metal>; 3],
+ ru_s2: [Array<Metal>; 3],
+ ru_c2_w: [Array<Metal>; 3],
+ ru_c2_b: [Array<Metal>; 3],
+ }
+
+ let dec_blocks: Vec<DecBlockWeights> = (0..4)
+ .map(|i| {
+ let cin = DEC_CHANNELS[i];
+ let cout = DEC_CHANNELS[i + 1];
+ let stride = DEC_STRIDES[i];
+ let ksize = stride * 2;
+ DecBlockWeights {
+ snake_a: context.create_array(&[cin], dt, &format!("d{i}_sa")),
+ tconv_w: context.create_array(&[cin * cout * ksize], dt, &format!("d{i}_tw")),
+ tconv_b: context.create_array(&[cout], dt, &format!("d{i}_tb")),
+ ru_s1: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s1"))),
+ ru_c1_w: std::array::from_fn(|j| {
+ context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))
+ }),
+ ru_c1_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c1b"))),
+ ru_s2: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s2"))),
+ ru_c2_w: std::array::from_fn(|j| {
+ context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))
+ }),
+ ru_c2_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c2b"))),
+ }
+ })
+ .collect();
+
+ // final snake + conv + tanh
+ let final_snake_a = context.create_array(&[DEC_CHANNELS[4]], dt, "final_sa");
+ let final_conv_w = context.create_array(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");
+ let final_conv_b = context.create_array(&[1], dt, "final_cb");
+
+ // Length arrays
+ let len_up0 = make_lengths(&context, batch_size, f_up0 as i32);
+ let len_up1 = make_lengths(&context, batch_size, f_up1 as i32);
+ let len_dec0 = make_lengths(&context, batch_size, f_dec0 as i32);
+ let len_dec1 = make_lengths(&context, batch_size, f_dec1 as i32);
+ let len_dec2 = make_lengths(&context, batch_size, f_dec2 as i32);
+ let len_dec3 = make_lengths(&context, batch_size, f_dec3 as i32);
+ let dec_lens = [&len_dec0, &len_dec1, &len_dec2, &len_dec3];
+ let dec_frame_sizes = [f_dec0, f_dec1, f_dec2, f_dec3];
+ let dilations = [1_i32, 3, 9];
+
+ let ms = measure("Full pipeline (quant -> 2 up -> first_conv -> 4 dec -> final)", || {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();
+ let sa = scratch_a.buffer();
+ let sb = scratch_b.buffer();
+
+ // --- Quantizer: output [B, T, input_dim] (NSC) ---
+ {
+ let tok = p_q_tok.buffer();
+ let tok = tok.borrow();
+ let len = p_q_len.buffer();
+ let len = len.borrow();
+ let scb = p_q_sem_cb.buffer();
+ let scb = scb.borrow();
+ let sp = p_q_sem_proj.buffer();
+ let sp = sp.borrow();
+ let sbi = p_q_sem_bias.buffer();
+ let sbi = sbi.borrow();
+ let rcb = p_q_res_cbs.buffer();
+ let rcb = rcb.borrow();
+ let rp = p_q_res_proj.buffer();
+ let rp = rp.borrow();
+ let rb = p_q_res_bias.buffer();
+ let rb = rb.borrow();
+ let mut out = sa.borrow_mut();
+ k_quant.encode(
+ &*tok,
+ &*len,
+ &*scb,
+ &*sp,
+ &*sbi,
+ &*rcb,
+ &*rp,
+ &*rb,
+ &mut *out,
+ batch_size as i32,
+ TOTAL_CODEBOOKS as i32,
+ frames as i32,
+ INPUT_DIM as i32,
+ CODEBOOK_DIM as i32,
+ RESIDUAL_QUANTIZERS as i32,
+ SEMANTIC_CARDINALITY as i32,
+ RESIDUAL_CARDINALITY as i32,
+ &mut encoder,
+ );
+ }
+
+ // --- Upsample block 0: 1024->512, stride=2, NSC input ---
+ {
+ let i = sa.borrow();
+ let w = u0_w.buffer();
+ let w = w.borrow();
+ let b = u0_b.buffer();
+ let b = b.borrow();
+ let mut o = sb.borrow_mut();
+ let l = len_up0.buffer();
+ let l = l.borrow();
+ k_tconv.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ UP_CHANNELS[0] as i32,
+ UP_CHANNELS[1] as i32,
+ frames as i32,
+ f_up0 as i32,
+ (UP_STRIDES[0] * 2) as i32,
+ UP_STRIDES[0] as i32,
+ 1_i32,
+ 1_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ // ConvNeXt 0
+ {
+ let i = sb.borrow();
+ let w = c0_dw_w.buffer();
+ let w = w.borrow();
+ let b = c0_dw_b.buffer();
+ let b = b.borrow();
+ let mut o = sa.borrow_mut();
+ let l = len_up0.buffer();
+ let l = l.borrow();
+ k_conv_grp.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ c0_ch as i32,
+ c0_ch as i32,
+ f_up0 as i32,
+ 7_i32,
+ 1_i32,
+ c0_ch as i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let i = sa.borrow();
+ let s = c0_ns.buffer();
+ let s = s.borrow();
+ let b = c0_nb.buffer();
+ let b = b.borrow();
+ let mut o = sb.borrow_mut();
+ let l = len_up0.buffer();
+ let l = l.borrow();
+ k_norm.encode(
+ &*i,
+ &*s,
+ &*b,
+ &mut *o,
+ &*l,
+ c0_ch as i32,
+ f_up0 as i32,
+ 1e-6_f32,
+ 1_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let i = sb.borrow();
+ let w = c0_p1_w.buffer();
+ let w = w.borrow();
+ let b = c0_p1_b.buffer();
+ let b = b.borrow();
+ let mut o = sa.borrow_mut();
+ let l = len_up0.buffer();
+ let l = l.borrow();
+ k_conv1d.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ c0_ch as i32,
+ c0_ch as i32,
+ f_up0 as i32,
+ f_up0 as i32,
+ 1_i32,
+ 1_i32,
+ 1_i32,
+ 0_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let i = sa.borrow();
+ let mut o = sb.borrow_mut();
+ k_act.encode(Some(&*i), &mut *o, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);
+ }
+ {
+ let i = sb.borrow();
+ let w = c0_p2_w.buffer();
+ let w = w.borrow();
+ let b = c0_p2_b.buffer();
+ let b = b.borrow();
+ let mut o = sa.borrow_mut();
+ let l = len_up0.buffer();
+ let l = l.borrow();
+ k_conv1d.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ c0_ch as i32,
+ c0_ch as i32,
+ f_up0 as i32,
+ f_up0 as i32,
+ 1_i32,
+ 1_i32,
+ 1_i32,
+ 0_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let a = sa.borrow();
+ let mut o = sb.borrow_mut();
+ k_add.encode(&*a, &*a, &mut *o, (c0_ch * f_up0) as i32, &mut encoder);
+ }
+
+ // --- Upsample block 1: 512->256, stride=2, NCS input ---
+ {
+ let i = sb.borrow();
+ let w = u1_w.buffer();
+ let w = w.borrow();
+ let b = u1_b.buffer();
+ let b = b.borrow();
+ let mut o = sa.borrow_mut();
+ let l = len_up1.buffer();
+ let l = l.borrow();
+ k_tconv.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ UP_CHANNELS[1] as i32,
+ UP_CHANNELS[2] as i32,
+ f_up0 as i32,
+ f_up1 as i32,
+ (UP_STRIDES[1] * 2) as i32,
+ UP_STRIDES[1] as i32,
+ 1_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ // ConvNeXt 1
+ {
+ let i = sa.borrow();
+ let w = c1_dw_w.buffer();
+ let w = w.borrow();
+ let b = c1_dw_b.buffer();
+ let b = b.borrow();
+ let mut o = sb.borrow_mut();
+ let l = len_up1.buffer();
+ let l = l.borrow();
+ k_conv_grp.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ c1_ch as i32,
+ c1_ch as i32,
+ f_up1 as i32,
+ 7_i32,
+ 1_i32,
+ c1_ch as i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let i = sb.borrow();
+ let s = c1_ns.buffer();
+ let s = s.borrow();
+ let b = c1_nb.buffer();
+ let b = b.borrow();
+ let mut o = sa.borrow_mut();
+ let l = len_up1.buffer();
+ let l = l.borrow();
+ k_norm.encode(
+ &*i,
+ &*s,
+ &*b,
+ &mut *o,
+ &*l,
+ c1_ch as i32,
+ f_up1 as i32,
+ 1e-6_f32,
+ 1_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let i = sa.borrow();
+ let w = c1_p1_w.buffer();
+ let w = w.borrow();
+ let b = c1_p1_b.buffer();
+ let b = b.borrow();
+ let mut o = sb.borrow_mut();
+ let l = len_up1.buffer();
+ let l = l.borrow();
+ k_conv1d.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ c1_ch as i32,
+ c1_ch as i32,
+ f_up1 as i32,
+ f_up1 as i32,
+ 1_i32,
+ 1_i32,
+ 1_i32,
+ 0_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let i = sb.borrow();
+ let mut o = sa.borrow_mut();
+ k_act.encode(Some(&*i), &mut *o, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);
+ }
+ {
+ let i = sa.borrow();
+ let w = c1_p2_w.buffer();
+ let w = w.borrow();
+ let b = c1_p2_b.buffer();
+ let b = b.borrow();
+ let mut o = sb.borrow_mut();
+ let l = len_up1.buffer();
+ let l = l.borrow();
+ k_conv1d.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ c1_ch as i32,
+ c1_ch as i32,
+ f_up1 as i32,
+ f_up1 as i32,
+ 1_i32,
+ 1_i32,
+ 1_i32,
+ 0_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ {
+ let a = sb.borrow();
+ let mut o = sa.borrow_mut();
+ k_add.encode(&*a, &*a, &mut *o, (c1_ch * f_up1) as i32, &mut encoder);
+ }
+
+ // --- first_conv: 256->1536, k=7 ---
+ {
+ let i = sa.borrow();
+ let w = fc_w.buffer();
+ let w = w.borrow();
+ let b = fc_b.buffer();
+ let b = b.borrow();
+ let mut o = sb.borrow_mut();
+ let l = len_up1.buffer();
+ let l = l.borrow();
+ k_conv1d_causal.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ UP_CHANNELS[2] as i32,
+ DECODER_DIM as i32,
+ f_up1 as i32,
+ 7_i32,
+ 1_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+
+ // --- 4 Decoder blocks ---
+ // State in sb after first_conv
+ let mut cur_in_sa = false; // data is in sb
+ let mut cur_frames = f_up1;
+
+ for block_idx in 0..4 {
+ let blk = &dec_blocks[block_idx];
+ let cin = DEC_CHANNELS[block_idx];
+ let cout = DEC_CHANNELS[block_idx + 1];
+ let stride = DEC_STRIDES[block_idx];
+ let ksize = stride * 2;
+ let next_frames = dec_frame_sizes[block_idx];
+ let dec_len = dec_lens[block_idx];
+
+ // snake (in-place swap)
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let a = blk.snake_a.buffer();
+ let a = a.borrow();
+ let mut o = o_buf.borrow_mut();
+ k_snake.encode(
+ &*i,
+ &*a,
+ &mut *o,
+ cin as i32,
+ cur_frames as i32,
+ cin as i32,
+ 0.0_f32,
+ 1e-9_f32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+
+ // trans_conv
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let w = blk.tconv_w.buffer();
+ let w = w.borrow();
+ let b = blk.tconv_b.buffer();
+ let b = b.borrow();
+ let mut o = o_buf.borrow_mut();
+ let l = dec_len.buffer();
+ let l = l.borrow();
+ k_tconv.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ cin as i32,
+ cout as i32,
+ cur_frames as i32,
+ next_frames as i32,
+ ksize as i32,
+ stride as i32,
+ 1_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+ cur_frames = next_frames;
+
+ // 3 residual units
+ let sr = scratch_r.buffer();
+ for ru_idx in 0..3 {
+ // snake1
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let a = blk.ru_s1[ru_idx].buffer();
+ let a = a.borrow();
+ let mut o = o_buf.borrow_mut();
+ k_snake.encode(
+ &*i,
+ &*a,
+ &mut *o,
+ cout as i32,
+ cur_frames as i32,
+ cout as i32,
+ 0.0_f32,
+ 1e-9_f32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+
+ // conv1 (dilation varies: 1, 3, 9)
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let w = blk.ru_c1_w[ru_idx].buffer();
+ let w = w.borrow();
+ let b = blk.ru_c1_b[ru_idx].buffer();
+ let b = b.borrow();
+ let mut o = o_buf.borrow_mut();
+ let l = dec_len.buffer();
+ let l = l.borrow();
+ k_conv1d_causal.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ cout as i32,
+ cout as i32,
+ cur_frames as i32,
+ 7_i32,
+ dilations[ru_idx],
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+
+ // snake2
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let a = blk.ru_s2[ru_idx].buffer();
+ let a = a.borrow();
+ let mut o = o_buf.borrow_mut();
+ k_snake.encode(
+ &*i,
+ &*a,
+ &mut *o,
+ cout as i32,
+ cur_frames as i32,
+ cout as i32,
+ 0.0_f32,
+ 1e-9_f32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+
+ // conv2 + residual add
+ // Use scratch_r as the residual source (separate buffer to
+ // avoid RefCell borrow conflict with the output destination).
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let r = sr.borrow();
+ let w = blk.ru_c2_w[ru_idx].buffer();
+ let w = w.borrow();
+ let b = blk.ru_c2_b[ru_idx].buffer();
+ let b = b.borrow();
+ let mut o = o_buf.borrow_mut();
+ let l = dec_len.buffer();
+ let l = l.borrow();
+ k_conv_res.encode(
+ &*i,
+ &*r,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ cout as i32,
+ cout as i32,
+ cur_frames as i32,
+ 7_i32,
+ 1_i32,
+ 1_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+ }
+ }
+
+ // --- final snake + conv + tanh ---
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let a = final_snake_a.buffer();
+ let a = a.borrow();
+ let mut o = o_buf.borrow_mut();
+ k_snake.encode(
+ &*i,
+ &*a,
+ &mut *o,
+ DEC_CHANNELS[4] as i32,
+ f_dec3 as i32,
+ DEC_CHANNELS[4] as i32,
+ 0.0_f32,
+ 1e-9_f32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let w = final_conv_w.buffer();
+ let w = w.borrow();
+ let b = final_conv_b.buffer();
+ let b = b.borrow();
+ let mut o = o_buf.borrow_mut();
+ let l = len_dec3.buffer();
+ let l = l.borrow();
+ k_conv1d_causal.encode(
+ &*i,
+ &*w,
+ &*b,
+ &mut *o,
+ &*l,
+ DEC_CHANNELS[4] as i32,
+ 1_i32,
+ f_dec3 as i32,
+ 7_i32,
+ 1_i32,
+ 0_i32,
+ batch_size as i32,
+ &mut encoder,
+ );
+ }
+ cur_in_sa = !cur_in_sa;
+ // tanh
+ {
+ let (i_buf, o_buf) = if cur_in_sa {
+ (sa.clone(), sb.clone())
+ } else {
+ (sb.clone(), sa.clone())
+ };
+ let i = i_buf.borrow();
+ let mut o = o_buf.borrow_mut();
+ k_act.encode(Some(&*i), &mut *o, f_dec3 as u32, ActivationType::TANH, &mut encoder);
+ }
+
+ encoder.end_encoding().submit().wait_until_completed().unwrap();
+ });
+ results.push(("Full pipeline".into(), ms));
+
+ // =======================================================================
+ // Summary
+ // =======================================================================
+ println!();
+ println!(" === SUMMARY ===");
+ println!(" {:<65} {:>8}", "Kernel", "Mean (ms)");
+ println!(" {:-<65} {:-<8}", "", "");
+ let mut total_individual = 0.0;
+ for (name, ms) in &results {
+ println!(" {name:<65} {ms:>8.3}");
+ if name != "Full pipeline" {
+ total_individual += ms;
+ }
+ }
+ println!(" {:-<65} {:-<8}", "", "");
+ println!(" {:<65} {:>8.3}", "Sum of individual kernels", total_individual);
+ if let Some((_, pipeline_ms)) = results.iter().find(|(n, _)| n == "Full pipeline") {
+ println!(" {:<65} {:>8.3}", "Full pipeline (single CB)", pipeline_ms);
+ }
+ println!();
+}
diff --git a/crates/uzu/tests/audio_nanocodec_fsq_test.rs b/crates/uzu/tests/audio_nanocodec_fsq_test.rs
new file mode 100644
index 000000000..a03d71722
--- /dev/null
+++ b/crates/uzu/tests/audio_nanocodec_fsq_test.rs
@@ -0,0 +1,32 @@
+mod common;
+
+use common::audio_nanocodec_fsq_reference::{fsq_decode_reference, fsq_encode_reference};
+
+#[test]
+fn fsq_decode_reference_masks_values_beyond_lengths() {
+ let output = fsq_decode_reference(&[0, 7, 11, 3, 5, 9], &[2], 1, 2, 3, 2, &[8, 5]).expect("decode");
+
+ assert_eq!(output.len(), 12);
+ let masked_t2 = [output[2], output[5], output[8], output[11]];
+ assert_eq!(masked_t2, [0.0, 0.0, 0.0, 0.0]);
+}
+
+#[test]
+fn fsq_encode_reference_masks_values_beyond_lengths() {
+ let tokens = fsq_encode_reference(
+ &[-0.9, -0.1, 0.2, 0.7, 0.8, 0.3, -0.4, -0.7, -0.2, 0.0, 0.4, 0.9, 0.5, -0.5, 0.1, -0.1],
+ &[3],
+ 1,
+ 2,
+ 4,
+ 2,
+ &[8, 6],
+ &[1, 8],
+ 1e-3,
+ )
+ .expect("encode");
+
+ assert_eq!(tokens.len(), 8);
+ assert_eq!(tokens[3], 0);
+ assert_eq!(tokens[7], 0);
+}
diff --git a/crates/uzu/tests/audio_nanocodec_runtime_test.rs b/crates/uzu/tests/audio_nanocodec_runtime_test.rs
new file mode 100644
index 000000000..bb0767544
--- /dev/null
+++ b/crates/uzu/tests/audio_nanocodec_runtime_test.rs
@@ -0,0 +1,177 @@
+#![cfg(all(feature = "audio-runtime", metal_backend))]
+
+mod common;
+
+use common::audio_nanocodec_fsq_reference::{fsq_decode_reference, fsq_encode_reference};
+use uzu::{
+ NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig,
+ backends::metal::Metal,
+ prelude::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioTokenGrid},
+};
+
+fn create_runtime() -> NanoCodecFsqRuntime<Metal> {
+ let config =
+ NanoCodecFsqRuntimeConfig::new(24_000, 2, vec![8, 6].into_boxed_slice(), 1e-3).expect("valid runtime config");
+ NanoCodecFsqRuntime::new(config)
+}
+
+fn make_pcm(
+ channels: usize,
+ lengths: &[usize],
+) -> AudioPcmBatch {
+ let sample_count = lengths.iter().sum::<usize>() * channels;
+ let samples: Vec<f32> = (0..sample_count).map(|index| (index as f32 * 0.071 - 1.3).sin() * 0.9).collect();
+ AudioPcmBatch::new(samples.into_boxed_slice(), 24_000, channels, lengths.to_vec().into_boxed_slice()).expect("pcm")
+}
+
+fn pack_for_reference(pcm: &AudioPcmBatch) -> (Vec<f32>, Vec<i32>, usize) {
+ let batch_size = pcm.batch_size();
+ let channels = pcm.channels();
+ let frames = pcm.lengths().iter().copied().max().unwrap_or(0);
+ let mut padded = vec![0.0_f32; batch_size * channels * frames];
+ let mut lengths_i32 = Vec::with_capacity(batch_size);
+
+ let mut src_offset = 0usize;
+ for (batch, &length) in pcm.lengths().iter().enumerate() {
+ lengths_i32.push(length as i32);
+ for frame in 0..length {
+ for channel in 0..channels {
+ let src_index = src_offset + frame * channels + channel;
+ let dst_index = (batch * channels + channel) * frames + frame;
+ padded[dst_index] = pcm.samples()[src_index];
+ }
+ }
+ src_offset += length * channels;
+ }
+
+ (padded, lengths_i32, frames)
+}
+
+fn unpack_reference_output(
+ padded: &[f32],
+ channels: usize,
+ frames: usize,
+ lengths: &[usize],
+) -> Vec<f32> {
+ let total = lengths.iter().sum::<usize>() * channels;
+ let mut out = vec![0.0_f32; total];
+ let mut dst_offset = 0usize;
+
+ for (batch, &length) in lengths.iter().enumerate() {
+ for frame in 0..length {
+ for channel in 0..channels {
+ let src_index = (batch * channels + channel) * frames + frame;
+ let dst_index = dst_offset + frame * channels + channel;
+ out[dst_index] = padded[src_index];
+ }
+ }
+ dst_offset += length * channels;
+ }
+
+ out
+}
+
+#[test]
+fn nanocodec_runtime_encode_matches_fsq_reference() {
+ let runtime = create_runtime();
+ let pcm = make_pcm(runtime.config().channels(), &[3, 5]);
+ let tokens = runtime.encode(&pcm).expect("encode");
+
+ let (padded_input, lengths_i32, frames) = pack_for_reference(&pcm);
+ let expected_i32 = fsq_encode_reference(
+ &padded_input,
+ &lengths_i32,
+ pcm.batch_size(),
+ runtime.config().num_groups(),
+ frames,
+ runtime.config().codebook_dim_per_group(),
+ runtime.config().num_levels_per_group(),
+ runtime.config().dim_base_index(),
+ runtime.config().eps(),
+ )
+ .expect("fsq reference encode");
+ let expected_u32: Vec<u32> = expected_i32.into_iter().map(|value| value as u32).collect();
+ let expected = AudioTokenGrid::new(
+ expected_u32.into_boxed_slice(),
+ pcm.batch_size(),
+ runtime.config().num_groups(),
+ frames,
+ pcm.lengths().to_vec().into_boxed_slice(),
+ )
+ .expect("expected token grid");
+
+ assert_eq!(tokens.batch_size(), expected.batch_size());
+ assert_eq!(tokens.codebooks(), expected.codebooks());
+ assert_eq!(tokens.frames(), expected.frames());
+ assert_eq!(tokens.lengths(), expected.lengths());
+ assert_eq!(tokens.tokens(), expected.tokens());
+}
+
+#[test]
+fn nanocodec_runtime_decode_matches_fsq_reference() {
+ let runtime = create_runtime();
+ let pcm = make_pcm(runtime.config().channels(), &[4, 2]);
+ let encoded = runtime.encode(&pcm).expect("encode");
+ let decoded = runtime.decode(&encoded).expect("decode");
+
+ let reference_tokens_i32: Vec<i32> = encoded.tokens().iter().map(|&token| token as i32).collect();
+ let lengths_i32: Vec<i32> = encoded.lengths().iter().map(|&length| length as i32).collect();
+ let reference_padded = fsq_decode_reference(
+ &reference_tokens_i32,
+ &lengths_i32,
+ encoded.batch_size(),
+ runtime.config().num_groups(),
+ encoded.frames(),
+ runtime.config().codebook_dim_per_group(),
+ runtime.config().num_levels_per_group(),
+ )
+ .expect("fsq reference decode");
+ let expected_samples =
+ unpack_reference_output(&reference_padded, runtime.config().channels(), encoded.frames(), encoded.lengths());
+
+ assert_eq!(decoded.sample_rate(), runtime.config().sample_rate());
+ assert_eq!(decoded.channels(), runtime.config().channels());
+ assert_eq!(decoded.lengths(), encoded.lengths());
+
+ for (index, (&got, &expected)) in decoded.samples().iter().zip(expected_samples.iter()).enumerate() {
+ let delta = (got - expected).abs();
+ assert!(delta <= 1e-5_f32, "decode mismatch at index={index}: got={got}, expected={expected}, delta={delta}");
+ }
+}
+
+#[test]
+fn nanocodec_runtime_decode_rejects_out_of_range_token() {
+ let runtime = create_runtime();
+ let cardinality = runtime.config().codec_cardinality();
+ let tokens = AudioTokenGrid::new(
+ vec![0, cardinality, 1, 2].into_boxed_slice(),
+ 1,
+ runtime.config().num_groups(),
+ 2,
+ vec![2usize].into_boxed_slice(),
+ )
+ .expect("token grid");
+
+ let error = runtime.decode(&tokens).expect_err("decode should fail");
+ assert_eq!(
+ error,
+ AudioError::InvalidCodecToken {
+ token: cardinality,
+ cardinality,
+ }
+ );
+}
+
+#[test]
+fn nanocodec_runtime_encode_rejects_channel_mismatch() {
+ let runtime = create_runtime();
+ let pcm = make_pcm(runtime.config().channels() - 1, &[2]);
+
+ let error = runtime.encode(&pcm).expect_err("encode should fail");
+ match error {
+ AudioError::Runtime(message) => {
+ assert!(message.contains("channel mismatch"), "unexpected error message: {message}");
+ },
+ other => panic!("unexpected error variant: {other:?}"),
+ }
+}
diff --git a/crates/uzu/tests/common/audio_nanocodec_fsq_reference.rs b/crates/uzu/tests/common/audio_nanocodec_fsq_reference.rs
new file mode 100644
index 000000000..433a6aa52
--- /dev/null
+++ b/crates/uzu/tests/common/audio_nanocodec_fsq_reference.rs
@@ -0,0 +1,197 @@
+use uzu::prelude::{AudioError, AudioResult};
+
+fn checked_product(values: &[usize]) -> AudioResult<usize> {
+ values
+ .iter()
+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))
+ .ok_or(AudioError::Runtime("dimension product overflow".to_string()))
+}
+
+fn round_ties_to_even(value: f32) -> f32 {
+ let floor = value.floor();
+ let fraction = value - floor;
+
+ if fraction < 0.5 {
+ floor
+ } else if fraction > 0.5 {
+ floor + 1.0
+ } else if (floor as i64 & 1) != 0 {
+ floor + 1.0
+ } else {
+ floor
+ }
+}
+
+fn validate_fsq_common(
+ batch_size: usize,
+ num_groups: usize,
+ seq_len: usize,
+ codebook_dim: usize,
+ num_levels: &[i32],
+ lengths: &[i32],
+) -> AudioResult<()> {
+ if num_groups == 0 || codebook_dim == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+
+ if num_levels.len() != codebook_dim {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: codebook_dim,
+ actual_tokens: num_levels.len(),
+ });
+ }
+
+ for &levels in num_levels {
+ if levels <= 1 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ }
+
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+
+ for &length in lengths {
+ if length < 0 || length as usize > seq_len {
+ return Err(AudioError::InvalidTokenLengthValue {
+ length: length.max(0) as usize,
+ frames: seq_len,
+ });
+ }
+ }
+
+ Ok(())
+}
+
+pub fn fsq_decode_reference(
+ tokens: &[i32],
+ lengths: &[i32],
+ batch_size: usize,
+ num_groups: usize,
+ seq_len: usize,
+ codebook_dim: usize,
+ num_levels: &[i32],
+) -> AudioResult<Vec<f32>> {
+ validate_fsq_common(batch_size, num_groups, seq_len, codebook_dim, num_levels, lengths)?;
+
+ let expected_tokens = checked_product(&[batch_size, num_groups, seq_len])?;
+ if tokens.len() != expected_tokens {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens,
+ actual_tokens: tokens.len(),
+ });
+ }
+
+ let output_len = checked_product(&[batch_size, num_groups * codebook_dim, seq_len])?;
+ let mut output = vec![0.0_f32; output_len];
+
+ for batch in 0..batch_size {
+ let length = lengths[batch] as usize;
+ for group in 0..num_groups {
+ for time in 0..seq_len {
+ if time >= length {
+ continue;
+ }
+
+ let token_index = (batch * num_groups + group) * seq_len + time;
+ let token = tokens[token_index];
+ let mut base = 1_i32;
+
+ for dim in 0..codebook_dim {
+ let levels = num_levels[dim];
+ let scale = levels / 2;
+ let offset = scale;
+
+ let div = token / base;
+ let mut code_nonnegative = div % levels;
+ if code_nonnegative < 0 {
+ code_nonnegative += levels;
+ }
+
+ let value = (code_nonnegative - offset) as f32 / scale as f32;
+ let channel = group * codebook_dim + dim;
+ let output_index = (batch * (num_groups * codebook_dim) + channel) * seq_len + time;
+ output[output_index] = value;
+
+ base =
+ base.checked_mul(levels).ok_or(AudioError::Runtime("fsq decode base overflow".to_string()))?;
+ }
+ }
+ }
+ }
+
+ Ok(output)
+}
+
+pub fn fsq_encode_reference(
+ input: &[f32],
+ lengths: &[i32],
+ batch_size: usize,
+ num_groups: usize,
+ seq_len: usize,
+ codebook_dim: usize,
+ num_levels: &[i32],
+ dim_base_index: &[i32],
+ eps: f32,
+) -> AudioResult<Vec<i32>> {
+ validate_fsq_common(batch_size, num_groups, seq_len, codebook_dim, num_levels, lengths)?;
+
+ if dim_base_index.len() != codebook_dim {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: codebook_dim,
+ actual_tokens: dim_base_index.len(),
+ });
+ }
+
+ let expected_input = checked_product(&[batch_size, num_groups * codebook_dim, seq_len])?;
+ if input.len() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.len(),
+ });
+ }
+
+ let output_len = checked_product(&[batch_size, num_groups, seq_len])?;
+ let mut tokens = vec![0_i32; output_len];
+
+ for batch in 0..batch_size {
+ let length = lengths[batch] as usize;
+ for group in 0..num_groups {
+ for time in 0..seq_len {
+ let output_index = (batch * num_groups + group) * seq_len + time;
+ if time >= length {
+ tokens[output_index] = 0;
+ continue;
+ }
+
+ let mut token = 0_i32;
+ for dim in 0..codebook_dim {
+ let levels = num_levels[dim];
+ let scale_i = levels / 2;
+
+ let output_scale = (levels - 1) as f32 * 0.5 * (1.0 - eps);
+ let output_offset = if levels % 2 == 0 {
+ 0.5
+ } else {
+ 0.0
+ };
+ let input_shift = (output_offset / output_scale).tan();
+
+ let input_index =
+ (batch * (num_groups * codebook_dim) + group * codebook_dim + dim) * seq_len + time;
+ let compressed = output_scale * (input[input_index] + input_shift).tanh() - output_offset;
+ let rounded = round_ties_to_even(compressed);
+ let code_nonnegative = (rounded as i32 + scale_i).clamp(0, levels - 1);
+ token += code_nonnegative * dim_base_index[dim];
+ }
+
+ tokens[output_index] = token;
+ }
+ }
+ }
+
+ Ok(tokens)
+}
diff --git a/crates/uzu/tests/common/audio_nanocodec_ops_reference.rs b/crates/uzu/tests/common/audio_nanocodec_ops_reference.rs
new file mode 100644
index 000000000..6b642dc47
--- /dev/null
+++ b/crates/uzu/tests/common/audio_nanocodec_ops_reference.rs
@@ -0,0 +1,512 @@
+use uzu::prelude::{AudioError, AudioResult};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PadMode {
+ Zeros,
+ Replicate,
+}
+
+fn checked_product(values: &[usize]) -> AudioResult<usize> {
+ values
+ .iter()
+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))
+ .ok_or(AudioError::Runtime("dimension product overflow".to_string()))
+}
+
+fn validate_lengths(
+ lengths: &[i32],
+ batch_size: usize,
+ frames: usize,
+) -> AudioResult<()> {
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+
+ for &length in lengths {
+ if length < 0 || length as usize > frames {
+ return Err(AudioError::InvalidTokenLengthValue {
+ length: length.max(0) as usize,
+ frames,
+ });
+ }
+ }
+
+ Ok(())
+}
+
+pub struct Conv1dSpec<'a> {
+ pub input: &'a [f32],
+ pub weight: &'a [f32],
+ pub bias: &'a [f32],
+ pub lengths: &'a [i32],
+ pub batch_size: usize,
+ pub cin: usize,
+ pub cout: usize,
+ pub seq_len_in: usize,
+ pub seq_len_out: usize,
+ pub kernel_size: usize,
+ pub stride: usize,
+ pub dilation: usize,
+ pub padding: usize,
+ pub pad_mode: PadMode,
+}
+
+pub fn conv1d_reference(spec: Conv1dSpec<'_>) -> AudioResult<Vec<f32>> {
+ let Conv1dSpec {
+ input,
+ weight,
+ bias,
+ lengths,
+ batch_size,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ kernel_size,
+ stride,
+ dilation,
+ padding,
+ pad_mode,
+ } = spec;
+
+ validate_lengths(lengths, batch_size, seq_len_out)?;
+
+ let expected_input = checked_product(&[batch_size, cin, seq_len_in])?;
+ if input.len() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.len(),
+ });
+ }
+
+ let expected_weight = checked_product(&[cout, cin, kernel_size])?;
+ if weight.len() != expected_weight {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_weight,
+ actual_tokens: weight.len(),
+ });
+ }
+
+ if bias.len() != cout {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: cout,
+ actual_tokens: bias.len(),
+ });
+ }
+
+ let output_len = checked_product(&[batch_size, cout, seq_len_out])?;
+ let mut output = vec![0.0_f32; output_len];
+
+ for batch in 0..batch_size {
+ let length = lengths[batch] as usize;
+ for out_channel in 0..cout {
+ for out_time in 0..seq_len_out {
+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;
+ if out_time >= length {
+ output[out_index] = 0.0;
+ continue;
+ }
+
+ let mut acc = bias[out_channel];
+ let base = (out_time * stride) as isize - padding as isize;
+
+ for in_channel in 0..cin {
+ let input_base = (batch * cin + in_channel) * seq_len_in;
+ let weight_base = (out_channel * cin + in_channel) * kernel_size;
+ for kernel_offset in 0..kernel_size {
+ let x_time_raw = base + (kernel_offset * dilation) as isize;
+ let x_time = match pad_mode {
+ PadMode::Zeros => {
+ if x_time_raw < 0 || x_time_raw >= seq_len_in as isize {
+ continue;
+ }
+ x_time_raw as usize
+ },
+ PadMode::Replicate => x_time_raw.clamp(0, seq_len_in as isize - 1) as usize,
+ };
+
+ let x_index = input_base + x_time;
+ let w_index = weight_base + kernel_offset;
+ acc += weight[w_index] * input[x_index];
+ }
+ }
+
+ output[out_index] = acc;
+ }
+ }
+ }
+
+ Ok(output)
+}
+
+pub struct CausalConv1dSpec<'a> {
+ pub input: &'a [f32],
+ pub weight: &'a [f32],
+ pub bias: &'a [f32],
+ pub lengths: &'a [i32],
+ pub batch_size: usize,
+ pub cin: usize,
+ pub cout: usize,
+ pub seq_len: usize,
+ pub kernel_size: usize,
+ pub dilation: usize,
+}
+
+pub fn causal_conv1d_reference(spec: CausalConv1dSpec<'_>) -> AudioResult<Vec<f32>> {
+ let CausalConv1dSpec {
+ input,
+ weight,
+ bias,
+ lengths,
+ batch_size,
+ cin,
+ cout,
+ seq_len,
+ kernel_size,
+ dilation,
+ } = spec;
+
+ validate_lengths(lengths, batch_size, seq_len)?;
+
+ let expected_input = checked_product(&[batch_size, cin, seq_len])?;
+ if input.len() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.len(),
+ });
+ }
+
+ let expected_weight = checked_product(&[cout, cin, kernel_size])?;
+ if weight.len() != expected_weight {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_weight,
+ actual_tokens: weight.len(),
+ });
+ }
+
+ if bias.len() != cout {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: cout,
+ actual_tokens: bias.len(),
+ });
+ }
+
+ let output_len = checked_product(&[batch_size, cout, seq_len])?;
+ let mut output = vec![0.0_f32; output_len];
+ let pad = (kernel_size - 1) * dilation;
+
+ for batch in 0..batch_size {
+ let length = lengths[batch] as usize;
+ for out_channel in 0..cout {
+ for time in 0..seq_len {
+ let out_index = (batch * cout + out_channel) * seq_len + time;
+ if time >= length {
+ output[out_index] = 0.0;
+ continue;
+ }
+
+ let mut acc = bias[out_channel];
+ for in_channel in 0..cin {
+ let input_base = (batch * cin + in_channel) * seq_len;
+ let weight_base = (out_channel * cin + in_channel) * kernel_size;
+ for kernel_offset in 0..kernel_size {
+ let x_time = time as isize + (kernel_offset * dilation) as isize - pad as isize;
+ if x_time < 0 || x_time >= seq_len as isize {
+ continue;
+ }
+
+ let x_index = input_base + x_time as usize;
+ let w_index = weight_base + kernel_offset;
+ acc += weight[w_index] * input[x_index];
+ }
+ }
+
+ output[out_index] = acc;
+ }
+ }
+ }
+
+ Ok(output)
+}
+
+pub struct CausalConvTranspose1dSpec<'a> {
+ pub input: &'a [f32],
+ pub weight: &'a [f32],
+ pub bias: &'a [f32],
+ pub lengths: &'a [i32],
+ pub batch_size: usize,
+ pub cin: usize,
+ pub cout: usize,
+ pub seq_len_in: usize,
+ pub seq_len_out: usize,
+ pub stride: usize,
+ pub groups: usize,
+}
+
+pub fn causal_conv_transpose1d_reference(spec: CausalConvTranspose1dSpec<'_>) -> AudioResult<Vec<f32>> {
+ let CausalConvTranspose1dSpec {
+ input,
+ weight,
+ bias,
+ lengths,
+ batch_size,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ stride,
+ groups,
+ } = spec;
+
+ validate_lengths(lengths, batch_size, seq_len_out)?;
+ if groups == 0 || cin == 0 || cout == 0 || cin % groups != 0 || cout % groups != 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+
+ let expected_input = checked_product(&[batch_size, cin, seq_len_in])?;
+ if input.len() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.len(),
+ });
+ }
+
+ let kernel_size = 2 * stride;
+ let cin_per_group = cin / groups;
+ let expected_weight = checked_product(&[cout, cin_per_group, kernel_size])?;
+ if weight.len() != expected_weight {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_weight,
+ actual_tokens: weight.len(),
+ });
+ }
+
+ if bias.len() != cout {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: cout,
+ actual_tokens: bias.len(),
+ });
+ }
+
+ let output_len = checked_product(&[batch_size, cout, seq_len_out])?;
+ let mut output = vec![0.0_f32; output_len];
+ let cout_per_group = cout / groups;
+
+ for batch in 0..batch_size {
+ let length = lengths[batch] as usize;
+ for out_channel in 0..cout {
+ let group_index = out_channel / cout_per_group;
+ let in_channel_begin = group_index * cin_per_group;
+ let in_channel_end = in_channel_begin + cin_per_group;
+
+ for out_time in 0..seq_len_out {
+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;
+ if out_time >= length {
+ output[out_index] = 0.0;
+ continue;
+ }
+
+ let q = out_time / stride;
+ let r = out_time % stride;
+ let mut acc = bias[out_channel];
+
+ for in_channel in in_channel_begin..in_channel_end {
+ if q >= seq_len_in {
+ continue;
+ }
+
+ let input_base = (batch * cin + in_channel) * seq_len_in;
+ let in_channel_in_group = in_channel - in_channel_begin;
+ let weight_base = (out_channel * cin_per_group + in_channel_in_group) * kernel_size;
+
+ acc += input[input_base + q] * weight[weight_base + r];
+ if q > 0 {
+ acc += input[input_base + (q - 1)] * weight[weight_base + (stride + r)];
+ }
+ }
+
+ output[out_index] = acc;
+ }
+ }
+ }
+
+ Ok(output)
+}
+
+pub fn causal_conv_transpose1d_causal_pad_reference(spec: CausalConvTranspose1dSpec<'_>) -> AudioResult<Vec<f32>> {
+ let CausalConvTranspose1dSpec {
+ input,
+ weight,
+ bias,
+ lengths,
+ batch_size,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ stride,
+ groups,
+ } = spec;
+
+ validate_lengths(lengths, batch_size, seq_len_out)?;
+ if groups == 0 || stride == 0 || cin == 0 || cout == 0 || cin % groups != 0 || cout % groups != 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+
+ let expected_input = checked_product(&[batch_size, cin, seq_len_in])?;
+ if input.len() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.len(),
+ });
+ }
+
+ let cout_per_group = cout / groups;
+ let cin_per_group = cin / groups;
+ let weight_plane = checked_product(&[cout, cin_per_group])?;
+ if weight_plane == 0 || weight.len() % weight_plane != 0 {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: weight_plane,
+ actual_tokens: weight.len(),
+ });
+ }
+
+ let kernel_size = weight.len() / weight_plane;
+ if kernel_size == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+
+ if bias.len() != cout {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: cout,
+ actual_tokens: bias.len(),
+ });
+ }
+
+ let output_len = checked_product(&[batch_size, cout, seq_len_out])?;
+ let mut output = vec![0.0_f32; output_len];
+ let seq_len_expanded = if seq_len_in == 0 {
+ 0
+ } else {
+ (seq_len_in - 1)
+ .checked_mul(stride)
+ .and_then(|value| value.checked_add(1))
+ .ok_or(AudioError::Runtime("expanded sequence length overflow".to_string()))?
+ };
+ let left_pad = kernel_size - 1;
+
+ for batch in 0..batch_size {
+ let length = lengths[batch] as usize;
+ for out_channel in 0..cout {
+ let group_index = out_channel / cout_per_group;
+ let in_channel_begin = group_index * cin_per_group;
+ let in_channel_end = in_channel_begin + cin_per_group;
+
+ for out_time in 0..seq_len_out {
+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;
+ if out_time >= length {
+ output[out_index] = 0.0;
+ continue;
+ }
+
+ let mut acc = bias[out_channel];
+
+ for in_channel in in_channel_begin..in_channel_end {
+ let input_base = (batch * cin + in_channel) * seq_len_in;
+ let in_channel_in_group = in_channel - in_channel_begin;
+ let weight_base = (out_channel * cin_per_group + in_channel_in_group) * kernel_size;
+
+ for kernel_offset in 0..kernel_size {
+ let expanded_time = out_time as isize + kernel_offset as isize - left_pad as isize;
+ if expanded_time < 0 || expanded_time >= seq_len_expanded as isize {
+ continue;
+ }
+
+ let expanded_time = expanded_time as usize;
+ if expanded_time % stride != 0 {
+ continue;
+ }
+
+ let src_time = expanded_time / stride;
+ if src_time >= seq_len_in {
+ continue;
+ }
+
+ acc += input[input_base + src_time] * weight[weight_base + kernel_offset];
+ }
+ }
+
+ output[out_index] = acc;
+ }
+ }
+ }
+
+ Ok(output)
+}
+
+pub struct HalfSnakeSpec<'a> {
+ pub input: &'a [f32],
+ pub alpha: &'a [f32],
+ pub batch_size: usize,
+ pub channels: usize,
+ pub seq_len: usize,
+ pub snake_channels: usize,
+ pub negative_slope: f32,
+ pub eps: f32,
+}
+
+pub fn half_snake_reference(spec: HalfSnakeSpec<'_>) -> AudioResult<Vec<f32>> {
+ let HalfSnakeSpec {
+ input,
+ alpha,
+ batch_size,
+ channels,
+ seq_len,
+ snake_channels,
+ negative_slope,
+ eps,
+ } = spec;
+
+ if snake_channels > channels {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+
+ let expected_input = checked_product(&[batch_size, channels, seq_len])?;
+ if input.len() != expected_input {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: expected_input,
+ actual_tokens: input.len(),
+ });
+ }
+
+ if alpha.len() != snake_channels {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens: snake_channels,
+ actual_tokens: alpha.len(),
+ });
+ }
+
+ let mut output = vec![0.0_f32; input.len()];
+ for batch in 0..batch_size {
+ for channel in 0..channels {
+ for time in 0..seq_len {
+ let index = (batch * channels + channel) * seq_len + time;
+ let x = input[index];
+ output[index] = if channel < snake_channels {
+ let a = alpha[channel];
+ let sine = (a * x).sin();
+ x + (sine * sine) / (a + eps)
+ } else if x >= 0.0 {
+ x
+ } else {
+ negative_slope * x
+ };
+ }
+ }
+ }
+
+ Ok(output)
+}
diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rs
index 6ed5da08a..ac66869ed 100644
--- a/crates/uzu/tests/common/mod.rs
+++ b/crates/uzu/tests/common/mod.rs
@@ -1,6 +1,8 @@
#![allow(dead_code)]
pub mod assert;
+pub mod audio_nanocodec_fsq_reference;
+pub mod audio_nanocodec_ops_reference;
pub mod proptest;
/// Invokes `$body` once per available backend, with `$B` bound to each backend type.
diff --git a/crates/uzu/tests/integration/parameters/loader_test.rs b/crates/uzu/tests/integration/parameters/loader_test.rs
index 463beae4b..a717f1a3b 100644
--- a/crates/uzu/tests/integration/parameters/loader_test.rs
+++ b/crates/uzu/tests/integration/parameters/loader_test.rs
@@ -22,7 +22,7 @@ fn test_parameter_loader_basic() {
let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");
let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");
- let embeddings = loader.get("embedding.weights").expect("weights embeddings");
+ let embeddings = loader.tree().leaf_array("embedding.weights").expect("weights embeddings");
let view = embeddings.as_view::<bf16>();
assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));
diff --git a/crates/uzu/tests/unit/audio/types_test.rs b/crates/uzu/tests/unit/audio/types_test.rs
index bc66cb593..89814d78e 100644
--- a/crates/uzu/tests/unit/audio/types_test.rs
+++ b/crates/uzu/tests/unit/audio/types_test.rs
@@ -1,7 +1,7 @@
-use super::{AudioError, AudioTokenGrid, AudioTokenPacking};
+use super::{AudioError, AudioTokenGrid};
#[test]
-fn packing_conversion_roundtrip_is_lossless() {
+fn token_grid_get_reads_expected_slots() {
let grid = AudioTokenGrid::new(
vec![
0, 1, 2, 3, // b0 f0, f1
@@ -12,27 +12,19 @@ fn packing_conversion_roundtrip_is_lossless() {
2,
2,
vec![2, 2].into_boxed_slice(),
- AudioTokenPacking::FrameMajor,
)
.expect("valid grid");
- let converted = grid.to_packing(AudioTokenPacking::CodebookMajor);
- let restored = converted.to_packing(AudioTokenPacking::FrameMajor);
-
- assert_eq!(restored, grid);
+ assert_eq!(grid.get(0, 0, 0), 0);
+ assert_eq!(grid.get(0, 1, 1), 3);
+ assert_eq!(grid.get(1, 0, 0), 4);
+ assert_eq!(grid.get(1, 1, 1), 7);
}
#[test]
fn invalid_grid_shape_is_rejected() {
- let error = AudioTokenGrid::new(
- vec![0, 1, 2].into_boxed_slice(),
- 1,
- 2,
- 2,
- vec![2].into_boxed_slice(),
- AudioTokenPacking::FrameMajor,
- )
- .expect_err("shape mismatch should fail");
+ let error = AudioTokenGrid::new(vec![0, 1, 2].into_boxed_slice(), 1, 2, 2, vec![2].into_boxed_slice())
+ .expect_err("shape mismatch should fail");
assert!(matches!(
error,
diff --git a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
index a9955faed..34f9dcc9e 100644
--- a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
+++ b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs
@@ -343,7 +343,7 @@ fn run_conv_scan_once(
if let Some(ref scratch) = scratch_buf {
let bytes = channels * tap_count * size_of::<f32>();
if bytes > 0 {
- encoder.encode_copy(scratch, &mut state_buf, bytes);
+ encoder.encode_copy(scratch, 0..bytes, &mut state_buf, 0..bytes);
}
}
diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rs
index 9f070d94e..d60d905be 100644
--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs
@@ -1198,15 +1198,15 @@ fn perf_two_pass_attention() {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time() {
- Some(gpu_time) => {
+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
+ Some(gpu_time_ms) => {
println!(
"Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",
num_heads,
seq_len - suffix_length,
suffix_length,
head_dim,
- gpu_time.as_secs_f64() * 1e3,
+ gpu_time_ms,
host_elapsed_ms
);
},
diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
index 329d2fbb3..952f4dfdc 100644
--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs
+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs
@@ -17,7 +17,7 @@ use crate::{
},
metal::Metal,
},
- session::parameter::SamplingMethod,
+ session::parameter::{SamplingMethod, SamplingProcessingOrder},
};
// Constant seed for reproducible test results
@@ -187,15 +187,11 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time() {
- Some(gpu_time) => {
+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
+ Some(gpu_time_ms) => {
println!(
"Argmax sampling perf (batch={}, vocab={}, strategy={:?}): GPU={:.2} ms, Host-side={:.2} ms",
- BATCH,
- VOCAB,
- strategy,
- gpu_time.as_secs_f64() * 1e3,
- host_elapsed_ms
+ BATCH, VOCAB, strategy, gpu_time_ms, host_elapsed_ms
);
},
None => {
@@ -296,6 +292,7 @@ fn test_categorical_sampling() {
top_k: None,
top_p: None,
min_p: None,
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
batch_size,
vocab_size,
@@ -432,6 +429,7 @@ fn test_categorical_sampling_statistical() {
top_k: None,
top_p: None,
min_p: None,
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
BATCH,
VOCAB,
@@ -533,6 +531,7 @@ fn perf_categorical_128k_vocab() {
top_k: None,
top_p: None,
min_p: None,
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
BATCH,
VOCAB,
@@ -544,14 +543,11 @@ fn perf_categorical_128k_vocab() {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time() {
- Some(gpu_time) => {
+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
+ Some(gpu_time_ms) => {
println!(
"Categorical sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",
- BATCH,
- VOCAB,
- gpu_time.as_secs_f64() * 1e3,
- host_elapsed_ms
+ BATCH, VOCAB, gpu_time_ms, host_elapsed_ms
);
},
None => {
@@ -760,6 +756,7 @@ fn test_minp_sampling_exact_match(
top_k: None,
top_p: None,
min_p: Some(min_p),
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
batch_size,
vocab_size,
diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
index bda04a3e9..b52980d71 100644
--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs
@@ -1,8 +1,7 @@
#![cfg(metal_backend)]
use crate::{
- DataType,
- array::ArrayContextExt,
+ ArrayContextExt, DataType,
backends::{
common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},
metal::Metal,
diff --git a/crates/uzu/tests/unit/kernel/audio/audio_test.rs b/crates/uzu/tests/unit/kernel/audio/audio_test.rs
index 48aff9f65..fc876cf92 100644
--- a/crates/uzu/tests/unit/kernel/audio/audio_test.rs
+++ b/crates/uzu/tests/unit/kernel/audio/audio_test.rs
@@ -1,29 +1,57 @@
#![cfg(all(feature = "audio-runtime", metal_backend))]
-use std::{
- ops::{Deref, DerefMut},
- rc::Rc,
-};
-
use uzu::{
ArrayContextExt, DataType,
backends::{
common::{
Backend, Context, Encoder, Kernels,
+ gpu_types::ActivationType,
kernel::{
- AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,
- AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,
- AudioLeakyReluKernel, AudioScaleKernel, AudioTanhKernel,
+ ActivationKernel, AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dCausalPadKernel,
+ AudioCausalConvTranspose1dKernel, AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel,
+ AudioHalfSnakeKernel, AudioNormNcsKernel,
},
},
metal::Metal,
},
};
-fn create_test_context() -> Rc<<Metal as Backend>::Context> {
+use super::super::common::{
+ audio_nanocodec_fsq_reference::{fsq_decode_reference, fsq_encode_reference},
+ audio_nanocodec_ops_reference::{
+ CausalConv1dSpec, CausalConvTranspose1dSpec, Conv1dSpec, HalfSnakeSpec, PadMode,
+ causal_conv_transpose1d_causal_pad_reference, causal_conv_transpose1d_reference, causal_conv1d_reference,
+ conv1d_reference, half_snake_reference,
+ },
+};
+
+macro_rules! borrow_array_buffer {
+ ($name:ident = $array:expr) => {
+ let $name = $array.buffer();
+ let $name = $name.borrow();
+ };
+}
+
+macro_rules! borrow_array_buffer_mut {
+ ($name:ident = $array:expr) => {
+ let $name = $array.buffer();
+ let mut $name = $name.borrow_mut();
+ };
+}
+
+fn create_test_context() -> std::rc::Rc<<Metal as Backend>::Context> {
<Metal as Backend>::Context::new().expect("MetalContext")
}
+fn run_command_buffer(
+ context: &std::rc::Rc<<Metal as Backend>::Context>,
+ encode: impl FnOnce(&mut Encoder<'_, Metal>),
+) {
+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).expect("command buffer");
+ encode(&mut encoder);
+ encoder.end_encoding().submit().wait_until_completed().expect("command buffer completed");
+}
+
#[test]
fn audio_conv1d_replicate_matches_reference_f32() {
let context = create_test_context();
@@ -65,62 +93,49 @@ fn audio_conv1d_replicate_matches_reference_f32() {
lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
let output = context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");
-
- let mut expected = vec![0.0f32; output_len];
- for batch in 0..batch_size {
- for out_channel in 0..cout {
- for out_time in 0..seq_len_out {
- let out_index = (batch * cout + out_channel) * seq_len_out + out_time;
- if (out_time as i32) >= lengths_out[batch] {
- expected[out_index] = 0.0;
- continue;
- }
-
- let mut acc = bias_values[out_channel];
- let base = (out_time * stride) as isize - padding as isize;
-
- for in_channel in 0..cin {
- let x_base = (batch * cin + in_channel) * seq_len_in;
- let w_base = (out_channel * cin + in_channel) * kernel_size;
- for kernel_offset in 0..kernel_size {
- let mut x_time = base + (kernel_offset * dilation) as isize;
- if x_time < 0 {
- x_time = 0;
- } else if x_time >= seq_len_in as isize {
- x_time = seq_len_in as isize - 1;
- }
-
- let x_index = x_base + x_time as usize;
- let w_index = w_base + kernel_offset;
- acc += weight_values[w_index] * input_values[x_index];
- }
- }
-
- expected[out_index] = acc;
- }
- }
- }
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input.buffer().borrow().deref(),
- weight.buffer().borrow().deref(),
- bias.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- lengths.buffer().borrow().deref(),
- cin as i32,
- cout as i32,
- seq_len_in as i32,
- seq_len_out as i32,
- kernel_size as i32,
- stride as i32,
- dilation as i32,
- padding as i32,
- 1_i32,
- batch_size as i32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ let expected = conv1d_reference(Conv1dSpec {
+ input: &input_values,
+ weight: &weight_values,
+ bias: &bias_values,
+ lengths: &lengths_out,
+ batch_size,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ kernel_size,
+ stride,
+ dilation,
+ padding,
+ pad_mode: PadMode::Replicate,
+ })
+ .expect("reference conv1d");
+
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer!(weight_buffer = weight);
+ borrow_array_buffer!(bias_buffer = bias);
+ borrow_array_buffer_mut!(output_buffer = output);
+ borrow_array_buffer!(lengths_buffer = lengths);
+ kernel.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin as i32,
+ cout as i32,
+ seq_len_in as i32,
+ seq_len_out as i32,
+ kernel_size as i32,
+ stride as i32,
+ dilation as i32,
+ padding as i32,
+ 1_i32,
+ batch_size as i32,
+ command_buffer,
+ );
+ });
let got = output.as_slice::<f32>();
for index in 0..output_len {
@@ -146,7 +161,6 @@ fn audio_causal_conv1d_matches_reference_f32() {
let seq_len = 8usize;
let kernel_size = 5usize;
let dilation = 2usize;
- let pad = (kernel_size - 1) * dilation;
let lengths: [i32; 2] = [5, 8];
@@ -172,52 +186,42 @@ fn audio_causal_conv1d_matches_reference_f32() {
lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);
let output = context.create_array(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");
-
- let mut expected = vec![0.0f32; output_len];
- for batch in 0..batch_size {
- for out_channel in 0..cout {
- for time in 0..seq_len {
- let out_index = (batch * cout + out_channel) * seq_len + time;
- if (time as i32) >= lengths[batch] {
- expected[out_index] = 0.0;
- continue;
- }
-
- let mut acc = bias_values[out_channel];
- for in_channel in 0..cin {
- for kernel_offset in 0..kernel_size {
- let x_time = time as isize + (kernel_offset * dilation) as isize - pad as isize;
- if x_time < 0 || x_time >= seq_len as isize {
- continue;
- }
-
- let x_index = (batch * cin + in_channel) * seq_len + x_time as usize;
- let w_index = (out_channel * cin + in_channel) * kernel_size + kernel_offset;
- acc += weight_values[w_index] * input_values[x_index];
- }
- }
-
- expected[out_index] = acc;
- }
- }
- }
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input.buffer().borrow().deref(),
- weight.buffer().borrow().deref(),
- bias.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- lengths_array.buffer().borrow().deref(),
- cin as i32,
- cout as i32,
- seq_len as i32,
- kernel_size as i32,
- dilation as i32,
- batch_size as i32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ let expected = causal_conv1d_reference(CausalConv1dSpec {
+ input: &input_values,
+ weight: &weight_values,
+ bias: &bias_values,
+ lengths: &lengths,
+ batch_size,
+ cin,
+ cout,
+ seq_len,
+ kernel_size,
+ dilation,
+ })
+ .expect("reference causal conv1d");
+
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer!(weight_buffer = weight);
+ borrow_array_buffer!(bias_buffer = bias);
+ borrow_array_buffer_mut!(output_buffer = output);
+ borrow_array_buffer!(lengths_buffer = lengths_array);
+ kernel.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin as i32,
+ cout as i32,
+ seq_len as i32,
+ kernel_size as i32,
+ dilation as i32,
+ 0,
+ batch_size as i32,
+ command_buffer,
+ );
+ });
let got = output.as_slice::<f32>();
for index in 0..output_len {
@@ -251,7 +255,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
let lengths_out: [i32; 2] = [9, 15];
let input_len = batch_size * cin * seq_len_in;
- let weight_len = cin * (cout / groups) * kernel_size;
+ let weight_len = cout * (cin / groups) * kernel_size;
let output_len = batch_size * cout * seq_len_out;
let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.013).sin() * 0.5).collect();
@@ -263,7 +267,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
let mut weight =
- context.create_array(&[cin, cout / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");
+ context.create_array(&[cout, cin / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");
weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");
@@ -274,98 +278,208 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {
let output =
context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_causal_conv_transpose_output");
+ let expected = causal_conv_transpose1d_reference(CausalConvTranspose1dSpec {
+ input: &input_values,
+ weight: &weight_values,
+ bias: &bias_values,
+ lengths: &lengths_out,
+ batch_size,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ stride,
+ groups,
+ })
+ .expect("reference causal conv transpose1d");
+
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer!(weight_buffer = weight);
+ borrow_array_buffer!(bias_buffer = bias);
+ borrow_array_buffer_mut!(output_buffer = output);
+ borrow_array_buffer!(lengths_buffer = lengths);
+ kernel.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin as i32,
+ cout as i32,
+ seq_len_in as i32,
+ seq_len_out as i32,
+ stride as i32,
+ groups as i32,
+ batch_size as i32,
+ command_buffer,
+ );
+ });
- let cout_per_group = cout / groups;
- let cin_per_group = cin / groups;
- let mut expected = vec![0.0f32; output_len];
-
- for batch in 0..batch_size {
- for out_channel in 0..cout {
- let group_index = out_channel / cout_per_group;
- let out_channel_in_group = out_channel % cout_per_group;
- let in_channel_begin = group_index * cin_per_group;
- let in_channel_end = in_channel_begin + cin_per_group;
-
- for out_time in 0..seq_len_out {
- let out_index = (batch * cout + out_channel) * seq_len_out + out_time;
- if (out_time as i32) >= lengths_out[batch] {
- expected[out_index] = 0.0;
- continue;
- }
+ let got = output.as_slice::<f32>();
+ for index in 0..output_len {
+ let delta = (expected[index] - got[index]).abs();
+ assert!(
+ delta <= 1e-4,
+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",
+ expected[index],
+ got[index]
+ );
+ }
+}
- let q = out_time / stride;
- let r = out_time % stride;
- let mut acc = bias_values[out_channel];
+#[test]
+fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(
+ &context,
+ DataType::F32,
+ )
+ .expect("audio runtime");
- for in_channel in in_channel_begin..in_channel_end {
- let input_base = (batch * cin + in_channel) * seq_len_in;
- let weight_base = (in_channel * cout_per_group + out_channel_in_group) * kernel_size;
+ let batch_size = 1usize;
+ let cin = 2usize;
+ let groups = 1usize;
+ let cout = 2usize;
+ let stride = 2usize;
+ let kernel_size = 3usize;
- acc += input_values[input_base + q] * weight_values[weight_base + r];
- if q > 0 {
- acc += input_values[input_base + (q - 1)] * weight_values[weight_base + (stride + r)];
- }
- }
+ let seq_len_in = 5usize;
+ let seq_len_out = 10usize;
+ let lengths_out: [i32; 1] = [10];
+
+ let input_len = batch_size * cin * seq_len_in;
+ let weight_len = cout * (cin / groups) * kernel_size;
+ let output_len = batch_size * cout * seq_len_out;
- expected[out_index] = acc;
+ let input_values_ncs: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.017).sin() * 0.5).collect();
+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.09).cos() * 0.2).collect();
+ let bias_values: Vec<f32> = vec![0.01, -0.02];
+ let expected = causal_conv_transpose1d_causal_pad_reference(CausalConvTranspose1dSpec {
+ input: &input_values_ncs,
+ weight: &weight_values,
+ bias: &bias_values,
+ lengths: &lengths_out,
+ batch_size,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ stride,
+ groups,
+ })
+ .expect("reference causal conv transpose lalamo");
+
+ let mut input_values_nsc = vec![0.0_f32; input_len];
+ for b in 0..batch_size {
+ for c in 0..cin {
+ for t in 0..seq_len_in {
+ let src_index = (b * cin + c) * seq_len_in + t;
+ let dst_index = (b * seq_len_in + t) * cin + c;
+ input_values_nsc[dst_index] = input_values_ncs[src_index];
}
}
}
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input.buffer().borrow().deref(),
- weight.buffer().borrow().deref(),
- bias.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- lengths.buffer().borrow().deref(),
- cin as i32,
- cout as i32,
- seq_len_in as i32,
- seq_len_out as i32,
- stride as i32,
- groups as i32,
- batch_size as i32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ for (layout_name, layout_code, input_shape, input_values) in [
+ ("NCS", 0_i32, [batch_size, cin, seq_len_in], input_values_ncs.as_slice()),
+ ("NSC", 1_i32, [batch_size, seq_len_in, cin], input_values_nsc.as_slice()),
+ ] {
+ let mut input = context.create_array(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");
+ input.as_slice_mut::<f32>().copy_from_slice(input_values);
+
+ let mut weight = context.create_array(
+ &[cout, cin / groups, kernel_size],
+ DataType::F32,
+ "audio_causal_conv_transpose_lalamo_weight",
+ );
+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
- let got = output.as_slice::<f32>();
- for index in 0..output_len {
- let delta = (expected[index] - got[index]).abs();
- assert!(
- delta <= 1e-4,
- "Mismatch at index={index}: expected {}, got {}, delta={delta}",
- expected[index],
- got[index]
+ let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");
+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
+
+ let mut lengths =
+ context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");
+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
+
+ let output = context.create_array(
+ &[batch_size, cout, seq_len_out],
+ DataType::F32,
+ "audio_causal_conv_transpose_lalamo_output",
);
+
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer!(weight_buffer = weight);
+ borrow_array_buffer!(bias_buffer = bias);
+ borrow_array_buffer_mut!(output_buffer = output);
+ borrow_array_buffer!(lengths_buffer = lengths);
+ kernel.encode(
+ &*input_buffer,
+ &*weight_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ cin as i32,
+ cout as i32,
+ seq_len_in as i32,
+ seq_len_out as i32,
+ kernel_size as i32,
+ stride as i32,
+ groups as i32,
+ layout_code,
+ batch_size as i32,
+ command_buffer,
+ );
+ });
+
+ let got = output.as_slice::<f32>();
+ for index in 0..output_len {
+ let delta = (expected[index] - got[index]).abs();
+ assert!(
+ delta <= 1e-4,
+ "layout={layout_name}, index={index}: expected {}, got {}, delta={delta}",
+ expected[index],
+ got[index]
+ );
+ }
}
}
#[test]
fn audio_leaky_relu_matches_reference_f32() {
let context = create_test_context();
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioLeakyReluKernel::new(&context, DataType::F32)
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, DataType::F32)
.expect("audio runtime");
let n = 1024usize;
- let slope = 0.01f32;
+ let slope = 0.0f32;
+ let eps = 1e-6f32;
let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
- let mut input = context.create_array(&[n], DataType::F32, "audio_leaky_relu_input");
+ let mut input = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");
input.as_slice_mut::<f32>().copy_from_slice(&input_values);
- let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- n as i32,
- slope,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ let alpha = context.create_array(&[1], DataType::F32, "audio_leaky_relu_alpha");
+ let output = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");
+
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer!(alpha_buffer = alpha);
+ borrow_array_buffer_mut!(output_buffer = output);
+ kernel.encode(
+ &*input_buffer,
+ &*alpha_buffer,
+ &mut *output_buffer,
+ 1,
+ n as i32,
+ 0,
+ slope,
+ eps,
+ 1,
+ command_buffer,
+ );
+ });
let got = output.as_slice::<f32>();
for index in 0..n {
@@ -382,8 +496,8 @@ fn audio_leaky_relu_matches_reference_f32() {
#[test]
fn audio_tanh_matches_reference_f32() {
let context = create_test_context();
- let kernel =
- <<Metal as Backend>::Kernels as Kernels>::AudioTanhKernel::new(&context, DataType::F32).expect("audio runtime");
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, DataType::F32, false)
+ .expect("audio runtime");
let n = 1024usize;
let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
@@ -393,9 +507,11 @@ fn audio_tanh_matches_reference_f32() {
let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(input.buffer().borrow().deref(), output.buffer().borrow_mut().deref_mut(), n as i32, &mut encoder);
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer_mut!(output_buffer = output);
+ kernel.encode(Some(&*input_buffer), &mut *output_buffer, n as u32, ActivationType::TANH, command_buffer);
+ });
let got = output.as_slice::<f32>();
for index in 0..n {
@@ -406,15 +522,12 @@ fn audio_tanh_matches_reference_f32() {
}
#[test]
-fn audio_add_and_scale_match_reference_f32() {
+fn audio_add_matches_reference_f32() {
let context = create_test_context();
let add_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioAddKernel::new(&context, DataType::F32)
.expect("audio add kernel");
- let scale_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioScaleKernel::new(&context, DataType::F32)
- .expect("audio scale kernel");
let n = 2048usize;
- let scale_value = 1.0f32 / 3.0f32;
let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();
let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();
@@ -425,80 +538,25 @@ fn audio_add_and_scale_match_reference_f32() {
b.as_slice_mut::<f32>().copy_from_slice(&b_values);
let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");
- let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- add_kernel.encode(
- a.buffer().borrow().deref(),
- b.buffer().borrow().deref(),
- sum.buffer().borrow_mut().deref_mut(),
- n as i32,
- &mut encoder,
- );
- scale_kernel.encode(
- sum.buffer().borrow().deref(),
- scaled.buffer().borrow_mut().deref_mut(),
- n as i32,
- scale_value,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(a_buffer = a);
+ borrow_array_buffer!(b_buffer = b);
+ borrow_array_buffer_mut!(sum_buffer = sum);
+ add_kernel.encode(&*a_buffer, &*b_buffer, &mut *sum_buffer, n as i32, command_buffer);
+ });
let sum_got = sum.as_slice::<f32>();
- let scaled_got = scaled.as_slice::<f32>();
for index in 0..n {
let expected_sum = a_values[index] + b_values[index];
- let expected_scaled = expected_sum * scale_value;
-
let sum_delta = (expected_sum - sum_got[index]).abs();
- let scaled_delta = (expected_scaled - scaled_got[index]).abs();
assert!(
sum_delta <= 1e-6,
"Add mismatch at index={index}: expected {expected_sum}, got {}, delta={sum_delta}",
sum_got[index]
);
- assert!(
- scaled_delta <= 1e-6,
- "Scale mismatch at index={index}: expected {expected_scaled}, got {}, delta={scaled_delta}",
- scaled_got[index]
- );
- }
-}
-
-#[test]
-fn audio_clamp_matches_reference_f32() {
- let context = create_test_context();
- let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioClampKernel::new(&context, DataType::F32)
- .expect("audio runtime");
-
- let n = 2048usize;
- let min_value = -1.0f32;
- let max_value = 1.0f32;
- let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 10.24).collect();
-
- let mut input = context.create_array(&[n], DataType::F32, "audio_clamp_input");
- input.as_slice_mut::<f32>().copy_from_slice(&input_values);
-
- let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");
-
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- n as i32,
- min_value,
- max_value,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
-
- let got = output.as_slice::<f32>();
- for index in 0..n {
- let expected = input_values[index].clamp(min_value, max_value);
- let delta = (expected - got[index]).abs();
- assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);
}
}
@@ -528,45 +586,153 @@ fn audio_half_snake_matches_reference_f32() {
let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input.buffer().borrow().deref(),
- alpha.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- channels as i32,
- seq_len as i32,
- snake_channels as i32,
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer!(alpha_buffer = alpha);
+ borrow_array_buffer_mut!(output_buffer = output);
+ kernel.encode(
+ &*input_buffer,
+ &*alpha_buffer,
+ &mut *output_buffer,
+ channels as i32,
+ seq_len as i32,
+ snake_channels as i32,
+ negative_slope,
+ eps,
+ batch_size as i32,
+ command_buffer,
+ );
+ });
+
+ let got = output.as_slice::<f32>();
+ let expected = half_snake_reference(HalfSnakeSpec {
+ input: &input_values,
+ alpha: &alpha_values,
+ batch_size,
+ channels,
+ seq_len,
+ snake_channels,
negative_slope,
eps,
- batch_size as i32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ })
+ .expect("reference half snake");
+ for index in 0..n {
+ let delta = (expected[index] - got[index]).abs();
+ assert!(
+ delta <= 1e-5,
+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",
+ expected[index],
+ got[index]
+ );
+ }
+}
- let got = output.as_slice::<f32>();
- for batch in 0..batch_size {
- for channel in 0..channels {
- for time in 0..seq_len {
- let index = (batch * channels + channel) * seq_len + time;
- let x = input_values[index];
- let expected = if channel < snake_channels {
- let alpha = alpha_values[channel];
- let sine = (alpha * x).sin();
- x + (sine * sine) / (alpha + eps)
- } else if x >= 0.0 {
- x
- } else {
- negative_slope * x
- };
-
- let delta = (expected - got[index]).abs();
- assert!(
- delta <= 1e-5,
- "Mismatch at index={index}: expected {expected}, got {}, delta={delta}",
- got[index]
- );
+#[test]
+fn audio_norm_ncs_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioNormNcsKernel::new(&context, DataType::F32)
+ .expect("audio runtime");
+
+ let batch_size = 2usize;
+ let channels = 7usize;
+ let seq_len = 11usize;
+ let lengths_values: [i32; 2] = [8, 11];
+
+ let input_len = batch_size * channels * seq_len;
+ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.013 - 3.0).sin() * 0.7).collect();
+ let scales_values: Vec<f32> = (0..channels).map(|i| 0.7 + i as f32 * 0.05).collect();
+ let bias_values: Vec<f32> = (0..channels).map(|i| -0.2 + i as f32 * 0.03).collect();
+ let epsilon = 1e-5f32;
+
+ let reference = |subtract_mean: bool| -> Vec<f32> {
+ let mut out = vec![0.0_f32; input_values.len()];
+ for batch in 0..batch_size {
+ let active_len = lengths_values[batch].max(0) as usize;
+ for t in 0..active_len {
+ let mut mean = 0.0_f32;
+ if subtract_mean {
+ for c in 0..channels {
+ let idx = (batch * channels + c) * seq_len + t;
+ mean += input_values[idx];
+ }
+ mean /= channels as f32;
+ }
+ let mut variance_sum = 0.0_f32;
+ for c in 0..channels {
+ let idx = (batch * channels + c) * seq_len + t;
+ let centered = if subtract_mean {
+ input_values[idx] - mean
+ } else {
+ input_values[idx]
+ };
+ variance_sum += centered * centered;
+ }
+ let inv_std = 1.0_f32 / (variance_sum / channels as f32 + epsilon).sqrt();
+ for c in 0..channels {
+ let idx = (batch * channels + c) * seq_len + t;
+ let centered = if subtract_mean {
+ input_values[idx] - mean
+ } else {
+ input_values[idx]
+ };
+ out[idx] = centered * inv_std * scales_values[c] + bias_values[c];
+ }
}
}
+ out
+ };
+
+ for subtract_mean in [false, true] {
+ let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let mut scales = context.create_array(&[channels], DataType::F32, "audio_norm_scales");
+ scales.as_slice_mut::<f32>().copy_from_slice(&scales_values);
+
+ let mut bias = context.create_array(&[channels], DataType::F32, "audio_norm_bias");
+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
+
+ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_norm_lengths");
+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);
+
+ let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");
+
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer!(scales_buffer = scales);
+ borrow_array_buffer!(bias_buffer = bias);
+ borrow_array_buffer_mut!(output_buffer = output);
+ borrow_array_buffer!(lengths_buffer = lengths);
+ kernel.encode(
+ &*input_buffer,
+ &*scales_buffer,
+ &*bias_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ channels as i32,
+ seq_len as i32,
+ epsilon,
+ if subtract_mean {
+ 1
+ } else {
+ 0
+ },
+ batch_size as i32,
+ command_buffer,
+ );
+ });
+
+ let got = output.as_slice::<f32>();
+ let expected = reference(subtract_mean);
+ for index in 0..input_len {
+ let delta = (expected[index] - got[index]).abs();
+ assert!(
+ delta <= 1e-4,
+ "subtract_mean={subtract_mean}, index={index}: expected {}, got {}, delta={delta}",
+ expected[index],
+ got[index]
+ );
+ }
}
}
@@ -581,6 +747,7 @@ fn audio_fsq_decode_matches_reference() {
let seq_len = 3usize;
let codebook_dim = 2usize;
let num_levels: Vec<i32> = vec![8, 5];
+ let dim_base_index: Vec<i32> = vec![1, 8];
let mut tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");
tokens.as_slice_mut::<i32>().copy_from_slice(&[
@@ -594,22 +761,27 @@ fn audio_fsq_decode_matches_reference() {
let output =
context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- tokens.buffer().borrow().deref(),
- output.buffer().borrow_mut().deref_mut(),
- lengths.buffer().borrow().deref(),
- num_groups as i32,
- seq_len as i32,
- codebook_dim as i32,
- &num_levels,
- batch_size as i32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(tokens_buffer = tokens);
+ borrow_array_buffer_mut!(output_buffer = output);
+ borrow_array_buffer!(lengths_buffer = lengths);
+ kernel.encode(
+ &*tokens_buffer,
+ &mut *output_buffer,
+ &*lengths_buffer,
+ num_groups as i32,
+ seq_len as i32,
+ codebook_dim as i32,
+ &num_levels,
+ &dim_base_index,
+ batch_size as i32,
+ command_buffer,
+ );
+ });
let expected =
- reference_fsq_decode(&[0, 7, 11, 3, 5, 9], 2, batch_size, num_groups, seq_len, codebook_dim, &num_levels);
+ fsq_decode_reference(&[0, 7, 11, 3, 5, 9], &[2], batch_size, num_groups, seq_len, codebook_dim, &num_levels)
+ .expect("reference decode");
let actual = output.as_slice::<f32>();
for (index, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() {
@@ -652,23 +824,26 @@ fn audio_fsq_encode_matches_reference() {
let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");
- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");
- kernel.encode(
- input.buffer().borrow().deref(),
- tokens.buffer().borrow_mut().deref_mut(),
- lengths.buffer().borrow().deref(),
- num_groups as i32,
- seq_len as i32,
- codebook_dim as i32,
- &num_levels,
- &dim_base_index,
- eps,
- batch_size as i32,
- &mut encoder,
- );
- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");
+ run_command_buffer(&context, |command_buffer| {
+ borrow_array_buffer!(input_buffer = input);
+ borrow_array_buffer_mut!(tokens_buffer = tokens);
+ borrow_array_buffer!(lengths_buffer = lengths);
+ kernel.encode(
+ &*input_buffer,
+ &mut *tokens_buffer,
+ &*lengths_buffer,
+ num_groups as i32,
+ seq_len as i32,
+ codebook_dim as i32,
+ &num_levels,
+ &dim_base_index,
+ eps,
+ batch_size as i32,
+ command_buffer,
+ );
+ });
- let expected = reference_fsq_encode(
+ let expected = fsq_encode_reference(
&input_values,
&lengths_values,
batch_size,
@@ -678,118 +853,9 @@ fn audio_fsq_encode_matches_reference() {
&num_levels,
&dim_base_index,
eps,
- );
+ )
+ .expect("reference encode");
let actual = tokens.as_slice::<i32>();
assert_eq!(actual, expected);
}
-
-fn round_ties_to_even(value: f32) -> f32 {
- let floor = value.floor();
- let fraction = value - floor;
- if fraction < 0.5 {
- floor
- } else if fraction > 0.5 {
- floor + 1.0
- } else if (floor as i64 & 1) != 0 {
- floor + 1.0
- } else {
- floor
- }
-}
-
-fn reference_fsq_encode(
- input: &[f32],
- lengths: &[i32],
- batch_size: usize,
- num_groups: usize,
- seq_len: usize,
- codebook_dim: usize,
- num_levels: &[i32],
- dim_base_index: &[i32],
- eps: f32,
-) -> Vec<i32> {
- let mut tokens = vec![0_i32; batch_size * num_groups * seq_len];
-
- for batch in 0..batch_size {
- for group in 0..num_groups {
- for time in 0..seq_len {
- let output_index = (batch * num_groups + group) * seq_len + time;
- if (time as i32) >= lengths[batch] {
- tokens[output_index] = 0;
- continue;
- }
-
- let mut token = 0_i32;
- for dim in 0..codebook_dim {
- let levels = num_levels[dim];
- let scale_i = levels / 2;
- let output_scale = (levels - 1) as f32 * 0.5 * (1.0 - eps);
- let output_offset = if levels % 2 == 0 {
- 0.5
- } else {
- 0.0
- };
- let input_shift = (output_offset / output_scale).tan();
-
- let input_index =
- (batch * (num_groups * codebook_dim) + group * codebook_dim + dim) * seq_len + time;
- let compressed = output_scale * (input[input_index] + input_shift).tanh() - output_offset;
- let rounded = round_ties_to_even(compressed);
- let code_nonnegative = (rounded as i32 + scale_i).clamp(0, levels - 1);
- token += code_nonnegative * dim_base_index[dim];
- }
-
- tokens[output_index] = token;
- }
- }
- }
-
- tokens
-}
-
-fn reference_fsq_decode(
- tokens: &[i32],
- length: usize,
- batch_size: usize,
- num_groups: usize,
- seq_len: usize,
- codebook_dim: usize,
- num_levels: &[i32],
-) -> Vec<f32> {
- let mut out = vec![0.0f32; batch_size * num_groups * codebook_dim * seq_len];
-
- for b in 0..batch_size {
- for g in 0..num_groups {
- for t in 0..seq_len {
- if t >= length {
- continue;
- }
-
- let token_index = (b * num_groups + g) * seq_len + t;
- let token = tokens[token_index];
-
- let mut base = 1i32;
- for d in 0..codebook_dim {
- let levels = num_levels[d];
- let scale = levels / 2;
- let offset = scale;
- let div = token / base;
- let mut code_nonneg = div % levels;
- if code_nonneg < 0 {
- code_nonneg += levels;
- }
- let value = (code_nonneg - offset) as f32 / scale as f32;
-
- let out_channel = g * codebook_dim + d;
- let out_index = (b * (num_groups * codebook_dim) + out_channel) * seq_len + t;
- out[out_index] = value;
-
- base *= levels;
- }
- }
- }
- }
-
- out
-}
diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
index 0e6a47fd5..b35d4facb 100644
--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs
@@ -16,7 +16,7 @@ use uzu::{
},
cpu::Cpu,
},
- session::parameter::SamplingMethod,
+ session::parameter::{SamplingMethod, SamplingProcessingOrder},
};
const TEST_SAMPLING_SEED: u64 = 42;
@@ -293,6 +293,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(
top_k: None,
top_p: Some(p),
min_p: None,
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
batch_size,
vocab_size,
@@ -397,6 +398,7 @@ fn test_topp_sampling_statistical_large() {
top_k: None,
top_p: Some(TOP_P),
min_p: None,
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
BATCH,
VOCAB,
@@ -473,6 +475,7 @@ fn perf_topp_128k_vocab() {
top_k: None,
top_p: Some(TOP_P),
min_p: None,
+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,
},
BATCH,
VOCAB,
@@ -484,14 +487,14 @@ fn perf_topp_128k_vocab() {
let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();
let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;
- match completed.gpu_execution_time() {
- Some(gpu_time) => {
+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {
+ Some(gpu_time_ms) => {
println!(
"Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",
BATCH,
VOCAB,
std::any::type_name::<B>(),
- gpu_time.as_secs_f64() * 1e3,
+ gpu_time_ms,
host_elapsed_ms
);
},