A high-performance inference engine for AI models
Fish Audio TTS (#266)
LuckyIYIb7a964e6b4fc
diff --git a/crates/uzu/src/audio/codec.rs b/crates/uzu/src/audio/codec.rsindex e973653ed..231fc1735 100644--- a/crates/uzu/src/audio/codec.rs+++ b/crates/uzu/src/audio/codec.rs@@ -1,6 +1,6 @@use super::{AudioPcmBatch, AudioResult, AudioTokenGrid};-pub trait AudioCodecRuntime: Send + Sync {+pub trait AudioCodecRuntime {fn encode(&self,pcm: &AudioPcmBatch,diff --git a/crates/uzu/src/audio/context.rs b/crates/uzu/src/audio/context.rsnew file mode 100644index 000000000..e85c62d66--- /dev/null+++ b/crates/uzu/src/audio/context.rs@@ -0,0 +1,56 @@+#![cfg(feature = "audio-runtime")]++use std::{path::Path, rc::Rc};++use super::{AudioResult, NanoCodecFsqRuntime};+use crate::{backends::common::Backend, config::TtsConfig};++#[derive(Clone)]+pub struct AudioGenerationContext<B: Backend> {+ runtime: Rc<NanoCodecFsqRuntime<B>>,+ codec_cardinality: usize,+ semantic_codec_cardinality: Option<usize>,+ num_codebooks: usize,+ sample_rate: u32,+}++impl<B: Backend> AudioGenerationContext<B> {+ pub fn from_tts_config_and_model_path(+ tts_config: &TtsConfig,+ model_path: &Path,+ ) -> AudioResult<Self> {+ Self::with_runtime(NanoCodecFsqRuntime::from_tts_config_and_model_path(tts_config, model_path)?)+ }++ fn with_runtime(runtime: NanoCodecFsqRuntime<B>) -> AudioResult<Self> {+ let codec_cardinality = usize::try_from(runtime.config().codec_cardinality())+ .map_err(|_| super::AudioError::Runtime("audio codec cardinality exceeds usize".to_string()))?;+ Ok(Self {+ codec_cardinality,+ semantic_codec_cardinality: runtime.config().semantic_codec_cardinality(),+ num_codebooks: runtime.config().num_groups(),+ sample_rate: runtime.config().sample_rate(),+ runtime: Rc::new(runtime),+ })+ }++ pub fn runtime(&self) -> &NanoCodecFsqRuntime<B> {+ self.runtime.as_ref()+ }++ pub fn codec_cardinality(&self) -> usize {+ self.codec_cardinality+ }++ pub fn semantic_codec_cardinality(&self) -> Option<usize> {+ self.semantic_codec_cardinality+ }++ pub fn num_codebooks(&self) -> usize {+ self.num_codebooks+ }++ pub fn sample_rate(&self) -> u32 {+ self.sample_rate+ }+}diff --git a/crates/uzu/src/audio/mod.rs b/crates/uzu/src/audio/mod.rsindex 9b68e6d6d..68052a2e0 100644--- a/crates/uzu/src/audio/mod.rs+++ b/crates/uzu/src/audio/mod.rs@@ -1,5 +1,12 @@mod codec;+#[cfg(all(feature = "audio-runtime", metal_backend))]+mod context;+pub mod nanocodec;mod types;pub use codec::AudioCodecRuntime;-pub use types::{AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid, AudioTokenPacking};+#[cfg(all(feature = "audio-runtime", metal_backend))]+pub use context::AudioGenerationContext;+#[cfg(all(feature = "audio-runtime", metal_backend))]+pub use nanocodec::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};+pub use types::{AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid};diff --git a/crates/uzu/src/audio/nanocodec/mod.rs b/crates/uzu/src/audio/nanocodec/mod.rsnew file mode 100644index 000000000..f4ee66705--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/mod.rs@@ -0,0 +1,5 @@+#[cfg(all(feature = "audio-runtime", metal_backend))]+pub mod runtime;++#[cfg(all(feature = "audio-runtime", metal_backend))]+pub use runtime::{AudioDecodeStepStats, AudioDecodeStreamState, NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};diff --git a/crates/uzu/src/audio/nanocodec/runtime.rs b/crates/uzu/src/audio/nanocodec/runtime.rsnew file mode 100644index 000000000..a7b340ce6--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime.rs@@ -0,0 +1,809 @@+use std::{+ cell::{Cell, RefCell},+ collections::{BTreeMap, HashMap},+ fs::File,+ path::Path,+ rc::Rc,+};++use serde::Deserialize;++use crate::{+ DataType,+ array::{Array, ArrayContextExt, size_for_shape},+ audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid},+ backends::common::{+ Backend, Buffer, Context, Encoder, Kernels, Pending,+ kernel::{+ ActivationKernel, AudioAddKernel, AudioCausalConv1dGroupedKernel, AudioCausalConv1dGroupedResidualKernel,+ AudioCausalConv1dKernel, AudioCausalConvTranspose1dCausalPadKernel, AudioConv1dKernel,+ AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel, AudioNormNcsKernel,+ AudioQuantizerDecodeKernel,+ },+ },+ config::{+ ConfigDataType, DescriptAudioCodecConfig, DescriptAudioConvNeXtNormConfig, EmbeddingConfig,+ EmbeddingConfigCommon, InnerModelConfig, TtsAudioDecoderConfig, TtsConfig,+ },+ encodable_block::{Decoder, EncodingParameters, LayerExecutables, RMSNorm},+ forward_pass::{+ model_shape::ModelShape,+ scratch_buffers::ScratchBuffers,+ state::{ArrayId, ForwardPassState, SharedBuffers},+ },+ parameters::ParameterLoader,+};++mod loaders;+mod profile;+mod stream;+mod structured;+mod support;++use loaders::load_audio_runtime_from_tts_config;+pub(crate) use profile::PendingStreamPcmChunk;+pub use stream::{AudioDecodeStepStats, AudioDecodeStreamState};+use stream::{extract_delta_from_padded_with_offset_snapshot, pack_pcm_to_padded, unpack_padded_to_pcm};+use structured::{StructuredAudioCodecGraph, StructuredAudioRuntimeResources};+use support::{DecodedPaddedAudio, checked_product, convert_lengths_to_i32, usize_to_i32};++fn default_eps() -> f32 {+ 1e-3+}++fn compute_dim_base_index(num_levels: &[i32]) -> AudioResult<Box<[i32]>> {+ if num_levels.is_empty() {+ return Err(AudioError::InvalidTokenCardinality);+ }++ let mut out = vec![0_i32; num_levels.len()];+ let mut base = 1_i32;++ for (index, &levels) in num_levels.iter().enumerate() {+ if levels <= 1 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ out[index] = base;+ base = base.checked_mul(levels).ok_or(AudioError::Runtime("dim_base_index overflow".to_string()))?;+ }++ Ok(out.into_boxed_slice())+}++#[derive(Debug, Clone, Deserialize)]+#[serde(deny_unknown_fields)]+struct RuntimeConfigJson {+ sample_rate: u32,+ num_groups: usize,+ num_levels_per_group: Vec<i32>,+ #[serde(default = "default_eps")]+ eps: f32,+}++#[derive(Debug, Clone, PartialEq)]+pub struct NanoCodecFsqRuntimeConfig {+ sample_rate: u32,+ num_groups: usize,+ num_levels_per_group: Box<[i32]>,+ dim_base_index: Box<[i32]>,+ codebook_dim_per_group: usize,+ channels: usize,+ codec_cardinality: u32,+ eps: f32,+ structured_decoder: Option<StructuredAudioCodecGraph>,+}++impl NanoCodecFsqRuntimeConfig {+ pub fn from_tts_config_and_model_path(+ tts_config: &TtsConfig,+ model_path: &Path,+ ) -> AudioResult<Self> {+ let (runtime_config, decoder) = load_audio_runtime_from_tts_config(tts_config, model_path)?;+ let mut runtime = Self::from_runtime_config_json(runtime_config)?;+ runtime.structured_decoder = Some(decoder);+ Ok(runtime)+ }++ fn from_runtime_config_json(parsed: RuntimeConfigJson) -> AudioResult<Self> {+ Self::new(parsed.sample_rate, parsed.num_groups, parsed.num_levels_per_group.into_boxed_slice(), parsed.eps)+ }++ pub fn new(+ sample_rate: u32,+ num_groups: usize,+ num_levels_per_group: Box<[i32]>,+ eps: f32,+ ) -> AudioResult<Self> {+ if sample_rate == 0 {+ return Err(AudioError::InvalidSampleRate);+ }+ if num_groups == 0 || num_levels_per_group.is_empty() {+ return Err(AudioError::InvalidTokenCardinality);+ }+ if !eps.is_finite() || !(0.0..1.0).contains(&eps) {+ return Err(AudioError::Runtime("eps must be finite and satisfy 0.0 <= eps < 1.0".to_string()));+ }+ for &level in num_levels_per_group.iter() {+ if level <= 1 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ }++ let codebook_dim_per_group = num_levels_per_group.len();+ let channels = num_groups+ .checked_mul(codebook_dim_per_group)+ .ok_or(AudioError::Runtime("num_groups * codebook_dim_per_group overflow".to_string()))?;+ let dim_base_index = compute_dim_base_index(&num_levels_per_group)?;++ let mut codec_cardinality_u64 = 1_u64;+ for &level in num_levels_per_group.iter() {+ codec_cardinality_u64 = codec_cardinality_u64+ .checked_mul(level as u64)+ .ok_or(AudioError::Runtime("codec cardinality overflow".to_string()))?;+ }++ if codec_cardinality_u64 > u32::MAX as u64 {+ return Err(AudioError::Runtime("codec cardinality exceeds u32 range".to_string()));+ }++ let codec_cardinality = codec_cardinality_u64 as u32;+ if codec_cardinality_u64 > (i32::MAX as u64 + 1) {+ return Err(AudioError::Runtime("codec cardinality exceeds i32 token kernel range".to_string()));+ }++ Ok(Self {+ sample_rate,+ num_groups,+ num_levels_per_group,+ dim_base_index,+ codebook_dim_per_group,+ channels,+ codec_cardinality,+ eps,+ structured_decoder: None,+ })+ }++ pub fn sample_rate(&self) -> u32 {+ self.sample_rate+ }++ pub fn num_groups(&self) -> usize {+ self.num_groups+ }++ pub fn num_levels_per_group(&self) -> &[i32] {+ &self.num_levels_per_group+ }++ pub fn dim_base_index(&self) -> &[i32] {+ &self.dim_base_index+ }++ pub fn codebook_dim_per_group(&self) -> usize {+ self.codebook_dim_per_group+ }++ pub fn channels(&self) -> usize {+ self.channels+ }++ pub fn codec_cardinality(&self) -> u32 {+ self.codec_cardinality+ }++ pub fn semantic_codec_cardinality(&self) -> Option<usize> {+ self.structured_decoder.as_ref().map(|decoder| decoder.semantic_codebook_size)+ }++ pub fn eps(&self) -> f32 {+ self.eps+ }++ fn structured_decoder(&self) -> Option<&StructuredAudioCodecGraph> {+ self.structured_decoder.as_ref()+ }+}++pub struct NanoCodecFsqRuntime<B: Backend> {+ config: NanoCodecFsqRuntimeConfig,+ structured_resources: RefCell<Option<Rc<StructuredAudioRuntimeResources<B>>>>,+}++impl<B: Backend> Clone for NanoCodecFsqRuntime<B> {+ fn clone(&self) -> Self {+ Self {+ config: self.config.clone(),+ structured_resources: self.structured_resources.clone(),+ }+ }+}++impl<B: Backend> std::fmt::Debug for NanoCodecFsqRuntime<B> {+ fn fmt(+ &self,+ f: &mut std::fmt::Formatter<'_>,+ ) -> std::fmt::Result {+ f.debug_struct("NanoCodecFsqRuntime").field("config", &self.config).finish()+ }+}++impl<B: Backend> NanoCodecFsqRuntime<B> {+ pub fn new(config: NanoCodecFsqRuntimeConfig) -> Self {+ Self {+ config,+ structured_resources: RefCell::new(None),+ }+ }++ pub fn from_tts_config_and_model_path(+ tts_config: &TtsConfig,+ model_path: &Path,+ ) -> AudioResult<Self> {+ Ok(Self::new(NanoCodecFsqRuntimeConfig::from_tts_config_and_model_path(tts_config, model_path)?))+ }++ pub fn config(&self) -> &NanoCodecFsqRuntimeConfig {+ &self.config+ }++ fn validate_fishaudio_token_delta(+ &self,+ tokens: &AudioTokenGrid,+ fishaudio: &StructuredAudioCodecGraph,+ ) -> AudioResult<()> {+ let semantic_cardinality = u32::try_from(fishaudio.semantic_codebook_size).map_err(|_| {+ AudioError::Runtime("FishAudio semantic codebook cardinality exceeds u32 range".to_string())+ })?;+ let residual_cardinality = u32::try_from(fishaudio.codebook_size).map_err(|_| {+ AudioError::Runtime("FishAudio residual codebook cardinality exceeds u32 range".to_string())+ })?;+ let frames = tokens.frames();+ for batch in 0..tokens.batch_size() {+ for codebook in 0..tokens.codebooks() {+ let cardinality = if codebook == 0 {+ semantic_cardinality+ } else {+ residual_cardinality+ };+ for frame in 0..frames {+ let token = tokens.get(batch, codebook, frame);+ if token >= cardinality {+ return Err(AudioError::InvalidCodecToken {+ token,+ cardinality,+ });+ }+ }+ }+ }+ Ok(())+ }++ fn fsq_decode_padded(+ &self,+ tokens: &AudioTokenGrid,+ ) -> AudioResult<DecodedPaddedAudio> {+ if tokens.codebooks() != self.config.num_groups() {+ return Err(AudioError::Runtime(format!(+ "token codebook mismatch: expected {}, got {}",+ self.config.num_groups(),+ tokens.codebooks()+ )));+ }++ let batch_size = tokens.batch_size();+ let frames = tokens.frames();+ let lengths_usize = tokens.lengths().to_vec();+ let lengths_i32 = convert_lengths_to_i32(&lengths_usize, frames)?;++ if batch_size == 0 || frames == 0 {+ return Ok(DecodedPaddedAudio {+ samples: Vec::new(),+ channels: self.config.channels(),+ frames: 0,+ lengths: lengths_usize,+ });+ }++ let mut tokens_i32 = vec![0_i32; tokens.tokens().len()];+ for (index, &token) in tokens.tokens().iter().enumerate() {+ if token >= self.config.codec_cardinality() {+ return Err(AudioError::InvalidCodecToken {+ token,+ cardinality: self.config.codec_cardinality(),+ });+ }+ if token > i32::MAX as u32 {+ return Err(AudioError::Runtime(format!("token at index {index} exceeds i32 kernel range: {token}")));+ }+ tokens_i32[index] = token as i32;+ }++ let context = Self::create_context()?;+ let kernel = <B::Kernels as Kernels>::AudioFsqDecodeKernel::new(&context, DataType::F32)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize fsq decode kernel: {err}")))?;++ let mut tokens_array = context.create_array(+ &[batch_size, self.config.num_groups(), frames],+ DataType::I32,+ "nanocodec_fsq_decode_tokens",+ );+ tokens_array.as_slice_mut::<i32>().copy_from_slice(&tokens_i32);++ let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_decode_lengths");+ lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);++ let output = context.create_array(+ &[batch_size, self.config.channels(), frames],+ DataType::F32,+ "nanocodec_fsq_decode_output",+ );++ let mut encoder = Encoder::new(context.as_ref())+ .map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;++ let num_groups_i32 = usize_to_i32(self.config.num_groups(), "num_groups")?;+ let frames_i32 = usize_to_i32(frames, "frames")?;+ let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;+ let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;+ {+ let tokens_buffer = tokens_array.buffer();+ let tokens_buffer = tokens_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();++ kernel.encode(+ &*tokens_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ num_groups_i32,+ frames_i32,+ codebook_dim_i32,+ self.config.num_levels_per_group(),+ self.config.dim_base_index(),+ batch_size_i32,+ &mut encoder,+ );+ }++ encoder+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ decode command buffer: {err}")))?;++ Ok(DecodedPaddedAudio {+ samples: output.as_slice::<f32>().to_vec(),+ channels: self.config.channels(),+ frames,+ lengths: lengths_usize,+ })+ }++ fn fsq_encode(+ &self,+ pcm: &AudioPcmBatch,+ ) -> AudioResult<AudioTokenGrid> {+ if pcm.sample_rate() != self.config.sample_rate() {+ return Err(AudioError::Runtime(format!(+ "pcm sample-rate mismatch: expected {}, got {}",+ self.config.sample_rate(),+ pcm.sample_rate()+ )));+ }++ let (padded_input, lengths_usize, lengths_i32, frames) = pack_pcm_to_padded(pcm, self.config.channels())?;+ let batch_size = pcm.batch_size();++ if batch_size == 0 || frames == 0 {+ let empty_tokens = Vec::<u32>::new().into_boxed_slice();+ let grid = AudioTokenGrid::new(+ empty_tokens,+ batch_size,+ self.config.num_groups(),+ frames,+ lengths_usize.into_boxed_slice(),+ )?;+ return Ok(grid);+ }++ let context = Self::create_context()?;+ let kernel = <B::Kernels as Kernels>::AudioFsqEncodeKernel::new(&context, DataType::F32)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize fsq encode kernel: {err}")))?;++ let mut input = context.create_array(+ &[batch_size, self.config.channels(), frames],+ DataType::F32,+ "nanocodec_fsq_encode_input",+ );+ input.as_slice_mut::<f32>().copy_from_slice(&padded_input);++ let mut lengths = context.create_array(&[batch_size], DataType::I32, "nanocodec_fsq_encode_lengths");+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);++ let tokens = context.create_array(+ &[batch_size, self.config.num_groups(), frames],+ DataType::I32,+ "nanocodec_fsq_encode_tokens",+ );++ let mut encoder = Encoder::new(context.as_ref())+ .map_err(|err| AudioError::Runtime(format!("failed to create encoder: {err}")))?;++ let num_groups_i32 = usize_to_i32(self.config.num_groups(), "num_groups")?;+ let frames_i32 = usize_to_i32(frames, "frames")?;+ let codebook_dim_i32 = usize_to_i32(self.config.codebook_dim_per_group(), "codebook_dim_per_group")?;+ let batch_size_i32 = usize_to_i32(batch_size, "batch_size")?;+ {+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let tokens_buffer = tokens.buffer();+ let mut tokens_buffer = tokens_buffer.borrow_mut();+ let lengths_buffer = lengths.buffer();+ let lengths_buffer = lengths_buffer.borrow();++ kernel.encode(+ &*input_buffer,+ &mut *tokens_buffer,+ &*lengths_buffer,+ num_groups_i32,+ frames_i32,+ codebook_dim_i32,+ self.config.num_levels_per_group(),+ self.config.dim_base_index(),+ self.config.eps(),+ batch_size_i32,+ &mut encoder,+ );+ }++ encoder+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|err| AudioError::Runtime(format!("failed to wait for FSQ encode command buffer: {err}")))?;++ let mut tokens_u32 = vec![0_u32; tokens.num_elements()];+ for (index, &token) in tokens.as_slice::<i32>().iter().enumerate() {+ if token < 0 {+ return Err(AudioError::Runtime(format!(+ "fsq encode returned negative token at index {index}: {token}"+ )));+ }++ let token_u32 = token as u32;+ if token_u32 >= self.config.codec_cardinality() {+ return Err(AudioError::InvalidCodecToken {+ token: token_u32,+ cardinality: self.config.codec_cardinality(),+ });+ }+ tokens_u32[index] = token_u32;+ }++ AudioTokenGrid::new(+ tokens_u32.into_boxed_slice(),+ batch_size,+ self.config.num_groups(),+ frames,+ lengths_usize.into_boxed_slice(),+ )+ }++ pub fn begin_decode_stream(+ &self,+ batch_size: usize,+ codebooks: usize,+ ) -> AudioResult<AudioDecodeStreamState> {+ self.begin_decode_stream_with_options(batch_size, codebooks, 256)+ }++ pub fn begin_decode_stream_with_options(+ &self,+ batch_size: usize,+ codebooks: usize,+ max_workspace_frames: usize,+ ) -> AudioResult<AudioDecodeStreamState> {+ if codebooks != self.config.num_groups() {+ return Err(AudioError::Runtime(format!(+ "stream codebook mismatch: expected {}, got {}",+ self.config.num_groups(),+ codebooks+ )));+ }+ AudioDecodeStreamState::new(batch_size, codebooks, max_workspace_frames)+ }++ pub(crate) fn decoded_padded_to_pcm_batch(+ &self,+ decoded: &DecodedPaddedAudio,+ ) -> AudioResult<AudioPcmBatch> {+ let samples = unpack_padded_to_pcm(+ &decoded.samples,+ decoded.lengths.len(),+ decoded.channels,+ decoded.frames,+ &decoded.lengths,+ )?;+ AudioPcmBatch::new(+ samples.into_boxed_slice(),+ self.config.sample_rate(),+ decoded.channels,+ decoded.lengths.clone().into_boxed_slice(),+ )+ }++ pub fn end_decode_stream(+ &self,+ _state: AudioDecodeStreamState,+ ) -> AudioResult<()> {+ Ok(())+ }++ fn create_context() -> AudioResult<Rc<B::Context>> {+ B::Context::new().map_err(|err| AudioError::Runtime(format!("failed to create audio context: {err}")))+ }++ fn structured_resources(&self) -> AudioResult<Rc<StructuredAudioRuntimeResources<B>>> {+ if let Some(existing) = self.structured_resources.borrow().as_ref() {+ return Ok(existing.clone());+ }+ let context = Self::create_context()?;+ let created = Rc::new(StructuredAudioRuntimeResources::new(context));+ *self.structured_resources.borrow_mut() = Some(created.clone());+ Ok(created)+ }++ fn decode_structured_stream_delta(+ &self,+ state: &mut AudioDecodeStreamState,+ fishaudio: &StructuredAudioCodecGraph,+ input_frames: usize,+ ) -> AudioResult<DecodedPaddedAudio> {+ if state.total_frames() == 0 {+ state.record_last_step_stats(input_frames, 0, 0);+ return Ok(DecodedPaddedAudio {+ samples: Vec::new(),+ channels: 1,+ frames: 0,+ lengths: vec![0usize; state.batch_size],+ });+ }++ let resources = self.structured_resources()?;+ let Some(context_frames) = fishaudio.streaming_decode_context_frames(resources.as_ref())? else {+ let full_grid = state.to_full_grid()?;+ let full_padded = self.decode_padded(&full_grid)?;+ state.record_last_step_stats(input_frames, 0, state.total_frames());+ return state.extract_delta_from_padded_with_offset(&full_padded, 0, fishaudio.upsample_factor);+ };+ let mut window_start = state.total_frames();+ for &emitted in &state.emitted_semantic_lengths {+ window_start = window_start.min(emitted.saturating_sub(context_frames));+ }+ let window_end = state.total_frames();+ let batch_size = state.batch_size;+ let codebooks = state.codebooks;+ let (window_tokens, window_lengths, window_frames) = state.flatten_window(window_start, window_end)?;+ let decoded_window = fishaudio.decode_padded(+ resources.as_ref(),+ window_tokens,+ window_lengths,+ batch_size,+ codebooks,+ window_frames,+ )?;+ let audio_offset_frames = window_start+ .checked_mul(fishaudio.upsample_factor)+ .ok_or(AudioError::Runtime("stream audio offset overflow".to_string()))?;+ state.record_last_step_stats(input_frames, window_start, window_frames);+ state.extract_delta_from_padded_with_offset(&decoded_window, audio_offset_frames, fishaudio.upsample_factor)+ }++ pub(crate) fn decode_padded(+ &self,+ tokens: &AudioTokenGrid,+ ) -> AudioResult<DecodedPaddedAudio> {+ if tokens.codebooks() != self.config.num_groups() {+ return Err(AudioError::Runtime(format!(+ "token codebook mismatch: expected {}, got {}",+ self.config.num_groups(),+ tokens.codebooks()+ )));+ }++ let batch_size = tokens.batch_size();+ let frames = tokens.frames();+ let lengths_usize = tokens.lengths().to_vec();++ if batch_size == 0 || frames == 0 {+ let channels = if self.config.structured_decoder().is_some() {+ 1+ } else {+ self.config.channels()+ };+ let out_lengths = if let Some(decoder) = self.config.structured_decoder() {+ lengths_usize+ .iter()+ .map(|&length| {+ length+ .checked_mul(decoder.upsample_factor)+ .ok_or(AudioError::Runtime("decoder length scaling overflow".to_string()))+ })+ .collect::<AudioResult<Vec<_>>>()?+ } else {+ lengths_usize.clone()+ };+ return Ok(DecodedPaddedAudio {+ samples: Vec::new(),+ channels,+ frames: out_lengths.iter().copied().max().unwrap_or(0),+ lengths: out_lengths,+ });+ }++ if let Some(fishaudio) = self.config.structured_decoder() {+ self.validate_fishaudio_token_delta(tokens, fishaudio)?;+ let resources = self.structured_resources()?;+ let decoded = fishaudio.decode_padded(+ resources.as_ref(),+ tokens.tokens(),+ &lengths_usize,+ batch_size,+ tokens.codebooks(),+ frames,+ )?;+ return Ok(decoded);+ }++ self.fsq_decode_padded(tokens)+ }++ pub(crate) fn decode_stream_step(+ &self,+ state: &mut AudioDecodeStreamState,+ new_tokens: &AudioTokenGrid,+ _is_final: bool,+ ) -> AudioResult<DecodedPaddedAudio> {+ if new_tokens.codebooks() != state.codebooks {+ return Err(AudioError::Runtime(format!(+ "stream delta codebook mismatch: expected {}, got {}",+ state.codebooks,+ new_tokens.codebooks()+ )));+ }+ if new_tokens.batch_size() != state.batch_size {+ return Err(AudioError::Runtime(format!(+ "stream delta batch mismatch: expected {}, got {}",+ state.batch_size,+ new_tokens.batch_size()+ )));+ }+ if new_tokens.frames() == 0 {+ state.record_last_step_stats(0, state.total_frames(), 0);+ return Ok(DecodedPaddedAudio {+ samples: Vec::new(),+ channels: if self.config.structured_decoder().is_some() {+ 1+ } else {+ self.config.channels()+ },+ frames: 0,+ lengths: vec![0; state.batch_size],+ });+ }++ if let Some(fishaudio) = self.config.structured_decoder() {+ self.validate_fishaudio_token_delta(new_tokens, fishaudio)?;+ state.append_delta(new_tokens)?;+ return self.decode_structured_stream_delta(state, fishaudio, new_tokens.frames());+ }++ state.append_delta(new_tokens)?;+ let full_grid = state.to_full_grid()?;+ let full_padded = self.fsq_decode_padded(&full_grid)?;+ state.record_last_step_stats(new_tokens.frames(), 0, state.total_frames());+ state.extract_delta_from_padded_with_offset(&full_padded, 0, 1)+ }++ pub(crate) fn submit_decode_stream_step(+ &self,+ state: &mut AudioDecodeStreamState,+ new_tokens: &AudioTokenGrid,+ is_final: bool,+ ) -> AudioResult<Option<PendingStreamPcmChunk<B>>> {+ if is_final {+ return Ok(None);+ }+ if new_tokens.codebooks() != state.codebooks {+ return Err(AudioError::Runtime(format!(+ "stream delta codebook mismatch: expected {}, got {}",+ state.codebooks,+ new_tokens.codebooks()+ )));+ }+ if new_tokens.batch_size() != state.batch_size {+ return Err(AudioError::Runtime(format!(+ "stream delta batch mismatch: expected {}, got {}",+ state.batch_size,+ new_tokens.batch_size()+ )));+ }+ if new_tokens.frames() == 0 {+ return Ok(None);+ }++ let Some(fishaudio) = self.config.structured_decoder() else {+ return Ok(None);+ };+ let resources = self.structured_resources()?;+ let Some(context_frames) = fishaudio.streaming_decode_context_frames(resources.as_ref())? else {+ return Ok(None);+ };++ self.validate_fishaudio_token_delta(new_tokens, fishaudio)?;+ state.append_delta(new_tokens)?;++ let mut window_start = state.total_frames();+ for &emitted in &state.emitted_semantic_lengths {+ window_start = window_start.min(emitted.saturating_sub(context_frames));+ }+ let window_end = state.total_frames();+ let batch_size = state.batch_size;+ let codebooks = state.codebooks;+ let previous_audio_lengths = state.emitted_audio_lengths.clone().into_boxed_slice();+ let semantic_lengths = state.semantic_lengths.clone().into_boxed_slice();+ let (window_tokens, window_lengths, window_frames) = state.flatten_window(window_start, window_end)?;+ let submitted = fishaudio.submit_decode_padded(+ resources.as_ref(),+ window_tokens,+ window_lengths,+ batch_size,+ codebooks,+ window_frames,+ )?;+ // The scratch buffers inside the workspace were encoded into the+ // submitted command buffer. Reset the workspace so the next decode+ // pass allocates fresh buffers instead of reusing ones the in-flight+ // command buffer may still be reading.+ resources.reset_for_pending();+ let audio_offset_frames = window_start+ .checked_mul(fishaudio.upsample_factor)+ .ok_or(AudioError::Runtime("stream audio offset overflow".to_string()))?;+ state.record_last_step_stats(new_tokens.frames(), window_start, window_frames);+ state.mark_submitted_audio_window(&semantic_lengths, fishaudio.upsample_factor)?;++ Ok(Some(PendingStreamPcmChunk {+ runtime: self.clone(),+ submitted,+ previous_audio_lengths,+ semantic_lengths,+ audio_offset_frames,+ upsample_factor: fishaudio.upsample_factor,+ step_stats: state.last_step_stats(),+ }))+ }+}++impl<B: Backend> AudioCodecRuntime for NanoCodecFsqRuntime<B> {+ fn encode(+ &self,+ pcm: &AudioPcmBatch,+ ) -> AudioResult<AudioTokenGrid> {+ if self.config.structured_decoder().is_some() {+ return Err(AudioError::Runtime("encode is not supported when a decoder graph is configured".to_string()));+ }+ self.fsq_encode(pcm)+ }++ fn decode(+ &self,+ tokens: &AudioTokenGrid,+ ) -> AudioResult<AudioPcmBatch> {+ let decoded = self.decode_padded(tokens)?;+ self.decoded_padded_to_pcm_batch(&decoded)+ }+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/loaders.rs b/crates/uzu/src/audio/nanocodec/runtime/loaders.rsnew file mode 100644index 000000000..a81ea9359--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/loaders.rs@@ -0,0 +1,93 @@+use super::*;++pub(super) fn resolve_descript_audio_codec_vocoder_data_type(+ top_level_precision: Option<ConfigDataType>,+ config: &DescriptAudioCodecConfig,+) -> AudioResult<DataType> {+ let mut resolved_precision: Option<ConfigDataType> = None;+ for (field_name, precision) in [+ ("tts_config.activation_precision", top_level_precision),+ ("tts_config.audio_decoder_config.precision", Some(config.precision)),+ ("tts_config.audio_decoder_config.quantizer_config.precision", config.quantizer_config.precision),+ ] {+ if let Some(precision) = precision {+ if let Some(existing) = resolved_precision {+ if existing != precision {+ return Err(AudioError::Runtime(format!(+ "conflicting DescriptAudioCodec precision in Lalamo export: {field_name}={precision:?} conflicts with {existing:?}"+ )));+ }+ } else {+ resolved_precision = Some(precision);+ }+ }+ }++ let precision = resolved_precision.ok_or(AudioError::Runtime(+ "missing DescriptAudioCodec precision in Lalamo export; expected one of tts_config.activation_precision, \+tts_config.audio_decoder_config.precision, or tts_config.audio_decoder_config.quantizer_config.precision"+ .to_string(),+ ))?;+ let data_type: DataType = precision.into();+ if !matches!(data_type, DataType::F32 | DataType::F16 | DataType::BF16) {+ return Err(AudioError::Runtime(format!(+ "unsupported DescriptAudioCodec vocoder precision in Lalamo export: {precision:?} (expected float32/float16/bfloat16)"+ )));+ }+ Ok(data_type)+}++pub(super) fn load_audio_runtime_from_tts_config(+ tts_config: &TtsConfig,+ model_path: &Path,+) -> AudioResult<(RuntimeConfigJson, StructuredAudioCodecGraph)> {+ let cfg = match &tts_config.audio_decoder_config {+ TtsAudioDecoderConfig::DescriptAudioCodecConfig {+ config,+ } => config,+ };+ let fishaudio_weights = model_path.join("model.safetensors");+ if !fishaudio_weights.is_file() {+ return Err(AudioError::Runtime(format!(+ "missing exported FishAudio decoder weights '{}'",+ fishaudio_weights.display()+ )));+ }++ let vocoder_data_type = resolve_descript_audio_codec_vocoder_data_type(tts_config.activation_precision, cfg)?;++ let total_codebooks = cfg+ .n_codebooks+ .checked_add(1)+ .ok_or(AudioError::Runtime("FishAudio codebook count overflow while building runtime config".to_string()))?;+ let codebook_size_i32 = i32::try_from(cfg.codebook_size)+ .map_err(|_| AudioError::Runtime("FishAudio codebook_size exceeds i32 kernel range".to_string()))?;+ if codebook_size_i32 <= 1 {+ return Err(AudioError::InvalidTokenCardinality);+ }++ let upsample_factor = cfg+ .downsample_factor+ .iter()+ .chain(cfg.decoder_rates.iter())+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))+ .ok_or(AudioError::Runtime("FishAudio upsample factor overflow".to_string()))?;++ let runtime = RuntimeConfigJson {+ sample_rate: cfg.samplerate,+ num_groups: total_codebooks,+ num_levels_per_group: vec![codebook_size_i32],+ eps: default_eps(),+ };+ let decoder = StructuredAudioCodecGraph {+ config: cfg.clone(),+ weights_path: fishaudio_weights.display().to_string(),+ codebook_size: cfg.codebook_size,+ semantic_codebook_size: cfg.semantic_codebook_size,+ input_dim: cfg.input_dim,+ total_codebooks,+ upsample_factor,+ vocoder_data_type,+ };+ Ok((runtime, decoder))+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/profile.rs b/crates/uzu/src/audio/nanocodec/runtime/profile.rsnew file mode 100644index 000000000..8c19f5df3--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/profile.rs@@ -0,0 +1,75 @@+use std::sync::mpsc::Receiver;++use super::*;++pub struct SubmittedDecodedPaddedAudio<B: Backend> {+ pub(in crate::audio::nanocodec::runtime) output: Array<B>,+ pub(in crate::audio::nanocodec::runtime) channels: usize,+ pub(in crate::audio::nanocodec::runtime) frames: usize,+ pub(in crate::audio::nanocodec::runtime) lengths: Vec<usize>,+ pub(in crate::audio::nanocodec::runtime) final_command_buffer: Option<Pending<B>>,+ pub(in crate::audio::nanocodec::runtime) completion_notification: Option<Receiver<()>>,+}++impl<B: Backend> SubmittedDecodedPaddedAudio<B> {+ pub(crate) fn is_complete(&self) -> bool {+ use std::sync::mpsc::TryRecvError;++ self.completion_notification.as_ref().is_none_or(|notification| match notification.try_recv() {+ Ok(()) | Err(TryRecvError::Disconnected) => true,+ Err(TryRecvError::Empty) => false,+ })+ }++ pub(in crate::audio::nanocodec::runtime) fn resolve(mut self) -> AudioResult<DecodedPaddedAudio> {+ if let Some(command_buffer) = self.final_command_buffer.take() {+ command_buffer.wait_until_completed().map_err(|err| {+ AudioError::Runtime(format!("failed to wait for FishAudio decoder command buffer: {err}"))+ })?;+ }+ let samples: Vec<f32> = match self.output.data_type() {+ DataType::F32 => self.output.as_slice::<f32>().to_vec(),+ DataType::F16 => self.output.as_slice::<half::f16>().iter().map(|&v| f32::from(v)).collect(),+ DataType::BF16 => self.output.as_slice::<half::bf16>().iter().map(|&v| f32::from(v)).collect(),+ dt => return Err(AudioError::Runtime(format!("unsupported vocoder output dtype: {dt:?}"))),+ };+ Ok(DecodedPaddedAudio {+ samples,+ channels: self.channels,+ frames: self.frames,+ lengths: self.lengths,+ })+ }+}++pub(crate) struct PendingStreamPcmChunk<B: Backend> {+ pub(in crate::audio::nanocodec::runtime) runtime: NanoCodecFsqRuntime<B>,+ pub(in crate::audio::nanocodec::runtime) submitted: SubmittedDecodedPaddedAudio<B>,+ pub(in crate::audio::nanocodec::runtime) previous_audio_lengths: Box<[usize]>,+ pub(in crate::audio::nanocodec::runtime) semantic_lengths: Box<[usize]>,+ pub(in crate::audio::nanocodec::runtime) audio_offset_frames: usize,+ pub(in crate::audio::nanocodec::runtime) upsample_factor: usize,+ pub(in crate::audio::nanocodec::runtime) step_stats: AudioDecodeStepStats,+}++impl<B: Backend> PendingStreamPcmChunk<B> {+ pub(crate) fn step_stats(&self) -> AudioDecodeStepStats {+ self.step_stats+ }++ pub(crate) fn is_complete(&self) -> bool {+ self.submitted.is_complete()+ }++ pub(crate) fn resolve(self) -> AudioResult<AudioPcmBatch> {+ let decoded_window = self.submitted.resolve()?;+ let delta = extract_delta_from_padded_with_offset_snapshot(+ &decoded_window,+ &self.previous_audio_lengths,+ &self.semantic_lengths,+ self.audio_offset_frames,+ self.upsample_factor,+ )?;+ self.runtime.decoded_padded_to_pcm_batch(&delta)+ }+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/stream.rs b/crates/uzu/src/audio/nanocodec/runtime/stream.rsnew file mode 100644index 000000000..330eb0804--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/stream.rs@@ -0,0 +1,481 @@+use super::*;++#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct AudioDecodeStepStats {+ pub input_frames: usize,+ pub total_semantic_frames: usize,+ pub decoded_window_start_frame: usize,+ pub decoded_window_frames: usize,+}++/// Ring-buffer backed stream state for incremental audio decoding.+///+/// Token rows are stored in fixed-capacity circular buffers so that evicting+/// old frames is O(1) (just advance the head pointer) instead of O(N) via+/// `Vec::drain`.+#[derive(Debug, Clone)]+pub struct AudioDecodeStreamState {+ pub(super) batch_size: usize,+ pub(super) codebooks: usize,+ retained_capacity: usize,+ stored_frame_start: usize,+ stored_frames: usize,+ total_frames_generated: usize,+ row_tokens: Vec<Vec<u32>>,+ flattened_tokens: Vec<u32>,+ window_lengths: Vec<usize>,+ pub(super) semantic_lengths: Vec<usize>,+ pub(super) emitted_semantic_lengths: Vec<usize>,+ pub(super) emitted_audio_lengths: Vec<usize>,+ last_step_stats: AudioDecodeStepStats,+}++impl AudioDecodeStreamState {+ pub(super) fn new(+ batch_size: usize,+ codebooks: usize,+ max_workspace_frames: usize,+ ) -> AudioResult<Self> {+ if batch_size == 0 {+ return Err(AudioError::Runtime("stream state batch_size must be > 0".to_string()));+ }+ if codebooks == 0 {+ return Err(AudioError::Runtime("stream state codebooks must be > 0".to_string()));+ }+ if max_workspace_frames == 0 {+ return Err(AudioError::Runtime("stream state max_workspace_frames must be > 0".to_string()));+ }++ let row_count = batch_size+ .checked_mul(codebooks)+ .ok_or(AudioError::Runtime("stream state row count overflow".to_string()))?;+ // Double the workspace for ring capacity so we have room for context+ // overlap without immediate eviction.+ let retained_capacity = max_workspace_frames+ .checked_mul(2)+ .ok_or(AudioError::Runtime("stream retained capacity overflow".to_string()))?;+ let mut row_tokens = Vec::with_capacity(row_count);+ for _ in 0..row_count {+ row_tokens.push(vec![0u32; retained_capacity]);+ }++ Ok(Self {+ batch_size,+ codebooks,+ retained_capacity,+ stored_frame_start: 0,+ stored_frames: 0,+ total_frames_generated: 0,+ row_tokens,+ flattened_tokens: Vec::with_capacity(+ row_count+ .checked_mul(max_workspace_frames)+ .ok_or(AudioError::Runtime("stream token capacity overflow".to_string()))?,+ ),+ window_lengths: Vec::with_capacity(batch_size),+ semantic_lengths: vec![0; batch_size],+ emitted_semantic_lengths: vec![0; batch_size],+ emitted_audio_lengths: vec![0; batch_size],+ last_step_stats: AudioDecodeStepStats::default(),+ })+ }++ pub(super) fn total_frames(&self) -> usize {+ self.total_frames_generated+ }++ pub fn last_step_stats(&self) -> AudioDecodeStepStats {+ self.last_step_stats+ }++ /// Global frame index one past the last retained frame.+ fn stored_frame_end(&self) -> usize {+ self.stored_frame_start.saturating_add(self.stored_frames)+ }++ /// Physical ring slot for a global frame index.+ #[inline]+ fn slot(+ &self,+ global_frame: usize,+ ) -> usize {+ global_frame % self.retained_capacity+ }++ pub(super) fn append_delta(+ &mut self,+ delta_tokens: &AudioTokenGrid,+ ) -> AudioResult<()> {+ if delta_tokens.batch_size() != self.batch_size {+ return Err(AudioError::Runtime(format!(+ "stream delta batch mismatch: expected {}, got {}",+ self.batch_size,+ delta_tokens.batch_size()+ )));+ }+ if delta_tokens.codebooks() != self.codebooks {+ return Err(AudioError::Runtime(format!(+ "stream delta codebook mismatch: expected {}, got {}",+ self.codebooks,+ delta_tokens.codebooks()+ )));+ }+ if delta_tokens.frames() == 0 {+ return Ok(());+ }++ let delta_frames = delta_tokens.frames();+ let tokens = delta_tokens.tokens();+ let delta_lengths = delta_tokens.lengths();++ for frame_offset in 0..delta_frames {+ let global_frame = self.stored_frame_start + self.stored_frames;+ // Evict oldest frame if ring is full.+ if self.stored_frames == self.retained_capacity {+ self.stored_frame_start += 1;+ self.stored_frames -= 1;+ }+ let physical = self.slot(global_frame);++ for batch in 0..self.batch_size {+ for codebook in 0..self.codebooks {+ let row_index = batch * self.codebooks + codebook;+ let src_index = row_index * delta_frames + frame_offset;+ self.row_tokens[row_index][physical] = tokens[src_index];+ }+ }+ self.stored_frames += 1;+ }+ self.total_frames_generated = self.total_frames_generated.saturating_add(delta_frames);++ for (length, &delta_len) in self.semantic_lengths.iter_mut().zip(delta_lengths.iter()) {+ *length = length+ .checked_add(delta_len)+ .ok_or(AudioError::Runtime("stream semantic length overflow".to_string()))?;+ }++ Ok(())+ }++ pub(super) fn to_full_grid(&self) -> AudioResult<AudioTokenGrid> {+ if self.stored_frame_start != 0 {+ return Err(AudioError::Runtime(format!(+ "full-grid decode requires retained prefix, but {} frames were evicted",+ self.stored_frame_start+ )));+ }+ let total_frames = self.stored_frames;++ let expected_tokens = self+ .batch_size+ .checked_mul(self.codebooks)+ .and_then(|value| value.checked_mul(total_frames))+ .ok_or(AudioError::Runtime("stream token count overflow".to_string()))?;+ let mut tokens = Vec::with_capacity(expected_tokens);++ for row in &self.row_tokens {+ // When stored_frame_start == 0 and no wrap has occurred, frames+ // 0..total_frames map to physical slots 0..total_frames.+ for frame in 0..total_frames {+ let physical = self.slot(frame);+ tokens.push(row[physical]);+ }+ }+ if tokens.len() != expected_tokens {+ return Err(AudioError::Runtime("stream flattened token count mismatch".to_string()));+ }++ AudioTokenGrid::new(+ tokens.into_boxed_slice(),+ self.batch_size,+ self.codebooks,+ total_frames,+ self.semantic_lengths.clone().into_boxed_slice(),+ )+ }++ pub(super) fn flatten_window(+ &mut self,+ start_frame: usize,+ end_frame: usize,+ ) -> AudioResult<(&[u32], &[usize], usize)> {+ if end_frame < start_frame {+ return Err(AudioError::Runtime(format!(+ "invalid stream window: start_frame={start_frame}, end_frame={end_frame}"+ )));+ }+ let total_frames = self.total_frames();+ if end_frame > total_frames {+ return Err(AudioError::Runtime(format!(+ "stream window end exceeds token frames: end={end_frame}, total={total_frames}"+ )));+ }+ if start_frame < self.stored_frame_start || end_frame > self.stored_frame_end() {+ return Err(AudioError::Runtime(format!(+ "stream window [{start_frame}, {end_frame}) exceeds retained workspace [{}, {})",+ self.stored_frame_start,+ self.stored_frame_end()+ )));+ }++ let window_frames = end_frame.saturating_sub(start_frame);+ let row_count = self+ .batch_size+ .checked_mul(self.codebooks)+ .ok_or(AudioError::Runtime("stream row count overflow".to_string()))?;+ let required_capacity = row_count+ .checked_mul(window_frames)+ .ok_or(AudioError::Runtime("stream window token capacity overflow".to_string()))?;++ // Grow the scratch buffer if needed (only happens once in practice).+ if self.flattened_tokens.capacity() < required_capacity {+ self.flattened_tokens.reserve(required_capacity - self.flattened_tokens.capacity());+ }+ self.flattened_tokens.clear();++ // Read from ring in logical order via modulo indexing.+ for row in &self.row_tokens {+ for global_frame in start_frame..end_frame {+ let physical = self.slot(global_frame);+ self.flattened_tokens.push(row[physical]);+ }+ }++ self.window_lengths.clear();+ for &length in &self.semantic_lengths {+ self.window_lengths.push(length.saturating_sub(start_frame));+ }++ Ok((&self.flattened_tokens, &self.window_lengths, window_frames))+ }++ pub(super) fn extract_delta_from_padded_with_offset(+ &mut self,+ full_padded: &DecodedPaddedAudio,+ audio_offset_frames: usize,+ upsample_factor: usize,+ ) -> AudioResult<DecodedPaddedAudio> {+ let delta = extract_delta_from_padded_with_offset_snapshot(+ full_padded,+ &self.emitted_audio_lengths,+ &self.semantic_lengths,+ audio_offset_frames,+ upsample_factor,+ )?;+ for index in 0..self.semantic_lengths.len() {+ let semantic_length = self.semantic_lengths[index];+ let full_audio_length = semantic_length+ .checked_mul(upsample_factor)+ .ok_or(AudioError::Runtime("stream audio length overflow".to_string()))?;+ self.emitted_audio_lengths[index] = full_audio_length;+ self.emitted_semantic_lengths[index] = semantic_length;+ }+ Ok(delta)+ }++ pub(super) fn mark_submitted_audio_window(+ &mut self,+ semantic_lengths: &[usize],+ upsample_factor: usize,+ ) -> AudioResult<()> {+ if semantic_lengths.len() != self.batch_size {+ return Err(AudioError::Runtime(format!(+ "stream semantic lengths mismatch: expected {}, got {}",+ self.batch_size,+ semantic_lengths.len()+ )));+ }+ for (index, &semantic_length) in semantic_lengths.iter().enumerate() {+ let full_audio_length = semantic_length+ .checked_mul(upsample_factor)+ .ok_or(AudioError::Runtime("stream audio length overflow".to_string()))?;+ self.emitted_audio_lengths[index] = full_audio_length;+ self.emitted_semantic_lengths[index] = semantic_length;+ }+ Ok(())+ }++ pub(super) fn record_last_step_stats(+ &mut self,+ input_frames: usize,+ decoded_window_start_frame: usize,+ decoded_window_frames: usize,+ ) {+ self.last_step_stats = AudioDecodeStepStats {+ input_frames,+ total_semantic_frames: self.total_frames_generated,+ decoded_window_start_frame,+ decoded_window_frames,+ };+ }+}++pub(super) fn extract_delta_from_padded_with_offset_snapshot(+ full_padded: &DecodedPaddedAudio,+ previous_audio_lengths: &[usize],+ semantic_lengths: &[usize],+ audio_offset_frames: usize,+ upsample_factor: usize,+) -> AudioResult<DecodedPaddedAudio> {+ if full_padded.lengths.len() != semantic_lengths.len() || previous_audio_lengths.len() != semantic_lengths.len() {+ return Err(AudioError::Runtime(format!(+ "stream decoded batch mismatch: full_lengths={}, previous_lengths={}, semantic_lengths={}",+ full_padded.lengths.len(),+ previous_audio_lengths.len(),+ semantic_lengths.len()+ )));+ }+ if upsample_factor == 0 {+ return Err(AudioError::Runtime("stream upsample_factor must be > 0".to_string()));+ }++ let batch_size = semantic_lengths.len();+ let channels = full_padded.channels;+ let decoded_frames = full_padded.frames;+ let mut delta_lengths = vec![0usize; batch_size];+ let mut max_delta_frames = 0usize;+ let mut local_delta_ranges = vec![(0usize, 0usize); batch_size];++ for batch in 0..batch_size {+ let full_audio_length = semantic_lengths[batch]+ .checked_mul(upsample_factor)+ .ok_or(AudioError::Runtime("stream audio length overflow".to_string()))?;+ let previous_audio_length = previous_audio_lengths[batch].min(full_audio_length);+ let decoded_batch_length = full_padded.lengths[batch];+ let target_end_local = full_audio_length.saturating_sub(audio_offset_frames).min(decoded_batch_length);+ let target_start_local = previous_audio_length.saturating_sub(audio_offset_frames).min(target_end_local);+ let delta = target_end_local.saturating_sub(target_start_local);+ local_delta_ranges[batch] = (target_start_local, target_end_local);+ delta_lengths[batch] = delta;+ max_delta_frames = max_delta_frames.max(delta);+ }++ let padded_len = checked_product(&[batch_size, channels, max_delta_frames])?;+ let mut delta_padded = vec![0.0_f32; padded_len];+ for (batch, &(start_local, end_local)) in local_delta_ranges.iter().enumerate() {+ let delta_len = end_local.saturating_sub(start_local);+ if delta_len == 0 {+ continue;+ }+ for channel in 0..channels {+ let src_start = (batch * channels + channel)+ .checked_mul(decoded_frames)+ .and_then(|value| value.checked_add(start_local))+ .ok_or(AudioError::Runtime("stream padded source offset overflow".to_string()))?;+ let src_end = src_start+ .checked_add(delta_len)+ .ok_or(AudioError::Runtime("stream padded source offset overflow".to_string()))?;+ let dst_start = (batch * channels + channel)+ .checked_mul(max_delta_frames)+ .ok_or(AudioError::Runtime("stream padded destination offset overflow".to_string()))?;+ let dst_end = dst_start+ .checked_add(delta_len)+ .ok_or(AudioError::Runtime("stream padded destination offset overflow".to_string()))?;+ delta_padded[dst_start..dst_end].copy_from_slice(&full_padded.samples[src_start..src_end]);+ }+ }++ Ok(DecodedPaddedAudio {+ samples: delta_padded,+ channels,+ frames: max_delta_frames,+ lengths: delta_lengths,+ })+}++pub(super) fn pack_pcm_to_padded(+ pcm: &AudioPcmBatch,+ expected_channels: usize,+) -> AudioResult<(Vec<f32>, Vec<usize>, Vec<i32>, usize)> {+ if pcm.channels() != expected_channels {+ return Err(AudioError::Runtime(format!(+ "pcm channel mismatch: expected {expected_channels}, got {}",+ pcm.channels()+ )));+ }++ let lengths = pcm.lengths().to_vec();+ let frames = lengths.iter().copied().max().unwrap_or(0);+ let lengths_i32 = convert_lengths_to_i32(&lengths, frames)?;+ let padded_len = checked_product(&[pcm.batch_size(), expected_channels, frames])?;+ let mut padded = vec![0.0_f32; padded_len];++ let mut src_offset = 0usize;+ let samples = pcm.samples();+ for (batch, &frame_count) in lengths.iter().enumerate() {+ let sample_count = frame_count+ .checked_mul(expected_channels)+ .ok_or(AudioError::Runtime("pcm frame-count overflow".to_string()))?;+ let src_end =+ src_offset.checked_add(sample_count).ok_or(AudioError::Runtime("pcm indexing overflow".to_string()))?;+ if src_end > samples.len() {+ return Err(AudioError::Runtime("pcm indexing out of bounds".to_string()));+ }++ for frame in 0..frame_count {+ let src_frame_base = src_offset + frame * expected_channels;+ for channel in 0..expected_channels {+ let dst_index = (batch * expected_channels + channel) * frames + frame;+ padded[dst_index] = samples[src_frame_base + channel];+ }+ }++ src_offset = src_end;+ }++ Ok((padded, lengths, lengths_i32, frames))+}++pub(super) fn unpack_padded_to_pcm(+ padded: &[f32],+ batch_size: usize,+ channels: usize,+ frames: usize,+ lengths: &[usize],+) -> AudioResult<Vec<f32>> {+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }++ let expected_padded = checked_product(&[batch_size, channels, frames])?;+ if padded.len() != expected_padded {+ return Err(AudioError::InvalidPcmShape {+ expected_samples: expected_padded,+ actual_samples: padded.len(),+ });+ }++ let total_frames = lengths.iter().sum::<usize>();+ let packed_len = checked_product(&[total_frames, channels])?;+ let mut packed = vec![0.0_f32; packed_len];+ let mut dst_offset = 0usize;++ for (batch, &frame_count) in lengths.iter().enumerate() {+ if frame_count > frames {+ return Err(AudioError::InvalidTokenLengthValue {+ length: frame_count,+ frames,+ });+ }++ for frame in 0..frame_count {+ for channel in 0..channels {+ let src_index = (batch * channels + channel) * frames + frame;+ let dst_index = dst_offset + frame * channels + channel;+ packed[dst_index] = padded[src_index];+ }+ }+ dst_offset = dst_offset+ .checked_add(+ frame_count+ .checked_mul(channels)+ .ok_or(AudioError::Runtime("pcm destination indexing overflow".to_string()))?,+ )+ .ok_or(AudioError::Runtime("pcm destination indexing overflow".to_string()))?;+ }++ Ok(packed)+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rsnew file mode 100644index 000000000..034414d0c--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/build.rs@@ -0,0 +1,358 @@+use super::*;++pub(super) type BackendParameterTree<'loader, B> = crate::parameters::ParameterTree<'loader, <B as Backend>::Context>;++pub(super) fn read_float_array<B: Backend, const RANK: usize>(+ tree: &BackendParameterTree<'_, B>,+ name: &str,+ expected_data_type: DataType,+) -> AudioResult<([usize; RANK], Array<B>)> {+ let array = tree.leaf_array(name)?;+ if array.data_type() != expected_data_type {+ return Err(AudioError::Runtime(format!(+ "tensor '{name}' dtype mismatch: expected {expected_data_type:?}, got {:?}",+ array.data_type()+ )));+ }+ if array.shape().len() != RANK {+ return Err(AudioError::Runtime(format!(+ "expected rank-{RANK} tensor for '{name}', got shape {:?}",+ array.shape()+ )));+ }+ let mut dims = [0usize; RANK];+ dims.copy_from_slice(array.shape());+ Ok((dims, array))+}++pub(super) fn read_float_vector_exact<B: Backend>(+ tree: &BackendParameterTree<'_, B>,+ name: &str,+ expected_len: usize,+ expected_data_type: DataType,+) -> AudioResult<Array<B>> {+ let (shape, array) = read_float_array::<B, 1>(tree, name, expected_data_type)?;+ if shape[0] != expected_len {+ return Err(AudioError::Runtime(format!(+ "tensor '{name}' shape mismatch: expected [{expected_len}], got {:?}",+ shape+ )));+ }+ Ok(array)+}++pub(super) fn read_float_matrix_exact<B: Backend>(+ tree: &BackendParameterTree<'_, B>,+ name: &str,+ expected_rows: usize,+ expected_cols: usize,+ expected_data_type: DataType,+) -> AudioResult<Array<B>> {+ let (shape, array) = read_float_array::<B, 2>(tree, name, expected_data_type)?;+ if shape != [expected_rows, expected_cols] {+ return Err(AudioError::Runtime(format!(+ "tensor '{name}' shape mismatch: expected [{expected_rows}, {expected_cols}], got {:?}",+ shape+ )));+ }+ Ok(array)+}++pub(super) fn outer_axis_view<B: Backend>(+ array: &Array<B>,+ index: usize,+ slice_shape: &[usize],+) -> AudioResult<Array<B>> {+ let slice_bytes = size_for_shape(slice_shape, array.data_type());+ let offset = index+ .checked_mul(slice_bytes)+ .and_then(|value| value.checked_add(array.offset()))+ .ok_or(AudioError::Runtime("array outer-axis view offset overflow".to_string()))?;+ Ok(unsafe { Array::from_parts(array.buffer(), offset, slice_shape, array.data_type()) })+}++pub(super) fn read_conv1d_layer<B: Backend>(+ tree: &BackendParameterTree<'_, B>,+ data_type: DataType,+ dilation: usize,+ groups: usize,+) -> AudioResult<StructuredAudioConv1d<B>> {+ let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;+ let bias = read_float_vector_exact::<B>(tree, "biases", shape[0], data_type)?;+ if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ if shape[0] % groups != 0 {+ return Err(AudioError::Runtime(format!(+ "invalid grouped conv weights for 'weights': out_channels {} not divisible by groups {groups}",+ shape[0]+ )));+ }+ Ok(StructuredAudioConv1d {+ weight,+ bias,+ cin: shape[1].checked_mul(groups).ok_or(AudioError::Runtime("conv input channel overflow".to_string()))?,+ cout: shape[0],+ kernel_size: shape[2],+ dilation,+ groups,+ })+}++pub(super) fn read_conv_transpose1d_layer<B: Backend>(+ tree: &BackendParameterTree<'_, B>,+ data_type: DataType,+ stride: usize,+ groups: usize,+) -> AudioResult<StructuredAudioConvTranspose1d<B>> {+ if stride == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ let (shape, weight) = read_float_array::<B, 3>(tree, "weights", data_type)?;+ if groups == 0 || shape[0] == 0 || shape[1] == 0 || shape[2] == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ let export_cout = shape[0];+ let export_cin =+ shape[1].checked_mul(groups).ok_or(AudioError::Runtime("transpose conv input channel overflow".to_string()))?;+ if export_cout % groups != 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ let bias = read_float_vector_exact::<B>(tree, "biases", export_cout, data_type)?;+ let expected_weight_shape = [export_cout, export_cin / groups, shape[2]];+ if shape != expected_weight_shape {+ return Err(AudioError::Runtime(format!(+ "transpose conv export-layout weight shape mismatch: expected {:?}, got {:?}",+ expected_weight_shape, shape+ )));+ }++ Ok(StructuredAudioConvTranspose1d {+ weight,+ bias,+ cin: export_cin,+ cout: export_cout,+ kernel_size: shape[2],+ stride,+ groups,+ })+}++pub(super) fn read_pointwise_conv_layer<B: Backend>(+ tree: &BackendParameterTree<'_, B>,+ data_type: DataType,+ expected_out_dim: usize,+ expected_in_dim: usize,+) -> AudioResult<StructuredAudioPointwiseConv<B>> {+ let weight = read_float_matrix_exact::<B>(tree, "weights", expected_out_dim, expected_in_dim, data_type)?;+ let bias = read_float_vector_exact::<B>(tree, "biases", expected_out_dim, data_type)?;+ Ok(StructuredAudioPointwiseConv {+ weight: weight.view(&[expected_out_dim, expected_in_dim, 1]),+ bias,+ cin: expected_in_dim,+ cout: expected_out_dim,+ })+}++pub(super) fn read_norm_layer<B: Backend>(+ context: &Rc<B::Context>,+ tree: &BackendParameterTree<'_, B>,+ data_type: DataType,+ channels: usize,+ epsilon: f32,+ subtract_mean: bool,+ use_bias: bool,+ label_prefix: &str,+) -> AudioResult<StructuredAudioNorm<B>> {+ let scales = read_float_vector_exact::<B>(tree, "scales", channels, data_type)?;+ let bias = if use_bias {+ read_float_vector_exact::<B>(tree, "biases", channels, data_type)?+ } else {+ context.create_array(&[channels], data_type, &format!("{label_prefix}_bias"))+ };+ Ok(StructuredAudioNorm {+ scales,+ bias,+ epsilon,+ subtract_mean,+ })+}++pub(super) fn read_convnext_layer<B: Backend>(+ context: &Rc<B::Context>,+ tree: &BackendParameterTree<'_, B>,+ data_type: DataType,+ norm_config: &DescriptAudioConvNeXtNormConfig,+) -> AudioResult<StructuredAudioConvNeXt<B>> {+ let depthwise_tree = tree.subtree("dwconv")?;+ let (depthwise_shape, depthwise_weight) = read_float_array::<B, 3>(&depthwise_tree, "weights", data_type)?;+ if depthwise_shape[1] != 1 {+ return Err(AudioError::Runtime(format!(+ "ConvNeXt depthwise weight in_channels_per_group must be 1 at 'dwconv.weights', got {}",+ depthwise_shape[1]+ )));+ }+ let depthwise_conv = StructuredAudioConv1d {+ weight: depthwise_weight,+ bias: read_float_vector_exact::<B>(&depthwise_tree, "biases", depthwise_shape[0], data_type)?,+ cin: depthwise_shape[0],+ cout: depthwise_shape[0],+ kernel_size: depthwise_shape[2],+ dilation: 1,+ groups: depthwise_shape[0],+ };+ if depthwise_conv.bias.shape()[0] != depthwise_conv.cout {+ return Err(AudioError::Runtime(format!(+ "ConvNeXt depthwise bias mismatch at 'dwconv.biases': expected {}, got {}",+ depthwise_conv.cout,+ depthwise_conv.bias.shape()[0]+ )));+ }+ if depthwise_conv.cout == 0 || depthwise_conv.kernel_size == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ if depthwise_conv.cin != depthwise_conv.cout {+ return Err(AudioError::Runtime(format!(+ "ConvNeXt depthwise conv expects cin==cout, got {} vs {} at 'dwconv.weights'",+ depthwise_conv.cin, depthwise_conv.cout+ )));+ }+ let channels = depthwise_conv.cout;+ let norm = read_norm_layer::<B>(+ context,+ &tree.subtree("norm")?,+ data_type,+ channels,+ norm_config.epsilon,+ norm_config.subtract_mean,+ norm_config.use_bias,+ "structured_audio_convnext_norm",+ )?;+ let (pwconv1_shape, _) = read_float_array::<B, 2>(&tree.subtree("pwconv1")?, "weights", data_type)?;+ if pwconv1_shape[1] != channels {+ return Err(AudioError::Runtime(format!(+ "ConvNeXt pwconv1 input mismatch at 'pwconv1.weights': expected {}, got {}",+ channels, pwconv1_shape[1]+ )));+ }+ let pwconv1 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv1")?, data_type, pwconv1_shape[0], channels)?;+ let pwconv2 = read_pointwise_conv_layer::<B>(&tree.subtree("pwconv2")?, data_type, channels, pwconv1_shape[0])?;+ Ok(StructuredAudioConvNeXt {+ depthwise_conv,+ norm,+ pwconv1,+ pwconv2,+ })+}++pub(super) fn read_residual_unit_layer<B: Backend>(+ tree: &BackendParameterTree<'_, B>,+ data_type: DataType,+ dilation: usize,+) -> AudioResult<StructuredAudioResidualUnit<B>> {+ let snake1_alpha = {+ let snake1_tree = tree.subtree("snake1")?;+ let (shape, alpha) = read_float_array::<B, 1>(&snake1_tree, "alpha", data_type)?;+ if shape[0] == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ alpha+ };+ let conv1 = read_conv1d_layer::<B>(&tree.subtree("conv1")?, data_type, dilation, 1)?;+ let snake2_alpha = {+ let snake2_tree = tree.subtree("snake2")?;+ let (shape, alpha) = read_float_array::<B, 1>(&snake2_tree, "alpha", data_type)?;+ if shape[0] != conv1.cout {+ return Err(AudioError::Runtime(format!(+ "residual snake2 alpha mismatch: expected {}, got {}",+ conv1.cout, shape[0]+ )));+ }+ alpha+ };+ if snake1_alpha.shape()[0] != conv1.cin {+ return Err(AudioError::Runtime(format!(+ "residual snake1 alpha mismatch: expected {}, got {}",+ conv1.cin,+ snake1_alpha.shape()[0]+ )));+ }+ let conv2 = read_conv1d_layer::<B>(&tree.subtree("conv2")?, data_type, 1, 1)?;+ Ok(StructuredAudioResidualUnit {+ snake1_alpha,+ conv1,+ snake2_alpha,+ conv2,+ })+}++pub(super) fn build_vocoder_graph_from_tree<B: Backend>(+ context: &Rc<B::Context>,+ root: &BackendParameterTree<'_, B>,+ config: &DescriptAudioCodecConfig,+ data_type: DataType,+) -> AudioResult<StructuredAudioDecoderGraph<B>> {+ let audio_decoder_tree = root.subtree("audio_decoder")?;+ let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;+ let decoder_tree = audio_decoder_tree.subtree("decoder")?;++ let first_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("first_conv")?, data_type, 1, 1)?;+ let final_conv = read_conv1d_layer::<B>(&decoder_tree.subtree("final_conv")?, data_type, 1, 1)?;+ let final_snake_alpha =+ read_float_vector_exact::<B>(&decoder_tree.subtree("final_snake")?, "alpha", final_conv.cin, data_type)?;++ let mut upsample_blocks = Vec::with_capacity(config.downsample_factor.len());+ for (index, &stride) in config.downsample_factor.iter().rev().enumerate() {+ let block_tree = quantizer_tree.subtree("upsampler")?.subtree("blocks")?.subtree(&index.to_string())?;+ let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;+ let convnext = read_convnext_layer::<B>(+ context,+ &block_tree.subtree("convnext")?,+ data_type,+ &config.quantizer_config.upsampler_config.block_configs[index].convnext_config.norm_config,+ )?;+ if convnext.depthwise_conv.cin != trans_conv.cout {+ return Err(AudioError::Runtime(format!(+ "structured audio upsampler convnext channel mismatch at block {index}: trans_conv out {} vs convnext in {}",+ trans_conv.cout, convnext.depthwise_conv.cin+ )));+ }+ upsample_blocks.push((trans_conv, convnext));+ }++ let mut decoder_blocks = Vec::with_capacity(config.decoder_rates.len());+ for (index, &stride) in config.decoder_rates.iter().enumerate() {+ let block_tree = decoder_tree.subtree("decoder_blocks")?.subtree(&index.to_string())?;+ let trans_conv = read_conv_transpose1d_layer::<B>(&block_tree.subtree("trans_conv")?, data_type, stride, 1)?;+ let snake_alpha =+ read_float_vector_exact::<B>(&block_tree.subtree("snake")?, "alpha", trans_conv.cin, data_type)?;+ let channels = trans_conv.cout;+ let res_unit1 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit1")?, data_type, 1)?;+ let res_unit2 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit2")?, data_type, 3)?;+ let res_unit3 = read_residual_unit_layer::<B>(&block_tree.subtree("res_unit3")?, data_type, 9)?;+ if snake_alpha.shape()[0] != trans_conv.cin+ || res_unit1.conv1.cin != channels+ || res_unit2.conv1.cin != channels+ || res_unit3.conv1.cin != channels+ {+ return Err(AudioError::Runtime(format!(+ "structured audio decoder block {index} channel mismatch in exported weights"+ )));+ }+ decoder_blocks.push(StructuredAudioDecoderBlock {+ snake_alpha,+ trans_conv,+ res_unit1,+ res_unit2,+ res_unit3,+ });+ }++ Ok(StructuredAudioDecoderGraph {+ first_conv,+ upsample_blocks,+ decoder_blocks,+ final_snake_alpha,+ final_conv,+ })+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rsnew file mode 100644index 000000000..f07975c82--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/context.rs@@ -0,0 +1,206 @@+use super::*;++impl StructuredAudioCodecGraph {+ fn conv1d_input_context<B: Backend>(layer: &StructuredAudioConv1d<B>) -> AudioResult<usize> {+ layer+ .kernel_size+ .checked_sub(1)+ .and_then(|value| value.checked_mul(layer.dilation))+ .ok_or(AudioError::Runtime("conv1d streaming context overflow".to_string()))+ }++ fn convtranspose_input_context<B: Backend>(+ output_context: usize,+ layer: &StructuredAudioConvTranspose1d<B>,+ ) -> AudioResult<usize> {+ let kernel_minus_one = layer.kernel_size.saturating_sub(1);+ let numerator = output_context+ .checked_add(kernel_minus_one)+ .ok_or(AudioError::Runtime("transpose streaming context overflow".to_string()))?;+ checked_div_ceil(numerator, layer.stride)+ }++ fn residual_unit_input_context<B: Backend>(+ output_context: usize,+ layer: &StructuredAudioResidualUnit<B>,+ ) -> AudioResult<usize> {+ let conv2 = Self::conv1d_input_context(&layer.conv2)?;+ let conv1 = Self::conv1d_input_context(&layer.conv1)?;+ output_context+ .checked_add(conv2)+ .and_then(|value| value.checked_add(conv1))+ .ok_or(AudioError::Runtime("residual-unit streaming context overflow".to_string()))+ }++ fn convnext_input_context<B: Backend>(+ output_context: usize,+ layer: &StructuredAudioConvNeXt<B>,+ ) -> AudioResult<usize> {+ output_context+ .checked_add(Self::conv1d_input_context(&layer.depthwise_conv)?)+ .ok_or(AudioError::Runtime("convnext streaming context overflow".to_string()))+ }++ fn decoder_block_input_context<B: Backend>(+ output_context: usize,+ layer: &StructuredAudioDecoderBlock<B>,+ ) -> AudioResult<usize> {+ let after_res3 = Self::residual_unit_input_context(output_context, &layer.res_unit3)?;+ let after_res2 = Self::residual_unit_input_context(after_res3, &layer.res_unit2)?;+ let after_res1 = Self::residual_unit_input_context(after_res2, &layer.res_unit1)?;+ Self::convtranspose_input_context(after_res1, &layer.trans_conv)+ }++ fn streaming_vocoder_context_frames<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ ) -> AudioResult<usize> {+ let vocoder = self.vocoder_graph(resources)?;+ let mut required = Self::conv1d_input_context(&vocoder.final_conv)?;++ for block in vocoder.decoder_blocks.iter().rev() {+ required = Self::decoder_block_input_context(required, block)?;+ }++ required = required+ .checked_add(Self::conv1d_input_context(&vocoder.first_conv)?)+ .ok_or(AudioError::Runtime("decoder-first streaming context overflow".to_string()))?;++ for (trans_conv, convnext) in vocoder.upsample_blocks.iter().rev() {+ required = Self::convnext_input_context(required, convnext)?;+ required = Self::convtranspose_input_context(required, trans_conv)?;+ }++ Ok(required)+ }++ fn post_module_streaming_context_frames(&self) -> Option<usize> {+ let mut context = 0usize;+ for layer in &self.config.quantizer_config.post_module_config.layer_configs {+ let window = layer.mixer_config.sliding_window_size()?;+ context = context.max(window.saturating_sub(1));+ }+ Some(context)+ }++ pub(in crate::audio::nanocodec::runtime) fn streaming_decode_context_frames<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ ) -> AudioResult<Option<usize>> {+ let vocoder_context = self.streaming_vocoder_context_frames(resources)?;+ let Some(post_module_context) = self.post_module_streaming_context_frames() else {+ return Ok(None);+ };+ Ok(Some(vocoder_context.max(post_module_context)))+ }++ pub(super) fn decode_quantizer_to_nsc_array_enqueued<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ context: &Rc<B::Context>,+ encoder: &mut Encoder<B>,+ tokens: &[u32],+ lengths: &[usize],+ batch_size: usize,+ codebooks: usize,+ frames: usize,+ ) -> AudioResult<Array<B>> {+ if codebooks != self.total_codebooks {+ return Err(AudioError::Runtime(format!(+ "structured audio codebook mismatch: expected {}, got {codebooks}",+ self.total_codebooks+ )));+ }+ let expected_tokens = checked_product(&[batch_size, codebooks, frames])?;+ if tokens.len() != expected_tokens {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens,+ actual_tokens: tokens.len(),+ });+ }+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }+ for &length in lengths {+ if length > frames {+ return Err(AudioError::InvalidTokenLengthValue {+ length,+ frames,+ });+ }+ }+ let quantizer_resources = self.quantizer_resources(resources)?;+ if quantizer_resources.residual_quantizers + 1 != codebooks {+ return Err(AudioError::Runtime(format!(+ "structured audio residual quantizer count mismatch: expected {}, got {}",+ codebooks.saturating_sub(1),+ quantizer_resources.residual_quantizers+ )));+ }++ let lengths_i32 = convert_lengths_to_i32(lengths, frames)?;++ let mut tokens_array = resources.token_staging(batch_size * codebooks * frames);+ tokens_array.as_slice_mut::<u32>()[..tokens.len()].copy_from_slice(tokens);+ let mut lengths_array = resources.length_staging(batch_size);+ lengths_array.as_slice_mut::<i32>()[..lengths_i32.len()].copy_from_slice(&lengths_i32);++ let output = context.create_array(+ &[batch_size, frames, self.input_dim],+ quantizer_resources.data_type,+ "structured_audio_quantizer_output_nsc",+ );+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;+ let codebooks_i32 = usize_to_i32(codebooks, "codebooks")?;+ let frames_i32 = usize_to_i32(frames, "frames")?;+ let input_dim_i32 = usize_to_i32(self.input_dim, "input_dim")?;+ let codebook_dim_i32 = usize_to_i32(quantizer_resources.codebook_dim, "codebook_dim")?;+ let residual_quantizers_i32 = usize_to_i32(quantizer_resources.residual_quantizers, "residual_quantizers")?;+ let semantic_cardinality_i32 = usize_to_i32(quantizer_resources.semantic_cardinality, "semantic_cardinality")?;+ let residual_cardinality_i32 = usize_to_i32(quantizer_resources.residual_cardinality, "residual_cardinality")?;++ let tokens_buffer = tokens_array.buffer();+ let tokens_buffer = tokens_buffer.borrow();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();+ let semantic_codebook_buffer = quantizer_resources.semantic_codebook.buffer();+ let semantic_codebook_buffer = semantic_codebook_buffer.borrow();+ let semantic_out_proj_buffer = quantizer_resources.semantic_out_proj.buffer();+ let semantic_out_proj_buffer = semantic_out_proj_buffer.borrow();+ let semantic_out_bias_buffer = quantizer_resources.semantic_out_bias.buffer();+ let semantic_out_bias_buffer = semantic_out_bias_buffer.borrow();+ let residual_codebooks_buffer = quantizer_resources.residual_codebooks.buffer();+ let residual_codebooks_buffer = residual_codebooks_buffer.borrow();+ let residual_out_proj_buffer = quantizer_resources.residual_out_proj.buffer();+ let residual_out_proj_buffer = residual_out_proj_buffer.borrow();+ let residual_out_bias_buffer = quantizer_resources.residual_out_bias.buffer();+ let residual_out_bias_buffer = residual_out_bias_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ quantizer_resources.kernel.encode(+ &*tokens_buffer,+ &*lengths_buffer,+ &*semantic_codebook_buffer,+ &*semantic_out_proj_buffer,+ &*semantic_out_bias_buffer,+ &*residual_codebooks_buffer,+ &*residual_out_proj_buffer,+ &*residual_out_bias_buffer,+ &mut *output_buffer,+ batch_i32,+ codebooks_i32,+ frames_i32,+ input_dim_i32,+ codebook_dim_i32,+ residual_quantizers_i32,+ semantic_cardinality_i32,+ residual_cardinality_i32,+ encoder,+ );++ Ok(output)+ }+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rsnew file mode 100644index 000000000..494ab745b--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/decode.rs@@ -0,0 +1,386 @@+use std::sync::mpsc::channel;++use super::*;+use crate::audio::nanocodec::runtime::profile::SubmittedDecodedPaddedAudio;++impl StructuredAudioCodecGraph {+ fn run_residual_unit_enqueued<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ unit: &StructuredAudioResidualUnit<B>,+ lengths: &[i32],+ lengths_array: &Array<B>,+ batch_size: usize,+ channels: usize,+ seq_len: usize,+ kernels: &StructuredAudioKernelCache<B>,+ ) -> AudioResult<Array<B>> {+ let ws = &resources.decode_workspace;+ let ctx = resources.context();+ let data_type = input.data_type();++ let residual = input.clone();+ let x = snake1d_enqueue(+ encoder,+ input,+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake1"),+ &unit.snake1_alpha,+ batch_size,+ channels,+ seq_len,+ kernels,+ )?;+ let x = causal_conv1d_grouped_enqueue(+ encoder,+ &x,+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv1"),+ &unit.conv1,+ SequenceLayout::Ncs,+ lengths,+ lengths_array,+ batch_size,+ seq_len,+ kernels,+ )?;+ let x = snake1d_enqueue(+ encoder,+ &x,+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_snake2"),+ &unit.snake2_alpha,+ batch_size,+ channels,+ seq_len,+ kernels,+ )?;+ causal_conv1d_grouped_residual_enqueue(+ encoder,+ &x,+ &residual,+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "res_conv2"),+ &unit.conv2,+ lengths,+ lengths_array,+ batch_size,+ seq_len,+ kernels,+ )+ }++ pub(in crate::audio::nanocodec::runtime) fn submit_decode_padded<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ tokens: &[u32],+ lengths: &[usize],+ batch_size: usize,+ codebooks: usize,+ frames: usize,+ ) -> AudioResult<SubmittedDecodedPaddedAudio<B>> {+ if batch_size == 0 || frames == 0 {+ let out_lengths = lengths+ .iter()+ .map(|&length| {+ length+ .checked_mul(self.upsample_factor)+ .ok_or(AudioError::Runtime("structured audio length scaling overflow".to_string()))+ })+ .collect::<AudioResult<Vec<_>>>()?;+ let context = resources.context().clone();+ return Ok(SubmittedDecodedPaddedAudio {+ output: context.create_array(&[0], DataType::F32, "structured_audio_empty_decode_output"),+ channels: 1,+ frames: out_lengths.iter().copied().max().unwrap_or(0),+ lengths: out_lengths,+ final_command_buffer: None,+ completion_notification: None,+ });+ }++ let mut lengths_i32 = lengths+ .iter()+ .map(|&length| {+ i32::try_from(length)+ .map_err(|_| AudioError::Runtime("structured audio length exceeds i32 range".to_string()))+ })+ .collect::<AudioResult<Vec<_>>>()?;+ let context = resources.context().clone();+ let mut encoder = Encoder::new(context.as_ref()).map_err(|err| {+ AudioError::Runtime(format!("failed to create structured audio decode command buffer: {err}"))+ })?;++ let ws = &resources.decode_workspace;++ let mut x;+ let mut x_layout = SequenceLayout::Nsc;+ let quantized_nsc = self.decode_quantizer_to_nsc_array_enqueued(+ resources,+ &context,+ &mut encoder,+ tokens,+ lengths,+ batch_size,+ codebooks,+ frames,+ )?;+ let _ = codebooks;+ x = self.apply_post_module_enqueued(+ resources,+ &context,+ &mut encoder,+ &quantized_nsc,+ lengths,+ batch_size,+ frames,+ )?;++ let vocoder_graph = self.vocoder_graph(resources)?;+ let kernels = resources.kernels(self.vocoder_data_type)?;+ let mut current_channels = self.input_dim;+ let mut current_frames = frames;+ let mut next_lengths_i32 = vec![0_i32; lengths_i32.len()];+ let mut lengths_array = ws.lengths_array(&context, 0, lengths_i32.len());+ lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths_i32);+ let mut next_lengths_array = ws.lengths_array(&context, 1, lengths_i32.len());++ let data_type = x.data_type();++ for (block_index, (trans_conv, convnext)) in vocoder_graph.upsample_blocks.iter().enumerate() {+ if trans_conv.cin != current_channels {+ return Err(AudioError::Runtime(format!(+ "structured audio upsampler input channel mismatch: expected {}, got {}",+ trans_conv.cin, current_channels+ )));+ }+ let next_frames = current_frames+ .checked_mul(trans_conv.stride)+ .ok_or(AudioError::Runtime("structured audio upsampler frame overflow".to_string()))?;+ scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, trans_conv.stride)?;+ next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);++ x = causal_conv_transpose1d_causal_pad_enqueue(+ &mut encoder,+ &x,+ ws.next_scratch(&context, &[batch_size, trans_conv.cout, next_frames], data_type, "up_tconv"),+ trans_conv,+ &next_lengths_i32,+ batch_size,+ current_frames,+ next_frames,+ x_layout,+ &next_lengths_array,+ &kernels,+ )+ .map_err(|err| {+ AudioError::Runtime(format!(+ "structured audio upsample block {block_index} transpose_conv failed: {err} (x_len={}, batch_size={}, cin={}, seq_len_in={}, seq_len_out={})",+ x.num_elements(),+ batch_size,+ trans_conv.cin,+ current_frames,+ next_frames+ ))+ })?;+ x = self+ .apply_convnext_ncs_enqueued(+ resources,+ &mut encoder,+ &x,+ convnext,+ &next_lengths_i32,+ &next_lengths_array,+ batch_size,+ trans_conv.cout,+ next_frames,+ &kernels,+ )+ .map_err(|err| {+ AudioError::Runtime(format!("structured audio upsample block {block_index} convnext failed: {err}"))+ })?;++ x_layout = SequenceLayout::Ncs;+ current_frames = next_frames;+ current_channels = trans_conv.cout;+ std::mem::swap(&mut lengths_i32, &mut next_lengths_i32);+ std::mem::swap(&mut lengths_array, &mut next_lengths_array);+ }++ if vocoder_graph.first_conv.cin != current_channels {+ return Err(AudioError::Runtime(format!(+ "structured audio decoder input channels mismatch: expected {}, got {}",+ vocoder_graph.first_conv.cin, current_channels+ )));+ }+ x = causal_conv1d_grouped_enqueue(+ &mut encoder,+ &x,+ ws.next_scratch(+ &context,+ &[batch_size, vocoder_graph.first_conv.cout, current_frames],+ data_type,+ "dec_first_conv",+ ),+ &vocoder_graph.first_conv,+ x_layout,+ &lengths_i32,+ &lengths_array,+ batch_size,+ current_frames,+ &kernels,+ )?;+ current_channels = vocoder_graph.first_conv.cout;++ for block in vocoder_graph.decoder_blocks.iter() {+ if block.trans_conv.cin != current_channels {+ return Err(AudioError::Runtime(format!(+ "structured audio decoder block input mismatch: expected {}, got {}",+ block.trans_conv.cin, current_channels+ )));+ }+ x = snake1d_enqueue(+ &mut encoder,+ &x,+ ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "dec_snake"),+ &block.snake_alpha,+ batch_size,+ current_channels,+ current_frames,+ &kernels,+ )?;++ let next_frames = current_frames+ .checked_mul(block.trans_conv.stride)+ .ok_or(AudioError::Runtime("structured audio decoder frame overflow".to_string()))?;+ scale_lengths_i32_in_place(&lengths_i32, &mut next_lengths_i32, block.trans_conv.stride)?;+ next_lengths_array.as_slice_mut::<i32>().copy_from_slice(&next_lengths_i32);++ x = causal_conv_transpose1d_causal_pad_enqueue(+ &mut encoder,+ &x,+ ws.next_scratch(&context, &[batch_size, block.trans_conv.cout, next_frames], data_type, "dec_tconv"),+ &block.trans_conv,+ &next_lengths_i32,+ batch_size,+ current_frames,+ next_frames,+ SequenceLayout::Ncs,+ &next_lengths_array,+ &kernels,+ )?;++ current_frames = next_frames;+ current_channels = block.trans_conv.cout;+ std::mem::swap(&mut lengths_i32, &mut next_lengths_i32);+ std::mem::swap(&mut lengths_array, &mut next_lengths_array);++ x = self.run_residual_unit_enqueued(+ resources,+ &mut encoder,+ &x,+ &block.res_unit1,+ &lengths_i32,+ &lengths_array,+ batch_size,+ current_channels,+ current_frames,+ &kernels,+ )?;+ x = self.run_residual_unit_enqueued(+ resources,+ &mut encoder,+ &x,+ &block.res_unit2,+ &lengths_i32,+ &lengths_array,+ batch_size,+ current_channels,+ current_frames,+ &kernels,+ )?;+ x = self.run_residual_unit_enqueued(+ resources,+ &mut encoder,+ &x,+ &block.res_unit3,+ &lengths_i32,+ &lengths_array,+ batch_size,+ current_channels,+ current_frames,+ &kernels,+ )?;+ }++ x = snake1d_enqueue(+ &mut encoder,+ &x,+ ws.next_scratch(&context, &[batch_size, current_channels, current_frames], data_type, "final_snake"),+ &vocoder_graph.final_snake_alpha,+ batch_size,+ current_channels,+ current_frames,+ &kernels,+ )?;+ x = causal_conv1d_grouped_enqueue(+ &mut encoder,+ &x,+ ws.next_scratch(+ &context,+ &[batch_size, vocoder_graph.final_conv.cout, current_frames],+ data_type,+ "final_conv",+ ),+ &vocoder_graph.final_conv,+ SequenceLayout::Ncs,+ &lengths_i32,+ &lengths_array,+ batch_size,+ current_frames,+ &kernels,+ )?;+ // The final output is returned and held by the caller, so allocate a+ // fresh array instead of using a scratch buffer.+ x = tanh_enqueue(+ &mut encoder,+ &x,+ context.create_array(x.shape(), data_type, "structured_audio_tanh_output"),+ &kernels,+ )?;++ let (completion_sender, completion_notification) = channel();+ encoder.add_completion_handler({+ move |_| {+ let _ = completion_sender.send(());+ }+ });+ let final_command_buffer = Some(encoder.end_encoding().submit());+ let out_lengths = lengths_i32+ .into_iter()+ .map(|length| {+ usize::try_from(length).map_err(|_| {+ AudioError::Runtime("structured audio decoder produced invalid negative length".to_string())+ })+ })+ .collect::<AudioResult<Vec<_>>>()?;+ Ok(SubmittedDecodedPaddedAudio {+ output: x,+ channels: vocoder_graph.final_conv.cout,+ frames: current_frames,+ lengths: out_lengths,+ final_command_buffer,+ completion_notification: Some(completion_notification),+ })+ }++ pub(in crate::audio::nanocodec::runtime) fn decode_padded<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ tokens: &[u32],+ lengths: &[usize],+ batch_size: usize,+ codebooks: usize,+ frames: usize,+ ) -> AudioResult<DecodedPaddedAudio> {+ self.submit_decode_padded(resources, tokens, lengths, batch_size, codebooks, frames)?.resolve()+ }+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/enqueue.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/enqueue.rsnew file mode 100644index 000000000..5f0051723--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/enqueue.rs@@ -0,0 +1,565 @@+use super::*;+use crate::backends::common::gpu_types::ActivationType;++pub(super) fn snake1d_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ output: Array<B>,+ alpha: &Array<B>,+ batch_size: usize,+ channels: usize,+ seq_len: usize,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ let expected_input = checked_product(&[batch_size, channels, seq_len])?;+ if input.num_elements() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.num_elements(),+ });+ }+ if alpha.shape() != [channels] {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: channels,+ actual_tokens: alpha.num_elements(),+ });+ }++ let data_type = input.data_type();+ if alpha.data_type() != data_type {+ return Err(AudioError::Runtime(format!(+ "snake alpha dtype mismatch: expected {:?}, got {:?}",+ data_type,+ alpha.data_type()+ )));+ }++ let channels_i32 = usize_to_i32(channels, "channels")?;+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let alpha_buffer = alpha.buffer();+ let alpha_buffer = alpha_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ kernels.half_snake.encode(+ &*input_buffer,+ &*alpha_buffer,+ &mut *output_buffer,+ channels_i32,+ seq_len_i32,+ channels_i32,+ 0.0,+ 1e-9,+ batch_i32,+ encoder,+ );++ Ok(output)+}++fn validate_grouped_conv_params<B: Backend>(+ input: &Array<B>,+ layer: &StructuredAudioConv1d<B>,+ lengths: &[i32],+ batch_size: usize,+ seq_len: usize,+) -> AudioResult<()> {+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }+ if layer.groups == 0 || layer.cin % layer.groups != 0 || layer.cout % layer.groups != 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];+ if layer.weight.shape() != expected_weight_shape {+ return Err(AudioError::Runtime(format!(+ "causal conv1d weight shape mismatch: expected {:?}, got {:?}",+ expected_weight_shape,+ layer.weight.shape()+ )));+ }+ if layer.bias.shape() != [layer.cout] {+ return Err(AudioError::Runtime(format!(+ "causal conv1d bias shape mismatch: expected [{}], got {:?}",+ layer.cout,+ layer.bias.shape()+ )));+ }+ let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;+ if input.num_elements() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.num_elements(),+ });+ }+ let data_type = input.data_type();+ if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {+ return Err(AudioError::Runtime("causal conv1d dtype mismatch".to_string()));+ }+ Ok(())+}++pub(super) fn causal_conv1d_grouped_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ output: Array<B>,+ layer: &StructuredAudioConv1d<B>,+ input_layout: SequenceLayout,+ lengths: &[i32],+ lengths_array: &Array<B>,+ batch_size: usize,+ seq_len: usize,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;++ let cin_i32 = usize_to_i32(layer.cin, "cin")?;+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;+ let kernel_size_i32 = usize_to_i32(layer.kernel_size, "kernel_size")?;+ let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;+ let input_layout_i32 = input_layout.as_i32();+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;+ if layer.groups == 1 {+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let weight_buffer = layer.weight.buffer();+ let weight_buffer = weight_buffer.borrow();+ let bias_buffer = layer.bias.buffer();+ let bias_buffer = bias_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();+ kernels.causal_conv1d.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin_i32,+ cout_i32,+ seq_len_i32,+ kernel_size_i32,+ dilation_i32,+ input_layout_i32,+ batch_i32,+ encoder,+ );+ } else {+ let groups_i32 = usize_to_i32(layer.groups, "groups")?;+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let weight_buffer = layer.weight.buffer();+ let weight_buffer = weight_buffer.borrow();+ let bias_buffer = layer.bias.buffer();+ let bias_buffer = bias_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();+ kernels.causal_conv1d_grouped.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin_i32,+ cout_i32,+ seq_len_i32,+ kernel_size_i32,+ dilation_i32,+ groups_i32,+ input_layout_i32,+ batch_i32,+ encoder,+ );+ }++ Ok(output)+}++pub(super) fn causal_conv1d_grouped_residual_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ residual: &Array<B>,+ output: Array<B>,+ layer: &StructuredAudioConv1d<B>,+ lengths: &[i32],+ lengths_array: &Array<B>,+ batch_size: usize,+ seq_len: usize,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ validate_grouped_conv_params(input, layer, lengths, batch_size, seq_len)?;+ let expected_residual = checked_product(&[batch_size, layer.cout, seq_len])?;+ if residual.num_elements() != expected_residual {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_residual,+ actual_tokens: residual.num_elements(),+ });+ }+ let cin_i32 = usize_to_i32(layer.cin, "cin")?;+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;+ let kernel_size_i32 = usize_to_i32(layer.kernel_size, "kernel_size")?;+ let dilation_i32 = usize_to_i32(layer.dilation, "dilation")?;+ let groups_i32 = usize_to_i32(layer.groups, "groups")?;+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;+ let data_type = input.data_type();+ if residual.data_type() != data_type {+ return Err(AudioError::Runtime("causal conv1d residual dtype mismatch".to_string()));+ }++ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let residual_buffer = residual.buffer();+ let residual_buffer = residual_buffer.borrow();+ let weight_buffer = layer.weight.buffer();+ let weight_buffer = weight_buffer.borrow();+ let bias_buffer = layer.bias.buffer();+ let bias_buffer = bias_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();+ kernels.causal_conv1d_grouped_residual.encode(+ &*input_buffer,+ &*residual_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin_i32,+ cout_i32,+ seq_len_i32,+ kernel_size_i32,+ dilation_i32,+ groups_i32,+ batch_i32,+ encoder,+ );++ Ok(output)+}++pub(super) fn causal_conv_transpose1d_causal_pad_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ output: Array<B>,+ layer: &StructuredAudioConvTranspose1d<B>,+ lengths: &[i32],+ batch_size: usize,+ seq_len_in: usize,+ seq_len_out: usize,+ input_layout: SequenceLayout,+ lengths_array: &Array<B>,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }+ let expected_input = checked_product(&[batch_size, layer.cin, seq_len_in])?;+ if input.num_elements() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.num_elements(),+ });+ }+ if layer.groups == 0 || layer.cin % layer.groups != 0 || layer.cout % layer.groups != 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ let expected_weight_shape = [layer.cout, layer.cin / layer.groups, layer.kernel_size];+ if layer.weight.shape() != expected_weight_shape {+ return Err(AudioError::Runtime(format!(+ "causal transpose weight shape mismatch: expected {:?}, got {:?}",+ expected_weight_shape,+ layer.weight.shape()+ )));+ }+ if layer.bias.shape() != [layer.cout] {+ return Err(AudioError::Runtime(format!(+ "causal transpose bias shape mismatch: expected [{}], got {:?}",+ layer.cout,+ layer.bias.shape()+ )));+ }++ let data_type = input.data_type();+ if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {+ return Err(AudioError::Runtime("causal transpose dtype mismatch".to_string()));+ }++ let cin_i32 = usize_to_i32(layer.cin, "cin")?;+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;+ let seq_in_i32 = usize_to_i32(seq_len_in, "seq_len_in")?;+ let seq_out_i32 = usize_to_i32(seq_len_out, "seq_len_out")?;+ let kernel_size_i32 = usize_to_i32(layer.kernel_size, "kernel_size")?;+ let stride_i32 = usize_to_i32(layer.stride, "stride")?;+ let groups_i32 = usize_to_i32(layer.groups, "groups")?;+ let input_layout_i32 = input_layout.as_i32();+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;++ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let weight_buffer = layer.weight.buffer();+ let weight_buffer = weight_buffer.borrow();+ let bias_buffer = layer.bias.buffer();+ let bias_buffer = bias_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();+ kernels.causal_conv_transpose1d_causal_pad.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin_i32,+ cout_i32,+ seq_in_i32,+ seq_out_i32,+ kernel_size_i32,+ stride_i32,+ groups_i32,+ input_layout_i32,+ batch_i32,+ encoder,+ );++ Ok(output)+}++pub(super) fn conv1d_pointwise_ncs_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ output: Array<B>,+ layer: &StructuredAudioPointwiseConv<B>,+ lengths: &[i32],+ lengths_array: &Array<B>,+ batch_size: usize,+ seq_len: usize,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }+ let expected_input = checked_product(&[batch_size, layer.cin, seq_len])?;+ if input.num_elements() != expected_input {+ return Err(AudioError::Runtime(format!(+ "pointwise conv input shape mismatch: expected {expected_input} elements ([batch={batch_size}, cin={}, seq_len={seq_len}]), got {}",+ layer.cin,+ input.num_elements()+ )));+ }+ if layer.weight.shape() != [layer.cout, layer.cin, 1] {+ return Err(AudioError::Runtime(format!(+ "pointwise conv weight shape mismatch: expected [{}, {}, 1], got {:?}",+ layer.cout,+ layer.cin,+ layer.weight.shape()+ )));+ }+ if layer.bias.shape() != [layer.cout] {+ return Err(AudioError::Runtime(format!(+ "pointwise conv bias shape mismatch: expected [{}], got {:?}",+ layer.cout,+ layer.bias.shape()+ )));+ }++ let data_type = input.data_type();+ if layer.weight.data_type() != data_type || layer.bias.data_type() != data_type {+ return Err(AudioError::Runtime("pointwise conv dtype mismatch".to_string()));+ }++ let cin_i32 = usize_to_i32(layer.cin, "cin")?;+ let cout_i32 = usize_to_i32(layer.cout, "cout")?;+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let weight_buffer = layer.weight.buffer();+ let weight_buffer = weight_buffer.borrow();+ let bias_buffer = layer.bias.buffer();+ let bias_buffer = bias_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();+ kernels.conv1d.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin_i32,+ cout_i32,+ seq_len_i32,+ seq_len_i32,+ 1,+ 1,+ 1,+ 0,+ 0,+ batch_i32,+ encoder,+ );++ Ok(output)+}++pub(super) fn norm_ncs_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ output: Array<B>,+ norm: &StructuredAudioNorm<B>,+ lengths: &[i32],+ lengths_array: &Array<B>,+ batch_size: usize,+ channels: usize,+ seq_len: usize,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }+ if norm.scales.shape() != [channels] {+ return Err(AudioError::Runtime(format!(+ "norm scale shape mismatch: expected [{channels}], got {:?}",+ norm.scales.shape()+ )));+ }+ if norm.bias.shape() != [channels] {+ return Err(AudioError::Runtime(format!(+ "norm bias shape mismatch: expected [{channels}], got {:?}",+ norm.bias.shape()+ )));+ }++ let expected_input = checked_product(&[batch_size, channels, seq_len])?;+ if input.num_elements() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.num_elements(),+ });+ }++ let data_type = input.data_type();+ if norm.scales.data_type() != data_type || norm.bias.data_type() != data_type {+ return Err(AudioError::Runtime("norm dtype mismatch".to_string()));+ }++ let channels_i32 = usize_to_i32(channels, "channels")?;+ let seq_len_i32 = usize_to_i32(seq_len, "seq_len")?;+ let batch_i32 = usize_to_i32(batch_size, "batch_size")?;+ let subtract_mean = if norm.subtract_mean {+ 1+ } else {+ 0+ };+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let scales_buffer = norm.scales.buffer();+ let scales_buffer = scales_buffer.borrow();+ let bias_buffer = norm.bias.buffer();+ let bias_buffer = bias_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ let lengths_buffer = lengths_array.buffer();+ let lengths_buffer = lengths_buffer.borrow();+ kernels.norm_ncs.encode(+ &*input_buffer,+ &*scales_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ channels_i32,+ seq_len_i32,+ norm.epsilon,+ subtract_mean,+ batch_i32,+ encoder,+ );++ Ok(output)+}++pub(super) fn gelu_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ output: Array<B>,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ let n_u32 = u32::try_from(input.num_elements())+ .map_err(|_| AudioError::Runtime("gelu element count exceeds u32 range".to_string()))?;+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::GELU, encoder);+ Ok(output)+}++pub(super) fn add_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ a: &Array<B>,+ b: &Array<B>,+ output: Array<B>,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ if a.num_elements() != b.num_elements() {+ return Err(AudioError::Runtime(format!(+ "elementwise add shape mismatch: {} vs {}",+ a.num_elements(),+ b.num_elements()+ )));+ }+ if a.data_type() != b.data_type() {+ return Err(AudioError::Runtime(format!(+ "elementwise add dtype mismatch: {:?} vs {:?}",+ a.data_type(),+ b.data_type()+ )));+ }+ let n_i32 = usize_to_i32(a.num_elements(), "n")?;+ let a_buffer = a.buffer();+ let a_buffer = a_buffer.borrow();+ let b_buffer = b.buffer();+ let b_buffer = b_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ kernels.add.encode(&*a_buffer, &*b_buffer, &mut *output_buffer, n_i32, encoder);+ Ok(output)+}++pub(super) fn tanh_enqueue<B: Backend>(+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ output: Array<B>,+ kernels: &StructuredAudioKernelCache<B>,+) -> AudioResult<Array<B>> {+ let n_u32 = u32::try_from(input.num_elements())+ .map_err(|_| AudioError::Runtime("tanh element count exceeds u32 range".to_string()))?;+ let input_buffer = input.buffer();+ let input_buffer = input_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ kernels.activation.encode(Some(&*input_buffer), &mut *output_buffer, n_u32, ActivationType::TANH, encoder);+ Ok(output)+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rsnew file mode 100644index 000000000..a30f99533--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/fishaudio.rs@@ -0,0 +1,257 @@+use super::*;++pub(super) fn structured_audio_dtype_key(data_type: DataType) -> u8 {+ match data_type {+ DataType::F16 => 1,+ DataType::BF16 => 2,+ _ => 0,+ }+}++pub(super) struct StructuredAudioPostModuleRuntime<B: Backend> {+ pub(super) context: Rc<B::Context>,+ pub(super) model_shape: ModelShape,+ pub(super) scratch_buffers: ScratchBuffers<B>,+ pub(super) shared_buffers: Rc<RefCell<SharedBuffers<B>>>,+ pub(super) layers: Box<[LayerExecutables<B>]>,+ pub(super) output_norm: RMSNorm<B>,+ pub(super) max_sequence_length: usize,+}++pub(super) struct StructuredAudioKernelCache<B: Backend> {+ pub(super) half_snake: <B::Kernels as Kernels>::AudioHalfSnakeKernel,+ pub(super) causal_conv1d: <B::Kernels as Kernels>::AudioCausalConv1dKernel,+ pub(super) causal_conv1d_grouped: <B::Kernels as Kernels>::AudioCausalConv1dGroupedKernel,+ pub(super) causal_conv1d_grouped_residual: <B::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel,+ pub(super) causal_conv_transpose1d_causal_pad: <B::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel,+ pub(super) conv1d: <B::Kernels as Kernels>::AudioConv1dKernel,+ pub(super) norm_ncs: <B::Kernels as Kernels>::AudioNormNcsKernel,+ pub(super) activation: <B::Kernels as Kernels>::ActivationKernel,+ pub(super) add: <B::Kernels as Kernels>::AudioAddKernel,+}++pub(super) struct FishAudioQuantizerResources<B: Backend> {+ pub(super) data_type: DataType,+ pub(super) codebook_dim: usize,+ pub(super) residual_quantizers: usize,+ pub(super) semantic_cardinality: usize,+ pub(super) residual_cardinality: usize,+ pub(super) kernel: <B::Kernels as Kernels>::AudioQuantizerDecodeKernel,+ pub(super) semantic_codebook: Array<B>,+ pub(super) semantic_out_proj: Array<B>,+ pub(super) semantic_out_bias: Array<B>,+ pub(super) residual_codebooks: Array<B>,+ pub(super) residual_out_proj: Array<B>,+ pub(super) residual_out_bias: Array<B>,+}++/// Ping-pong scratch pool for the sequential decode pipeline.+///+/// At any point in the pipeline at most two data buffers are live (the current+/// input and the current output). `next_scratch` alternates between two slots,+/// re-using a buffer when its `Rc` strong-count shows it is exclusively owned+/// by the pool and it is large enough. When a residual connection keeps an+/// extra reference alive, a fresh buffer is allocated transparently.+pub(super) struct DecodeWorkspace<B: Backend> {+ slots: [RefCell<Option<Rc<RefCell<B::Buffer>>>>; 2],+ toggle: Cell<bool>,+ /// Pair of reusable I32 arrays for per-stage sequence lengths. Cached to+ /// avoid allocating new GPU buffers on every decode call.+ lengths_pair: [RefCell<Option<Array<B>>>; 2],+}++impl<B: Backend> DecodeWorkspace<B> {+ fn new() -> Self {+ Self {+ slots: [RefCell::new(None), RefCell::new(None)],+ toggle: Cell::new(false),+ lengths_pair: [RefCell::new(None), RefCell::new(None)],+ }+ }++ /// Return a reusable I32 array of at least `min_elements` entries for+ /// sequence length data. `index` must be 0 or 1 (ping-pong pair).+ pub(super) fn lengths_array(+ &self,+ context: &B::Context,+ index: usize,+ min_elements: usize,+ ) -> Array<B> {+ debug_assert!(index < 2);+ let mut slot = self.lengths_pair[index].borrow_mut();+ if let Some(existing) = slot.as_ref() {+ if existing.num_elements() >= min_elements {+ return existing.clone();+ }+ }+ let label = if index == 0 {+ "structured_audio_lengths_a"+ } else {+ "structured_audio_lengths_b"+ };+ let array = context.create_array(&[min_elements], DataType::I32, label);+ *slot = Some(array.clone());+ array+ }++ /// Return a scratch [`Array`] backed by the next slot in the ping-pong+ /// pair. The underlying GPU buffer is reused when possible.+ pub(super) fn next_scratch(+ &self,+ context: &B::Context,+ shape: &[usize],+ data_type: DataType,+ label: &str,+ ) -> Array<B> {+ let idx = usize::from(self.toggle.get());+ self.toggle.set(!self.toggle.get());++ let needed = size_for_shape(shape, data_type);+ let mut slot = self.slots[idx].borrow_mut();++ // Try to reuse the existing buffer if nobody else holds a reference+ // and it is large enough.+ if let Some(buf_rc) = slot.as_ref() {+ if Rc::strong_count(buf_rc) == 1 && buf_rc.borrow().length() >= needed {+ let array = unsafe { Array::from_parts(buf_rc.clone(), 0, shape, data_type) };+ return array;+ }+ }++ // Allocate a fresh buffer and cache it.+ let mut buffer = context.create_buffer(needed).expect("Failed to create scratch buffer");+ buffer.set_label(Some(label));+ let buf_rc = Rc::new(RefCell::new(buffer));+ *slot = Some(buf_rc.clone());+ unsafe { Array::from_parts(buf_rc, 0, shape, data_type) }+ }++ /// Clear both scratch slots, forcing the next call to allocate fresh+ /// buffers. This must be called after submitting a command buffer that+ /// references scratch intermediates so that the in-flight GPU work is+ /// not corrupted by a subsequent decode pass reusing the same buffers.+ pub(super) fn reset(&self) {+ *self.slots[0].borrow_mut() = None;+ *self.slots[1].borrow_mut() = None;+ self.toggle.set(false);+ *self.lengths_pair[0].borrow_mut() = None;+ *self.lengths_pair[1].borrow_mut() = None;+ }+}++pub(in crate::audio::nanocodec::runtime) struct StructuredAudioRuntimeResources<B: Backend> {+ context: Rc<B::Context>,+ pub(super) kernels_by_dtype: RefCell<HashMap<u8, Rc<StructuredAudioKernelCache<B>>>>,+ pub(super) post_module_runtime: RefCell<Option<Rc<StructuredAudioPostModuleRuntime<B>>>>,+ pub(super) vocoder_graph: RefCell<Option<Rc<StructuredAudioDecoderGraph<B>>>>,+ pub(super) quantizer_resources: RefCell<Option<Rc<FishAudioQuantizerResources<B>>>>,+ token_staging: RefCell<Option<Array<B>>>,+ length_staging: RefCell<Option<Array<B>>>,+ pub(super) decode_workspace: DecodeWorkspace<B>,+}++impl<B: Backend> StructuredAudioRuntimeResources<B> {+ pub(in crate::audio::nanocodec::runtime) fn new(context: Rc<B::Context>) -> Self {+ Self {+ context,+ kernels_by_dtype: RefCell::new(HashMap::new()),+ post_module_runtime: RefCell::new(None),+ vocoder_graph: RefCell::new(None),+ quantizer_resources: RefCell::new(None),+ token_staging: RefCell::new(None),+ length_staging: RefCell::new(None),+ decode_workspace: DecodeWorkspace::new(),+ }+ }++ pub(super) fn context(&self) -> &Rc<B::Context> {+ &self.context+ }++ /// Reset the decode workspace and staging arrays so that subsequent+ /// decode passes allocate fresh buffers. Call this after submitting a+ /// command buffer whose encoded work references the current buffers.+ pub(in crate::audio::nanocodec::runtime) fn reset_for_pending(&self) {+ self.decode_workspace.reset();+ *self.token_staging.borrow_mut() = None;+ *self.length_staging.borrow_mut() = None;+ }++ pub(super) fn kernels(+ &self,+ data_type: DataType,+ ) -> AudioResult<Rc<StructuredAudioKernelCache<B>>> {+ let key = structured_audio_dtype_key(data_type);+ if let Some(existing) = self.kernels_by_dtype.borrow().get(&key) {+ return Ok(existing.clone());+ }+ let created = Rc::new(build_structured_audio_kernels(&self.context, data_type)?);+ self.kernels_by_dtype.borrow_mut().insert(key, created.clone());+ Ok(created)+ }++ pub(super) fn token_staging(+ &self,+ min_elements: usize,+ ) -> Array<B> {+ let mut slot = self.token_staging.borrow_mut();+ if let Some(existing) = slot.as_ref() {+ if existing.num_elements() >= min_elements {+ return existing.clone();+ }+ }+ let array = self.context.create_array(&[min_elements], DataType::U32, "quantizer_token_staging");+ *slot = Some(array.clone());+ array+ }++ pub(super) fn length_staging(+ &self,+ min_elements: usize,+ ) -> Array<B> {+ let mut slot = self.length_staging.borrow_mut();+ if let Some(existing) = slot.as_ref() {+ if existing.num_elements() >= min_elements {+ return existing.clone();+ }+ }+ let array = self.context.create_array(&[min_elements], DataType::I32, "quantizer_length_staging");+ *slot = Some(array.clone());+ array+ }+}++pub(super) fn build_structured_audio_kernels<B: Backend>(+ context: &Rc<B::Context>,+ data_type: DataType,+) -> AudioResult<StructuredAudioKernelCache<B>> {+ Ok(StructuredAudioKernelCache {+ half_snake: <B::Kernels as Kernels>::AudioHalfSnakeKernel::new(context.as_ref(), data_type)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize snake1d kernel: {err}")))?,+ causal_conv1d: <B::Kernels as Kernels>::AudioCausalConv1dKernel::new(context.as_ref(), data_type)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize causal conv1d kernel: {err}")))?,+ causal_conv1d_grouped: <B::Kernels as Kernels>::AudioCausalConv1dGroupedKernel::new(+ context.as_ref(),+ data_type,+ )+ .map_err(|err| AudioError::Runtime(format!("failed to initialize grouped causal conv1d kernel: {err}")))?,+ causal_conv1d_grouped_residual: <B::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel::new(+ context.as_ref(),+ data_type,+ )+ .map_err(|err| AudioError::Runtime(format!("failed to initialize grouped residual conv1d kernel: {err}")))?,+ causal_conv_transpose1d_causal_pad: <B::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(+ context.as_ref(),+ data_type,+ )+ .map_err(|err| AudioError::Runtime(format!("failed to initialize causal transpose-conv kernel: {err}")))?,+ conv1d: <B::Kernels as Kernels>::AudioConv1dKernel::new(context.as_ref(), data_type)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize conv1d kernel: {err}")))?,+ norm_ncs: <B::Kernels as Kernels>::AudioNormNcsKernel::new(context.as_ref(), data_type)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize norm kernel: {err}")))?,+ activation: <B::Kernels as Kernels>::ActivationKernel::new(context.as_ref(), data_type, false)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize activation kernel: {err}")))?,+ add: <B::Kernels as Kernels>::AudioAddKernel::new(context.as_ref(), data_type)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize add kernel: {err}")))?,+ })+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/mod.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/mod.rsnew file mode 100644index 000000000..1d204b8fd--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/mod.rs@@ -0,0 +1,20 @@+use super::{+ support::{checked_div_ceil, scale_lengths_i32_in_place},+ *,+};++mod build;+mod context;+mod decode;+mod enqueue;+mod fishaudio;+mod post_module;+mod shared;++use build::*;+use enqueue::*;+// Re-export to parent (runtime.rs)+pub(in crate::audio::nanocodec::runtime) use fishaudio::StructuredAudioRuntimeResources;+use fishaudio::*;+pub(super) use shared::StructuredAudioCodecGraph;+use shared::*;diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rsnew file mode 100644index 000000000..054db8d82--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/post_module.rs@@ -0,0 +1,532 @@+use super::*;++impl StructuredAudioCodecGraph {+ pub(super) fn apply_convnext_ncs_enqueued<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ encoder: &mut Encoder<B>,+ input: &Array<B>,+ layer: &StructuredAudioConvNeXt<B>,+ lengths: &[i32],+ lengths_array: &Array<B>,+ batch_size: usize,+ channels: usize,+ seq_len: usize,+ kernels: &StructuredAudioKernelCache<B>,+ ) -> AudioResult<Array<B>> {+ let ws = &resources.decode_workspace;+ let ctx = resources.context();+ let data_type = input.data_type();++ let residual = input.clone();+ let x = causal_conv1d_grouped_enqueue(+ encoder,+ input,+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_dw"),+ &layer.depthwise_conv,+ SequenceLayout::Ncs,+ lengths,+ lengths_array,+ batch_size,+ seq_len,+ kernels,+ )?;+ let x = norm_ncs_enqueue(+ encoder,+ &x,+ ws.next_scratch(ctx, &[batch_size, channels, seq_len], data_type, "cnxt_norm"),+ &layer.norm,+ lengths,+ lengths_array,+ batch_size,+ channels,+ seq_len,+ kernels,+ )?;+ let x = conv1d_pointwise_ncs_enqueue(+ encoder,+ &x,+ ws.next_scratch(ctx, &[batch_size, layer.pwconv1.cout, seq_len], data_type, "cnxt_pw1"),+ &layer.pwconv1,+ lengths,+ lengths_array,+ batch_size,+ seq_len,+ kernels,+ )?;+ let x = gelu_enqueue(encoder, &x, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_gelu"), kernels)?;+ let x = conv1d_pointwise_ncs_enqueue(+ encoder,+ &x,+ ws.next_scratch(ctx, &[batch_size, layer.pwconv2.cout, seq_len], data_type, "cnxt_pw2"),+ &layer.pwconv2,+ lengths,+ lengths_array,+ batch_size,+ seq_len,+ kernels,+ )?;+ add_enqueue(encoder, &x, &residual, ws.next_scratch(ctx, x.shape(), data_type, "cnxt_add"), kernels)+ }++ pub(super) fn build_post_module_runtime<B: Backend>(+ &self,+ context: Rc<B::Context>,+ required_sequence_length: usize,+ ) -> AudioResult<StructuredAudioPostModuleRuntime<B>> {+ let inner_model_config = InnerModelConfig {+ embedding_config: EmbeddingConfig::Untied {+ common: EmbeddingConfigCommon {+ input_scale: None,+ logit_soft_cap: None,+ },+ precision: self.vocoder_data_type.into(),+ },+ transformer_config: self.config.quantizer_config.post_module_config.clone(),+ vocab_size: 1,+ };+ let decoder_config = Rc::new(inner_model_config.to_decoder_config().map_err(|_| {+ AudioError::Runtime("failed to build structured audio post_module decoder config".to_string())+ })?);+ let model_shape = ModelShape::from_decoder_config(&decoder_config);++ let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {+ AudioError::Runtime(format!("failed to open post_module weights '{}': {err}", self.weights_path))+ })?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|err| {+ AudioError::Runtime(format!("failed to load post_module weights '{}': {err}", self.weights_path))+ })?;+ let root_loader_view = loader.tree();+ let transformer_subtree_name = "audio_decoder.quantizer.post_module";+ root_loader_view+ .subtree(transformer_subtree_name)+ .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;++ let max_sequence_length = decoder_config.context_length.max(required_sequence_length.max(1));+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), &decoder_config, &model_shape)));+ {+ let transformer_tree = root_loader_view+ .subtree(transformer_subtree_name)+ .map_err(|err| AudioError::Runtime(format!("missing structured audio post_module subtree: {err}")))?;+ let mut shared_buffers = shared_buffers.borrow_mut();+ if let Some(global_rope) = &mut shared_buffers.global_rope {+ global_rope.update_data(&transformer_tree, "global_rope");+ }+ if let Some(local_rope) = &mut shared_buffers.local_rope {+ local_rope.update_data(&transformer_tree, "local_rope");+ }+ }+ let scratch_buffers = ScratchBuffers::new(+ context.as_ref(),+ &decoder_config,+ &model_shape,+ max_sequence_length,+ max_sequence_length,+ );+ let (layers, output_norm) = Decoder::build_transformer_layers_and_norm(+ context.clone(),+ decoder_config,+ &root_loader_view,+ transformer_subtree_name,+ );++ Ok(StructuredAudioPostModuleRuntime {+ context,+ model_shape,+ scratch_buffers,+ shared_buffers,+ layers,+ output_norm,+ max_sequence_length,+ })+ }++ pub(super) fn post_module_runtime<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ required_sequence_length: usize,+ ) -> AudioResult<Rc<StructuredAudioPostModuleRuntime<B>>> {+ {+ let cached = resources.post_module_runtime.borrow();+ if let Some(runtime) = cached.as_ref() {+ if runtime.max_sequence_length >= required_sequence_length.max(1) {+ return Ok(runtime.clone());+ }+ }+ }+ let runtime = Rc::new(self.build_post_module_runtime(resources.context().clone(), required_sequence_length)?);+ *resources.post_module_runtime.borrow_mut() = Some(runtime.clone());+ Ok(runtime)+ }++ pub(super) fn build_quantizer_resources<B: Backend>(+ &self,+ context: &Rc<B::Context>,+ ) -> AudioResult<FishAudioQuantizerResources<B>> {+ if self.semantic_codebook_size == 0 || self.codebook_size == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ let data_type = self.vocoder_data_type;+ let kernel = <B::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(context.as_ref(), data_type)+ .map_err(|err| AudioError::Runtime(format!("failed to initialize quantizer decode kernel: {err}")))?;+ let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {+ AudioError::Runtime(format!("failed to open structured audio weights '{}': {err}", self.weights_path))+ })?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|err| {+ AudioError::Runtime(format!("failed to load structured audio weights '{}': {err}", self.weights_path))+ })?;+ let root = loader.tree();+ let audio_decoder_tree = root.subtree("audio_decoder")?;+ let quantizer_tree = audio_decoder_tree.subtree("quantizer")?;+ let semantic_tree = quantizer_tree.subtree("semantic_quantizer")?.subtree("quantizers")?.subtree("0")?;+ let semantic_codebook = read_float_matrix_exact::<B>(+ &semantic_tree.subtree("codebook")?,+ "weights",+ self.semantic_codebook_size,+ self.config.codebook_dim,+ data_type,+ )?;+ let codebook_dim = semantic_codebook.shape()[1];+ let semantic_out_proj = read_float_matrix_exact::<B>(+ &semantic_tree.subtree("out_proj")?,+ "weights",+ self.input_dim,+ codebook_dim,+ data_type,+ )?;+ let semantic_out_bias =+ read_float_vector_exact::<B>(&semantic_tree.subtree("out_proj")?, "biases", self.input_dim, data_type)?;++ let residual_quantizers = self.config.n_codebooks;+ let residual_count_for_shape = residual_quantizers.max(1);+ let residual_codebook_rows_for_shape = self.codebook_size.max(1);+ let residual_codebooks = context.create_array(+ &[residual_count_for_shape, residual_codebook_rows_for_shape, codebook_dim],+ data_type,+ "structured_audio_quantizer_residual_codebooks",+ );+ let residual_out_proj = context.create_array(+ &[residual_count_for_shape, self.input_dim, codebook_dim],+ data_type,+ "structured_audio_quantizer_residual_out_proj",+ );+ let residual_out_bias = context.create_array(+ &[residual_count_for_shape, self.input_dim],+ data_type,+ "structured_audio_quantizer_residual_out_bias",+ );++ let residual_root = quantizer_tree.subtree("quantizer")?.subtree("quantizers")?;+ for index in 0..residual_quantizers {+ let quantizer_tree = residual_root.subtree(&index.to_string())?;+ let codebook = read_float_matrix_exact::<B>(+ &quantizer_tree.subtree("codebook")?,+ "weights",+ self.codebook_size,+ codebook_dim,+ data_type,+ )?;+ let out_proj = read_float_matrix_exact::<B>(+ &quantizer_tree.subtree("out_proj")?,+ "weights",+ self.input_dim,+ codebook_dim,+ data_type,+ )?;+ let out_bias = read_float_vector_exact::<B>(+ &quantizer_tree.subtree("out_proj")?,+ "biases",+ self.input_dim,+ data_type,+ )?;++ let mut dst_codebook = outer_axis_view(&residual_codebooks, index, &[self.codebook_size, codebook_dim])?;+ dst_codebook.copy_from_array(&codebook);+ let mut dst_out_proj = outer_axis_view(&residual_out_proj, index, &[self.input_dim, codebook_dim])?;+ dst_out_proj.copy_from_array(&out_proj);+ let mut dst_out_bias = outer_axis_view(&residual_out_bias, index, &[self.input_dim])?;+ dst_out_bias.copy_from_array(&out_bias);+ }++ Ok(FishAudioQuantizerResources {+ data_type,+ codebook_dim,+ residual_quantizers,+ semantic_cardinality: self.semantic_codebook_size,+ residual_cardinality: self.codebook_size,+ kernel,+ semantic_codebook,+ semantic_out_proj,+ semantic_out_bias,+ residual_codebooks,+ residual_out_proj,+ residual_out_bias,+ })+ }++ pub(super) fn quantizer_resources<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ ) -> AudioResult<Rc<FishAudioQuantizerResources<B>>> {+ if let Some(existing) = resources.quantizer_resources.borrow().as_ref() {+ return Ok(existing.clone());+ }+ let created = Rc::new(self.build_quantizer_resources(resources.context())?);+ *resources.quantizer_resources.borrow_mut() = Some(created.clone());+ Ok(created)+ }++ pub(super) fn build_vocoder_graph<B: Backend>(+ &self,+ context: &Rc<B::Context>,+ ) -> AudioResult<StructuredAudioDecoderGraph<B>> {+ let weights_file = File::open(self.weights_path.as_str()).map_err(|err| {+ AudioError::Runtime(format!("failed to open structured audio weights '{}': {err}", self.weights_path))+ })?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|err| {+ AudioError::Runtime(format!("failed to load structured audio weights '{}': {err}", self.weights_path))+ })?;+ let root = loader.tree();+ build_vocoder_graph_from_tree::<B>(context, &root, &self.config, self.vocoder_data_type)+ }++ pub(super) fn vocoder_graph<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ ) -> AudioResult<Rc<StructuredAudioDecoderGraph<B>>> {+ if let Some(existing) = resources.vocoder_graph.borrow().as_ref() {+ return Ok(existing.clone());+ }+ let created = Rc::new(self.build_vocoder_graph(resources.context())?);+ *resources.vocoder_graph.borrow_mut() = Some(created.clone());+ Ok(created)+ }++ pub(super) fn encode_post_module_layers<B: Backend>(+ runtime: &StructuredAudioPostModuleRuntime<B>,+ state: &mut ForwardPassState<B>,+ encoder: &mut Encoder<B>,+ ) -> AudioResult<()> {+ let encoding_parameters = EncodingParameters::new();+ for layer in runtime.layers.iter() {+ layer+ .encode(state, &encoding_parameters, encoder)+ .map_err(|err| AudioError::Runtime(format!("post_module layer encode failed: {err}")))?;+ }+ runtime+ .output_norm+ .encode(state, encoder)+ .map_err(|err| AudioError::Runtime(format!("post_module output norm encode failed: {err}")))?;+ Ok(())+ }++ pub(super) fn apply_post_module_single_batch_enqueued<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ encoder: &mut Encoder<B>,+ latent_nsc: &Array<B>,+ frames: usize,+ ) -> AudioResult<Array<B>> {+ if frames == 0 {+ return Ok(latent_nsc.clone());+ }++ let expected_elements = checked_product(&[frames, self.input_dim])?;+ if latent_nsc.num_elements() != expected_elements {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_elements,+ actual_tokens: latent_nsc.num_elements(),+ });+ }++ let runtime = self.post_module_runtime(resources, frames.max(1))?;+ let token_ids = vec![0_u64; frames];+ let token_positions = (0..frames).collect::<Vec<_>>();+ let mut state = ForwardPassState::new_classifier(+ runtime.context.clone(),+ &runtime.model_shape,+ &runtime.scratch_buffers,+ runtime.shared_buffers.clone(),+ &token_ids,+ &token_positions,+ false,+ 1,+ );++ let main = state.arrays(&[ArrayId::Main])[0].clone();+ let main_output = {+ if main.shape() != [frames, self.input_dim] {+ return Err(AudioError::Runtime(format!(+ "post_module main shape mismatch: expected [{frames}, {}], got {:?}",+ self.input_dim,+ main.shape()+ )));+ }+ if main.data_type() != latent_nsc.data_type() {+ return Err(AudioError::Runtime(format!(+ "post_module dtype mismatch: main={:?}, latent={:?}",+ main.data_type(),+ latent_nsc.data_type()+ )));+ }+ main.clone()+ };++ let copy_bytes = latent_nsc+ .num_elements()+ .checked_mul(latent_nsc.data_type().size_in_bytes())+ .ok_or(AudioError::Runtime("post_module copy size overflow".to_string()))?;+ {+ let latent_buffer = latent_nsc.buffer();+ let main_output_buffer = main_output.buffer();+ let latent_buffer = latent_buffer.borrow();+ let mut main_output_buffer = main_output_buffer.borrow_mut();+ encoder.encode_copy(+ &latent_buffer,+ latent_nsc.offset()..latent_nsc.offset() + copy_bytes,+ &mut main_output_buffer,+ main_output.offset()..main_output.offset() + copy_bytes,+ );+ }+ Self::encode_post_module_layers(&runtime, &mut state, encoder)?;++ Ok(main_output)+ }++ pub(super) fn apply_post_module_enqueued<B: Backend>(+ &self,+ resources: &StructuredAudioRuntimeResources<B>,+ context: &Rc<B::Context>,+ encoder: &mut Encoder<B>,+ latent_nsc: &Array<B>,+ lengths: &[usize],+ batch_size: usize,+ frames: usize,+ ) -> AudioResult<Array<B>> {+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }+ let expected_elements = checked_product(&[batch_size, frames, self.input_dim])?;+ if latent_nsc.num_elements() != expected_elements {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_elements,+ actual_tokens: latent_nsc.num_elements(),+ });+ }+ if batch_size == 1 && lengths.first().copied() == Some(frames) {+ return self.apply_post_module_single_batch_enqueued(resources, encoder, latent_nsc, frames);+ }++ let output = context.create_array(+ &[batch_size, frames, self.input_dim],+ latent_nsc.data_type(),+ "structured_audio_post_module_output_nsc",+ );+ let mut batch_indices_by_length = BTreeMap::<usize, Vec<usize>>::new();+ for (batch_index, &active_len) in lengths.iter().enumerate() {+ if active_len == 0 {+ continue;+ }+ if active_len > frames {+ return Err(AudioError::InvalidTokenLengthValue {+ length: active_len,+ frames,+ });+ }+ batch_indices_by_length.entry(active_len).or_default().push(batch_index);+ }++ if batch_indices_by_length.is_empty() {+ let mut output = output;+ output.copy_from_array(latent_nsc);+ return Ok(output);+ }++ let full_copy_bytes = latent_nsc.size();+ let mut copied_output_prefix = false;+ for (active_len, batch_indices) in batch_indices_by_length {+ let runtime = self.post_module_runtime(resources, active_len.max(1))?;+ let token_ids = vec![0_u64; active_len];+ let token_positions = (0..active_len).collect::<Vec<_>>();+ let mut state = ForwardPassState::new_classifier(+ runtime.context.clone(),+ &runtime.model_shape,+ &runtime.scratch_buffers,+ runtime.shared_buffers.clone(),+ &token_ids,+ &token_positions,+ false,+ 1,+ );++ let main = state.arrays(&[ArrayId::Main])[0].clone();+ let main_output = {+ if main.shape() != [active_len, self.input_dim] {+ return Err(AudioError::Runtime(format!(+ "post_module main shape mismatch: expected [{active_len}, {}], got {:?}",+ self.input_dim,+ main.shape()+ )));+ }+ if main.data_type() != latent_nsc.data_type() {+ return Err(AudioError::Runtime(format!(+ "post_module dtype mismatch: main={:?}, latent={:?}",+ main.data_type(),+ latent_nsc.data_type()+ )));+ }+ main.clone()+ };++ for &batch_index in &batch_indices {+ if !copied_output_prefix {+ {+ let latent_buffer = latent_nsc.buffer();+ let output_buffer = output.buffer();+ let latent_buffer = latent_buffer.borrow();+ let mut output_buffer = output_buffer.borrow_mut();+ encoder.encode_copy(+ &latent_buffer,+ latent_nsc.offset()..latent_nsc.offset() + full_copy_bytes,+ &mut output_buffer,+ output.offset()..output.offset() + full_copy_bytes,+ );+ }+ copied_output_prefix = true;+ }+ let source = array_batch_view(latent_nsc, batch_index, frames, self.input_dim, active_len)?;+ {+ let source_buffer = source.buffer();+ let main_output_buffer = main_output.buffer();+ let source_buffer = source_buffer.borrow();+ let mut main_output_buffer = main_output_buffer.borrow_mut();+ encoder.encode_copy(+ &source_buffer,+ source.offset()..source.offset() + source.size(),+ &mut main_output_buffer,+ main_output.offset()..main_output.offset() + source.size(),+ );+ }+ Self::encode_post_module_layers(&runtime, &mut state, encoder)?;+ let destination = array_batch_view(&output, batch_index, frames, self.input_dim, active_len)?;+ {+ let main_output_buffer = main_output.buffer();+ let destination_buffer = destination.buffer();+ let main_output_buffer = main_output_buffer.borrow();+ let mut destination_buffer = destination_buffer.borrow_mut();+ encoder.encode_copy(+ &main_output_buffer,+ main_output.offset()..main_output.offset() + destination.size(),+ &mut destination_buffer,+ destination.offset()..destination.offset() + destination.size(),+ );+ }+ }+ }++ Ok(output)+ }+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/structured/shared.rs b/crates/uzu/src/audio/nanocodec/runtime/structured/shared.rsnew file mode 100644index 000000000..249ee8573--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/structured/shared.rs@@ -0,0 +1,118 @@+use super::*;++pub(super) fn array_batch_view<B: Backend>(+ array: &Array<B>,+ batch_index: usize,+ frames: usize,+ channels: usize,+ active_frames: usize,+) -> AudioResult<Array<B>> {+ let batch_stride_bytes = size_for_shape(&[frames, channels], array.data_type());+ let batch_offset = batch_index+ .checked_mul(batch_stride_bytes)+ .and_then(|value| value.checked_add(array.offset()))+ .ok_or(AudioError::Runtime("array batch view offset overflow".to_string()))?;+ if active_frames > frames {+ return Err(AudioError::Runtime("array batch view active_frames exceeds frames".to_string()));+ }+ Ok(unsafe { Array::from_parts(array.buffer(), batch_offset, &[active_frames, channels], array.data_type()) })+}++#[derive(Clone, Copy, Debug, PartialEq, Eq)]+pub(super) enum SequenceLayout {+ Ncs,+ Nsc,+}++impl SequenceLayout {+ pub(super) fn as_i32(self) -> i32 {+ match self {+ Self::Ncs => 0,+ Self::Nsc => 1,+ }+ }+}++#[derive(Debug, Clone, PartialEq)]+pub struct StructuredAudioCodecGraph {+ pub(in crate::audio::nanocodec::runtime) config: DescriptAudioCodecConfig,+ pub(in crate::audio::nanocodec::runtime) weights_path: String,+ pub(in crate::audio::nanocodec::runtime) codebook_size: usize,+ pub(in crate::audio::nanocodec::runtime) semantic_codebook_size: usize,+ pub(in crate::audio::nanocodec::runtime) input_dim: usize,+ pub(in crate::audio::nanocodec::runtime) total_codebooks: usize,+ pub(in crate::audio::nanocodec::runtime) upsample_factor: usize,+ pub(in crate::audio::nanocodec::runtime) vocoder_data_type: DataType,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioConv1d<B: Backend> {+ pub(super) weight: Array<B>,+ pub(super) bias: Array<B>,+ pub(super) cin: usize,+ pub(super) cout: usize,+ pub(super) kernel_size: usize,+ pub(super) dilation: usize,+ pub(super) groups: usize,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioConvTranspose1d<B: Backend> {+ pub(super) weight: Array<B>,+ pub(super) bias: Array<B>,+ pub(super) cin: usize,+ pub(super) cout: usize,+ pub(super) kernel_size: usize,+ pub(super) stride: usize,+ pub(super) groups: usize,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioPointwiseConv<B: Backend> {+ pub(super) weight: Array<B>,+ pub(super) bias: Array<B>,+ pub(super) cin: usize,+ pub(super) cout: usize,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioNorm<B: Backend> {+ pub(super) scales: Array<B>,+ pub(super) bias: Array<B>,+ pub(super) epsilon: f32,+ pub(super) subtract_mean: bool,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioConvNeXt<B: Backend> {+ pub(super) depthwise_conv: StructuredAudioConv1d<B>,+ pub(super) norm: StructuredAudioNorm<B>,+ pub(super) pwconv1: StructuredAudioPointwiseConv<B>,+ pub(super) pwconv2: StructuredAudioPointwiseConv<B>,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioResidualUnit<B: Backend> {+ pub(super) snake1_alpha: Array<B>,+ pub(super) conv1: StructuredAudioConv1d<B>,+ pub(super) snake2_alpha: Array<B>,+ pub(super) conv2: StructuredAudioConv1d<B>,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioDecoderBlock<B: Backend> {+ pub(super) snake_alpha: Array<B>,+ pub(super) trans_conv: StructuredAudioConvTranspose1d<B>,+ pub(super) res_unit1: StructuredAudioResidualUnit<B>,+ pub(super) res_unit2: StructuredAudioResidualUnit<B>,+ pub(super) res_unit3: StructuredAudioResidualUnit<B>,+}++#[derive(Debug, Clone)]+pub(super) struct StructuredAudioDecoderGraph<B: Backend> {+ pub(super) first_conv: StructuredAudioConv1d<B>,+ pub(super) upsample_blocks: Vec<(StructuredAudioConvTranspose1d<B>, StructuredAudioConvNeXt<B>)>,+ pub(super) decoder_blocks: Vec<StructuredAudioDecoderBlock<B>>,+ pub(super) final_snake_alpha: Array<B>,+ pub(super) final_conv: StructuredAudioConv1d<B>,+}diff --git a/crates/uzu/src/audio/nanocodec/runtime/support.rs b/crates/uzu/src/audio/nanocodec/runtime/support.rsnew file mode 100644index 000000000..08e8198c6--- /dev/null+++ b/crates/uzu/src/audio/nanocodec/runtime/support.rs@@ -0,0 +1,82 @@+use super::*;++#[derive(Debug, Clone, PartialEq)]+pub struct DecodedPaddedAudio {+ pub samples: Vec<f32>,+ pub channels: usize,+ pub frames: usize,+ pub lengths: Vec<usize>,+}++pub(in crate::audio::nanocodec::runtime) fn checked_product(values: &[usize]) -> AudioResult<usize> {+ values+ .iter()+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))+ .ok_or(AudioError::Runtime("dimension product overflow".to_string()))+}++pub(in crate::audio::nanocodec::runtime) fn usize_to_i32(+ value: usize,+ name: &str,+) -> AudioResult<i32> {+ i32::try_from(value).map_err(|_| AudioError::Runtime(format!("{name} exceeds i32 range")))+}++pub(in crate::audio::nanocodec::runtime) fn convert_lengths_to_i32(+ lengths: &[usize],+ frames: usize,+) -> AudioResult<Vec<i32>> {+ let mut out = Vec::with_capacity(lengths.len());+ for &length in lengths {+ if length > frames {+ return Err(AudioError::InvalidTokenLengthValue {+ length,+ frames,+ });+ }+ out.push(usize_to_i32(length, "length")?);+ }+ Ok(out)+}++pub(in crate::audio::nanocodec::runtime) fn checked_mul_i32(+ value: i32,+ mul: usize,+) -> AudioResult<i32> {+ i32::try_from(mul)+ .map_err(|_| AudioError::Runtime("length scaling factor exceeds i32 range".to_string()))?+ .checked_mul(value)+ .ok_or(AudioError::Runtime("scaled length overflow".to_string()))+}++pub(in crate::audio::nanocodec::runtime) fn checked_div_ceil(+ numerator: usize,+ denominator: usize,+) -> AudioResult<usize> {+ if denominator == 0 {+ return Err(AudioError::Runtime("division by zero".to_string()));+ }+ let addend = denominator.saturating_sub(1);+ numerator+ .checked_add(addend)+ .ok_or(AudioError::Runtime("ceil-division overflow".to_string()))+ .map(|value| value / denominator)+}++pub(in crate::audio::nanocodec::runtime) fn scale_lengths_i32_in_place(+ source: &[i32],+ destination: &mut [i32],+ factor: usize,+) -> AudioResult<()> {+ if destination.len() != source.len() {+ return Err(AudioError::Runtime(format!(+ "scaled length buffer mismatch: expected {}, got {}",+ source.len(),+ destination.len()+ )));+ }+ for (dst, &src) in destination.iter_mut().zip(source.iter()) {+ *dst = checked_mul_i32(src, factor)?;+ }+ Ok(())+}diff --git a/crates/uzu/src/audio/types.rs b/crates/uzu/src/audio/types.rsindex be933455d..a06ef2049 100644--- a/crates/uzu/src/audio/types.rs+++ b/crates/uzu/src/audio/types.rs@@ -1,5 +1,7 @@use std::num::{NonZeroU32, NonZeroUsize};+use crate::{backends::common::Backend, parameters::ParameterLoaderError};+#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)]pub enum AudioError {#[error("sample_rate must be > 0")]@@ -47,11 +49,10 @@ pub enum AudioError {pub type AudioResult<T> = Result<T, AudioError>;-#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]-pub enum AudioTokenPacking {- #[default]- FrameMajor,- CodebookMajor,+impl<B: Backend> From<ParameterLoaderError<B>> for AudioError {+ fn from(value: ParameterLoaderError<B>) -> Self {+ Self::Runtime(value.to_string())+ }}#[derive(Debug, Clone, PartialEq)]@@ -123,7 +124,6 @@ pub struct AudioTokenGrid {codebooks: usize,frames: usize,lengths: Box<[usize]>,- packing: AudioTokenPacking,}impl AudioTokenGrid {@@ -133,7 +133,6 @@ impl AudioTokenGrid {codebooks: usize,frames: usize,lengths: Box<[usize]>,- packing: AudioTokenPacking,) -> AudioResult<Self> {if codebooks == 0 {return Err(AudioError::InvalidTokenCardinality);@@ -173,7 +172,6 @@ impl AudioTokenGrid {codebooks,frames,lengths,- packing,})}@@ -193,16 +191,12 @@ impl AudioTokenGrid {&self.lengths}- pub fn packing(&self) -> AudioTokenPacking {- self.packing- }-pub fn tokens(&self) -> &[u32] {&self.tokens}- pub fn into_parts(self) -> (Box<[u32]>, usize, usize, usize, Box<[usize]>, AudioTokenPacking) {- (self.tokens, self.batch_size, self.codebooks, self.frames, self.lengths, self.packing)+ pub fn into_parts(self) -> (Box<[u32]>, usize, usize, usize, Box<[usize]>) {+ (self.tokens, self.batch_size, self.codebooks, self.frames, self.lengths)}pub fn get(@@ -211,73 +205,18 @@ impl AudioTokenGrid {codebook: usize,frame: usize,) -> u32 {- let index =- Self::token_index(self.packing, self.batch_size, self.codebooks, self.frames, batch, codebook, frame);+ let index = Self::token_index(self.codebooks, self.frames, batch, codebook, frame);self.tokens[index]}- pub fn to_packing(- &self,- packing: AudioTokenPacking,- ) -> Self {- if packing == self.packing {- return self.clone();- }-- let mut tokens = vec![0u32; self.tokens.len()];- for batch in 0..self.batch_size {- for frame in 0..self.frames {- for codebook in 0..self.codebooks {- let src_idx = Self::token_index(- self.packing,- self.batch_size,- self.codebooks,- self.frames,- batch,- codebook,- frame,- );- let dst_idx = Self::token_index(- packing,- self.batch_size,- self.codebooks,- self.frames,- batch,- codebook,- frame,- );- tokens[dst_idx] = self.tokens[src_idx];- }- }- }-- Self {- tokens: tokens.into_boxed_slice(),- batch_size: self.batch_size,- codebooks: self.codebooks,- frames: self.frames,- lengths: self.lengths.clone(),- packing,- }- }-fn token_index(- packing: AudioTokenPacking,- batch_size: usize,codebooks: usize,frames: usize,batch: usize,codebook: usize,frame: usize,) -> usize {- debug_assert!(batch < batch_size);- debug_assert!(codebook < codebooks);- debug_assert!(frame < frames);-- match packing {- AudioTokenPacking::FrameMajor => ((batch * frames + frame) * codebooks) + codebook,- AudioTokenPacking::CodebookMajor => ((batch * codebooks + codebook) * frames) + frame,- }+ ((batch * codebooks + codebook) * frames) + frame}}diff --git a/crates/uzu/src/backends/common/command_buffer.rs b/crates/uzu/src/backends/common/command_buffer.rsindex 523aff8cf..7abf67225 100644--- a/crates/uzu/src/backends/common/command_buffer.rs+++ b/crates/uzu/src/backends/common/command_buffer.rs@@ -73,8 +73,9 @@ pub trait CommandBufferEncoding {fn encode_copy(&mut self,src: &<<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,+ src_range: Range<usize>,dst: &mut <<Self::CommandBuffer as CommandBuffer>::Backend as Backend>::Buffer,- size: usize,+ dst_range: Range<usize>,);fn encode_fill(@@ -135,6 +136,5 @@ pub trait CommandBufferPending {pub trait CommandBufferCompleted {type CommandBuffer: CommandBuffer<Completed = Self>;- /// Returns GPU execution time.fn gpu_execution_time(&self) -> Option<Duration>;}diff --git a/crates/uzu/src/backends/common/encoder.rs b/crates/uzu/src/backends/common/encoder.rsindex 7c0136060..7717599d9 100644--- a/crates/uzu/src/backends/common/encoder.rs+++ b/crates/uzu/src/backends/common/encoder.rs@@ -59,20 +59,23 @@ impl<'encoding, B: Backend> Encoder<'encoding, B> {pub fn encode_copy(&mut self,src: &B::Buffer,+ src_range: Range<usize>,dst: &mut B::Buffer,- size: usize,+ dst_range: Range<usize>,) {+ let size = src_range.end - src_range.start;+ debug_assert_eq!(size, dst_range.end - dst_range.start);self.access(&[Access {- range: src.gpu_address_subrange(0..size),+ range: src.gpu_address_subrange(src_range.clone()),flags: AccessFlags::copy_read(),},Access {- range: dst.gpu_address_subrange(0..size),+ range: dst.gpu_address_subrange(dst_range.clone()),flags: AccessFlags::copy_write(),},]);- self.command_buffer.encode_copy(src, dst, size);+ self.command_buffer.encode_copy(src, src_range, dst, dst_range);}pub fn encode_fill(diff --git a/crates/uzu/src/backends/common/gpu_types/activation_type.rs b/crates/uzu/src/backends/common/gpu_types/activation_type.rsindex 939965237..09ebe4657 100644--- a/crates/uzu/src/backends/common/gpu_types/activation_type.rs+++ b/crates/uzu/src/backends/common/gpu_types/activation_type.rs@@ -8,6 +8,7 @@ const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)pub enum ActivationType {SILU,GELU,+ TANH,IDENTITY,}@@ -19,6 +20,7 @@ impl ActivationType {match self {ActivationType::SILU => silu(x),ActivationType::GELU => gelu(x),+ ActivationType::TANH => tanh_activation(x),ActivationType::IDENTITY => x,}}@@ -36,3 +38,8 @@ fn gelu<T: Float>(x: T) -> T {let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());T::from(y_float).unwrap()}++fn tanh_activation<T: Float>(x: T) -> T {+ let x_float = x.to_f32().unwrap();+ T::from(x_float.tanh()).unwrap()+}diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rsindex f782acf4a..b5a284288 100644--- a/crates/uzu/src/backends/common/kernel/mod.rs+++ b/crates/uzu/src/backends/common/kernel/mod.rs@@ -3,7 +3,6 @@include!(concat!(env!("OUT_DIR"), "/traits.rs"));pub mod attention;-pub mod audio;pub mod kv_cache_update;pub mod matmul;pub mod mlp_gate_act_mul;diff --git a/crates/uzu/src/backends/common/kernel/sampling.rs b/crates/uzu/src/backends/common/kernel/sampling.rsindex 400faa501..491431e61 100644--- a/crates/uzu/src/backends/common/kernel/sampling.rs+++ b/crates/uzu/src/backends/common/kernel/sampling.rs@@ -16,7 +16,7 @@ use crate::{TemperatureKernel, TopKKernel, TopPKernel,},},- session::parameter::SamplingMethod,+ session::parameter::{SamplingMethod, SamplingProcessingOrder},};#[derive(Debug, Clone, Copy, PartialEq)]@@ -177,9 +177,12 @@ impl<B: Backend> SamplingKernel<B> {top_k,top_p,min_p,+ processing_order,} = sampling_method{- if let Some(temperature) = temperature {+ if let Some(temperature) = temperature+ && processing_order == SamplingProcessingOrder::TemperatureThenFilters+ {self.temperature.encode(None::<&B::Buffer>,logits_buffer.deref_mut(),@@ -200,7 +203,6 @@ impl<B: Backend> SamplingKernel<B> {encoder,);}-if let Some(top_p) = top_p {self.topp.encode(None::<&B::Buffer>,@@ -211,7 +213,6 @@ impl<B: Backend> SamplingKernel<B> {encoder,);}-if let Some(min_p) = min_p {self.minp.encode(None::<&B::Buffer>,@@ -223,6 +224,19 @@ impl<B: Backend> SamplingKernel<B> {);}+ if let Some(temperature) = temperature+ && processing_order == SamplingProcessingOrder::FiltersThenTemperature+ {+ self.temperature.encode(+ None::<&B::Buffer>,+ logits_buffer.deref_mut(),+ batch_size as u32,+ vocab_size as u32,+ temperature,+ encoder,+ );+ }+self.gumbel.encode(None::<&B::Buffer>,(seeds_buffer.ok_or(SamplingError::StochasticWithoutSeed)?, seeds_offset),diff --git a/crates/uzu/src/backends/cpu/command_buffer.rs b/crates/uzu/src/backends/cpu/command_buffer.rsindex 8ce9063bd..086877080 100644--- a/crates/uzu/src/backends/cpu/command_buffer.rs+++ b/crates/uzu/src/backends/cpu/command_buffer.rs@@ -59,13 +59,19 @@ impl CommandBufferEncoding for CpuCommandBuffer {fn encode_copy(&mut self,src: &UnsafeCell<Box<[u8]>>,+ src_range: std::ops::Range<usize>,dst: &mut UnsafeCell<Box<[u8]>>,- size: usize,+ dst_range: std::ops::Range<usize>,) {- let src = unsafe { &*src.get() }.as_ptr();- let dst = dst.get_mut().as_mut_ptr();+ let size = src_range.end - src_range.start;+ assert_eq!(size, dst_range.end - dst_range.start);+ assert!(unsafe { &*src.get() }.len() >= src_range.end);+ assert!(dst.get_mut().len() >= dst_range.end);++ let src_ptr = unsafe { (&*src.get()).as_ptr().add(src_range.start) };+ let dst_ptr = unsafe { dst.get_mut().as_mut_ptr().add(dst_range.start) };self.push_command(Box::new(move || unsafe {- std::ptr::copy(src, dst, size);+ std::ptr::copy(src_ptr, dst_ptr, size);}))}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/add.rs b/crates/uzu/src/backends/cpu/kernel/audio/add.rsnew file mode 100644index 000000000..7e711eb22--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/add.rs@@ -0,0 +1,16 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioAdd)]+#[variants(T, f32, f16, bf16)]+pub fn audio_add<T: ArrayElement + Float>(+ #[allow(unused)] a: *const T,+ #[allow(unused)] b: *const T,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] n: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/audio.rs b/crates/uzu/src/backends/cpu/kernel/audio/audio.rsdeleted file mode 100644index 81a2fd57a..000000000--- a/crates/uzu/src/backends/cpu/kernel/audio/audio.rs+++ /dev/null@@ -1,167 +0,0 @@-use dsl::kernel;-use half::{bf16, f16};-use num_traits::Float;--use crate::ArrayElement;--#[kernel(AudioFsqDecode)]-#[variants(T, f32, f16, bf16)]-pub fn audio_fsq_decode<T: ArrayElement + Float>(- #[allow(unused)] tokens: *const i32,- #[allow(unused)] out: *mut T,- #[allow(unused)] lengths: *const i32,- #[allow(unused)] num_groups: i32,- #[allow(unused)] seq_len: i32,- #[allow(unused)] codebook_dim: i32,- #[allow(unused)] num_levels: &[i32],- #[allow(unused)] batch_size: i32,-) {- todo!()-}--#[kernel(AudioLeakyRelu)]-#[variants(T, f32, f16, bf16)]-pub fn audio_leaky_relu<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] n: i32,- #[allow(unused)] negative_slope: f32,-) {- todo!()-}--#[kernel(AudioTanh)]-#[variants(T, f32, f16, bf16)]-pub fn audio_tanh<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] n: i32,-) {- todo!()-}--#[kernel(AudioAdd)]-#[variants(T, f32, f16, bf16)]-pub fn audio_add<T: ArrayElement + Float>(- #[allow(unused)] a: *const T,- #[allow(unused)] b: *const T,- #[allow(unused)] out: *mut T,- #[allow(unused)] n: i32,-) {- todo!()-}--#[kernel(AudioScale)]-#[variants(T, f32, f16, bf16)]-pub fn audio_scale<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] n: i32,- #[allow(unused)] scale: f32,-) {- todo!()-}--#[kernel(AudioCausalConv1d)]-#[variants(T, f32, f16, bf16)]-pub fn audio_causal_conv1d<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] weight: *const T,- #[allow(unused)] bias: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] lengths: *const i32,- #[allow(unused)] cin: i32,- #[allow(unused)] cout: i32,- #[allow(unused)] seq_len: i32,- #[allow(unused)] kernel_size: i32,- #[allow(unused)] dilation: i32,- #[allow(unused)] batch_size: i32,-) {- todo!()-}--#[kernel(AudioCausalConvTranspose1d)]-#[variants(T, f32, f16, bf16)]-pub fn audio_causal_conv_transpose1d<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] weight: *const T,- #[allow(unused)] bias: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] lengths: *const i32,- #[allow(unused)] cin: i32,- #[allow(unused)] cout: i32,- #[allow(unused)] seq_len_in: i32,- #[allow(unused)] seq_len_out: i32,- #[allow(unused)] stride: i32,- #[allow(unused)] groups: i32,- #[allow(unused)] batch_size: i32,-) {- todo!()-}--#[kernel(AudioHalfSnake)]-#[variants(T, f32, f16, bf16)]-pub fn audio_half_snake<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] alpha: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] channels: i32,- #[allow(unused)] seq_len: i32,- #[allow(unused)] snake_channels: i32,- #[allow(unused)] negative_slope: f32,- #[allow(unused)] eps: f32,- #[allow(unused)] batch_size: i32,-) {- todo!()-}--#[kernel(AudioClamp)]-#[variants(T, f32, f16, bf16)]-pub fn audio_clamp<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] n: i32,- #[allow(unused)] min_value: f32,- #[allow(unused)] max_value: f32,-) {- todo!()-}--#[kernel(AudioConv1d)]-#[variants(T, f32, f16, bf16)]-pub fn audio_conv1d<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] weight: *const T,- #[allow(unused)] bias: *const T,- #[allow(unused)] output: *mut T,- #[allow(unused)] lengths: *const i32,- #[allow(unused)] cin: i32,- #[allow(unused)] cout: i32,- #[allow(unused)] seq_len_in: i32,- #[allow(unused)] seq_len_out: i32,- #[allow(unused)] kernel_size: i32,- #[allow(unused)] stride: i32,- #[allow(unused)] dilation: i32,- #[allow(unused)] padding: i32,- #[allow(unused)] pad_mode: i32,- #[allow(unused)] batch_size: i32,-) {- todo!()-}--#[kernel(AudioFsqEncode)]-#[variants(T, f32, f16, bf16)]-pub fn audio_fsq_encode<T: ArrayElement + Float>(- #[allow(unused)] input: *const T,- #[allow(unused)] tokens: *mut i32,- #[allow(unused)] lengths: *const i32,- #[allow(unused)] num_groups: i32,- #[allow(unused)] seq_len: i32,- #[allow(unused)] codebook_dim: i32,- #[allow(unused)] num_levels: &[i32],- #[allow(unused)] dim_base_index: &[i32],- #[allow(unused)] eps: f32,- #[allow(unused)] batch_size: i32,-) {- todo!()-}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d.rsnew file mode 100644index 000000000..a7725f357--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d.rs@@ -0,0 +1,24 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioCausalConv1d)]+#[variants(T, f32, f16, bf16)]+pub fn audio_causal_conv1d<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] kernel_size: i32,+ #[allow(unused)] dilation: i32,+ #[allow(unused)] input_layout: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped.rsnew file mode 100644index 000000000..26c6c521f--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped.rs@@ -0,0 +1,25 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioCausalConv1dGrouped)]+#[variants(T, f32, f16, bf16)]+pub fn audio_causal_conv1d_grouped<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] kernel_size: i32,+ #[allow(unused)] dilation: i32,+ #[allow(unused)] groups: i32,+ #[allow(unused)] input_layout: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped_residual.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped_residual.rsnew file mode 100644index 000000000..cc247a183--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv1d_grouped_residual.rs@@ -0,0 +1,25 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioCausalConv1dGroupedResidual)]+#[variants(T, f32, f16, bf16)]+pub fn audio_causal_conv1d_grouped_residual<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] residual: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] kernel_size: i32,+ #[allow(unused)] dilation: i32,+ #[allow(unused)] groups: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d.rsnew file mode 100644index 000000000..e143dc71b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d.rs@@ -0,0 +1,24 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioCausalConvTranspose1d)]+#[variants(T, f32, f16, bf16)]+pub fn audio_causal_conv_transpose1d<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len_in: i32,+ #[allow(unused)] seq_len_out: i32,+ #[allow(unused)] stride: i32,+ #[allow(unused)] groups: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d_causal_pad.rs b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d_causal_pad.rsnew file mode 100644index 000000000..d9f33326b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/causal_conv_transpose1d_causal_pad.rs@@ -0,0 +1,26 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioCausalConvTranspose1dCausalPad)]+#[variants(T, f32, f16, bf16)]+pub fn audio_causal_conv_transpose1d_causal_pad<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len_in: i32,+ #[allow(unused)] seq_len_out: i32,+ #[allow(unused)] kernel_size: i32,+ #[allow(unused)] stride: i32,+ #[allow(unused)] groups: i32,+ #[allow(unused)] input_layout: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/audio/conv1d.rsnew file mode 100644index 000000000..5096e3994--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/conv1d.rs@@ -0,0 +1,27 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioConv1d)]+#[variants(T, f32, f16, bf16)]+pub fn audio_conv1d<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] weight: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] cin: i32,+ #[allow(unused)] cout: i32,+ #[allow(unused)] seq_len_in: i32,+ #[allow(unused)] seq_len_out: i32,+ #[allow(unused)] kernel_size: i32,+ #[allow(unused)] stride: i32,+ #[allow(unused)] dilation: i32,+ #[allow(unused)] padding: i32,+ #[allow(unused)] pad_mode: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/fsq_decode.rs b/crates/uzu/src/backends/cpu/kernel/audio/fsq_decode.rsnew file mode 100644index 000000000..8da21f019--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/fsq_decode.rs@@ -0,0 +1,21 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioFsqDecode)]+#[variants(T, f32, f16, bf16)]+pub fn audio_fsq_decode<T: ArrayElement + Float>(+ #[allow(unused)] tokens: *const i32,+ #[allow(unused)] out: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] num_groups: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] codebook_dim: i32,+ #[allow(unused)] num_levels: &[i32],+ #[allow(unused)] dim_base_index: &[i32],+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/fsq_encode.rs b/crates/uzu/src/backends/cpu/kernel/audio/fsq_encode.rsnew file mode 100644index 000000000..580cabc38--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/fsq_encode.rs@@ -0,0 +1,22 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioFsqEncode)]+#[variants(T, f32, f16, bf16)]+pub fn audio_fsq_encode<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] tokens: *mut i32,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] num_groups: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] codebook_dim: i32,+ #[allow(unused)] num_levels: &[i32],+ #[allow(unused)] dim_base_index: &[i32],+ #[allow(unused)] eps: f32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/half_snake.rs b/crates/uzu/src/backends/cpu/kernel/audio/half_snake.rsnew file mode 100644index 000000000..d3aaeace0--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/half_snake.rs@@ -0,0 +1,21 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioHalfSnake)]+#[variants(T, f32, f16, bf16)]+pub fn audio_half_snake<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] alpha: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] channels: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] snake_channels: i32,+ #[allow(unused)] negative_slope: f32,+ #[allow(unused)] eps: f32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/mod.rs b/crates/uzu/src/backends/cpu/kernel/audio/mod.rsindex fa3e4e468..cd7577077 100644--- a/crates/uzu/src/backends/cpu/kernel/audio/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/audio/mod.rs@@ -1 +1,12 @@-pub mod audio;+pub mod add;+pub mod causal_conv1d;+pub mod causal_conv1d_grouped;+pub mod causal_conv1d_grouped_residual;+pub mod causal_conv_transpose1d;+pub mod causal_conv_transpose1d_causal_pad;+pub mod conv1d;+pub mod fsq_decode;+pub mod fsq_encode;+pub mod half_snake;+pub mod norm_ncs;+pub mod quantizer_decode;diff --git a/crates/uzu/src/backends/cpu/kernel/audio/norm_ncs.rs b/crates/uzu/src/backends/cpu/kernel/audio/norm_ncs.rsnew file mode 100644index 000000000..9830f981b--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/norm_ncs.rs@@ -0,0 +1,22 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioNormNcs)]+#[variants(T, f32, f16, bf16)]+pub fn audio_norm_ncs<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] scales: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] channels: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] epsilon: f32,+ #[allow(unused)] subtract_mean: i32,+ #[allow(unused)] batch_size: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/audio/quantizer_decode.rs b/crates/uzu/src/backends/cpu/kernel/audio/quantizer_decode.rsnew file mode 100644index 000000000..7bb16000e--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/audio/quantizer_decode.rs@@ -0,0 +1,29 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(AudioQuantizerDecode)]+#[variants(T, f32, f16, bf16)]+pub fn audio_quantizer_decode<T: ArrayElement + Float>(+ #[allow(unused)] tokens: *const u32,+ #[allow(unused)] lengths: *const i32,+ #[allow(unused)] semantic_codebook: *const T,+ #[allow(unused)] semantic_out_proj: *const T,+ #[allow(unused)] semantic_out_bias: *const T,+ #[allow(unused)] residual_codebooks: *const T,+ #[allow(unused)] residual_out_proj: *const T,+ #[allow(unused)] residual_out_bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] batch_size: i32,+ #[allow(unused)] total_codebooks: i32,+ #[allow(unused)] seq_len: i32,+ #[allow(unused)] input_dim: i32,+ #[allow(unused)] codebook_dim: i32,+ #[allow(unused)] residual_quantizers: i32,+ #[allow(unused)] semantic_cardinality: i32,+ #[allow(unused)] residual_cardinality: i32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rsnew file mode 100644index 000000000..eccc8c7f1--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/embedding/embedding_rows_sum.rs@@ -0,0 +1,19 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(EmbeddingRowsSum)]+#[variants(T, f32, f16, bf16)]+pub fn embedding_rows_sum<T: ArrayElement + Float>(+ #[allow(unused)] token_indices: *const u32,+ #[allow(unused)] weights: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] num_rows: u32,+ #[allow(unused)] total_rows: u32,+ #[allow(unused)] model_dim: u32,+ #[allow(unused)] codebook_stride: u32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rsindex 93f7cf1f0..556c0cfb7 100644--- a/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/embedding/mod.rs@@ -1,2 +1,3 @@+pub mod embedding_rows_sum;pub mod full_precision_embedding;pub mod quant_embedding;diff --git a/crates/uzu/src/backends/cpu/kernel/mod.rs b/crates/uzu/src/backends/cpu/kernel/mod.rsindex 287488d31..2d9020b1f 100644--- a/crates/uzu/src/backends/cpu/kernel/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/mod.rs@@ -18,6 +18,7 @@ mod short_conv;mod sigmoid;mod ssm;mod tensor_add_bias;+mod tensor_add_scale;mod tensor_add_swap;mod tensor_copy;mod token_copy;diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/mod.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/mod.rsnew file mode 100644index 000000000..287ef450d--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/mod.rs@@ -0,0 +1 @@+pub mod tensor_add_scale;diff --git a/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rsnew file mode 100644index 000000000..e0d2289b5--- /dev/null+++ b/crates/uzu/src/backends/cpu/kernel/tensor_add_scale/tensor_add_scale.rs@@ -0,0 +1,18 @@+use dsl::kernel;+use half::{bf16, f16};+use num_traits::Float;++use crate::ArrayElement;++#[kernel(TensorAddScale)]+#[variants(T, f32, f16, bf16)]+pub fn tensor_add_scale<T: ArrayElement + Float>(+ #[allow(unused)] input: *const T,+ #[allow(unused)] bias: *const T,+ #[allow(unused)] output: *mut T,+ #[allow(unused)] num_cols: u32,+ #[allow(unused)] length: u32,+ #[allow(unused)] scale: f32,+) {+ todo!()+}diff --git a/crates/uzu/src/backends/metal/command_buffer.rs b/crates/uzu/src/backends/metal/command_buffer.rsindex 894b5eb96..aa3de6135 100644--- a/crates/uzu/src/backends/metal/command_buffer.rs+++ b/crates/uzu/src/backends/metal/command_buffer.rs@@ -121,12 +121,15 @@ impl CommandBufferEncoding for MetalCommandBufferEncoding {fn encode_copy(&mut self,src: &Retained<ProtocolObject<dyn MTLBuffer>>,+ src_range: std::ops::Range<usize>,dst: &mut Retained<ProtocolObject<dyn MTLBuffer>>,- size: usize,+ dst_range: std::ops::Range<usize>,) {- assert!(src.length() >= size && dst.length() >= size);+ let size = src_range.end - src_range.start;+ assert_eq!(size, dst_range.end - dst_range.start);+ assert!(src.length() >= src_range.end && dst.length() >= dst_range.end);- self.ensure_blit().copy_buffer_to_buffer(src, 0, dst, 0, size);+ self.ensure_blit().copy_buffer_to_buffer(src, src_range.start, dst, dst_range.start, size);}fn encode_fill(@@ -231,7 +234,7 @@ impl CommandBufferCompleted for MetalCommandBufferCompleted {fn gpu_execution_time(&self) -> Option<Duration> {match (self.command_buffer.gpu_start_time(), self.command_buffer.gpu_end_time()) {- (Some(start), Some(end)) => Some(Duration::from_secs_f64(end - start)),+ (Some(start), Some(end)) if end > start => Some(Duration::from_secs_f64(end - start)),_ => None,}}diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.hindex 741264acd..290f091ea 100644--- a/crates/uzu/src/backends/metal/generated/activation_type.h+++ b/crates/uzu/src/backends/metal/generated/activation_type.h@@ -17,6 +17,7 @@ namespace activation_type {enum ActivationType : uint32_t {SILU,GELU,+ TANH,IDENTITY,};diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.hindex 68ee975d5..b8c053e85 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activations.h+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h@@ -21,6 +21,18 @@ inline T activate_gelu(T x) {return T(yf);}+template <typename T>+inline T activate_leaky_relu(T x, float alpha) {+ const float xf = float(x);+ const float yf = (xf >= 0.0f) ? xf : (alpha * xf);+ return static_cast<T>(yf);+}++template <typename T>+inline T activate_tanh(T x) {+ return static_cast<T>(metal::tanh(float(x)));+}+template <typename T>inline T activate(T x, ActivationType type) {switch (type) {@@ -28,6 +40,8 @@ inline T activate(T x, ActivationType type) {return activate_silu(x);case GELU:return activate_gelu(x);+ case TANH:+ return activate_tanh(x);case IDENTITY:return x;}diff --git a/crates/uzu/src/backends/metal/kernel/audio/add.metal b/crates/uzu/src/backends/metal/kernel/audio/add.metalnew file mode 100644index 000000000..545920cd5--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/add.metal@@ -0,0 +1,30 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++template <typename T>+void add_tensors(+ device const T* a,+ device const T* b,+ device T* out,+ const constant int& n,+ const uint tid+) {+ if ((int)tid >= n) {+ return;+ }+ out[tid] = a[tid] + b[tid];+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioAdd)(+ device const T* a,+ device const T* b,+ device T* out,+ const constant int& n,+ uint tid AXIS(n, 256)+) {+ add_tensors<T>(a, b, out, n, tid);+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metaldeleted file mode 100644index e95cb27f0..000000000--- a/crates/uzu/src/backends/metal/kernel/audio/audio.metal+++ /dev/null@@ -1,738 +0,0 @@-#include <metal_stdlib>-#include "../common/dsl.h"--using namespace metal;--// === FSQ (Finite Scalar Quantization) decode ===-//-// Implements NeMo's FiniteScalarQuantizer.decode() applied per group:-// - tokens are stored as [B, G, T] int32-// - output codes are stored as [B, G*D, T] where D = codebook_dim_per_group-//-// NeMo reference:-// codes_nonnegative = (indices // dim_base_index) % num_levels-// dequantized = (codes_nonnegative - (num_levels//2)) / (num_levels//2)-//-// Here dim_base_index is derived from num_levels:-// base[0]=1, base[d]=prod_{k<d} num_levels[k]--template <typename T>-void fsq_decode(- device const int* tokens,- device T* out,- device const int* lengths,- const constant int& num_groups,- const constant int& seq_len,- const constant int& codebook_dim,- const constant int* num_levels,- const uint3 gid-) {- const uint t = gid.x;- const uint g = gid.y;- const uint b = gid.z;-- if (t >= (uint)seq_len || g >= (uint)num_groups) {- return;- }-- const int len_b = lengths ? lengths[b] : seq_len;- if ((int)t >= len_b) {- // Zero all D codes for this token when masked.- for (int d = 0; d < codebook_dim; ++d) {- const uint out_c = g * (uint)codebook_dim + (uint)d;- const uint out_idx =- (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;- out[out_idx] = (T)0;- }- return;- }-- const uint token_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;- const int token = tokens[token_idx];-- // Compute dim bases (small D, do it per-thread).- int base = 1;- for (int d = 0; d < codebook_dim; ++d) {- const int levels = num_levels[d];- const int scale = levels / 2; // integer division (matches PyTorch // 2)- const int offset = scale;-- // Euclidean modulo to keep result in [0, levels) even if token is negative- // (tokens are expected nonnegative, but keep this robust).- const int div = token / base;- int code_nonneg = div % levels;- if (code_nonneg < 0) {- code_nonneg += levels;- }- const float code = ((float)(code_nonneg - offset)) / (float)scale;-- const uint out_c = g * (uint)codebook_dim + (uint)d;- const uint out_idx =- (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;- out[out_idx] = (T)code;-- base *= levels;- }-}--// === Elementwise ops for codec blocks ===--template <typename T>-void audio_codec_leaky_relu(- device const T* input,- device T* output,- const constant int& n,- const constant float& negative_slope,- const uint tid-) {- if ((int)tid >= n) {- return;- }- const float x = float(input[tid]);- const float y = (x >= 0.0f) ? x : (negative_slope * x);- output[tid] = (T)y;-}--template <typename T>-void audio_codec_tanh(- device const T* input,- device T* output,- const constant int& n,- const uint tid-) {- if ((int)tid >= n) {- return;- }- const float x = float(input[tid]);- output[tid] = (T)tanh(x);-}--template <typename T>-void audio_codec_add(- device const T* a,- device const T* b,- device T* out,- const constant int& n,- const uint tid-) {- if ((int)tid >= n) {- return;- }- out[tid] = a[tid] + b[tid];-}--template <typename T>-void audio_codec_scale(- device const T* input,- device T* output,- const constant int& n,- const constant float& scale,- const uint tid-) {- if ((int)tid >= n) {- return;- }- const float x = float(input[tid]);- output[tid] = (T)(x * scale);-}--// === Causal Conv1d (stride=1, groups=1) ===-//-// Matches NeMo's CausalConv1dNorm forward for the decoder path where-// stride=1:-// - left padding by padding_total = (kernel_size_eff - 1) where-// kernel_size_eff=(K-1)*dilation+1-// - no right padding (extra_padding=0 for stride=1)-// - output length equals input length-template <typename T>-void audio_codec_causal_conv1d(- device const T* input, // [B, Cin, T]- device const T* weight, // [Cout, Cin, K]- device const T* bias, // [Cout]- device T* output, // [B, Cout, T]- device const int* lengths, // [B]- const constant int& cin,- const constant int& cout,- const constant int& seq_len,- const constant int& kernel_size,- const constant int& dilation,- const uint3 gid-) {- const uint t = gid.x;- const uint oc = gid.y;- const uint b = gid.z;-- if (t >= (uint)seq_len || oc >= (uint)cout) {- return;- }-- const int len_b = lengths ? lengths[b] : seq_len;- const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len + t;- if ((int)t >= len_b) {- output[out_idx] = (T)0;- return;- }-- const int pad = (kernel_size - 1) * dilation;- float acc = float(bias[oc]);-- for (int ic = 0; ic < cin; ++ic) {- const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;- const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;- for (int k = 0; k < kernel_size; ++k) {- const int x_t = (int)t + k * dilation - pad;- if (x_t < 0) {- continue;- }- // For stride=1 and causal padding, x_t never exceeds len_b for valid t,- // but for safety clamp to seq_len.- if (x_t >= seq_len) {- continue;- }- acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);- }- }-- output[out_idx] = (T)acc;-}--// === Causal ConvTranspose1d upsample (kernel_size = 2*stride) ===-//-// Matches NeMo's CausalConvTranspose1dNorm forward for trim_right_ratio=1:-// - ConvTranspose1d with padding=0, output_padding=0, dilation=1-// - kernel_size = 2 * stride-// - unpad: trim_right = kernel_size - stride (= stride), trim_left = 0-// Resulting output length is input_len * stride.-template <typename T>-void audio_codec_causal_conv_transpose1d(- device const T* input, // [B, Cin, Tin]- device const T* weight, // [Cin, Cout_per_group, 2*stride]- device const T* bias, // [Cout]- device T* output, // [B, Cout, Tout]- device const int* lengths, // [B] (output lengths)- const constant int& cin,- const constant int& cout,- const constant int& seq_len_in,- const constant int& seq_len_out,- const constant int& stride,- const constant int& groups,- const uint3 gid-) {- const uint t_out = gid.x;- const uint oc = gid.y;- const uint b = gid.z;-- if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {- return;- }-- const int len_b = lengths ? lengths[b] : seq_len_out;- const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;- if ((int)t_out >= len_b) {- output[out_idx] = (T)0;- return;- }-- const int cout_per_group = cout / groups;- const int cin_per_group = cin / groups;- const int group_idx = (int)oc / cout_per_group;- const int oc_in_group = (int)oc - group_idx * cout_per_group;-- const int q = (int)t_out / stride;- const int r = (int)t_out - q * stride;-- float acc = float(bias[oc]);-- const uint in_base_b = b * (uint)cin * (uint)seq_len_in;- const uint w_base_oc = (uint)oc_in_group * (uint)(2 * stride);-- const int ic_begin = group_idx * cin_per_group;- const int ic_end = ic_begin + cin_per_group;-- for (int ic = ic_begin; ic < ic_end; ++ic) {- if (q >= seq_len_in) {- continue;- }- const uint in_base = in_base_b + (uint)ic * (uint)seq_len_in;-- // Weight layout: [Cin, Cout_per_group, K]- const uint w_base =- ((uint)ic * (uint)cout_per_group) * (uint)(2 * stride) + w_base_oc;-- // Contribution from input[q] with k=r- acc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);-- // Contribution from input[q-1] with k=stride+r- if (q > 0) {- acc += float(weight[w_base + (uint)(stride + r)]) *- float(input[in_base + (uint)(q - 1)]);- }- }-- output[out_idx] = (T)acc;-}--// === Snake / HalfSnake / Clamp (NeMo common.parts.utils) ===-//-// snake(x, alpha, eps) = x + (alpha + eps)^-1 * sin(alpha * x)^2-// HalfSnake applies snake to the first half channels and LeakyReLU to the-// rest.--template <typename T>-void audio_codec_half_snake(- device const T* input, // [B, C, T]- device const T* alpha, // [1, C_snake, 1] (contiguous, index by channel)- device T* output, // [B, C, T]- const constant int& channels,- const constant int& seq_len,- const constant int& snake_channels,- const constant float& negative_slope,- const constant float& eps,- const uint3 gid-) {- const uint t = gid.x;- const uint c = gid.y;- const uint b = gid.z;-- if (t >= (uint)seq_len || c >= (uint)channels) {- return;- }-- const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;- const float x = float(input[idx]);-- if ((int)c < snake_channels) {- const float a = float(alpha[c]);- const float ax = a * x;- const float s = sin(ax);- const float y = x + (1.0f / (a + eps)) * (s * s);- output[idx] = (T)y;- } else {- const float y = (x >= 0.0f) ? x : (negative_slope * x);- output[idx] = (T)y;- }-}--template <typename T>-void audio_codec_clamp(- device const T* input,- device T* output,- const constant int& n,- const constant float& min_value,- const constant float& max_value,- const uint tid-) {- if ((int)tid >= n) {- return;- }- const float x = float(input[tid]);- const float y = clamp(x, min_value, max_value);- output[tid] = (T)y;-}--// === Non-causal Conv1d with padding + stride (for encoder) ===-//-// Matches torch.nn.Conv1d with explicit padding and padding_mode:-// out[t_out] = bias + sum_{ic,k} w[oc,ic,k] * x[ic, t_out*stride - padding-// + k*dilation]-// - pad_mode=0: zeros-// - pad_mode=1: replicate (clamp index to [0, seq_len_in-1])-template <typename T>-void audio_codec_conv1d(- device const T* input, // [B, Cin, Tin]- device const T* weight, // [Cout, Cin, K]- device const T* bias, // [Cout]- device T* output, // [B, Cout, Tout]- device const int* lengths, // [B] (Tout lengths)- const constant int& cin,- const constant int& cout,- const constant int& seq_len_in,- const constant int& seq_len_out,- const constant int& kernel_size,- const constant int& stride,- const constant int& dilation,- const constant int& padding,- const constant int& pad_mode,- const uint3 gid-) {- const uint t_out = gid.x;- const uint oc = gid.y;- const uint b = gid.z;-- if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {- return;- }-- const int len_b = lengths ? lengths[b] : seq_len_out;- const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;- if ((int)t_out >= len_b) {- output[out_idx] = (T)0;- return;- }-- const int base = (int)t_out * stride - padding;- float acc = float(bias[oc]);-- for (int ic = 0; ic < cin; ++ic) {- const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;- const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;- for (int k = 0; k < kernel_size; ++k) {- int x_t = base + k * dilation;- if (pad_mode == 0) {- if (x_t < 0 || x_t >= seq_len_in) {- continue;- }- } else {- x_t = clamp(x_t, 0, seq_len_in - 1);- }- acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);- }- }-- output[out_idx] = (T)acc;-}--// === FSQ (Finite Scalar Quantization) encode ===-//-// Implements GroupFiniteScalarQuantizer.encode() for inference:-// - input: [B, G*D, T] float-// - output: [B, G, T] int32-//-// NeMo reference (FiniteScalarQuantizer):-// output_scale = (num_levels - 1) / 2 * (1 - eps)-// output_offset = 0.5 if even else 0-// input_shift = tan(output_offset / output_scale)-// compressed = output_scale * tanh(inputs + input_shift) - output_offset-// rounded = round(compressed) # half away from zero-// code_nonneg = rounded + (num_levels//2)-// token = sum_d code_nonneg[d] * dim_base_index[d]--// Match torch.round() semantics (ties to even).-inline float round_ties_to_even(const float x) {- const float f = floor(x);- const float frac = x - f;- if (frac < 0.5f) {- return f;- }- if (frac > 0.5f) {- return f + 1.0f;- }- const int fi = (int)f; // f is integral- return ((fi & 1) != 0) ? (f + 1.0f) : f;-}--template <typename T>-void fsq_encode(- device const T* input, // [B, G*D, T]- device int* tokens, // [B, G, T]- device const int* lengths, // [B]- const constant int& num_groups,- const constant int& seq_len,- const constant int& codebook_dim,- const constant int* num_levels,- const constant int* dim_base_index,- const constant float& eps,- const uint3 gid-) {- const uint t = gid.x;- const uint g = gid.y;- const uint b = gid.z;-- if (t >= (uint)seq_len || g >= (uint)num_groups) {- return;- }-- const int len_b = lengths ? lengths[b] : seq_len;- const uint out_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;- if ((int)t >= len_b) {- tokens[out_idx] = 0;- return;- }-- const uint in_base =- (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *- (uint)seq_len +- t;-- int token = 0;- for (int d = 0; d < codebook_dim; ++d) {- const int levels = num_levels[d];- const int scale_i = levels / 2;- const float output_scale = ((float)(levels - 1)) * 0.5f * (1.0f - eps);- const float output_offset = (levels % 2 == 0) ? 0.5f : 0.0f;- const float input_shift = tan(output_offset / output_scale);-- const float x = float(input[in_base + (uint)d * (uint)seq_len]);- const float compressed =- output_scale * tanh(x + input_shift) - output_offset;- const float rounded = round_ties_to_even(compressed);-- int code_nonneg = (int)rounded + scale_i;- code_nonneg = clamp(code_nonneg, 0, levels - 1);- token += code_nonneg * dim_base_index[d];- }-- tokens[out_idx] = token;-}--// === DSL kernel wrappers ===-//-// Keep the scalar helper implementations above and expose a DSL-annotated-// surface so audio kernels are integrated through the same generated wrapper-// path as other kernels.--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioFsqDecode)(- device const int* tokens,- device T* out,- device const int* lengths,- const constant int& num_groups,- const constant int& seq_len,- const constant int& codebook_dim,- const constant int* num_levels,- const constant int& batch_size,- uint t AXIS(seq_len, 32),- uint g AXIS(num_groups, 1),- uint b AXIS(batch_size, 1)-) {- fsq_decode<T>(- tokens,- out,- lengths,- num_groups,- seq_len,- codebook_dim,- num_levels,- uint3(t, g, b)- );-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioLeakyRelu)(- device const T* input,- device T* output,- const constant int& n,- const constant float& negative_slope,- uint tid AXIS(n, 256)-) {- audio_codec_leaky_relu<T>(input, output, n, negative_slope, tid);-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioTanh)(- device const T* input,- device T* output,- const constant int& n,- uint tid AXIS(n, 256)-) {- audio_codec_tanh<T>(input, output, n, tid);-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioAdd)(- device const T* a,- device const T* b,- device T* out,- const constant int& n,- uint tid AXIS(n, 256)-) {- audio_codec_add<T>(a, b, out, n, tid);-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioScale)(- device const T* input,- device T* output,- const constant int& n,- const constant float& scale,- uint tid AXIS(n, 256)-) {- audio_codec_scale<T>(input, output, n, scale, tid);-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioCausalConv1d)(- device const T* input,- device const T* weight,- device const T* bias,- device T* output,- device const int* lengths,- const constant int& cin,- const constant int& cout,- const constant int& seq_len,- const constant int& kernel_size,- const constant int& dilation,- const constant int& batch_size,- uint t AXIS(seq_len, 32),- uint oc AXIS(cout, 1),- uint b AXIS(batch_size, 1)-) {- audio_codec_causal_conv1d<T>(- input,- weight,- bias,- output,- lengths,- cin,- cout,- seq_len,- kernel_size,- dilation,- uint3(t, oc, b)- );-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioCausalConvTranspose1d)(- device const T* input,- device const T* weight,- device const T* bias,- device T* output,- device const int* lengths,- const constant int& cin,- const constant int& cout,- const constant int& seq_len_in,- const constant int& seq_len_out,- const constant int& stride,- const constant int& groups,- const constant int& batch_size,- uint t_out AXIS(seq_len_out, 32),- uint oc AXIS(cout, 1),- uint b AXIS(batch_size, 1)-) {- audio_codec_causal_conv_transpose1d<T>(- input,- weight,- bias,- output,- lengths,- cin,- cout,- seq_len_in,- seq_len_out,- stride,- groups,- uint3(t_out, oc, b)- );-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioHalfSnake)(- device const T* input,- device const T* alpha,- device T* output,- const constant int& channels,- const constant int& seq_len,- const constant int& snake_channels,- const constant float& negative_slope,- const constant float& eps,- const constant int& batch_size,- uint t AXIS(seq_len, 32),- uint c AXIS(channels, 1),- uint b AXIS(batch_size, 1)-) {- audio_codec_half_snake<T>(- input,- alpha,- output,- channels,- seq_len,- snake_channels,- negative_slope,- eps,- uint3(t, c, b)- );-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioClamp)(- device const T* input,- device T* output,- const constant int& n,- const constant float& min_value,- const constant float& max_value,- uint tid AXIS(n, 256)-) {- audio_codec_clamp<T>(input, output, n, min_value, max_value, tid);-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioConv1d)(- device const T* input,- device const T* weight,- device const T* bias,- device T* output,- device const int* lengths,- const constant int& cin,- const constant int& cout,- const constant int& seq_len_in,- const constant int& seq_len_out,- const constant int& kernel_size,- const constant int& stride,- const constant int& dilation,- const constant int& padding,- const constant int& pad_mode,- const constant int& batch_size,- uint t_out AXIS(seq_len_out, 32),- uint oc AXIS(cout, 1),- uint b AXIS(batch_size, 1)-) {- audio_codec_conv1d<T>(- input,- weight,- bias,- output,- lengths,- cin,- cout,- seq_len_in,- seq_len_out,- kernel_size,- stride,- dilation,- padding,- pad_mode,- uint3(t_out, oc, b)- );-}--template <typename T>-VARIANTS(T, float, half, bfloat)-PUBLIC KERNEL(AudioFsqEncode)(- device const T* input,- device int* tokens,- device const int* lengths,- const constant int& num_groups,- const constant int& seq_len,- const constant int& codebook_dim,- const constant int* num_levels,- const constant int* dim_base_index,- const constant float& eps,- const constant int& batch_size,- uint t AXIS(seq_len, 32),- uint g AXIS(num_groups, 1),- uint b AXIS(batch_size, 1)-) {- fsq_encode<T>(- input,- tokens,- lengths,- num_groups,- seq_len,- codebook_dim,- num_levels,- dim_base_index,- eps,- uint3(t, g, b)- );-}diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d.metalnew file mode 100644index 000000000..cfcd6e82f--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d.metal@@ -0,0 +1,168 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++constant uint AUDIO_TIME_TILE = 4;+constant int AUDIO_LAYOUT_NCS = 0;+constant int AUDIO_LAYOUT_NSC = 1;++template <typename T>+void causal_conv1d(+ device const T* input, // [B, Cin, T] when NCS, [B, T, Cin] when NSC+ device const T* weight, // [Cout, Cin, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, T]+ device const int* lengths, // [B]+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& input_layout,+ const uint3 gid+) {+ const uint t = gid.x * AUDIO_TIME_TILE;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || oc >= (uint)cout) {+ return;+ }++ if (input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;+ int lane_count = seq_len - (int)t;+ if (lane_count > (int)AUDIO_TIME_TILE) {+ lane_count = (int)AUDIO_TIME_TILE;+ }+ if (lane_count <= 0) {+ return;+ }+ int valid_count = len_b - (int)t;+ if (valid_count > lane_count) {+ valid_count = lane_count;+ }+ if (valid_count < 0) {+ valid_count = 0;+ }++ const int pad = (kernel_size - 1) * dilation;+ const float bias_value = float(bias[oc]);+ float acc0 = bias_value;+ float acc1 = bias_value;+ float acc2 = bias_value;+ float acc3 = bias_value;++ if (valid_count > 0) {+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;+ for (int ic = 0; ic < cin; ++ic) {+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;+ const uint x_base_ncs = (b * (uint)cin + (uint)ic) * (uint)seq_len;+ for (int k = 0; k < kernel_size; ++k) {+ const float w = float(weight[w_base + (uint)k]);+ const int x_t = (int)t + k * dilation - pad;+ if (full_tile && x_t >= 0 && x_t + 3 < seq_len) {+ if (input_layout == AUDIO_LAYOUT_NCS) {+ const uint x_idx = x_base_ncs + (uint)x_t;+ acc0 += w * float(input[x_idx]);+ acc1 += w * float(input[x_idx + 1]);+ acc2 += w * float(input[x_idx + 2]);+ acc3 += w * float(input[x_idx + 3]);+ } else {+ const uint x_idx0 =+ (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;+ acc0 += w * float(input[x_idx0]);+ acc1 += w * float(input[x_idx0 + (uint)cin]);+ acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);+ acc3 += w * float(input[x_idx0 + (uint)(3 * cin)]);+ }+ continue;+ }+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {+ const uint x_idx =+ (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)x_t)+ : ((b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic);+ acc0 += w * float(input[x_idx]);+ }+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {+ const uint x_idx =+ (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)(x_t + 1))+ : ((b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin ++ (uint)ic);+ acc1 += w * float(input[x_idx]);+ }+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {+ const uint x_idx =+ (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)(x_t + 2))+ : ((b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin ++ (uint)ic);+ acc2 += w * float(input[x_idx]);+ }+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {+ const uint x_idx =+ (input_layout == AUDIO_LAYOUT_NCS)+ ? (x_base_ncs + (uint)(x_t + 3))+ : ((b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin ++ (uint)ic);+ acc3 += w * float(input[x_idx]);+ }+ }+ }+ }++ if (lane_count > 0) {+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;+ }+ if (lane_count > 1) {+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;+ }+ if (lane_count > 2) {+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;+ }+ if (lane_count > 3) {+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;+ }+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioCausalConv1d)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& input_layout,+ const constant int& batch_size,+ uint t AXIS((seq_len + 3) / 4, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ causal_conv1d<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len,+ kernel_size,+ dilation,+ input_layout,+ uint3(t, oc, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metalnew file mode 100644index 000000000..8b6840366--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped.metal@@ -0,0 +1,361 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++constant uint AUDIO_TIME_TILE = 8;+constant int AUDIO_LAYOUT_NCS = 0;+constant int AUDIO_LAYOUT_NSC = 1;++template <typename T>+void causal_conv1d_grouped_ncs(+ device const T* input, // [B, Cin, T]+ device const T* weight, // [Cout, Cin/groups, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, T]+ device const int* lengths, // [B]+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& groups,+ const uint3 gid+) {+ const uint t = gid.x * AUDIO_TIME_TILE;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || oc >= (uint)cout) {+ return;+ }++ if (groups <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;+ int lane_count = seq_len - (int)t;+ if (lane_count > (int)AUDIO_TIME_TILE) {+ lane_count = (int)AUDIO_TIME_TILE;+ }+ if (lane_count <= 0) {+ return;+ }+ int valid_count = len_b - (int)t;+ if (valid_count > lane_count) {+ valid_count = lane_count;+ }+ if (valid_count < 0) {+ valid_count = 0;+ }++ const int cin_per_group = cin / groups;+ const int cout_per_group = cout / groups;+ const int group_idx = (int)oc / cout_per_group;+ const int oc_in_group = (int)oc - group_idx * cout_per_group;+ const int in_begin = group_idx * cin_per_group;++ const int pad = (kernel_size - 1) * dilation;+ const float bias_value = float(bias[oc]);+ float acc0 = bias_value;+ float acc1 = bias_value;+ float acc2 = bias_value;+ float acc3 = bias_value;+ float acc4 = bias_value;+ float acc5 = bias_value;+ float acc6 = bias_value;+ float acc7 = bias_value;++ if (valid_count > 0) {+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;+ for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {+ const uint w_base =+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group ++ (uint)ic_local) *+ (uint)kernel_size;+ const int ic = in_begin + ic_local;+ const uint x_base_ncs = (b * (uint)cin + (uint)ic) * (uint)seq_len;++ for (int k = 0; k < kernel_size; ++k) {+ const float w = float(weight[w_base + (uint)k]);+ const int x_t = (int)t + k * dilation - pad;+ if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {+ // Fast path: all 8 lanes in bounds, use vectorized loads+ const uint x_idx = x_base_ncs + (uint)x_t;+ vec<T, 4> v0 = *(device const vec<T, 4>*)(input + x_idx);+ vec<T, 4> v1 = *(device const vec<T, 4>*)(input + x_idx + 4);+ acc0 += w * float(v0.x);+ acc1 += w * float(v0.y);+ acc2 += w * float(v0.z);+ acc3 += w * float(v0.w);+ acc4 += w * float(v1.x);+ acc5 += w * float(v1.y);+ acc6 += w * float(v1.z);+ acc7 += w * float(v1.w);+ continue;+ }+ // Slow path: per-lane boundary checks+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {+ acc0 += w * float(input[x_base_ncs + (uint)x_t]);+ }+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {+ acc1 += w * float(input[x_base_ncs + (uint)(x_t + 1)]);+ }+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {+ acc2 += w * float(input[x_base_ncs + (uint)(x_t + 2)]);+ }+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {+ acc3 += w * float(input[x_base_ncs + (uint)(x_t + 3)]);+ }+ if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {+ acc4 += w * float(input[x_base_ncs + (uint)(x_t + 4)]);+ }+ if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {+ acc5 += w * float(input[x_base_ncs + (uint)(x_t + 5)]);+ }+ if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {+ acc6 += w * float(input[x_base_ncs + (uint)(x_t + 6)]);+ }+ if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {+ acc7 += w * float(input[x_base_ncs + (uint)(x_t + 7)]);+ }+ }+ }+ }++ if (lane_count > 0) {+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;+ }+ if (lane_count > 1) {+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;+ }+ if (lane_count > 2) {+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;+ }+ if (lane_count > 3) {+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;+ }+ if (lane_count > 4) {+ output[out_base + 4] = valid_count > 4 ? (T)acc4 : (T)0;+ }+ if (lane_count > 5) {+ output[out_base + 5] = valid_count > 5 ? (T)acc5 : (T)0;+ }+ if (lane_count > 6) {+ output[out_base + 6] = valid_count > 6 ? (T)acc6 : (T)0;+ }+ if (lane_count > 7) {+ output[out_base + 7] = valid_count > 7 ? (T)acc7 : (T)0;+ }+}++template <typename T>+void causal_conv1d_grouped_nsc(+ device const T* input, // [B, T, Cin]+ device const T* weight, // [Cout, Cin/groups, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, T]+ device const int* lengths, // [B]+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& groups,+ const uint3 gid+) {+ const uint t = gid.x * AUDIO_TIME_TILE;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || oc >= (uint)cout) {+ return;+ }++ if (groups <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;+ int lane_count = seq_len - (int)t;+ if (lane_count > (int)AUDIO_TIME_TILE) {+ lane_count = (int)AUDIO_TIME_TILE;+ }+ if (lane_count <= 0) {+ return;+ }+ int valid_count = len_b - (int)t;+ if (valid_count > lane_count) {+ valid_count = lane_count;+ }+ if (valid_count < 0) {+ valid_count = 0;+ }++ const int cin_per_group = cin / groups;+ const int cout_per_group = cout / groups;+ const int group_idx = (int)oc / cout_per_group;+ const int oc_in_group = (int)oc - group_idx * cout_per_group;+ const int in_begin = group_idx * cin_per_group;++ const int pad = (kernel_size - 1) * dilation;+ const float bias_value = float(bias[oc]);+ float acc0 = bias_value;+ float acc1 = bias_value;+ float acc2 = bias_value;+ float acc3 = bias_value;+ float acc4 = bias_value;+ float acc5 = bias_value;+ float acc6 = bias_value;+ float acc7 = bias_value;++ if (valid_count > 0) {+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;+ for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {+ const int ic = in_begin + ic_local;+ const uint w_base =+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group ++ (uint)ic_local) *+ (uint)kernel_size;++ for (int k = 0; k < kernel_size; ++k) {+ const float w = float(weight[w_base + (uint)k]);+ const int x_t = (int)t + k * dilation - pad;+ if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {+ const uint x_idx0 =+ (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;+ acc0 += w * float(input[x_idx0]);+ acc1 += w * float(input[x_idx0 + (uint)cin]);+ acc2 += w * float(input[x_idx0 + (uint)(2 * cin)]);+ acc3 += w * float(input[x_idx0 + (uint)(3 * cin)]);+ acc4 += w * float(input[x_idx0 + (uint)(4 * cin)]);+ acc5 += w * float(input[x_idx0 + (uint)(5 * cin)]);+ acc6 += w * float(input[x_idx0 + (uint)(6 * cin)]);+ acc7 += w * float(input[x_idx0 + (uint)(7 * cin)]);+ continue;+ }+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)x_t) * (uint)cin + (uint)ic;+ acc0 += w * float(input[x_idx]);+ }+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)(x_t + 1)) * (uint)cin + (uint)ic;+ acc1 += w * float(input[x_idx]);+ }+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)(x_t + 2)) * (uint)cin + (uint)ic;+ acc2 += w * float(input[x_idx]);+ }+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)(x_t + 3)) * (uint)cin + (uint)ic;+ acc3 += w * float(input[x_idx]);+ }+ if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)(x_t + 4)) * (uint)cin + (uint)ic;+ acc4 += w * float(input[x_idx]);+ }+ if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)(x_t + 5)) * (uint)cin + (uint)ic;+ acc5 += w * float(input[x_idx]);+ }+ if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)(x_t + 6)) * (uint)cin + (uint)ic;+ acc6 += w * float(input[x_idx]);+ }+ if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {+ const uint x_idx =+ (b * (uint)seq_len + (uint)(x_t + 7)) * (uint)cin + (uint)ic;+ acc7 += w * float(input[x_idx]);+ }+ }+ }+ }++ if (lane_count > 0) {+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;+ }+ if (lane_count > 1) {+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;+ }+ if (lane_count > 2) {+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;+ }+ if (lane_count > 3) {+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;+ }+ if (lane_count > 4) {+ output[out_base + 4] = valid_count > 4 ? (T)acc4 : (T)0;+ }+ if (lane_count > 5) {+ output[out_base + 5] = valid_count > 5 ? (T)acc5 : (T)0;+ }+ if (lane_count > 6) {+ output[out_base + 6] = valid_count > 6 ? (T)acc6 : (T)0;+ }+ if (lane_count > 7) {+ output[out_base + 7] = valid_count > 7 ? (T)acc7 : (T)0;+ }+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioCausalConv1dGrouped)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& groups,+ const constant int& input_layout,+ const constant int& batch_size,+ uint t AXIS((seq_len + 7) / 8, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ if (input_layout == AUDIO_LAYOUT_NCS) {+ causal_conv1d_grouped_ncs<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len,+ kernel_size,+ dilation,+ groups,+ uint3(t, oc, b)+ );+ } else {+ causal_conv1d_grouped_nsc<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len,+ kernel_size,+ dilation,+ groups,+ uint3(t, oc, b)+ );+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metalnew file mode 100644index 000000000..bfc694c9f--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv1d_grouped_residual.metal@@ -0,0 +1,195 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++constant uint AUDIO_TIME_TILE = 8;++template <typename T>+void causal_conv1d_grouped_residual(+ device const T* input, // [B, Cin, T]+ device const T* residual, // [B, Cout, T]+ device const T* weight, // [Cout, Cin/groups, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, T]+ device const int* lengths, // [B]+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& groups,+ const uint3 gid+) {+ const uint t = gid.x * AUDIO_TIME_TILE;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || oc >= (uint)cout) {+ return;+ }++ if (groups <= 0 || (cin % groups) != 0 || (cout % groups) != 0) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len + t;+ int lane_count = seq_len - (int)t;+ if (lane_count > (int)AUDIO_TIME_TILE) {+ lane_count = (int)AUDIO_TIME_TILE;+ }+ if (lane_count <= 0) {+ return;+ }+ int valid_count = len_b - (int)t;+ if (valid_count > lane_count) {+ valid_count = lane_count;+ }+ if (valid_count < 0) {+ valid_count = 0;+ }++ const float bias_value = float(bias[oc]);+ const float residual0 = lane_count > 0 ? float(residual[out_base]) : 0.0f;+ const float residual1 = lane_count > 1 ? float(residual[out_base + 1]) : 0.0f;+ const float residual2 = lane_count > 2 ? float(residual[out_base + 2]) : 0.0f;+ const float residual3 = lane_count > 3 ? float(residual[out_base + 3]) : 0.0f;+ const float residual4 = lane_count > 4 ? float(residual[out_base + 4]) : 0.0f;+ const float residual5 = lane_count > 5 ? float(residual[out_base + 5]) : 0.0f;+ const float residual6 = lane_count > 6 ? float(residual[out_base + 6]) : 0.0f;+ const float residual7 = lane_count > 7 ? float(residual[out_base + 7]) : 0.0f;+ float acc0 = bias_value + residual0;+ float acc1 = bias_value + residual1;+ float acc2 = bias_value + residual2;+ float acc3 = bias_value + residual3;+ float acc4 = bias_value + residual4;+ float acc5 = bias_value + residual5;+ float acc6 = bias_value + residual6;+ float acc7 = bias_value + residual7;++ const int cin_per_group = cin / groups;+ const int cout_per_group = cout / groups;+ const int group_idx = (int)oc / cout_per_group;+ const int oc_in_group = (int)oc - group_idx * cout_per_group;+ const int in_begin = group_idx * cin_per_group;++ const int pad = (kernel_size - 1) * dilation;+ if (valid_count > 0) {+ const bool full_tile = valid_count == (int)AUDIO_TIME_TILE;+ for (int ic_local = 0; ic_local < cin_per_group; ++ic_local) {+ const int ic = in_begin + ic_local;+ const uint w_base =+ ((group_idx * cout_per_group + oc_in_group) * (uint)cin_per_group ++ (uint)ic_local) *+ (uint)kernel_size;+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;++ for (int k = 0; k < kernel_size; ++k) {+ const float w = float(weight[w_base + (uint)k]);+ const int x_t = (int)t + k * dilation - pad;+ if (full_tile && x_t >= 0 && x_t + 7 < seq_len) {+ // Fast path: all 8 lanes in bounds, use vectorized loads+ const uint x_idx = x_base + (uint)x_t;+ vec<T, 4> v0 = *(device const vec<T, 4>*)(input + x_idx);+ vec<T, 4> v1 = *(device const vec<T, 4>*)(input + x_idx + 4);+ acc0 += w * float(v0.x);+ acc1 += w * float(v0.y);+ acc2 += w * float(v0.z);+ acc3 += w * float(v0.w);+ acc4 += w * float(v1.x);+ acc5 += w * float(v1.y);+ acc6 += w * float(v1.z);+ acc7 += w * float(v1.w);+ continue;+ }+ if (valid_count > 0 && x_t >= 0 && x_t < seq_len) {+ acc0 += w * float(input[x_base + (uint)x_t]);+ }+ if (valid_count > 1 && x_t + 1 >= 0 && x_t + 1 < seq_len) {+ acc1 += w * float(input[x_base + (uint)(x_t + 1)]);+ }+ if (valid_count > 2 && x_t + 2 >= 0 && x_t + 2 < seq_len) {+ acc2 += w * float(input[x_base + (uint)(x_t + 2)]);+ }+ if (valid_count > 3 && x_t + 3 >= 0 && x_t + 3 < seq_len) {+ acc3 += w * float(input[x_base + (uint)(x_t + 3)]);+ }+ if (valid_count > 4 && x_t + 4 >= 0 && x_t + 4 < seq_len) {+ acc4 += w * float(input[x_base + (uint)(x_t + 4)]);+ }+ if (valid_count > 5 && x_t + 5 >= 0 && x_t + 5 < seq_len) {+ acc5 += w * float(input[x_base + (uint)(x_t + 5)]);+ }+ if (valid_count > 6 && x_t + 6 >= 0 && x_t + 6 < seq_len) {+ acc6 += w * float(input[x_base + (uint)(x_t + 6)]);+ }+ if (valid_count > 7 && x_t + 7 >= 0 && x_t + 7 < seq_len) {+ acc7 += w * float(input[x_base + (uint)(x_t + 7)]);+ }+ }+ }+ }++ if (lane_count > 0) {+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)residual0;+ }+ if (lane_count > 1) {+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)residual1;+ }+ if (lane_count > 2) {+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)residual2;+ }+ if (lane_count > 3) {+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)residual3;+ }+ if (lane_count > 4) {+ output[out_base + 4] = valid_count > 4 ? (T)acc4 : (T)residual4;+ }+ if (lane_count > 5) {+ output[out_base + 5] = valid_count > 5 ? (T)acc5 : (T)residual5;+ }+ if (lane_count > 6) {+ output[out_base + 6] = valid_count > 6 ? (T)acc6 : (T)residual6;+ }+ if (lane_count > 7) {+ output[out_base + 7] = valid_count > 7 ? (T)acc7 : (T)residual7;+ }+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioCausalConv1dGroupedResidual)(+ device const T* input,+ device const T* residual,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& groups,+ const constant int& batch_size,+ uint t AXIS((seq_len + 7) / 8, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ causal_conv1d_grouped_residual<T>(+ input,+ residual,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len,+ kernel_size,+ dilation,+ groups,+ uint3(t, oc, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metalnew file mode 100644index 000000000..016716aaa--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d.metal@@ -0,0 +1,106 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++template <typename T>+void causal_conv_transpose1d(+ device const T* input, // [B, Cin, Tin]+ device const T* weight, // [Cout, Cin_per_group, 2*stride]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, Tout]+ device const int* lengths, // [B] (output lengths)+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& stride,+ const constant int& groups,+ const uint3 gid+) {+ const uint t_out = gid.x;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len_out;+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;+ if ((int)t_out >= len_b) {+ output[out_idx] = (T)0;+ return;+ }++ const int cout_per_group = cout / groups;+ const int cin_per_group = cin / groups;+ const int group_idx = (int)oc / cout_per_group;++ const int q = (int)t_out / stride;+ const int r = (int)t_out - q * stride;++ float acc = float(bias[oc]);++ const uint in_base_b = b * (uint)cin * (uint)seq_len_in;+ const int ic_begin = group_idx * cin_per_group;+ const int ic_end = ic_begin + cin_per_group;++ for (int ic = ic_begin; ic < ic_end; ++ic) {+ if (q >= seq_len_in) {+ continue;+ }+ const uint in_base = in_base_b + (uint)ic * (uint)seq_len_in;++ // Weight layout: [Cout, Cin_per_group, K]+ const uint ic_local = (uint)(ic - ic_begin);+ const uint w_base =+ ((uint)oc * (uint)cin_per_group + ic_local) * (uint)(2 * stride);++ // Contribution from input[q] with k=r+ acc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);++ // Contribution from input[q-1] with k=stride+r+ if (q > 0) {+ acc += float(weight[w_base + (uint)(stride + r)]) *+ float(input[in_base + (uint)(q - 1)]);+ }+ }++ output[out_idx] = (T)acc;+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioCausalConvTranspose1d)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& stride,+ const constant int& groups,+ const constant int& batch_size,+ uint t_out AXIS(seq_len_out, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ causal_conv_transpose1d<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ stride,+ groups,+ uint3(t_out, oc, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metalnew file mode 100644index 000000000..46d3bed81--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/causal_conv_transpose1d_causal_pad.metal@@ -0,0 +1,479 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++constant uint AUDIO_TIME_TILE = 4;++// Input layout for causal_conv_transpose1d_causal_pad:+// 0 = NCS ([B, Cin, Tin]), 1 = NSC ([B, Tin, Cin]).+constant int AUDIO_LAYOUT_NCS = 0;+constant int AUDIO_LAYOUT_NSC = 1;++// === Causal ConvTranspose1d (causal-pad semantics, arbitrary kernel_size) ===+//+// This matches causal_conv_transpose1d_causal_pad_reference():+// - Expand input by stride with zeros inserted between samples.+// - Left pad by (kernel_size - 1), no right pad.+// - Convolve with provided kernel along expanded timeline.+template <typename T>+void causal_conv_transpose1d_causal_pad(+ device const T* input, // [B, Cin, Tin] when NCS, [B, Tin, Cin] when NSC+ device const T* weight, // [Cout, Cin_per_group, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, Tout]+ device const int* lengths, // [B]+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& kernel_size,+ const constant int& stride,+ const constant int& groups,+ const constant int& input_layout,+ const uint3 gid+) {+ const uint t_out = gid.x * AUDIO_TIME_TILE;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {+ return;+ }+ if ((input_layout != AUDIO_LAYOUT_NCS && input_layout != AUDIO_LAYOUT_NSC) ||+ groups <= 0 || stride <= 0 || kernel_size <= 0 || (cin % groups) != 0 ||+ (cout % groups) != 0) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len_out;+ const uint out_base = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;+ int lane_count = seq_len_out - (int)t_out;+ if (lane_count > (int)AUDIO_TIME_TILE) {+ lane_count = (int)AUDIO_TIME_TILE;+ }+ if (lane_count <= 0) {+ return;+ }+ int valid_count = len_b - (int)t_out;+ if (valid_count > lane_count) {+ valid_count = lane_count;+ }+ if (valid_count < 0) {+ valid_count = 0;+ }++ const int cout_per_group = cout / groups;+ const int cin_per_group = cin / groups;+ const int group_idx = (int)oc / cout_per_group;+ const int ic_begin = group_idx * cin_per_group;+ const int ic_end = ic_begin + cin_per_group;++ const int seq_len_expanded =+ (seq_len_in > 0) ? (((seq_len_in - 1) * stride) + 1) : 0;+ const int left_pad = kernel_size - 1;+ const bool fast_two_tap = (kernel_size == (stride * 2));++ const float bias_value = float(bias[oc]);+ float acc0 = bias_value;+ float acc1 = bias_value;+ float acc2 = bias_value;+ float acc3 = bias_value;++ // Process each lane that is valid (within sequence length)+ if (valid_count > 0) {+ // Precompute q and r for lane 0; consecutive lanes have t_out+1, t_out+2,+ // t_out+3.+ const int t0 = (int)t_out;+ const int q0 = t0 / stride;+ const int r0 = t0 % stride;++ if (input_layout == AUDIO_LAYOUT_NCS) {+ for (int ic = ic_begin; ic < ic_end; ++ic) {+ const uint in_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;+ const uint ic_local = (uint)(ic - ic_begin);+ const uint w_base =+ ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;+ if (fast_two_tap) {+ // When stride >= AUDIO_TIME_TILE (common for decoder blocks with+ // strides 4,8), all 4 lanes share the same q value. This means+ // they all read from the same input positions (q and q-1), just+ // with different weight indices. We load each input value once+ // and broadcast it across lanes.+ if (stride >= (int)AUDIO_TIME_TILE) {+ // All lanes have same q since t_out+3 < t_out+stride means+ // (t_out+3)/stride == t_out/stride.+ const float in_qm1 = (q0 > 0 && q0 - 1 < seq_len_in)+ ? float(input[in_base + (uint)(q0 - 1)])+ : 0.0f;+ const float in_q0 = (q0 >= 0 && q0 < seq_len_in)+ ? float(input[in_base + (uint)q0])+ : 0.0f;+ // Lane 0: r = r0+ if (valid_count > 0) {+ acc0 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);+ acc0 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r0)]);+ }+ // Lane 1: r = r0+1+ if (valid_count > 1) {+ const int r1 = r0 + 1;+ acc1 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);+ acc1 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ }+ // Lane 2: r = r0+2+ if (valid_count > 2) {+ const int r2 = r0 + 2;+ acc2 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);+ acc2 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ }+ // Lane 3: r = r0+3+ if (valid_count > 3) {+ const int r3 = r0 + 3;+ acc3 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);+ acc3 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ }+ } else {+ // stride < 4 (e.g., stride=2 for upsampler blocks).+ // Lanes may cross stride boundaries (different q values).+ if (valid_count > 0) {+ const int k_lo = (stride - 1) - r0;+ const int k_hi = (kernel_size - 1) - r0;+ if (q0 > 0 && q0 - 1 < seq_len_in) {+ acc0 += float(input[in_base + (uint)(q0 - 1)]) *+ float(weight[w_base + (uint)k_lo]);+ }+ if (q0 >= 0 && q0 < seq_len_in) {+ acc0 += float(input[in_base + (uint)q0]) *+ float(weight[w_base + (uint)k_hi]);+ }+ }+ if (valid_count > 1) {+ const int q1 = (t0 + 1) / stride;+ const int r1 = (t0 + 1) % stride;+ if (q1 > 0 && q1 - 1 < seq_len_in) {+ acc1 += float(input[in_base + (uint)(q1 - 1)]) *+ float(weight[w_base + (uint)((stride - 1) - r1)]);+ }+ if (q1 >= 0 && q1 < seq_len_in) {+ acc1 += float(input[in_base + (uint)q1]) *+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ }+ }+ if (valid_count > 2) {+ const int q2 = (t0 + 2) / stride;+ const int r2 = (t0 + 2) % stride;+ if (q2 > 0 && q2 - 1 < seq_len_in) {+ acc2 += float(input[in_base + (uint)(q2 - 1)]) *+ float(weight[w_base + (uint)((stride - 1) - r2)]);+ }+ if (q2 >= 0 && q2 < seq_len_in) {+ acc2 += float(input[in_base + (uint)q2]) *+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ }+ }+ if (valid_count > 3) {+ const int q3 = (t0 + 3) / stride;+ const int r3 = (t0 + 3) % stride;+ if (q3 > 0 && q3 - 1 < seq_len_in) {+ acc3 += float(input[in_base + (uint)(q3 - 1)]) *+ float(weight[w_base + (uint)((stride - 1) - r3)]);+ }+ if (q3 >= 0 && q3 < seq_len_in) {+ acc3 += float(input[in_base + (uint)q3]) *+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ }+ }+ }+ } else {+ for (int k = 0; k < kernel_size; ++k) {+ const float w = float(weight[w_base + (uint)k]);+ // Lane 0+ if (valid_count > 0) {+ const int expanded_time0 = (int)t_out + k - left_pad;+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&+ (expanded_time0 % stride) == 0) {+ const int src_time0 = expanded_time0 / stride;+ if (src_time0 >= 0 && src_time0 < seq_len_in) {+ acc0 += float(input[in_base + (uint)src_time0]) * w;+ }+ }+ }+ // Lane 1+ if (valid_count > 1) {+ const int expanded_time1 = (int)t_out + 1 + k - left_pad;+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&+ (expanded_time1 % stride) == 0) {+ const int src_time1 = expanded_time1 / stride;+ if (src_time1 >= 0 && src_time1 < seq_len_in) {+ acc1 += float(input[in_base + (uint)src_time1]) * w;+ }+ }+ }+ // Lane 2+ if (valid_count > 2) {+ const int expanded_time2 = (int)t_out + 2 + k - left_pad;+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&+ (expanded_time2 % stride) == 0) {+ const int src_time2 = expanded_time2 / stride;+ if (src_time2 >= 0 && src_time2 < seq_len_in) {+ acc2 += float(input[in_base + (uint)src_time2]) * w;+ }+ }+ }+ // Lane 3+ if (valid_count > 3) {+ const int expanded_time3 = (int)t_out + 3 + k - left_pad;+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&+ (expanded_time3 % stride) == 0) {+ const int src_time3 = expanded_time3 / stride;+ if (src_time3 >= 0 && src_time3 < seq_len_in) {+ acc3 += float(input[in_base + (uint)src_time3]) * w;+ }+ }+ }+ }+ }+ }+ } else {+ // NSC layout+ for (int ic = ic_begin; ic < ic_end; ++ic) {+ const uint ic_local = (uint)(ic - ic_begin);+ const uint w_base =+ ((uint)oc * (uint)cin_per_group + ic_local) * (uint)kernel_size;+ if (fast_two_tap) {+ if (stride >= (int)AUDIO_TIME_TILE) {+ // All lanes share q0; load input once per (q-1, q) pair.+ const float in_qm1 =+ (q0 > 0 && q0 - 1 < seq_len_in)+ ? float(input+ [(b * (uint)seq_len_in + (uint)(q0 - 1)) *+ (uint)cin ++ (uint)ic])+ : 0.0f;+ const float in_q0 =+ (q0 >= 0 && q0 < seq_len_in)+ ? float(input+ [(b * (uint)seq_len_in + (uint)q0) * (uint)cin ++ (uint)ic])+ : 0.0f;+ if (valid_count > 0) {+ acc0 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r0)]);+ acc0 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r0)]);+ }+ if (valid_count > 1) {+ const int r1 = r0 + 1;+ acc1 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r1)]);+ acc1 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ }+ if (valid_count > 2) {+ const int r2 = r0 + 2;+ acc2 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r2)]);+ acc2 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ }+ if (valid_count > 3) {+ const int r3 = r0 + 3;+ acc3 +=+ in_qm1 * float(weight[w_base + (uint)((stride - 1) - r3)]);+ acc3 += in_q0 *+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ }+ } else {+ // stride < 4: lanes may have different q values.+ if (valid_count > 0) {+ if (q0 > 0 && q0 - 1 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)(q0 - 1)) * (uint)cin ++ (uint)ic;+ acc0 += float(input[in_idx]) *+ float(weight[w_base + (uint)((stride - 1) - r0)]);+ }+ if (q0 >= 0 && q0 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)q0) * (uint)cin + (uint)ic;+ acc0 += float(input[in_idx]) *+ float(weight[w_base + (uint)((kernel_size - 1) - r0)]);+ }+ }+ if (valid_count > 1) {+ const int q1 = (t0 + 1) / stride;+ const int r1 = (t0 + 1) % stride;+ if (q1 > 0 && q1 - 1 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)(q1 - 1)) * (uint)cin ++ (uint)ic;+ acc1 += float(input[in_idx]) *+ float(weight[w_base + (uint)((stride - 1) - r1)]);+ }+ if (q1 >= 0 && q1 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)q1) * (uint)cin + (uint)ic;+ acc1 += float(input[in_idx]) *+ float(weight[w_base + (uint)((kernel_size - 1) - r1)]);+ }+ }+ if (valid_count > 2) {+ const int q2 = (t0 + 2) / stride;+ const int r2 = (t0 + 2) % stride;+ if (q2 > 0 && q2 - 1 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)(q2 - 1)) * (uint)cin ++ (uint)ic;+ acc2 += float(input[in_idx]) *+ float(weight[w_base + (uint)((stride - 1) - r2)]);+ }+ if (q2 >= 0 && q2 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)q2) * (uint)cin + (uint)ic;+ acc2 += float(input[in_idx]) *+ float(weight[w_base + (uint)((kernel_size - 1) - r2)]);+ }+ }+ if (valid_count > 3) {+ const int q3 = (t0 + 3) / stride;+ const int r3 = (t0 + 3) % stride;+ if (q3 > 0 && q3 - 1 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)(q3 - 1)) * (uint)cin ++ (uint)ic;+ acc3 += float(input[in_idx]) *+ float(weight[w_base + (uint)((stride - 1) - r3)]);+ }+ if (q3 >= 0 && q3 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)q3) * (uint)cin + (uint)ic;+ acc3 += float(input[in_idx]) *+ float(weight[w_base + (uint)((kernel_size - 1) - r3)]);+ }+ }+ }+ } else {+ for (int k = 0; k < kernel_size; ++k) {+ const float w = float(weight[w_base + (uint)k]);+ // Lane 0+ if (valid_count > 0) {+ const int expanded_time0 = (int)t_out + k - left_pad;+ if (expanded_time0 >= 0 && expanded_time0 < seq_len_expanded &&+ (expanded_time0 % stride) == 0) {+ const int src_time0 = expanded_time0 / stride;+ if (src_time0 >= 0 && src_time0 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)src_time0) * (uint)cin ++ (uint)ic;+ acc0 += float(input[in_idx]) * w;+ }+ }+ }+ // Lane 1+ if (valid_count > 1) {+ const int expanded_time1 = (int)t_out + 1 + k - left_pad;+ if (expanded_time1 >= 0 && expanded_time1 < seq_len_expanded &&+ (expanded_time1 % stride) == 0) {+ const int src_time1 = expanded_time1 / stride;+ if (src_time1 >= 0 && src_time1 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)src_time1) * (uint)cin ++ (uint)ic;+ acc1 += float(input[in_idx]) * w;+ }+ }+ }+ // Lane 2+ if (valid_count > 2) {+ const int expanded_time2 = (int)t_out + 2 + k - left_pad;+ if (expanded_time2 >= 0 && expanded_time2 < seq_len_expanded &&+ (expanded_time2 % stride) == 0) {+ const int src_time2 = expanded_time2 / stride;+ if (src_time2 >= 0 && src_time2 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)src_time2) * (uint)cin ++ (uint)ic;+ acc2 += float(input[in_idx]) * w;+ }+ }+ }+ // Lane 3+ if (valid_count > 3) {+ const int expanded_time3 = (int)t_out + 3 + k - left_pad;+ if (expanded_time3 >= 0 && expanded_time3 < seq_len_expanded &&+ (expanded_time3 % stride) == 0) {+ const int src_time3 = expanded_time3 / stride;+ if (src_time3 >= 0 && src_time3 < seq_len_in) {+ const uint in_idx =+ (b * (uint)seq_len_in + (uint)src_time3) * (uint)cin ++ (uint)ic;+ acc3 += float(input[in_idx]) * w;+ }+ }+ }+ }+ }+ }+ }+ }++ if (lane_count > 0) {+ output[out_base] = valid_count > 0 ? (T)acc0 : (T)0;+ }+ if (lane_count > 1) {+ output[out_base + 1] = valid_count > 1 ? (T)acc1 : (T)0;+ }+ if (lane_count > 2) {+ output[out_base + 2] = valid_count > 2 ? (T)acc2 : (T)0;+ }+ if (lane_count > 3) {+ output[out_base + 3] = valid_count > 3 ? (T)acc3 : (T)0;+ }+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioCausalConvTranspose1dCausalPad)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& kernel_size,+ const constant int& stride,+ const constant int& groups,+ const constant int& input_layout,+ const constant int& batch_size,+ uint t_out AXIS((seq_len_out + 3) / 4, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ causal_conv_transpose1d_causal_pad<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ kernel_size,+ stride,+ groups,+ input_layout,+ uint3(t_out, oc, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/conv1d.metal b/crates/uzu/src/backends/metal/kernel/audio/conv1d.metalnew file mode 100644index 000000000..33e61d901--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/conv1d.metal@@ -0,0 +1,100 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++template <typename T>+void conv1d(+ device const T* input, // [B, Cin, Tin]+ device const T* weight, // [Cout, Cin, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, Tout]+ device const int* lengths, // [B] (Tout lengths)+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& kernel_size,+ const constant int& stride,+ const constant int& dilation,+ const constant int& padding,+ const constant int& pad_mode,+ const uint3 gid+) {+ const uint t_out = gid.x;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len_out;+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;+ if ((int)t_out >= len_b) {+ output[out_idx] = (T)0;+ return;+ }++ const int base = (int)t_out * stride - padding;+ float acc = float(bias[oc]);++ for (int ic = 0; ic < cin; ++ic) {+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;+ for (int k = 0; k < kernel_size; ++k) {+ int x_t = base + k * dilation;+ if (pad_mode == 0) {+ if (x_t < 0 || x_t >= seq_len_in) {+ continue;+ }+ } else {+ x_t = clamp(x_t, 0, seq_len_in - 1);+ }+ acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);+ }+ }++ output[out_idx] = (T)acc;+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioConv1d)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& kernel_size,+ const constant int& stride,+ const constant int& dilation,+ const constant int& padding,+ const constant int& pad_mode,+ const constant int& batch_size,+ uint t_out AXIS(seq_len_out, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ conv1d<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ kernel_size,+ stride,+ dilation,+ padding,+ pad_mode,+ uint3(t_out, oc, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/fsq_decode.metal b/crates/uzu/src/backends/metal/kernel/audio/fsq_decode.metalnew file mode 100644index 000000000..95e3fb2db--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/fsq_decode.metal@@ -0,0 +1,83 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++template <typename T>+void fsq_decode(+ device const int* tokens,+ device T* out,+ device const int* lengths,+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const constant int* dim_base_index,+ const uint3 gid+) {+ const uint t = gid.x;+ const uint gd = gid.y;+ const uint b = gid.z;+ const uint channels = (uint)(num_groups * codebook_dim);++ if (t >= (uint)seq_len || gd >= channels) {+ return;+ }++ const uint g = gd / (uint)codebook_dim;+ const uint d = gd % (uint)codebook_dim;++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_idx = (b * channels + gd) * (uint)seq_len + t;+ if ((int)t >= len_b) {+ out[out_idx] = (T)0;+ return;+ }++ const uint token_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;+ const int token = tokens[token_idx];++ const int levels = num_levels[d];+ const int scale = levels / 2; // integer division (matches PyTorch // 2)+ const int offset = scale;+ const int base = dim_base_index[d];++ // Euclidean modulo to keep result in [0, levels) even if token is negative+ // (tokens are expected nonnegative, but keep this robust).+ const int div = token / base;+ int code_nonneg = div % levels;+ if (code_nonneg < 0) {+ code_nonneg += levels;+ }+ const float code = ((float)(code_nonneg - offset)) / (float)scale;+ out[out_idx] = (T)code;+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioFsqDecode)(+ device const int* tokens,+ device T* out,+ device const int* lengths,+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const constant int* dim_base_index,+ const constant int& batch_size,+ uint t AXIS(seq_len, 32),+ uint gd AXIS(num_groups * codebook_dim, 1),+ uint b AXIS(batch_size, 1)+) {+ fsq_decode<T>(+ tokens,+ out,+ lengths,+ num_groups,+ seq_len,+ codebook_dim,+ num_levels,+ dim_base_index,+ uint3(t, gd, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/fsq_encode.metal b/crates/uzu/src/backends/metal/kernel/audio/fsq_encode.metalnew file mode 100644index 000000000..34320fcae--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/fsq_encode.metal@@ -0,0 +1,102 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++inline float round_ties_to_even(const float x) {+ const float f = floor(x);+ const float frac = x - f;+ if (frac < 0.5f) {+ return f;+ }+ if (frac > 0.5f) {+ return f + 1.0f;+ }+ const int fi = (int)f; // f is integral+ return ((fi & 1) != 0) ? (f + 1.0f) : f;+}++template <typename T>+void fsq_encode(+ device const T* input, // [B, G*D, T]+ device int* tokens, // [B, G, T]+ device const int* lengths, // [B]+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const constant int* dim_base_index,+ const constant float& eps,+ const uint3 gid+) {+ const uint t = gid.x;+ const uint g = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || g >= (uint)num_groups) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;+ if ((int)t >= len_b) {+ tokens[out_idx] = 0;+ return;+ }++ const uint in_base =+ (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *+ (uint)seq_len ++ t;++ int token = 0;+ for (int d = 0; d < codebook_dim; ++d) {+ const int levels = num_levels[d];+ const int scale_i = levels / 2;+ const float output_scale = ((float)(levels - 1)) * 0.5f * (1.0f - eps);+ const float output_offset = (levels % 2 == 0) ? 0.5f : 0.0f;+ const float input_shift = tan(output_offset / output_scale);++ const float x = float(input[in_base + (uint)d * (uint)seq_len]);+ const float compressed =+ output_scale * tanh(x + input_shift) - output_offset;+ const float rounded = round_ties_to_even(compressed);++ int code_nonneg = (int)rounded + scale_i;+ code_nonneg = clamp(code_nonneg, 0, levels - 1);+ token += code_nonneg * dim_base_index[d];+ }++ tokens[out_idx] = token;+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioFsqEncode)(+ device const T* input,+ device int* tokens,+ device const int* lengths,+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const constant int* dim_base_index,+ const constant float& eps,+ const constant int& batch_size,+ uint t AXIS(seq_len, 32),+ uint g AXIS(num_groups, 1),+ uint b AXIS(batch_size, 1)+) {+ fsq_encode<T>(+ input,+ tokens,+ lengths,+ num_groups,+ seq_len,+ codebook_dim,+ num_levels,+ dim_base_index,+ eps,+ uint3(t, g, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/half_snake.metal b/crates/uzu/src/backends/metal/kernel/audio/half_snake.metalnew file mode 100644index 000000000..84e866842--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/half_snake.metal@@ -0,0 +1,68 @@+#include <metal_stdlib>+#include "../common/dsl.h"+#include "../activation/activations.h"++using namespace metal;++template <typename T>+void half_snake(+ device const T* input, // [B, C, T]+ device const T* alpha, // [1, C_snake, 1] (contiguous, index by channel)+ device T* output, // [B, C, T]+ const constant int& channels,+ const constant int& seq_len,+ const constant int& snake_channels,+ const constant float& negative_slope,+ const constant float& eps,+ const uint3 gid+) {+ const uint t = gid.x;+ const uint c = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || c >= (uint)channels) {+ return;+ }++ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;+ const float x = float(input[idx]);++ if ((int)c < snake_channels) {+ const float a = float(alpha[c]);+ const float ax = a * x;+ const float s = sin(ax);+ const float y = x + (1.0f / (a + eps)) * (s * s);+ output[idx] = (T)y;+ } else {+ output[idx] = activate_leaky_relu((T)x, negative_slope);+ }+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioHalfSnake)(+ device const T* input,+ device const T* alpha,+ device T* output,+ const constant int& channels,+ const constant int& seq_len,+ const constant int& snake_channels,+ const constant float& negative_slope,+ const constant float& eps,+ const constant int& batch_size,+ uint t AXIS(seq_len, 32),+ uint c AXIS(channels, 1),+ uint b AXIS(batch_size, 1)+) {+ half_snake<T>(+ input,+ alpha,+ output,+ channels,+ seq_len,+ snake_channels,+ negative_slope,+ eps,+ uint3(t, c, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metalnew file mode 100644index 000000000..46d59eb11--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/norm_ncs.metal@@ -0,0 +1,123 @@+#include <metal_stdlib>+#include "../common/dsl.h"+#include "../common/thread_context.h"+#include "../common/threadgroup_reduce.h"++using namespace metal;++#define AUDIO_NORM_NCS_BLOCK_SIZE 256+#define AUDIO_NORM_NCS_MAX_SIMDS 32++template <typename T>+void norm_ncs(+ device const T* input, // [B, C, T]+ device const T* scales, // [C]+ device const T* bias, // [C]+ device T* output, // [B, C, T]+ device const int* lengths, // [B]+ const constant int& channels,+ const constant int& seq_len,+ const constant float& epsilon,+ const constant int& subtract_mean,+ threadgroup float* shared_mean,+ threadgroup float* shared_variance,+ const thread ThreadContext& thread_context,+ const uint b,+ const uint t,+ const uint lid+) {+ if (t >= (uint)seq_len || channels <= 0) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;++ if ((int)t >= len_b) {+ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {+ const uint out_idx = (b * (uint)channels + c) * (uint)seq_len + t;+ output[out_idx] = (T)0;+ }+ return;+ }++ float partial_sum = 0.0f;+ if (subtract_mean != 0) {+ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;+ partial_sum += float(input[idx]);+ }+ }++ const float sum =+ threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(+ partial_sum,+ shared_mean,+ (ushort)lid,+ thread_context+ );+ const float mean = (subtract_mean != 0) ? (sum / (float)channels) : 0.0f;++ float partial_variance = 0.0f;+ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;+ const float x = float(input[idx]);+ const float centered = (subtract_mean != 0) ? (x - mean) : x;+ partial_variance += centered * centered;+ }++ const float variance_sum =+ threadgroup_cooperative_reduce_sum<AUDIO_NORM_NCS_BLOCK_SIZE>(+ partial_variance,+ shared_variance,+ (ushort)lid,+ thread_context+ );+ const float inv_std = rsqrt(variance_sum / (float)channels + epsilon);++ for (uint c = lid; c < (uint)channels; c += AUDIO_NORM_NCS_BLOCK_SIZE) {+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;+ const float x = float(input[idx]);+ const float centered = (subtract_mean != 0) ? (x - mean) : x;+ const float y = centered * inv_std * float(scales[c]) + float(bias[c]);+ output[idx] = (T)y;+ }+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioNormNcs)(+ device const T* input,+ device const T* scales,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& channels,+ const constant int& seq_len,+ const constant float& epsilon,+ const constant int& subtract_mean,+ const constant int& batch_size,+ threadgroup float shared_mean[AUDIO_NORM_NCS_MAX_SIMDS],+ threadgroup float shared_variance[AUDIO_NORM_NCS_MAX_SIMDS],+ const ThreadContext thread_context,+ const uint b GROUPS(batch_size),+ const uint t GROUPS(seq_len),+ const uint lid THREADS(AUDIO_NORM_NCS_BLOCK_SIZE)+) {+ norm_ncs<T>(+ input,+ scales,+ bias,+ output,+ lengths,+ channels,+ seq_len,+ epsilon,+ subtract_mean,+ shared_mean,+ shared_variance,+ thread_context,+ b,+ t,+ lid+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/audio/quantizer_decode.metal b/crates/uzu/src/backends/metal/kernel/audio/quantizer_decode.metalnew file mode 100644index 000000000..ec147ca1d--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/quantizer_decode.metal@@ -0,0 +1,120 @@+#include <metal_stdlib>+#include "../common/dsl.h"++using namespace metal;++template <typename T>+void quantizer_decode(+ device const uint* tokens, // [B, K, T]+ device const int* lengths, // [B]+ device const T* semantic_codebook, // [semantic_cardinality, codebook_dim]+ device const T* semantic_out_proj, // [input_dim, codebook_dim]+ device const T* semantic_out_bias, // [input_dim]+ device const T*+ residual_codebooks, // [R, residual_cardinality, codebook_dim]+ device const T* residual_out_proj, // [R, input_dim, codebook_dim]+ device const T* residual_out_bias, // [R, input_dim]+ device T* output, // [B, T, input_dim]+ const constant int& batch_size,+ const constant int& total_codebooks,+ const constant int& seq_len,+ const constant int& input_dim,+ const constant int& codebook_dim,+ const constant int& residual_quantizers,+ const constant int& semantic_cardinality,+ const constant int& residual_cardinality,+ const uint3 gid+) {+ const uint d = gid.x;+ const uint t = gid.y;+ const uint b = gid.z;++ if (d >= (uint)input_dim || t >= (uint)seq_len || b >= (uint)batch_size) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_idx = (b * (uint)seq_len + t) * (uint)input_dim + d;+ if ((int)t >= len_b) {+ output[out_idx] = (T)0;+ return;+ }++ const uint token_base = (b * (uint)total_codebooks) * (uint)seq_len + t;+ uint semantic_token = tokens[token_base];+ semantic_token = min(semantic_token, (uint)(semantic_cardinality - 1));++ float acc = float(semantic_out_bias[d]);+ const uint semantic_code_base = (uint)semantic_token * (uint)codebook_dim;+ const uint semantic_proj_base = d * (uint)codebook_dim;+ for (uint k = 0; k < (uint)codebook_dim; ++k) {+ acc += float(semantic_out_proj[semantic_proj_base + k]) *+ float(semantic_codebook[semantic_code_base + k]);+ }++ for (int r = 0; r < residual_quantizers; ++r) {+ const uint token_idx = token_base + (uint)(r + 1) * (uint)seq_len;+ uint residual_token = tokens[token_idx];+ residual_token = min(residual_token, (uint)(residual_cardinality - 1));++ const uint bias_idx = (uint)r * (uint)input_dim + d;+ acc += float(residual_out_bias[bias_idx]);++ const uint code_base =+ ((uint)r * (uint)residual_cardinality + (uint)residual_token) *+ (uint)codebook_dim;+ const uint proj_base = ((uint)r * (uint)input_dim + d) * (uint)codebook_dim;+ for (uint k = 0; k < (uint)codebook_dim; ++k) {+ acc += float(residual_out_proj[proj_base + k]) *+ float(residual_codebooks[code_base + k]);+ }+ }++ output[out_idx] = (T)acc;+}++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(AudioQuantizerDecode)(+ device const uint* tokens,+ device const int* lengths,+ device const T* semantic_codebook,+ device const T* semantic_out_proj,+ device const T* semantic_out_bias,+ device const T* residual_codebooks,+ device const T* residual_out_proj,+ device const T* residual_out_bias,+ device T* output,+ const constant int& batch_size,+ const constant int& total_codebooks,+ const constant int& seq_len,+ const constant int& input_dim,+ const constant int& codebook_dim,+ const constant int& residual_quantizers,+ const constant int& semantic_cardinality,+ const constant int& residual_cardinality,+ uint d AXIS(input_dim, 32),+ uint t AXIS(seq_len, 1),+ uint b AXIS(batch_size, 1)+) {+ quantizer_decode<T>(+ tokens,+ lengths,+ semantic_codebook,+ semantic_out_proj,+ semantic_out_bias,+ residual_codebooks,+ residual_out_proj,+ residual_out_bias,+ output,+ batch_size,+ total_codebooks,+ seq_len,+ input_dim,+ codebook_dim,+ residual_quantizers,+ semantic_cardinality,+ residual_cardinality,+ uint3(d, t, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/embedding/embedding_rows_sum.metal b/crates/uzu/src/backends/metal/kernel/embedding/embedding_rows_sum.metalnew file mode 100644index 000000000..fa08f9da0--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/embedding/embedding_rows_sum.metal@@ -0,0 +1,27 @@+#include <metal_stdlib>+#include "../common/dsl.h"++#define BLOCK_SIZE 256++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(EmbeddingRowsSum)(+ const device uint32_t* token_indices, // [num_rows]+ const device T* weights, // [total_rows, model_dim]+ device T* output, // [model_dim]+ constant uint32_t& num_rows,+ constant uint32_t& total_rows,+ constant uint32_t& model_dim,+ constant uint32_t& codebook_stride,+ uint dim_idx AXIS(model_dim, BLOCK_SIZE)+) {+ float sum = 0.0f;+ for (uint row_idx = 0; row_idx < num_rows; row_idx++) {+ uint token = token_indices[row_idx];+ uint row = row_idx * codebook_stride + token;+ if (row < total_rows) {+ sum += float(weights[row * model_dim + dim_idx]);+ }+ }+ output[dim_idx] = T(sum);+}diff --git a/crates/uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal b/crates/uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metalnew file mode 100644index 000000000..45afc5789--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/tensor_add_scale/tensor_add_scale.metal@@ -0,0 +1,18 @@+#include <metal_stdlib>+#include "../common/dsl.h"++template <typename T>+VARIANTS(T, float, half, bfloat)+PUBLIC KERNEL(TensorAddScale)(+ const device T* input,+ const device T* bias,+ device T* output,+ constant uint& num_cols,+ constant uint& length,+ constant float& scale,+ const uint position AXIS(length, 32)+) {+ uint col = position % num_cols;+ float value = (float(input[position]) + float(bias[col])) * scale;+ output[position] = T(value);+}diff --git a/crates/uzu/src/config/mod.rs b/crates/uzu/src/config/mod.rsindex c44e73c80..61c4abf13 100644--- a/crates/uzu/src/config/mod.rs+++ b/crates/uzu/src/config/mod.rs@@ -20,6 +20,7 @@ mod rope;mod short_conv;mod transformer;mod transformer_layer;+mod tts_model;pub use attention::*;pub use classifier::*;@@ -43,3 +44,4 @@ pub use rope::*;pub use short_conv::*;pub use transformer::*;pub use transformer_layer::*;+pub use tts_model::*;diff --git a/crates/uzu/src/config/model_config.rs b/crates/uzu/src/config/model_config.rsindex 90587d598..51673d313 100644--- a/crates/uzu/src/config/model_config.rs+++ b/crates/uzu/src/config/model_config.rs@@ -1,12 +1,13 @@use serde::{Deserialize, Serialize};-use super::{ClassifierModelConfig, LanguageModelConfig};+use super::{ClassifierModelConfig, LanguageModelConfig, TtsModelConfig};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]#[serde(untagged)]pub enum ModelConfig {LanguageModel(LanguageModelConfig),Classifier(ClassifierModelConfig),+ Tts(TtsModelConfig),}impl ModelConfig {@@ -24,6 +25,13 @@ impl ModelConfig {}}+ pub fn as_tts(&self) -> Option<&TtsModelConfig> {+ match self {+ ModelConfig::Tts(config) => Some(config),+ _ => None,+ }+ }+pub fn is_language_model(&self) -> bool {matches!(self, ModelConfig::LanguageModel(_))}@@ -31,4 +39,8 @@ impl ModelConfig {pub fn is_classifier(&self) -> bool {matches!(self, ModelConfig::Classifier(_))}++ pub fn is_tts(&self) -> bool {+ matches!(self, ModelConfig::Tts(_))+ }}diff --git a/crates/uzu/src/config/model_type.rs b/crates/uzu/src/config/model_type.rsindex e5dc7d691..1e27bf44d 100644--- a/crates/uzu/src/config/model_type.rs+++ b/crates/uzu/src/config/model_type.rs@@ -5,4 +5,5 @@ use serde::{Deserialize, Serialize};pub enum ModelType {LanguageModel,ClassifierModel,+ TtsModel,}diff --git a/crates/uzu/src/config/tts_model.rs b/crates/uzu/src/config/tts_model.rsnew file mode 100644index 000000000..23e407898--- /dev/null+++ b/crates/uzu/src/config/tts_model.rs@@ -0,0 +1,551 @@+use std::collections::BTreeMap;+#[cfg(all(feature = "audio-runtime", metal_backend))]+use std::path::Path;++use serde::{Deserialize, Serialize};++use super::{EmbeddingConfig, EmbeddingConfigCommon, LinearConfig, TransformerConfig};+use crate::ConfigDataType;++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct TtsMessageProcessorConfig {+ pub prompt_template: String,+ #[serde(default = "default_drop_initial_newline")]+ pub drop_initial_newline: bool,+ #[serde(default = "default_system_role_name")]+ pub system_role_name: String,+ #[serde(default = "default_user_role_name")]+ pub user_role_name: String,+ #[serde(default = "default_assistant_role_name")]+ pub assistant_role_name: String,+ #[serde(default = "default_tts_message_fields")]+ pub default_message_fields: BTreeMap<String, String>,+}++fn default_drop_initial_newline() -> bool {+ true+}++fn default_system_role_name() -> String {+ String::from("system")+}++fn default_user_role_name() -> String {+ String::from("user")+}++fn default_assistant_role_name() -> String {+ String::from("assistant")+}++fn default_tts_message_fields() -> BTreeMap<String, String> {+ BTreeMap::new()+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct TtsConfig {+ pub text_decoder_config: TtsTextDecoderConfig,+ pub audio_decoder_config: TtsAudioDecoderConfig,+ #[serde(default)]+ pub activation_precision: Option<ConfigDataType>,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(tag = "type")]+pub enum TtsTextDecoderConfig {+ FishAudioTextDecoderConfig {+ #[serde(flatten)]+ config: FishAudioTextDecoderConfig,+ },+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct FishAudioTextDecoderConfig {+ pub slow_embeddings_config: FishAudioEmbeddingConfig,+ pub slow_model_config: TransformerConfig,+ pub slow_readout_config: FishAudioLinearConfig,+ pub fast_embeddings_config: FishAudioEmbeddingConfig,+ pub fast_model_config: TransformerConfig,+ pub fast_readout_config: FishAudioLinearConfig,+ pub codebook_embeddings_config: FishAudioEmbeddingConfig,+ pub fast_model_projection_config: Option<FishAudioLinearConfig>,+ pub semantic_token_begin_id: i64,+ pub semantic_token_end_id: i64,+ pub im_end_token_id: i64,+ pub codebook_size: usize,+ pub vocab_size: usize,+ pub slow_model_dim: usize,+ pub fast_model_dim: usize,+ pub num_codebooks: usize,+ pub max_seq_len: usize,+ pub scale_codebook_embeddings: bool,+ #[serde(default)]+ pub precision: Option<ConfigDataType>,+ pub short_logits_size: usize,+ pub repeat_window_size: usize,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(untagged)]+pub enum FishAudioLinearConfig {+ Tagged(LinearConfig),+ UntaggedFullPrecision {+ #[serde(rename = "precision")]+ precision: ConfigDataType,+ },+}++impl FishAudioLinearConfig {+ pub fn is_full_precision(&self) -> bool {+ matches!(+ self,+ FishAudioLinearConfig::Tagged(LinearConfig::FullPrecision { .. })+ | FishAudioLinearConfig::UntaggedFullPrecision { .. }+ )+ }+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(untagged)]+pub enum FishAudioEmbeddingConfig {+ Tagged(EmbeddingConfig),+ UntaggedFullPrecision {+ input_scale: Option<f32>,+ logit_soft_cap: Option<f32>,+ precision: ConfigDataType,+ },+}++impl FishAudioEmbeddingConfig {+ pub fn to_embedding_config(&self) -> EmbeddingConfig {+ match self {+ FishAudioEmbeddingConfig::Tagged(config) => config.clone(),+ FishAudioEmbeddingConfig::UntaggedFullPrecision {+ input_scale,+ logit_soft_cap,+ precision,+ } => EmbeddingConfig::Tied {+ common: EmbeddingConfigCommon {+ input_scale: *input_scale,+ logit_soft_cap: *logit_soft_cap,+ },+ precision: *precision,+ },+ }+ }++ #[cfg(all(feature = "audio-runtime", metal_backend))]+ pub(crate) fn to_text_decoder_embedding_config(&self) -> EmbeddingConfig {+ match self {+ FishAudioEmbeddingConfig::Tagged(config) => config.clone(),+ FishAudioEmbeddingConfig::UntaggedFullPrecision {+ input_scale,+ logit_soft_cap,+ precision,+ } => EmbeddingConfig::Untied {+ common: EmbeddingConfigCommon {+ input_scale: *input_scale,+ logit_soft_cap: *logit_soft_cap,+ },+ precision: *precision,+ },+ }+ }+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(tag = "type")]+pub enum TtsAudioDecoderConfig {+ DescriptAudioCodecConfig {+ #[serde(flatten)]+ config: DescriptAudioCodecConfig,+ },+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct DescriptAudioCodecConfig {+ pub precision: ConfigDataType,+ pub quantizer_config: DescriptAudioQuantizerConfig,+ pub decoder_config: DescriptAudioDacDecoderConfig,+ pub samplerate: u32,+ pub encoder_dim: usize,+ pub encoder_rates: Vec<usize>,+ pub decoder_dim: usize,+ pub decoder_rates: Vec<usize>,+ pub input_dim: usize,+ pub n_codebooks: usize,+ pub codebook_dim: usize,+ pub downsample_factor: Vec<usize>,+ pub codebook_size: usize,+ pub semantic_codebook_size: usize,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(deny_unknown_fields)]+pub struct DescriptAudioDacDecoderConfig {+ pub precision: ConfigDataType,+ pub conv_config: DescriptAudioCausalConv1dConfig,+ pub snake_config: DescriptAudioSnake1dConfig,+ pub decoder_block_config: DescriptAudioDacDecoderBlockConfig,+ pub causal: bool,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(deny_unknown_fields)]+pub struct DescriptAudioDacDecoderBlockConfig {+ pub precision: ConfigDataType,+ pub snake_config: DescriptAudioSnake1dConfig,+ pub trans_conv_config: DescriptAudioCausalTransposeConv1dConfig,+ pub res_unit_config: DescriptAudioResidualUnitConfig,+ pub causal: bool,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(deny_unknown_fields)]+pub struct DescriptAudioResidualUnitConfig {+ pub precision: ConfigDataType,+ pub snake_config: DescriptAudioSnake1dConfig,+ pub conv_config: DescriptAudioCausalConv1dConfig,+ pub causal: bool,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(deny_unknown_fields)]+pub struct DescriptAudioCausalConv1dConfig {+ pub precision: ConfigDataType,+ pub has_biases: bool,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(deny_unknown_fields)]+pub struct DescriptAudioCausalTransposeConv1dConfig {+ pub precision: ConfigDataType,+ pub has_biases: bool,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+#[serde(deny_unknown_fields)]+pub struct DescriptAudioSnake1dConfig {+ pub precision: ConfigDataType,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct DescriptAudioQuantizerConfig {+ #[serde(default)]+ pub precision: Option<ConfigDataType>,+ pub post_module_config: TransformerConfig,+ pub upsampler_config: DescriptAudioUpsamplerConfig,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct DescriptAudioUpsamplerConfig {+ pub block_configs: Vec<DescriptAudioUpsamplingBlockConfig>,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct DescriptAudioUpsamplingBlockConfig {+ pub convnext_config: DescriptAudioConvNeXtConfig,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct DescriptAudioConvNeXtConfig {+ pub norm_config: DescriptAudioConvNeXtNormConfig,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct DescriptAudioConvNeXtNormConfig {+ pub epsilon: f32,+ #[serde(default)]+ pub subtract_mean: bool,+ #[serde(default)]+ pub use_bias: bool,+}++#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]+pub struct TtsModelConfig {+ pub tts_config: TtsConfig,+ pub message_processor_config: TtsMessageProcessorConfig,+}++#[cfg(all(feature = "audio-runtime", metal_backend))]+impl TtsModelConfig {+ pub fn create_audio_generation_context_with_model_path<B: crate::backends::common::Backend>(+ &self,+ model_path: &Path,+ ) -> crate::audio::AudioResult<crate::audio::AudioGenerationContext<B>> {+ crate::audio::AudioGenerationContext::from_tts_config_and_model_path(&self.tts_config, model_path)+ }+}++#[cfg(test)]+mod tests {+ use crate::config::{ModelConfig, ModelMetadata, ModelType};++ fn descript_audio_decoder_config_json() -> serde_json::Value {+ serde_json::json!({+ "precision": "float16",+ "conv_config": {+ "precision": "float16",+ "has_biases": true+ },+ "snake_config": {+ "precision": "float16"+ },+ "decoder_block_config": {+ "precision": "float16",+ "snake_config": {+ "precision": "float16"+ },+ "trans_conv_config": {+ "precision": "float16",+ "has_biases": true+ },+ "res_unit_config": {+ "precision": "float16",+ "snake_config": {+ "precision": "float16"+ },+ "conv_config": {+ "precision": "float16",+ "has_biases": true+ },+ "causal": true+ },+ "causal": true+ },+ "causal": true+ })+ }++ fn fishaudio_text_decoder_config_json() -> serde_json::Value {+ let norm_config = serde_json::json!({+ "scale_precision": "float32",+ "accumulation_precision": "float32",+ "epsilon": 1e-5,+ "scale_offset": null,+ "upcast_mode": "full_layer",+ "subtract_mean": true+ });+ let transformer_config = serde_json::json!({+ "global_rope_config": null,+ "local_rope_config": null,+ "layer_configs": [],+ "output_norm_config": norm_config,+ "model_dim": 256,+ "hidden_dim": 256,+ "context_length": 512+ });+ serde_json::json!({+ "type": "FishAudioTextDecoderConfig",+ "slow_embeddings_config": { "precision": "float16", "input_scale": null, "logit_soft_cap": null },+ "slow_model_config": transformer_config,+ "slow_readout_config": { "precision": "float16" },+ "fast_embeddings_config": { "precision": "float16", "input_scale": null, "logit_soft_cap": null },+ "fast_model_config": transformer_config,+ "fast_readout_config": { "precision": "float16" },+ "codebook_embeddings_config": { "precision": "float16", "input_scale": null, "logit_soft_cap": null },+ "fast_model_projection_config": null,+ "semantic_token_begin_id": 100,+ "semantic_token_end_id": 1123,+ "im_end_token_id": 4,+ "codebook_size": 1024,+ "vocab_size": 2048,+ "slow_model_dim": 256,+ "fast_model_dim": 256,+ "num_codebooks": 8,+ "max_seq_len": 4096,+ "scale_codebook_embeddings": false,+ "short_logits_size": 1124,+ "repeat_window_size": 16+ })+ }++ #[test]+ fn metadata_with_tts_model_type_parses() {+ let config_json = serde_json::json!({+ "toolchain_version": "0.0.0-test",+ "vendor": "NVIDIA",+ "family": "nanocodec",+ "name": "nemo-nano-codec-22khz-1.78kbps-12.5fps",+ "size": "0.1B",+ "quantization": null,+ "repo": "nvidia/nemo-nano-codec-22khz-1.78kbps-12.5fps",+ "use_cases": [],+ "model_type": "tts_model",+ "model_config": {+ "tts_config": {+ "text_decoder_config": fishaudio_text_decoder_config_json(),+ "audio_decoder_config": {+ "type": "DescriptAudioCodecConfig",+ "precision": "float16",+ "samplerate": 44100,+ "encoder_dim": 1024,+ "encoder_rates": [2, 4, 8, 8],+ "decoder_dim": 1536,+ "n_codebooks": 9,+ "codebook_dim": 8,+ "codebook_size": 1024,+ "semantic_codebook_size": 4096,+ "input_dim": 1024,+ "downsample_factor": [2, 2],+ "decoder_rates": [8, 8, 4, 2],+ "decoder_config": descript_audio_decoder_config_json(),+ "quantizer_config": {+ "post_module_config": {+ "global_rope_config": null,+ "local_rope_config": null,+ "layer_configs": [],+ "output_norm_config": {+ "scale_precision": "float32",+ "accumulation_precision": "float32",+ "epsilon": 1e-5,+ "scale_offset": null,+ "upcast_mode": "full_layer",+ "subtract_mean": true+ },+ "model_dim": 1024,+ "hidden_dim": 1024,+ "context_length": 1+ },+ "upsampler_config": {+ "block_configs": []+ }+ }+ },+ },+ "message_processor_config": {+ "prompt_template": "{% for message in messages %}{{message.content}}{% endfor %}",+ "drop_initial_newline": true+ }+ }+ });++ let metadata: ModelMetadata = serde_json::from_value(config_json).expect("metadata parse");+ assert_eq!(metadata.model_type, ModelType::TtsModel);++ let tts_config = metadata.model_config.as_tts().expect("tts config");+ assert_eq!(+ tts_config.message_processor_config.prompt_template,+ "{% for message in messages %}{{message.content}}{% endfor %}"+ );+ assert!(tts_config.message_processor_config.drop_initial_newline);+ assert!(tts_config.message_processor_config.default_message_fields.is_empty());+ }++ #[test]+ fn tts_model_config_helpers_work() {+ let model_config_json = serde_json::json!({+ "tts_config": {+ "text_decoder_config": fishaudio_text_decoder_config_json(),+ "audio_decoder_config": {+ "type": "DescriptAudioCodecConfig",+ "precision": "float16",+ "samplerate": 44100,+ "encoder_dim": 1024,+ "encoder_rates": [2, 4, 8, 8],+ "decoder_dim": 1536,+ "n_codebooks": 9,+ "codebook_dim": 8,+ "codebook_size": 1024,+ "semantic_codebook_size": 4096,+ "input_dim": 1024,+ "downsample_factor": [2, 2],+ "decoder_rates": [8, 8, 4, 2],+ "decoder_config": descript_audio_decoder_config_json(),+ "quantizer_config": {+ "post_module_config": {+ "global_rope_config": null,+ "local_rope_config": null,+ "layer_configs": [],+ "output_norm_config": {+ "scale_precision": "float32",+ "accumulation_precision": "float32",+ "epsilon": 1e-5,+ "scale_offset": null,+ "upcast_mode": "full_layer",+ "subtract_mean": true+ },+ "model_dim": 1024,+ "hidden_dim": 1024,+ "context_length": 1+ },+ "upsampler_config": {+ "block_configs": []+ }+ }+ },+ },+ "message_processor_config": {+ "prompt_template": "{{messages[0].content}}"+ }+ });++ let model_config: ModelConfig = serde_json::from_value(model_config_json).expect("model config parse");+ assert!(model_config.is_tts());+ assert!(model_config.as_tts().is_some());+ assert!(!model_config.is_classifier());+ assert!(!model_config.is_language_model());++ let tts = model_config.as_tts().expect("tts variant");+ assert_eq!(tts.message_processor_config.prompt_template, "{{messages[0].content}}");+ assert!(tts.message_processor_config.drop_initial_newline);+ assert!(tts.message_processor_config.default_message_fields.is_empty());+ }++ #[test]+ fn tts_model_config_rejects_untyped_decoder_config() {+ let model_config_json = serde_json::json!({+ "tts_config": {+ "text_decoder_config": fishaudio_text_decoder_config_json(),+ "audio_decoder_config": {+ "type": "DescriptAudioCodecConfig",+ "precision": "float16",+ "samplerate": 44100,+ "encoder_dim": 1024,+ "encoder_rates": [2, 4, 8, 8],+ "decoder_dim": 1536,+ "n_codebooks": 9,+ "codebook_dim": 8,+ "codebook_size": 1024,+ "semantic_codebook_size": 4096,+ "input_dim": 1024,+ "downsample_factor": [2, 2],+ "decoder_rates": [8, 8, 4, 2],+ "decoder_config": {},+ "quantizer_config": {+ "post_module_config": {+ "global_rope_config": null,+ "local_rope_config": null,+ "layer_configs": [],+ "output_norm_config": {+ "scale_precision": "float32",+ "accumulation_precision": "float32",+ "epsilon": 1e-5,+ "scale_offset": null,+ "upcast_mode": "full_layer",+ "subtract_mean": true+ },+ "model_dim": 1024,+ "hidden_dim": 1024,+ "context_length": 1+ },+ "upsampler_config": {+ "block_configs": []+ }+ }+ },+ },+ "message_processor_config": {+ "prompt_template": "{{messages[0].content}}"+ }+ });++ let error =+ serde_json::from_value::<ModelConfig>(model_config_json).expect_err("decoder_config should be typed");+ let error_text = error.to_string();+ assert!(+ error_text.contains("did not match any variant") || error_text.contains("conv_config"),+ "unexpected error: {error}"+ );+ }+}diff --git a/crates/uzu/src/encodable_block/decoder.rs b/crates/uzu/src/encodable_block/decoder.rsindex b2d4f700e..305d89a84 100644--- a/crates/uzu/src/encodable_block/decoder.rs+++ b/crates/uzu/src/encodable_block/decoder.rs@@ -37,17 +37,62 @@ impl<B: Backend> Decoder<B> {decoder_config: Rc<DecoderConfig>,root_weight_loader: &ParameterTree<B::Context>,) -> Self {- let decoder_weight_loader = root_weight_loader.subtree("transformer").expect("transformer subtree not found");+ Self::new_with_embedding_and_readout_subtrees(+ context,+ decoder_config,+ root_weight_loader,+ "transformer",+ "embedding",+ "embedding",+ )+ }++ /// Used by models whose token lookup weights and logits readout weights+ pub fn new_with_embedding_and_readout_subtrees(+ context: Rc<B::Context>,+ decoder_config: Rc<DecoderConfig>,+ root_weight_loader: &ParameterTree<B::Context>,+ transformer_subtree: &str,+ embedding_subtree: &str,+ readout_subtree: &str,+ ) -> Self {+ let embedding_weight_loader =+ root_weight_loader.subtree(embedding_subtree).expect("Failed to get embedding subtree");+ let readout_weight_loader = root_weight_loader.subtree(readout_subtree).expect("Failed to get readout subtree");- let embed = Embedding::new(+ let embed = Embedding::new_with_lookup_and_readout_trees(context.as_ref(),decoder_config.vocab_size as u32,decoder_config.model_dim as u32,&decoder_config.embedding_config,- &root_weight_loader.subtree("embedding").expect("Failed to get embedding subtree"),+ &embedding_weight_loader,+ &readout_weight_loader,).expect("Failed to create embedding");+ let (layers, norm) = Self::build_transformer_layers_and_norm(+ context,+ decoder_config.clone(),+ root_weight_loader,+ transformer_subtree,+ );++ Self {+ embed,+ layers,+ norm,+ }+ }++ pub(crate) fn build_transformer_layers_and_norm(+ context: Rc<B::Context>,+ decoder_config: Rc<DecoderConfig>,+ root_weight_loader: &ParameterTree<B::Context>,+ transformer_subtree: &str,+ ) -> (Box<[LayerExecutables<B>]>, RMSNorm<B>) {+ let decoder_weight_loader =+ root_weight_loader.subtree(transformer_subtree).expect("transformer subtree not found");+let attention_data_type = Self::attention_data_type(&decoder_config);let norm_reference_layer =decoder_config.layer_configs.as_ref().map(|configs| &configs[0]).unwrap_or(&decoder_config.layer_config);@@ -134,11 +179,7 @@ impl<B: Backend> Decoder<B> {.map(RMSNorm::with_sampling_range).expect("Failed to create output RMS norm kernel");- Self {- embed,- layers: layers.into_boxed_slice(),- norm: norm_block,- }+ (layers.into_boxed_slice(), norm_block)}fn create_rope_block(diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex cf281bbe5..92c68ed77 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -102,6 +102,33 @@ pub struct Embedding<B: Backend> {model_dim: u32,}+struct UntiedWeightKeys {+ lookup_weights: &'static str,+ lookup_scales: &'static str,+ lookup_biases: &'static str,+ readout_weights: &'static str,+ readout_scales: &'static str,+ readout_biases: &'static str,+}++const SHARED_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {+ lookup_weights: "input_weights",+ lookup_scales: "input_scales",+ lookup_biases: "input_biases",+ readout_weights: "output_weights",+ readout_scales: "output_scales",+ readout_biases: "output_biases",+};++const SPLIT_TREE_UNTIED_KEYS: UntiedWeightKeys = UntiedWeightKeys {+ lookup_weights: "weights",+ lookup_scales: "scales",+ lookup_biases: "biases",+ readout_weights: "weights",+ readout_scales: "scales",+ readout_biases: "biases",+};+fn validate_tensor<'file, 'context, 'leaf, B: Backend>(weights_leaf: &ParameterLeaf<'file, 'context, 'leaf, B::Context>,expected_shape: [usize; 2],@@ -129,6 +156,45 @@ impl<B: Backend> Embedding<B> {model_dim: u32,config: &EmbeddingConfig,parameter_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, EmbeddingError<B>> {+ Self::new_with_weight_trees(+ context,+ vocab_size,+ model_dim,+ config,+ parameter_tree,+ parameter_tree,+ &SHARED_TREE_UNTIED_KEYS,+ )+ }++ pub(crate) fn new_with_lookup_and_readout_trees(+ context: &B::Context,+ vocab_size: u32,+ model_dim: u32,+ config: &EmbeddingConfig,+ lookup_tree: &ParameterTree<B::Context>,+ readout_tree: &ParameterTree<B::Context>,+ ) -> Result<Self, EmbeddingError<B>> {+ Self::new_with_weight_trees(+ context,+ vocab_size,+ model_dim,+ config,+ lookup_tree,+ readout_tree,+ &SPLIT_TREE_UNTIED_KEYS,+ )+ }++ fn new_with_weight_trees(+ context: &B::Context,+ vocab_size: u32,+ model_dim: u32,+ config: &EmbeddingConfig,+ lookup_tree: &ParameterTree<B::Context>,+ readout_tree: &ParameterTree<B::Context>,+ untied_keys: &UntiedWeightKeys,) -> Result<Self, EmbeddingError<B>> {let common = config.common();@@ -139,9 +205,8 @@ impl<B: Backend> Embedding<B> {} => {let data_type = (*precision).into();- let weights_leaf = parameter_tree.leaf("weights")?;+ let weights_leaf = lookup_tree.leaf("weights")?;validate_tensor(&weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;-let weights = weights_leaf.read_buffer()?;let lookup = <B::Kernels as Kernels>::FullPrecisionEmbeddingLookupKernel::new(context, data_type)@@ -162,9 +227,9 @@ impl<B: Backend> Embedding<B> {} => {let data_type = (*precision).into();- let input_weights_leaf = parameter_tree.leaf("input_weights")?;+ let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;- let output_weights_leaf = parameter_tree.leaf("output_weights")?;+ let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;validate_tensor(&output_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;let input_weights = input_weights_leaf.read_buffer()?;@@ -199,15 +264,15 @@ impl<B: Backend> Embedding<B> {let num_groups = (model_dim as usize).div_ceil(*group_size);- let weights_leaf = parameter_tree.leaf("weights")?;+ let weights_leaf = lookup_tree.leaf("weights")?;validate_tensor(&weights_leaf,[vocab_size as usize, model_dim as usize / packing_divisor],storage_data_type,)?;- let scales_leaf = parameter_tree.leaf("scales")?;+ let scales_leaf = lookup_tree.leaf("scales")?;validate_tensor(&scales_leaf, [vocab_size as usize, num_groups], data_type)?;- let biases_leaf = parameter_tree.leaf("biases")?;+ let biases_leaf = lookup_tree.leaf("biases")?;validate_tensor(&biases_leaf, [vocab_size as usize, num_groups], data_type)?;let weights = weights_leaf.read_buffer()?;@@ -264,26 +329,26 @@ impl<B: Backend> Embedding<B> {let num_groups = (model_dim as usize).div_ceil(*group_size);- let input_weights_leaf = parameter_tree.leaf("input_weights")?;+ let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;validate_tensor(&input_weights_leaf,[vocab_size as usize, model_dim as usize / packing_divisor],storage_data_type,)?;- let input_scales_leaf = parameter_tree.leaf("input_scales")?;+ let input_scales_leaf = lookup_tree.leaf(untied_keys.lookup_scales)?;validate_tensor(&input_scales_leaf, [vocab_size as usize, num_groups], data_type)?;- let input_biases_leaf = parameter_tree.leaf("input_biases")?;+ let input_biases_leaf = lookup_tree.leaf(untied_keys.lookup_biases)?;validate_tensor(&input_biases_leaf, [vocab_size as usize, num_groups], data_type)?;- let output_weights_leaf = parameter_tree.leaf("output_weights")?;+ let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;validate_tensor(&output_weights_leaf,[vocab_size as usize, model_dim as usize / packing_divisor],storage_data_type,)?;- let output_scales_leaf = parameter_tree.leaf("output_scales")?;+ let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;- let output_biases_leaf = parameter_tree.leaf("output_biases")?;+ let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;let input_weights = input_weights_leaf.read_buffer()?;@@ -349,18 +414,18 @@ impl<B: Backend> Embedding<B> {let num_groups = (model_dim as usize).div_ceil(*group_size);- let input_weights_leaf = parameter_tree.leaf("input_weights")?;+ let input_weights_leaf = lookup_tree.leaf(untied_keys.lookup_weights)?;validate_tensor(&input_weights_leaf, [vocab_size as usize, model_dim as usize], data_type)?;- let output_weights_leaf = parameter_tree.leaf("output_weights")?;+ let output_weights_leaf = readout_tree.leaf(untied_keys.readout_weights)?;validate_tensor(&output_weights_leaf,[vocab_size as usize, model_dim as usize / packing_divisor],storage_data_type,)?;- let output_scales_leaf = parameter_tree.leaf("output_scales")?;+ let output_scales_leaf = readout_tree.leaf(untied_keys.readout_scales)?;validate_tensor(&output_scales_leaf, [vocab_size as usize, num_groups], data_type)?;- let output_biases_leaf = parameter_tree.leaf("output_biases")?;+ let output_biases_leaf = readout_tree.leaf(untied_keys.readout_biases)?;validate_tensor(&output_biases_leaf, [vocab_size as usize, num_groups], data_type)?;let input_weights = input_weights_leaf.read_buffer()?;diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rsindex c3bb078ea..6ab607cd7 100644--- a/crates/uzu/src/encodable_block/mlp/moe.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -183,6 +183,9 @@ impl<B: Backend> MoeBlock<B> {ActivationType::SILU {..} => 2,+ ActivationType::TANH => {+ panic!("Tanh activation is not supported for MoE kernels")+ },ActivationType::IDENTITY => {panic!("Identity activation is not supported for MoE kernels")},diff --git a/crates/uzu/src/forward_pass/state/state.rs b/crates/uzu/src/forward_pass/state/state.rsindex d1a6ae5b6..f2b84ea6f 100644--- a/crates/uzu/src/forward_pass/state/state.rs+++ b/crates/uzu/src/forward_pass/state/state.rs@@ -757,6 +757,11 @@ impl<B: Backend> ForwardPassState<B> {let copy_size_bytes = destination_array.size();debug_assert_eq!(destination_array.size(), source_ref.size());- encoder.encode_copy(src_buf_rc.borrow().deref(), dst_buf_rc.borrow_mut().deref_mut(), copy_size_bytes);+ encoder.encode_copy(+ src_buf_rc.borrow().deref(),+ 0..copy_size_bytes,+ dst_buf_rc.borrow_mut().deref_mut(),+ 0..copy_size_bytes,+ );}}diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rsindex 23cda3918..fdb352ea3 100644--- a/crates/uzu/src/lib.rs+++ b/crates/uzu/src/lib.rs@@ -18,6 +18,8 @@ pub mod prelude;pub mod session;pub use array::{Array, ArrayContextExt};+#[cfg(all(feature = "audio-runtime", metal_backend))]+pub use audio::{NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig};pub use config::ConfigDataType;pub use data_type::{ArrayElement, DataType};pub use language_model::gumbel::{gumbel_float, revidx};diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rsindex 612fa1549..2f33b9151 100644--- a/crates/uzu/src/prelude.rs+++ b/crates/uzu/src/prelude.rs@@ -9,9 +9,13 @@// Session Parameters// Session Types// Speculators+#[cfg(metal_backend)]+pub use crate::backends::metal::MetalContext;+#[cfg(all(feature = "audio-runtime", metal_backend))]+pub use crate::session::TtsSession;pub use crate::{VERSION,- audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid, AudioTokenPacking},+ audio::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid},parameters::{HeaderLoadingError, ParameterLeaf, ParameterLoaderError, ParameterTree},session::{ChatSession, ClassificationSession,diff --git a/crates/uzu/src/session/config/mod.rs b/crates/uzu/src/session/config/mod.rsindex 50dfc113c..d87a0831c 100644--- a/crates/uzu/src/session/config/mod.rs+++ b/crates/uzu/src/session/config/mod.rs@@ -2,8 +2,14 @@ mod decoding_config;mod grammar_config;mod run_config;mod speculator_config;+mod tts_run_config;+mod tts_session_options;pub use decoding_config::DecodingConfig;pub use grammar_config::GrammarConfig;pub use run_config::RunConfig;pub use speculator_config::SpeculatorConfig;+pub use tts_run_config::{TtsChunkPolicy, TtsNonStreamingMode, TtsPerformanceConfig, TtsRunConfig, TtsRunConfigError};+#[cfg(all(feature = "audio-runtime", metal_backend))]+pub use tts_session_options::TtsSessionOptions;+pub use tts_session_options::{TextDecoderRuntimeConfig, TextSamplingConfig};diff --git a/crates/uzu/src/session/config/tts_run_config.rs b/crates/uzu/src/session/config/tts_run_config.rsnew file mode 100644index 000000000..16453a895--- /dev/null+++ b/crates/uzu/src/session/config/tts_run_config.rs@@ -0,0 +1,222 @@+use thiserror::Error;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum TtsChunkPolicy {+ Fixed,+ Adaptive,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum TtsNonStreamingMode {+ FullDecode,+ ChunkedIfNeeded,+}++#[derive(Debug, Clone, Copy, PartialEq)]+pub struct TtsRunConfig {+ pub streaming_enabled: bool,+ pub target_emit_latency_ms: u32,+ pub initial_chunk_frames: usize,+ pub min_chunk_frames: usize,+ pub max_chunk_frames: usize,+ pub max_stream_workspace_frames: usize,+ pub max_semantic_frames: usize,+ pub chunk_policy: TtsChunkPolicy,+ pub non_streaming_mode: TtsNonStreamingMode,+ pub default_seed: u64,+ pub chunk_ema_alpha: f64,+ pub chunk_hysteresis_fraction: f64,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]+pub enum TtsRunConfigError {+ #[error("initial_chunk_frames must be greater than zero")]+ InitialChunkFramesMustBePositive,+ #[error("min_chunk_frames must be greater than zero")]+ MinChunkFramesMustBePositive,+ #[error("max_chunk_frames must be greater than or equal to min_chunk_frames")]+ MaxChunkFramesBelowMinChunkFrames,+ #[error("max_stream_workspace_frames must be greater than zero")]+ MaxStreamWorkspaceFramesMustBePositive,+ #[error("target_emit_latency_ms must be greater than zero")]+ TargetEmitLatencyMustBePositive,+ #[error("max_semantic_frames must be greater than zero")]+ MaxSemanticFramesMustBePositive,+ #[error("max_stream_workspace_frames must be greater than or equal to max_chunk_frames")]+ MaxStreamWorkspaceFramesBelowMaxChunkFrames,+}++impl TtsRunConfig {+ pub fn validate(&self) -> Result<(), TtsRunConfigError> {+ if self.initial_chunk_frames == 0 {+ return Err(TtsRunConfigError::InitialChunkFramesMustBePositive);+ }+ if self.min_chunk_frames == 0 {+ return Err(TtsRunConfigError::MinChunkFramesMustBePositive);+ }+ if self.max_chunk_frames < self.min_chunk_frames {+ return Err(TtsRunConfigError::MaxChunkFramesBelowMinChunkFrames);+ }+ if self.max_stream_workspace_frames == 0 {+ return Err(TtsRunConfigError::MaxStreamWorkspaceFramesMustBePositive);+ }+ if self.target_emit_latency_ms == 0 {+ return Err(TtsRunConfigError::TargetEmitLatencyMustBePositive);+ }+ if self.max_semantic_frames == 0 {+ return Err(TtsRunConfigError::MaxSemanticFramesMustBePositive);+ }+ Ok(())+ }++ pub fn validate_stream_decode(&self) -> Result<(), TtsRunConfigError> {+ self.validate()?;+ if self.max_stream_workspace_frames < self.max_chunk_frames {+ return Err(TtsRunConfigError::MaxStreamWorkspaceFramesBelowMaxChunkFrames);+ }+ Ok(())+ }++ pub fn fixed_chunk_frames(chunk_frames: usize) -> Self {+ Self {+ chunk_policy: TtsChunkPolicy::Fixed,+ initial_chunk_frames: chunk_frames.max(1),+ min_chunk_frames: chunk_frames.max(1),+ max_chunk_frames: chunk_frames.max(1),+ ..Self::default()+ }+ }+}++impl Default for TtsRunConfig {+ fn default() -> Self {+ Self {+ streaming_enabled: true,+ target_emit_latency_ms: 80,+ initial_chunk_frames: 1,+ min_chunk_frames: 16,+ max_chunk_frames: 256,+ max_stream_workspace_frames: 768,+ max_semantic_frames: 768,+ chunk_policy: TtsChunkPolicy::Adaptive,+ non_streaming_mode: TtsNonStreamingMode::ChunkedIfNeeded,+ default_seed: 123,+ chunk_ema_alpha: 0.2,+ chunk_hysteresis_fraction: 0.25,+ }+ }+}++#[derive(Debug, Clone, PartialEq)]+pub struct TtsPerformanceConfig {+ pub streaming: TtsRunConfig,+ pub non_streaming: TtsRunConfig,+ pub non_streaming_chunked_threshold_frames: usize,+}++impl Default for TtsPerformanceConfig {+ fn default() -> Self {+ let streaming = TtsRunConfig::default();+ let non_streaming = TtsRunConfig {+ streaming_enabled: false,+ chunk_policy: TtsChunkPolicy::Fixed,+ non_streaming_mode: TtsNonStreamingMode::ChunkedIfNeeded,+ initial_chunk_frames: 128,+ min_chunk_frames: 128,+ max_chunk_frames: 128,+ ..TtsRunConfig::default()+ };+ Self {+ streaming,+ non_streaming,+ non_streaming_chunked_threshold_frames: 4096,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::{TtsChunkPolicy, TtsNonStreamingMode, TtsPerformanceConfig, TtsRunConfig};++ #[test]+ fn default_run_config_matches_production_defaults() {+ let config = TtsRunConfig::default();+ assert!(config.streaming_enabled);+ assert_eq!(config.target_emit_latency_ms, 80);+ assert_eq!(config.initial_chunk_frames, 1);+ assert_eq!(config.min_chunk_frames, 16);+ assert_eq!(config.max_chunk_frames, 256);+ assert_eq!(config.max_stream_workspace_frames, 768);+ assert_eq!(config.max_semantic_frames, 768);+ assert_eq!(config.chunk_policy, TtsChunkPolicy::Adaptive);+ assert_eq!(config.non_streaming_mode, TtsNonStreamingMode::ChunkedIfNeeded);+ assert_eq!(config.default_seed, 123);+ assert!((config.chunk_ema_alpha - 0.2).abs() < f64::EPSILON);+ assert!((config.chunk_hysteresis_fraction - 0.25).abs() < f64::EPSILON);+ }++ #[test]+ fn fixed_chunk_constructor_sets_bounds() {+ let config = TtsRunConfig::fixed_chunk_frames(64);+ assert_eq!(config.chunk_policy, TtsChunkPolicy::Fixed);+ assert_eq!(config.initial_chunk_frames, 64);+ assert_eq!(config.min_chunk_frames, 64);+ assert_eq!(config.max_chunk_frames, 64);+ }++ #[test]+ fn run_config_validation_rejects_invalid_bounds() {+ let invalid = TtsRunConfig {+ min_chunk_frames: 32,+ max_chunk_frames: 16,+ ..TtsRunConfig::default()+ };+ assert!(invalid.validate().is_err());+ }++ #[test]+ fn run_config_validation_rejects_zero_initial_chunk() {+ let invalid = TtsRunConfig {+ initial_chunk_frames: 0,+ ..TtsRunConfig::default()+ };+ assert!(invalid.validate().is_err());+ }++ #[test]+ fn run_config_validation_rejects_zero_semantic_cap() {+ let invalid = TtsRunConfig {+ max_semantic_frames: 0,+ ..TtsRunConfig::default()+ };+ assert!(invalid.validate().is_err());+ }++ #[test]+ fn run_config_validation_rejects_zero_workspace_frames() {+ let invalid = TtsRunConfig {+ max_stream_workspace_frames: 0,+ ..TtsRunConfig::default()+ };+ assert!(invalid.validate().is_err());+ }++ #[test]+ fn run_config_validation_rejects_workspace_smaller_than_max_chunk() {+ let invalid = TtsRunConfig {+ max_chunk_frames: 256,+ max_stream_workspace_frames: 128,+ ..TtsRunConfig::default()+ };+ assert!(invalid.validate_stream_decode().is_err());+ }++ #[test]+ fn performance_config_defaults_are_consistent() {+ let config = TtsPerformanceConfig::default();+ assert_eq!(config.streaming.chunk_policy, TtsChunkPolicy::Adaptive);+ assert_eq!(config.non_streaming.chunk_policy, TtsChunkPolicy::Fixed);+ assert_eq!(config.non_streaming.min_chunk_frames, config.non_streaming.max_chunk_frames);+ assert_eq!(config.non_streaming.non_streaming_mode, TtsNonStreamingMode::ChunkedIfNeeded);+ }+}diff --git a/crates/uzu/src/session/config/tts_session_options.rs b/crates/uzu/src/session/config/tts_session_options.rsnew file mode 100644index 000000000..5a3e72e0d--- /dev/null+++ b/crates/uzu/src/session/config/tts_session_options.rs@@ -0,0 +1,41 @@+#[derive(Debug, Clone, Copy, PartialEq)]+pub struct TextSamplingConfig {+ pub temperature: f32,+ pub top_p: f32,+}++impl Default for TextSamplingConfig {+ fn default() -> Self {+ Self {+ temperature: 0.8008,+ top_p: 0.8008,+ }+ }+}++#[derive(Debug, Clone, PartialEq)]+pub struct TextDecoderRuntimeConfig {+ pub sampling: TextSamplingConfig,+ pub min_frames_before_im_end: usize,+ pub max_new_tokens_override: Option<usize>,+ pub force_semantic_sampling_mask: Option<bool>,+ pub prefill_step_size: usize,+}++impl Default for TextDecoderRuntimeConfig {+ fn default() -> Self {+ Self {+ sampling: TextSamplingConfig::default(),+ min_frames_before_im_end: 8,+ max_new_tokens_override: None,+ force_semantic_sampling_mask: None,+ prefill_step_size: 128,+ }+ }+}++#[cfg(all(feature = "audio-runtime", metal_backend))]+#[derive(Debug, Clone, PartialEq, Default)]+pub struct TtsSessionOptions {+ pub text_decoder: TextDecoderRuntimeConfig,+}diff --git a/crates/uzu/src/session/mod.rs b/crates/uzu/src/session/mod.rsindex 8633ea0d9..0ff8b0af5 100644--- a/crates/uzu/src/session/mod.rs+++ b/crates/uzu/src/session/mod.rs@@ -3,7 +3,11 @@ pub mod classification_session;pub mod config;pub mod helpers;pub mod parameter;+#[cfg(all(feature = "audio-runtime", metal_backend))]+mod tts_session;pub mod types;pub use chat_session::{ChatSession, ChatSession as Session};pub use classification_session::ClassificationSession;+#[cfg(all(feature = "audio-runtime", metal_backend))]+pub use tts_session::TtsSession;diff --git a/crates/uzu/src/session/parameter/mod.rs b/crates/uzu/src/session/parameter/mod.rsindex 6470f6549..ba653526b 100644--- a/crates/uzu/src/session/parameter/mod.rs+++ b/crates/uzu/src/session/parameter/mod.rs@@ -10,5 +10,5 @@ pub use context_length::ContextLength;pub use context_mode::ContextMode;pub use prefill_step_size::PrefillStepSize;pub use resolvable_value::{ConfigResolvableValue, ResolvableValue};-pub use sampling_policy::{SamplingMethod, SamplingPolicy};+pub use sampling_policy::{SamplingMethod, SamplingPolicy, SamplingProcessingOrder};pub use sampling_seed::SamplingSeed;diff --git a/crates/uzu/src/session/parameter/sampling_policy.rs b/crates/uzu/src/session/parameter/sampling_policy.rsindex 94d8d5b73..e60c6e1e3 100644--- a/crates/uzu/src/session/parameter/sampling_policy.rs+++ b/crates/uzu/src/session/parameter/sampling_policy.rs@@ -1,5 +1,11 @@use crate::{config::LanguageModelConfig, session::parameter::ConfigResolvableValue};+#[derive(Debug, Copy, Clone, PartialEq, Eq)]+pub enum SamplingProcessingOrder {+ TemperatureThenFilters,+ FiltersThenTemperature,+}+#[derive(Debug, Copy, Clone, PartialEq)]pub enum SamplingMethod {Greedy,@@ -8,6 +14,7 @@ pub enum SamplingMethod {top_k: Option<u32>,top_p: Option<f32>,min_p: Option<f32>,+ processing_order: SamplingProcessingOrder,},}@@ -43,6 +50,7 @@ impl ConfigResolvableValue<LanguageModelConfig, SamplingMethod> for SamplingPolitop_k: generation_config.top_k,top_p: generation_config.top_p,min_p: generation_config.min_p,+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},SamplingPolicy::Custom {value,diff --git a/crates/uzu/src/session/tts_session.rs b/crates/uzu/src/session/tts_session.rsnew file mode 100644index 000000000..040cefdd3--- /dev/null+++ b/crates/uzu/src/session/tts_session.rs@@ -0,0 +1,236 @@+#![cfg(all(feature = "audio-runtime", metal_backend))]++mod audio_backend;+mod backend_factory;+mod decoder_runtime;+mod decoder_support;+mod fishaudio;+mod generator;++use std::{+ cell::RefCell,+ collections::HashMap,+ fs::File,+ path::{Path, PathBuf},+ rc::Rc,+ time::Instant,+};++use backend_factory::load_tts_runtime;+use decoder_runtime::*;+use decoder_support::*;+use minijinja::{Environment, context};+use rand::{RngExt, SeedableRng, rngs::StdRng};+use tokenizers::Tokenizer;++use crate::{+ DataType,+ array::ArrayContextExt,+ audio::{+ AudioCodecRuntime, AudioGenerationContext, AudioPcmBatch, AudioTokenGrid,+ nanocodec::{AudioDecodeStepStats, AudioDecodeStreamState},+ },+ backends::common::{+ Backend, Context as BackendContext, Encoder, Kernels,+ kernel::{+ EmbeddingRowsSumKernel, TensorAddScaleKernel, TensorCopyKernel, TokenCopySampledKernel,+ TokenCopyToResultsKernel,+ kv_cache_update::KVCacheUpdate,+ matmul::{MatmulArguments, MatmulKernel, MatmulKernels},+ },+ },+ config::{InnerModelConfig, ModelMetadata, TtsMessageProcessorConfig},+ encodable_block::{Decoder, EncodingParameters, Sampling as GpuSampling},+ forward_pass::{+ cache_layers::CacheLayers,+ model_shape::ModelShape,+ scratch_buffers::ScratchBuffers,+ state::{ArrayId, ForwardPassState, SharedBuffers},+ },+ parameters::ParameterLoader,+ session::{+ config::{TextDecoderRuntimeConfig, TextSamplingConfig, TtsChunkPolicy, TtsRunConfig, TtsSessionOptions},+ parameter::{ConfigResolvableValue, SamplingMethod, SamplingProcessingOrder},+ types::{Error, Input},+ },+};++fn unable_to_create_context<E: std::error::Error + 'static>(err: E) -> Error {+ Error::UnableToCreateContext(Box::new(err))+}++#[derive(Debug, Clone, Default, PartialEq)]+pub struct TtsExecutionStats {+ pub semantic_decode_seconds: f64,+ pub audio_decode_seconds: f64,+ pub callback_seconds: f64,+ pub first_chunk_seconds: f64,+ pub command_buffers_submitted: usize,+ pub host_waits: usize,+ pub semantic_frames: usize,+ pub first_chunk_frames: usize,+ pub emitted_chunks: usize,+ pub audio_decode_calls: usize,+ pub audio_input_frames: usize,+ pub audio_decoded_window_frames: usize,+ pub audio_max_decoded_window_frames: usize,+}++#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub(super) struct RunnerInstrumentation {+ pub(super) command_buffers_submitted: usize,+ pub(super) host_waits: usize,+}++#[derive(Debug, Clone, Copy)]+pub(super) struct AdaptiveChunkController {+ pub(super) ema_ms_per_frame: Option<f64>,+ pub(super) current_chunk_frames: usize,+}++pub struct TtsSession<B: Backend> {+ tokenizer: Tokenizer,+ audio: AudioGenerationContext<B>,+ audio_decoder: Box<dyn AudioDecoderBackend>,+ message_processor_config: TtsMessageProcessorConfig,+ text_decoder: Box<dyn SemanticDecoderBackend>,+ last_execution_stats: Option<TtsExecutionStats>,+}++trait SemanticDecoderBackend {+ fn generate_semantic_tokens(+ &mut self,+ text_tokens: &[u64],+ codec_cardinality: usize,+ seed: u64,+ max_semantic_frames: usize,+ ) -> Result<AudioTokenGrid, Error>;++ fn generate_semantic_tokens_with_callback(+ &mut self,+ text_tokens: &[u64],+ codec_cardinality: usize,+ seed: u64,+ max_semantic_frames: usize,+ on_frame: &mut dyn FnMut(&[u32]) -> Result<(), Error>,+ ) -> Result<AudioTokenGrid, Error>;++ fn take_instrumentation(&mut self) -> RunnerInstrumentation;+}++trait AudioDecoderBackend {+ fn codec_cardinality(&self) -> usize;++ fn num_codebooks(&self) -> usize;++ fn sample_rate(&self) -> u32;++ fn decode(+ &self,+ tokens: &AudioTokenGrid,+ ) -> Result<AudioPcmBatch, Error>;++ fn begin_stream(+ &self,+ batch_size: usize,+ codebooks: usize,+ max_workspace_frames: usize,+ ) -> Result<Box<dyn AudioDecoderStreamBackend>, Error>;+}++trait AudioDecoderStreamBackend {+ fn decode_step(+ &mut self,+ new_tokens: &AudioTokenGrid,+ is_final: bool,+ ) -> Result<AudioPcmBatch, Error>;++ fn decode_step_pending(+ &mut self,+ new_tokens: &AudioTokenGrid,+ is_final: bool,+ ) -> Result<Box<dyn PendingAudioChunkBackend>, Error> {+ let pcm = self.decode_step(new_tokens, is_final)?;+ Ok(Box::new(ImmediatePendingAudioChunk {+ pcm: Some(pcm),+ step_stats: self.last_step_stats(),+ }))+ }++ fn last_step_stats(&self) -> Option<AudioDecodeStepStats>;++ fn finish(self: Box<Self>) -> Result<(), Error>;+}++pub(super) trait PendingAudioChunkBackend {+ fn step_stats(&self) -> Option<AudioDecodeStepStats>;++ fn try_resolve(&mut self) -> Result<Option<AudioPcmBatch>, Error>;++ fn resolve(self: Box<Self>) -> Result<AudioPcmBatch, Error>;++ fn resolve_with_decode_duration(self: Box<Self>) -> Result<(AudioPcmBatch, std::time::Duration), Error> {+ let start = Instant::now();+ let pcm = self.resolve()?;+ Ok((pcm, start.elapsed()))+ }++ fn try_resolve_with_decode_duration(&mut self) -> Result<Option<(AudioPcmBatch, std::time::Duration)>, Error> {+ let start = Instant::now();+ let Some(pcm) = self.try_resolve()? else {+ return Ok(None);+ };+ Ok(Some((pcm, start.elapsed())))+ }+}++struct ImmediatePendingAudioChunk {+ pcm: Option<AudioPcmBatch>,+ step_stats: Option<AudioDecodeStepStats>,+}++impl PendingAudioChunkBackend for ImmediatePendingAudioChunk {+ fn step_stats(&self) -> Option<AudioDecodeStepStats> {+ self.step_stats+ }++ fn try_resolve(&mut self) -> Result<Option<AudioPcmBatch>, Error> {+ Ok(Some(self.pcm.take().ok_or(Error::GenerateFailed)?))+ }++ fn resolve(mut self: Box<Self>) -> Result<AudioPcmBatch, Error> {+ self.pcm.take().ok_or(Error::GenerateFailed)+ }+}++struct NanoCodecPendingAudioChunk<B: Backend> {+ inner: Option<crate::audio::nanocodec::runtime::PendingStreamPcmChunk<B>>,+}++impl<B: Backend> PendingAudioChunkBackend for NanoCodecPendingAudioChunk<B> {+ fn step_stats(&self) -> Option<AudioDecodeStepStats> {+ self.inner.as_ref().map(|pending| pending.step_stats())+ }++ fn try_resolve(&mut self) -> Result<Option<AudioPcmBatch>, Error> {+ let ready = self.inner.as_ref().is_none_or(|pending| pending.is_complete());+ if !ready {+ return Ok(None);+ }++ let pending = self.inner.take().ok_or(Error::GenerateFailed)?;+ pending.resolve().map(Some).map_err(Error::from)+ }++ fn resolve(mut self: Box<Self>) -> Result<AudioPcmBatch, Error> {+ let pending = self.inner.take().ok_or(Error::GenerateFailed)?;+ pending.resolve().map_err(Error::from)+ }+}++pub(super) struct PendingStreamingChunk {+ pub(super) submission_decode_duration: std::time::Duration,+ pub(super) ready_frames: usize,+ pub(super) next_chunk_frames: usize,+ pub(super) chunk: Box<dyn PendingAudioChunkBackend>,+}diff --git a/crates/uzu/src/session/tts_session/audio_backend.rs b/crates/uzu/src/session/tts_session/audio_backend.rsnew file mode 100644index 000000000..5e3ea6663--- /dev/null+++ b/crates/uzu/src/session/tts_session/audio_backend.rs@@ -0,0 +1,102 @@+use super::*;++pub(super) struct NanoCodecAudioDecoderBackend<B: Backend> {+ audio: AudioGenerationContext<B>,+ codec_cardinality: usize,+}++pub(super) struct NanoCodecAudioDecoderStream<B: Backend> {+ audio: AudioGenerationContext<B>,+ state: Option<AudioDecodeStreamState>,+}++impl<B: Backend> NanoCodecAudioDecoderBackend<B> {+ pub(super) fn new(audio: AudioGenerationContext<B>) -> Self {+ let codec_cardinality = audio.codec_cardinality();+ Self {+ audio,+ codec_cardinality,+ }+ }+}++impl<B: Backend + Send + Sync> AudioDecoderBackend for NanoCodecAudioDecoderBackend<B> {+ fn codec_cardinality(&self) -> usize {+ self.codec_cardinality+ }++ fn num_codebooks(&self) -> usize {+ self.audio.runtime().config().num_groups()+ }++ fn sample_rate(&self) -> u32 {+ self.audio.sample_rate()+ }++ fn decode(+ &self,+ tokens: &AudioTokenGrid,+ ) -> Result<AudioPcmBatch, Error> {+ self.audio.runtime().decode(tokens).map_err(Error::from)+ }++ fn begin_stream(+ &self,+ batch_size: usize,+ codebooks: usize,+ max_workspace_frames: usize,+ ) -> Result<Box<dyn AudioDecoderStreamBackend>, Error> {+ let state = self+ .audio+ .runtime()+ .begin_decode_stream_with_options(batch_size, codebooks, max_workspace_frames)+ .map_err(Error::from)?;+ Ok(Box::new(NanoCodecAudioDecoderStream {+ audio: self.audio.clone(),+ state: Some(state),+ }))+ }+}++impl<B: Backend> AudioDecoderStreamBackend for NanoCodecAudioDecoderStream<B> {+ fn decode_step(+ &mut self,+ new_tokens: &AudioTokenGrid,+ is_final: bool,+ ) -> Result<AudioPcmBatch, Error> {+ let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;+ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;+ self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)+ }++ fn decode_step_pending(+ &mut self,+ new_tokens: &AudioTokenGrid,+ is_final: bool,+ ) -> Result<Box<dyn PendingAudioChunkBackend>, Error> {+ let state = self.state.as_mut().ok_or(Error::GenerateFailed)?;+ if let Some(pending) =+ self.audio.runtime().submit_decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?+ {+ return Ok(Box::new(NanoCodecPendingAudioChunk {+ inner: Some(pending),+ }));+ }++ let decoded = self.audio.runtime().decode_stream_step(state, new_tokens, is_final).map_err(Error::from)?;+ let pcm = self.audio.runtime().decoded_padded_to_pcm_batch(&decoded).map_err(Error::from)?;+ Ok(Box::new(ImmediatePendingAudioChunk {+ pcm: Some(pcm),+ step_stats: Some(state.last_step_stats()),+ }))+ }++ fn last_step_stats(&self) -> Option<AudioDecodeStepStats> {+ self.state.as_ref().map(AudioDecodeStreamState::last_step_stats)+ }++ fn finish(mut self: Box<Self>) -> Result<(), Error> {+ let state = self.state.take().ok_or(Error::GenerateFailed)?;+ self.audio.runtime().end_decode_stream(state).map_err(Error::from)+ }+}diff --git a/crates/uzu/src/session/tts_session/backend_factory.rs b/crates/uzu/src/session/tts_session/backend_factory.rsnew file mode 100644index 000000000..82b2f11fe--- /dev/null+++ b/crates/uzu/src/session/tts_session/backend_factory.rs@@ -0,0 +1,63 @@+use super::*;+use crate::{+ config::{ModelType, TtsModelConfig, TtsTextDecoderConfig},+ session::types::TtsModelConfigError,+};++pub(super) struct LoadedTtsRuntime<B: Backend> {+ pub(super) audio: AudioGenerationContext<B>,+ pub(super) audio_decoder: Box<dyn AudioDecoderBackend>,+ pub(super) text_decoder: Box<dyn SemanticDecoderBackend>,+ pub(super) message_processor_config: TtsMessageProcessorConfig,+}++pub(super) fn build_audio_decoder_backend<B: Backend + Send + Sync>(+ audio: &AudioGenerationContext<B>+) -> Result<Box<dyn AudioDecoderBackend>, Error> {+ Ok(Box::new(audio_backend::NanoCodecAudioDecoderBackend::new(audio.clone())))+}++pub(super) fn load_tts_runtime<B: Backend + Send + Sync>(+ model_path: &Path,+ model_metadata: &ModelMetadata,+ options: &TtsSessionOptions,+) -> Result<LoadedTtsRuntime<B>, Error> {+ if model_metadata.model_type != ModelType::TtsModel {+ return Err(TtsModelConfigError::ModelTypeMismatch {+ expected: ModelType::TtsModel,+ actual: model_metadata.model_type.clone(),+ }+ .into());+ }++ let tts_model_config =+ model_metadata.model_config.as_tts().ok_or(TtsModelConfigError::MissingTtsModelConfig)?.clone();+ let audio: AudioGenerationContext<B> =+ tts_model_config.create_audio_generation_context_with_model_path(model_path)?;+ let text_decoder = build_text_decoder_backend(&tts_model_config, &audio, model_path, options)?;+ let audio_decoder = build_audio_decoder_backend(&audio)?;+ let message_processor_config = tts_model_config.message_processor_config.clone();++ Ok(LoadedTtsRuntime {+ audio,+ audio_decoder,+ text_decoder,+ message_processor_config,+ })+}++pub(super) fn build_text_decoder_backend<B: Backend>(+ tts_model_config: &TtsModelConfig,+ audio: &AudioGenerationContext<B>,+ model_path: &Path,+ options: &TtsSessionOptions,+) -> Result<Box<dyn SemanticDecoderBackend>, Error> {+ match &tts_model_config.tts_config.text_decoder_config {+ TtsTextDecoderConfig::FishAudioTextDecoderConfig {+ config,+ } => {+ fishaudio::validate_fishaudio_message_processor_config(&tts_model_config.message_processor_config)?;+ fishaudio::build_fishaudio_text_decoder_runtime(config, audio, model_path, &options.text_decoder)+ },+ }+}diff --git a/crates/uzu/src/session/tts_session/decoder_runtime.rs b/crates/uzu/src/session/tts_session/decoder_runtime.rsnew file mode 100644index 000000000..a236b2207--- /dev/null+++ b/crates/uzu/src/session/tts_session/decoder_runtime.rs@@ -0,0 +1,1168 @@+use super::{decoder_support::*, *};+use crate::{array::Array, backends::common::Buffer, session::types::TtsModelConfigError};++struct TokenDecoderLoadedModel<B: Backend> {+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,+ scratch_buffers: ScratchBuffers<B>,+ executables: Decoder<B>,+ sampler: GpuSampling<B>,+ token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,+ token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,+}++impl<B: Backend> TokenDecoderLoadedModel<B> {+ fn load(+ context: &Rc<B::Context>,+ model_path: &Path,+ decoder_config: &Rc<crate::config::DecoderConfig>,+ model_shape: &ModelShape,+ transformer_subtree: &str,+ embedding_subtree: &str,+ readout_subtree: &str,+ max_prefix_length: usize,+ max_suffix_length: usize,+ ) -> Result<Self, Error> {+ let weights_path = model_path.join("model.safetensors");+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;+ let loader = ParameterLoader::new(&weights_file, context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;+ let root_loader_view = loader.tree();++ let shared_buffers = TokenDecoderContext::<B>::build_shared_buffers(+ context,+ decoder_config,+ model_shape,+ &root_loader_view,+ transformer_subtree,+ )?;+ let scratch_buffers =+ ScratchBuffers::new(context.as_ref(), decoder_config, model_shape, max_prefix_length, max_suffix_length);+ let executables = Decoder::new_with_embedding_and_readout_subtrees(+ context.clone(),+ decoder_config.clone(),+ &root_loader_view,+ transformer_subtree,+ embedding_subtree,+ readout_subtree,+ );+ let logits_data_type = scratch_buffers.logits.data_type();+ let sampler =+ GpuSampling::new(context.as_ref(), logits_data_type, max_suffix_length, decoder_config.vocab_size)+ .map_err(unable_to_create_context)?;+ let token_copy_sampled =+ <B::Kernels as Kernels>::TokenCopySampledKernel::new(context.as_ref()).map_err(unable_to_create_context)?;+ let token_copy_results = <B::Kernels as Kernels>::TokenCopyToResultsKernel::new(context.as_ref())+ .map_err(unable_to_create_context)?;++ Ok(Self {+ shared_buffers,+ scratch_buffers,+ executables,+ sampler,+ token_copy_sampled,+ token_copy_results,+ })+ }+}++struct TokenDecoderContext<B: Backend> {+ context: Rc<B::Context>,+ cache_layers: Rc<RefCell<CacheLayers<B>>>,+ shared_buffers: Rc<RefCell<SharedBuffers<B>>>,+ scratch_buffers: ScratchBuffers<B>,+ model_shape: ModelShape,+ decoder_config: Rc<crate::config::DecoderConfig>,+ executables: Decoder<B>,+ sampler: GpuSampling<B>,+ kv_cache_update: KVCacheUpdate<B>,+ token_copy_sampled: <B::Kernels as Kernels>::TokenCopySampledKernel,+ token_copy_results: <B::Kernels as Kernels>::TokenCopyToResultsKernel,+ async_chain_positions: Rc<RefCell<B::Buffer>>,+ async_chain_seeds: Rc<RefCell<B::Buffer>>,+ async_chain_results: Rc<RefCell<B::Buffer>>,+ async_chain_capacity: usize,+}++impl<B: Backend> TokenDecoderContext<B> {+ pub(super) fn context(&self) -> &Rc<B::Context> {+ &self.context+ }++ fn new(+ context: Rc<B::Context>,+ model_path: &Path,+ decoder_config: Rc<crate::config::DecoderConfig>,+ transformer_subtree: &str,+ embedding_subtree: &str,+ readout_subtree: &str,+ runtime_config: &TextDecoderRuntimeConfig,+ ) -> Result<(Self, DataType, bool), Error> {+ let model_shape = ModelShape::from_decoder_config(&decoder_config);+ let max_prefix_length = decoder_config.context_length;+ let max_suffix_length = text_decoder_prefill_step_size(runtime_config, decoder_config.context_length).max(32);+ let should_fill_attention_bias =+ model_shape.sliding_window_length_per_layer.iter().any(|value| value.is_some());+ let activation_data_type = model_shape.activation_data_type();+ let loaded_model = TokenDecoderLoadedModel::<B>::load(+ &context,+ model_path,+ &decoder_config,+ &model_shape,+ transformer_subtree,+ embedding_subtree,+ readout_subtree,+ max_prefix_length,+ max_suffix_length,+ )?;+ let async_chain_capacity = max_suffix_length.max(1);+ let (async_chain_positions, async_chain_seeds, async_chain_results) =+ Self::build_async_chain_buffers(&context, async_chain_capacity)?;+ let cache_layers = Rc::new(RefCell::new(CacheLayers::new(+ context.as_ref(),+ &model_shape,+ max_prefix_length,+ max_suffix_length,+ )));+ let intermediate_data_type: DataType = decoder_config.output_norm_config.scale_precision.into();+ let kv_cache_update = KVCacheUpdate::new(context.as_ref(), intermediate_data_type, max_prefix_length)+ .map_err(unable_to_create_context)?;++ Ok((+ Self {+ context,+ cache_layers,+ shared_buffers: loaded_model.shared_buffers,+ scratch_buffers: loaded_model.scratch_buffers,+ model_shape,+ decoder_config,+ executables: loaded_model.executables,+ sampler: loaded_model.sampler,+ kv_cache_update,+ token_copy_sampled: loaded_model.token_copy_sampled,+ token_copy_results: loaded_model.token_copy_results,+ async_chain_positions,+ async_chain_seeds,+ async_chain_results,+ async_chain_capacity,+ },+ activation_data_type,+ should_fill_attention_bias,+ ))+ }++ fn build_shared_buffers(+ context: &Rc<B::Context>,+ decoder_config: &Rc<crate::config::DecoderConfig>,+ model_shape: &ModelShape,+ root_loader_view: &crate::parameters::ParameterTree<B::Context>,+ transformer_subtree: &str,+ ) -> Result<Rc<RefCell<SharedBuffers<B>>>, Error> {+ let shared_buffers = Rc::new(RefCell::new(SharedBuffers::new(context.as_ref(), decoder_config, model_shape)));+ let transformer_tree = root_loader_view.subtree(transformer_subtree).map_err(|_| Error::UnableToLoadWeights)?;+ let mut shared_buffers_borrow = shared_buffers.borrow_mut();+ if let Some(global_rope) = &mut shared_buffers_borrow.global_rope {+ global_rope.update_data(&transformer_tree, "global_rope");+ }+ if let Some(local_rope) = &mut shared_buffers_borrow.local_rope {+ local_rope.update_data(&transformer_tree, "local_rope");+ }+ drop(shared_buffers_borrow);+ Ok(shared_buffers)+ }++ fn build_async_chain_buffers(+ context: &Rc<B::Context>,+ async_chain_capacity: usize,+ ) -> Result<(Rc<RefCell<B::Buffer>>, Rc<RefCell<B::Buffer>>, Rc<RefCell<B::Buffer>>), Error> {+ let positions = Rc::new(RefCell::new(+ context+ .create_buffer(async_chain_capacity * std::mem::size_of::<i32>())+ .map_err(unable_to_create_context)?,+ ));+ let seeds = Rc::new(RefCell::new(+ context+ .create_buffer(async_chain_capacity * std::mem::size_of::<u64>())+ .map_err(unable_to_create_context)?,+ ));+ let results = Rc::new(RefCell::new(+ context+ .create_buffer(async_chain_capacity * std::mem::size_of::<u32>())+ .map_err(unable_to_create_context)?,+ ));+ Ok((positions, seeds, results))+ }+}++pub(super) struct TokenDecoderRunner<B: Backend> {+ ctx: TokenDecoderContext<B>,+ pub(super) single_hidden_capture: Array<B>,+ pub(super) single_override_embedding: Array<B>,+ tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,+ tensor_add_scale: <B::Kernels as Kernels>::TensorAddScaleKernel,+ single_token_vocab_masks: HashMap<usize, Box<[u32]>>,+ two_token_vocab_masks: HashMap<usize, Box<[u32]>>,+ should_fill_attention_bias: bool,+ next_position: usize,+ instrumentation: RunnerInstrumentation,+}++impl<B: Backend> TokenDecoderRunner<B> {+ pub(super) fn context(&self) -> &Rc<B::Context> {+ self.ctx.context()+ }++ pub(super) fn new_with_context(+ context: Rc<B::Context>,+ model_path: &Path,+ decoder_config: Rc<crate::config::DecoderConfig>,+ transformer_subtree: &str,+ embedding_subtree: &str,+ readout_subtree: &str,+ runtime_config: &TextDecoderRuntimeConfig,+ ) -> Result<Self, Error> {+ let (ctx, activation_data_type, should_fill_attention_bias) = TokenDecoderContext::new(+ context,+ model_path,+ decoder_config,+ transformer_subtree,+ embedding_subtree,+ readout_subtree,+ runtime_config,+ )?;+ let context: Rc<B::Context> = Rc::clone(ctx.context());+ let model_dim = ctx.decoder_config.model_dim;+ let tensor_copy = <B::Kernels as Kernels>::TensorCopyKernel::new(context.as_ref(), activation_data_type)+ .map_err(unable_to_create_context)?;+ let tensor_add_scale =+ <B::Kernels as Kernels>::TensorAddScaleKernel::new(context.as_ref(), activation_data_type)+ .map_err(unable_to_create_context)?;+ let single_hidden_capture =+ context.create_array(&[1, model_dim], activation_data_type, "tts_single_hidden_capture");+ let single_override_embedding =+ context.create_array(&[1, model_dim], activation_data_type, "tts_single_override_embedding");++ Ok(Self {+ ctx,+ single_hidden_capture,+ single_override_embedding,+ tensor_copy,+ tensor_add_scale,+ single_token_vocab_masks: HashMap::new(),+ two_token_vocab_masks: HashMap::new(),+ should_fill_attention_bias,+ next_position: 0,+ instrumentation: RunnerInstrumentation::default(),+ })+ }++ pub(super) fn reset(&mut self) {+ self.ctx.cache_layers.borrow_mut().clear();+ self.next_position = 0;+ }++ pub(super) fn prefill_without_sampling(+ &mut self,+ token_ids: &[u64],+ ) -> Result<(), Error> {+ if token_ids.is_empty() {+ return Ok(());+ }++ objc2::rc::autoreleasepool(|_| {+ let token_count = token_ids.len();++ let positions: Vec<usize> = (self.next_position..self.next_position + token_count).collect();+ let token_seeds = vec![0_u64; token_count];++ let mut state = ForwardPassState::new_llm(+ self.ctx.context.clone(),+ &self.ctx.decoder_config,+ &self.ctx.model_shape,+ &self.ctx.scratch_buffers,+ self.ctx.cache_layers.clone(),+ self.ctx.shared_buffers.clone(),+ token_ids,+ &positions,+ None, // no bitmask+ &token_seeds,+ token_count,+ 0, // sampling_start: irrelevant+ 0, // sampling_length: no sampling+ false,+ None,+ false,+ self.should_fill_attention_bias,+ None,+ None,+ );++ let encoding_parameters = EncodingParameters::new();++ let context = Rc::clone(&self.ctx.context);+ let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;++ self.ctx+ .executables+ .embed+ .encode_lookup(&mut state, &mut encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ for layer in self.ctx.executables.layers.iter() {+ layer+ .encode(&mut state, &encoding_parameters, &mut encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ }+ self.encode_cache_acceptance_update_on(&mut encoder, token_count);++ self.submit_and_wait_command_buffer(encoder)?;+ self.register_positions_and_advance(token_count);+ Ok(())+ })+ }++ pub(super) fn decode_next_token_with_hidden_capture(+ &mut self,+ token_ids: &[u64],+ embedding_injection: EmbeddingInjection,+ sampling: &mut TextSamplingState,+ precomputed_token_bitmask: Option<&[u32]>,+ ) -> Result<u64, Error> {+ self.decode_next_step(token_ids, embedding_injection, None, sampling, precomputed_token_bitmask, true, None)+ }++ /// Encode the first fast token plus followup passes onto the given+ /// command buffer, without submitting or reading results. The caller+ /// must submit the command buffer later (potentially after encoding+ /// more work from another runner).+ ///+ /// Returns `total_count = 1 + followup_count`. After the eventual+ /// submit+wait the results live in `async_chain_results[0..total_count]`.+ pub(super) fn encode_first_and_followup_tokens_on(+ &mut self,+ encoder: &mut Encoder<B>,+ initial_token_ids: &[u64],+ initial_embedding_injection: EmbeddingInjection,+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ followup_count: usize,+ vocab_limit: Option<usize>,+ sampling: &mut TextSamplingState,+ ) -> Result<usize, Error> {+ let total_count =+ 1_usize.checked_add(followup_count).ok_or(TtsModelConfigError::AsyncChainFollowupCountOverflow)?;+ if total_count > self.ctx.async_chain_capacity {+ return Err(TtsModelConfigError::AsyncChainTokenCountExceedsCapacity {+ total_count,+ capacity: self.ctx.async_chain_capacity,+ }+ .into());+ }++ let vocab_mask_limit = if let Some(limit_raw) = vocab_limit {+ let limit = limit_raw.min(self.ctx.decoder_config.vocab_size);+ if limit == 0 || limit >= self.ctx.decoder_config.vocab_size {+ None+ } else {+ self.prepare_two_token_vocab_mask(limit)?;+ self.prepare_single_token_vocab_mask(limit)?;+ Some(limit)+ }+ } else {+ None+ };++ let initial_token_count = initial_token_ids.len();+ if initial_token_count == 0 {+ return Err(Error::GenerateFailed);+ }++ // Resolve the bitmask into a local copy so that it does not borrow+ // from `self` when we later call `encode_single_forward_pass`.+ let initial_token_bitmask_owned: Option<Box<[u32]>> = if let Some(limit) = vocab_mask_limit {+ if initial_token_count == 1 {+ self.get_single_token_vocab_mask(limit).map(|m| m.into())+ } else if initial_token_count == 2 {+ self.get_two_token_vocab_mask(limit).map(|m| m.into())+ } else {+ return Err(TtsModelConfigError::UnsupportedInitialTokenCountForVocabMask {+ initial_token_count,+ }+ .into());+ }+ } else {+ None+ };+ let initial_token_bitmask: Option<&[u32]> = initial_token_bitmask_owned.as_deref();++ // Consume the initial seed FIRST to preserve RNG ordering with the+ // old two-submit path (initial token consumed its seed before followups).+ let initial_seed = if !matches!(sampling.method(), SamplingMethod::Greedy) {+ Some(sampling.next_seed())+ } else {+ None+ };++ // Pre-fill followup positions and seeds into the async chain buffers.+ let followup_base_position = self.next_position + initial_token_ids.len();+ {+ let positions_borrow = self.ctx.async_chain_positions.borrow();+ let positions_ptr = positions_borrow.cpu_ptr().as_ptr() as *mut i32;+ for pass in 0..followup_count {+ unsafe {+ *positions_ptr.add(pass) = (followup_base_position + pass) as i32;+ }+ }+ }+ {+ let seeds_borrow = self.ctx.async_chain_seeds.borrow();+ let seeds_ptr = seeds_borrow.cpu_ptr().as_ptr() as *mut u64;+ if !matches!(sampling.method(), SamplingMethod::Greedy) {+ for pass in 0..followup_count {+ unsafe {+ *seeds_ptr.add(pass) = sampling.next_seed();+ }+ }+ } else {+ for pass in 0..followup_count {+ unsafe {+ *seeds_ptr.add(pass) = 0;+ }+ }+ }+ }++ // Encode the initial forward pass (no new CB, no submit).+ let state = self.encode_single_forward_pass_on(+ encoder,+ initial_token_ids,+ initial_embedding_injection,+ sampling,+ initial_token_bitmask,+ false, // capture_hidden+ pre_injection_encode,+ initial_seed,+ )?;++ // Copy the sampled token for chaining and into results[0].+ {+ let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;+ let sampling_output_buffer = sampling_output.buffer();+ let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();++ let sampling_output_buffer = sampling_output_buffer.borrow();+ let mut token_ids_buffer = token_ids_buffer.borrow_mut();+ if followup_count > 0 {+ self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);+ }+ let mut results_buffer = self.ctx.async_chain_results.borrow_mut();+ self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);+ }++ self.encode_cache_acceptance_update_on(encoder, initial_token_count);+ self.register_positions_and_advance(initial_token_count);++ if followup_count > 0 {+ self.encode_followup_passes_on(encoder, followup_count, 0, vocab_mask_limit, sampling.method(), 1)?;+ }++ self.next_position = followup_base_position.saturating_add(followup_count);+ // Do NOT submit -- the caller will submit after encoding more work.+ Ok(total_count)+ }++ /// Encode `followup_count` autoregressive passes onto the current command+ /// buffer (which must already be open). `first_token` is the embedding+ /// input for the first pass; subsequent passes read from the+ /// `token_copy_sampled` output of the preceding pass.+ ///+ /// `results_offset_slots` is the slot index in `async_chain_results` where+ /// the first pass's result should be written (0 for standalone, 1+ for+ /// merged paths).+ fn encode_followup_passes_on(+ &mut self,+ encoder: &mut Encoder<B>,+ followup_count: usize,+ first_token: u64,+ vocab_mask_limit: Option<usize>,+ sampling_method: SamplingMethod,+ results_offset_slots: usize,+ ) -> Result<(), Error> {+ let encoding_parameters = EncodingParameters::new();+ let context_rc = self.ctx.context.clone();+ let cache_layers_rc = self.ctx.cache_layers.clone();+ let shared_buffers_rc = self.ctx.shared_buffers.clone();+ let positions_rc = self.ctx.async_chain_positions.clone();+ let seeds_rc = self.ctx.async_chain_seeds.clone();+ let token_bitmask = vocab_mask_limit.and_then(|limit| self.get_single_token_vocab_mask(limit));++ for pass in 0..followup_count {+ let results_slot = results_offset_slots + pass;+ let token_ids = [if pass == 0 {+ first_token+ } else {+ 0+ }];+ let mut state = ForwardPassState::new_llm(+ context_rc.clone(),+ &self.ctx.decoder_config,+ &self.ctx.model_shape,+ &self.ctx.scratch_buffers,+ cache_layers_rc.clone(),+ shared_buffers_rc.clone(),+ &token_ids,+ &[self.next_position + pass],+ token_bitmask,+ &[0],+ 1,+ 0,+ 1,+ false,+ None,+ pass > 0 || results_offset_slots > 0,+ self.should_fill_attention_bias,+ Some((positions_rc.clone(), pass)),+ Some((seeds_rc.clone(), pass)),+ );+ if let Some(method) = state.sampling_method_mut() {+ *method = Some(sampling_method);+ }++ {+ self.ctx+ .executables+ .embed+ .encode_lookup(&mut state, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ for layer in self.ctx.executables.layers.iter() {+ layer+ .encode(&mut state, &encoding_parameters, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ }+ self.ctx+ .executables+ .norm+ .encode(&mut state, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ self.ctx+ .executables+ .embed+ .encode_readout(&mut state, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ }++ let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;+ let sampling_output_buffer = sampling_output.buffer();+ let token_ids_buffer = self.ctx.scratch_buffers.token_ids.buffer();++ {+ let sampling_output_buffer = sampling_output_buffer.borrow();+ let mut token_ids_buffer = token_ids_buffer.borrow_mut();+ if pass + 1 < followup_count {+ self.ctx.token_copy_sampled.encode(&*sampling_output_buffer, &mut *token_ids_buffer, encoder);+ }+ let results_offset = results_slot * std::mem::size_of::<u32>();+ let mut results_buffer = self.ctx.async_chain_results.borrow_mut();+ self.ctx.token_copy_results.encode(+ &*sampling_output_buffer,+ (&mut *results_buffer, results_offset),+ encoder,+ );+ }++ self.ctx.cache_layers.borrow_mut().update_after_acceptance(&[0], None, encoder, &self.ctx.kv_cache_update);+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&[self.next_position + pass]);+ }+ Ok(())+ }++ pub(super) fn prepare_single_token_vocab_mask(+ &mut self,+ vocab_limit: usize,+ ) -> Result<(), Error> {+ let limit = vocab_limit.min(self.ctx.decoder_config.vocab_size);+ if limit == 0 || limit >= self.ctx.decoder_config.vocab_size {+ return Ok(());+ }+ if self.single_token_vocab_masks.contains_key(&limit) {+ return Ok(());+ }+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);+ let mut mask = vec![0_u32; row_words];+ for token_index in 0..limit {+ let word = token_index / 32;+ let bit = token_index % 32;+ mask[word] |= 1_u32 << bit;+ }+ self.single_token_vocab_masks.insert(limit, mask.into_boxed_slice());+ Ok(())+ }++ pub(super) fn prepare_two_token_vocab_mask(+ &mut self,+ vocab_limit: usize,+ ) -> Result<(), Error> {+ let limit = vocab_limit.min(self.ctx.decoder_config.vocab_size);+ if limit == 0 || limit >= self.ctx.decoder_config.vocab_size {+ return Ok(());+ }+ if self.two_token_vocab_masks.contains_key(&limit) {+ return Ok(());+ }+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);+ let mut mask = vec![+ 0_u32;+ row_words.checked_mul(2).ok_or(TtsModelConfigError::TwoTokenVocabMaskSizeOverflow {+ row_words+ })?+ ];+ for token_index in 0..limit {+ let word = token_index / 32;+ let bit = token_index % 32;+ mask[row_words + word] |= 1_u32 << bit;+ }+ self.two_token_vocab_masks.insert(limit, mask.into_boxed_slice());+ Ok(())+ }++ fn get_single_token_vocab_mask(+ &self,+ vocab_limit: usize,+ ) -> Option<&[u32]> {+ self.single_token_vocab_masks.get(&vocab_limit).map(|mask| mask.as_ref())+ }++ fn get_two_token_vocab_mask(+ &self,+ vocab_limit: usize,+ ) -> Option<&[u32]> {+ self.two_token_vocab_masks.get(&vocab_limit).map(|mask| mask.as_ref())+ }++ fn encode_single_forward_pass_on(+ &mut self,+ encoder: &mut Encoder<B>,+ token_ids: &[u64],+ embedding_injection: EmbeddingInjection,+ sampling: &mut TextSamplingState,+ token_bitmask: Option<&[u32]>,+ capture_hidden: bool,+ mut pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ preconsumed_seed: Option<u64>,+ ) -> Result<ForwardPassState<B>, Error> {+ if token_ids.is_empty() {+ return Err(Error::GenerateFailed);+ }++ let token_count = token_ids.len();+ let sampling_start = token_count - 1;+ let sampling_length = 1usize;++ let mut single_position = [0_usize; 1];+ let mut two_positions = [0_usize; 2];+ let positions_storage;+ let positions: &[usize] = if token_count == 1 {+ single_position[0] = self.next_position;+ &single_position+ } else if token_count == 2 {+ two_positions[0] = self.next_position;+ two_positions[1] = self.next_position + 1;+ &two_positions+ } else {+ positions_storage = (self.next_position..self.next_position + token_count).collect::<Vec<_>>();+ positions_storage.as_slice()+ };++ let mut single_seed = [0_u64; 1];+ let mut two_seeds = [0_u64; 2];+ let mut token_seeds_storage;+ let token_seeds: &mut [u64] = if token_count == 1 {+ &mut single_seed+ } else if token_count == 2 {+ &mut two_seeds+ } else {+ token_seeds_storage = vec![0_u64; token_count];+ token_seeds_storage.as_mut_slice()+ };+ if !matches!(sampling.method(), SamplingMethod::Greedy) {+ token_seeds[sampling_start] = preconsumed_seed.unwrap_or_else(|| sampling.next_seed());+ }++ if let Some(mask) = token_bitmask {+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);+ let expected_words =+ token_count.checked_mul(row_words).ok_or(TtsModelConfigError::TokenBitmaskSizeOverflow)?;+ if mask.len() != expected_words {+ return Err(TtsModelConfigError::TokenBitmaskLengthMismatch {+ actual_words: mask.len(),+ expected_words,+ token_count,+ row_words,+ }+ .into());+ }+ }++ if matches!(embedding_injection, EmbeddingInjection::OverrideFirstRowInternal) && capture_hidden {+ return Err(Error::GenerateFailed);+ }++ let mut state = ForwardPassState::new_llm(+ self.ctx.context.clone(),+ &self.ctx.decoder_config,+ &self.ctx.model_shape,+ &self.ctx.scratch_buffers,+ self.ctx.cache_layers.clone(),+ self.ctx.shared_buffers.clone(),+ token_ids,+ positions,+ token_bitmask,+ token_seeds,+ token_count,+ sampling_start,+ sampling_length,+ false,+ None,+ false,+ self.should_fill_attention_bias,+ None,+ None,+ );+ if let Some(method) = state.sampling_method_mut() {+ *method = Some(sampling.method());+ }++ let encoding_parameters = EncodingParameters::new();+ self.ctx+ .executables+ .embed+ .encode_lookup(&mut state, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ if let Some(pre_encode) = pre_injection_encode.as_mut() {+ pre_encode(self, &state, encoder)?;+ }+ match embedding_injection {+ EmbeddingInjection::None => {},+ EmbeddingInjection::AddPreloaded {+ post_scale,+ } => {+ self.encode_add_scale_from_single_bias_on(encoder, &state, token_count, post_scale.unwrap_or(1.0))?;+ },+ EmbeddingInjection::OverrideFirstRowInternal => {+ self.encode_override_first_row_from_device_on(encoder, &state, &self.single_override_embedding)?;+ },+ }+ for layer in self.ctx.executables.layers.iter() {+ layer+ .encode(&mut state, &encoding_parameters, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ }+ if capture_hidden {+ self.encode_capture_last_hidden_into_single_buffer_on(encoder, &state, token_count)?;+ }+ self.ctx.executables.norm.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ self.ctx+ .executables+ .embed+ .encode_readout(&mut state, encoder)+ .map_err(|err| Error::EncodeFailed(Box::new(err)))?;+ self.ctx.sampler.encode(&mut state, encoder).map_err(|err| Error::EncodeFailed(Box::new(err)))?;++ Ok(state)+ }++ /// Encode the KV-cache acceptance update for `token_count` tokens onto+ /// the current command buffer.+ fn encode_cache_acceptance_update_on(+ &mut self,+ encoder: &mut Encoder<B>,+ token_count: usize,+ ) {+ let mut single_accepted = [0_usize; 1];+ let two_accepted = [0_usize, 1_usize];+ let accepted_suffix_indices_storage;+ let accepted_suffix_indices: &[usize] = if token_count == 1 {+ single_accepted[0] = 0;+ &single_accepted+ } else if token_count == 2 {+ &two_accepted+ } else {+ accepted_suffix_indices_storage = (0..token_count).collect::<Vec<_>>();+ accepted_suffix_indices_storage.as_slice()+ };+ self.ctx.cache_layers.borrow_mut().update_after_acceptance(+ accepted_suffix_indices,+ None,+ encoder,+ &self.ctx.kv_cache_update,+ );+ }++ /// Register accepted token positions in the KV cache and advance+ /// `next_position` by `token_count`.+ fn register_positions_and_advance(+ &mut self,+ token_count: usize,+ ) {+ let positions_storage;+ let positions: &[usize] = if token_count == 1 {+ &[self.next_position]+ } else if token_count == 2 {+ positions_storage = [self.next_position, self.next_position + 1];+ &positions_storage[..]+ } else {+ let v: Vec<usize> = (self.next_position..self.next_position + token_count).collect();+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(&v);+ self.next_position = self.next_position.saturating_add(token_count);+ return;+ };+ self.ctx.cache_layers.borrow_mut().register_accepted_tokens(positions);+ self.next_position = self.next_position.saturating_add(token_count);+ }++ pub(super) fn decode_next_step(+ &mut self,+ token_ids: &[u64],+ embedding_injection: EmbeddingInjection,+ vocab_limit: Option<usize>,+ sampling: &mut TextSamplingState,+ precomputed_token_bitmask: Option<&[u32]>,+ capture_hidden: bool,+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ ) -> Result<u64, Error> {+ objc2::rc::autoreleasepool(|_| {+ if token_ids.is_empty() {+ return Err(Error::GenerateFailed);+ }++ let token_count = token_ids.len();++ // Resolve the token bitmask into a local copy so that it does not+ // borrow from `self` when we later call `encode_single_forward_pass`.+ let sampling_start = token_count - 1;+ let row_words = self.ctx.decoder_config.vocab_size.div_ceil(32);+ let token_bitmask_owned: Option<Box<[u32]>> = if let Some(mask) = precomputed_token_bitmask {+ let expected_words =+ token_count.checked_mul(row_words).ok_or(TtsModelConfigError::PrecomputedBitmaskSizeOverflow)?;+ if mask.len() != expected_words {+ return Err(TtsModelConfigError::PrecomputedBitmaskLengthMismatch {+ actual_words: mask.len(),+ expected_words,+ }+ .into());+ }+ Some(mask.into())+ } else if let Some(limit_raw) = vocab_limit {+ let limit = limit_raw.min(self.ctx.decoder_config.vocab_size);+ if limit == 0 {+ return Err(TtsModelConfigError::VocabLimitResolvedToZero.into());+ }+ if limit >= self.ctx.decoder_config.vocab_size {+ None+ } else if token_count == 1 {+ if let Some(mask) = self.get_single_token_vocab_mask(limit) {+ Some(mask.into())+ } else {+ let mut mask = vec![0_u32; row_words];+ for token_index in 0..limit {+ let word = token_index / 32;+ let bit = token_index % 32;+ mask[word] |= 1_u32 << bit;+ }+ Some(mask.into_boxed_slice())+ }+ } else if token_count == 2 {+ if let Some(mask) = self.get_two_token_vocab_mask(limit) {+ Some(mask.into())+ } else {+ let total_words = token_count.checked_mul(row_words).ok_or(+ TtsModelConfigError::InlineBitmaskSizeOverflow {+ token_count,+ row_words,+ },+ )?;+ let mut mask = vec![0_u32; total_words];+ for token_index in 0..limit {+ let word = token_index / 32;+ let bit = token_index % 32;+ mask[sampling_start * row_words + word] |= 1_u32 << bit;+ }+ Some(mask.into_boxed_slice())+ }+ } else {+ // token_count > 2 with vocab_limit requires a precomputed bitmask;+ // silently dropping the limit would produce incorrect sampling.+ return Err(TtsModelConfigError::VocabLimitRequiresPrecomputedBitmask {+ token_count,+ }+ .into());+ }+ } else {+ None+ };++ let token_bitmask: Option<&[u32]> = token_bitmask_owned.as_deref();++ let context = Rc::clone(&self.ctx.context);+ let mut encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;++ let state = self.encode_single_forward_pass_on(+ &mut encoder,+ token_ids,+ embedding_injection,+ sampling,+ token_bitmask,+ capture_hidden,+ pre_injection_encode,+ None,+ )?;+ self.encode_cache_acceptance_update_on(&mut encoder, token_count);++ self.submit_and_wait_command_buffer(encoder)?;+ let token = read_sampled_token_from_sampling_output(&state)?;+ self.register_positions_and_advance(token_count);+ Ok(token)+ })+ }++ fn encode_capture_last_hidden_into_single_buffer_on(+ &self,+ encoder: &mut Encoder<B>,+ state: &ForwardPassState<B>,+ token_count: usize,+ ) -> Result<(), Error> {+ if token_count == 0 {+ return Err(Error::GenerateFailed);+ }+ let model_dim = self.ctx.decoder_config.model_dim;+ let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {+ model_dim,+ })?;+ let main = state.arrays(&[ArrayId::Main])[0].clone();+ let bytes_per_element = main.data_type().size_in_bytes();+ let row_offset = (token_count - 1)+ .checked_mul(model_dim)+ .and_then(|value| value.checked_mul(bytes_per_element))+ .ok_or(TtsModelConfigError::HiddenCaptureRowOffsetOverflow {+ token_count,+ model_dim,+ })?;+ let src_offset =+ main.offset().checked_add(row_offset).ok_or(TtsModelConfigError::HiddenCaptureSourceOffsetOverflow)?;+ let capture = &self.single_hidden_capture;+ if capture.shape() != [1, model_dim] || capture.data_type() != main.data_type() {+ return Err(TtsModelConfigError::HiddenCaptureTensorMismatch {+ expected_shape: [1, model_dim].into(),+ expected_data_type: main.data_type(),+ actual_shape: capture.shape().into(),+ actual_data_type: capture.data_type(),+ }+ .into());+ }++ let main_buffer = main.buffer();+ let main_buffer = main_buffer.borrow();+ let capture_buffer = capture.buffer();+ let mut capture_buffer = capture_buffer.borrow_mut();+ self.tensor_copy.encode((&*main_buffer, src_offset), &mut *capture_buffer, model_dim_u32, encoder);+ Ok(())+ }++ fn encode_override_first_row_from_device_on(+ &self,+ encoder: &mut Encoder<B>,+ state: &ForwardPassState<B>,+ override_embedding: &Array<B>,+ ) -> Result<(), Error> {+ let model_dim = self.ctx.decoder_config.model_dim;+ let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {+ model_dim,+ })?;+ let main = state.arrays(&[ArrayId::Main])[0].clone();+ if override_embedding.shape() != [1, model_dim] || override_embedding.data_type() != main.data_type() {+ return Err(TtsModelConfigError::OverrideEmbeddingTensorMismatch {+ expected_shape: [1, model_dim].into(),+ expected_data_type: main.data_type(),+ actual_shape: override_embedding.shape().into(),+ actual_data_type: override_embedding.data_type(),+ }+ .into());+ }++ let override_buffer = override_embedding.buffer();+ let override_buffer = override_buffer.borrow();+ let main_buffer = main.buffer();+ let mut main_buffer = main_buffer.borrow_mut();+ self.tensor_copy.encode(+ (&*override_buffer, override_embedding.offset()),+ (&mut *main_buffer, main.offset()),+ model_dim_u32,+ encoder,+ );+ Ok(())+ }++ fn encode_add_scale_from_single_bias_on(+ &self,+ encoder: &mut Encoder<B>,+ state: &ForwardPassState<B>,+ token_count: usize,+ scale: f32,+ ) -> Result<(), Error> {+ if token_count == 0 {+ return Err(Error::GenerateFailed);+ }+ let model_dim = self.ctx.decoder_config.model_dim;+ let model_dim_u32 = u32::try_from(model_dim).map_err(|_| TtsModelConfigError::ModelDimExceedsU32 {+ model_dim,+ })?;+ let total_len = token_count.checked_mul(model_dim).ok_or(TtsModelConfigError::AddScaleTotalLengthOverflow {+ token_count,+ model_dim,+ })?;+ let total_len_u32 =+ u32::try_from(total_len).map_err(|_| TtsModelConfigError::AddScaleTotalLengthExceedsU32 {+ total_len,+ })?;++ let main = state.arrays(&[ArrayId::Main])[0].clone();+ let bias = &self.single_override_embedding;+ if bias.shape() != [1, model_dim] || bias.data_type() != main.data_type() {+ return Err(TtsModelConfigError::AddScaleBiasTensorMismatch {+ expected_shape: [1, model_dim].into(),+ expected_data_type: main.data_type(),+ actual_shape: bias.shape().into(),+ actual_data_type: bias.data_type(),+ }+ .into());+ }++ let bias_buffer = bias.buffer();+ let bias_buffer = bias_buffer.borrow();+ let main_output_buffer = main.buffer();+ let mut main_output_buffer = main_output_buffer.borrow_mut();+ // TensorAddScale is elementwise, so in-place read/write aliasing is valid here.+ let main_input_buffer: &B::Buffer = unsafe { &*(&*main_output_buffer as *const B::Buffer) };+ self.tensor_add_scale.encode(+ (main_input_buffer, main.offset()),+ &*bias_buffer,+ (&mut *main_output_buffer, main.offset()),+ model_dim_u32,+ total_len_u32,+ scale,+ encoder,+ );+ Ok(())+ }++ /// Encode a single decode step (embedding, transformer, readout, sampling,+ /// and optionally hidden capture) onto the **current** command buffer.+ /// Does NOT create a new command buffer, does NOT submit.+ ///+ /// The caller is responsible for submitting the command buffer and reading+ /// the sampled token from `async_chain_results[0]` after the eventual+ /// submit.+ pub(super) fn encode_next_step_on(+ &mut self,+ encoder: &mut Encoder<B>,+ token_ids: &[u64],+ embedding_injection: EmbeddingInjection,+ sampling: &mut TextSamplingState,+ precomputed_token_bitmask: Option<&[u32]>,+ capture_hidden: bool,+ pre_injection_encode: Option<&mut PreInjectionEncodeCallback<'_, B>>,+ ) -> Result<(), Error> {+ let token_count = token_ids.len();++ let state = self.encode_single_forward_pass_on(+ encoder,+ token_ids,+ embedding_injection,+ sampling,+ precomputed_token_bitmask,+ capture_hidden,+ pre_injection_encode,+ None,+ )?;++ // Copy sampled token to async_chain_results[0] so the caller can+ // read it after the eventual submit.+ {+ let sampling_output = state.sampling_output().ok_or(Error::SamplingFailed)?;+ let sampling_output_buffer = sampling_output.buffer();++ let sampling_output_buffer = sampling_output_buffer.borrow();+ let mut results_buffer = self.ctx.async_chain_results.borrow_mut();+ self.ctx.token_copy_results.encode(&*sampling_output_buffer, (&mut *results_buffer, 0), encoder);+ }++ self.encode_cache_acceptance_update_on(encoder, token_count);+ // Do NOT submit -- the caller will submit.+ self.register_positions_and_advance(token_count);+ Ok(())+ }++ pub(super) fn submit_and_wait_command_buffer(+ &mut self,+ encoder: Encoder<B>,+ ) -> Result<(), Error> {+ self.instrumentation.command_buffers_submitted += 1;+ encoder+ .end_encoding()+ .submit()+ .wait_until_completed()+ .map_err(|err| Error::CommandBufferFailed(Box::new(err)))?;+ self.instrumentation.host_waits += 1;+ Ok(())+ }++ pub(super) fn copy_async_chain_results_to_on(+ &self,+ encoder: &mut Encoder<B>,+ src_slot: usize,+ dst: (&mut B::Buffer, usize),+ count: usize,+ ) -> Result<(), Error> {+ let src_offset = src_slot.checked_mul(std::mem::size_of::<u32>()).ok_or(+ TtsModelConfigError::AsyncChainCopySourceOffsetOverflow {+ src_slot,+ },+ )?;+ let copy_size =+ count.checked_mul(std::mem::size_of::<u32>()).ok_or(TtsModelConfigError::AsyncChainCopySizeOverflow {+ count,+ })?;+ let results_buffer = self.ctx.async_chain_results.borrow();+ encoder.encode_copy(&results_buffer, src_offset..src_offset + copy_size, dst.0, dst.1..dst.1 + copy_size);+ Ok(())+ }++ pub(super) fn read_async_chain_result(+ &self,+ slot: usize,+ ) -> Result<u32, Error> {+ let results_buffer = self.ctx.async_chain_results.borrow();+ let ptr = results_buffer.cpu_ptr().as_ptr() as *const u32;+ let capacity = self.ctx.async_chain_capacity;+ if slot >= capacity {+ return Err(TtsModelConfigError::AsyncChainResultSlotOutOfBounds {+ slot,+ capacity,+ }+ .into());+ }+ Ok(unsafe { *ptr.add(slot) })+ }++ pub(super) fn take_instrumentation(&mut self) -> RunnerInstrumentation {+ std::mem::take(&mut self.instrumentation)+ }++ pub(super) fn clear_instrumentation(&mut self) {+ self.instrumentation = RunnerInstrumentation::default();+ }+}++fn read_sampled_token_from_sampling_output<B: Backend>(state: &ForwardPassState<B>) -> Result<u64, Error> {+ let output = state.sampling_output().ok_or(Error::SamplingFailed)?;+ let tokens = output.as_slice::<u32>();+ let token = tokens.first().copied().ok_or(Error::SamplingFailed)?;+ Ok(u64::from(token))+}diff --git a/crates/uzu/src/session/tts_session/decoder_support.rs b/crates/uzu/src/session/tts_session/decoder_support.rsnew file mode 100644index 000000000..773bf7e39--- /dev/null+++ b/crates/uzu/src/session/tts_session/decoder_support.rs@@ -0,0 +1,598 @@+use super::*;++pub(super) type PreInjectionEncodeCallback<'a, B> =+ dyn FnMut(&TokenDecoderRunner<B>, &ForwardPassState<B>, &mut Encoder<B>) -> Result<(), Error> + 'a;++pub(super) enum EmbeddingInjection {+ None,+ AddPreloaded {+ post_scale: Option<f32>,+ },+ OverrideFirstRowInternal,+}++pub(super) struct TextSamplingState {+ rng: StdRng,+ method: SamplingMethod,+}++impl TextSamplingState {+ pub(super) fn from_config(+ seed: u64,+ config: &TextSamplingConfig,+ ) -> Self {+ Self::with_params(seed, config.temperature, config.top_p)+ }++ pub(super) fn with_params(+ seed: u64,+ temperature: f32,+ top_p: f32,+ ) -> Self {+ let method = if temperature <= 0.0 || top_p <= 0.0 {+ SamplingMethod::Greedy+ } else {+ SamplingMethod::Stochastic {+ temperature: Some(temperature),+ top_k: None,+ top_p: Some(top_p),+ min_p: None,+ processing_order: SamplingProcessingOrder::FiltersThenTemperature,+ }+ };+ Self {+ rng: StdRng::seed_from_u64(seed),+ method,+ }+ }++ pub(super) fn method(&self) -> SamplingMethod {+ self.method+ }++ pub(super) fn next_seed(&mut self) -> u64 {+ self.rng.random::<u64>()+ }+}++pub(super) fn text_decoder_prefill_step_size(+ config: &TextDecoderRuntimeConfig,+ context_length: usize,+) -> usize {+ config.prefill_step_size.min(context_length.max(1)).max(1)+}++pub(super) fn normalize_rendered_prompt(+ mut rendered: String,+ template: &str,+ drop_initial_newline: bool,+) -> String {+ if drop_initial_newline && rendered.starts_with('\n') {+ rendered.remove(0);+ }+ if template.ends_with('\n') && rendered.ends_with('\n') {+ rendered.pop();+ }+ rendered+}++impl AdaptiveChunkController {+ pub(super) fn new(config: &TtsRunConfig) -> Self {+ Self {+ ema_ms_per_frame: None,+ current_chunk_frames: config.min_chunk_frames.max(1),+ }+ }++ pub(super) fn target_frames(+ &self,+ config: &TtsRunConfig,+ ) -> usize {+ let min_frames = config.min_chunk_frames.max(1);+ let max_frames = config.max_chunk_frames.max(min_frames);+ match config.chunk_policy {+ TtsChunkPolicy::Fixed => min_frames,+ TtsChunkPolicy::Adaptive => {+ let Some(ema_ms_per_frame) = self.ema_ms_per_frame else {+ return min_frames;+ };+ let raw = (config.target_emit_latency_ms as f64 / ema_ms_per_frame).round();+ let candidate = raw.max(min_frames as f64).min(max_frames as f64) as usize;+ if self.current_chunk_frames == 0 {+ return candidate;+ }+ if candidate <= self.current_chunk_frames {+ return self.current_chunk_frames;+ }+ let change = ((candidate as f64 - self.current_chunk_frames as f64).abs()+ / self.current_chunk_frames as f64)+ .max(0.0);+ if change < config.chunk_hysteresis_fraction {+ self.current_chunk_frames+ } else {+ candidate+ }+ },+ }+ }++ pub(super) fn observe(+ &mut self,+ config: &TtsRunConfig,+ frames: usize,+ decode_elapsed: std::time::Duration,+ next_chunk_frames: usize,+ ) {+ if frames == 0 {+ return;+ }+ let alpha = config.chunk_ema_alpha;+ let ms_per_frame = (decode_elapsed.as_secs_f64() * 1000.0) / frames as f64;+ self.ema_ms_per_frame = Some(match self.ema_ms_per_frame {+ Some(previous) => previous * (1.0 - alpha) + ms_per_frame * alpha,+ None => ms_per_frame,+ });+ self.current_chunk_frames = next_chunk_frames.max(1);+ }++ pub(super) fn adapt_up_for_realtime(+ &mut self,+ config: &TtsRunConfig,+ generated_frames: usize,+ sample_rate: u32,+ decode_elapsed: std::time::Duration,+ emitted_audio_frames: usize,+ ) {+ if generated_frames == 0 || emitted_audio_frames == 0 || sample_rate == 0 {+ return;+ }+ let min_frames = config.min_chunk_frames.max(1);+ let max_frames = config.max_chunk_frames.max(min_frames);+ let decode_ms = decode_elapsed.as_secs_f64() * 1000.0;+ let produced_audio_ms = (emitted_audio_frames as f64) * 1000.0 / f64::from(sample_rate);+ if produced_audio_ms <= 0.0 {+ return;+ }+ let realtime_ratio = decode_ms / produced_audio_ms;+ if realtime_ratio <= 1.0 {+ return;+ }+ let scaled = ((generated_frames as f64) * realtime_ratio * 1.1).ceil() as usize;+ let clamped = scaled.clamp(min_frames, max_frames);+ self.current_chunk_frames = self.current_chunk_frames.max(clamped);+ }++ pub(super) fn promote_to_max_chunk(+ &mut self,+ config: &TtsRunConfig,+ ) {+ let min_frames = config.min_chunk_frames.max(1);+ let max_frames = config.max_chunk_frames.max(min_frames);+ self.current_chunk_frames = max_frames;+ }+}++pub(super) fn next_startup_target_frames(+ current_target_frames: usize,+ startup_cap_frames: usize,+) -> usize {+ let startup_cap_frames = startup_cap_frames.max(1);+ current_target_frames.max(1).saturating_mul(2).min(startup_cap_frames)+}++pub(super) struct StreamingTokenAccumulator {+ by_codebook: Vec<Vec<u32>>,+}++impl StreamingTokenAccumulator {+ pub(super) fn new(num_codebooks: usize) -> Result<Self, Error> {+ if num_codebooks == 0 {+ return Err(Error::UnableToLoadConfig);+ }+ Ok(Self {+ by_codebook: vec![Vec::new(); num_codebooks],+ })+ }++ pub(super) fn push_frame(+ &mut self,+ frame_codes: &[u32],+ ) -> Result<(), Error> {+ if frame_codes.len() != self.by_codebook.len() {+ return Err(Error::GenerateFailed);+ }+ for (codebook, &token) in self.by_codebook.iter_mut().zip(frame_codes.iter()) {+ codebook.push(token);+ }+ Ok(())+ }++ pub(super) fn frames(&self) -> usize {+ self.by_codebook.first().map_or(0, Vec::len)+ }++ pub(super) fn to_grid_range(+ &self,+ frame_start: usize,+ frame_end: usize,+ ) -> Result<AudioTokenGrid, Error> {+ let frames = self.frames();+ if frame_start > frame_end || frame_end > frames {+ return Err(Error::GenerateFailed);+ }+ let range_frames = frame_end - frame_start;+ let mut tokens = Vec::with_capacity(self.by_codebook.len() * range_frames);+ for codebook in &self.by_codebook {+ if codebook.len() != frames {+ return Err(Error::GenerateFailed);+ }+ tokens.extend_from_slice(&codebook[frame_start..frame_end]);+ }++ AudioTokenGrid::new(+ tokens.into_boxed_slice(),+ 1,+ self.by_codebook.len(),+ range_frames,+ vec![range_frames].into_boxed_slice(),+ )+ .map_err(Error::from)+ }+}++pub(super) fn slice_grid_range(+ grid: &AudioTokenGrid,+ frame_start: usize,+ frame_end: usize,+) -> Result<AudioTokenGrid, Error> {+ if frame_start > frame_end || frame_end > grid.frames() {+ return Err(Error::GenerateFailed);+ }+ let range_frames = frame_end.saturating_sub(frame_start);+ let batch_size = grid.batch_size();+ let codebooks = grid.codebooks();+ let frames = grid.frames();++ let mut tokens = Vec::with_capacity(batch_size * codebooks * range_frames);+ let row_stride = frames;+ for batch in 0..batch_size {+ for codebook in 0..codebooks {+ let row_index = batch+ .checked_mul(codebooks)+ .and_then(|value| value.checked_add(codebook))+ .ok_or(Error::GenerateFailed)?;+ let row_start = row_index.checked_mul(row_stride).ok_or(Error::GenerateFailed)?;+ let src_start = row_start.checked_add(frame_start).ok_or(Error::GenerateFailed)?;+ let src_end = row_start.checked_add(frame_end).ok_or(Error::GenerateFailed)?;+ tokens.extend_from_slice(&grid.tokens()[src_start..src_end]);+ }+ }++ let mut lengths = Vec::with_capacity(batch_size);+ for &length in grid.lengths() {+ lengths.push(length.saturating_sub(frame_start).min(range_frames));+ }++ AudioTokenGrid::new(tokens.into_boxed_slice(), batch_size, codebooks, range_frames, lengths.into_boxed_slice())+ .map_err(Error::from)+}++pub(super) fn accumulate_audio_decode_step_stats(+ decode_calls: &mut usize,+ input_frames: &mut usize,+ decoded_window_frames: &mut usize,+ max_decoded_window_frames: &mut usize,+ step: Option<AudioDecodeStepStats>,+) {+ let Some(step) = step else {+ return;+ };+ if step.input_frames == 0 && step.decoded_window_frames == 0 {+ return;+ }+ *decode_calls = decode_calls.saturating_add(1);+ *input_frames = input_frames.saturating_add(step.input_frames);+ *decoded_window_frames = decoded_window_frames.saturating_add(step.decoded_window_frames);+ *max_decoded_window_frames = (*max_decoded_window_frames).max(step.decoded_window_frames);+}++pub(super) struct StreamingSynthesisState<'a, F: FnMut(&AudioPcmBatch)> {+ pub(super) on_chunk: &'a mut F,+ pub(super) pending_chunk: Option<PendingStreamingChunk>,+ pub(super) callback_seconds: f64,+ pub(super) audio_decode_seconds_in_loop: f64,+ pub(super) audio_decode_seconds: f64,+ pub(super) output_samples: Vec<f32>,+ pub(super) output_frames: usize,+ pub(super) output_sample_rate: u32,+ pub(super) output_channels: usize,+ pub(super) emitted_chunks: usize,+ pub(super) first_emit_pending: bool,+ pub(super) first_chunk_seconds: Option<f64>,+ pub(super) first_chunk_frames: usize,+ pub(super) stream_start: Instant,+ pub(super) startup_target_frames: usize,+ pub(super) startup_cap_frames: usize,+ pub(super) chunk_controller: AdaptiveChunkController,+ pub(super) audio_decode_calls: usize,+ pub(super) audio_input_frames: usize,+ pub(super) audio_decoded_window_frames: usize,+ pub(super) audio_max_decoded_window_frames: usize,+}++impl<'a, F: FnMut(&AudioPcmBatch)> StreamingSynthesisState<'a, F> {+ pub(super) fn maybe_flush_pending(+ &mut self,+ force: bool,+ count_in_loop: bool,+ config: &TtsRunConfig,+ ) -> Result<(), Error> {+ let (+ step_stats,+ ready_frames,+ next_chunk_frames,+ submission_decode_duration,+ partial_pcm,+ resolve_decode_duration,+ ) = if force {+ let pending = self.pending_chunk.take().ok_or(Error::GenerateFailed)?;+ let step_stats = pending.chunk.step_stats();+ let ready_frames = pending.ready_frames;+ let next_chunk_frames = pending.next_chunk_frames;+ let submission_decode_duration = pending.submission_decode_duration;+ let (partial_pcm, resolve_decode_duration) = pending.chunk.resolve_with_decode_duration()?;+ (+ step_stats,+ ready_frames,+ next_chunk_frames,+ submission_decode_duration,+ partial_pcm,+ resolve_decode_duration,+ )+ } else {+ let Some(pending) = self.pending_chunk.as_mut() else {+ return Ok(());+ };+ let Some((partial_pcm, resolve_decode_duration)) = pending.chunk.try_resolve_with_decode_duration()? else {+ return Ok(());+ };+ let step_stats = pending.chunk.step_stats();+ let ready_frames = pending.ready_frames;+ let next_chunk_frames = pending.next_chunk_frames;+ let submission_decode_duration = pending.submission_decode_duration;+ let _ = self.pending_chunk.take().ok_or(Error::GenerateFailed)?;+ (+ step_stats,+ ready_frames,+ next_chunk_frames,+ submission_decode_duration,+ partial_pcm,+ resolve_decode_duration,+ )+ };++ accumulate_audio_decode_step_stats(+ &mut self.audio_decode_calls,+ &mut self.audio_input_frames,+ &mut self.audio_decoded_window_frames,+ &mut self.audio_max_decoded_window_frames,+ step_stats,+ );+ let decode_elapsed = submission_decode_duration.saturating_add(resolve_decode_duration);+ if count_in_loop {+ self.audio_decode_seconds_in_loop += decode_elapsed.as_secs_f64();+ }+ self.audio_decode_seconds += decode_elapsed.as_secs_f64();++ let partial_sample_rate = partial_pcm.sample_rate();+ let callback_start = Instant::now();+ let emitted_frames = if partial_pcm.lengths().len() == 1 {+ partial_pcm.lengths()[0]+ } else {+ return Err(Error::GenerateFailed);+ };+ if emitted_frames > 0 {+ (self.on_chunk)(&partial_pcm);+ }+ self.callback_seconds += callback_start.elapsed().as_secs_f64();+ self.chunk_controller.observe(config, ready_frames, decode_elapsed, next_chunk_frames);++ if emitted_frames > 0 {+ self.chunk_controller.adapt_up_for_realtime(+ config,+ ready_frames,+ partial_sample_rate,+ decode_elapsed,+ emitted_frames,+ );+ self.output_samples.extend_from_slice(partial_pcm.samples());+ self.output_frames = self.output_frames.saturating_add(emitted_frames);+ self.output_sample_rate = partial_pcm.sample_rate();+ self.output_channels = partial_pcm.channels();+ self.emitted_chunks = self.emitted_chunks.saturating_add(1);+ if self.first_emit_pending {+ self.first_emit_pending = false;+ self.first_chunk_seconds = Some(self.stream_start.elapsed().as_secs_f64());+ self.first_chunk_frames = emitted_frames;+ self.chunk_controller.promote_to_max_chunk(config);+ }+ } else if self.first_emit_pending {+ self.startup_target_frames =+ next_startup_target_frames(self.startup_target_frames, self.startup_cap_frames);+ }++ Ok(())+ }+}++impl<B: Backend + Send + Sync> TtsSession<B> {+ pub fn new(model_path: PathBuf) -> Result<Self, Error> {+ Self::new_with_options(model_path, TtsSessionOptions::default())+ }++ pub fn new_with_options(+ model_path: PathBuf,+ options: TtsSessionOptions,+ ) -> Result<Self, Error> {+ if !model_path.exists() {+ return Err(Error::ModelFolderNotFound);+ }++ let config_path = model_path.join("config.json");+ if !config_path.exists() {+ return Err(Error::UnableToLoadConfig);+ }+ let config_file = File::open(&config_path).map_err(|_| Error::UnableToLoadConfig)?;+ let model_metadata: ModelMetadata =+ serde_json::from_reader(std::io::BufReader::new(config_file)).map_err(|_| Error::UnableToLoadConfig)?;++ Self::from_model_metadata_with_options(model_path, model_metadata, options)+ }++ pub fn last_execution_stats(&self) -> Option<TtsExecutionStats> {+ self.last_execution_stats.clone()+ }++ pub fn sample_rate(&self) -> u32 {+ self.audio.sample_rate()+ }++ fn from_model_metadata_with_options(+ model_path: PathBuf,+ model_metadata: ModelMetadata,+ options: TtsSessionOptions,+ ) -> Result<Self, Error> {+ let tokenizer_path = model_path.join("tokenizer.json");+ if !tokenizer_path.exists() {+ return Err(Error::UnableToLoadTokenizer);+ }+ let tokenizer = Tokenizer::from_file(&tokenizer_path).map_err(|_| Error::UnableToLoadTokenizer)?;++ let loaded_runtime = load_tts_runtime(&model_path, &model_metadata, &options)?;++ Ok(Self {+ tokenizer,+ audio: loaded_runtime.audio,+ audio_decoder: loaded_runtime.audio_decoder,+ message_processor_config: loaded_runtime.message_processor_config,+ text_decoder: loaded_runtime.text_decoder,+ last_execution_stats: None,+ })+ }++ pub(super) fn render_prompt(+ &self,+ input: &Input,+ ) -> Result<String, Error> {+ let messages = input+ .get_messages()+ .into_iter()+ .map(|message| message.resolve(&self.message_processor_config))+ .collect::<Vec<_>>();++ let template_name = "tts_prompt_template";+ let mut environment = Environment::new();+ environment+ .add_template(template_name, self.message_processor_config.prompt_template.as_str())+ .map_err(|_| Error::UnableToLoadPromptTemplate)?;+ let template = environment.get_template(template_name).map_err(|_| Error::UnableToLoadPromptTemplate)?;++ let result = template+ .render(context!(+ messages => messages+ ))+ .map_err(|_| Error::UnableToRenderPromptTemplate)?;++ Ok(normalize_rendered_prompt(+ result,+ self.message_processor_config.prompt_template.as_str(),+ self.message_processor_config.drop_initial_newline,+ ))+ }+}++pub(super) fn semantic_token_to_code(+ semantic_token: u64,+ semantic_begin: i64,+ semantic_end: i64,+ token_upper_bound: usize,+) -> u32 {+ if semantic_begin > semantic_end || token_upper_bound == 0 {+ return 0;+ }++ let semantic = semantic_token as i64;+ if semantic < semantic_begin || semantic > semantic_end {+ return 0;+ }++ let relative = usize::try_from(semantic - semantic_begin).unwrap_or(0);+ let clamped = relative.min(token_upper_bound.saturating_sub(1));+ u32::try_from(clamped).unwrap_or(0)+}++pub(super) fn build_semantic_sampling_mask_row(+ vocab_size: usize,+ semantic_begin: i64,+ semantic_end: i64,+ im_end: i64,+) -> Result<Box<[u32]>, Error> {+ if vocab_size == 0 || semantic_begin > semantic_end {+ return Err(Error::UnableToLoadConfig);+ }++ let max_token_id = i64::try_from(vocab_size.saturating_sub(1)).map_err(|_| Error::UnableToLoadConfig)?;+ if semantic_begin < 0 || semantic_end < 0 || semantic_end > max_token_id || im_end < 0 || im_end > max_token_id {+ return Err(Error::UnableToLoadConfig);+ }++ let row_words = vocab_size.div_ceil(32);+ let mut mask = vec![0_u32; row_words];+ for token_index in semantic_begin..=semantic_end {+ let token = usize::try_from(token_index).map_err(|_| Error::UnableToLoadConfig)?;+ let word = token / 32;+ let bit = token % 32;+ mask[word] |= 1_u32 << bit;+ }+ let im_end_token = usize::try_from(im_end).map_err(|_| Error::UnableToLoadConfig)?;+ let word = im_end_token / 32;+ let bit = im_end_token % 32;+ mask[word] |= 1_u32 << bit;+ Ok(mask.into_boxed_slice())+}++pub(super) fn clear_token_in_sampling_mask(+ mask: &mut [u32],+ token: i64,+) -> Result<(), Error> {+ if token < 0 {+ return Err(Error::UnableToLoadConfig);+ }+ let token = usize::try_from(token).map_err(|_| Error::UnableToLoadConfig)?;+ let word = token / 32;+ let bit = token % 32;+ if word >= mask.len() {+ return Err(Error::UnableToLoadConfig);+ }+ mask[word] &= !(1_u32 << bit);+ Ok(())+}++pub(super) fn expand_token_mask_for_sampling_row(+ row_mask: &[u32],+ token_count: usize,+) -> Result<Box<[u32]>, Error> {+ if token_count == 0 || row_mask.is_empty() {+ return Err(Error::GenerateFailed);+ }+ if token_count == 1 {+ return Ok(row_mask.to_vec().into_boxed_slice());+ }++ let row_words = row_mask.len();+ let total_words = token_count.checked_mul(row_words).ok_or(Error::GenerateFailed)?;+ let mut expanded = vec![u32::MAX; total_words];+ let offset = (token_count - 1).checked_mul(row_words).ok_or(Error::GenerateFailed)?;+ expanded[offset..offset + row_words].copy_from_slice(row_mask);+ Ok(expanded.into_boxed_slice())+}diff --git a/crates/uzu/src/session/tts_session/fishaudio.rs b/crates/uzu/src/session/tts_session/fishaudio.rsnew file mode 100644index 000000000..52afd9d61--- /dev/null+++ b/crates/uzu/src/session/tts_session/fishaudio.rs@@ -0,0 +1,839 @@+use std::{collections::BTreeSet, sync::LazyLock};++use regex::Regex;++use super::*;+use crate::{+ array::Array,+ config::TtsMessageProcessorConfig,+ session::types::{TtsModelConfigError, TtsPromptConfigError},+};++pub(super) struct FishAudioTextDecoderRuntime<B: Backend> {+ slow_runner: TokenDecoderRunner<B>,+ fast_runner: TokenDecoderRunner<B>,+ semantic_bridge: FishAudioSemanticBridge<B>,+ runtime_config: TextDecoderRuntimeConfig,+ semantic_token_begin_id: i64,+ semantic_token_end_id: i64,+ semantic_cardinality: usize,+ im_end_token_id: i64,+ codebook_size: usize,+ num_codebooks: usize,+ slow_model_dim: usize,+ fast_model_dim: usize,+ max_seq_len: usize,+ scale_codebook_embeddings: bool,+ fast_vocab_limit: usize,+ apply_semantic_sampling_mask: bool,+ semantic_sampling_mask_row: Box<[u32]>,+ semantic_sampling_mask_without_im_end_row: Box<[u32]>,+ current_codes_scratch: Vec<u32>,+ instrumentation: RunnerInstrumentation,+}++struct FishAudioSemanticBridge<B: Backend> {+ embedding_rows_sum: <B::Kernels as Kernels>::EmbeddingRowsSumKernel,+ projection: <B::Kernels as MatmulKernels>::MatmulKernel,+ tensor_copy: <B::Kernels as Kernels>::TensorCopyKernel,+ codebook_embeddings: Array<B>,+ codebook_token_indices: Array<B>,+ projection_weights: Option<Array<B>>,+}++impl<B: Backend> FishAudioSemanticBridge<B> {+ fn load(+ context: &B::Context,+ parameter_tree: &crate::parameters::ParameterTree<B::Context>,+ config: &crate::config::FishAudioTextDecoderConfig,+ data_type: DataType,+ ) -> Result<Self, Error> {+ let codebook_embeddings = load_float_tensor_array(+ parameter_tree,+ "text_decoder.codebook_embeddings.weights",+ [+ config.codebook_size.checked_mul(config.num_codebooks).ok_or(Error::UnableToLoadConfig)?,+ config.slow_model_dim,+ ],+ data_type,+ )?;+ let fast_model_projection = if config.fast_model_projection_config.is_some() {+ Some(load_float_tensor_array(+ parameter_tree,+ "text_decoder.fast_model_projection.weights",+ [config.fast_model_dim, config.slow_model_dim],+ data_type,+ )?)+ } else {+ None+ };++ Self::new(+ context,+ data_type,+ codebook_embeddings,+ fast_model_projection,+ config.num_codebooks,+ config.codebook_size,+ config.slow_model_dim,+ config.fast_model_dim,+ )+ }++ fn new(+ context: &B::Context,+ data_type: DataType,+ codebook_embeddings: Array<B>,+ fast_model_projection: Option<Array<B>>,+ num_codebooks: usize,+ codebook_size: usize,+ slow_model_dim: usize,+ fast_model_dim: usize,+ ) -> Result<Self, Error> {+ if codebook_embeddings.shape()+ != [num_codebooks.checked_mul(codebook_size).ok_or(Error::UnableToLoadConfig)?, slow_model_dim]+ || codebook_embeddings.data_type() != data_type+ {+ return Err(Error::UnableToLoadConfig);+ }+ if let Some(ref projection) = fast_model_projection+ && (projection.shape() != [fast_model_dim, slow_model_dim] || projection.data_type() != data_type)+ {+ return Err(Error::UnableToLoadConfig);+ }++ let embedding_rows_sum = <B::Kernels as Kernels>::EmbeddingRowsSumKernel::new(context, data_type)+ .map_err(unable_to_create_context)?;+ let projection = <<B::Kernels as MatmulKernels>::MatmulKernel as MatmulKernel>::new(context, data_type)+ .map_err(unable_to_create_context)?;+ let tensor_copy =+ <B::Kernels as Kernels>::TensorCopyKernel::new(context, data_type).map_err(unable_to_create_context)?;+ let codebook_token_indices =+ context.create_array(&[num_codebooks], DataType::U32, "tts_codebook_token_indices");++ Ok(Self {+ embedding_rows_sum,+ projection,+ tensor_copy,+ codebook_embeddings,+ codebook_token_indices,+ projection_weights: fast_model_projection,+ })+ }+}++fn load_float_tensor_array<B: Backend>(+ parameter_tree: &crate::parameters::ParameterTree<B::Context>,+ key: &str,+ expected_shape: [usize; 2],+ target_data_type: DataType,+) -> Result<Array<B>, Error> {+ let array = parameter_tree.leaf_array(key).map_err(|_| Error::UnableToLoadWeights)?;+ if array.shape() != expected_shape {+ return Err(Error::UnableToLoadConfig);+ }+ if array.data_type() != target_data_type {+ return Err(TtsModelConfigError::FishAudioTensorDataTypeMismatch {+ key: key.into(),+ expected: target_data_type,+ actual: array.data_type(),+ }+ .into());+ }+ Ok(array)+}++fn validate_fishaudio_decoder_contract(+ num_codebooks: usize,+ codebook_size: usize,+ max_seq_len: usize,+ semantic_token_begin_id: i64,+ semantic_token_end_id: i64,+ audio_num_codebooks: usize,+ audio_codec_cardinality: usize,+ audio_semantic_cardinality: usize,+) -> Result<usize, Error> {+ if num_codebooks <= 1 || codebook_size == 0 || max_seq_len == 0 {+ return Err(TtsModelConfigError::FishAudioDecoderCoreParametersInvalid {+ num_codebooks,+ codebook_size,+ max_seq_len,+ }+ .into());+ }+ if num_codebooks != audio_num_codebooks {+ return Err(TtsModelConfigError::FishAudioDecoderNumCodebooksMismatch {+ decoder_num_codebooks: num_codebooks,+ audio_num_codebooks,+ }+ .into());+ }+ if audio_codec_cardinality == 0 || audio_codec_cardinality > codebook_size {+ return Err(TtsModelConfigError::FishAudioDecoderCodebookSizeTooSmall {+ codebook_size,+ audio_codec_cardinality,+ }+ .into());+ }+ if semantic_token_begin_id > semantic_token_end_id {+ return Err(TtsModelConfigError::FishAudioSemanticTokenRangeInvalid {+ begin: semantic_token_begin_id,+ end: semantic_token_end_id,+ }+ .into());+ }++ let semantic_cardinality = usize::try_from(semantic_token_end_id - semantic_token_begin_id + 1).map_err(|_| {+ TtsModelConfigError::FishAudioSemanticTokenRangeOverflow {+ begin: semantic_token_begin_id,+ end: semantic_token_end_id,+ }+ })?;+ if semantic_cardinality == 0 || semantic_cardinality != audio_semantic_cardinality {+ return Err(TtsModelConfigError::FishAudioSemanticCodecCardinalityMismatch {+ semantic_cardinality,+ audio_semantic_cardinality,+ }+ .into());+ }++ Ok(semantic_cardinality)+}++static MESSAGE_FIELD_REF_REGEX: LazyLock<Regex> =+ LazyLock::new(|| Regex::new(r"message\.([A-Za-z_][A-Za-z0-9_]*)").expect("message field regex"));++pub(super) fn validate_fishaudio_message_processor_config(config: &TtsMessageProcessorConfig) -> Result<(), Error> {+ let builtin_fields =+ ["role", "content", "reasoning_content", "speaker_id", "style"].into_iter().collect::<BTreeSet<_>>();+ let referenced_fields = MESSAGE_FIELD_REF_REGEX+ .captures_iter(config.prompt_template.as_str())+ .filter_map(|capture| capture.get(1).map(|field| field.as_str().to_owned()))+ .collect::<BTreeSet<_>>();++ for field in referenced_fields {+ if !builtin_fields.contains(field.as_str()) && !config.default_message_fields.contains_key(field.as_str()) {+ return Err(TtsPromptConfigError::MissingDefaultMessageField {+ field: field.into_boxed_str(),+ }+ .into());+ }+ }++ Ok(())+}++pub(super) fn build_fishaudio_text_decoder_runtime<B: Backend>(+ config: &crate::config::FishAudioTextDecoderConfig,+ audio: &AudioGenerationContext<B>,+ model_path: &Path,+ runtime_config: &TextDecoderRuntimeConfig,+) -> Result<Box<dyn SemanticDecoderBackend>, Error> {+ let audio_semantic_cardinality =+ audio.runtime().config().semantic_codec_cardinality().ok_or(Error::UnableToLoadConfig)?;+ let semantic_cardinality = validate_fishaudio_decoder_contract(+ config.num_codebooks,+ config.codebook_size,+ config.max_seq_len,+ config.semantic_token_begin_id,+ config.semantic_token_end_id,+ audio.num_codebooks(),+ audio.codec_cardinality(),+ audio_semantic_cardinality,+ )?;+ if !config.slow_readout_config.is_full_precision() {+ return Err(Error::UnableToLoadConfig);+ }+ if !config.fast_readout_config.is_full_precision() {+ return Err(Error::UnableToLoadConfig);+ }+ if config.slow_model_config.model_dim != config.slow_model_dim {+ return Err(Error::UnableToLoadConfig);+ }+ if config.fast_model_config.model_dim != config.fast_model_dim {+ return Err(Error::UnableToLoadConfig);+ }+ if config.short_logits_size == 0 {+ return Err(Error::UnableToLoadConfig);+ }+ let semantic_sampling_mask_row = build_semantic_sampling_mask_row(+ config.vocab_size,+ config.semantic_token_begin_id,+ config.semantic_token_end_id,+ config.im_end_token_id,+ )?;+ let mut semantic_sampling_mask_without_im_end_row = semantic_sampling_mask_row.to_vec();+ clear_token_in_sampling_mask(&mut semantic_sampling_mask_without_im_end_row, config.im_end_token_id)?;++ let slow_transformer_config = config.slow_model_config.clone();+ let fast_transformer_config = config.fast_model_config.clone();++ let slow_inner_config = InnerModelConfig {+ embedding_config: config.slow_embeddings_config.to_text_decoder_embedding_config(),+ transformer_config: slow_transformer_config,+ vocab_size: config.vocab_size,+ };+ let fast_inner_config = InnerModelConfig {+ embedding_config: config.fast_embeddings_config.to_text_decoder_embedding_config(),+ transformer_config: fast_transformer_config,+ vocab_size: config.codebook_size,+ };++ let slow_decoder_config = Rc::new(slow_inner_config.to_decoder_config().map_err(|_| Error::UnableToLoadConfig)?);+ let fast_decoder_config = Rc::new(fast_inner_config.to_decoder_config().map_err(|_| Error::UnableToLoadConfig)?);+ let text_decoder_context = B::Context::new().map_err(unable_to_create_context)?;++ let slow_runner = TokenDecoderRunner::new_with_context(+ text_decoder_context.clone(),+ model_path,+ slow_decoder_config,+ "text_decoder.transformer_slow",+ "text_decoder.embeddings_slow",+ "text_decoder.readout_slow",+ runtime_config,+ )?;++ let fast_runner = TokenDecoderRunner::new_with_context(+ text_decoder_context.clone(),+ model_path,+ fast_decoder_config,+ "text_decoder.transformer_fast",+ "text_decoder.embeddings_fast",+ "text_decoder.readout_fast",+ runtime_config,+ )?;++ let activation_data_type = slow_runner.single_hidden_capture.data_type();+ let fast_data_type = fast_runner.single_override_embedding.data_type();+ if fast_data_type != activation_data_type {+ return Err(Error::UnableToLoadConfig);+ }++ let weights_path = model_path.join("model.safetensors");+ let weights_file = File::open(&weights_path).map_err(|_| Error::UnableToLoadWeights)?;+ let loader =+ ParameterLoader::new(&weights_file, text_decoder_context.as_ref()).map_err(|_| Error::UnableToLoadWeights)?;+ let root_weights = loader.tree();++ let apply_semantic_sampling_mask =+ runtime_config.force_semantic_sampling_mask.unwrap_or(!matches!(activation_data_type, DataType::F32));++ let semantic_bridge =+ FishAudioSemanticBridge::load(text_decoder_context.as_ref(), &root_weights, config, activation_data_type)?;++ Ok(Box::new(FishAudioTextDecoderRuntime::<B> {+ slow_runner,+ fast_runner,+ semantic_bridge,+ runtime_config: runtime_config.clone(),+ semantic_token_begin_id: config.semantic_token_begin_id,+ semantic_token_end_id: config.semantic_token_end_id,+ semantic_cardinality,+ im_end_token_id: config.im_end_token_id,+ codebook_size: config.codebook_size,+ num_codebooks: config.num_codebooks,+ slow_model_dim: config.slow_model_dim,+ fast_model_dim: config.fast_model_dim,+ max_seq_len: config.max_seq_len,+ scale_codebook_embeddings: config.scale_codebook_embeddings,+ fast_vocab_limit: config.short_logits_size.min(config.codebook_size),+ apply_semantic_sampling_mask,+ semantic_sampling_mask_row,+ semantic_sampling_mask_without_im_end_row: semantic_sampling_mask_without_im_end_row.into_boxed_slice(),+ current_codes_scratch: vec![0_u32; config.num_codebooks],+ instrumentation: RunnerInstrumentation::default(),+ }))+}++impl<B: Backend> FishAudioTextDecoderRuntime<B> {+ fn generate_semantic_tokens_internal(+ &mut self,+ text_tokens: &[u64],+ codec_cardinality: usize,+ seed: u64,+ max_semantic_frames: usize,+ mut on_frame: Option<&mut dyn FnMut(&[u32]) -> Result<(), Error>>,+ ) -> Result<AudioTokenGrid, Error> {+ if text_tokens.is_empty() {+ return AudioTokenGrid::new(+ Vec::new().into_boxed_slice(),+ 1,+ self.num_codebooks,+ 0,+ vec![0].into_boxed_slice(),+ )+ .map_err(Error::from);+ }+ if text_tokens.len() >= self.max_seq_len {+ return Err(Error::GenerateFailed);+ }++ if codec_cardinality == 0 || codec_cardinality > self.codebook_size {+ return Err(Error::UnableToLoadConfig);+ }+ let semantic_token_upper_bound = self.semantic_cardinality;+ let residual_token_upper_bound = codec_cardinality;+ if semantic_token_upper_bound == 0 || residual_token_upper_bound == 0 {+ return Err(Error::UnableToLoadConfig);+ }++ self.reset_generation_state();+ let mut sampling = TextSamplingState::from_config(seed, &self.runtime_config.sampling);+ let mut current_semantic_token = self.decode_initial_semantic_token(text_tokens, &mut sampling)?;+ let post_scale = if self.scale_codebook_embeddings {+ Some(1.0 / ((self.num_codebooks + 1) as f32).sqrt())+ } else {+ None+ };++ let mut max_new_tokens = self.max_seq_len.saturating_sub(text_tokens.len());+ max_new_tokens = max_new_tokens.min(max_semantic_frames.max(1));+ if let Some(limit) = self.runtime_config.max_new_tokens_override {+ max_new_tokens = max_new_tokens.min(limit);+ }+ let mut by_codebook =+ (0..self.num_codebooks).map(|_| Vec::<u32>::with_capacity(max_new_tokens)).collect::<Vec<_>>();+ if self.current_codes_scratch.len() != self.num_codebooks {+ self.current_codes_scratch = vec![0_u32; self.num_codebooks];+ }++ self.prepare_fast_runner_masks(residual_token_upper_bound)?;++ for _step in 0..max_new_tokens {+ if current_semantic_token as i64 == self.im_end_token_id {+ break;+ }+ let first_code = semantic_token_to_code(+ current_semantic_token,+ self.semantic_token_begin_id,+ self.semantic_token_end_id,+ semantic_token_upper_bound,+ );+ by_codebook[0].push(first_code);+ self.current_codes_scratch[0] = first_code;++ current_semantic_token = self.decode_frame_merged(+ current_semantic_token,+ first_code,+ residual_token_upper_bound,+ post_scale,+ &mut sampling,+ &mut by_codebook,+ )?;++ if let Some(callback) = on_frame.as_mut() {+ callback(&self.current_codes_scratch)?;+ }+ }++ self.instrumentation = self.slow_runner.take_instrumentation();+ let fast = self.fast_runner.take_instrumentation();+ self.instrumentation.command_buffers_submitted += fast.command_buffers_submitted;+ self.instrumentation.host_waits += fast.host_waits;++ self.finish_semantic_grid(&by_codebook)+ }++ fn reset_generation_state(&mut self) {+ self.instrumentation = RunnerInstrumentation::default();+ self.slow_runner.reset();+ self.slow_runner.clear_instrumentation();+ self.fast_runner.clear_instrumentation();+ }++ fn decode_initial_semantic_token(+ &mut self,+ text_tokens: &[u64],+ sampling: &mut TextSamplingState,+ ) -> Result<u64, Error> {+ let slow_runner = &mut self.slow_runner;+ let initial_sampling_row = if self.apply_semantic_sampling_mask {+ if self.runtime_config.min_frames_before_im_end > 0 {+ self.semantic_sampling_mask_without_im_end_row.as_ref()+ } else {+ self.semantic_sampling_mask_row.as_ref()+ }+ } else {+ &[]+ };+ let prefill_step_size = text_decoder_prefill_step_size(&self.runtime_config, self.max_seq_len);+ if text_tokens.len() > prefill_step_size {+ for chunk in text_tokens[..text_tokens.len() - prefill_step_size].chunks(prefill_step_size) {+ slow_runner.prefill_without_sampling(chunk)?;+ }+ }+ let prefill_tail_start = text_tokens.len().saturating_sub(prefill_step_size);+ let prefill_tail = &text_tokens[prefill_tail_start..];+ let prefill_mask = if initial_sampling_row.is_empty() {+ None+ } else {+ Some(expand_token_mask_for_sampling_row(initial_sampling_row, prefill_tail.len())?)+ };+ slow_runner.decode_next_token_with_hidden_capture(+ prefill_tail,+ EmbeddingInjection::None,+ sampling,+ prefill_mask.as_deref(),+ )+ }++ fn prepare_fast_runner_masks(+ &mut self,+ residual_token_upper_bound: usize,+ ) -> Result<(), Error> {+ let fast_vocab_limit = self.fast_vocab_limit.min(residual_token_upper_bound);+ if fast_vocab_limit == 0 {+ return Err(Error::UnableToLoadConfig);+ }+ self.fast_runner.prepare_single_token_vocab_mask(fast_vocab_limit)?;+ self.fast_runner.prepare_two_token_vocab_mask(fast_vocab_limit)?;+ Ok(())+ }++ fn finish_semantic_grid(+ &self,+ by_codebook: &[Vec<u32>],+ ) -> Result<AudioTokenGrid, Error> {+ let frames = by_codebook.first().map_or(0, Vec::len);+ let mut tokens = Vec::with_capacity(self.num_codebooks * frames);+ for codebook_tokens in by_codebook {+ if codebook_tokens.len() != frames {+ return Err(Error::GenerateFailed);+ }+ tokens.extend_from_slice(codebook_tokens);+ }+ AudioTokenGrid::new(tokens.into_boxed_slice(), 1, self.num_codebooks, frames, vec![frames].into_boxed_slice())+ .map_err(Error::from)+ }++ fn encode_project_slow_hidden_to_fast_on(+ context: &B::Context,+ semantic_bridge: &mut FishAudioSemanticBridge<B>,+ slow_hidden_capture: &Array<B>,+ output_embedding: &Array<B>,+ slow_model_dim: usize,+ fast_model_dim: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<(), Error> {+ let model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;++ if let Some(weights) = semantic_bridge.projection_weights.as_ref() {+ let hidden = slow_hidden_capture;+ let weights = weights;+ let output = output_embedding;+ if hidden.shape() != [1, slow_model_dim]+ || output.shape() != [1, fast_model_dim]+ || weights.shape() != [fast_model_dim, slow_model_dim]+ {+ return Err(Error::GenerateFailed);+ }++ let hidden_buffer = hidden.buffer();+ let weights_buffer = weights.buffer();+ let output_buffer = output.buffer();+ let hidden_buffer = hidden_buffer.borrow();+ let weights_buffer = weights_buffer.borrow();+ let mut output_buffer = output_buffer.borrow_mut();+ semantic_bridge.projection.encode(+ context,+ MatmulArguments {+ a: &hidden_buffer,+ a_offset: hidden.offset() as u64,+ b: &weights_buffer,+ d: &mut output_buffer,+ bias: None,+ batch: 1,+ input_dim: slow_model_dim as i32,+ output_dim: fast_model_dim as i32,+ leading_dimension_a: slow_model_dim as i32,+ leading_dimension_b: fast_model_dim as i32,+ leading_dimension_d: fast_model_dim as i32,+ transpose_b: false,+ },+ encoder,+ );+ return Ok(());+ }++ if slow_model_dim != fast_model_dim {+ return Err(Error::UnableToLoadConfig);+ }++ let hidden = slow_hidden_capture;+ let output = output_embedding;+ if hidden.shape() != [1, slow_model_dim] || output.shape() != [1, fast_model_dim] {+ return Err(Error::GenerateFailed);+ }+ let hidden_buffer = hidden.buffer();+ let hidden_buffer = hidden_buffer.borrow();+ let output_buffer = output.buffer();+ let mut output_buffer = output_buffer.borrow_mut();+ semantic_bridge.tensor_copy.encode(+ (&*hidden_buffer, hidden.offset()),+ (&mut *output_buffer, output.offset()),+ model_dim_u32,+ encoder,+ );+ Ok(())+ }++ /// Encode the codebook sum reading residual+ /// codes directly from a GPU buffer instead of from CPU memory.+ ///+ /// `first_code` is written into slot 0 of the staging buffer from CPU.+ /// Slots 1..num_codebooks are filled by a GPU blit from+ /// `gpu_tokens_buffer` at the given byte offset.+ fn encode_slow_codebook_sum_from_gpu_tokens_on(+ semantic_bridge: &mut FishAudioSemanticBridge<B>,+ slow_sum_embedding: &Array<B>,+ first_code: u32,+ source_runner: &TokenDecoderRunner<B>,+ source_slot: usize,+ num_codebooks: usize,+ codebook_size: usize,+ slow_model_dim: usize,+ encoder: &mut Encoder<B>,+ ) -> Result<(), Error> {+ if num_codebooks == 0 {+ return Err(Error::UnableToLoadConfig);+ }++ // Write first_code at position 0 from CPU.+ {+ let token_indices = &mut semantic_bridge.codebook_token_indices;+ if token_indices.shape() != [num_codebooks] || token_indices.data_type() != DataType::U32 {+ return Err(Error::GenerateFailed);+ }+ let indices_slice = token_indices.as_slice_mut::<u32>();+ indices_slice[0] = first_code;+ }++ // Blit residual codes from GPU: async_chain_results[0..N-2] -> codebook_token_indices[1..N-1]+ let residual_count = num_codebooks.saturating_sub(1);+ if residual_count > 0 {+ let dst_byte_offset = std::mem::size_of::<u32>(); // skip slot 0+ let token_indices_buffer = semantic_bridge.codebook_token_indices.buffer();+ let mut token_indices_buffer = token_indices_buffer.borrow_mut();+ source_runner.copy_async_chain_results_to_on(+ encoder,+ source_slot,+ (&mut *token_indices_buffer, semantic_bridge.codebook_token_indices.offset() + dst_byte_offset),+ residual_count,+ )?;+ }++ let total_vocab = num_codebooks.checked_mul(codebook_size).ok_or(Error::GenerateFailed)?;+ let total_vocab_u32 = u32::try_from(total_vocab).map_err(|_| Error::GenerateFailed)?;+ let num_codebooks_u32 = u32::try_from(num_codebooks).map_err(|_| Error::GenerateFailed)?;+ let slow_model_dim_u32 = u32::try_from(slow_model_dim).map_err(|_| Error::GenerateFailed)?;+ let codebook_size_u32 = u32::try_from(codebook_size).map_err(|_| Error::GenerateFailed)?;++ let codebook_token_indices = &semantic_bridge.codebook_token_indices;+ let codebook_embeddings = &semantic_bridge.codebook_embeddings;+ let slow_sum = slow_sum_embedding;+ if codebook_token_indices.shape() != [num_codebooks]+ || codebook_embeddings.shape() != [total_vocab, slow_model_dim]+ || slow_sum.shape() != [1, slow_model_dim]+ || codebook_embeddings.data_type() != slow_sum.data_type()+ {+ return Err(Error::GenerateFailed);+ }++ let codebook_token_indices_buffer = codebook_token_indices.buffer();+ let codebook_token_indices_buffer = codebook_token_indices_buffer.borrow();+ let codebook_embeddings_buffer = codebook_embeddings.buffer();+ let codebook_embeddings_buffer = codebook_embeddings_buffer.borrow();+ let slow_sum_buffer = slow_sum.buffer();+ let mut slow_sum_buffer = slow_sum_buffer.borrow_mut();+ semantic_bridge.embedding_rows_sum.encode(+ (&*codebook_token_indices_buffer, codebook_token_indices.offset()),+ (&*codebook_embeddings_buffer, codebook_embeddings.offset()),+ (&mut *slow_sum_buffer, slow_sum.offset()),+ num_codebooks_u32,+ total_vocab_u32,+ slow_model_dim_u32,+ codebook_size_u32,+ encoder,+ );+ Ok(())+ }++ /// Merged fast+slow frame: encodes all fast runner forward passes, the+ /// codebook sum (reading GPU tokens directly), and the slow runner+ /// forward pass into a **single** command buffer submission.+ ///+ /// Returns the next semantic token.+ fn decode_frame_merged(+ &mut self,+ current_semantic_token: u64,+ first_code: u32,+ residual_token_upper_bound: usize,+ post_scale: Option<f32>,+ sampling: &mut TextSamplingState,+ by_codebook: &mut [Vec<u32>],+ ) -> Result<u64, Error> {+ let fast_vocab_limit = self.fast_vocab_limit.min(residual_token_upper_bound);+ let followup_count = self.num_codebooks.saturating_sub(2);++ self.fast_runner.reset();++ let context = Rc::clone(self.fast_runner.context());+ let mut shared_encoder = Encoder::new(context.as_ref()).map_err(unable_to_create_context)?;++ let total_fast_count =+ self.encode_fast_passes_on(&mut shared_encoder, first_code, fast_vocab_limit, followup_count, sampling)?;+ self.encode_slow_pass_on(+ &mut shared_encoder,+ current_semantic_token,+ first_code,+ post_scale,+ sampling,+ by_codebook,+ )?;+ self.slow_runner.submit_and_wait_command_buffer(shared_encoder)?;++ for pass in 0..total_fast_count {+ let sampled = self.fast_runner.read_async_chain_result(pass)?;+ let codebook_index = pass + 1;+ let clamped = (sampled as usize).min(residual_token_upper_bound.saturating_sub(1));+ let clamped = u32::try_from(clamped).map_err(|_| Error::GenerateFailed)?;+ by_codebook[codebook_index].push(clamped);+ self.current_codes_scratch[codebook_index] = clamped;+ }++ let slow_token = u64::from(self.slow_runner.read_async_chain_result(0)?);+ Ok(slow_token)+ }++ /// Encode the fast runner's initial + followup passes onto the shared+ /// command buffer.+ fn encode_fast_passes_on(+ &mut self,+ encoder: &mut Encoder<B>,+ first_code: u32,+ fast_vocab_limit: usize,+ followup_count: usize,+ sampling: &mut TextSamplingState,+ ) -> Result<usize, Error> {+ let (slow_runner, fast_runner, semantic_bridge) =+ (&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);+ let slow_hidden_capture = &slow_runner.single_hidden_capture;+ let slow_model_dim = self.slow_model_dim;+ let fast_model_dim = self.fast_model_dim;++ let mut pre_projection =+ |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {+ Self::encode_project_slow_hidden_to_fast_on(+ runner.context().as_ref(),+ semantic_bridge,+ slow_hidden_capture,+ &runner.single_override_embedding,+ slow_model_dim,+ fast_model_dim,+ encoder,+ )+ };++ fast_runner.encode_first_and_followup_tokens_on(+ encoder,+ &[0, u64::from(first_code)],+ EmbeddingInjection::OverrideFirstRowInternal,+ Some(&mut pre_projection),+ followup_count,+ Some(fast_vocab_limit),+ sampling,+ )+ }++ /// Encode the slow runner's forward pass onto the shared command buffer.+ fn encode_slow_pass_on(+ &mut self,+ encoder: &mut Encoder<B>,+ current_semantic_token: u64,+ first_code: u32,+ post_scale: Option<f32>,+ sampling: &mut TextSamplingState,+ by_codebook: &[Vec<u32>],+ ) -> Result<(), Error> {+ let (slow_runner, fast_runner, semantic_bridge) =+ (&mut self.slow_runner, &mut self.fast_runner, &mut self.semantic_bridge);+ let num_codebooks = self.num_codebooks;+ let codebook_size = self.codebook_size;+ let slow_model_dim = self.slow_model_dim;++ let sampled_frames = by_codebook[0].len();+ let slow_sampling_mask = if self.apply_semantic_sampling_mask {+ if sampled_frames < self.runtime_config.min_frames_before_im_end {+ Some(self.semantic_sampling_mask_without_im_end_row.as_ref())+ } else {+ Some(self.semantic_sampling_mask_row.as_ref())+ }+ } else {+ None+ };++ let mut pre_codebook_sum =+ |runner: &TokenDecoderRunner<B>, _state: &ForwardPassState<B>, encoder: &mut Encoder<B>| {+ Self::encode_slow_codebook_sum_from_gpu_tokens_on(+ semantic_bridge,+ &runner.single_override_embedding,+ first_code,+ fast_runner,+ 0,+ num_codebooks,+ codebook_size,+ slow_model_dim,+ encoder,+ )+ };++ slow_runner.encode_next_step_on(+ encoder,+ &[current_semantic_token],+ EmbeddingInjection::AddPreloaded {+ post_scale,+ },+ sampling,+ slow_sampling_mask,+ true, // capture_hidden+ Some(&mut pre_codebook_sum),+ )+ }++ fn take_instrumentation(&mut self) -> RunnerInstrumentation {+ std::mem::take(&mut self.instrumentation)+ }+}++impl<B: Backend> SemanticDecoderBackend for FishAudioTextDecoderRuntime<B> {+ fn generate_semantic_tokens(+ &mut self,+ text_tokens: &[u64],+ codec_cardinality: usize,+ seed: u64,+ max_semantic_frames: usize,+ ) -> Result<AudioTokenGrid, Error> {+ self.generate_semantic_tokens_internal(text_tokens, codec_cardinality, seed, max_semantic_frames, None)+ }++ fn generate_semantic_tokens_with_callback(+ &mut self,+ text_tokens: &[u64],+ codec_cardinality: usize,+ seed: u64,+ max_semantic_frames: usize,+ on_frame: &mut dyn FnMut(&[u32]) -> Result<(), Error>,+ ) -> Result<AudioTokenGrid, Error> {+ self.generate_semantic_tokens_internal(+ text_tokens,+ codec_cardinality,+ seed,+ max_semantic_frames,+ Some(on_frame),+ )+ }++ fn take_instrumentation(&mut self) -> RunnerInstrumentation {+ FishAudioTextDecoderRuntime::take_instrumentation(self)+ }+}diff --git a/crates/uzu/src/session/tts_session/generator.rs b/crates/uzu/src/session/tts_session/generator.rsnew file mode 100644index 000000000..690b21fe2--- /dev/null+++ b/crates/uzu/src/session/tts_session/generator.rs@@ -0,0 +1,426 @@+use super::*;++impl<B: Backend + Send + Sync> TtsSession<B> {+ pub fn synthesize(+ &mut self,+ input: Input,+ ) -> Result<AudioPcmBatch, Error> {+ self.synthesize_with_seed_and_config(input, TtsRunConfig::default().default_seed, &TtsRunConfig::default())+ }++ pub fn synthesize_with_seed(+ &mut self,+ input: Input,+ seed: u64,+ ) -> Result<AudioPcmBatch, Error> {+ self.synthesize_with_seed_and_config(input, seed, &TtsRunConfig::default())+ }++ pub fn synthesize_with_config(+ &mut self,+ input: Input,+ config: &TtsRunConfig,+ ) -> Result<AudioPcmBatch, Error> {+ self.synthesize_with_seed_and_config(input, config.default_seed, config)+ }++ pub fn synthesize_with_seed_and_config(+ &mut self,+ input: Input,+ seed: u64,+ config: &TtsRunConfig,+ ) -> Result<AudioPcmBatch, Error> {+ config.validate()?;++ let prompt = self.render_prompt(&input)?;+ let text_tokens: Vec<u64> = self+ .tokenizer+ .encode(prompt.as_str(), false)+ .map_err(|_| Error::UnableToEncodeText)?+ .get_ids()+ .iter()+ .map(|&token| token as u64)+ .collect();++ let semantic_start = Instant::now();+ let semantic_tokens = self.generate_semantic_tokens(&text_tokens, seed, config.max_semantic_frames)?;+ let semantic_decode_seconds = semantic_start.elapsed().as_secs_f64();+ let instrumentation = self.take_text_decoder_instrumentation();++ let audio_start = Instant::now();+ let mut audio_decode_calls = 0usize;+ let mut audio_input_frames = 0usize;+ let mut audio_decoded_window_frames = 0usize;+ let mut audio_max_decoded_window_frames = 0usize;+ let pcm = match config.non_streaming_mode {+ crate::session::config::TtsNonStreamingMode::FullDecode => {+ audio_decode_calls = usize::from(semantic_tokens.frames() > 0);+ audio_input_frames = semantic_tokens.frames();+ audio_decoded_window_frames = semantic_tokens.frames();+ audio_max_decoded_window_frames = semantic_tokens.frames();+ self.audio_decoder.decode(&semantic_tokens)?+ },+ crate::session::config::TtsNonStreamingMode::ChunkedIfNeeded => {+ if semantic_tokens.batch_size() != 1 {+ return Err(Error::GenerateFailed);+ }+ config.validate_stream_decode()?;+ let total_frames = semantic_tokens.frames();+ let chunked_threshold = config.max_stream_workspace_frames.max(config.max_chunk_frames.max(1));+ if total_frames <= chunked_threshold {+ audio_decode_calls = usize::from(total_frames > 0);+ audio_input_frames = total_frames;+ audio_decoded_window_frames = total_frames;+ audio_max_decoded_window_frames = total_frames;+ self.audio_decoder.decode(&semantic_tokens)?+ } else {+ let chunk_frames = config.max_chunk_frames.max(config.min_chunk_frames.max(1));+ let mut stream = self.audio_decoder.begin_stream(+ semantic_tokens.batch_size(),+ semantic_tokens.codebooks(),+ config.max_stream_workspace_frames,+ )?;++ let mut all_samples = Vec::<f32>::with_capacity(total_frames.saturating_mul(512));+ let mut accumulated_lengths = vec![0usize; semantic_tokens.batch_size()];+ let mut sample_rate = self.audio_decoder.sample_rate();+ let mut channels = 1usize;++ let mut frame_start = 0usize;+ while frame_start < total_frames {+ let frame_end = (frame_start + chunk_frames).min(total_frames);+ let delta_grid = slice_grid_range(&semantic_tokens, frame_start, frame_end)?;+ let partial_pcm = stream.decode_step(&delta_grid, frame_end == total_frames)?;+ accumulate_audio_decode_step_stats(+ &mut audio_decode_calls,+ &mut audio_input_frames,+ &mut audio_decoded_window_frames,+ &mut audio_max_decoded_window_frames,+ stream.last_step_stats(),+ );+ if partial_pcm.lengths().len() != accumulated_lengths.len() {+ return Err(Error::GenerateFailed);+ }+ for (acc, &len) in accumulated_lengths.iter_mut().zip(partial_pcm.lengths().iter()) {+ *acc = acc.saturating_add(len);+ }+ if !partial_pcm.samples().is_empty() {+ all_samples.extend_from_slice(partial_pcm.samples());+ }+ sample_rate = partial_pcm.sample_rate();+ channels = partial_pcm.channels();+ frame_start = frame_end;+ }+ stream.finish()?;+ AudioPcmBatch::new(+ all_samples.into_boxed_slice(),+ sample_rate,+ channels,+ accumulated_lengths.into_boxed_slice(),+ )+ .map_err(Error::from)?+ }+ },+ };+ let audio_decode_seconds = audio_start.elapsed().as_secs_f64();++ self.record_last_execution_stats(TtsExecutionStats {+ semantic_decode_seconds,+ audio_decode_seconds,+ callback_seconds: 0.0,+ first_chunk_seconds: 0.0,+ command_buffers_submitted: instrumentation.command_buffers_submitted,+ host_waits: instrumentation.host_waits,+ semantic_frames: semantic_tokens.frames(),+ first_chunk_frames: 0,+ emitted_chunks: usize::from(config.streaming_enabled),+ audio_decode_calls,+ audio_input_frames,+ audio_decoded_window_frames,+ audio_max_decoded_window_frames,+ });++ Ok(pcm)+ }++ pub fn generate_semantic_tokens_with_seed(+ &mut self,+ input: Input,+ seed: u64,+ ) -> Result<AudioTokenGrid, Error> {+ self.generate_semantic_tokens_with_seed_and_config(input, seed, &TtsRunConfig::default())+ }++ pub fn generate_semantic_tokens_with_seed_and_config(+ &mut self,+ input: Input,+ seed: u64,+ config: &TtsRunConfig,+ ) -> Result<AudioTokenGrid, Error> {+ config.validate()?;+ let prompt = self.render_prompt(&input)?;+ let text_tokens: Vec<u64> = self+ .tokenizer+ .encode(prompt.as_str(), false)+ .map_err(|_| Error::UnableToEncodeText)?+ .get_ids()+ .iter()+ .map(|&token| token as u64)+ .collect();++ self.generate_semantic_tokens(&text_tokens, seed, config.max_semantic_frames)+ }++ pub fn synthesize_streaming<F>(+ &mut self,+ input: Input,+ chunk_frames: usize,+ on_chunk: F,+ ) -> Result<AudioPcmBatch, Error>+ where+ F: FnMut(&AudioPcmBatch),+ {+ let config = TtsRunConfig::default();+ self.synthesize_streaming_with_seed(input, config.default_seed, chunk_frames, on_chunk)+ }++ pub fn synthesize_streaming_with_seed<F>(+ &mut self,+ input: Input,+ seed: u64,+ chunk_frames: usize,+ on_chunk: F,+ ) -> Result<AudioPcmBatch, Error>+ where+ F: FnMut(&AudioPcmBatch),+ {+ let config = TtsRunConfig::fixed_chunk_frames(chunk_frames);+ self.synthesize_streaming_with_seed_and_config(input, seed, &config, on_chunk)+ }++ pub fn synthesize_streaming_with_config<F>(+ &mut self,+ input: Input,+ config: &TtsRunConfig,+ on_chunk: F,+ ) -> Result<AudioPcmBatch, Error>+ where+ F: FnMut(&AudioPcmBatch),+ {+ self.synthesize_streaming_with_seed_and_config(input, config.default_seed, config, on_chunk)+ }++ pub fn synthesize_streaming_with_seed_and_config<F>(+ &mut self,+ input: Input,+ seed: u64,+ config: &TtsRunConfig,+ mut on_chunk: F,+ ) -> Result<AudioPcmBatch, Error>+ where+ F: FnMut(&AudioPcmBatch),+ {+ if !config.streaming_enabled {+ let pcm = self.synthesize_with_seed_and_config(input, seed, config)?;+ on_chunk(&pcm);+ return Ok(pcm);+ }+ config.validate_stream_decode()?;++ let prompt = self.render_prompt(&input)?;+ let text_tokens: Vec<u64> = self+ .tokenizer+ .encode(prompt.as_str(), false)+ .map_err(|_| Error::UnableToEncodeText)?+ .get_ids()+ .iter()+ .map(|&token| token as u64)+ .collect();++ let mut streamed_tokens = StreamingTokenAccumulator::new(self.audio_decoder.num_codebooks())?;+ let mut audio_stream = self.audio_decoder.begin_stream(+ 1,+ self.audio_decoder.num_codebooks(),+ config.max_stream_workspace_frames,+ )?;+ let mut last_decoded_frames = 0usize;+ let startup_cap_frames = config.max_chunk_frames.max(config.min_chunk_frames.max(1));+ let initial_chunk_frames = config.initial_chunk_frames.max(1).min(startup_cap_frames);++ let mut ss = StreamingSynthesisState {+ on_chunk: &mut on_chunk,+ pending_chunk: None,+ callback_seconds: 0.0,+ audio_decode_seconds_in_loop: 0.0,+ audio_decode_seconds: 0.0,+ output_samples: Vec::with_capacity(config.max_semantic_frames.saturating_mul(512)),+ output_frames: 0,+ output_sample_rate: self.audio_decoder.sample_rate(),+ output_channels: 1,+ emitted_chunks: 0,+ first_emit_pending: true,+ first_chunk_seconds: None,+ first_chunk_frames: 0,+ stream_start: Instant::now(),+ startup_target_frames: initial_chunk_frames,+ startup_cap_frames,+ chunk_controller: AdaptiveChunkController::new(config),+ audio_decode_calls: 0,+ audio_input_frames: 0,+ audio_decoded_window_frames: 0,+ audio_max_decoded_window_frames: 0,+ };++ let semantic_start = Instant::now();+ let semantic_tokens = self.generate_semantic_tokens_with_callback(+ &text_tokens,+ seed,+ config.max_semantic_frames,+ &mut |codes| {+ ss.maybe_flush_pending(false, true, config)?;+ streamed_tokens.push_frame(codes)?;+ let ready_frames = streamed_tokens.frames().saturating_sub(last_decoded_frames);+ let adaptive_target_frames = ss.chunk_controller.target_frames(config);+ let target_frames = if ss.first_emit_pending {+ ss.startup_target_frames+ } else {+ adaptive_target_frames+ };+ if ready_frames >= target_frames && ss.pending_chunk.is_none() {+ let partial_grid = streamed_tokens.to_grid_range(last_decoded_frames, streamed_tokens.frames())?;+ last_decoded_frames = streamed_tokens.frames();+ let pending_next_chunk_frames = if ss.first_emit_pending {+ adaptive_target_frames+ } else {+ target_frames+ };+ let decode_start = Instant::now();+ let pending_audio_chunk = audio_stream.decode_step_pending(&partial_grid, false)?;+ ss.pending_chunk = Some(PendingStreamingChunk {+ submission_decode_duration: decode_start.elapsed(),+ ready_frames,+ next_chunk_frames: pending_next_chunk_frames,+ chunk: pending_audio_chunk,+ });+ ss.maybe_flush_pending(false, true, config)?;+ }+ Ok(())+ },+ )?;+ let semantic_loop_seconds = semantic_start.elapsed().as_secs_f64();++ ss.maybe_flush_pending(true, false, config)?;++ if last_decoded_frames < semantic_tokens.frames() {+ let final_decode_start = Instant::now();+ let final_delta_grid = streamed_tokens.to_grid_range(last_decoded_frames, semantic_tokens.frames())?;+ let final_pcm = audio_stream.decode_step(&final_delta_grid, true)?;+ accumulate_audio_decode_step_stats(+ &mut ss.audio_decode_calls,+ &mut ss.audio_input_frames,+ &mut ss.audio_decoded_window_frames,+ &mut ss.audio_max_decoded_window_frames,+ audio_stream.last_step_stats(),+ );+ ss.audio_decode_seconds += final_decode_start.elapsed().as_secs_f64();+ let callback_start = Instant::now();+ let emitted_frames = if final_pcm.lengths().len() == 1 {+ final_pcm.lengths()[0]+ } else {+ return Err(Error::GenerateFailed);+ };+ if emitted_frames > 0 {+ (ss.on_chunk)(&final_pcm);+ }+ ss.callback_seconds += callback_start.elapsed().as_secs_f64();+ if emitted_frames > 0 {+ ss.output_samples.extend_from_slice(final_pcm.samples());+ ss.output_frames = ss.output_frames.saturating_add(emitted_frames);+ ss.output_sample_rate = final_pcm.sample_rate();+ ss.output_channels = final_pcm.channels();+ ss.emitted_chunks += 1;+ if ss.first_emit_pending {+ ss.first_chunk_seconds = Some(ss.stream_start.elapsed().as_secs_f64());+ ss.first_chunk_frames = emitted_frames;+ }+ }+ }+ let semantic_decode_seconds =+ (semantic_loop_seconds - ss.audio_decode_seconds_in_loop - ss.callback_seconds).max(0.0);+ audio_stream.finish()?;++ let stats = TtsExecutionStats {+ semantic_decode_seconds,+ audio_decode_seconds: ss.audio_decode_seconds,+ callback_seconds: ss.callback_seconds,+ first_chunk_seconds: ss.first_chunk_seconds.unwrap_or(0.0),+ command_buffers_submitted: 0,+ host_waits: 0,+ semantic_frames: semantic_tokens.frames(),+ first_chunk_frames: ss.first_chunk_frames,+ emitted_chunks: ss.emitted_chunks,+ audio_decode_calls: ss.audio_decode_calls,+ audio_input_frames: ss.audio_input_frames,+ audio_decoded_window_frames: ss.audio_decoded_window_frames,+ audio_max_decoded_window_frames: ss.audio_max_decoded_window_frames,+ };++ let full_pcm = AudioPcmBatch::new(+ ss.output_samples.into_boxed_slice(),+ ss.output_sample_rate,+ ss.output_channels,+ vec![ss.output_frames].into_boxed_slice(),+ )+ .map_err(Error::from)?;++ let instrumentation = self.take_text_decoder_instrumentation();+ self.record_last_execution_stats(TtsExecutionStats {+ command_buffers_submitted: instrumentation.command_buffers_submitted,+ host_waits: instrumentation.host_waits,+ ..stats+ });++ Ok(full_pcm)+ }++ fn generate_semantic_tokens(+ &mut self,+ text_tokens: &[u64],+ seed: u64,+ max_semantic_frames: usize,+ ) -> Result<AudioTokenGrid, Error> {+ let codec_cardinality = self.audio_decoder.codec_cardinality();+ self.text_decoder.generate_semantic_tokens(text_tokens, codec_cardinality, seed, max_semantic_frames)+ }++ fn generate_semantic_tokens_with_callback<F>(+ &mut self,+ text_tokens: &[u64],+ seed: u64,+ max_semantic_frames: usize,+ on_frame: &mut F,+ ) -> Result<AudioTokenGrid, Error>+ where+ F: FnMut(&[u32]) -> Result<(), Error>,+ {+ let codec_cardinality = self.audio_decoder.codec_cardinality();+ self.text_decoder.generate_semantic_tokens_with_callback(+ text_tokens,+ codec_cardinality,+ seed,+ max_semantic_frames,+ on_frame,+ )+ }++ fn take_text_decoder_instrumentation(&mut self) -> RunnerInstrumentation {+ self.text_decoder.take_instrumentation()+ }++ fn record_last_execution_stats(+ &mut self,+ stats: TtsExecutionStats,+ ) {+ self.last_execution_stats.replace(stats);+ }+}diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsindex 3e634ed4c..23191ee09 100644--- a/crates/uzu/src/session/types/error.rs+++ b/crates/uzu/src/session/types/error.rs@@ -1,4 +1,172 @@-use crate::classifier::ClassifierError;+use crate::{DataType, classifier::ClassifierError, config::ModelType, session::config::TtsRunConfigError};++#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]+pub enum TtsModelConfigError {+ #[error("expected model_type={expected:?}, got {actual:?}")]+ ModelTypeMismatch {+ expected: ModelType,+ actual: ModelType,+ },+ #[error("missing TTS model config")]+ MissingTtsModelConfig,+ #[error("FishAudio tensor {key} dtype mismatch: expected {expected:?}, got {actual:?}")]+ FishAudioTensorDataTypeMismatch {+ key: Box<str>,+ expected: DataType,+ actual: DataType,+ },+ #[error(+ "FishAudio decoder requires num_codebooks > 1 plus positive codebook_size and max_seq_len, got num_codebooks={num_codebooks}, codebook_size={codebook_size}, max_seq_len={max_seq_len}"+ )]+ FishAudioDecoderCoreParametersInvalid {+ num_codebooks: usize,+ codebook_size: usize,+ max_seq_len: usize,+ },+ #[error(+ "FishAudio decoder num_codebooks={decoder_num_codebooks} does not match audio num_codebooks={audio_num_codebooks}"+ )]+ FishAudioDecoderNumCodebooksMismatch {+ decoder_num_codebooks: usize,+ audio_num_codebooks: usize,+ },+ #[error(+ "FishAudio decoder codebook_size={codebook_size} must be at least audio residual codec cardinality={audio_codec_cardinality}"+ )]+ FishAudioDecoderCodebookSizeTooSmall {+ codebook_size: usize,+ audio_codec_cardinality: usize,+ },+ #[error("FishAudio semantic token range is invalid: begin={begin}, end={end}")]+ FishAudioSemanticTokenRangeInvalid {+ begin: i64,+ end: i64,+ },+ #[error("FishAudio semantic token range overflow: begin={begin}, end={end}")]+ FishAudioSemanticTokenRangeOverflow {+ begin: i64,+ end: i64,+ },+ #[error(+ "FishAudio semantic codec cardinality={semantic_cardinality} does not match audio semantic codec cardinality={audio_semantic_cardinality}"+ )]+ FishAudioSemanticCodecCardinalityMismatch {+ semantic_cardinality: usize,+ audio_semantic_cardinality: usize,+ },+ #[error("followup_count overflow in async chain")]+ AsyncChainFollowupCountOverflow,+ #[error("async chain token count {total_count} exceeds capacity {capacity}")]+ AsyncChainTokenCountExceedsCapacity {+ total_count: usize,+ capacity: usize,+ },+ #[error("vocab mask not supported for initial_token_count={initial_token_count} (expected 1 or 2)")]+ UnsupportedInitialTokenCountForVocabMask {+ initial_token_count: usize,+ },+ #[error("two-token vocab mask size overflow (row_words={row_words})")]+ TwoTokenVocabMaskSizeOverflow {+ row_words: usize,+ },+ #[error("token bitmask size overflow")]+ TokenBitmaskSizeOverflow,+ #[error(+ "token bitmask length {actual_words}, expected {expected_words} (token_count={token_count}, row_words={row_words})"+ )]+ TokenBitmaskLengthMismatch {+ actual_words: usize,+ expected_words: usize,+ token_count: usize,+ row_words: usize,+ },+ #[error("precomputed bitmask size overflow")]+ PrecomputedBitmaskSizeOverflow,+ #[error("precomputed bitmask length {actual_words}, expected {expected_words}")]+ PrecomputedBitmaskLengthMismatch {+ actual_words: usize,+ expected_words: usize,+ },+ #[error("vocab_limit resolved to 0")]+ VocabLimitResolvedToZero,+ #[error("inline bitmask size overflow (token_count={token_count}, row_words={row_words})")]+ InlineBitmaskSizeOverflow {+ token_count: usize,+ row_words: usize,+ },+ #[error("vocab_limit with token_count={token_count} requires a precomputed bitmask")]+ VocabLimitRequiresPrecomputedBitmask {+ token_count: usize,+ },+ #[error("model_dim {model_dim} exceeds u32 range")]+ ModelDimExceedsU32 {+ model_dim: usize,+ },+ #[error("hidden capture row offset overflow (token_count={token_count}, model_dim={model_dim})")]+ HiddenCaptureRowOffsetOverflow {+ token_count: usize,+ model_dim: usize,+ },+ #[error("hidden capture src offset overflow")]+ HiddenCaptureSourceOffsetOverflow,+ #[error(+ "hidden capture shape/dtype mismatch: expected {expected_shape:?} @ {expected_data_type:?}, got {actual_shape:?} @ {actual_data_type:?}"+ )]+ HiddenCaptureTensorMismatch {+ expected_shape: Box<[usize]>,+ expected_data_type: DataType,+ actual_shape: Box<[usize]>,+ actual_data_type: DataType,+ },+ #[error(+ "override embedding shape/dtype mismatch: expected {expected_shape:?} @ {expected_data_type:?}, got {actual_shape:?} @ {actual_data_type:?}"+ )]+ OverrideEmbeddingTensorMismatch {+ expected_shape: Box<[usize]>,+ expected_data_type: DataType,+ actual_shape: Box<[usize]>,+ actual_data_type: DataType,+ },+ #[error("add_scale total_len overflow (token_count={token_count}, model_dim={model_dim})")]+ AddScaleTotalLengthOverflow {+ token_count: usize,+ model_dim: usize,+ },+ #[error("add_scale total_len {total_len} exceeds u32 range")]+ AddScaleTotalLengthExceedsU32 {+ total_len: usize,+ },+ #[error(+ "add_scale bias shape/dtype mismatch: expected {expected_shape:?} @ {expected_data_type:?}, got {actual_shape:?} @ {actual_data_type:?}"+ )]+ AddScaleBiasTensorMismatch {+ expected_shape: Box<[usize]>,+ expected_data_type: DataType,+ actual_shape: Box<[usize]>,+ actual_data_type: DataType,+ },+ #[error("async chain copy src_offset overflow (src_slot={src_slot})")]+ AsyncChainCopySourceOffsetOverflow {+ src_slot: usize,+ },+ #[error("async chain copy size overflow (count={count})")]+ AsyncChainCopySizeOverflow {+ count: usize,+ },+ #[error("async chain result slot {slot} out of bounds (capacity={capacity})")]+ AsyncChainResultSlotOutOfBounds {+ slot: usize,+ capacity: usize,+ },+}++#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]+pub enum TtsPromptConfigError {+ #[error("FishAudio prompt template references message.{field}, but default_message_fields does not provide it")]+ MissingDefaultMessageField {+ field: Box<str>,+ },+}#[derive(Debug, thiserror::Error)]pub enum Error {@@ -12,6 +180,12 @@ pub enum Error {UnableToCreateCommandBuffer(Box<dyn std::error::Error>),#[error("Unable to load model configuration")]UnableToLoadConfig,+ #[error("Invalid TTS model config: {0}")]+ InvalidTtsModelConfig(#[from] TtsModelConfigError),+ #[error("Invalid TTS run config: {0}")]+ InvalidTtsRunConfig(#[from] TtsRunConfigError),+ #[error("Invalid TTS prompt config: {0}")]+ InvalidTtsPromptConfig(#[from] TtsPromptConfigError),#[error("Unable to load model weights")]UnableToLoadWeights,#[error("Unable to load tokenizer")]@@ -58,4 +232,6 @@ pub enum Error {CaptureFailed,#[error("Classifier error: {0}")]Classifier(#[from] ClassifierError),+ #[error("Audio codec error: {0}")]+ AudioCodec(#[from] crate::audio::AudioError),}diff --git a/crates/uzu/src/session/types/message.rs b/crates/uzu/src/session/types/message.rsindex f36961d3e..6f4afd26c 100644--- a/crates/uzu/src/session/types/message.rs+++ b/crates/uzu/src/session/types/message.rs@@ -3,10 +3,13 @@ use std::collections::HashMap;use serde::{Deserialize, Serialize};use crate::{- config::MessageProcessorConfig,+ config::{MessageProcessorConfig, TtsMessageProcessorConfig},session::{parameter::ConfigResolvableValue, types::Role},};+pub(crate) const DEFAULT_TTS_SPEAKER_ID: &str = "speaker:0";+pub(crate) const DEFAULT_TTS_STYLE: &str = "interleave";+#[derive(Clone, Debug, Deserialize, Serialize)]pub struct Message {pub role: Role,@@ -61,3 +64,25 @@ impl ConfigResolvableValue<MessageProcessorConfig, HashMap<String, String>> forresult}}++impl ConfigResolvableValue<TtsMessageProcessorConfig, HashMap<String, String>> for Message {+ fn resolve(+ &self,+ config: &TtsMessageProcessorConfig,+ ) -> HashMap<String, String> {+ let role = match self.role {+ Role::System => config.system_role_name.clone(),+ Role::User => config.user_role_name.clone(),+ Role::Assistant => config.assistant_role_name.clone(),+ };+ let mut result: HashMap<String, String> = config.default_message_fields.clone().into_iter().collect();+ result.entry(String::from("speaker_id")).or_insert_with(|| String::from(DEFAULT_TTS_SPEAKER_ID));+ result.entry(String::from("style")).or_insert_with(|| String::from(DEFAULT_TTS_STYLE));+ result.insert(String::from("role"), role);+ result.insert(String::from("content"), self.content.clone());+ if let Some(reasoning_content) = self.reasoning_content.clone() {+ result.insert(String::from("reasoning_content"), reasoning_content);+ }+ result+ }+}diff --git a/crates/uzu/src/session/types/mod.rs b/crates/uzu/src/session/types/mod.rsindex c1dab1298..85e4668b4 100644--- a/crates/uzu/src/session/types/mod.rs+++ b/crates/uzu/src/session/types/mod.rs@@ -5,7 +5,7 @@ mod output;mod role;mod stats;mod text;-pub use error::Error;+pub use error::{Error, TtsModelConfigError, TtsPromptConfigError};pub use input::Input;pub use message::Message;pub use output::{FinishReason, Output};diff --git a/crates/uzu/tests/audio_kernel_perf_test.rs b/crates/uzu/tests/audio_kernel_perf_test.rsnew file mode 100644index 000000000..034bf919c--- /dev/null+++ b/crates/uzu/tests/audio_kernel_perf_test.rs@@ -0,0 +1,1582 @@+#![cfg(all(feature = "audio-runtime", metal_backend))]++use std::time::Instant;++use uzu::{+ Array, ArrayContextExt, DataType,+ backends::{+ common::{+ Backend, Context, Encoder, Kernels,+ gpu_types::ActivationType,+ kernel::{+ ActivationKernel, AudioAddKernel, AudioCausalConv1dGroupedKernel,+ AudioCausalConv1dGroupedResidualKernel, AudioCausalConv1dKernel,+ AudioCausalConvTranspose1dCausalPadKernel, AudioConv1dKernel, AudioHalfSnakeKernel, AudioNormNcsKernel,+ AudioQuantizerDecodeKernel,+ },+ },+ metal::Metal,+ },+};++type Ctx = <Metal as Backend>::Context;++const WARMUP: usize = 5;+const ITERATIONS: usize = 20;++fn measure<F: FnMut()>(+ label: &str,+ mut f: F,+) -> f64 {+ for _ in 0..WARMUP {+ f();+ }+ let start = Instant::now();+ for _ in 0..ITERATIONS {+ f();+ }+ let elapsed = start.elapsed();+ let mean_ms = elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64;+ println!(" {label:<65} {mean_ms:>8.3} ms");+ mean_ms+}++fn make_lengths(+ context: &Ctx,+ batch_size: usize,+ value: i32,+) -> Array<Metal> {+ context.create_array_from(&[batch_size], &vec![value; batch_size], "lengths")+}++// ==========================================================================+// FishAudio s1-mini vocoder configuration (from model config JSON)+//+// input_dim = 1024+// decoder_dim = 1536+// downsample_factor = [2, 2] (reversed for upsampler -> strides [2, 2])+// decoder_rates = [8, 8, 4, 2]+// n_codebooks = 9 (+ 1 semantic = 10 total)+// codebook_dim = 8+// codebook_size = 1024+// semantic_codebook_size = 4096+// samplerate = 44100+// upsample_factor = 2*2 * 8*8*4*2 = 2048+//+// Channel progression (standard DAC with decoder_dim=1536):+// Upsampler: 1024 -> 512 (s=2) -> 256 (s=2)+// first_conv: 256 -> 1536 (k=7)+// Decoder: 1536 -> 768 (s=8) -> 384 (s=8) -> 192 (s=4) -> 96 (s=2)+// final_conv: 96 -> 1 (k=7)+//+// Residual units per decoder block: 3, with dilations [1, 3, 9], k=7+// ConvNeXt depthwise: k=7, per-channel+// Transpose conv kernel_size = stride * 2 (fast_two_tap path)+//+// For 10 seconds at 44100 Hz:+// 441000 samples / 2048 upsample_factor = ~215 semantic frames+// We use 215 frames as the starting point.+// ==========================================================================++const INPUT_DIM: usize = 1024;+const DECODER_DIM: usize = 1536;+const CODEBOOK_DIM: usize = 8;+const TOTAL_CODEBOOKS: usize = 10;+const RESIDUAL_QUANTIZERS: usize = 9;+const SEMANTIC_CARDINALITY: usize = 4096;+const RESIDUAL_CARDINALITY: usize = 1024;++// Upsample block channel progression: input_dim -> input_dim/2 -> input_dim/4+const UP_CHANNELS: [usize; 3] = [1024, 512, 256];+const UP_STRIDES: [usize; 2] = [2, 2]; // downsample_factor reversed++// Decoder block channel progression: decoder_dim -> decoder_dim/2 -> /4 -> /8 -> /16+const DEC_CHANNELS: [usize; 5] = [1536, 768, 384, 192, 96];+const DEC_STRIDES: [usize; 4] = [8, 8, 4, 2]; // decoder_rates++const SEMANTIC_FRAMES_10S: usize = 215; // ~10s at 44100 Hz with upsample_factor=2048++#[test]+#[ignore] // Heavy benchmark (~240s); run explicitly with --ignored+fn audio_kernel_perf() {+ let context = <Metal as Backend>::Context::new().unwrap();+ let dt = DataType::F16;+ let batch_size: usize = 1;++ let mut results: Vec<(String, f64)> = Vec::new();++ println!();+ println!(" FishAudio s1-mini vocoder kernel benchmark (10s audio, 44.1kHz, F16)");+ println!(" =====================================================================");+ println!();+ println!(" {:<65} {:>8}", "Kernel", "Mean (ms)");+ println!(" {:-<65} {:-<8}", "", "");++ // -----------------------------------------------------------------------+ // 1. AudioQuantizerDecodeKernel+ // tokens: [B, K, T], output: [B, T, input_dim]+ // -----------------------------------------------------------------------+ {+ let frames = SEMANTIC_FRAMES_10S;++ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();++ let tokens = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "perf_quant_tokens");+ let lengths = make_lengths(&context, batch_size, frames as i32);+ let semantic_codebook = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "perf_sem_cb");+ let semantic_out_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "perf_sem_proj");+ let semantic_out_bias = context.create_array(&[INPUT_DIM], dt, "perf_sem_bias");+ let residual_codebooks =+ context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "perf_res_cbs");+ let residual_out_proj =+ context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "perf_res_proj");+ let residual_out_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "perf_res_bias");+ let output = context.create_array(&[batch_size * frames * INPUT_DIM], dt, "perf_quant_out");++ let label = format!(+ "QuantizerDecode [B={}, T={}, K={}, dim={}, cdim={}]",+ batch_size, frames, TOTAL_CODEBOOKS, INPUT_DIM, CODEBOOK_DIM+ );+ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let tok = tokens.buffer();+ let tok = tok.borrow();+ let len = lengths.buffer();+ let len = len.borrow();+ let scb = semantic_codebook.buffer();+ let scb = scb.borrow();+ let sp = semantic_out_proj.buffer();+ let sp = sp.borrow();+ let sb = semantic_out_bias.buffer();+ let sb = sb.borrow();+ let rcb = residual_codebooks.buffer();+ let rcb = rcb.borrow();+ let rp = residual_out_proj.buffer();+ let rp = rp.borrow();+ let rb = residual_out_bias.buffer();+ let rb = rb.borrow();+ let out = output.buffer();+ let mut out = out.borrow_mut();+ kernel.encode(+ &*tok,+ &*len,+ &*scb,+ &*sp,+ &*sb,+ &*rcb,+ &*rp,+ &*rb,+ &mut *out,+ batch_size as i32,+ TOTAL_CODEBOOKS as i32,+ frames as i32,+ INPUT_DIM as i32,+ CODEBOOK_DIM as i32,+ RESIDUAL_QUANTIZERS as i32,+ SEMANTIC_CARDINALITY as i32,+ RESIDUAL_CARDINALITY as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push(("QuantizerDecode".into(), ms));+ }++ // -----------------------------------------------------------------------+ // 2. ConvTranspose1dCausalPad — all 6 stages where it appears+ // Upsample: 1024->512 s=2, 512->256 s=2+ // Decoder: 1536->768 s=8, 768->384 s=8, 384->192 s=4, 192->96 s=2+ // -----------------------------------------------------------------------+ {+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(&context, dt)+ .unwrap();++ // Upsample blocks (NSC input for first, NCS for rest)+ let up_configs: [(usize, usize, usize, i32, &str); 2] = [+ (UP_CHANNELS[0], UP_CHANNELS[1], UP_STRIDES[0], 1, "up0 NSC"), // 1024->512, s=2+ (UP_CHANNELS[1], UP_CHANNELS[2], UP_STRIDES[1], 0, "up1 NCS"), // 512->256, s=2+ ];+ let mut frames = SEMANTIC_FRAMES_10S;+ for (cin, cout, stride, layout, tag) in up_configs {+ let seq_in = frames;+ let seq_out = seq_in * stride;+ let ksize = stride * 2;+ let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");+ let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");+ let bias = context.create_array(&[cout], dt, "tconv_b");+ let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");+ let lengths = make_lengths(&context, batch_size, seq_out as i32);++ let label = format!("TransConv {tag} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");+ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ kernel.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ cin as i32,+ cout as i32,+ seq_in as i32,+ seq_out as i32,+ ksize as i32,+ stride as i32,+ 1_i32,+ layout,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("TransConv {tag}"), ms));+ frames = seq_out;+ }++ // Decoder blocks (all NCS input)+ frames = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1]; // 860+ for block_idx in 0..4 {+ let cin = DEC_CHANNELS[block_idx];+ let cout = DEC_CHANNELS[block_idx + 1];+ let stride = DEC_STRIDES[block_idx];+ let seq_in = frames;+ let seq_out = seq_in * stride;+ let ksize = stride * 2;+ let input = context.create_array(&[batch_size * cin * seq_in], dt, "tconv_in");+ let weight = context.create_array(&[cin * cout * ksize], dt, "tconv_w");+ let bias = context.create_array(&[cout], dt, "tconv_b");+ let output = context.create_array(&[batch_size * cout * seq_out], dt, "tconv_out");+ let lengths = make_lengths(&context, batch_size, seq_out as i32);++ let label = format!("TransConv dec{block_idx} [{cin}->{cout}, s={stride}, k={ksize}, {seq_in}->{seq_out}]");+ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ kernel.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ cin as i32,+ cout as i32,+ seq_in as i32,+ seq_out as i32,+ ksize as i32,+ stride as i32,+ 1_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("TransConv dec{block_idx}"), ms));+ frames = seq_out;+ }+ }++ // -----------------------------------------------------------------------+ // 3. CausalConv1d (groups=1) — first_conv, final_conv, and residual conv1+ // first_conv: 256->1536, k=7+ // residual conv1: same-channel, k=7, dilations [1,3,9]+ // final_conv: 96->1, k=7+ // -----------------------------------------------------------------------+ {+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dKernel::new(&context, dt).unwrap();++ // first_conv after upsampler: 256->1536, k=7+ let fc_seq = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1]; // 860+ {+ let cin = UP_CHANNELS[2]; // 256+ let cout = DECODER_DIM; // 1536+ let ksize = 7;+ let input = context.create_array(&[batch_size * cin * fc_seq], dt, "fc_in");+ let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");+ let bias = context.create_array(&[cout], dt, "fc_b");+ let output = context.create_array(&[batch_size * cout * fc_seq], dt, "fc_out");+ let lengths = make_lengths(&context, batch_size, fc_seq as i32);++ let label = format!("CausalConv1d first_conv [{cin}->{cout}, k={ksize}, T={fc_seq}]");+ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ kernel.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ cin as i32,+ cout as i32,+ fc_seq as i32,+ ksize as i32,+ 1_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push(("CausalConv1d first_conv".into(), ms));+ }++ // Residual conv1 at each decoder block resolution and dilation+ let mut dec_frames = fc_seq;+ for block_idx in 0..4 {+ dec_frames *= DEC_STRIDES[block_idx];+ let ch = DEC_CHANNELS[block_idx + 1];+ let ksize = 7;++ for &dilation in &[1, 3, 9] {+ let label =+ format!("CausalConv1d res dec{block_idx} [{ch}->{ch}, k={ksize}, d={dilation}, T={dec_frames}]");+ let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");+ let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");+ let bias = context.create_array(&[ch], dt, "res_b");+ let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");+ let lengths = make_lengths(&context, batch_size, dec_frames as i32);++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ kernel.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ ch as i32,+ ch as i32,+ dec_frames as i32,+ ksize as i32,+ dilation as i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("CausalConv1d res dec{block_idx} d={dilation}"), ms));+ }+ }++ // final_conv: 96->1, k=7+ {+ let cin = DEC_CHANNELS[4]; // 96+ let cout = 1;+ let ksize = 7;+ let final_frames = SEMANTIC_FRAMES_10S * 2 * 2 * 8 * 8 * 4 * 2; // full resolution+ let input = context.create_array(&[batch_size * cin * final_frames], dt, "fc_in");+ let weight = context.create_array(&[cout * cin * ksize], dt, "fc_w");+ let bias = context.create_array(&[cout], dt, "fc_b");+ let output = context.create_array(&[batch_size * cout * final_frames], dt, "fc_out");+ let lengths = make_lengths(&context, batch_size, final_frames as i32);++ let label = format!("CausalConv1d final_conv [{cin}->{cout}, k={ksize}, T={final_frames}]");+ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ kernel.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ cin as i32,+ cout as i32,+ final_frames as i32,+ ksize as i32,+ 1_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push(("CausalConv1d final_conv".into(), ms));+ }+ }++ // -----------------------------------------------------------------------+ // 4. CausalConv1dGroupedResidual — residual conv2 at each decoder stage+ // Same dimensions as conv1 but with residual add, dilation=1+ // -----------------------------------------------------------------------+ {+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel::new(&context, dt)+ .unwrap();++ let mut dec_frames = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1];+ for block_idx in 0..4 {+ dec_frames *= DEC_STRIDES[block_idx];+ let ch = DEC_CHANNELS[block_idx + 1];+ let ksize = 7;++ let label = format!("CausalConv1dResidual dec{block_idx} [{ch}->{ch}, k={ksize}, T={dec_frames}]");+ let input = context.create_array(&[batch_size * ch * dec_frames], dt, "res_in");+ let residual = context.create_array(&[batch_size * ch * dec_frames], dt, "res_res");+ let weight = context.create_array(&[ch * ch * ksize], dt, "res_w");+ let bias = context.create_array(&[ch], dt, "res_b");+ let output = context.create_array(&[batch_size * ch * dec_frames], dt, "res_out");+ let lengths = make_lengths(&context, batch_size, dec_frames as i32);++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let r = residual.buffer();+ let r = r.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ kernel.encode(+ &*i,+ &*r,+ &*w,+ &*b,+ &mut *o,+ &*l,+ ch as i32,+ ch as i32,+ dec_frames as i32,+ ksize as i32,+ 1_i32,+ 1_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("CausalConv1dResidual dec{block_idx}"), ms));+ }+ }++ // -----------------------------------------------------------------------+ // 5. CausalConv1dGrouped (depthwise) — ConvNeXt depthwise at upsampler resolutions+ // -----------------------------------------------------------------------+ {+ let kernel =+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedKernel::new(&context, dt).unwrap();++ let mut frames = SEMANTIC_FRAMES_10S;+ for block_idx in 0..2 {+ frames *= UP_STRIDES[block_idx];+ let ch = UP_CHANNELS[block_idx + 1];+ let ksize = 7;++ let label = format!("CausalConv1dGrouped dw up{block_idx} [{ch}, g={ch}, k={ksize}, T={frames}]");+ let input = context.create_array(&[batch_size * ch * frames], dt, "dw_in");+ let weight = context.create_array(&[ch * 1 * ksize], dt, "dw_w");+ let bias = context.create_array(&[ch], dt, "dw_b");+ let output = context.create_array(&[batch_size * ch * frames], dt, "dw_out");+ let lengths = make_lengths(&context, batch_size, frames as i32);++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ kernel.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ ch as i32,+ ch as i32,+ frames as i32,+ ksize as i32,+ 1_i32,+ ch as i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("CausalConv1dGrouped dw up{block_idx}"), ms));+ }+ }++ // -----------------------------------------------------------------------+ // 6. HalfSnake — at each decoder stage resolution+ // -----------------------------------------------------------------------+ {+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, dt).unwrap();++ // Before decoder blocks: snake on decoder_dim channels at upsampled frames+ let fc_frames = SEMANTIC_FRAMES_10S * UP_STRIDES[0] * UP_STRIDES[1];+ let mut dec_frames = fc_frames;+ for block_idx in 0..4 {+ // Snake before trans_conv (at input resolution)+ let ch_in = DEC_CHANNELS[block_idx];+ {+ let label = format!("HalfSnake pre-tconv dec{block_idx} [{ch_in}ch, T={dec_frames}]");+ let input = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_in");+ let alpha = context.create_array(&[ch_in], dt, "sn_a");+ let output = context.create_array(&[batch_size * ch_in * dec_frames], dt, "sn_out");++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let a = alpha.buffer();+ let a = a.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ kernel.encode(+ &*i,+ &*a,+ &mut *o,+ ch_in as i32,+ dec_frames as i32,+ ch_in as i32,+ 0.0_f32,+ 1e-9_f32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("HalfSnake pre dec{block_idx}"), ms));+ }++ dec_frames *= DEC_STRIDES[block_idx];+ // Snake in residual units (at output resolution)+ let ch_out = DEC_CHANNELS[block_idx + 1];+ {+ let label = format!("HalfSnake res dec{block_idx} [{ch_out}ch, T={dec_frames}]");+ let input = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_in");+ let alpha = context.create_array(&[ch_out], dt, "sn_a");+ let output = context.create_array(&[batch_size * ch_out * dec_frames], dt, "sn_out");++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let a = alpha.buffer();+ let a = a.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ kernel.encode(+ &*i,+ &*a,+ &mut *o,+ ch_out as i32,+ dec_frames as i32,+ ch_out as i32,+ 0.0_f32,+ 1e-9_f32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("HalfSnake res dec{block_idx}"), ms));+ }+ }+ }++ // -----------------------------------------------------------------------+ // 7. NormNcs and Conv1d pointwise — ConvNeXt blocks in upsampler+ // -----------------------------------------------------------------------+ {+ let k_norm = <<Metal as Backend>::Kernels as Kernels>::AudioNormNcsKernel::new(&context, dt).unwrap();+ let k_conv1d = <<Metal as Backend>::Kernels as Kernels>::AudioConv1dKernel::new(&context, dt).unwrap();+ let k_act = <<Metal as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, dt, false).unwrap();++ let mut frames = SEMANTIC_FRAMES_10S;+ for block_idx in 0..2 {+ frames *= UP_STRIDES[block_idx];+ let ch = UP_CHANNELS[block_idx + 1];++ // Norm+ {+ let label = format!("NormNcs up{block_idx} [{ch}ch, T={frames}]");+ let input = context.create_array(&[batch_size * ch * frames], dt, "norm_in");+ let scales = context.create_array(&[ch], dt, "norm_s");+ let bias = context.create_array(&[ch], dt, "norm_b");+ let output = context.create_array(&[batch_size * ch * frames], dt, "norm_out");+ let lengths = make_lengths(&context, batch_size, frames as i32);++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let s = scales.buffer();+ let s = s.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ k_norm.encode(+ &*i,+ &*s,+ &*b,+ &mut *o,+ &*l,+ ch as i32,+ frames as i32,+ 1e-6_f32,+ 1_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("NormNcs up{block_idx}"), ms));+ }++ // Pointwise conv1 (ch -> ch)+ {+ let label = format!("Conv1d pw up{block_idx} [{ch}->{ch}, k=1, T={frames}]");+ let input = context.create_array(&[batch_size * ch * frames], dt, "pw_in");+ let weight = context.create_array(&[ch * ch], dt, "pw_w");+ let bias = context.create_array(&[ch], dt, "pw_b");+ let output = context.create_array(&[batch_size * ch * frames], dt, "pw_out");+ let lengths = make_lengths(&context, batch_size, frames as i32);++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let w = weight.buffer();+ let w = w.borrow();+ let b = bias.buffer();+ let b = b.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ let l = lengths.buffer();+ let l = l.borrow();+ k_conv1d.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ ch as i32,+ ch as i32,+ frames as i32,+ frames as i32,+ 1_i32,+ 1_i32,+ 1_i32,+ 0_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("Conv1d pw up{block_idx}"), ms));+ }++ // GELU+ {+ let n = ch * frames;+ let label = format!("GELU up{block_idx} [n={n}]");+ let input = context.create_array(&[n], dt, "gelu_in");+ let output = context.create_array(&[n], dt, "gelu_out");++ let ms = measure(&label, || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ {+ let i = input.buffer();+ let i = i.borrow();+ let o = output.buffer();+ let mut o = o.borrow_mut();+ k_act.encode(Some(&*i), &mut *o, n as u32, ActivationType::GELU, &mut encoder);+ }+ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push((format!("GELU up{block_idx}"), ms));+ }+ }+ }++ // =======================================================================+ // Full vocoder pipeline simulation — single command buffer+ //+ // Matches submit_decode_padded flow:+ // quantizer -> 2 upsample blocks -> first_conv -> 4 decoder blocks -> final snake+conv+tanh+ // =======================================================================+ println!();+ println!(" {:-<65} {:-<8}", "", "");+ println!(" {:<65} {:>8}", "Full Pipeline Simulation (single command buffer)", "Mean (ms)");+ println!(" {:-<65} {:-<8}", "", "");++ // Build all kernels+ let k_quant = <<Metal as Backend>::Kernels as Kernels>::AudioQuantizerDecodeKernel::new(&context, dt).unwrap();+ let k_tconv =+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(&context, dt).unwrap();+ let k_norm = <<Metal as Backend>::Kernels as Kernels>::AudioNormNcsKernel::new(&context, dt).unwrap();+ let k_conv1d = <<Metal as Backend>::Kernels as Kernels>::AudioConv1dKernel::new(&context, dt).unwrap();+ let k_act = <<Metal as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, dt, false).unwrap();+ let k_add = <<Metal as Backend>::Kernels as Kernels>::AudioAddKernel::new(&context, dt).unwrap();+ let k_snake = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, dt).unwrap();+ let k_conv_grp =+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedKernel::new(&context, dt).unwrap();+ let k_conv1d_causal = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dKernel::new(&context, dt).unwrap();+ let k_conv_res =+ <<Metal as Backend>::Kernels as Kernels>::AudioCausalConv1dGroupedResidualKernel::new(&context, dt).unwrap();++ let frames = SEMANTIC_FRAMES_10S;++ // Quantizer arrays+ let p_q_tok = context.create_array(&[batch_size * TOTAL_CODEBOOKS * frames], DataType::U32, "pipe_q_tok");+ let p_q_len = make_lengths(&context, batch_size, frames as i32);+ let p_q_sem_cb = context.create_array(&[SEMANTIC_CARDINALITY * CODEBOOK_DIM], dt, "p_sem_cb");+ let p_q_sem_proj = context.create_array(&[INPUT_DIM * CODEBOOK_DIM], dt, "p_sem_proj");+ let p_q_sem_bias = context.create_array(&[INPUT_DIM], dt, "p_sem_bias");+ let p_q_res_cbs =+ context.create_array(&[RESIDUAL_QUANTIZERS * RESIDUAL_CARDINALITY * CODEBOOK_DIM], dt, "p_res_cbs");+ let p_q_res_proj = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM * CODEBOOK_DIM], dt, "p_res_proj");+ let p_q_res_bias = context.create_array(&[RESIDUAL_QUANTIZERS * INPUT_DIM], dt, "p_res_bias");++ // Compute frame counts at each stage+ let f_up0 = frames * UP_STRIDES[0]; // 430+ let f_up1 = f_up0 * UP_STRIDES[1]; // 860+ let f_dec0 = f_up1 * DEC_STRIDES[0]; // 6880+ let f_dec1 = f_dec0 * DEC_STRIDES[1]; // 55040+ let f_dec2 = f_dec1 * DEC_STRIDES[2]; // 220160+ let f_dec3 = f_dec2 * DEC_STRIDES[3]; // 440320++ // Pipeline scratch: large enough for the biggest intermediate+ // Biggest is decoder_dim * f_up1 or ch * f_dec stages+ let max_elems: usize = batch_size * DECODER_DIM * f_dec3.max(f_up1);+ let scratch_a = context.create_array(&[max_elems], dt, "pipe_sa");+ let scratch_b = context.create_array(&[max_elems], dt, "pipe_sb");+ // Third scratch buffer for residual connections (avoids RefCell borrow conflict)+ let scratch_r = context.create_array(&[max_elems], dt, "pipe_sr");++ // Upsample block 0: 1024->512, stride=2+ let u0_w = context.create_array(&[UP_CHANNELS[0] * UP_CHANNELS[1] * (UP_STRIDES[0] * 2)], dt, "u0_w");+ let u0_b = context.create_array(&[UP_CHANNELS[1]], dt, "u0_b");+ // ConvNeXt 0 (ch=512)+ let c0_ch = UP_CHANNELS[1];+ let c0_dw_w = context.create_array(&[c0_ch * 1 * 7], dt, "c0_dw_w");+ let c0_dw_b = context.create_array(&[c0_ch], dt, "c0_dw_b");+ let c0_ns = context.create_array(&[c0_ch], dt, "c0_ns");+ let c0_nb = context.create_array(&[c0_ch], dt, "c0_nb");+ let c0_p1_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p1_w");+ let c0_p1_b = context.create_array(&[c0_ch], dt, "c0_p1_b");+ let c0_p2_w = context.create_array(&[c0_ch * c0_ch], dt, "c0_p2_w");+ let c0_p2_b = context.create_array(&[c0_ch], dt, "c0_p2_b");++ // Upsample block 1: 512->256, stride=2+ let u1_w = context.create_array(&[UP_CHANNELS[1] * UP_CHANNELS[2] * (UP_STRIDES[1] * 2)], dt, "u1_w");+ let u1_b = context.create_array(&[UP_CHANNELS[2]], dt, "u1_b");+ // ConvNeXt 1 (ch=256)+ let c1_ch = UP_CHANNELS[2];+ let c1_dw_w = context.create_array(&[c1_ch * 1 * 7], dt, "c1_dw_w");+ let c1_dw_b = context.create_array(&[c1_ch], dt, "c1_dw_b");+ let c1_ns = context.create_array(&[c1_ch], dt, "c1_ns");+ let c1_nb = context.create_array(&[c1_ch], dt, "c1_nb");+ let c1_p1_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p1_w");+ let c1_p1_b = context.create_array(&[c1_ch], dt, "c1_p1_b");+ let c1_p2_w = context.create_array(&[c1_ch * c1_ch], dt, "c1_p2_w");+ let c1_p2_b = context.create_array(&[c1_ch], dt, "c1_p2_b");++ // first_conv: 256->1536, k=7+ let fc_w = context.create_array(&[DECODER_DIM * UP_CHANNELS[2] * 7], dt, "fc_w");+ let fc_b = context.create_array(&[DECODER_DIM], dt, "fc_b");++ // Decoder block weights for all 4 blocks+ struct DecBlockWeights {+ snake_a: Array<Metal>,+ tconv_w: Array<Metal>,+ tconv_b: Array<Metal>,+ // 3 residual units, each with snake1, conv1, snake2, conv2+ ru_s1: [Array<Metal>; 3],+ ru_c1_w: [Array<Metal>; 3],+ ru_c1_b: [Array<Metal>; 3],+ ru_s2: [Array<Metal>; 3],+ ru_c2_w: [Array<Metal>; 3],+ ru_c2_b: [Array<Metal>; 3],+ }++ let dec_blocks: Vec<DecBlockWeights> = (0..4)+ .map(|i| {+ let cin = DEC_CHANNELS[i];+ let cout = DEC_CHANNELS[i + 1];+ let stride = DEC_STRIDES[i];+ let ksize = stride * 2;+ DecBlockWeights {+ snake_a: context.create_array(&[cin], dt, &format!("d{i}_sa")),+ tconv_w: context.create_array(&[cin * cout * ksize], dt, &format!("d{i}_tw")),+ tconv_b: context.create_array(&[cout], dt, &format!("d{i}_tb")),+ ru_s1: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s1"))),+ ru_c1_w: std::array::from_fn(|j| {+ context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c1w"))+ }),+ ru_c1_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c1b"))),+ ru_s2: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_s2"))),+ ru_c2_w: std::array::from_fn(|j| {+ context.create_array(&[cout * cout * 7], dt, &format!("d{i}_r{j}_c2w"))+ }),+ ru_c2_b: std::array::from_fn(|j| context.create_array(&[cout], dt, &format!("d{i}_r{j}_c2b"))),+ }+ })+ .collect();++ // final snake + conv + tanh+ let final_snake_a = context.create_array(&[DEC_CHANNELS[4]], dt, "final_sa");+ let final_conv_w = context.create_array(&[1 * DEC_CHANNELS[4] * 7], dt, "final_cw");+ let final_conv_b = context.create_array(&[1], dt, "final_cb");++ // Length arrays+ let len_up0 = make_lengths(&context, batch_size, f_up0 as i32);+ let len_up1 = make_lengths(&context, batch_size, f_up1 as i32);+ let len_dec0 = make_lengths(&context, batch_size, f_dec0 as i32);+ let len_dec1 = make_lengths(&context, batch_size, f_dec1 as i32);+ let len_dec2 = make_lengths(&context, batch_size, f_dec2 as i32);+ let len_dec3 = make_lengths(&context, batch_size, f_dec3 as i32);+ let dec_lens = [&len_dec0, &len_dec1, &len_dec2, &len_dec3];+ let dec_frame_sizes = [f_dec0, f_dec1, f_dec2, f_dec3];+ let dilations = [1_i32, 3, 9];++ let ms = measure("Full pipeline (quant -> 2 up -> first_conv -> 4 dec -> final)", || {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).unwrap();+ let sa = scratch_a.buffer();+ let sb = scratch_b.buffer();++ // --- Quantizer: output [B, T, input_dim] (NSC) ---+ {+ let tok = p_q_tok.buffer();+ let tok = tok.borrow();+ let len = p_q_len.buffer();+ let len = len.borrow();+ let scb = p_q_sem_cb.buffer();+ let scb = scb.borrow();+ let sp = p_q_sem_proj.buffer();+ let sp = sp.borrow();+ let sbi = p_q_sem_bias.buffer();+ let sbi = sbi.borrow();+ let rcb = p_q_res_cbs.buffer();+ let rcb = rcb.borrow();+ let rp = p_q_res_proj.buffer();+ let rp = rp.borrow();+ let rb = p_q_res_bias.buffer();+ let rb = rb.borrow();+ let mut out = sa.borrow_mut();+ k_quant.encode(+ &*tok,+ &*len,+ &*scb,+ &*sp,+ &*sbi,+ &*rcb,+ &*rp,+ &*rb,+ &mut *out,+ batch_size as i32,+ TOTAL_CODEBOOKS as i32,+ frames as i32,+ INPUT_DIM as i32,+ CODEBOOK_DIM as i32,+ RESIDUAL_QUANTIZERS as i32,+ SEMANTIC_CARDINALITY as i32,+ RESIDUAL_CARDINALITY as i32,+ &mut encoder,+ );+ }++ // --- Upsample block 0: 1024->512, stride=2, NSC input ---+ {+ let i = sa.borrow();+ let w = u0_w.buffer();+ let w = w.borrow();+ let b = u0_b.buffer();+ let b = b.borrow();+ let mut o = sb.borrow_mut();+ let l = len_up0.buffer();+ let l = l.borrow();+ k_tconv.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ UP_CHANNELS[0] as i32,+ UP_CHANNELS[1] as i32,+ frames as i32,+ f_up0 as i32,+ (UP_STRIDES[0] * 2) as i32,+ UP_STRIDES[0] as i32,+ 1_i32,+ 1_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ // ConvNeXt 0+ {+ let i = sb.borrow();+ let w = c0_dw_w.buffer();+ let w = w.borrow();+ let b = c0_dw_b.buffer();+ let b = b.borrow();+ let mut o = sa.borrow_mut();+ let l = len_up0.buffer();+ let l = l.borrow();+ k_conv_grp.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ c0_ch as i32,+ c0_ch as i32,+ f_up0 as i32,+ 7_i32,+ 1_i32,+ c0_ch as i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let i = sa.borrow();+ let s = c0_ns.buffer();+ let s = s.borrow();+ let b = c0_nb.buffer();+ let b = b.borrow();+ let mut o = sb.borrow_mut();+ let l = len_up0.buffer();+ let l = l.borrow();+ k_norm.encode(+ &*i,+ &*s,+ &*b,+ &mut *o,+ &*l,+ c0_ch as i32,+ f_up0 as i32,+ 1e-6_f32,+ 1_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let i = sb.borrow();+ let w = c0_p1_w.buffer();+ let w = w.borrow();+ let b = c0_p1_b.buffer();+ let b = b.borrow();+ let mut o = sa.borrow_mut();+ let l = len_up0.buffer();+ let l = l.borrow();+ k_conv1d.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ c0_ch as i32,+ c0_ch as i32,+ f_up0 as i32,+ f_up0 as i32,+ 1_i32,+ 1_i32,+ 1_i32,+ 0_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let i = sa.borrow();+ let mut o = sb.borrow_mut();+ k_act.encode(Some(&*i), &mut *o, (c0_ch * f_up0) as u32, ActivationType::GELU, &mut encoder);+ }+ {+ let i = sb.borrow();+ let w = c0_p2_w.buffer();+ let w = w.borrow();+ let b = c0_p2_b.buffer();+ let b = b.borrow();+ let mut o = sa.borrow_mut();+ let l = len_up0.buffer();+ let l = l.borrow();+ k_conv1d.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ c0_ch as i32,+ c0_ch as i32,+ f_up0 as i32,+ f_up0 as i32,+ 1_i32,+ 1_i32,+ 1_i32,+ 0_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let a = sa.borrow();+ let mut o = sb.borrow_mut();+ k_add.encode(&*a, &*a, &mut *o, (c0_ch * f_up0) as i32, &mut encoder);+ }++ // --- Upsample block 1: 512->256, stride=2, NCS input ---+ {+ let i = sb.borrow();+ let w = u1_w.buffer();+ let w = w.borrow();+ let b = u1_b.buffer();+ let b = b.borrow();+ let mut o = sa.borrow_mut();+ let l = len_up1.buffer();+ let l = l.borrow();+ k_tconv.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ UP_CHANNELS[1] as i32,+ UP_CHANNELS[2] as i32,+ f_up0 as i32,+ f_up1 as i32,+ (UP_STRIDES[1] * 2) as i32,+ UP_STRIDES[1] as i32,+ 1_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ // ConvNeXt 1+ {+ let i = sa.borrow();+ let w = c1_dw_w.buffer();+ let w = w.borrow();+ let b = c1_dw_b.buffer();+ let b = b.borrow();+ let mut o = sb.borrow_mut();+ let l = len_up1.buffer();+ let l = l.borrow();+ k_conv_grp.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ c1_ch as i32,+ c1_ch as i32,+ f_up1 as i32,+ 7_i32,+ 1_i32,+ c1_ch as i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let i = sb.borrow();+ let s = c1_ns.buffer();+ let s = s.borrow();+ let b = c1_nb.buffer();+ let b = b.borrow();+ let mut o = sa.borrow_mut();+ let l = len_up1.buffer();+ let l = l.borrow();+ k_norm.encode(+ &*i,+ &*s,+ &*b,+ &mut *o,+ &*l,+ c1_ch as i32,+ f_up1 as i32,+ 1e-6_f32,+ 1_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let i = sa.borrow();+ let w = c1_p1_w.buffer();+ let w = w.borrow();+ let b = c1_p1_b.buffer();+ let b = b.borrow();+ let mut o = sb.borrow_mut();+ let l = len_up1.buffer();+ let l = l.borrow();+ k_conv1d.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ c1_ch as i32,+ c1_ch as i32,+ f_up1 as i32,+ f_up1 as i32,+ 1_i32,+ 1_i32,+ 1_i32,+ 0_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let i = sb.borrow();+ let mut o = sa.borrow_mut();+ k_act.encode(Some(&*i), &mut *o, (c1_ch * f_up1) as u32, ActivationType::GELU, &mut encoder);+ }+ {+ let i = sa.borrow();+ let w = c1_p2_w.buffer();+ let w = w.borrow();+ let b = c1_p2_b.buffer();+ let b = b.borrow();+ let mut o = sb.borrow_mut();+ let l = len_up1.buffer();+ let l = l.borrow();+ k_conv1d.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ c1_ch as i32,+ c1_ch as i32,+ f_up1 as i32,+ f_up1 as i32,+ 1_i32,+ 1_i32,+ 1_i32,+ 0_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ {+ let a = sb.borrow();+ let mut o = sa.borrow_mut();+ k_add.encode(&*a, &*a, &mut *o, (c1_ch * f_up1) as i32, &mut encoder);+ }++ // --- first_conv: 256->1536, k=7 ---+ {+ let i = sa.borrow();+ let w = fc_w.buffer();+ let w = w.borrow();+ let b = fc_b.buffer();+ let b = b.borrow();+ let mut o = sb.borrow_mut();+ let l = len_up1.buffer();+ let l = l.borrow();+ k_conv1d_causal.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ UP_CHANNELS[2] as i32,+ DECODER_DIM as i32,+ f_up1 as i32,+ 7_i32,+ 1_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }++ // --- 4 Decoder blocks ---+ // State in sb after first_conv+ let mut cur_in_sa = false; // data is in sb+ let mut cur_frames = f_up1;++ for block_idx in 0..4 {+ let blk = &dec_blocks[block_idx];+ let cin = DEC_CHANNELS[block_idx];+ let cout = DEC_CHANNELS[block_idx + 1];+ let stride = DEC_STRIDES[block_idx];+ let ksize = stride * 2;+ let next_frames = dec_frame_sizes[block_idx];+ let dec_len = dec_lens[block_idx];++ // snake (in-place swap)+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let a = blk.snake_a.buffer();+ let a = a.borrow();+ let mut o = o_buf.borrow_mut();+ k_snake.encode(+ &*i,+ &*a,+ &mut *o,+ cin as i32,+ cur_frames as i32,+ cin as i32,+ 0.0_f32,+ 1e-9_f32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;++ // trans_conv+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let w = blk.tconv_w.buffer();+ let w = w.borrow();+ let b = blk.tconv_b.buffer();+ let b = b.borrow();+ let mut o = o_buf.borrow_mut();+ let l = dec_len.buffer();+ let l = l.borrow();+ k_tconv.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ cin as i32,+ cout as i32,+ cur_frames as i32,+ next_frames as i32,+ ksize as i32,+ stride as i32,+ 1_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;+ cur_frames = next_frames;++ // 3 residual units+ let sr = scratch_r.buffer();+ for ru_idx in 0..3 {+ // snake1+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let a = blk.ru_s1[ru_idx].buffer();+ let a = a.borrow();+ let mut o = o_buf.borrow_mut();+ k_snake.encode(+ &*i,+ &*a,+ &mut *o,+ cout as i32,+ cur_frames as i32,+ cout as i32,+ 0.0_f32,+ 1e-9_f32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;++ // conv1 (dilation varies: 1, 3, 9)+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let w = blk.ru_c1_w[ru_idx].buffer();+ let w = w.borrow();+ let b = blk.ru_c1_b[ru_idx].buffer();+ let b = b.borrow();+ let mut o = o_buf.borrow_mut();+ let l = dec_len.buffer();+ let l = l.borrow();+ k_conv1d_causal.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ cout as i32,+ cout as i32,+ cur_frames as i32,+ 7_i32,+ dilations[ru_idx],+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;++ // snake2+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let a = blk.ru_s2[ru_idx].buffer();+ let a = a.borrow();+ let mut o = o_buf.borrow_mut();+ k_snake.encode(+ &*i,+ &*a,+ &mut *o,+ cout as i32,+ cur_frames as i32,+ cout as i32,+ 0.0_f32,+ 1e-9_f32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;++ // conv2 + residual add+ // Use scratch_r as the residual source (separate buffer to+ // avoid RefCell borrow conflict with the output destination).+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let r = sr.borrow();+ let w = blk.ru_c2_w[ru_idx].buffer();+ let w = w.borrow();+ let b = blk.ru_c2_b[ru_idx].buffer();+ let b = b.borrow();+ let mut o = o_buf.borrow_mut();+ let l = dec_len.buffer();+ let l = l.borrow();+ k_conv_res.encode(+ &*i,+ &*r,+ &*w,+ &*b,+ &mut *o,+ &*l,+ cout as i32,+ cout as i32,+ cur_frames as i32,+ 7_i32,+ 1_i32,+ 1_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;+ }+ }++ // --- final snake + conv + tanh ---+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let a = final_snake_a.buffer();+ let a = a.borrow();+ let mut o = o_buf.borrow_mut();+ k_snake.encode(+ &*i,+ &*a,+ &mut *o,+ DEC_CHANNELS[4] as i32,+ f_dec3 as i32,+ DEC_CHANNELS[4] as i32,+ 0.0_f32,+ 1e-9_f32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let w = final_conv_w.buffer();+ let w = w.borrow();+ let b = final_conv_b.buffer();+ let b = b.borrow();+ let mut o = o_buf.borrow_mut();+ let l = len_dec3.buffer();+ let l = l.borrow();+ k_conv1d_causal.encode(+ &*i,+ &*w,+ &*b,+ &mut *o,+ &*l,+ DEC_CHANNELS[4] as i32,+ 1_i32,+ f_dec3 as i32,+ 7_i32,+ 1_i32,+ 0_i32,+ batch_size as i32,+ &mut encoder,+ );+ }+ cur_in_sa = !cur_in_sa;+ // tanh+ {+ let (i_buf, o_buf) = if cur_in_sa {+ (sa.clone(), sb.clone())+ } else {+ (sb.clone(), sa.clone())+ };+ let i = i_buf.borrow();+ let mut o = o_buf.borrow_mut();+ k_act.encode(Some(&*i), &mut *o, f_dec3 as u32, ActivationType::TANH, &mut encoder);+ }++ encoder.end_encoding().submit().wait_until_completed().unwrap();+ });+ results.push(("Full pipeline".into(), ms));++ // =======================================================================+ // Summary+ // =======================================================================+ println!();+ println!(" === SUMMARY ===");+ println!(" {:<65} {:>8}", "Kernel", "Mean (ms)");+ println!(" {:-<65} {:-<8}", "", "");+ let mut total_individual = 0.0;+ for (name, ms) in &results {+ println!(" {name:<65} {ms:>8.3}");+ if name != "Full pipeline" {+ total_individual += ms;+ }+ }+ println!(" {:-<65} {:-<8}", "", "");+ println!(" {:<65} {:>8.3}", "Sum of individual kernels", total_individual);+ if let Some((_, pipeline_ms)) = results.iter().find(|(n, _)| n == "Full pipeline") {+ println!(" {:<65} {:>8.3}", "Full pipeline (single CB)", pipeline_ms);+ }+ println!();+}diff --git a/crates/uzu/tests/audio_nanocodec_fsq_test.rs b/crates/uzu/tests/audio_nanocodec_fsq_test.rsnew file mode 100644index 000000000..a03d71722--- /dev/null+++ b/crates/uzu/tests/audio_nanocodec_fsq_test.rs@@ -0,0 +1,32 @@+mod common;++use common::audio_nanocodec_fsq_reference::{fsq_decode_reference, fsq_encode_reference};++#[test]+fn fsq_decode_reference_masks_values_beyond_lengths() {+ let output = fsq_decode_reference(&[0, 7, 11, 3, 5, 9], &[2], 1, 2, 3, 2, &[8, 5]).expect("decode");++ assert_eq!(output.len(), 12);+ let masked_t2 = [output[2], output[5], output[8], output[11]];+ assert_eq!(masked_t2, [0.0, 0.0, 0.0, 0.0]);+}++#[test]+fn fsq_encode_reference_masks_values_beyond_lengths() {+ let tokens = fsq_encode_reference(+ &[-0.9, -0.1, 0.2, 0.7, 0.8, 0.3, -0.4, -0.7, -0.2, 0.0, 0.4, 0.9, 0.5, -0.5, 0.1, -0.1],+ &[3],+ 1,+ 2,+ 4,+ 2,+ &[8, 6],+ &[1, 8],+ 1e-3,+ )+ .expect("encode");++ assert_eq!(tokens.len(), 8);+ assert_eq!(tokens[3], 0);+ assert_eq!(tokens[7], 0);+}diff --git a/crates/uzu/tests/audio_nanocodec_runtime_test.rs b/crates/uzu/tests/audio_nanocodec_runtime_test.rsnew file mode 100644index 000000000..bb0767544--- /dev/null+++ b/crates/uzu/tests/audio_nanocodec_runtime_test.rs@@ -0,0 +1,177 @@+#![cfg(all(feature = "audio-runtime", metal_backend))]++mod common;++use common::audio_nanocodec_fsq_reference::{fsq_decode_reference, fsq_encode_reference};+use uzu::{+ NanoCodecFsqRuntime, NanoCodecFsqRuntimeConfig,+ backends::metal::Metal,+ prelude::{AudioCodecRuntime, AudioError, AudioPcmBatch, AudioTokenGrid},+};++fn create_runtime() -> NanoCodecFsqRuntime<Metal> {+ let config =+ NanoCodecFsqRuntimeConfig::new(24_000, 2, vec![8, 6].into_boxed_slice(), 1e-3).expect("valid runtime config");+ NanoCodecFsqRuntime::new(config)+}++fn make_pcm(+ channels: usize,+ lengths: &[usize],+) -> AudioPcmBatch {+ let sample_count = lengths.iter().sum::<usize>() * channels;+ let samples: Vec<f32> = (0..sample_count).map(|index| (index as f32 * 0.071 - 1.3).sin() * 0.9).collect();+ AudioPcmBatch::new(samples.into_boxed_slice(), 24_000, channels, lengths.to_vec().into_boxed_slice()).expect("pcm")+}++fn pack_for_reference(pcm: &AudioPcmBatch) -> (Vec<f32>, Vec<i32>, usize) {+ let batch_size = pcm.batch_size();+ let channels = pcm.channels();+ let frames = pcm.lengths().iter().copied().max().unwrap_or(0);+ let mut padded = vec![0.0_f32; batch_size * channels * frames];+ let mut lengths_i32 = Vec::with_capacity(batch_size);++ let mut src_offset = 0usize;+ for (batch, &length) in pcm.lengths().iter().enumerate() {+ lengths_i32.push(length as i32);+ for frame in 0..length {+ for channel in 0..channels {+ let src_index = src_offset + frame * channels + channel;+ let dst_index = (batch * channels + channel) * frames + frame;+ padded[dst_index] = pcm.samples()[src_index];+ }+ }+ src_offset += length * channels;+ }++ (padded, lengths_i32, frames)+}++fn unpack_reference_output(+ padded: &[f32],+ channels: usize,+ frames: usize,+ lengths: &[usize],+) -> Vec<f32> {+ let total = lengths.iter().sum::<usize>() * channels;+ let mut out = vec![0.0_f32; total];+ let mut dst_offset = 0usize;++ for (batch, &length) in lengths.iter().enumerate() {+ for frame in 0..length {+ for channel in 0..channels {+ let src_index = (batch * channels + channel) * frames + frame;+ let dst_index = dst_offset + frame * channels + channel;+ out[dst_index] = padded[src_index];+ }+ }+ dst_offset += length * channels;+ }++ out+}++#[test]+fn nanocodec_runtime_encode_matches_fsq_reference() {+ let runtime = create_runtime();+ let pcm = make_pcm(runtime.config().channels(), &[3, 5]);+ let tokens = runtime.encode(&pcm).expect("encode");++ let (padded_input, lengths_i32, frames) = pack_for_reference(&pcm);+ let expected_i32 = fsq_encode_reference(+ &padded_input,+ &lengths_i32,+ pcm.batch_size(),+ runtime.config().num_groups(),+ frames,+ runtime.config().codebook_dim_per_group(),+ runtime.config().num_levels_per_group(),+ runtime.config().dim_base_index(),+ runtime.config().eps(),+ )+ .expect("fsq reference encode");+ let expected_u32: Vec<u32> = expected_i32.into_iter().map(|value| value as u32).collect();+ let expected = AudioTokenGrid::new(+ expected_u32.into_boxed_slice(),+ pcm.batch_size(),+ runtime.config().num_groups(),+ frames,+ pcm.lengths().to_vec().into_boxed_slice(),+ )+ .expect("expected token grid");++ assert_eq!(tokens.batch_size(), expected.batch_size());+ assert_eq!(tokens.codebooks(), expected.codebooks());+ assert_eq!(tokens.frames(), expected.frames());+ assert_eq!(tokens.lengths(), expected.lengths());+ assert_eq!(tokens.tokens(), expected.tokens());+}++#[test]+fn nanocodec_runtime_decode_matches_fsq_reference() {+ let runtime = create_runtime();+ let pcm = make_pcm(runtime.config().channels(), &[4, 2]);+ let encoded = runtime.encode(&pcm).expect("encode");+ let decoded = runtime.decode(&encoded).expect("decode");++ let reference_tokens_i32: Vec<i32> = encoded.tokens().iter().map(|&token| token as i32).collect();+ let lengths_i32: Vec<i32> = encoded.lengths().iter().map(|&length| length as i32).collect();+ let reference_padded = fsq_decode_reference(+ &reference_tokens_i32,+ &lengths_i32,+ encoded.batch_size(),+ runtime.config().num_groups(),+ encoded.frames(),+ runtime.config().codebook_dim_per_group(),+ runtime.config().num_levels_per_group(),+ )+ .expect("fsq reference decode");+ let expected_samples =+ unpack_reference_output(&reference_padded, runtime.config().channels(), encoded.frames(), encoded.lengths());++ assert_eq!(decoded.sample_rate(), runtime.config().sample_rate());+ assert_eq!(decoded.channels(), runtime.config().channels());+ assert_eq!(decoded.lengths(), encoded.lengths());++ for (index, (&got, &expected)) in decoded.samples().iter().zip(expected_samples.iter()).enumerate() {+ let delta = (got - expected).abs();+ assert!(delta <= 1e-5_f32, "decode mismatch at index={index}: got={got}, expected={expected}, delta={delta}");+ }+}++#[test]+fn nanocodec_runtime_decode_rejects_out_of_range_token() {+ let runtime = create_runtime();+ let cardinality = runtime.config().codec_cardinality();+ let tokens = AudioTokenGrid::new(+ vec![0, cardinality, 1, 2].into_boxed_slice(),+ 1,+ runtime.config().num_groups(),+ 2,+ vec![2usize].into_boxed_slice(),+ )+ .expect("token grid");++ let error = runtime.decode(&tokens).expect_err("decode should fail");+ assert_eq!(+ error,+ AudioError::InvalidCodecToken {+ token: cardinality,+ cardinality,+ }+ );+}++#[test]+fn nanocodec_runtime_encode_rejects_channel_mismatch() {+ let runtime = create_runtime();+ let pcm = make_pcm(runtime.config().channels() - 1, &[2]);++ let error = runtime.encode(&pcm).expect_err("encode should fail");+ match error {+ AudioError::Runtime(message) => {+ assert!(message.contains("channel mismatch"), "unexpected error message: {message}");+ },+ other => panic!("unexpected error variant: {other:?}"),+ }+}diff --git a/crates/uzu/tests/common/audio_nanocodec_fsq_reference.rs b/crates/uzu/tests/common/audio_nanocodec_fsq_reference.rsnew file mode 100644index 000000000..433a6aa52--- /dev/null+++ b/crates/uzu/tests/common/audio_nanocodec_fsq_reference.rs@@ -0,0 +1,197 @@+use uzu::prelude::{AudioError, AudioResult};++fn checked_product(values: &[usize]) -> AudioResult<usize> {+ values+ .iter()+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))+ .ok_or(AudioError::Runtime("dimension product overflow".to_string()))+}++fn round_ties_to_even(value: f32) -> f32 {+ let floor = value.floor();+ let fraction = value - floor;++ if fraction < 0.5 {+ floor+ } else if fraction > 0.5 {+ floor + 1.0+ } else if (floor as i64 & 1) != 0 {+ floor + 1.0+ } else {+ floor+ }+}++fn validate_fsq_common(+ batch_size: usize,+ num_groups: usize,+ seq_len: usize,+ codebook_dim: usize,+ num_levels: &[i32],+ lengths: &[i32],+) -> AudioResult<()> {+ if num_groups == 0 || codebook_dim == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }++ if num_levels.len() != codebook_dim {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: codebook_dim,+ actual_tokens: num_levels.len(),+ });+ }++ for &levels in num_levels {+ if levels <= 1 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ }++ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }++ for &length in lengths {+ if length < 0 || length as usize > seq_len {+ return Err(AudioError::InvalidTokenLengthValue {+ length: length.max(0) as usize,+ frames: seq_len,+ });+ }+ }++ Ok(())+}++pub fn fsq_decode_reference(+ tokens: &[i32],+ lengths: &[i32],+ batch_size: usize,+ num_groups: usize,+ seq_len: usize,+ codebook_dim: usize,+ num_levels: &[i32],+) -> AudioResult<Vec<f32>> {+ validate_fsq_common(batch_size, num_groups, seq_len, codebook_dim, num_levels, lengths)?;++ let expected_tokens = checked_product(&[batch_size, num_groups, seq_len])?;+ if tokens.len() != expected_tokens {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens,+ actual_tokens: tokens.len(),+ });+ }++ let output_len = checked_product(&[batch_size, num_groups * codebook_dim, seq_len])?;+ let mut output = vec![0.0_f32; output_len];++ for batch in 0..batch_size {+ let length = lengths[batch] as usize;+ for group in 0..num_groups {+ for time in 0..seq_len {+ if time >= length {+ continue;+ }++ let token_index = (batch * num_groups + group) * seq_len + time;+ let token = tokens[token_index];+ let mut base = 1_i32;++ for dim in 0..codebook_dim {+ let levels = num_levels[dim];+ let scale = levels / 2;+ let offset = scale;++ let div = token / base;+ let mut code_nonnegative = div % levels;+ if code_nonnegative < 0 {+ code_nonnegative += levels;+ }++ let value = (code_nonnegative - offset) as f32 / scale as f32;+ let channel = group * codebook_dim + dim;+ let output_index = (batch * (num_groups * codebook_dim) + channel) * seq_len + time;+ output[output_index] = value;++ base =+ base.checked_mul(levels).ok_or(AudioError::Runtime("fsq decode base overflow".to_string()))?;+ }+ }+ }+ }++ Ok(output)+}++pub fn fsq_encode_reference(+ input: &[f32],+ lengths: &[i32],+ batch_size: usize,+ num_groups: usize,+ seq_len: usize,+ codebook_dim: usize,+ num_levels: &[i32],+ dim_base_index: &[i32],+ eps: f32,+) -> AudioResult<Vec<i32>> {+ validate_fsq_common(batch_size, num_groups, seq_len, codebook_dim, num_levels, lengths)?;++ if dim_base_index.len() != codebook_dim {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: codebook_dim,+ actual_tokens: dim_base_index.len(),+ });+ }++ let expected_input = checked_product(&[batch_size, num_groups * codebook_dim, seq_len])?;+ if input.len() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.len(),+ });+ }++ let output_len = checked_product(&[batch_size, num_groups, seq_len])?;+ let mut tokens = vec![0_i32; output_len];++ for batch in 0..batch_size {+ let length = lengths[batch] as usize;+ for group in 0..num_groups {+ for time in 0..seq_len {+ let output_index = (batch * num_groups + group) * seq_len + time;+ if time >= length {+ tokens[output_index] = 0;+ continue;+ }++ let mut token = 0_i32;+ for dim in 0..codebook_dim {+ let levels = num_levels[dim];+ let scale_i = levels / 2;++ let output_scale = (levels - 1) as f32 * 0.5 * (1.0 - eps);+ let output_offset = if levels % 2 == 0 {+ 0.5+ } else {+ 0.0+ };+ let input_shift = (output_offset / output_scale).tan();++ let input_index =+ (batch * (num_groups * codebook_dim) + group * codebook_dim + dim) * seq_len + time;+ let compressed = output_scale * (input[input_index] + input_shift).tanh() - output_offset;+ let rounded = round_ties_to_even(compressed);+ let code_nonnegative = (rounded as i32 + scale_i).clamp(0, levels - 1);+ token += code_nonnegative * dim_base_index[dim];+ }++ tokens[output_index] = token;+ }+ }+ }++ Ok(tokens)+}diff --git a/crates/uzu/tests/common/audio_nanocodec_ops_reference.rs b/crates/uzu/tests/common/audio_nanocodec_ops_reference.rsnew file mode 100644index 000000000..6b642dc47--- /dev/null+++ b/crates/uzu/tests/common/audio_nanocodec_ops_reference.rs@@ -0,0 +1,512 @@+use uzu::prelude::{AudioError, AudioResult};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum PadMode {+ Zeros,+ Replicate,+}++fn checked_product(values: &[usize]) -> AudioResult<usize> {+ values+ .iter()+ .try_fold(1usize, |acc, &value| acc.checked_mul(value))+ .ok_or(AudioError::Runtime("dimension product overflow".to_string()))+}++fn validate_lengths(+ lengths: &[i32],+ batch_size: usize,+ frames: usize,+) -> AudioResult<()> {+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }++ for &length in lengths {+ if length < 0 || length as usize > frames {+ return Err(AudioError::InvalidTokenLengthValue {+ length: length.max(0) as usize,+ frames,+ });+ }+ }++ Ok(())+}++pub struct Conv1dSpec<'a> {+ pub input: &'a [f32],+ pub weight: &'a [f32],+ pub bias: &'a [f32],+ pub lengths: &'a [i32],+ pub batch_size: usize,+ pub cin: usize,+ pub cout: usize,+ pub seq_len_in: usize,+ pub seq_len_out: usize,+ pub kernel_size: usize,+ pub stride: usize,+ pub dilation: usize,+ pub padding: usize,+ pub pad_mode: PadMode,+}++pub fn conv1d_reference(spec: Conv1dSpec<'_>) -> AudioResult<Vec<f32>> {+ let Conv1dSpec {+ input,+ weight,+ bias,+ lengths,+ batch_size,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ kernel_size,+ stride,+ dilation,+ padding,+ pad_mode,+ } = spec;++ validate_lengths(lengths, batch_size, seq_len_out)?;++ let expected_input = checked_product(&[batch_size, cin, seq_len_in])?;+ if input.len() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.len(),+ });+ }++ let expected_weight = checked_product(&[cout, cin, kernel_size])?;+ if weight.len() != expected_weight {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_weight,+ actual_tokens: weight.len(),+ });+ }++ if bias.len() != cout {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: cout,+ actual_tokens: bias.len(),+ });+ }++ let output_len = checked_product(&[batch_size, cout, seq_len_out])?;+ let mut output = vec![0.0_f32; output_len];++ for batch in 0..batch_size {+ let length = lengths[batch] as usize;+ for out_channel in 0..cout {+ for out_time in 0..seq_len_out {+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;+ if out_time >= length {+ output[out_index] = 0.0;+ continue;+ }++ let mut acc = bias[out_channel];+ let base = (out_time * stride) as isize - padding as isize;++ for in_channel in 0..cin {+ let input_base = (batch * cin + in_channel) * seq_len_in;+ let weight_base = (out_channel * cin + in_channel) * kernel_size;+ for kernel_offset in 0..kernel_size {+ let x_time_raw = base + (kernel_offset * dilation) as isize;+ let x_time = match pad_mode {+ PadMode::Zeros => {+ if x_time_raw < 0 || x_time_raw >= seq_len_in as isize {+ continue;+ }+ x_time_raw as usize+ },+ PadMode::Replicate => x_time_raw.clamp(0, seq_len_in as isize - 1) as usize,+ };++ let x_index = input_base + x_time;+ let w_index = weight_base + kernel_offset;+ acc += weight[w_index] * input[x_index];+ }+ }++ output[out_index] = acc;+ }+ }+ }++ Ok(output)+}++pub struct CausalConv1dSpec<'a> {+ pub input: &'a [f32],+ pub weight: &'a [f32],+ pub bias: &'a [f32],+ pub lengths: &'a [i32],+ pub batch_size: usize,+ pub cin: usize,+ pub cout: usize,+ pub seq_len: usize,+ pub kernel_size: usize,+ pub dilation: usize,+}++pub fn causal_conv1d_reference(spec: CausalConv1dSpec<'_>) -> AudioResult<Vec<f32>> {+ let CausalConv1dSpec {+ input,+ weight,+ bias,+ lengths,+ batch_size,+ cin,+ cout,+ seq_len,+ kernel_size,+ dilation,+ } = spec;++ validate_lengths(lengths, batch_size, seq_len)?;++ let expected_input = checked_product(&[batch_size, cin, seq_len])?;+ if input.len() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.len(),+ });+ }++ let expected_weight = checked_product(&[cout, cin, kernel_size])?;+ if weight.len() != expected_weight {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_weight,+ actual_tokens: weight.len(),+ });+ }++ if bias.len() != cout {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: cout,+ actual_tokens: bias.len(),+ });+ }++ let output_len = checked_product(&[batch_size, cout, seq_len])?;+ let mut output = vec![0.0_f32; output_len];+ let pad = (kernel_size - 1) * dilation;++ for batch in 0..batch_size {+ let length = lengths[batch] as usize;+ for out_channel in 0..cout {+ for time in 0..seq_len {+ let out_index = (batch * cout + out_channel) * seq_len + time;+ if time >= length {+ output[out_index] = 0.0;+ continue;+ }++ let mut acc = bias[out_channel];+ for in_channel in 0..cin {+ let input_base = (batch * cin + in_channel) * seq_len;+ let weight_base = (out_channel * cin + in_channel) * kernel_size;+ for kernel_offset in 0..kernel_size {+ let x_time = time as isize + (kernel_offset * dilation) as isize - pad as isize;+ if x_time < 0 || x_time >= seq_len as isize {+ continue;+ }++ let x_index = input_base + x_time as usize;+ let w_index = weight_base + kernel_offset;+ acc += weight[w_index] * input[x_index];+ }+ }++ output[out_index] = acc;+ }+ }+ }++ Ok(output)+}++pub struct CausalConvTranspose1dSpec<'a> {+ pub input: &'a [f32],+ pub weight: &'a [f32],+ pub bias: &'a [f32],+ pub lengths: &'a [i32],+ pub batch_size: usize,+ pub cin: usize,+ pub cout: usize,+ pub seq_len_in: usize,+ pub seq_len_out: usize,+ pub stride: usize,+ pub groups: usize,+}++pub fn causal_conv_transpose1d_reference(spec: CausalConvTranspose1dSpec<'_>) -> AudioResult<Vec<f32>> {+ let CausalConvTranspose1dSpec {+ input,+ weight,+ bias,+ lengths,+ batch_size,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ stride,+ groups,+ } = spec;++ validate_lengths(lengths, batch_size, seq_len_out)?;+ if groups == 0 || cin == 0 || cout == 0 || cin % groups != 0 || cout % groups != 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }++ let expected_input = checked_product(&[batch_size, cin, seq_len_in])?;+ if input.len() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.len(),+ });+ }++ let kernel_size = 2 * stride;+ let cin_per_group = cin / groups;+ let expected_weight = checked_product(&[cout, cin_per_group, kernel_size])?;+ if weight.len() != expected_weight {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_weight,+ actual_tokens: weight.len(),+ });+ }++ if bias.len() != cout {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: cout,+ actual_tokens: bias.len(),+ });+ }++ let output_len = checked_product(&[batch_size, cout, seq_len_out])?;+ let mut output = vec![0.0_f32; output_len];+ let cout_per_group = cout / groups;++ for batch in 0..batch_size {+ let length = lengths[batch] as usize;+ for out_channel in 0..cout {+ let group_index = out_channel / cout_per_group;+ let in_channel_begin = group_index * cin_per_group;+ let in_channel_end = in_channel_begin + cin_per_group;++ for out_time in 0..seq_len_out {+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;+ if out_time >= length {+ output[out_index] = 0.0;+ continue;+ }++ let q = out_time / stride;+ let r = out_time % stride;+ let mut acc = bias[out_channel];++ for in_channel in in_channel_begin..in_channel_end {+ if q >= seq_len_in {+ continue;+ }++ let input_base = (batch * cin + in_channel) * seq_len_in;+ let in_channel_in_group = in_channel - in_channel_begin;+ let weight_base = (out_channel * cin_per_group + in_channel_in_group) * kernel_size;++ acc += input[input_base + q] * weight[weight_base + r];+ if q > 0 {+ acc += input[input_base + (q - 1)] * weight[weight_base + (stride + r)];+ }+ }++ output[out_index] = acc;+ }+ }+ }++ Ok(output)+}++pub fn causal_conv_transpose1d_causal_pad_reference(spec: CausalConvTranspose1dSpec<'_>) -> AudioResult<Vec<f32>> {+ let CausalConvTranspose1dSpec {+ input,+ weight,+ bias,+ lengths,+ batch_size,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ stride,+ groups,+ } = spec;++ validate_lengths(lengths, batch_size, seq_len_out)?;+ if groups == 0 || stride == 0 || cin == 0 || cout == 0 || cin % groups != 0 || cout % groups != 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }++ let expected_input = checked_product(&[batch_size, cin, seq_len_in])?;+ if input.len() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.len(),+ });+ }++ let cout_per_group = cout / groups;+ let cin_per_group = cin / groups;+ let weight_plane = checked_product(&[cout, cin_per_group])?;+ if weight_plane == 0 || weight.len() % weight_plane != 0 {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: weight_plane,+ actual_tokens: weight.len(),+ });+ }++ let kernel_size = weight.len() / weight_plane;+ if kernel_size == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }++ if bias.len() != cout {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: cout,+ actual_tokens: bias.len(),+ });+ }++ let output_len = checked_product(&[batch_size, cout, seq_len_out])?;+ let mut output = vec![0.0_f32; output_len];+ let seq_len_expanded = if seq_len_in == 0 {+ 0+ } else {+ (seq_len_in - 1)+ .checked_mul(stride)+ .and_then(|value| value.checked_add(1))+ .ok_or(AudioError::Runtime("expanded sequence length overflow".to_string()))?+ };+ let left_pad = kernel_size - 1;++ for batch in 0..batch_size {+ let length = lengths[batch] as usize;+ for out_channel in 0..cout {+ let group_index = out_channel / cout_per_group;+ let in_channel_begin = group_index * cin_per_group;+ let in_channel_end = in_channel_begin + cin_per_group;++ for out_time in 0..seq_len_out {+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;+ if out_time >= length {+ output[out_index] = 0.0;+ continue;+ }++ let mut acc = bias[out_channel];++ for in_channel in in_channel_begin..in_channel_end {+ let input_base = (batch * cin + in_channel) * seq_len_in;+ let in_channel_in_group = in_channel - in_channel_begin;+ let weight_base = (out_channel * cin_per_group + in_channel_in_group) * kernel_size;++ for kernel_offset in 0..kernel_size {+ let expanded_time = out_time as isize + kernel_offset as isize - left_pad as isize;+ if expanded_time < 0 || expanded_time >= seq_len_expanded as isize {+ continue;+ }++ let expanded_time = expanded_time as usize;+ if expanded_time % stride != 0 {+ continue;+ }++ let src_time = expanded_time / stride;+ if src_time >= seq_len_in {+ continue;+ }++ acc += input[input_base + src_time] * weight[weight_base + kernel_offset];+ }+ }++ output[out_index] = acc;+ }+ }+ }++ Ok(output)+}++pub struct HalfSnakeSpec<'a> {+ pub input: &'a [f32],+ pub alpha: &'a [f32],+ pub batch_size: usize,+ pub channels: usize,+ pub seq_len: usize,+ pub snake_channels: usize,+ pub negative_slope: f32,+ pub eps: f32,+}++pub fn half_snake_reference(spec: HalfSnakeSpec<'_>) -> AudioResult<Vec<f32>> {+ let HalfSnakeSpec {+ input,+ alpha,+ batch_size,+ channels,+ seq_len,+ snake_channels,+ negative_slope,+ eps,+ } = spec;++ if snake_channels > channels {+ return Err(AudioError::InvalidTokenCardinality);+ }++ let expected_input = checked_product(&[batch_size, channels, seq_len])?;+ if input.len() != expected_input {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: expected_input,+ actual_tokens: input.len(),+ });+ }++ if alpha.len() != snake_channels {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens: snake_channels,+ actual_tokens: alpha.len(),+ });+ }++ let mut output = vec![0.0_f32; input.len()];+ for batch in 0..batch_size {+ for channel in 0..channels {+ for time in 0..seq_len {+ let index = (batch * channels + channel) * seq_len + time;+ let x = input[index];+ output[index] = if channel < snake_channels {+ let a = alpha[channel];+ let sine = (a * x).sin();+ x + (sine * sine) / (a + eps)+ } else if x >= 0.0 {+ x+ } else {+ negative_slope * x+ };+ }+ }+ }++ Ok(output)+}diff --git a/crates/uzu/tests/common/mod.rs b/crates/uzu/tests/common/mod.rsindex 6ed5da08a..ac66869ed 100644--- a/crates/uzu/tests/common/mod.rs+++ b/crates/uzu/tests/common/mod.rs@@ -1,6 +1,8 @@#![allow(dead_code)]pub mod assert;+pub mod audio_nanocodec_fsq_reference;+pub mod audio_nanocodec_ops_reference;pub mod proptest;/// Invokes `$body` once per available backend, with `$B` bound to each backend type.diff --git a/crates/uzu/tests/integration/parameters/loader_test.rs b/crates/uzu/tests/integration/parameters/loader_test.rsindex 463beae4b..a717f1a3b 100644--- a/crates/uzu/tests/integration/parameters/loader_test.rs+++ b/crates/uzu/tests/integration/parameters/loader_test.rs@@ -22,7 +22,7 @@ fn test_parameter_loader_basic() {let file = std::fs::File::open(&weights_path).expect("Weights file not found; run download script");let loader = ParameterLoader::new(&file, context.as_ref()).expect("create loader");- let embeddings = loader.get("embedding.weights").expect("weights embeddings");+ let embeddings = loader.tree().leaf_array("embedding.weights").expect("weights embeddings");let view = embeddings.as_view::<bf16>();assert!(is_close!(view[[5usize, 3usize]], bf16::from_f32(-0.01819)));diff --git a/crates/uzu/tests/unit/audio/types_test.rs b/crates/uzu/tests/unit/audio/types_test.rsindex bc66cb593..89814d78e 100644--- a/crates/uzu/tests/unit/audio/types_test.rs+++ b/crates/uzu/tests/unit/audio/types_test.rs@@ -1,7 +1,7 @@-use super::{AudioError, AudioTokenGrid, AudioTokenPacking};+use super::{AudioError, AudioTokenGrid};#[test]-fn packing_conversion_roundtrip_is_lossless() {+fn token_grid_get_reads_expected_slots() {let grid = AudioTokenGrid::new(vec![0, 1, 2, 3, // b0 f0, f1@@ -12,27 +12,19 @@ fn packing_conversion_roundtrip_is_lossless() {2,2,vec![2, 2].into_boxed_slice(),- AudioTokenPacking::FrameMajor,).expect("valid grid");- let converted = grid.to_packing(AudioTokenPacking::CodebookMajor);- let restored = converted.to_packing(AudioTokenPacking::FrameMajor);-- assert_eq!(restored, grid);+ assert_eq!(grid.get(0, 0, 0), 0);+ assert_eq!(grid.get(0, 1, 1), 3);+ assert_eq!(grid.get(1, 0, 0), 4);+ assert_eq!(grid.get(1, 1, 1), 7);}#[test]fn invalid_grid_shape_is_rejected() {- let error = AudioTokenGrid::new(- vec![0, 1, 2].into_boxed_slice(),- 1,- 2,- 2,- vec![2].into_boxed_slice(),- AudioTokenPacking::FrameMajor,- )- .expect_err("shape mismatch should fail");+ let error = AudioTokenGrid::new(vec![0, 1, 2].into_boxed_slice(), 1, 2, 2, vec![2].into_boxed_slice())+ .expect_err("shape mismatch should fail");assert!(matches!(error,diff --git a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rsindex a9955faed..34f9dcc9e 100644--- a/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs+++ b/crates/uzu/tests/unit/backends/common/kernel/ssd_prefill_test.rs@@ -343,7 +343,7 @@ fn run_conv_scan_once(if let Some(ref scratch) = scratch_buf {let bytes = channels * tap_count * size_of::<f32>();if bytes > 0 {- encoder.encode_copy(scratch, &mut state_buf, bytes);+ encoder.encode_copy(scratch, 0..bytes, &mut state_buf, 0..bytes);}}diff --git a/crates/uzu/tests/unit/encodable_block/attention_test.rs b/crates/uzu/tests/unit/encodable_block/attention_test.rsindex 9f070d94e..d60d905be 100644--- a/crates/uzu/tests/unit/encodable_block/attention_test.rs+++ b/crates/uzu/tests/unit/encodable_block/attention_test.rs@@ -1198,15 +1198,15 @@ fn perf_two_pass_attention() {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time() {- Some(gpu_time) => {+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {+ Some(gpu_time_ms) => {println!("Two-pass attention perf (heads={}, prefix={}, suffix={}, head_dim={}): GPU={:.2} ms, Host-side={:.2} ms",num_heads,seq_len - suffix_length,suffix_length,head_dim,- gpu_time.as_secs_f64() * 1e3,+ gpu_time_ms,host_elapsed_ms);},diff --git a/crates/uzu/tests/unit/encodable_block/sampling_test.rs b/crates/uzu/tests/unit/encodable_block/sampling_test.rsindex 329d2fbb3..952f4dfdc 100644--- a/crates/uzu/tests/unit/encodable_block/sampling_test.rs+++ b/crates/uzu/tests/unit/encodable_block/sampling_test.rs@@ -17,7 +17,7 @@ use crate::{},metal::Metal,},- session::parameter::SamplingMethod,+ session::parameter::{SamplingMethod, SamplingProcessingOrder},};// Constant seed for reproducible test results@@ -187,15 +187,11 @@ fn perf_argmax_128k_vocab_with_strategy(strategy: ArgmaxStrategy) {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time() {- Some(gpu_time) => {+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {+ Some(gpu_time_ms) => {println!("Argmax sampling perf (batch={}, vocab={}, strategy={:?}): GPU={:.2} ms, Host-side={:.2} ms",- BATCH,- VOCAB,- strategy,- gpu_time.as_secs_f64() * 1e3,- host_elapsed_ms+ BATCH, VOCAB, strategy, gpu_time_ms, host_elapsed_ms);},None => {@@ -296,6 +292,7 @@ fn test_categorical_sampling() {top_k: None,top_p: None,min_p: None,+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},batch_size,vocab_size,@@ -432,6 +429,7 @@ fn test_categorical_sampling_statistical() {top_k: None,top_p: None,min_p: None,+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},BATCH,VOCAB,@@ -533,6 +531,7 @@ fn perf_categorical_128k_vocab() {top_k: None,top_p: None,min_p: None,+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},BATCH,VOCAB,@@ -544,14 +543,11 @@ fn perf_categorical_128k_vocab() {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time() {- Some(gpu_time) => {+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {+ Some(gpu_time_ms) => {println!("Categorical sampling perf (batch={}, vocab={}): GPU={:.2} ms, Host-side={:.2} ms",- BATCH,- VOCAB,- gpu_time.as_secs_f64() * 1e3,- host_elapsed_ms+ BATCH, VOCAB, gpu_time_ms, host_elapsed_ms);},None => {@@ -760,6 +756,7 @@ fn test_minp_sampling_exact_match(top_k: None,top_p: None,min_p: Some(min_p),+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},batch_size,vocab_size,diff --git a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rsindex bda04a3e9..b52980d71 100644--- a/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs+++ b/crates/uzu/tests/unit/forward_pass/kv_cache_state_test.rs@@ -1,8 +1,7 @@#![cfg(metal_backend)]use crate::{- DataType,- array::ArrayContextExt,+ ArrayContextExt, DataType,backends::{common::{Backend, Context, Encoder, kernel::kv_cache_update::KVCacheUpdate},metal::Metal,diff --git a/crates/uzu/tests/unit/kernel/audio/audio_test.rs b/crates/uzu/tests/unit/kernel/audio/audio_test.rsindex 48aff9f65..fc876cf92 100644--- a/crates/uzu/tests/unit/kernel/audio/audio_test.rs+++ b/crates/uzu/tests/unit/kernel/audio/audio_test.rs@@ -1,29 +1,57 @@#![cfg(all(feature = "audio-runtime", metal_backend))]-use std::{- ops::{Deref, DerefMut},- rc::Rc,-};-use uzu::{ArrayContextExt, DataType,backends::{common::{Backend, Context, Encoder, Kernels,+ gpu_types::ActivationType,kernel::{- AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,- AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,- AudioLeakyReluKernel, AudioScaleKernel, AudioTanhKernel,+ ActivationKernel, AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dCausalPadKernel,+ AudioCausalConvTranspose1dKernel, AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel,+ AudioHalfSnakeKernel, AudioNormNcsKernel,},},metal::Metal,},};-fn create_test_context() -> Rc<<Metal as Backend>::Context> {+use super::super::common::{+ audio_nanocodec_fsq_reference::{fsq_decode_reference, fsq_encode_reference},+ audio_nanocodec_ops_reference::{+ CausalConv1dSpec, CausalConvTranspose1dSpec, Conv1dSpec, HalfSnakeSpec, PadMode,+ causal_conv_transpose1d_causal_pad_reference, causal_conv_transpose1d_reference, causal_conv1d_reference,+ conv1d_reference, half_snake_reference,+ },+};++macro_rules! borrow_array_buffer {+ ($name:ident = $array:expr) => {+ let $name = $array.buffer();+ let $name = $name.borrow();+ };+}++macro_rules! borrow_array_buffer_mut {+ ($name:ident = $array:expr) => {+ let $name = $array.buffer();+ let mut $name = $name.borrow_mut();+ };+}++fn create_test_context() -> std::rc::Rc<<Metal as Backend>::Context> {<Metal as Backend>::Context::new().expect("MetalContext")}+fn run_command_buffer(+ context: &std::rc::Rc<<Metal as Backend>::Context>,+ encode: impl FnOnce(&mut Encoder<'_, Metal>),+) {+ let mut encoder = Encoder::<Metal>::new(context.as_ref()).expect("command buffer");+ encode(&mut encoder);+ encoder.end_encoding().submit().wait_until_completed().expect("command buffer completed");+}+#[test]fn audio_conv1d_replicate_matches_reference_f32() {let context = create_test_context();@@ -65,62 +93,49 @@ fn audio_conv1d_replicate_matches_reference_f32() {lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);let output = context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");-- let mut expected = vec![0.0f32; output_len];- for batch in 0..batch_size {- for out_channel in 0..cout {- for out_time in 0..seq_len_out {- let out_index = (batch * cout + out_channel) * seq_len_out + out_time;- if (out_time as i32) >= lengths_out[batch] {- expected[out_index] = 0.0;- continue;- }-- let mut acc = bias_values[out_channel];- let base = (out_time * stride) as isize - padding as isize;-- for in_channel in 0..cin {- let x_base = (batch * cin + in_channel) * seq_len_in;- let w_base = (out_channel * cin + in_channel) * kernel_size;- for kernel_offset in 0..kernel_size {- let mut x_time = base + (kernel_offset * dilation) as isize;- if x_time < 0 {- x_time = 0;- } else if x_time >= seq_len_in as isize {- x_time = seq_len_in as isize - 1;- }-- let x_index = x_base + x_time as usize;- let w_index = w_base + kernel_offset;- acc += weight_values[w_index] * input_values[x_index];- }- }-- expected[out_index] = acc;- }- }- }-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input.buffer().borrow().deref(),- weight.buffer().borrow().deref(),- bias.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- lengths.buffer().borrow().deref(),- cin as i32,- cout as i32,- seq_len_in as i32,- seq_len_out as i32,- kernel_size as i32,- stride as i32,- dilation as i32,- padding as i32,- 1_i32,- batch_size as i32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ let expected = conv1d_reference(Conv1dSpec {+ input: &input_values,+ weight: &weight_values,+ bias: &bias_values,+ lengths: &lengths_out,+ batch_size,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ kernel_size,+ stride,+ dilation,+ padding,+ pad_mode: PadMode::Replicate,+ })+ .expect("reference conv1d");++ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer!(weight_buffer = weight);+ borrow_array_buffer!(bias_buffer = bias);+ borrow_array_buffer_mut!(output_buffer = output);+ borrow_array_buffer!(lengths_buffer = lengths);+ kernel.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin as i32,+ cout as i32,+ seq_len_in as i32,+ seq_len_out as i32,+ kernel_size as i32,+ stride as i32,+ dilation as i32,+ padding as i32,+ 1_i32,+ batch_size as i32,+ command_buffer,+ );+ });let got = output.as_slice::<f32>();for index in 0..output_len {@@ -146,7 +161,6 @@ fn audio_causal_conv1d_matches_reference_f32() {let seq_len = 8usize;let kernel_size = 5usize;let dilation = 2usize;- let pad = (kernel_size - 1) * dilation;let lengths: [i32; 2] = [5, 8];@@ -172,52 +186,42 @@ fn audio_causal_conv1d_matches_reference_f32() {lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);let output = context.create_array(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");-- let mut expected = vec![0.0f32; output_len];- for batch in 0..batch_size {- for out_channel in 0..cout {- for time in 0..seq_len {- let out_index = (batch * cout + out_channel) * seq_len + time;- if (time as i32) >= lengths[batch] {- expected[out_index] = 0.0;- continue;- }-- let mut acc = bias_values[out_channel];- for in_channel in 0..cin {- for kernel_offset in 0..kernel_size {- let x_time = time as isize + (kernel_offset * dilation) as isize - pad as isize;- if x_time < 0 || x_time >= seq_len as isize {- continue;- }-- let x_index = (batch * cin + in_channel) * seq_len + x_time as usize;- let w_index = (out_channel * cin + in_channel) * kernel_size + kernel_offset;- acc += weight_values[w_index] * input_values[x_index];- }- }-- expected[out_index] = acc;- }- }- }-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input.buffer().borrow().deref(),- weight.buffer().borrow().deref(),- bias.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- lengths_array.buffer().borrow().deref(),- cin as i32,- cout as i32,- seq_len as i32,- kernel_size as i32,- dilation as i32,- batch_size as i32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ let expected = causal_conv1d_reference(CausalConv1dSpec {+ input: &input_values,+ weight: &weight_values,+ bias: &bias_values,+ lengths: &lengths,+ batch_size,+ cin,+ cout,+ seq_len,+ kernel_size,+ dilation,+ })+ .expect("reference causal conv1d");++ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer!(weight_buffer = weight);+ borrow_array_buffer!(bias_buffer = bias);+ borrow_array_buffer_mut!(output_buffer = output);+ borrow_array_buffer!(lengths_buffer = lengths_array);+ kernel.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin as i32,+ cout as i32,+ seq_len as i32,+ kernel_size as i32,+ dilation as i32,+ 0,+ batch_size as i32,+ command_buffer,+ );+ });let got = output.as_slice::<f32>();for index in 0..output_len {@@ -251,7 +255,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {let lengths_out: [i32; 2] = [9, 15];let input_len = batch_size * cin * seq_len_in;- let weight_len = cin * (cout / groups) * kernel_size;+ let weight_len = cout * (cin / groups) * kernel_size;let output_len = batch_size * cout * seq_len_out;let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.013).sin() * 0.5).collect();@@ -263,7 +267,7 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {input.as_slice_mut::<f32>().copy_from_slice(&input_values);let mut weight =- context.create_array(&[cin, cout / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");+ context.create_array(&[cout, cin / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");@@ -274,98 +278,208 @@ fn audio_causal_conv_transpose1d_matches_reference_f32() {let output =context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_causal_conv_transpose_output");+ let expected = causal_conv_transpose1d_reference(CausalConvTranspose1dSpec {+ input: &input_values,+ weight: &weight_values,+ bias: &bias_values,+ lengths: &lengths_out,+ batch_size,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ stride,+ groups,+ })+ .expect("reference causal conv transpose1d");++ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer!(weight_buffer = weight);+ borrow_array_buffer!(bias_buffer = bias);+ borrow_array_buffer_mut!(output_buffer = output);+ borrow_array_buffer!(lengths_buffer = lengths);+ kernel.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin as i32,+ cout as i32,+ seq_len_in as i32,+ seq_len_out as i32,+ stride as i32,+ groups as i32,+ batch_size as i32,+ command_buffer,+ );+ });- let cout_per_group = cout / groups;- let cin_per_group = cin / groups;- let mut expected = vec![0.0f32; output_len];-- for batch in 0..batch_size {- for out_channel in 0..cout {- let group_index = out_channel / cout_per_group;- let out_channel_in_group = out_channel % cout_per_group;- let in_channel_begin = group_index * cin_per_group;- let in_channel_end = in_channel_begin + cin_per_group;-- for out_time in 0..seq_len_out {- let out_index = (batch * cout + out_channel) * seq_len_out + out_time;- if (out_time as i32) >= lengths_out[batch] {- expected[out_index] = 0.0;- continue;- }+ let got = output.as_slice::<f32>();+ for index in 0..output_len {+ let delta = (expected[index] - got[index]).abs();+ assert!(+ delta <= 1e-4,+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",+ expected[index],+ got[index]+ );+ }+}- let q = out_time / stride;- let r = out_time % stride;- let mut acc = bias_values[out_channel];+#[test]+fn audio_causal_conv_transpose1d_causal_pad_matches_reference_f32() {+ let context = create_test_context();+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioCausalConvTranspose1dCausalPadKernel::new(+ &context,+ DataType::F32,+ )+ .expect("audio runtime");- for in_channel in in_channel_begin..in_channel_end {- let input_base = (batch * cin + in_channel) * seq_len_in;- let weight_base = (in_channel * cout_per_group + out_channel_in_group) * kernel_size;+ let batch_size = 1usize;+ let cin = 2usize;+ let groups = 1usize;+ let cout = 2usize;+ let stride = 2usize;+ let kernel_size = 3usize;- acc += input_values[input_base + q] * weight_values[weight_base + r];- if q > 0 {- acc += input_values[input_base + (q - 1)] * weight_values[weight_base + (stride + r)];- }- }+ let seq_len_in = 5usize;+ let seq_len_out = 10usize;+ let lengths_out: [i32; 1] = [10];++ let input_len = batch_size * cin * seq_len_in;+ let weight_len = cout * (cin / groups) * kernel_size;+ let output_len = batch_size * cout * seq_len_out;- expected[out_index] = acc;+ let input_values_ncs: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.017).sin() * 0.5).collect();+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.09).cos() * 0.2).collect();+ let bias_values: Vec<f32> = vec![0.01, -0.02];+ let expected = causal_conv_transpose1d_causal_pad_reference(CausalConvTranspose1dSpec {+ input: &input_values_ncs,+ weight: &weight_values,+ bias: &bias_values,+ lengths: &lengths_out,+ batch_size,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ stride,+ groups,+ })+ .expect("reference causal conv transpose lalamo");++ let mut input_values_nsc = vec![0.0_f32; input_len];+ for b in 0..batch_size {+ for c in 0..cin {+ for t in 0..seq_len_in {+ let src_index = (b * cin + c) * seq_len_in + t;+ let dst_index = (b * seq_len_in + t) * cin + c;+ input_values_nsc[dst_index] = input_values_ncs[src_index];}}}- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input.buffer().borrow().deref(),- weight.buffer().borrow().deref(),- bias.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- lengths.buffer().borrow().deref(),- cin as i32,- cout as i32,- seq_len_in as i32,- seq_len_out as i32,- stride as i32,- groups as i32,- batch_size as i32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ for (layout_name, layout_code, input_shape, input_values) in [+ ("NCS", 0_i32, [batch_size, cin, seq_len_in], input_values_ncs.as_slice()),+ ("NSC", 1_i32, [batch_size, seq_len_in, cin], input_values_nsc.as_slice()),+ ] {+ let mut input = context.create_array(&input_shape, DataType::F32, "audio_causal_conv_transpose_lalamo_input");+ input.as_slice_mut::<f32>().copy_from_slice(input_values);++ let mut weight = context.create_array(+ &[cout, cin / groups, kernel_size],+ DataType::F32,+ "audio_causal_conv_transpose_lalamo_weight",+ );+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);- let got = output.as_slice::<f32>();- for index in 0..output_len {- let delta = (expected[index] - got[index]).abs();- assert!(- delta <= 1e-4,- "Mismatch at index={index}: expected {}, got {}, delta={delta}",- expected[index],- got[index]+ let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_lalamo_bias");+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);++ let mut lengths =+ context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lalamo_lengths");+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);++ let output = context.create_array(+ &[batch_size, cout, seq_len_out],+ DataType::F32,+ "audio_causal_conv_transpose_lalamo_output",);++ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer!(weight_buffer = weight);+ borrow_array_buffer!(bias_buffer = bias);+ borrow_array_buffer_mut!(output_buffer = output);+ borrow_array_buffer!(lengths_buffer = lengths);+ kernel.encode(+ &*input_buffer,+ &*weight_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ cin as i32,+ cout as i32,+ seq_len_in as i32,+ seq_len_out as i32,+ kernel_size as i32,+ stride as i32,+ groups as i32,+ layout_code,+ batch_size as i32,+ command_buffer,+ );+ });++ let got = output.as_slice::<f32>();+ for index in 0..output_len {+ let delta = (expected[index] - got[index]).abs();+ assert!(+ delta <= 1e-4,+ "layout={layout_name}, index={index}: expected {}, got {}, delta={delta}",+ expected[index],+ got[index]+ );+ }}}#[test]fn audio_leaky_relu_matches_reference_f32() {let context = create_test_context();- let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioLeakyReluKernel::new(&context, DataType::F32)+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioHalfSnakeKernel::new(&context, DataType::F32).expect("audio runtime");let n = 1024usize;- let slope = 0.01f32;+ let slope = 0.0f32;+ let eps = 1e-6f32;let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();- let mut input = context.create_array(&[n], DataType::F32, "audio_leaky_relu_input");+ let mut input = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_input");input.as_slice_mut::<f32>().copy_from_slice(&input_values);- let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- n as i32,- slope,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ let alpha = context.create_array(&[1], DataType::F32, "audio_leaky_relu_alpha");+ let output = context.create_array(&[1, 1, n], DataType::F32, "audio_leaky_relu_output");++ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer!(alpha_buffer = alpha);+ borrow_array_buffer_mut!(output_buffer = output);+ kernel.encode(+ &*input_buffer,+ &*alpha_buffer,+ &mut *output_buffer,+ 1,+ n as i32,+ 0,+ slope,+ eps,+ 1,+ command_buffer,+ );+ });let got = output.as_slice::<f32>();for index in 0..n {@@ -382,8 +496,8 @@ fn audio_leaky_relu_matches_reference_f32() {#[test]fn audio_tanh_matches_reference_f32() {let context = create_test_context();- let kernel =- <<Metal as Backend>::Kernels as Kernels>::AudioTanhKernel::new(&context, DataType::F32).expect("audio runtime");+ let kernel = <<Metal as Backend>::Kernels as Kernels>::ActivationKernel::new(&context, DataType::F32, false)+ .expect("audio runtime");let n = 1024usize;let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();@@ -393,9 +507,11 @@ fn audio_tanh_matches_reference_f32() {let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(input.buffer().borrow().deref(), output.buffer().borrow_mut().deref_mut(), n as i32, &mut encoder);- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer_mut!(output_buffer = output);+ kernel.encode(Some(&*input_buffer), &mut *output_buffer, n as u32, ActivationType::TANH, command_buffer);+ });let got = output.as_slice::<f32>();for index in 0..n {@@ -406,15 +522,12 @@ fn audio_tanh_matches_reference_f32() {}#[test]-fn audio_add_and_scale_match_reference_f32() {+fn audio_add_matches_reference_f32() {let context = create_test_context();let add_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioAddKernel::new(&context, DataType::F32).expect("audio add kernel");- let scale_kernel = <<Metal as Backend>::Kernels as Kernels>::AudioScaleKernel::new(&context, DataType::F32)- .expect("audio scale kernel");let n = 2048usize;- let scale_value = 1.0f32 / 3.0f32;let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();@@ -425,80 +538,25 @@ fn audio_add_and_scale_match_reference_f32() {b.as_slice_mut::<f32>().copy_from_slice(&b_values);let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");- let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- add_kernel.encode(- a.buffer().borrow().deref(),- b.buffer().borrow().deref(),- sum.buffer().borrow_mut().deref_mut(),- n as i32,- &mut encoder,- );- scale_kernel.encode(- sum.buffer().borrow().deref(),- scaled.buffer().borrow_mut().deref_mut(),- n as i32,- scale_value,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");++ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(a_buffer = a);+ borrow_array_buffer!(b_buffer = b);+ borrow_array_buffer_mut!(sum_buffer = sum);+ add_kernel.encode(&*a_buffer, &*b_buffer, &mut *sum_buffer, n as i32, command_buffer);+ });let sum_got = sum.as_slice::<f32>();- let scaled_got = scaled.as_slice::<f32>();for index in 0..n {let expected_sum = a_values[index] + b_values[index];- let expected_scaled = expected_sum * scale_value;-let sum_delta = (expected_sum - sum_got[index]).abs();- let scaled_delta = (expected_scaled - scaled_got[index]).abs();assert!(sum_delta <= 1e-6,"Add mismatch at index={index}: expected {expected_sum}, got {}, delta={sum_delta}",sum_got[index]);- assert!(- scaled_delta <= 1e-6,- "Scale mismatch at index={index}: expected {expected_scaled}, got {}, delta={scaled_delta}",- scaled_got[index]- );- }-}--#[test]-fn audio_clamp_matches_reference_f32() {- let context = create_test_context();- let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioClampKernel::new(&context, DataType::F32)- .expect("audio runtime");-- let n = 2048usize;- let min_value = -1.0f32;- let max_value = 1.0f32;- let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 10.24).collect();-- let mut input = context.create_array(&[n], DataType::F32, "audio_clamp_input");- input.as_slice_mut::<f32>().copy_from_slice(&input_values);-- let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");-- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- n as i32,- min_value,- max_value,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");-- let got = output.as_slice::<f32>();- for index in 0..n {- let expected = input_values[index].clamp(min_value, max_value);- let delta = (expected - got[index]).abs();- assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);}}@@ -528,45 +586,153 @@ fn audio_half_snake_matches_reference_f32() {let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input.buffer().borrow().deref(),- alpha.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- channels as i32,- seq_len as i32,- snake_channels as i32,+ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer!(alpha_buffer = alpha);+ borrow_array_buffer_mut!(output_buffer = output);+ kernel.encode(+ &*input_buffer,+ &*alpha_buffer,+ &mut *output_buffer,+ channels as i32,+ seq_len as i32,+ snake_channels as i32,+ negative_slope,+ eps,+ batch_size as i32,+ command_buffer,+ );+ });++ let got = output.as_slice::<f32>();+ let expected = half_snake_reference(HalfSnakeSpec {+ input: &input_values,+ alpha: &alpha_values,+ batch_size,+ channels,+ seq_len,+ snake_channels,negative_slope,eps,- batch_size as i32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ })+ .expect("reference half snake");+ for index in 0..n {+ let delta = (expected[index] - got[index]).abs();+ assert!(+ delta <= 1e-5,+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",+ expected[index],+ got[index]+ );+ }+}- let got = output.as_slice::<f32>();- for batch in 0..batch_size {- for channel in 0..channels {- for time in 0..seq_len {- let index = (batch * channels + channel) * seq_len + time;- let x = input_values[index];- let expected = if channel < snake_channels {- let alpha = alpha_values[channel];- let sine = (alpha * x).sin();- x + (sine * sine) / (alpha + eps)- } else if x >= 0.0 {- x- } else {- negative_slope * x- };-- let delta = (expected - got[index]).abs();- assert!(- delta <= 1e-5,- "Mismatch at index={index}: expected {expected}, got {}, delta={delta}",- got[index]- );+#[test]+fn audio_norm_ncs_matches_reference_f32() {+ let context = create_test_context();+ let kernel = <<Metal as Backend>::Kernels as Kernels>::AudioNormNcsKernel::new(&context, DataType::F32)+ .expect("audio runtime");++ let batch_size = 2usize;+ let channels = 7usize;+ let seq_len = 11usize;+ let lengths_values: [i32; 2] = [8, 11];++ let input_len = batch_size * channels * seq_len;+ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.013 - 3.0).sin() * 0.7).collect();+ let scales_values: Vec<f32> = (0..channels).map(|i| 0.7 + i as f32 * 0.05).collect();+ let bias_values: Vec<f32> = (0..channels).map(|i| -0.2 + i as f32 * 0.03).collect();+ let epsilon = 1e-5f32;++ let reference = |subtract_mean: bool| -> Vec<f32> {+ let mut out = vec![0.0_f32; input_values.len()];+ for batch in 0..batch_size {+ let active_len = lengths_values[batch].max(0) as usize;+ for t in 0..active_len {+ let mut mean = 0.0_f32;+ if subtract_mean {+ for c in 0..channels {+ let idx = (batch * channels + c) * seq_len + t;+ mean += input_values[idx];+ }+ mean /= channels as f32;+ }+ let mut variance_sum = 0.0_f32;+ for c in 0..channels {+ let idx = (batch * channels + c) * seq_len + t;+ let centered = if subtract_mean {+ input_values[idx] - mean+ } else {+ input_values[idx]+ };+ variance_sum += centered * centered;+ }+ let inv_std = 1.0_f32 / (variance_sum / channels as f32 + epsilon).sqrt();+ for c in 0..channels {+ let idx = (batch * channels + c) * seq_len + t;+ let centered = if subtract_mean {+ input_values[idx] - mean+ } else {+ input_values[idx]+ };+ out[idx] = centered * inv_std * scales_values[c] + bias_values[c];+ }}}+ out+ };++ for subtract_mean in [false, true] {+ let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let mut scales = context.create_array(&[channels], DataType::F32, "audio_norm_scales");+ scales.as_slice_mut::<f32>().copy_from_slice(&scales_values);++ let mut bias = context.create_array(&[channels], DataType::F32, "audio_norm_bias");+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);++ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_norm_lengths");+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);++ let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_norm_output");++ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer!(scales_buffer = scales);+ borrow_array_buffer!(bias_buffer = bias);+ borrow_array_buffer_mut!(output_buffer = output);+ borrow_array_buffer!(lengths_buffer = lengths);+ kernel.encode(+ &*input_buffer,+ &*scales_buffer,+ &*bias_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ channels as i32,+ seq_len as i32,+ epsilon,+ if subtract_mean {+ 1+ } else {+ 0+ },+ batch_size as i32,+ command_buffer,+ );+ });++ let got = output.as_slice::<f32>();+ let expected = reference(subtract_mean);+ for index in 0..input_len {+ let delta = (expected[index] - got[index]).abs();+ assert!(+ delta <= 1e-4,+ "subtract_mean={subtract_mean}, index={index}: expected {}, got {}, delta={delta}",+ expected[index],+ got[index]+ );+ }}}@@ -581,6 +747,7 @@ fn audio_fsq_decode_matches_reference() {let seq_len = 3usize;let codebook_dim = 2usize;let num_levels: Vec<i32> = vec![8, 5];+ let dim_base_index: Vec<i32> = vec![1, 8];let mut tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");tokens.as_slice_mut::<i32>().copy_from_slice(&[@@ -594,22 +761,27 @@ fn audio_fsq_decode_matches_reference() {let output =context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- tokens.buffer().borrow().deref(),- output.buffer().borrow_mut().deref_mut(),- lengths.buffer().borrow().deref(),- num_groups as i32,- seq_len as i32,- codebook_dim as i32,- &num_levels,- batch_size as i32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(tokens_buffer = tokens);+ borrow_array_buffer_mut!(output_buffer = output);+ borrow_array_buffer!(lengths_buffer = lengths);+ kernel.encode(+ &*tokens_buffer,+ &mut *output_buffer,+ &*lengths_buffer,+ num_groups as i32,+ seq_len as i32,+ codebook_dim as i32,+ &num_levels,+ &dim_base_index,+ batch_size as i32,+ command_buffer,+ );+ });let expected =- reference_fsq_decode(&[0, 7, 11, 3, 5, 9], 2, batch_size, num_groups, seq_len, codebook_dim, &num_levels);+ fsq_decode_reference(&[0, 7, 11, 3, 5, 9], &[2], batch_size, num_groups, seq_len, codebook_dim, &num_levels)+ .expect("reference decode");let actual = output.as_slice::<f32>();for (index, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() {@@ -652,23 +824,26 @@ fn audio_fsq_encode_matches_reference() {let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");- let mut encoder = Encoder::new(context.as_ref()).expect("Failed to create encoder");- kernel.encode(- input.buffer().borrow().deref(),- tokens.buffer().borrow_mut().deref_mut(),- lengths.buffer().borrow().deref(),- num_groups as i32,- seq_len as i32,- codebook_dim as i32,- &num_levels,- &dim_base_index,- eps,- batch_size as i32,- &mut encoder,- );- encoder.end_encoding().submit().wait_until_completed().expect("Failed to wait command buffer");+ run_command_buffer(&context, |command_buffer| {+ borrow_array_buffer!(input_buffer = input);+ borrow_array_buffer_mut!(tokens_buffer = tokens);+ borrow_array_buffer!(lengths_buffer = lengths);+ kernel.encode(+ &*input_buffer,+ &mut *tokens_buffer,+ &*lengths_buffer,+ num_groups as i32,+ seq_len as i32,+ codebook_dim as i32,+ &num_levels,+ &dim_base_index,+ eps,+ batch_size as i32,+ command_buffer,+ );+ });- let expected = reference_fsq_encode(+ let expected = fsq_encode_reference(&input_values,&lengths_values,batch_size,@@ -678,118 +853,9 @@ fn audio_fsq_encode_matches_reference() {&num_levels,&dim_base_index,eps,- );+ )+ .expect("reference encode");let actual = tokens.as_slice::<i32>();assert_eq!(actual, expected);}--fn round_ties_to_even(value: f32) -> f32 {- let floor = value.floor();- let fraction = value - floor;- if fraction < 0.5 {- floor- } else if fraction > 0.5 {- floor + 1.0- } else if (floor as i64 & 1) != 0 {- floor + 1.0- } else {- floor- }-}--fn reference_fsq_encode(- input: &[f32],- lengths: &[i32],- batch_size: usize,- num_groups: usize,- seq_len: usize,- codebook_dim: usize,- num_levels: &[i32],- dim_base_index: &[i32],- eps: f32,-) -> Vec<i32> {- let mut tokens = vec![0_i32; batch_size * num_groups * seq_len];-- for batch in 0..batch_size {- for group in 0..num_groups {- for time in 0..seq_len {- let output_index = (batch * num_groups + group) * seq_len + time;- if (time as i32) >= lengths[batch] {- tokens[output_index] = 0;- continue;- }-- let mut token = 0_i32;- for dim in 0..codebook_dim {- let levels = num_levels[dim];- let scale_i = levels / 2;- let output_scale = (levels - 1) as f32 * 0.5 * (1.0 - eps);- let output_offset = if levels % 2 == 0 {- 0.5- } else {- 0.0- };- let input_shift = (output_offset / output_scale).tan();-- let input_index =- (batch * (num_groups * codebook_dim) + group * codebook_dim + dim) * seq_len + time;- let compressed = output_scale * (input[input_index] + input_shift).tanh() - output_offset;- let rounded = round_ties_to_even(compressed);- let code_nonnegative = (rounded as i32 + scale_i).clamp(0, levels - 1);- token += code_nonnegative * dim_base_index[dim];- }-- tokens[output_index] = token;- }- }- }-- tokens-}--fn reference_fsq_decode(- tokens: &[i32],- length: usize,- batch_size: usize,- num_groups: usize,- seq_len: usize,- codebook_dim: usize,- num_levels: &[i32],-) -> Vec<f32> {- let mut out = vec![0.0f32; batch_size * num_groups * codebook_dim * seq_len];-- for b in 0..batch_size {- for g in 0..num_groups {- for t in 0..seq_len {- if t >= length {- continue;- }-- let token_index = (b * num_groups + g) * seq_len + t;- let token = tokens[token_index];-- let mut base = 1i32;- for d in 0..codebook_dim {- let levels = num_levels[d];- let scale = levels / 2;- let offset = scale;- let div = token / base;- let mut code_nonneg = div % levels;- if code_nonneg < 0 {- code_nonneg += levels;- }- let value = (code_nonneg - offset) as f32 / scale as f32;-- let out_channel = g * codebook_dim + d;- let out_index = (b * (num_groups * codebook_dim) + out_channel) * seq_len + t;- out[out_index] = value;-- base *= levels;- }- }- }- }-- out-}diff --git a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rsindex 0e6a47fd5..b35d4facb 100644--- a/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs+++ b/crates/uzu/tests/unit/kernel/sampling/top_p_test.rs@@ -16,7 +16,7 @@ use uzu::{},cpu::Cpu,},- session::parameter::SamplingMethod,+ session::parameter::{SamplingMethod, SamplingProcessingOrder},};const TEST_SAMPLING_SEED: u64 = 42;@@ -293,6 +293,7 @@ fn test_topp_sampling_from_prob_exact_match_internal<B: Backend>(top_k: None,top_p: Some(p),min_p: None,+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},batch_size,vocab_size,@@ -397,6 +398,7 @@ fn test_topp_sampling_statistical_large() {top_k: None,top_p: Some(TOP_P),min_p: None,+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},BATCH,VOCAB,@@ -473,6 +475,7 @@ fn perf_topp_128k_vocab() {top_k: None,top_p: Some(TOP_P),min_p: None,+ processing_order: SamplingProcessingOrder::TemperatureThenFilters,},BATCH,VOCAB,@@ -484,14 +487,14 @@ fn perf_topp_128k_vocab() {let completed = encoder.end_encoding().submit().wait_until_completed().unwrap();let host_elapsed_ms = host_timer.elapsed().as_secs_f64() * 1e3;- match completed.gpu_execution_time() {- Some(gpu_time) => {+ match completed.gpu_execution_time().map(|d| d.as_secs_f64() * 1e3) {+ Some(gpu_time_ms) => {println!("Top-p sampling perf (batch={}, vocab={}, backend={}): GPU={:.2} ms, Host-side={:.2} ms",BATCH,VOCAB,std::any::type_name::<B>(),- gpu_time.as_secs_f64() * 1e3,+ gpu_time_ms,host_elapsed_ms);},