A high-performance inference engine for AI models
Audio Сodec bits (#211)
LuckyIYIdfacd688060a
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.tomlindex 89cfb7d9a..0fc8e792e 100644--- a/crates/uzu/Cargo.toml+++ b/crates/uzu/Cargo.toml@@ -78,6 +78,7 @@ metal = [metal-nax = []tracing = []+audio-runtime = [][[test]]name = "moe"diff --git a/crates/uzu/src/audio/codec.rs b/crates/uzu/src/audio/codec.rsnew file mode 100644index 000000000..e973653ed--- /dev/null+++ b/crates/uzu/src/audio/codec.rs@@ -0,0 +1,13 @@+use super::{AudioPcmBatch, AudioResult, AudioTokenGrid};++pub trait AudioCodecRuntime: Send + Sync {+ fn encode(+ &self,+ pcm: &AudioPcmBatch,+ ) -> AudioResult<AudioTokenGrid>;++ fn decode(+ &self,+ tokens: &AudioTokenGrid,+ ) -> AudioResult<AudioPcmBatch>;+}diff --git a/crates/uzu/src/audio/mod.rs b/crates/uzu/src/audio/mod.rsnew file mode 100644index 000000000..9b68e6d6d--- /dev/null+++ b/crates/uzu/src/audio/mod.rs@@ -0,0 +1,5 @@+mod codec;+mod types;++pub use codec::AudioCodecRuntime;+pub use types::{AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid, AudioTokenPacking};diff --git a/crates/uzu/src/audio/types.rs b/crates/uzu/src/audio/types.rsnew file mode 100644index 000000000..60364a049--- /dev/null+++ b/crates/uzu/src/audio/types.rs@@ -0,0 +1,330 @@+use std::num::{NonZeroU32, NonZeroUsize};++#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)]+pub enum AudioError {+ #[error("sample_rate must be > 0")]+ InvalidSampleRate,+ #[error("channels must be > 0")]+ InvalidChannelCount,+ #[error("pcm samples shape mismatch: expected {expected_samples} samples, got {actual_samples}")]+ InvalidPcmShape {+ expected_samples: usize,+ actual_samples: usize,+ },+ #[error("token lengths shape mismatch: expected {expected_lengths} lengths, got {actual_lengths}")]+ InvalidTokenLengths {+ expected_lengths: usize,+ actual_lengths: usize,+ },+ #[error("token grid shape mismatch: expected {expected_tokens} tokens, got {actual_tokens}")]+ InvalidTokenShape {+ expected_tokens: usize,+ actual_tokens: usize,+ },+ #[error("token length {length} exceeds frame count {frames}")]+ InvalidTokenLengthValue {+ length: usize,+ frames: usize,+ },+ #[error("audio token cardinality must be > 0")]+ InvalidTokenCardinality,+ #[error("codec token {token} is outside codec range 0..{cardinality}")]+ InvalidCodecToken {+ token: u32,+ cardinality: u32,+ },+ #[error("model token {token} is outside audio model range {range_start}..={range_end}")]+ InvalidModelToken {+ token: u64,+ range_start: u64,+ range_end: u64,+ },+ #[error("tokenizer operation failed: {0}")]+ Tokenizer(String),+ #[error("audio runtime rejected input: {0}")]+ Runtime(String),+}++pub type AudioResult<T> = Result<T, AudioError>;++#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]+pub enum AudioTokenPacking {+ #[default]+ FrameMajor,+ CodebookMajor,+}++#[derive(Debug, Clone, PartialEq)]+pub struct AudioPcmBatch {+ samples: Box<[f32]>,+ sample_rate: NonZeroU32,+ channels: NonZeroUsize,+ lengths: Box<[usize]>,+}++impl AudioPcmBatch {+ pub fn new(+ samples: Box<[f32]>,+ sample_rate: u32,+ channels: usize,+ lengths: Box<[usize]>,+ ) -> AudioResult<Self> {+ let sample_rate = NonZeroU32::new(sample_rate).ok_or(AudioError::InvalidSampleRate)?;+ let channels = NonZeroUsize::new(channels).ok_or(AudioError::InvalidChannelCount)?;+ let expected_samples = lengths.iter().sum::<usize>() * channels.get();+ if samples.len() != expected_samples {+ return Err(AudioError::InvalidPcmShape {+ expected_samples,+ actual_samples: samples.len(),+ });+ }++ Ok(Self {+ samples,+ sample_rate,+ channels,+ lengths,+ })+ }++ pub fn batch_size(&self) -> usize {+ self.lengths.len()+ }++ pub fn sample_rate(&self) -> u32 {+ self.sample_rate.get()+ }++ pub fn channels(&self) -> usize {+ self.channels.get()+ }++ pub fn lengths(&self) -> &[usize] {+ &self.lengths+ }++ pub fn total_frames(&self) -> usize {+ self.lengths.iter().sum()+ }++ pub fn samples(&self) -> &[f32] {+ &self.samples+ }++ pub fn into_parts(self) -> (Box<[f32]>, u32, usize, Box<[usize]>) {+ (self.samples, self.sample_rate.get(), self.channels.get(), self.lengths)+ }+}++#[derive(Debug, Clone, PartialEq)]+pub struct AudioTokenGrid {+ tokens: Box<[u32]>,+ batch_size: usize,+ codebooks: usize,+ frames: usize,+ lengths: Box<[usize]>,+ packing: AudioTokenPacking,+}++impl AudioTokenGrid {+ pub fn new(+ tokens: Box<[u32]>,+ batch_size: usize,+ codebooks: usize,+ frames: usize,+ lengths: Box<[usize]>,+ packing: AudioTokenPacking,+ ) -> AudioResult<Self> {+ if codebooks == 0 {+ return Err(AudioError::InvalidTokenCardinality);+ }+ if lengths.len() != batch_size {+ return Err(AudioError::InvalidTokenLengths {+ expected_lengths: batch_size,+ actual_lengths: lengths.len(),+ });+ }+ for &length in lengths.iter() {+ if length > frames {+ return Err(AudioError::InvalidTokenLengthValue {+ length,+ frames,+ });+ }+ }++ let expected_tokens = batch_size.checked_mul(codebooks).and_then(|n| n.checked_mul(frames)).ok_or(+ AudioError::InvalidTokenShape {+ expected_tokens: usize::MAX,+ actual_tokens: tokens.len(),+ },+ )?;++ if tokens.len() != expected_tokens {+ return Err(AudioError::InvalidTokenShape {+ expected_tokens,+ actual_tokens: tokens.len(),+ });+ }++ Ok(Self {+ tokens,+ batch_size,+ codebooks,+ frames,+ lengths,+ packing,+ })+ }++ pub fn batch_size(&self) -> usize {+ self.batch_size+ }++ pub fn codebooks(&self) -> usize {+ self.codebooks+ }++ pub fn frames(&self) -> usize {+ self.frames+ }++ pub fn lengths(&self) -> &[usize] {+ &self.lengths+ }++ pub fn packing(&self) -> AudioTokenPacking {+ self.packing+ }++ pub fn tokens(&self) -> &[u32] {+ &self.tokens+ }++ pub fn into_parts(self) -> (Box<[u32]>, usize, usize, usize, Box<[usize]>, AudioTokenPacking) {+ (self.tokens, self.batch_size, self.codebooks, self.frames, self.lengths, self.packing)+ }++ pub fn get(+ &self,+ batch: usize,+ codebook: usize,+ frame: usize,+ ) -> u32 {+ let index =+ Self::token_index(self.packing, self.batch_size, self.codebooks, self.frames, batch, codebook, frame);+ self.tokens[index]+ }++ pub fn to_packing(+ &self,+ packing: AudioTokenPacking,+ ) -> Self {+ if packing == self.packing {+ return self.clone();+ }++ let mut tokens = vec![0u32; self.tokens.len()];+ for batch in 0..self.batch_size {+ for frame in 0..self.frames {+ for codebook in 0..self.codebooks {+ let src_idx = Self::token_index(+ self.packing,+ self.batch_size,+ self.codebooks,+ self.frames,+ batch,+ codebook,+ frame,+ );+ let dst_idx = Self::token_index(+ packing,+ self.batch_size,+ self.codebooks,+ self.frames,+ batch,+ codebook,+ frame,+ );+ tokens[dst_idx] = self.tokens[src_idx];+ }+ }+ }++ Self {+ tokens: tokens.into_boxed_slice(),+ batch_size: self.batch_size,+ codebooks: self.codebooks,+ frames: self.frames,+ lengths: self.lengths.clone(),+ packing,+ }+ }++ fn token_index(+ packing: AudioTokenPacking,+ batch_size: usize,+ codebooks: usize,+ frames: usize,+ batch: usize,+ codebook: usize,+ frame: usize,+ ) -> usize {+ debug_assert!(batch < batch_size);+ debug_assert!(codebook < codebooks);+ debug_assert!(frame < frames);++ match packing {+ AudioTokenPacking::FrameMajor => ((batch * frames + frame) * codebooks) + codebook,+ AudioTokenPacking::CodebookMajor => ((batch * codebooks + codebook) * frames) + frame,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::{AudioError, AudioTokenGrid, AudioTokenPacking};++ #[test]+ fn packing_conversion_roundtrip_is_lossless() {+ let grid = AudioTokenGrid::new(+ vec![+ 0, 1, 2, 3, // b0 f0, f1+ 4, 5, 6, 7, // b1 f0, f1+ ]+ .into_boxed_slice(),+ 2,+ 2,+ 2,+ vec![2, 2].into_boxed_slice(),+ AudioTokenPacking::FrameMajor,+ )+ .expect("valid grid");++ let converted = grid.to_packing(AudioTokenPacking::CodebookMajor);+ let restored = converted.to_packing(AudioTokenPacking::FrameMajor);++ assert_eq!(restored, grid);+ }++ #[test]+ fn invalid_grid_shape_is_rejected() {+ let error = AudioTokenGrid::new(+ vec![0, 1, 2].into_boxed_slice(),+ 1,+ 2,+ 2,+ vec![2].into_boxed_slice(),+ AudioTokenPacking::FrameMajor,+ )+ .expect_err("shape mismatch should fail");++ assert!(matches!(+ error,+ AudioError::InvalidTokenShape {+ expected_tokens: 4,+ actual_tokens: 3+ }+ ));+ }+}diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rsnew file mode 100644index 000000000..174dd259c--- /dev/null+++ b/crates/uzu/src/backends/common/kernel/audio.rs@@ -0,0 +1,187 @@+use crate::backends::common::Backend;++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum AudioPadMode {+ Zeros,+ Replicate,+}++pub struct AudioFsqDecodeArguments<'a, B: Backend> {+ pub tokens: &'a B::NativeBuffer,+ pub lengths: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub batch_size: usize,+ pub num_groups: usize,+ pub seq_len: usize,+ pub codebook_dim_per_group: usize,+ pub num_levels_per_group: &'a [i32],+}++pub struct AudioFsqEncodeArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub tokens: &'a B::NativeBuffer,+ pub lengths: &'a B::NativeBuffer,+ pub batch_size: usize,+ pub num_groups: usize,+ pub seq_len: usize,+ pub codebook_dim_per_group: usize,+ pub num_levels_per_group: &'a [i32],+ pub dim_base_index: &'a [i32],+ pub eps: f32,+}++pub struct AudioConv1dArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub weight: &'a B::NativeBuffer,+ pub bias: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub lengths: &'a B::NativeBuffer,+ pub batch_size: usize,+ pub cin: usize,+ pub cout: usize,+ pub seq_len_in: usize,+ pub seq_len_out: usize,+ pub kernel_size: usize,+ pub stride: usize,+ pub dilation: usize,+ pub padding: usize,+ pub pad_mode: AudioPadMode,+}++pub struct AudioCausalConv1dArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub weight: &'a B::NativeBuffer,+ pub bias: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub lengths: &'a B::NativeBuffer,+ pub batch_size: usize,+ pub cin: usize,+ pub cout: usize,+ pub seq_len: usize,+ pub kernel_size: usize,+ pub dilation: usize,+}++pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub weight: &'a B::NativeBuffer,+ pub bias: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub lengths: &'a B::NativeBuffer,+ pub batch_size: usize,+ pub cin: usize,+ pub cout: usize,+ pub seq_len_in: usize,+ pub seq_len_out: usize,+ pub stride: usize,+ pub groups: usize,+}++pub struct AudioHalfSnakeArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub alpha: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub batch_size: usize,+ pub channels: usize,+ pub seq_len: usize,+ pub snake_channels: usize,+ pub negative_slope: f32,+ pub eps: f32,+}++pub struct AudioElementwiseArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub n: usize,+}++pub struct AudioAddArguments<'a, B: Backend> {+ pub a: &'a B::NativeBuffer,+ pub b: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub n: usize,+}++pub struct AudioScaleArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub n: usize,+ pub scale: f32,+}++pub struct AudioClampArguments<'a, B: Backend> {+ pub input: &'a B::NativeBuffer,+ pub output: &'a B::NativeBuffer,+ pub n: usize,+ pub min_value: f32,+ pub max_value: f32,+}++pub trait AudioKernelRuntime<B: Backend> {+ fn encode_fsq_decode(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioFsqDecodeArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_fsq_encode(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioFsqEncodeArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_conv1d(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioConv1dArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_causal_conv1d(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioCausalConv1dArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_causal_conv_transpose1d(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioCausalConvTranspose1dArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_half_snake(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioHalfSnakeArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_leaky_relu(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioElementwiseArguments<'_, B>,+ negative_slope: f32,+ ) -> Result<(), B::Error>;++ fn encode_tanh(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioElementwiseArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_add(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioAddArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_scale(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioScaleArguments<'_, B>,+ ) -> Result<(), B::Error>;++ fn encode_clamp(+ &self,+ compute_encoder: &B::ComputeEncoder,+ arguments: AudioClampArguments<'_, B>,+ ) -> Result<(), B::Error>;+}diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rsindex b5a284288..f782acf4a 100644--- a/crates/uzu/src/backends/common/kernel/mod.rs+++ b/crates/uzu/src/backends/common/kernel/mod.rs@@ -3,6 +3,7 @@include!(concat!(env!("OUT_DIR"), "/traits.rs"));pub mod attention;+pub mod audio;pub mod kv_cache_update;pub mod matmul;pub mod mlp_gate_act_mul;diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metalnew file mode 100644index 000000000..2fb00029a--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/audio/audio.metal@@ -0,0 +1,738 @@+#include <metal_stdlib>+#include "../definitions.metal"++using namespace metal;++// === FSQ (Finite Scalar Quantization) decode ===+//+// Implements NeMo's FiniteScalarQuantizer.decode() applied per group:+// - tokens are stored as [B, G, T] int32+// - output codes are stored as [B, G*D, T] where D = codebook_dim_per_group+//+// NeMo reference:+// codes_nonnegative = (indices // dim_base_index) % num_levels+// dequantized = (codes_nonnegative - (num_levels//2)) / (num_levels//2)+//+// Here dim_base_index is derived from num_levels:+// base[0]=1, base[d]=prod_{k<d} num_levels[k]++template <typename T>+void fsq_decode(+ device const int* tokens,+ device T* out,+ device const int* lengths,+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const uint3 gid+) {+ const uint t = gid.x;+ const uint g = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || g >= (uint)num_groups) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ if ((int)t >= len_b) {+ // Zero all D codes for this token when masked.+ for (int d = 0; d < codebook_dim; ++d) {+ const uint out_c = g * (uint)codebook_dim + (uint)d;+ const uint out_idx =+ (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;+ out[out_idx] = (T)0;+ }+ return;+ }++ const uint token_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;+ const int token = tokens[token_idx];++ // Compute dim bases (small D, do it per-thread).+ int base = 1;+ for (int d = 0; d < codebook_dim; ++d) {+ const int levels = num_levels[d];+ const int scale = levels / 2; // integer division (matches PyTorch // 2)+ const int offset = scale;++ // Euclidean modulo to keep result in [0, levels) even if token is negative+ // (tokens are expected nonnegative, but keep this robust).+ const int div = token / base;+ int code_nonneg = div % levels;+ if (code_nonneg < 0) {+ code_nonneg += levels;+ }+ const float code = ((float)(code_nonneg - offset)) / (float)scale;++ const uint out_c = g * (uint)codebook_dim + (uint)d;+ const uint out_idx =+ (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;+ out[out_idx] = (T)code;++ base *= levels;+ }+}++// === Elementwise ops for codec blocks ===++template <typename T>+void audio_codec_leaky_relu(+ device const T* input,+ device T* output,+ const constant int& n,+ const constant float& negative_slope,+ const uint tid+) {+ if ((int)tid >= n) {+ return;+ }+ const float x = float(input[tid]);+ const float y = (x >= 0.0f) ? x : (negative_slope * x);+ output[tid] = (T)y;+}++template <typename T>+void audio_codec_tanh(+ device const T* input,+ device T* output,+ const constant int& n,+ const uint tid+) {+ if ((int)tid >= n) {+ return;+ }+ const float x = float(input[tid]);+ output[tid] = (T)tanh(x);+}++template <typename T>+void audio_codec_add(+ device const T* a,+ device const T* b,+ device T* out,+ const constant int& n,+ const uint tid+) {+ if ((int)tid >= n) {+ return;+ }+ out[tid] = a[tid] + b[tid];+}++template <typename T>+void audio_codec_scale(+ device const T* input,+ device T* output,+ const constant int& n,+ const constant float& scale,+ const uint tid+) {+ if ((int)tid >= n) {+ return;+ }+ const float x = float(input[tid]);+ output[tid] = (T)(x * scale);+}++// === Causal Conv1d (stride=1, groups=1) ===+//+// Matches NeMo's CausalConv1dNorm forward for the decoder path where+// stride=1:+// - left padding by padding_total = (kernel_size_eff - 1) where+// kernel_size_eff=(K-1)*dilation+1+// - no right padding (extra_padding=0 for stride=1)+// - output length equals input length+template <typename T>+void audio_codec_causal_conv1d(+ device const T* input, // [B, Cin, T]+ device const T* weight, // [Cout, Cin, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, T]+ device const int* lengths, // [B]+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const uint3 gid+) {+ const uint t = gid.x;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || oc >= (uint)cout) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len + t;+ if ((int)t >= len_b) {+ output[out_idx] = (T)0;+ return;+ }++ const int pad = (kernel_size - 1) * dilation;+ float acc = float(bias[oc]);++ for (int ic = 0; ic < cin; ++ic) {+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;+ for (int k = 0; k < kernel_size; ++k) {+ const int x_t = (int)t + k * dilation - pad;+ if (x_t < 0) {+ continue;+ }+ // For stride=1 and causal padding, x_t never exceeds len_b for valid t,+ // but for safety clamp to seq_len.+ if (x_t >= seq_len) {+ continue;+ }+ acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);+ }+ }++ output[out_idx] = (T)acc;+}++// === Causal ConvTranspose1d upsample (kernel_size = 2*stride) ===+//+// Matches NeMo's CausalConvTranspose1dNorm forward for trim_right_ratio=1:+// - ConvTranspose1d with padding=0, output_padding=0, dilation=1+// - kernel_size = 2 * stride+// - unpad: trim_right = kernel_size - stride (= stride), trim_left = 0+// Resulting output length is input_len * stride.+template <typename T>+void audio_codec_causal_conv_transpose1d(+ device const T* input, // [B, Cin, Tin]+ device const T* weight, // [Cin, Cout_per_group, 2*stride]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, Tout]+ device const int* lengths, // [B] (output lengths)+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& stride,+ const constant int& groups,+ const uint3 gid+) {+ const uint t_out = gid.x;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len_out;+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;+ if ((int)t_out >= len_b) {+ output[out_idx] = (T)0;+ return;+ }++ const int cout_per_group = cout / groups;+ const int cin_per_group = cin / groups;+ const int group_idx = (int)oc / cout_per_group;+ const int oc_in_group = (int)oc - group_idx * cout_per_group;++ const int q = (int)t_out / stride;+ const int r = (int)t_out - q * stride;++ float acc = float(bias[oc]);++ const uint in_base_b = b * (uint)cin * (uint)seq_len_in;+ const uint w_base_oc = (uint)oc_in_group * (uint)(2 * stride);++ const int ic_begin = group_idx * cin_per_group;+ const int ic_end = ic_begin + cin_per_group;++ for (int ic = ic_begin; ic < ic_end; ++ic) {+ if (q >= seq_len_in) {+ continue;+ }+ const uint in_base = in_base_b + (uint)ic * (uint)seq_len_in;++ // Weight layout: [Cin, Cout_per_group, K]+ const uint w_base =+ ((uint)ic * (uint)cout_per_group) * (uint)(2 * stride) + w_base_oc;++ // Contribution from input[q] with k=r+ acc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);++ // Contribution from input[q-1] with k=stride+r+ if (q > 0) {+ acc += float(weight[w_base + (uint)(stride + r)]) *+ float(input[in_base + (uint)(q - 1)]);+ }+ }++ output[out_idx] = (T)acc;+}++// === Snake / HalfSnake / Clamp (NeMo common.parts.utils) ===+//+// snake(x, alpha, eps) = x + (alpha + eps)^-1 * sin(alpha * x)^2+// HalfSnake applies snake to the first half channels and LeakyReLU to the+// rest.++template <typename T>+void audio_codec_half_snake(+ device const T* input, // [B, C, T]+ device const T* alpha, // [1, C_snake, 1] (contiguous, index by channel)+ device T* output, // [B, C, T]+ const constant int& channels,+ const constant int& seq_len,+ const constant int& snake_channels,+ const constant float& negative_slope,+ const constant float& eps,+ const uint3 gid+) {+ const uint t = gid.x;+ const uint c = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || c >= (uint)channels) {+ return;+ }++ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;+ const float x = float(input[idx]);++ if ((int)c < snake_channels) {+ const float a = float(alpha[c]);+ const float ax = a * x;+ const float s = sin(ax);+ const float y = x + (1.0f / (a + eps)) * (s * s);+ output[idx] = (T)y;+ } else {+ const float y = (x >= 0.0f) ? x : (negative_slope * x);+ output[idx] = (T)y;+ }+}++template <typename T>+void audio_codec_clamp(+ device const T* input,+ device T* output,+ const constant int& n,+ const constant float& min_value,+ const constant float& max_value,+ const uint tid+) {+ if ((int)tid >= n) {+ return;+ }+ const float x = float(input[tid]);+ const float y = clamp(x, min_value, max_value);+ output[tid] = (T)y;+}++// === Non-causal Conv1d with padding + stride (for encoder) ===+//+// Matches torch.nn.Conv1d with explicit padding and padding_mode:+// out[t_out] = bias + sum_{ic,k} w[oc,ic,k] * x[ic, t_out*stride - padding+// + k*dilation]+// - pad_mode=0: zeros+// - pad_mode=1: replicate (clamp index to [0, seq_len_in-1])+template <typename T>+void audio_codec_conv1d(+ device const T* input, // [B, Cin, Tin]+ device const T* weight, // [Cout, Cin, K]+ device const T* bias, // [Cout]+ device T* output, // [B, Cout, Tout]+ device const int* lengths, // [B] (Tout lengths)+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& kernel_size,+ const constant int& stride,+ const constant int& dilation,+ const constant int& padding,+ const constant int& pad_mode,+ const uint3 gid+) {+ const uint t_out = gid.x;+ const uint oc = gid.y;+ const uint b = gid.z;++ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len_out;+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;+ if ((int)t_out >= len_b) {+ output[out_idx] = (T)0;+ return;+ }++ const int base = (int)t_out * stride - padding;+ float acc = float(bias[oc]);++ for (int ic = 0; ic < cin; ++ic) {+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;+ for (int k = 0; k < kernel_size; ++k) {+ int x_t = base + k * dilation;+ if (pad_mode == 0) {+ if (x_t < 0 || x_t >= seq_len_in) {+ continue;+ }+ } else {+ x_t = clamp(x_t, 0, seq_len_in - 1);+ }+ acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);+ }+ }++ output[out_idx] = (T)acc;+}++// === FSQ (Finite Scalar Quantization) encode ===+//+// Implements GroupFiniteScalarQuantizer.encode() for inference:+// - input: [B, G*D, T] float+// - output: [B, G, T] int32+//+// NeMo reference (FiniteScalarQuantizer):+// output_scale = (num_levels - 1) / 2 * (1 - eps)+// output_offset = 0.5 if even else 0+// input_shift = tan(output_offset / output_scale)+// compressed = output_scale * tanh(inputs + input_shift) - output_offset+// rounded = round(compressed) # half away from zero+// code_nonneg = rounded + (num_levels//2)+// token = sum_d code_nonneg[d] * dim_base_index[d]++// Match torch.round() semantics (ties to even).+inline float round_ties_to_even(const float x) {+ const float f = floor(x);+ const float frac = x - f;+ if (frac < 0.5f) {+ return f;+ }+ if (frac > 0.5f) {+ return f + 1.0f;+ }+ const int fi = (int)f; // f is integral+ return ((fi & 1) != 0) ? (f + 1.0f) : f;+}++template <typename T>+void fsq_encode(+ device const T* input, // [B, G*D, T]+ device int* tokens, // [B, G, T]+ device const int* lengths, // [B]+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const constant int* dim_base_index,+ const constant float& eps,+ const uint3 gid+) {+ const uint t = gid.x;+ const uint g = gid.y;+ const uint b = gid.z;++ if (t >= (uint)seq_len || g >= (uint)num_groups) {+ return;+ }++ const int len_b = lengths ? lengths[b] : seq_len;+ const uint out_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;+ if ((int)t >= len_b) {+ tokens[out_idx] = 0;+ return;+ }++ const uint in_base =+ (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *+ (uint)seq_len ++ t;++ int token = 0;+ for (int d = 0; d < codebook_dim; ++d) {+ const int levels = num_levels[d];+ const int scale_i = levels / 2;+ const float output_scale = ((float)(levels - 1)) * 0.5f * (1.0f - eps);+ const float output_offset = (levels % 2 == 0) ? 0.5f : 0.0f;+ const float input_shift = tan(output_offset / output_scale);++ const float x = float(input[in_base + (uint)d * (uint)seq_len]);+ const float compressed =+ output_scale * tanh(x + input_shift) - output_offset;+ const float rounded = round_ties_to_even(compressed);++ int code_nonneg = (int)rounded + scale_i;+ code_nonneg = clamp(code_nonneg, 0, levels - 1);+ token += code_nonneg * dim_base_index[d];+ }++ tokens[out_idx] = token;+}++// === DSL kernel wrappers ===+//+// Keep the scalar helper implementations above and expose a DSL-annotated+// surface so audio kernels are integrated through the same generated wrapper+// path as other kernels.++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioFsqDecode)(+ device const int* tokens,+ device T* out,+ device const int* lengths,+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const constant int& batch_size,+ uint t AXIS(seq_len, 32),+ uint g AXIS(num_groups, 1),+ uint b AXIS(batch_size, 1)+) {+ fsq_decode<T>(+ tokens,+ out,+ lengths,+ num_groups,+ seq_len,+ codebook_dim,+ num_levels,+ uint3(t, g, b)+ );+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioLeakyRelu)(+ device const T* input,+ device T* output,+ const constant int& n,+ const constant float& negative_slope,+ uint tid AXIS(n, 256)+) {+ audio_codec_leaky_relu<T>(input, output, n, negative_slope, tid);+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioTanh)(+ device const T* input,+ device T* output,+ const constant int& n,+ uint tid AXIS(n, 256)+) {+ audio_codec_tanh<T>(input, output, n, tid);+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioAdd)(+ device const T* a,+ device const T* b,+ device T* out,+ const constant int& n,+ uint tid AXIS(n, 256)+) {+ audio_codec_add<T>(a, b, out, n, tid);+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioScale)(+ device const T* input,+ device T* output,+ const constant int& n,+ const constant float& scale,+ uint tid AXIS(n, 256)+) {+ audio_codec_scale<T>(input, output, n, scale, tid);+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioCausalConv1d)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len,+ const constant int& kernel_size,+ const constant int& dilation,+ const constant int& batch_size,+ uint t AXIS(seq_len, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ audio_codec_causal_conv1d<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len,+ kernel_size,+ dilation,+ uint3(t, oc, b)+ );+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioCausalConvTranspose1d)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& stride,+ const constant int& groups,+ const constant int& batch_size,+ uint t_out AXIS(seq_len_out, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ audio_codec_causal_conv_transpose1d<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ stride,+ groups,+ uint3(t_out, oc, b)+ );+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioHalfSnake)(+ device const T* input,+ device const T* alpha,+ device T* output,+ const constant int& channels,+ const constant int& seq_len,+ const constant int& snake_channels,+ const constant float& negative_slope,+ const constant float& eps,+ const constant int& batch_size,+ uint t AXIS(seq_len, 32),+ uint c AXIS(channels, 1),+ uint b AXIS(batch_size, 1)+) {+ audio_codec_half_snake<T>(+ input,+ alpha,+ output,+ channels,+ seq_len,+ snake_channels,+ negative_slope,+ eps,+ uint3(t, c, b)+ );+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioClamp)(+ device const T* input,+ device T* output,+ const constant int& n,+ const constant float& min_value,+ const constant float& max_value,+ uint tid AXIS(n, 256)+) {+ audio_codec_clamp<T>(input, output, n, min_value, max_value, tid);+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioConv1d)(+ device const T* input,+ device const T* weight,+ device const T* bias,+ device T* output,+ device const int* lengths,+ const constant int& cin,+ const constant int& cout,+ const constant int& seq_len_in,+ const constant int& seq_len_out,+ const constant int& kernel_size,+ const constant int& stride,+ const constant int& dilation,+ const constant int& padding,+ const constant int& pad_mode,+ const constant int& batch_size,+ uint t_out AXIS(seq_len_out, 32),+ uint oc AXIS(cout, 1),+ uint b AXIS(batch_size, 1)+) {+ audio_codec_conv1d<T>(+ input,+ weight,+ bias,+ output,+ lengths,+ cin,+ cout,+ seq_len_in,+ seq_len_out,+ kernel_size,+ stride,+ dilation,+ padding,+ pad_mode,+ uint3(t_out, oc, b)+ );+}++template <typename T>+VARIANTS(T, float, half, bfloat)+KERNEL(AudioFsqEncode)(+ device const T* input,+ device int* tokens,+ device const int* lengths,+ const constant int& num_groups,+ const constant int& seq_len,+ const constant int& codebook_dim,+ const constant int* num_levels,+ const constant int* dim_base_index,+ const constant float& eps,+ const constant int& batch_size,+ uint t AXIS(seq_len, 32),+ uint g AXIS(num_groups, 1),+ uint b AXIS(batch_size, 1)+) {+ fsq_encode<T>(+ input,+ tokens,+ lengths,+ num_groups,+ seq_len,+ codebook_dim,+ num_levels,+ dim_base_index,+ eps,+ uint3(t, g, b)+ );+}diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rsindex bd94bb20f..8f0af55a2 100644--- a/crates/uzu/src/backends/metal/kernel/mod.rs+++ b/crates/uzu/src/backends/metal/kernel/mod.rs@@ -1,4 +1,5 @@use crate::backends::common::kernel::matmul::MatmulKernels;+pub mod dsl {include!(concat!(env!("OUT_DIR"), "/dsl.rs"));}diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rsindex 4adb560e6..1a0c2b99d 100644--- a/crates/uzu/src/lib.rs+++ b/crates/uzu/src/lib.rs@@ -1,4 +1,5 @@pub mod array;+pub mod audio;pub mod backends;pub mod classifier;pub mod config;diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rsindex a842d9cb5..a6b2cc3ab 100644--- a/crates/uzu/src/prelude.rs+++ b/crates/uzu/src/prelude.rs@@ -11,6 +11,7 @@// Speculatorspub use crate::{VERSION,+ audio::{AudioCodecRuntime, AudioPcmBatch, AudioTokenGrid, AudioTokenPacking},session::{ChatSession, ClassificationSession,config::{DecodingConfig, RunConfig, SpeculatorConfig},diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rsnew file mode 100644index 000000000..dd5e2475d--- /dev/null+++ b/crates/uzu/tests/audio_codec_test.rs@@ -0,0 +1,813 @@+#![cfg(all(feature = "audio-runtime", target_os = "macos"))]++use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};+use uzu::{+ DataType,+ array::ArrayContextExt,+ backends::{+ common::{+ Context,+ kernel::{+ AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,+ AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,+ AudioLeakyReluKernel, AudioScaleKernel, AudioTanhKernel,+ },+ },+ metal::{+ MetalContext,+ kernel::dsl::{+ AudioAddMetalKernel, AudioCausalConv1dMetalKernel, AudioCausalConvTranspose1dMetalKernel,+ AudioClampMetalKernel, AudioConv1dMetalKernel, AudioFsqDecodeMetalKernel, AudioFsqEncodeMetalKernel,+ AudioHalfSnakeMetalKernel, AudioLeakyReluMetalKernel, AudioScaleMetalKernel, AudioTanhMetalKernel,+ },+ },+ },+};++fn create_test_context() -> std::rc::Rc<MetalContext> {+ <MetalContext as Context>::new().expect("MetalContext")+}++#[test]+fn audio_conv1d_replicate_matches_reference_f32() {+ let context = create_test_context();+ let kernel = AudioConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let batch_size = 2usize;+ let cin = 3usize;+ let cout = 4usize;+ let seq_len_in = 9usize;++ let stride = 3usize;+ let kernel_size = 2 * stride;+ let dilation = 1usize;+ let padding = 2usize;++ let seq_len_out = ((seq_len_in + 2 * padding - dilation * (kernel_size - 1) - 1) / stride) + 1;+ let lengths_out: [i32; 2] = [2, seq_len_out as i32];++ let input_len = batch_size * cin * seq_len_in;+ let weight_len = cout * cin * kernel_size;+ let bias_len = cout;+ let output_len = batch_size * cout * seq_len_out;++ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07 - 1.3).cos() * 0.2).collect();+ let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.01 - 0.02).collect();++ let mut input = context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);++ let mut bias = context.create_array(&[cout], DataType::F32, "audio_conv1d_bias");+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);++ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_conv1d_lengths");+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);++ let output = context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");++ let mut expected = vec![0.0f32; output_len];+ for batch in 0..batch_size {+ for out_channel in 0..cout {+ for out_time in 0..seq_len_out {+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;+ if (out_time as i32) >= lengths_out[batch] {+ expected[out_index] = 0.0;+ continue;+ }++ let mut acc = bias_values[out_channel];+ let base = (out_time * stride) as isize - padding as isize;++ for in_channel in 0..cin {+ let x_base = (batch * cin + in_channel) * seq_len_in;+ let w_base = (out_channel * cin + in_channel) * kernel_size;+ for kernel_offset in 0..kernel_size {+ let mut x_time = base + (kernel_offset * dilation) as isize;+ if x_time < 0 {+ x_time = 0;+ } else if x_time >= seq_len_in as isize {+ x_time = seq_len_in as isize - 1;+ }++ let x_index = x_base + x_time as usize;+ let w_index = w_base + kernel_offset;+ acc += weight_values[w_index] * input_values[x_index];+ }+ }++ expected[out_index] = acc;+ }+ }+ }++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(+ input.buffer(),+ weight.buffer(),+ bias.buffer(),+ output.buffer(),+ lengths.buffer(),+ cin as i32,+ cout as i32,+ seq_len_in as i32,+ seq_len_out as i32,+ kernel_size as i32,+ stride as i32,+ dilation as i32,+ padding as i32,+ 1_i32,+ batch_size as i32,+ &encoder,+ );++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let got = output.as_slice::<f32>();+ for index in 0..output_len {+ let delta = (expected[index] - got[index]).abs();+ assert!(+ delta <= 1e-4,+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",+ expected[index],+ got[index]+ );+ }+}++#[test]+fn audio_causal_conv1d_matches_reference_f32() {+ let context = create_test_context();+ let kernel = AudioCausalConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let batch_size = 2usize;+ let cin = 3usize;+ let cout = 4usize;+ let seq_len = 8usize;+ let kernel_size = 5usize;+ let dilation = 2usize;+ let pad = (kernel_size - 1) * dilation;++ let lengths: [i32; 2] = [5, 8];++ let input_len = batch_size * cin * seq_len;+ let weight_len = cout * cin * kernel_size;+ let bias_len = cout;+ let output_len = batch_size * cout * seq_len;++ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.01).sin()).collect();+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.02 - 0.5).cos() * 0.1).collect();+ let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.001 - 0.002).collect();++ let mut input = context.create_array(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);++ let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv1d_bias");+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);++ let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");+ lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);++ let output = context.create_array(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");++ let mut expected = vec![0.0f32; output_len];+ for batch in 0..batch_size {+ for out_channel in 0..cout {+ for time in 0..seq_len {+ let out_index = (batch * cout + out_channel) * seq_len + time;+ if (time as i32) >= lengths[batch] {+ expected[out_index] = 0.0;+ continue;+ }++ let mut acc = bias_values[out_channel];+ for in_channel in 0..cin {+ for kernel_offset in 0..kernel_size {+ let x_time = time as isize + (kernel_offset * dilation) as isize - pad as isize;+ if x_time < 0 || x_time >= seq_len as isize {+ continue;+ }++ let x_index = (batch * cin + in_channel) * seq_len + x_time as usize;+ let w_index = (out_channel * cin + in_channel) * kernel_size + kernel_offset;+ acc += weight_values[w_index] * input_values[x_index];+ }+ }++ expected[out_index] = acc;+ }+ }+ }++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(+ input.buffer(),+ weight.buffer(),+ bias.buffer(),+ output.buffer(),+ lengths_array.buffer(),+ cin as i32,+ cout as i32,+ seq_len as i32,+ kernel_size as i32,+ dilation as i32,+ batch_size as i32,+ &encoder,+ );++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let got = output.as_slice::<f32>();+ for index in 0..output_len {+ let delta = (expected[index] - got[index]).abs();+ assert!(+ delta <= 1e-4,+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",+ expected[index],+ got[index]+ );+ }+}++#[test]+fn audio_causal_conv_transpose1d_matches_reference_f32() {+ let context = create_test_context();+ let kernel = AudioCausalConvTranspose1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let batch_size = 2usize;+ let cin = 4usize;+ let groups = 2usize;+ let cout = 2usize;+ let stride = 3usize;+ let kernel_size = 2 * stride;++ let seq_len_in = 5usize;+ let seq_len_out = seq_len_in * stride;++ let lengths_out: [i32; 2] = [9, 15];++ let input_len = batch_size * cin * seq_len_in;+ let weight_len = cin * (cout / groups) * kernel_size;+ let output_len = batch_size * cout * seq_len_out;++ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.013).sin() * 0.5).collect();+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07).cos() * 0.2).collect();+ let bias_values: Vec<f32> = vec![0.01, -0.02];++ let mut input =+ context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let mut weight =+ context.create_array(&[cin, cout / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);++ let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);++ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);++ let output =+ context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_causal_conv_transpose_output");++ let cout_per_group = cout / groups;+ let cin_per_group = cin / groups;+ let mut expected = vec![0.0f32; output_len];++ for batch in 0..batch_size {+ for out_channel in 0..cout {+ let group_index = out_channel / cout_per_group;+ let out_channel_in_group = out_channel % cout_per_group;+ let in_channel_begin = group_index * cin_per_group;+ let in_channel_end = in_channel_begin + cin_per_group;++ for out_time in 0..seq_len_out {+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;+ if (out_time as i32) >= lengths_out[batch] {+ expected[out_index] = 0.0;+ continue;+ }++ let q = out_time / stride;+ let r = out_time % stride;+ let mut acc = bias_values[out_channel];++ for in_channel in in_channel_begin..in_channel_end {+ let input_base = (batch * cin + in_channel) * seq_len_in;+ let weight_base = (in_channel * cout_per_group + out_channel_in_group) * kernel_size;++ acc += input_values[input_base + q] * weight_values[weight_base + r];+ if q > 0 {+ acc += input_values[input_base + (q - 1)] * weight_values[weight_base + (stride + r)];+ }+ }++ expected[out_index] = acc;+ }+ }+ }++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(+ input.buffer(),+ weight.buffer(),+ bias.buffer(),+ output.buffer(),+ lengths.buffer(),+ cin as i32,+ cout as i32,+ seq_len_in as i32,+ seq_len_out as i32,+ stride as i32,+ groups as i32,+ batch_size as i32,+ &encoder,+ );++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let got = output.as_slice::<f32>();+ for index in 0..output_len {+ let delta = (expected[index] - got[index]).abs();+ assert!(+ delta <= 1e-4,+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",+ expected[index],+ got[index]+ );+ }+}++#[test]+fn audio_leaky_relu_matches_reference_f32() {+ let context = create_test_context();+ let kernel = AudioLeakyReluMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let n = 1024usize;+ let slope = 0.01f32;+ let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();++ let mut input = context.create_array(&[n], DataType::F32, "audio_leaky_relu_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(input.buffer(), output.buffer(), n as i32, slope, &encoder);++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let got = output.as_slice::<f32>();+ for index in 0..n {+ let expected = if input_values[index] >= 0.0 {+ input_values[index]+ } else {+ slope * input_values[index]+ };+ let delta = (expected - got[index]).abs();+ assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);+ }+}++#[test]+fn audio_tanh_matches_reference_f32() {+ let context = create_test_context();+ let kernel = AudioTanhMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let n = 1024usize;+ let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();++ let mut input = context.create_array(&[n], DataType::F32, "audio_tanh_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(input.buffer(), output.buffer(), n as i32, &encoder);++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let got = output.as_slice::<f32>();+ for index in 0..n {+ let expected = input_values[index].tanh();+ let delta = (expected - got[index]).abs();+ assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);+ }+}++#[test]+fn audio_add_and_scale_match_reference_f32() {+ let context = create_test_context();+ let add_kernel = AudioAddMetalKernel::new(&context, DataType::F32).expect("audio add kernel");+ let scale_kernel = AudioScaleMetalKernel::new(&context, DataType::F32).expect("audio scale kernel");++ let n = 2048usize;+ let scale_value = 1.0f32 / 3.0f32;+ let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();+ let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();++ let mut a = context.create_array(&[n], DataType::F32, "audio_add_a");+ a.as_slice_mut::<f32>().copy_from_slice(&a_values);++ let mut b = context.create_array(&[n], DataType::F32, "audio_add_b");+ b.as_slice_mut::<f32>().copy_from_slice(&b_values);++ let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");+ let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ add_kernel.encode(a.buffer(), b.buffer(), sum.buffer(), n as i32, &encoder);++ scale_kernel.encode(sum.buffer(), scaled.buffer(), n as i32, scale_value, &encoder);++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let sum_got = sum.as_slice::<f32>();+ let scaled_got = scaled.as_slice::<f32>();++ for index in 0..n {+ let expected_sum = a_values[index] + b_values[index];+ let expected_scaled = expected_sum * scale_value;++ let sum_delta = (expected_sum - sum_got[index]).abs();+ let scaled_delta = (expected_scaled - scaled_got[index]).abs();++ assert!(+ sum_delta <= 1e-6,+ "Add mismatch at index={index}: expected {expected_sum}, got {}, delta={sum_delta}",+ sum_got[index]+ );+ assert!(+ scaled_delta <= 1e-6,+ "Scale mismatch at index={index}: expected {expected_scaled}, got {}, delta={scaled_delta}",+ scaled_got[index]+ );+ }+}++#[test]+fn audio_clamp_matches_reference_f32() {+ let context = create_test_context();+ let kernel = AudioClampMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let n = 2048usize;+ let min_value = -1.0f32;+ let max_value = 1.0f32;+ let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 10.24).collect();++ let mut input = context.create_array(&[n], DataType::F32, "audio_clamp_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(input.buffer(), output.buffer(), n as i32, min_value, max_value, &encoder);++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let got = output.as_slice::<f32>();+ for index in 0..n {+ let expected = input_values[index].clamp(min_value, max_value);+ let delta = (expected - got[index]).abs();+ assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);+ }+}++#[test]+fn audio_half_snake_matches_reference_f32() {+ let context = create_test_context();+ let kernel = AudioHalfSnakeMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let batch_size = 1usize;+ let channels = 6usize;+ let snake_channels = channels / 2;+ let seq_len = 64usize;+ let n = batch_size * channels * seq_len;++ let negative_slope = 0.01f32;+ let eps = 1e-9f32;++ let input_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.01 - 3.2).sin()).collect();+ let alpha_values: Vec<f32> = (0..snake_channels).map(|i| 0.5 + i as f32 * 0.1).collect();++ let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let mut alpha = context.create_array(&[snake_channels], DataType::F32, "audio_half_snake_alpha");+ alpha.as_slice_mut::<f32>().copy_from_slice(&alpha_values);++ let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(+ input.buffer(),+ alpha.buffer(),+ output.buffer(),+ channels as i32,+ seq_len as i32,+ snake_channels as i32,+ negative_slope,+ eps,+ batch_size as i32,+ &encoder,+ );++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let got = output.as_slice::<f32>();+ for batch in 0..batch_size {+ for channel in 0..channels {+ for time in 0..seq_len {+ let index = (batch * channels + channel) * seq_len + time;+ let x = input_values[index];+ let expected = if channel < snake_channels {+ let alpha = alpha_values[channel];+ let sine = (alpha * x).sin();+ x + (sine * sine) / (alpha + eps)+ } else if x >= 0.0 {+ x+ } else {+ negative_slope * x+ };++ let delta = (expected - got[index]).abs();+ assert!(+ delta <= 1e-5,+ "Mismatch at index={index}: expected {expected}, got {}, delta={delta}",+ got[index]+ );+ }+ }+ }+}++#[test]+fn audio_fsq_decode_matches_reference() {+ let context = create_test_context();+ let kernel = AudioFsqDecodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let batch_size = 1usize;+ let num_groups = 2usize;+ let seq_len = 3usize;+ let codebook_dim = 2usize;+ let num_levels: Vec<i32> = vec![8, 5];++ let mut tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");+ tokens.as_slice_mut::<i32>().copy_from_slice(&[+ 0, 7, 11, // g0 tokens at t0..t2+ 3, 5, 9, // g1 tokens at t0..t2+ ]);++ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_lengths");+ lengths.as_slice_mut::<i32>().copy_from_slice(&[2]);++ let output =+ context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(+ tokens.buffer(),+ output.buffer(),+ lengths.buffer(),+ num_groups as i32,+ seq_len as i32,+ codebook_dim as i32,+ &num_levels,+ batch_size as i32,+ &encoder,+ );++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let expected =+ reference_fsq_decode(&[0, 7, 11, 3, 5, 9], 2, batch_size, num_groups, seq_len, codebook_dim, &num_levels);+ let actual = output.as_slice::<f32>();++ for (index, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() {+ let delta = (a - e).abs();+ assert!(delta <= 1e-5, "index={index}, actual={a}, expected={e}, delta={delta}");+ }+}++#[test]+fn audio_fsq_encode_matches_reference() {+ let context = create_test_context();+ let kernel = AudioFsqEncodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");++ let batch_size = 1usize;+ let num_groups = 2usize;+ let codebook_dim = 2usize;+ let seq_len = 4usize;+ let eps = 1e-3_f32;+ let num_levels: Vec<i32> = vec![8, 6];+ let dim_base_index: Vec<i32> = vec![1, 8];++ let input_values = vec![+ -0.9, -0.1, 0.2, 0.7, // g0 d0+ 0.8, 0.3, -0.4, -0.7, // g0 d1+ -0.2, 0.0, 0.4, 0.9, // g1 d0+ 0.5, -0.5, 0.1, -0.1, // g1 d1+ ];+ let lengths_values = vec![3_i32];++ let mut input = context.create_array(+ &[batch_size, num_groups * codebook_dim, seq_len],+ DataType::F32,+ "audio_fsq_encode_input",+ );+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);++ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);++ let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");++ let command_buffer = context.command_queue.command_buffer().expect("command buffer");+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");++ kernel.encode(+ input.buffer(),+ tokens.buffer(),+ lengths.buffer(),+ num_groups as i32,+ seq_len as i32,+ codebook_dim as i32,+ &num_levels,+ &dim_base_index,+ eps,+ batch_size as i32,+ &encoder,+ );++ encoder.end_encoding();+ command_buffer.commit();+ command_buffer.wait_until_completed();++ let expected = reference_fsq_encode(+ &input_values,+ &lengths_values,+ batch_size,+ num_groups,+ seq_len,+ codebook_dim,+ &num_levels,+ &dim_base_index,+ eps,+ );+ let actual = tokens.as_slice::<i32>();++ assert_eq!(actual, expected);+}++fn round_ties_to_even(value: f32) -> f32 {+ let floor = value.floor();+ let fraction = value - floor;+ if fraction < 0.5 {+ floor+ } else if fraction > 0.5 {+ floor + 1.0+ } else if (floor as i64 & 1) != 0 {+ floor + 1.0+ } else {+ floor+ }+}++fn reference_fsq_encode(+ input: &[f32],+ lengths: &[i32],+ batch_size: usize,+ num_groups: usize,+ seq_len: usize,+ codebook_dim: usize,+ num_levels: &[i32],+ dim_base_index: &[i32],+ eps: f32,+) -> Vec<i32> {+ let mut tokens = vec![0_i32; batch_size * num_groups * seq_len];++ for batch in 0..batch_size {+ for group in 0..num_groups {+ for time in 0..seq_len {+ let output_index = (batch * num_groups + group) * seq_len + time;+ if (time as i32) >= lengths[batch] {+ tokens[output_index] = 0;+ continue;+ }++ let mut token = 0_i32;+ for dim in 0..codebook_dim {+ let levels = num_levels[dim];+ let scale_i = levels / 2;+ let output_scale = (levels - 1) as f32 * 0.5 * (1.0 - eps);+ let output_offset = if levels % 2 == 0 {+ 0.5+ } else {+ 0.0+ };+ let input_shift = (output_offset / output_scale).tan();++ let input_index =+ (batch * (num_groups * codebook_dim) + group * codebook_dim + dim) * seq_len + time;+ let compressed = output_scale * (input[input_index] + input_shift).tanh() - output_offset;+ let rounded = round_ties_to_even(compressed);+ let code_nonnegative = (rounded as i32 + scale_i).clamp(0, levels - 1);+ token += code_nonnegative * dim_base_index[dim];+ }++ tokens[output_index] = token;+ }+ }+ }++ tokens+}++fn reference_fsq_decode(+ tokens: &[i32],+ length: usize,+ batch_size: usize,+ num_groups: usize,+ seq_len: usize,+ codebook_dim: usize,+ num_levels: &[i32],+) -> Vec<f32> {+ let mut out = vec![0.0f32; batch_size * num_groups * codebook_dim * seq_len];++ for b in 0..batch_size {+ for g in 0..num_groups {+ for t in 0..seq_len {+ if t >= length {+ continue;+ }++ let token_index = (b * num_groups + g) * seq_len + t;+ let token = tokens[token_index];++ let mut base = 1i32;+ for d in 0..codebook_dim {+ let levels = num_levels[d];+ let scale = levels / 2;+ let offset = scale;+ let div = token / base;+ let mut code_nonneg = div % levels;+ if code_nonneg < 0 {+ code_nonneg += levels;+ }+ let value = (code_nonneg - offset) as f32 / scale as f32;++ let out_channel = g * codebook_dim + d;+ let out_index = (b * (num_groups * codebook_dim) + out_channel) * seq_len + t;+ out[out_index] = value;++ base *= levels;+ }+ }+ }+ }++ out+}