Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Audio Сodec bits (#211)

LuckyIYIdfacd688060a
diff --git a/crates/uzu/Cargo.toml b/crates/uzu/Cargo.toml
index 89cfb7d9a..0fc8e792e 100644
--- a/crates/uzu/Cargo.toml
+++ b/crates/uzu/Cargo.toml
@@ -78,6 +78,7 @@ metal = [
metal-nax = []
tracing = []
+audio-runtime = []
[[test]]
name = "moe"
diff --git a/crates/uzu/src/audio/codec.rs b/crates/uzu/src/audio/codec.rs
new file mode 100644
index 000000000..e973653ed
--- /dev/null
+++ b/crates/uzu/src/audio/codec.rs
@@ -0,0 +1,13 @@
+use super::{AudioPcmBatch, AudioResult, AudioTokenGrid};
+
+pub trait AudioCodecRuntime: Send + Sync {
+ fn encode(
+ &self,
+ pcm: &AudioPcmBatch,
+ ) -> AudioResult<AudioTokenGrid>;
+
+ fn decode(
+ &self,
+ tokens: &AudioTokenGrid,
+ ) -> AudioResult<AudioPcmBatch>;
+}
diff --git a/crates/uzu/src/audio/mod.rs b/crates/uzu/src/audio/mod.rs
new file mode 100644
index 000000000..9b68e6d6d
--- /dev/null
+++ b/crates/uzu/src/audio/mod.rs
@@ -0,0 +1,5 @@
+mod codec;
+mod types;
+
+pub use codec::AudioCodecRuntime;
+pub use types::{AudioError, AudioPcmBatch, AudioResult, AudioTokenGrid, AudioTokenPacking};
diff --git a/crates/uzu/src/audio/types.rs b/crates/uzu/src/audio/types.rs
new file mode 100644
index 000000000..60364a049
--- /dev/null
+++ b/crates/uzu/src/audio/types.rs
@@ -0,0 +1,330 @@
+use std::num::{NonZeroU32, NonZeroUsize};
+
+#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)]
+pub enum AudioError {
+ #[error("sample_rate must be > 0")]
+ InvalidSampleRate,
+ #[error("channels must be > 0")]
+ InvalidChannelCount,
+ #[error("pcm samples shape mismatch: expected {expected_samples} samples, got {actual_samples}")]
+ InvalidPcmShape {
+ expected_samples: usize,
+ actual_samples: usize,
+ },
+ #[error("token lengths shape mismatch: expected {expected_lengths} lengths, got {actual_lengths}")]
+ InvalidTokenLengths {
+ expected_lengths: usize,
+ actual_lengths: usize,
+ },
+ #[error("token grid shape mismatch: expected {expected_tokens} tokens, got {actual_tokens}")]
+ InvalidTokenShape {
+ expected_tokens: usize,
+ actual_tokens: usize,
+ },
+ #[error("token length {length} exceeds frame count {frames}")]
+ InvalidTokenLengthValue {
+ length: usize,
+ frames: usize,
+ },
+ #[error("audio token cardinality must be > 0")]
+ InvalidTokenCardinality,
+ #[error("codec token {token} is outside codec range 0..{cardinality}")]
+ InvalidCodecToken {
+ token: u32,
+ cardinality: u32,
+ },
+ #[error("model token {token} is outside audio model range {range_start}..={range_end}")]
+ InvalidModelToken {
+ token: u64,
+ range_start: u64,
+ range_end: u64,
+ },
+ #[error("tokenizer operation failed: {0}")]
+ Tokenizer(String),
+ #[error("audio runtime rejected input: {0}")]
+ Runtime(String),
+}
+
+pub type AudioResult<T> = Result<T, AudioError>;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum AudioTokenPacking {
+ #[default]
+ FrameMajor,
+ CodebookMajor,
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct AudioPcmBatch {
+ samples: Box<[f32]>,
+ sample_rate: NonZeroU32,
+ channels: NonZeroUsize,
+ lengths: Box<[usize]>,
+}
+
+impl AudioPcmBatch {
+ pub fn new(
+ samples: Box<[f32]>,
+ sample_rate: u32,
+ channels: usize,
+ lengths: Box<[usize]>,
+ ) -> AudioResult<Self> {
+ let sample_rate = NonZeroU32::new(sample_rate).ok_or(AudioError::InvalidSampleRate)?;
+ let channels = NonZeroUsize::new(channels).ok_or(AudioError::InvalidChannelCount)?;
+ let expected_samples = lengths.iter().sum::<usize>() * channels.get();
+ if samples.len() != expected_samples {
+ return Err(AudioError::InvalidPcmShape {
+ expected_samples,
+ actual_samples: samples.len(),
+ });
+ }
+
+ Ok(Self {
+ samples,
+ sample_rate,
+ channels,
+ lengths,
+ })
+ }
+
+ pub fn batch_size(&self) -> usize {
+ self.lengths.len()
+ }
+
+ pub fn sample_rate(&self) -> u32 {
+ self.sample_rate.get()
+ }
+
+ pub fn channels(&self) -> usize {
+ self.channels.get()
+ }
+
+ pub fn lengths(&self) -> &[usize] {
+ &self.lengths
+ }
+
+ pub fn total_frames(&self) -> usize {
+ self.lengths.iter().sum()
+ }
+
+ pub fn samples(&self) -> &[f32] {
+ &self.samples
+ }
+
+ pub fn into_parts(self) -> (Box<[f32]>, u32, usize, Box<[usize]>) {
+ (self.samples, self.sample_rate.get(), self.channels.get(), self.lengths)
+ }
+}
+
+#[derive(Debug, Clone, PartialEq)]
+pub struct AudioTokenGrid {
+ tokens: Box<[u32]>,
+ batch_size: usize,
+ codebooks: usize,
+ frames: usize,
+ lengths: Box<[usize]>,
+ packing: AudioTokenPacking,
+}
+
+impl AudioTokenGrid {
+ pub fn new(
+ tokens: Box<[u32]>,
+ batch_size: usize,
+ codebooks: usize,
+ frames: usize,
+ lengths: Box<[usize]>,
+ packing: AudioTokenPacking,
+ ) -> AudioResult<Self> {
+ if codebooks == 0 {
+ return Err(AudioError::InvalidTokenCardinality);
+ }
+ if lengths.len() != batch_size {
+ return Err(AudioError::InvalidTokenLengths {
+ expected_lengths: batch_size,
+ actual_lengths: lengths.len(),
+ });
+ }
+ for &length in lengths.iter() {
+ if length > frames {
+ return Err(AudioError::InvalidTokenLengthValue {
+ length,
+ frames,
+ });
+ }
+ }
+
+ let expected_tokens = batch_size.checked_mul(codebooks).and_then(|n| n.checked_mul(frames)).ok_or(
+ AudioError::InvalidTokenShape {
+ expected_tokens: usize::MAX,
+ actual_tokens: tokens.len(),
+ },
+ )?;
+
+ if tokens.len() != expected_tokens {
+ return Err(AudioError::InvalidTokenShape {
+ expected_tokens,
+ actual_tokens: tokens.len(),
+ });
+ }
+
+ Ok(Self {
+ tokens,
+ batch_size,
+ codebooks,
+ frames,
+ lengths,
+ packing,
+ })
+ }
+
+ pub fn batch_size(&self) -> usize {
+ self.batch_size
+ }
+
+ pub fn codebooks(&self) -> usize {
+ self.codebooks
+ }
+
+ pub fn frames(&self) -> usize {
+ self.frames
+ }
+
+ pub fn lengths(&self) -> &[usize] {
+ &self.lengths
+ }
+
+ pub fn packing(&self) -> AudioTokenPacking {
+ self.packing
+ }
+
+ pub fn tokens(&self) -> &[u32] {
+ &self.tokens
+ }
+
+ pub fn into_parts(self) -> (Box<[u32]>, usize, usize, usize, Box<[usize]>, AudioTokenPacking) {
+ (self.tokens, self.batch_size, self.codebooks, self.frames, self.lengths, self.packing)
+ }
+
+ pub fn get(
+ &self,
+ batch: usize,
+ codebook: usize,
+ frame: usize,
+ ) -> u32 {
+ let index =
+ Self::token_index(self.packing, self.batch_size, self.codebooks, self.frames, batch, codebook, frame);
+ self.tokens[index]
+ }
+
+ pub fn to_packing(
+ &self,
+ packing: AudioTokenPacking,
+ ) -> Self {
+ if packing == self.packing {
+ return self.clone();
+ }
+
+ let mut tokens = vec![0u32; self.tokens.len()];
+ for batch in 0..self.batch_size {
+ for frame in 0..self.frames {
+ for codebook in 0..self.codebooks {
+ let src_idx = Self::token_index(
+ self.packing,
+ self.batch_size,
+ self.codebooks,
+ self.frames,
+ batch,
+ codebook,
+ frame,
+ );
+ let dst_idx = Self::token_index(
+ packing,
+ self.batch_size,
+ self.codebooks,
+ self.frames,
+ batch,
+ codebook,
+ frame,
+ );
+ tokens[dst_idx] = self.tokens[src_idx];
+ }
+ }
+ }
+
+ Self {
+ tokens: tokens.into_boxed_slice(),
+ batch_size: self.batch_size,
+ codebooks: self.codebooks,
+ frames: self.frames,
+ lengths: self.lengths.clone(),
+ packing,
+ }
+ }
+
+ fn token_index(
+ packing: AudioTokenPacking,
+ batch_size: usize,
+ codebooks: usize,
+ frames: usize,
+ batch: usize,
+ codebook: usize,
+ frame: usize,
+ ) -> usize {
+ debug_assert!(batch < batch_size);
+ debug_assert!(codebook < codebooks);
+ debug_assert!(frame < frames);
+
+ match packing {
+ AudioTokenPacking::FrameMajor => ((batch * frames + frame) * codebooks) + codebook,
+ AudioTokenPacking::CodebookMajor => ((batch * codebooks + codebook) * frames) + frame,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{AudioError, AudioTokenGrid, AudioTokenPacking};
+
+ #[test]
+ fn packing_conversion_roundtrip_is_lossless() {
+ let grid = AudioTokenGrid::new(
+ vec![
+ 0, 1, 2, 3, // b0 f0, f1
+ 4, 5, 6, 7, // b1 f0, f1
+ ]
+ .into_boxed_slice(),
+ 2,
+ 2,
+ 2,
+ vec![2, 2].into_boxed_slice(),
+ AudioTokenPacking::FrameMajor,
+ )
+ .expect("valid grid");
+
+ let converted = grid.to_packing(AudioTokenPacking::CodebookMajor);
+ let restored = converted.to_packing(AudioTokenPacking::FrameMajor);
+
+ assert_eq!(restored, grid);
+ }
+
+ #[test]
+ fn invalid_grid_shape_is_rejected() {
+ let error = AudioTokenGrid::new(
+ vec![0, 1, 2].into_boxed_slice(),
+ 1,
+ 2,
+ 2,
+ vec![2].into_boxed_slice(),
+ AudioTokenPacking::FrameMajor,
+ )
+ .expect_err("shape mismatch should fail");
+
+ assert!(matches!(
+ error,
+ AudioError::InvalidTokenShape {
+ expected_tokens: 4,
+ actual_tokens: 3
+ }
+ ));
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/audio.rs b/crates/uzu/src/backends/common/kernel/audio.rs
new file mode 100644
index 000000000..174dd259c
--- /dev/null
+++ b/crates/uzu/src/backends/common/kernel/audio.rs
@@ -0,0 +1,187 @@
+use crate::backends::common::Backend;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AudioPadMode {
+ Zeros,
+ Replicate,
+}
+
+pub struct AudioFsqDecodeArguments<'a, B: Backend> {
+ pub tokens: &'a B::NativeBuffer,
+ pub lengths: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub batch_size: usize,
+ pub num_groups: usize,
+ pub seq_len: usize,
+ pub codebook_dim_per_group: usize,
+ pub num_levels_per_group: &'a [i32],
+}
+
+pub struct AudioFsqEncodeArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub tokens: &'a B::NativeBuffer,
+ pub lengths: &'a B::NativeBuffer,
+ pub batch_size: usize,
+ pub num_groups: usize,
+ pub seq_len: usize,
+ pub codebook_dim_per_group: usize,
+ pub num_levels_per_group: &'a [i32],
+ pub dim_base_index: &'a [i32],
+ pub eps: f32,
+}
+
+pub struct AudioConv1dArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub weight: &'a B::NativeBuffer,
+ pub bias: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub lengths: &'a B::NativeBuffer,
+ pub batch_size: usize,
+ pub cin: usize,
+ pub cout: usize,
+ pub seq_len_in: usize,
+ pub seq_len_out: usize,
+ pub kernel_size: usize,
+ pub stride: usize,
+ pub dilation: usize,
+ pub padding: usize,
+ pub pad_mode: AudioPadMode,
+}
+
+pub struct AudioCausalConv1dArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub weight: &'a B::NativeBuffer,
+ pub bias: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub lengths: &'a B::NativeBuffer,
+ pub batch_size: usize,
+ pub cin: usize,
+ pub cout: usize,
+ pub seq_len: usize,
+ pub kernel_size: usize,
+ pub dilation: usize,
+}
+
+pub struct AudioCausalConvTranspose1dArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub weight: &'a B::NativeBuffer,
+ pub bias: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub lengths: &'a B::NativeBuffer,
+ pub batch_size: usize,
+ pub cin: usize,
+ pub cout: usize,
+ pub seq_len_in: usize,
+ pub seq_len_out: usize,
+ pub stride: usize,
+ pub groups: usize,
+}
+
+pub struct AudioHalfSnakeArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub alpha: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub batch_size: usize,
+ pub channels: usize,
+ pub seq_len: usize,
+ pub snake_channels: usize,
+ pub negative_slope: f32,
+ pub eps: f32,
+}
+
+pub struct AudioElementwiseArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub n: usize,
+}
+
+pub struct AudioAddArguments<'a, B: Backend> {
+ pub a: &'a B::NativeBuffer,
+ pub b: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub n: usize,
+}
+
+pub struct AudioScaleArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub n: usize,
+ pub scale: f32,
+}
+
+pub struct AudioClampArguments<'a, B: Backend> {
+ pub input: &'a B::NativeBuffer,
+ pub output: &'a B::NativeBuffer,
+ pub n: usize,
+ pub min_value: f32,
+ pub max_value: f32,
+}
+
+pub trait AudioKernelRuntime<B: Backend> {
+ fn encode_fsq_decode(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioFsqDecodeArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_fsq_encode(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioFsqEncodeArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_conv1d(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioConv1dArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_causal_conv1d(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioCausalConv1dArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_causal_conv_transpose1d(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioCausalConvTranspose1dArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_half_snake(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioHalfSnakeArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_leaky_relu(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioElementwiseArguments<'_, B>,
+ negative_slope: f32,
+ ) -> Result<(), B::Error>;
+
+ fn encode_tanh(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioElementwiseArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_add(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioAddArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_scale(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioScaleArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+
+ fn encode_clamp(
+ &self,
+ compute_encoder: &B::ComputeEncoder,
+ arguments: AudioClampArguments<'_, B>,
+ ) -> Result<(), B::Error>;
+}
diff --git a/crates/uzu/src/backends/common/kernel/mod.rs b/crates/uzu/src/backends/common/kernel/mod.rs
index b5a284288..f782acf4a 100644
--- a/crates/uzu/src/backends/common/kernel/mod.rs
+++ b/crates/uzu/src/backends/common/kernel/mod.rs
@@ -3,6 +3,7 @@
include!(concat!(env!("OUT_DIR"), "/traits.rs"));
pub mod attention;
+pub mod audio;
pub mod kv_cache_update;
pub mod matmul;
pub mod mlp_gate_act_mul;
diff --git a/crates/uzu/src/backends/metal/kernel/audio/audio.metal b/crates/uzu/src/backends/metal/kernel/audio/audio.metal
new file mode 100644
index 000000000..2fb00029a
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/audio/audio.metal
@@ -0,0 +1,738 @@
+#include <metal_stdlib>
+#include "../definitions.metal"
+
+using namespace metal;
+
+// === FSQ (Finite Scalar Quantization) decode ===
+//
+// Implements NeMo's FiniteScalarQuantizer.decode() applied per group:
+// - tokens are stored as [B, G, T] int32
+// - output codes are stored as [B, G*D, T] where D = codebook_dim_per_group
+//
+// NeMo reference:
+// codes_nonnegative = (indices // dim_base_index) % num_levels
+// dequantized = (codes_nonnegative - (num_levels//2)) / (num_levels//2)
+//
+// Here dim_base_index is derived from num_levels:
+// base[0]=1, base[d]=prod_{k<d} num_levels[k]
+
+template <typename T>
+void fsq_decode(
+ device const int* tokens,
+ device T* out,
+ device const int* lengths,
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const uint3 gid
+) {
+ const uint t = gid.x;
+ const uint g = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || g >= (uint)num_groups) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ if ((int)t >= len_b) {
+ // Zero all D codes for this token when masked.
+ for (int d = 0; d < codebook_dim; ++d) {
+ const uint out_c = g * (uint)codebook_dim + (uint)d;
+ const uint out_idx =
+ (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;
+ out[out_idx] = (T)0;
+ }
+ return;
+ }
+
+ const uint token_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;
+ const int token = tokens[token_idx];
+
+ // Compute dim bases (small D, do it per-thread).
+ int base = 1;
+ for (int d = 0; d < codebook_dim; ++d) {
+ const int levels = num_levels[d];
+ const int scale = levels / 2; // integer division (matches PyTorch // 2)
+ const int offset = scale;
+
+ // Euclidean modulo to keep result in [0, levels) even if token is negative
+ // (tokens are expected nonnegative, but keep this robust).
+ const int div = token / base;
+ int code_nonneg = div % levels;
+ if (code_nonneg < 0) {
+ code_nonneg += levels;
+ }
+ const float code = ((float)(code_nonneg - offset)) / (float)scale;
+
+ const uint out_c = g * (uint)codebook_dim + (uint)d;
+ const uint out_idx =
+ (b * (uint)(num_groups * codebook_dim) + out_c) * (uint)seq_len + t;
+ out[out_idx] = (T)code;
+
+ base *= levels;
+ }
+}
+
+// === Elementwise ops for codec blocks ===
+
+template <typename T>
+void audio_codec_leaky_relu(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ const constant float& negative_slope,
+ const uint tid
+) {
+ if ((int)tid >= n) {
+ return;
+ }
+ const float x = float(input[tid]);
+ const float y = (x >= 0.0f) ? x : (negative_slope * x);
+ output[tid] = (T)y;
+}
+
+template <typename T>
+void audio_codec_tanh(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ const uint tid
+) {
+ if ((int)tid >= n) {
+ return;
+ }
+ const float x = float(input[tid]);
+ output[tid] = (T)tanh(x);
+}
+
+template <typename T>
+void audio_codec_add(
+ device const T* a,
+ device const T* b,
+ device T* out,
+ const constant int& n,
+ const uint tid
+) {
+ if ((int)tid >= n) {
+ return;
+ }
+ out[tid] = a[tid] + b[tid];
+}
+
+template <typename T>
+void audio_codec_scale(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ const constant float& scale,
+ const uint tid
+) {
+ if ((int)tid >= n) {
+ return;
+ }
+ const float x = float(input[tid]);
+ output[tid] = (T)(x * scale);
+}
+
+// === Causal Conv1d (stride=1, groups=1) ===
+//
+// Matches NeMo's CausalConv1dNorm forward for the decoder path where
+// stride=1:
+// - left padding by padding_total = (kernel_size_eff - 1) where
+// kernel_size_eff=(K-1)*dilation+1
+// - no right padding (extra_padding=0 for stride=1)
+// - output length equals input length
+template <typename T>
+void audio_codec_causal_conv1d(
+ device const T* input, // [B, Cin, T]
+ device const T* weight, // [Cout, Cin, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, T]
+ device const int* lengths, // [B]
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const uint3 gid
+) {
+ const uint t = gid.x;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || oc >= (uint)cout) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len + t;
+ if ((int)t >= len_b) {
+ output[out_idx] = (T)0;
+ return;
+ }
+
+ const int pad = (kernel_size - 1) * dilation;
+ float acc = float(bias[oc]);
+
+ for (int ic = 0; ic < cin; ++ic) {
+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;
+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len;
+ for (int k = 0; k < kernel_size; ++k) {
+ const int x_t = (int)t + k * dilation - pad;
+ if (x_t < 0) {
+ continue;
+ }
+ // For stride=1 and causal padding, x_t never exceeds len_b for valid t,
+ // but for safety clamp to seq_len.
+ if (x_t >= seq_len) {
+ continue;
+ }
+ acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);
+ }
+ }
+
+ output[out_idx] = (T)acc;
+}
+
+// === Causal ConvTranspose1d upsample (kernel_size = 2*stride) ===
+//
+// Matches NeMo's CausalConvTranspose1dNorm forward for trim_right_ratio=1:
+// - ConvTranspose1d with padding=0, output_padding=0, dilation=1
+// - kernel_size = 2 * stride
+// - unpad: trim_right = kernel_size - stride (= stride), trim_left = 0
+// Resulting output length is input_len * stride.
+template <typename T>
+void audio_codec_causal_conv_transpose1d(
+ device const T* input, // [B, Cin, Tin]
+ device const T* weight, // [Cin, Cout_per_group, 2*stride]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, Tout]
+ device const int* lengths, // [B] (output lengths)
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& stride,
+ const constant int& groups,
+ const uint3 gid
+) {
+ const uint t_out = gid.x;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len_out;
+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;
+ if ((int)t_out >= len_b) {
+ output[out_idx] = (T)0;
+ return;
+ }
+
+ const int cout_per_group = cout / groups;
+ const int cin_per_group = cin / groups;
+ const int group_idx = (int)oc / cout_per_group;
+ const int oc_in_group = (int)oc - group_idx * cout_per_group;
+
+ const int q = (int)t_out / stride;
+ const int r = (int)t_out - q * stride;
+
+ float acc = float(bias[oc]);
+
+ const uint in_base_b = b * (uint)cin * (uint)seq_len_in;
+ const uint w_base_oc = (uint)oc_in_group * (uint)(2 * stride);
+
+ const int ic_begin = group_idx * cin_per_group;
+ const int ic_end = ic_begin + cin_per_group;
+
+ for (int ic = ic_begin; ic < ic_end; ++ic) {
+ if (q >= seq_len_in) {
+ continue;
+ }
+ const uint in_base = in_base_b + (uint)ic * (uint)seq_len_in;
+
+ // Weight layout: [Cin, Cout_per_group, K]
+ const uint w_base =
+ ((uint)ic * (uint)cout_per_group) * (uint)(2 * stride) + w_base_oc;
+
+ // Contribution from input[q] with k=r
+ acc += float(weight[w_base + (uint)r]) * float(input[in_base + (uint)q]);
+
+ // Contribution from input[q-1] with k=stride+r
+ if (q > 0) {
+ acc += float(weight[w_base + (uint)(stride + r)]) *
+ float(input[in_base + (uint)(q - 1)]);
+ }
+ }
+
+ output[out_idx] = (T)acc;
+}
+
+// === Snake / HalfSnake / Clamp (NeMo common.parts.utils) ===
+//
+// snake(x, alpha, eps) = x + (alpha + eps)^-1 * sin(alpha * x)^2
+// HalfSnake applies snake to the first half channels and LeakyReLU to the
+// rest.
+
+template <typename T>
+void audio_codec_half_snake(
+ device const T* input, // [B, C, T]
+ device const T* alpha, // [1, C_snake, 1] (contiguous, index by channel)
+ device T* output, // [B, C, T]
+ const constant int& channels,
+ const constant int& seq_len,
+ const constant int& snake_channels,
+ const constant float& negative_slope,
+ const constant float& eps,
+ const uint3 gid
+) {
+ const uint t = gid.x;
+ const uint c = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || c >= (uint)channels) {
+ return;
+ }
+
+ const uint idx = (b * (uint)channels + c) * (uint)seq_len + t;
+ const float x = float(input[idx]);
+
+ if ((int)c < snake_channels) {
+ const float a = float(alpha[c]);
+ const float ax = a * x;
+ const float s = sin(ax);
+ const float y = x + (1.0f / (a + eps)) * (s * s);
+ output[idx] = (T)y;
+ } else {
+ const float y = (x >= 0.0f) ? x : (negative_slope * x);
+ output[idx] = (T)y;
+ }
+}
+
+template <typename T>
+void audio_codec_clamp(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ const constant float& min_value,
+ const constant float& max_value,
+ const uint tid
+) {
+ if ((int)tid >= n) {
+ return;
+ }
+ const float x = float(input[tid]);
+ const float y = clamp(x, min_value, max_value);
+ output[tid] = (T)y;
+}
+
+// === Non-causal Conv1d with padding + stride (for encoder) ===
+//
+// Matches torch.nn.Conv1d with explicit padding and padding_mode:
+// out[t_out] = bias + sum_{ic,k} w[oc,ic,k] * x[ic, t_out*stride - padding
+// + k*dilation]
+// - pad_mode=0: zeros
+// - pad_mode=1: replicate (clamp index to [0, seq_len_in-1])
+template <typename T>
+void audio_codec_conv1d(
+ device const T* input, // [B, Cin, Tin]
+ device const T* weight, // [Cout, Cin, K]
+ device const T* bias, // [Cout]
+ device T* output, // [B, Cout, Tout]
+ device const int* lengths, // [B] (Tout lengths)
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& kernel_size,
+ const constant int& stride,
+ const constant int& dilation,
+ const constant int& padding,
+ const constant int& pad_mode,
+ const uint3 gid
+) {
+ const uint t_out = gid.x;
+ const uint oc = gid.y;
+ const uint b = gid.z;
+
+ if (t_out >= (uint)seq_len_out || oc >= (uint)cout) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len_out;
+ const uint out_idx = (b * (uint)cout + oc) * (uint)seq_len_out + t_out;
+ if ((int)t_out >= len_b) {
+ output[out_idx] = (T)0;
+ return;
+ }
+
+ const int base = (int)t_out * stride - padding;
+ float acc = float(bias[oc]);
+
+ for (int ic = 0; ic < cin; ++ic) {
+ const uint w_base = (oc * (uint)cin + (uint)ic) * (uint)kernel_size;
+ const uint x_base = (b * (uint)cin + (uint)ic) * (uint)seq_len_in;
+ for (int k = 0; k < kernel_size; ++k) {
+ int x_t = base + k * dilation;
+ if (pad_mode == 0) {
+ if (x_t < 0 || x_t >= seq_len_in) {
+ continue;
+ }
+ } else {
+ x_t = clamp(x_t, 0, seq_len_in - 1);
+ }
+ acc += float(weight[w_base + (uint)k]) * float(input[x_base + (uint)x_t]);
+ }
+ }
+
+ output[out_idx] = (T)acc;
+}
+
+// === FSQ (Finite Scalar Quantization) encode ===
+//
+// Implements GroupFiniteScalarQuantizer.encode() for inference:
+// - input: [B, G*D, T] float
+// - output: [B, G, T] int32
+//
+// NeMo reference (FiniteScalarQuantizer):
+// output_scale = (num_levels - 1) / 2 * (1 - eps)
+// output_offset = 0.5 if even else 0
+// input_shift = tan(output_offset / output_scale)
+// compressed = output_scale * tanh(inputs + input_shift) - output_offset
+// rounded = round(compressed) # half away from zero
+// code_nonneg = rounded + (num_levels//2)
+// token = sum_d code_nonneg[d] * dim_base_index[d]
+
+// Match torch.round() semantics (ties to even).
+inline float round_ties_to_even(const float x) {
+ const float f = floor(x);
+ const float frac = x - f;
+ if (frac < 0.5f) {
+ return f;
+ }
+ if (frac > 0.5f) {
+ return f + 1.0f;
+ }
+ const int fi = (int)f; // f is integral
+ return ((fi & 1) != 0) ? (f + 1.0f) : f;
+}
+
+template <typename T>
+void fsq_encode(
+ device const T* input, // [B, G*D, T]
+ device int* tokens, // [B, G, T]
+ device const int* lengths, // [B]
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const constant int* dim_base_index,
+ const constant float& eps,
+ const uint3 gid
+) {
+ const uint t = gid.x;
+ const uint g = gid.y;
+ const uint b = gid.z;
+
+ if (t >= (uint)seq_len || g >= (uint)num_groups) {
+ return;
+ }
+
+ const int len_b = lengths ? lengths[b] : seq_len;
+ const uint out_idx = (b * (uint)num_groups + g) * (uint)seq_len + t;
+ if ((int)t >= len_b) {
+ tokens[out_idx] = 0;
+ return;
+ }
+
+ const uint in_base =
+ (b * (uint)(num_groups * codebook_dim) + g * (uint)codebook_dim) *
+ (uint)seq_len +
+ t;
+
+ int token = 0;
+ for (int d = 0; d < codebook_dim; ++d) {
+ const int levels = num_levels[d];
+ const int scale_i = levels / 2;
+ const float output_scale = ((float)(levels - 1)) * 0.5f * (1.0f - eps);
+ const float output_offset = (levels % 2 == 0) ? 0.5f : 0.0f;
+ const float input_shift = tan(output_offset / output_scale);
+
+ const float x = float(input[in_base + (uint)d * (uint)seq_len]);
+ const float compressed =
+ output_scale * tanh(x + input_shift) - output_offset;
+ const float rounded = round_ties_to_even(compressed);
+
+ int code_nonneg = (int)rounded + scale_i;
+ code_nonneg = clamp(code_nonneg, 0, levels - 1);
+ token += code_nonneg * dim_base_index[d];
+ }
+
+ tokens[out_idx] = token;
+}
+
+// === DSL kernel wrappers ===
+//
+// Keep the scalar helper implementations above and expose a DSL-annotated
+// surface so audio kernels are integrated through the same generated wrapper
+// path as other kernels.
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioFsqDecode)(
+ device const int* tokens,
+ device T* out,
+ device const int* lengths,
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const constant int& batch_size,
+ uint t AXIS(seq_len, 32),
+ uint g AXIS(num_groups, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ fsq_decode<T>(
+ tokens,
+ out,
+ lengths,
+ num_groups,
+ seq_len,
+ codebook_dim,
+ num_levels,
+ uint3(t, g, b)
+ );
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioLeakyRelu)(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ const constant float& negative_slope,
+ uint tid AXIS(n, 256)
+) {
+ audio_codec_leaky_relu<T>(input, output, n, negative_slope, tid);
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioTanh)(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ uint tid AXIS(n, 256)
+) {
+ audio_codec_tanh<T>(input, output, n, tid);
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioAdd)(
+ device const T* a,
+ device const T* b,
+ device T* out,
+ const constant int& n,
+ uint tid AXIS(n, 256)
+) {
+ audio_codec_add<T>(a, b, out, n, tid);
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioScale)(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ const constant float& scale,
+ uint tid AXIS(n, 256)
+) {
+ audio_codec_scale<T>(input, output, n, scale, tid);
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioCausalConv1d)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len,
+ const constant int& kernel_size,
+ const constant int& dilation,
+ const constant int& batch_size,
+ uint t AXIS(seq_len, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ audio_codec_causal_conv1d<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len,
+ kernel_size,
+ dilation,
+ uint3(t, oc, b)
+ );
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioCausalConvTranspose1d)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& stride,
+ const constant int& groups,
+ const constant int& batch_size,
+ uint t_out AXIS(seq_len_out, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ audio_codec_causal_conv_transpose1d<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ stride,
+ groups,
+ uint3(t_out, oc, b)
+ );
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioHalfSnake)(
+ device const T* input,
+ device const T* alpha,
+ device T* output,
+ const constant int& channels,
+ const constant int& seq_len,
+ const constant int& snake_channels,
+ const constant float& negative_slope,
+ const constant float& eps,
+ const constant int& batch_size,
+ uint t AXIS(seq_len, 32),
+ uint c AXIS(channels, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ audio_codec_half_snake<T>(
+ input,
+ alpha,
+ output,
+ channels,
+ seq_len,
+ snake_channels,
+ negative_slope,
+ eps,
+ uint3(t, c, b)
+ );
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioClamp)(
+ device const T* input,
+ device T* output,
+ const constant int& n,
+ const constant float& min_value,
+ const constant float& max_value,
+ uint tid AXIS(n, 256)
+) {
+ audio_codec_clamp<T>(input, output, n, min_value, max_value, tid);
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioConv1d)(
+ device const T* input,
+ device const T* weight,
+ device const T* bias,
+ device T* output,
+ device const int* lengths,
+ const constant int& cin,
+ const constant int& cout,
+ const constant int& seq_len_in,
+ const constant int& seq_len_out,
+ const constant int& kernel_size,
+ const constant int& stride,
+ const constant int& dilation,
+ const constant int& padding,
+ const constant int& pad_mode,
+ const constant int& batch_size,
+ uint t_out AXIS(seq_len_out, 32),
+ uint oc AXIS(cout, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ audio_codec_conv1d<T>(
+ input,
+ weight,
+ bias,
+ output,
+ lengths,
+ cin,
+ cout,
+ seq_len_in,
+ seq_len_out,
+ kernel_size,
+ stride,
+ dilation,
+ padding,
+ pad_mode,
+ uint3(t_out, oc, b)
+ );
+}
+
+template <typename T>
+VARIANTS(T, float, half, bfloat)
+KERNEL(AudioFsqEncode)(
+ device const T* input,
+ device int* tokens,
+ device const int* lengths,
+ const constant int& num_groups,
+ const constant int& seq_len,
+ const constant int& codebook_dim,
+ const constant int* num_levels,
+ const constant int* dim_base_index,
+ const constant float& eps,
+ const constant int& batch_size,
+ uint t AXIS(seq_len, 32),
+ uint g AXIS(num_groups, 1),
+ uint b AXIS(batch_size, 1)
+) {
+ fsq_encode<T>(
+ input,
+ tokens,
+ lengths,
+ num_groups,
+ seq_len,
+ codebook_dim,
+ num_levels,
+ dim_base_index,
+ eps,
+ uint3(t, g, b)
+ );
+}
diff --git a/crates/uzu/src/backends/metal/kernel/mod.rs b/crates/uzu/src/backends/metal/kernel/mod.rs
index bd94bb20f..8f0af55a2 100644
--- a/crates/uzu/src/backends/metal/kernel/mod.rs
+++ b/crates/uzu/src/backends/metal/kernel/mod.rs
@@ -1,4 +1,5 @@
use crate::backends::common::kernel::matmul::MatmulKernels;
+
pub mod dsl {
include!(concat!(env!("OUT_DIR"), "/dsl.rs"));
}
diff --git a/crates/uzu/src/lib.rs b/crates/uzu/src/lib.rs
index 4adb560e6..1a0c2b99d 100644
--- a/crates/uzu/src/lib.rs
+++ b/crates/uzu/src/lib.rs
@@ -1,4 +1,5 @@
pub mod array;
+pub mod audio;
pub mod backends;
pub mod classifier;
pub mod config;
diff --git a/crates/uzu/src/prelude.rs b/crates/uzu/src/prelude.rs
index a842d9cb5..a6b2cc3ab 100644
--- a/crates/uzu/src/prelude.rs
+++ b/crates/uzu/src/prelude.rs
@@ -11,6 +11,7 @@
// Speculators
pub use crate::{
VERSION,
+ audio::{AudioCodecRuntime, AudioPcmBatch, AudioTokenGrid, AudioTokenPacking},
session::{
ChatSession, ClassificationSession,
config::{DecodingConfig, RunConfig, SpeculatorConfig},
diff --git a/crates/uzu/tests/audio_codec_test.rs b/crates/uzu/tests/audio_codec_test.rs
new file mode 100644
index 000000000..dd5e2475d
--- /dev/null
+++ b/crates/uzu/tests/audio_codec_test.rs
@@ -0,0 +1,813 @@
+#![cfg(all(feature = "audio-runtime", target_os = "macos"))]
+
+use metal::{MTLCommandBuffer, MTLCommandEncoder, MTLCommandQueue};
+use uzu::{
+ DataType,
+ array::ArrayContextExt,
+ backends::{
+ common::{
+ Context,
+ kernel::{
+ AudioAddKernel, AudioCausalConv1dKernel, AudioCausalConvTranspose1dKernel, AudioClampKernel,
+ AudioConv1dKernel, AudioFsqDecodeKernel, AudioFsqEncodeKernel, AudioHalfSnakeKernel,
+ AudioLeakyReluKernel, AudioScaleKernel, AudioTanhKernel,
+ },
+ },
+ metal::{
+ MetalContext,
+ kernel::dsl::{
+ AudioAddMetalKernel, AudioCausalConv1dMetalKernel, AudioCausalConvTranspose1dMetalKernel,
+ AudioClampMetalKernel, AudioConv1dMetalKernel, AudioFsqDecodeMetalKernel, AudioFsqEncodeMetalKernel,
+ AudioHalfSnakeMetalKernel, AudioLeakyReluMetalKernel, AudioScaleMetalKernel, AudioTanhMetalKernel,
+ },
+ },
+ },
+};
+
+fn create_test_context() -> std::rc::Rc<MetalContext> {
+ <MetalContext as Context>::new().expect("MetalContext")
+}
+
+#[test]
+fn audio_conv1d_replicate_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = AudioConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let batch_size = 2usize;
+ let cin = 3usize;
+ let cout = 4usize;
+ let seq_len_in = 9usize;
+
+ let stride = 3usize;
+ let kernel_size = 2 * stride;
+ let dilation = 1usize;
+ let padding = 2usize;
+
+ let seq_len_out = ((seq_len_in + 2 * padding - dilation * (kernel_size - 1) - 1) / stride) + 1;
+ let lengths_out: [i32; 2] = [2, seq_len_out as i32];
+
+ let input_len = batch_size * cin * seq_len_in;
+ let weight_len = cout * cin * kernel_size;
+ let bias_len = cout;
+ let output_len = batch_size * cout * seq_len_out;
+
+ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.01).sin() * 0.5).collect();
+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07 - 1.3).cos() * 0.2).collect();
+ let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.01 - 0.02).collect();
+
+ let mut input = context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_conv1d_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_conv1d_weight");
+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
+
+ let mut bias = context.create_array(&[cout], DataType::F32, "audio_conv1d_bias");
+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
+
+ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_conv1d_lengths");
+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
+
+ let output = context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_conv1d_output");
+
+ let mut expected = vec![0.0f32; output_len];
+ for batch in 0..batch_size {
+ for out_channel in 0..cout {
+ for out_time in 0..seq_len_out {
+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;
+ if (out_time as i32) >= lengths_out[batch] {
+ expected[out_index] = 0.0;
+ continue;
+ }
+
+ let mut acc = bias_values[out_channel];
+ let base = (out_time * stride) as isize - padding as isize;
+
+ for in_channel in 0..cin {
+ let x_base = (batch * cin + in_channel) * seq_len_in;
+ let w_base = (out_channel * cin + in_channel) * kernel_size;
+ for kernel_offset in 0..kernel_size {
+ let mut x_time = base + (kernel_offset * dilation) as isize;
+ if x_time < 0 {
+ x_time = 0;
+ } else if x_time >= seq_len_in as isize {
+ x_time = seq_len_in as isize - 1;
+ }
+
+ let x_index = x_base + x_time as usize;
+ let w_index = w_base + kernel_offset;
+ acc += weight_values[w_index] * input_values[x_index];
+ }
+ }
+
+ expected[out_index] = acc;
+ }
+ }
+ }
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(
+ input.buffer(),
+ weight.buffer(),
+ bias.buffer(),
+ output.buffer(),
+ lengths.buffer(),
+ cin as i32,
+ cout as i32,
+ seq_len_in as i32,
+ seq_len_out as i32,
+ kernel_size as i32,
+ stride as i32,
+ dilation as i32,
+ padding as i32,
+ 1_i32,
+ batch_size as i32,
+ &encoder,
+ );
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let got = output.as_slice::<f32>();
+ for index in 0..output_len {
+ let delta = (expected[index] - got[index]).abs();
+ assert!(
+ delta <= 1e-4,
+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",
+ expected[index],
+ got[index]
+ );
+ }
+}
+
+#[test]
+fn audio_causal_conv1d_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = AudioCausalConv1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let batch_size = 2usize;
+ let cin = 3usize;
+ let cout = 4usize;
+ let seq_len = 8usize;
+ let kernel_size = 5usize;
+ let dilation = 2usize;
+ let pad = (kernel_size - 1) * dilation;
+
+ let lengths: [i32; 2] = [5, 8];
+
+ let input_len = batch_size * cin * seq_len;
+ let weight_len = cout * cin * kernel_size;
+ let bias_len = cout;
+ let output_len = batch_size * cout * seq_len;
+
+ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.01).sin()).collect();
+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.02 - 0.5).cos() * 0.1).collect();
+ let bias_values: Vec<f32> = (0..bias_len).map(|i| i as f32 * 0.001 - 0.002).collect();
+
+ let mut input = context.create_array(&[batch_size, cin, seq_len], DataType::F32, "audio_causal_conv1d_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let mut weight = context.create_array(&[cout, cin, kernel_size], DataType::F32, "audio_causal_conv1d_weight");
+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
+
+ let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv1d_bias");
+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
+
+ let mut lengths_array = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv1d_lengths");
+ lengths_array.as_slice_mut::<i32>().copy_from_slice(&lengths);
+
+ let output = context.create_array(&[batch_size, cout, seq_len], DataType::F32, "audio_causal_conv1d_output");
+
+ let mut expected = vec![0.0f32; output_len];
+ for batch in 0..batch_size {
+ for out_channel in 0..cout {
+ for time in 0..seq_len {
+ let out_index = (batch * cout + out_channel) * seq_len + time;
+ if (time as i32) >= lengths[batch] {
+ expected[out_index] = 0.0;
+ continue;
+ }
+
+ let mut acc = bias_values[out_channel];
+ for in_channel in 0..cin {
+ for kernel_offset in 0..kernel_size {
+ let x_time = time as isize + (kernel_offset * dilation) as isize - pad as isize;
+ if x_time < 0 || x_time >= seq_len as isize {
+ continue;
+ }
+
+ let x_index = (batch * cin + in_channel) * seq_len + x_time as usize;
+ let w_index = (out_channel * cin + in_channel) * kernel_size + kernel_offset;
+ acc += weight_values[w_index] * input_values[x_index];
+ }
+ }
+
+ expected[out_index] = acc;
+ }
+ }
+ }
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(
+ input.buffer(),
+ weight.buffer(),
+ bias.buffer(),
+ output.buffer(),
+ lengths_array.buffer(),
+ cin as i32,
+ cout as i32,
+ seq_len as i32,
+ kernel_size as i32,
+ dilation as i32,
+ batch_size as i32,
+ &encoder,
+ );
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let got = output.as_slice::<f32>();
+ for index in 0..output_len {
+ let delta = (expected[index] - got[index]).abs();
+ assert!(
+ delta <= 1e-4,
+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",
+ expected[index],
+ got[index]
+ );
+ }
+}
+
+#[test]
+fn audio_causal_conv_transpose1d_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = AudioCausalConvTranspose1dMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let batch_size = 2usize;
+ let cin = 4usize;
+ let groups = 2usize;
+ let cout = 2usize;
+ let stride = 3usize;
+ let kernel_size = 2 * stride;
+
+ let seq_len_in = 5usize;
+ let seq_len_out = seq_len_in * stride;
+
+ let lengths_out: [i32; 2] = [9, 15];
+
+ let input_len = batch_size * cin * seq_len_in;
+ let weight_len = cin * (cout / groups) * kernel_size;
+ let output_len = batch_size * cout * seq_len_out;
+
+ let input_values: Vec<f32> = (0..input_len).map(|i| (i as f32 * 0.013).sin() * 0.5).collect();
+ let weight_values: Vec<f32> = (0..weight_len).map(|i| ((i as f32) * 0.07).cos() * 0.2).collect();
+ let bias_values: Vec<f32> = vec![0.01, -0.02];
+
+ let mut input =
+ context.create_array(&[batch_size, cin, seq_len_in], DataType::F32, "audio_causal_conv_transpose_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let mut weight =
+ context.create_array(&[cin, cout / groups, kernel_size], DataType::F32, "audio_causal_conv_transpose_weight");
+ weight.as_slice_mut::<f32>().copy_from_slice(&weight_values);
+
+ let mut bias = context.create_array(&[cout], DataType::F32, "audio_causal_conv_transpose_bias");
+ bias.as_slice_mut::<f32>().copy_from_slice(&bias_values);
+
+ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_causal_conv_transpose_lengths");
+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_out);
+
+ let output =
+ context.create_array(&[batch_size, cout, seq_len_out], DataType::F32, "audio_causal_conv_transpose_output");
+
+ let cout_per_group = cout / groups;
+ let cin_per_group = cin / groups;
+ let mut expected = vec![0.0f32; output_len];
+
+ for batch in 0..batch_size {
+ for out_channel in 0..cout {
+ let group_index = out_channel / cout_per_group;
+ let out_channel_in_group = out_channel % cout_per_group;
+ let in_channel_begin = group_index * cin_per_group;
+ let in_channel_end = in_channel_begin + cin_per_group;
+
+ for out_time in 0..seq_len_out {
+ let out_index = (batch * cout + out_channel) * seq_len_out + out_time;
+ if (out_time as i32) >= lengths_out[batch] {
+ expected[out_index] = 0.0;
+ continue;
+ }
+
+ let q = out_time / stride;
+ let r = out_time % stride;
+ let mut acc = bias_values[out_channel];
+
+ for in_channel in in_channel_begin..in_channel_end {
+ let input_base = (batch * cin + in_channel) * seq_len_in;
+ let weight_base = (in_channel * cout_per_group + out_channel_in_group) * kernel_size;
+
+ acc += input_values[input_base + q] * weight_values[weight_base + r];
+ if q > 0 {
+ acc += input_values[input_base + (q - 1)] * weight_values[weight_base + (stride + r)];
+ }
+ }
+
+ expected[out_index] = acc;
+ }
+ }
+ }
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(
+ input.buffer(),
+ weight.buffer(),
+ bias.buffer(),
+ output.buffer(),
+ lengths.buffer(),
+ cin as i32,
+ cout as i32,
+ seq_len_in as i32,
+ seq_len_out as i32,
+ stride as i32,
+ groups as i32,
+ batch_size as i32,
+ &encoder,
+ );
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let got = output.as_slice::<f32>();
+ for index in 0..output_len {
+ let delta = (expected[index] - got[index]).abs();
+ assert!(
+ delta <= 1e-4,
+ "Mismatch at index={index}: expected {}, got {}, delta={delta}",
+ expected[index],
+ got[index]
+ );
+ }
+}
+
+#[test]
+fn audio_leaky_relu_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = AudioLeakyReluMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let n = 1024usize;
+ let slope = 0.01f32;
+ let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
+
+ let mut input = context.create_array(&[n], DataType::F32, "audio_leaky_relu_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let output = context.create_array(&[n], DataType::F32, "audio_leaky_relu_output");
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(input.buffer(), output.buffer(), n as i32, slope, &encoder);
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let got = output.as_slice::<f32>();
+ for index in 0..n {
+ let expected = if input_values[index] >= 0.0 {
+ input_values[index]
+ } else {
+ slope * input_values[index]
+ };
+ let delta = (expected - got[index]).abs();
+ assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);
+ }
+}
+
+#[test]
+fn audio_tanh_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = AudioTanhMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let n = 1024usize;
+ let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 5.12).collect();
+
+ let mut input = context.create_array(&[n], DataType::F32, "audio_tanh_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let output = context.create_array(&[n], DataType::F32, "audio_tanh_output");
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(input.buffer(), output.buffer(), n as i32, &encoder);
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let got = output.as_slice::<f32>();
+ for index in 0..n {
+ let expected = input_values[index].tanh();
+ let delta = (expected - got[index]).abs();
+ assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);
+ }
+}
+
+#[test]
+fn audio_add_and_scale_match_reference_f32() {
+ let context = create_test_context();
+ let add_kernel = AudioAddMetalKernel::new(&context, DataType::F32).expect("audio add kernel");
+ let scale_kernel = AudioScaleMetalKernel::new(&context, DataType::F32).expect("audio scale kernel");
+
+ let n = 2048usize;
+ let scale_value = 1.0f32 / 3.0f32;
+ let a_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.001 - 1.0).collect();
+ let b_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.002).sin()).collect();
+
+ let mut a = context.create_array(&[n], DataType::F32, "audio_add_a");
+ a.as_slice_mut::<f32>().copy_from_slice(&a_values);
+
+ let mut b = context.create_array(&[n], DataType::F32, "audio_add_b");
+ b.as_slice_mut::<f32>().copy_from_slice(&b_values);
+
+ let sum = context.create_array(&[n], DataType::F32, "audio_add_sum");
+ let scaled = context.create_array(&[n], DataType::F32, "audio_add_scaled");
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ add_kernel.encode(a.buffer(), b.buffer(), sum.buffer(), n as i32, &encoder);
+
+ scale_kernel.encode(sum.buffer(), scaled.buffer(), n as i32, scale_value, &encoder);
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let sum_got = sum.as_slice::<f32>();
+ let scaled_got = scaled.as_slice::<f32>();
+
+ for index in 0..n {
+ let expected_sum = a_values[index] + b_values[index];
+ let expected_scaled = expected_sum * scale_value;
+
+ let sum_delta = (expected_sum - sum_got[index]).abs();
+ let scaled_delta = (expected_scaled - scaled_got[index]).abs();
+
+ assert!(
+ sum_delta <= 1e-6,
+ "Add mismatch at index={index}: expected {expected_sum}, got {}, delta={sum_delta}",
+ sum_got[index]
+ );
+ assert!(
+ scaled_delta <= 1e-6,
+ "Scale mismatch at index={index}: expected {expected_scaled}, got {}, delta={scaled_delta}",
+ scaled_got[index]
+ );
+ }
+}
+
+#[test]
+fn audio_clamp_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = AudioClampMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let n = 2048usize;
+ let min_value = -1.0f32;
+ let max_value = 1.0f32;
+ let input_values: Vec<f32> = (0..n).map(|i| i as f32 * 0.01 - 10.24).collect();
+
+ let mut input = context.create_array(&[n], DataType::F32, "audio_clamp_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let output = context.create_array(&[n], DataType::F32, "audio_clamp_output");
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(input.buffer(), output.buffer(), n as i32, min_value, max_value, &encoder);
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let got = output.as_slice::<f32>();
+ for index in 0..n {
+ let expected = input_values[index].clamp(min_value, max_value);
+ let delta = (expected - got[index]).abs();
+ assert!(delta <= 1e-6, "Mismatch at index={index}: expected {expected}, got {}, delta={delta}", got[index]);
+ }
+}
+
+#[test]
+fn audio_half_snake_matches_reference_f32() {
+ let context = create_test_context();
+ let kernel = AudioHalfSnakeMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let batch_size = 1usize;
+ let channels = 6usize;
+ let snake_channels = channels / 2;
+ let seq_len = 64usize;
+ let n = batch_size * channels * seq_len;
+
+ let negative_slope = 0.01f32;
+ let eps = 1e-9f32;
+
+ let input_values: Vec<f32> = (0..n).map(|i| (i as f32 * 0.01 - 3.2).sin()).collect();
+ let alpha_values: Vec<f32> = (0..snake_channels).map(|i| 0.5 + i as f32 * 0.1).collect();
+
+ let mut input = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_input");
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let mut alpha = context.create_array(&[snake_channels], DataType::F32, "audio_half_snake_alpha");
+ alpha.as_slice_mut::<f32>().copy_from_slice(&alpha_values);
+
+ let output = context.create_array(&[batch_size, channels, seq_len], DataType::F32, "audio_half_snake_output");
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(
+ input.buffer(),
+ alpha.buffer(),
+ output.buffer(),
+ channels as i32,
+ seq_len as i32,
+ snake_channels as i32,
+ negative_slope,
+ eps,
+ batch_size as i32,
+ &encoder,
+ );
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let got = output.as_slice::<f32>();
+ for batch in 0..batch_size {
+ for channel in 0..channels {
+ for time in 0..seq_len {
+ let index = (batch * channels + channel) * seq_len + time;
+ let x = input_values[index];
+ let expected = if channel < snake_channels {
+ let alpha = alpha_values[channel];
+ let sine = (alpha * x).sin();
+ x + (sine * sine) / (alpha + eps)
+ } else if x >= 0.0 {
+ x
+ } else {
+ negative_slope * x
+ };
+
+ let delta = (expected - got[index]).abs();
+ assert!(
+ delta <= 1e-5,
+ "Mismatch at index={index}: expected {expected}, got {}, delta={delta}",
+ got[index]
+ );
+ }
+ }
+ }
+}
+
+#[test]
+fn audio_fsq_decode_matches_reference() {
+ let context = create_test_context();
+ let kernel = AudioFsqDecodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let batch_size = 1usize;
+ let num_groups = 2usize;
+ let seq_len = 3usize;
+ let codebook_dim = 2usize;
+ let num_levels: Vec<i32> = vec![8, 5];
+
+ let mut tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_tokens");
+ tokens.as_slice_mut::<i32>().copy_from_slice(&[
+ 0, 7, 11, // g0 tokens at t0..t2
+ 3, 5, 9, // g1 tokens at t0..t2
+ ]);
+
+ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_lengths");
+ lengths.as_slice_mut::<i32>().copy_from_slice(&[2]);
+
+ let output =
+ context.create_array(&[batch_size, num_groups * codebook_dim, seq_len], DataType::F32, "audio_fsq_output");
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(
+ tokens.buffer(),
+ output.buffer(),
+ lengths.buffer(),
+ num_groups as i32,
+ seq_len as i32,
+ codebook_dim as i32,
+ &num_levels,
+ batch_size as i32,
+ &encoder,
+ );
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let expected =
+ reference_fsq_decode(&[0, 7, 11, 3, 5, 9], 2, batch_size, num_groups, seq_len, codebook_dim, &num_levels);
+ let actual = output.as_slice::<f32>();
+
+ for (index, (&a, &e)) in actual.iter().zip(expected.iter()).enumerate() {
+ let delta = (a - e).abs();
+ assert!(delta <= 1e-5, "index={index}, actual={a}, expected={e}, delta={delta}");
+ }
+}
+
+#[test]
+fn audio_fsq_encode_matches_reference() {
+ let context = create_test_context();
+ let kernel = AudioFsqEncodeMetalKernel::new(&context, DataType::F32).expect("audio runtime");
+
+ let batch_size = 1usize;
+ let num_groups = 2usize;
+ let codebook_dim = 2usize;
+ let seq_len = 4usize;
+ let eps = 1e-3_f32;
+ let num_levels: Vec<i32> = vec![8, 6];
+ let dim_base_index: Vec<i32> = vec![1, 8];
+
+ let input_values = vec![
+ -0.9, -0.1, 0.2, 0.7, // g0 d0
+ 0.8, 0.3, -0.4, -0.7, // g0 d1
+ -0.2, 0.0, 0.4, 0.9, // g1 d0
+ 0.5, -0.5, 0.1, -0.1, // g1 d1
+ ];
+ let lengths_values = vec![3_i32];
+
+ let mut input = context.create_array(
+ &[batch_size, num_groups * codebook_dim, seq_len],
+ DataType::F32,
+ "audio_fsq_encode_input",
+ );
+ input.as_slice_mut::<f32>().copy_from_slice(&input_values);
+
+ let mut lengths = context.create_array(&[batch_size], DataType::I32, "audio_fsq_encode_lengths");
+ lengths.as_slice_mut::<i32>().copy_from_slice(&lengths_values);
+
+ let tokens = context.create_array(&[batch_size, num_groups, seq_len], DataType::I32, "audio_fsq_encode_tokens");
+
+ let command_buffer = context.command_queue.command_buffer().expect("command buffer");
+ let encoder = command_buffer.new_compute_command_encoder().expect("compute encoder");
+
+ kernel.encode(
+ input.buffer(),
+ tokens.buffer(),
+ lengths.buffer(),
+ num_groups as i32,
+ seq_len as i32,
+ codebook_dim as i32,
+ &num_levels,
+ &dim_base_index,
+ eps,
+ batch_size as i32,
+ &encoder,
+ );
+
+ encoder.end_encoding();
+ command_buffer.commit();
+ command_buffer.wait_until_completed();
+
+ let expected = reference_fsq_encode(
+ &input_values,
+ &lengths_values,
+ batch_size,
+ num_groups,
+ seq_len,
+ codebook_dim,
+ &num_levels,
+ &dim_base_index,
+ eps,
+ );
+ let actual = tokens.as_slice::<i32>();
+
+ assert_eq!(actual, expected);
+}
+
+fn round_ties_to_even(value: f32) -> f32 {
+ let floor = value.floor();
+ let fraction = value - floor;
+ if fraction < 0.5 {
+ floor
+ } else if fraction > 0.5 {
+ floor + 1.0
+ } else if (floor as i64 & 1) != 0 {
+ floor + 1.0
+ } else {
+ floor
+ }
+}
+
+fn reference_fsq_encode(
+ input: &[f32],
+ lengths: &[i32],
+ batch_size: usize,
+ num_groups: usize,
+ seq_len: usize,
+ codebook_dim: usize,
+ num_levels: &[i32],
+ dim_base_index: &[i32],
+ eps: f32,
+) -> Vec<i32> {
+ let mut tokens = vec![0_i32; batch_size * num_groups * seq_len];
+
+ for batch in 0..batch_size {
+ for group in 0..num_groups {
+ for time in 0..seq_len {
+ let output_index = (batch * num_groups + group) * seq_len + time;
+ if (time as i32) >= lengths[batch] {
+ tokens[output_index] = 0;
+ continue;
+ }
+
+ let mut token = 0_i32;
+ for dim in 0..codebook_dim {
+ let levels = num_levels[dim];
+ let scale_i = levels / 2;
+ let output_scale = (levels - 1) as f32 * 0.5 * (1.0 - eps);
+ let output_offset = if levels % 2 == 0 {
+ 0.5
+ } else {
+ 0.0
+ };
+ let input_shift = (output_offset / output_scale).tan();
+
+ let input_index =
+ (batch * (num_groups * codebook_dim) + group * codebook_dim + dim) * seq_len + time;
+ let compressed = output_scale * (input[input_index] + input_shift).tanh() - output_offset;
+ let rounded = round_ties_to_even(compressed);
+ let code_nonnegative = (rounded as i32 + scale_i).clamp(0, levels - 1);
+ token += code_nonnegative * dim_base_index[dim];
+ }
+
+ tokens[output_index] = token;
+ }
+ }
+ }
+
+ tokens
+}
+
+fn reference_fsq_decode(
+ tokens: &[i32],
+ length: usize,
+ batch_size: usize,
+ num_groups: usize,
+ seq_len: usize,
+ codebook_dim: usize,
+ num_levels: &[i32],
+) -> Vec<f32> {
+ let mut out = vec![0.0f32; batch_size * num_groups * codebook_dim * seq_len];
+
+ for b in 0..batch_size {
+ for g in 0..num_groups {
+ for t in 0..seq_len {
+ if t >= length {
+ continue;
+ }
+
+ let token_index = (b * num_groups + g) * seq_len + t;
+ let token = tokens[token_index];
+
+ let mut base = 1i32;
+ for d in 0..codebook_dim {
+ let levels = num_levels[d];
+ let scale = levels / 2;
+ let offset = scale;
+ let div = token / base;
+ let mut code_nonneg = div % levels;
+ if code_nonneg < 0 {
+ code_nonneg += levels;
+ }
+ let value = (code_nonneg - offset) as f32 / scale as f32;
+
+ let out_channel = g * codebook_dim + d;
+ let out_index = (b * (num_groups * codebook_dim) + out_channel) * seq_len + t;
+ out[out_index] = value;
+
+ base *= levels;
+ }
+ }
+ }
+ }
+
+ out
+}