A high-performance inference engine for AI models
Gpu types enum (#264)
Aleksandr Golokozf222b5a53d44
diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rsindex 30826ed30..e012805dd 100644--- a/crates/uzu/build/gpu_types/mod.rs+++ b/crates/uzu/build/gpu_types/mod.rs@@ -2,7 +2,7 @@ use std::{collections::HashMap, env, fs, path::PathBuf};use anyhow::Context;use async_trait::async_trait;-use syn::{Expr, ExprLit, Fields, Item, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};+use syn::{Attribute, Expr, ExprLit, Fields, Item, ItemEnum, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};use tokio::task::spawn_blocking;use crate::{@@ -91,26 +91,42 @@ impl GpuTypesCompiler {let syntax = syn::parse_file(&source).with_context(|| format!("cannot parse {}", src_path.display()))?;+ let mut repr_c_enums: Vec<ItemEnum> = Vec::new();let mut repr_c_structs: Vec<ItemStruct> = Vec::new();for item in syntax.items {- if let Item::Struct(item_struct) = item {- let has_repr_c = item_struct.attrs.iter().any(|attribute| {- if attribute.path().is_ident("repr") {- if let Ok(ident) = attribute.parse_args::<syn::Ident>() {- return ident == "C";- }- }- false- });-- if has_repr_c {+ if let Item::Enum(item_enum) = item {+ if have_repr_c_attr(&item_enum.attrs) {+ repr_c_enums.push(item_enum);+ }+ } else if let Item::Struct(item_struct) = item {+ if have_repr_c_attr(&item_struct.attrs) {repr_c_structs.push(item_struct);}}}- let mut c_struct_definitions = String::new();- for rust_struct in &repr_c_structs {+ let c_enum_definitions = Self::get_enum_definitions(&repr_c_enums);+ let c_struct_definitions = Self::get_struct_definitions(&repr_c_structs);++ let mut content = String::new();+ if !c_enum_definitions.is_empty() {+ content.push_str(&c_enum_definitions);+ }+ if !c_struct_definitions.is_empty() {+ if !content.is_empty() {+ content.push_str("\n\n");+ }+ content.push_str(&c_struct_definitions);+ }++ self.write_header(module_name, content.as_str())?;++ Ok(())+ }++ fn get_struct_definitions(c_structs: &Vec<ItemStruct>) -> String {+ let mut definitions = String::new();+ for (i, rust_struct) in c_structs.iter().enumerate() {for attribute in &rust_struct.attrs {if attribute.path().is_ident("doc") {if let Meta::NameValue(meta_name_value) = &attribute.meta {@@ -122,16 +138,16 @@ impl GpuTypesCompiler {let doc_comment = literal_string.value();let doc_comment = doc_comment.trim();if !doc_comment.is_empty() {- c_struct_definitions.push_str("/**");- c_struct_definitions.push_str(doc_comment);- c_struct_definitions.push_str(" */\n");+ definitions.push_str("/**");+ definitions.push_str(doc_comment);+ definitions.push_str(" */\n");}}}}}- c_struct_definitions.push_str("typedef struct {\n");+ definitions.push_str("typedef struct {\n");if let Fields::Named(named_fields) = &rust_struct.fields {for field in &named_fields.named {@@ -149,9 +165,9 @@ impl GpuTypesCompiler {let doc_comment = literal_string.value();let doc_comment = doc_comment.trim();if !doc_comment.is_empty() {- c_struct_definitions.push_str(" /**");- c_struct_definitions.push_str(doc_comment);- c_struct_definitions.push_str(" */\n");+ definitions.push_str(" /**");+ definitions.push_str(doc_comment);+ definitions.push_str(" */\n");}}}@@ -159,26 +175,49 @@ impl GpuTypesCompiler {}if let Some((base_type, array_size)) = c_type.split_once('[') {- c_struct_definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));+ definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));} else {- c_struct_definitions.push_str(&format!(" {c_type} {field_name};\n"));+ definitions.push_str(&format!(" {c_type} {field_name};\n"));}}}let struct_name = rust_struct.ident.to_string();- c_struct_definitions.push_str(&format!("}} {struct_name};\n\n"));+ definitions.push_str(&format!("}} {struct_name};"));+ if i < c_structs.len() - 1 {+ definitions.push_str("\n\n");+ }}- self.write_header(module_name, c_struct_definitions.trim())?;+ definitions+ }++ fn get_enum_definitions(enums: &Vec<ItemEnum>) -> String {+ let mut definitions = String::new();+ for (i, rust_enum) in enums.iter().enumerate() {+ let enum_name = rust_enum.ident.to_string();+ definitions.push_str("enum ");+ definitions.push_str(&enum_name);+ definitions.push_str(" {\n");++ for variant in &rust_enum.variants {+ definitions.push_str(" ");+ definitions.push_str(variant.ident.to_string().as_str());+ definitions.push_str(",\n");+ }- Ok(())+ definitions.push_str("};");+ if i < enums.len() - 1 {+ definitions.push_str("\n\n");+ }+ }+ definitions}fn write_header(&self,module_name: &str,- structs: &str,+ content: &str,) -> anyhow::Result<()> {let include_guard = format!("UZU_{}_H", module_name.to_uppercase());@@ -199,7 +238,7 @@ namespace {module_name} {{#include <stdint.h>#endif-{structs}+{content}#ifdef __METAL_VERSION__}} // namespace {module_name}@@ -266,6 +305,17 @@ fn rust_type_to_c(rust_type: &Type) -> String {}}+fn have_repr_c_attr(attrs: &Vec<Attribute>) -> bool {+ attrs.iter().any(|attr| {+ if attr.path().is_ident("repr") {+ if let Ok(ident) = attr.parse_args::<syn::Ident>() {+ return ident == "C";+ }+ }+ false+ })+}+#[async_trait]impl Compiler for GpuTypesCompiler {async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {diff --git a/crates/uzu/src/backends/common/activation_config.rs b/crates/uzu/src/backends/common/activation_config.rsnew file mode 100644index 000000000..c054a6e52--- /dev/null+++ b/crates/uzu/src/backends/common/activation_config.rs@@ -0,0 +1,49 @@+use serde::{Deserialize, Serialize};++use crate::backends::common::gpu_types::ActivationType;++#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]+#[serde(tag = "type")]+pub enum ActivationConfig {+ #[serde(rename = "SiLU")]+ SILU {+ #[serde(default = "default_silu_alpha")]+ alpha: f32,+ },+ #[serde(rename = "GELU")]+ GELU,+ #[serde(rename = "Identity")]+ IDENTITY,+}++impl ActivationConfig {+ pub fn silu_default() -> ActivationConfig {+ ActivationConfig::SILU {+ alpha: 1.0,+ }+ }++ pub fn act_type(&self) -> ActivationType {+ match self {+ ActivationConfig::SILU {+ ..+ } => ActivationType::SILU,+ ActivationConfig::GELU => ActivationType::GELU,+ ActivationConfig::IDENTITY => ActivationType::IDENTITY,+ }+ }++ pub fn alpha(&self) -> f32 {+ match self {+ ActivationConfig::SILU {+ alpha,+ } => *alpha,+ ActivationConfig::GELU => 1.0,+ ActivationConfig::IDENTITY => 1.0,+ }+ }+}++fn default_silu_alpha() -> f32 {+ 1.0+}diff --git a/crates/uzu/src/backends/common/gpu_types/activation_type.rs b/crates/uzu/src/backends/common/gpu_types/activation_type.rsnew file mode 100644index 000000000..939965237--- /dev/null+++ b/crates/uzu/src/backends/common/gpu_types/activation_type.rs@@ -0,0 +1,38 @@+use num_traits::Float;++const GELU_K0: f32 = 0.044715;+const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)++#[repr(C)]+#[derive(Debug, Copy, Clone, PartialEq)]+pub enum ActivationType {+ SILU,+ GELU,+ IDENTITY,+}++impl ActivationType {+ pub fn activate<T: Float>(+ &self,+ x: T,+ ) -> T {+ match self {+ ActivationType::SILU => silu(x),+ ActivationType::GELU => gelu(x),+ ActivationType::IDENTITY => x,+ }+ }+}++fn silu<T: Float>(x: T) -> T {+ let x_float = x.to_f32().unwrap();+ let y_float = x_float / (1.0f32 + (-x_float).exp());+ T::from(y_float).unwrap()+}++fn gelu<T: Float>(x: T) -> T {+ let x_float = x.to_f32().unwrap();+ let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);+ let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());+ T::from(y_float).unwrap()+}diff --git a/crates/uzu/src/backends/common/gpu_types/mod.rs b/crates/uzu/src/backends/common/gpu_types/mod.rsindex 8d996182d..1111d902c 100644--- a/crates/uzu/src/backends/common/gpu_types/mod.rs+++ b/crates/uzu/src/backends/common/gpu_types/mod.rs@@ -3,12 +3,16 @@//! These `#[repr(C)]` structs are the source of truth. The build system uses//! cbindgen to generate C headers for Metal shaders.+pub mod activation_type;pub mod argmax;pub mod attention;pub mod kv_cache_update;pub mod matmul;+pub mod quantization;+pub use activation_type::*;pub use argmax::*;pub use attention::*;pub use kv_cache_update::*;pub use matmul::*;+pub use quantization::*;diff --git a/crates/uzu/src/backends/common/gpu_types/quantization.rs b/crates/uzu/src/backends/common/gpu_types/quantization.rsnew file mode 100644index 000000000..617afbe2b--- /dev/null+++ b/crates/uzu/src/backends/common/gpu_types/quantization.rs@@ -0,0 +1,40 @@+use serde::{Deserialize, Serialize};++use crate::DataType;++#[repr(C)]+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]+#[serde(rename_all = "lowercase")]+pub enum QuantizationMode {+ UINT4,+ INT8,+ UINT8,+}++impl QuantizationMode {+ pub fn packing_divisor(&self) -> usize {+ match self {+ QuantizationMode::UINT4 => 2,+ QuantizationMode::INT8 => 1,+ QuantizationMode::UINT8 => 1,+ }+ }++ pub fn storage_type(&self) -> DataType {+ match self {+ QuantizationMode::UINT4 => DataType::U8,+ QuantizationMode::INT8 => DataType::I8,+ QuantizationMode::UINT8 => DataType::U8,+ }+ }+}++impl From<QuantizationMode> for DataType {+ fn from(val: QuantizationMode) -> Self {+ match val {+ QuantizationMode::UINT4 => DataType::U4,+ QuantizationMode::INT8 => DataType::I8,+ QuantizationMode::UINT8 => DataType::U8,+ }+ }+}diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rsindex 6e83fefa2..2c5f1863a 100644--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs@@ -1,12 +1,13 @@use crate::{DataType,- backends::common::{Backend, CommandBuffer, Kernels, kernel::MlpGateActMulKernel},- config::Activation,+ backends::common::{+ ActivationConfig, Backend, CommandBuffer, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,+ },};pub struct MlpGateActMulEncodable<B: Backend> {kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,- activation: Activation,+ activation: ActivationConfig,hidden_dim: usize,}@@ -14,7 +15,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {pub fn new(context: &B::Context,data_type: DataType,- activation: Activation,+ activation: ActivationConfig,hidden_dim: usize,) -> Result<Self, B::Error> {let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type)?;@@ -32,16 +33,10 @@ impl<B: Backend> MlpGateActMulEncodable<B> {hidden: &mut B::Buffer,m: i32,) -> Result<(), B::Error> {- let act_type = match self.activation {- Activation::SiLU {- ..- } => 0,- Activation::Gelu => 1,- Activation::Identity => {- panic!("Identity activation is not supported for kernel")- },- };- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, command_buffer);+ if self.activation.act_type() == ActivationType::IDENTITY {+ panic!("Identity activation is not supported for kernel")+ }+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), command_buffer);Ok(())}}diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rsindex 40daa27df..e63b12039 100644--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs@@ -4,12 +4,12 @@ use crate::{DataType,backends::common::{Backend, CommandBuffer, Kernels,+ gpu_types::QuantizationMode,kernel::{QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,},},- config::QuantizationMode,};#[derive(Debug, thiserror::Error)]@@ -466,8 +466,8 @@ fn select_matrix_matrix_family(fn quant_bits<B: Backend>(mode: QuantizationMode) -> Result<usize, QuantizedMatmulError<B>> {let bits = match mode {- QuantizationMode::UInt4 => 4,- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,+ QuantizationMode::UINT4 => 4,+ QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,};if matches!(bits, 4 | 8) {Ok(bits)diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rsindex 660d6a9f8..8557fd41d 100644--- a/crates/uzu/src/backends/common/mod.rs+++ b/crates/uzu/src/backends/common/mod.rs@@ -1,3 +1,4 @@+mod activation_config;mod backend;mod buffer;mod command_buffer;@@ -6,6 +7,7 @@ mod event;pub mod gpu_types;pub mod kernel;+pub use activation_config::ActivationConfig;pub use backend::Backend;pub use buffer::Buffer;pub use command_buffer::{diff --git a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu/src/backends/cpu/kernel/activation/activation.rsindex 66d94c2f7..2b967ff48 100644--- a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs+++ b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs@@ -12,7 +12,7 @@ pub fn activation<T: ArrayElement + Float>(input: Option<*const T>,#[allow(unused)] output: *mut T,#[allow(unused)] n: u32,- #[allow(unused)] act_type: u32,+ #[allow(unused)] act_type: crate::backends::common::gpu_types::activation_type::ActivationType,#[allow(unused)]#[specialize]in_place: bool,diff --git a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs b/crates/uzu/src/backends/cpu/kernel/activation/mod.rsindex e70a83196..3e9730314 100644--- a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs+++ b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs@@ -1,49 +1 @@-use num_traits::Float;--use crate::ArrayElement;-pub mod activation;--const GELU_K0: f32 = 0.044715f32;-const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)--#[repr(u32)]-pub enum ActivationType {- SiLU,- GELU,-}--impl TryFrom<u32> for ActivationType {- type Error = ();-- fn try_from(value: u32) -> Result<Self, Self::Error> {- match value {- 0 => Ok(Self::SiLU),- 1 => Ok(Self::GELU),- _ => Err(()),- }- }-}--fn silu<T: Float>(x: T) -> T {- let x_float = x.to_f32().unwrap();- let y_float = x_float / (1.0f32 + (-x_float).exp());- T::from(y_float).unwrap()-}--fn gelu<T: Float>(x: T) -> T {- let x_float = x.to_f32().unwrap();- let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);- let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());- T::from(y_float).unwrap()-}--pub fn activate<T: Float>(- x: T,- activation_type: &ActivationType,-) -> T {- match activation_type {- ActivationType::SiLU => silu(x),- ActivationType::GELU => gelu(x),- }-}diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 7fa0e433a..43d4d79c6 100644--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -19,9 +19,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[allow(unused)]#[specialize]group_size: u32,- #[allow(unused)]- #[specialize]- quant_mode: u32,+ #[allow(unused)] quant_mode: crate::backends::common::gpu_types::quantization::QuantizationMode,) {todo!()}diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rsindex f76ff38d6..477711f5c 100644--- a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs@@ -2,32 +2,24 @@ use dsl::kernel;use half::{bf16, f16};use num_traits::Float;-use crate::{- Activation, ArrayElement,- backends::cpu::kernel::activation::{ActivationType, activate},-};+use crate::ArrayElement;#[kernel(MlpGateActMul)]#[variants(T, f32, f16, bf16)]pub fn mlp_gate_act_mul<T: ArrayElement + Float>(- #[allow(unused)] fused_up: *const T,- #[allow(unused)] hidden: *mut T,- #[allow(unused)] h: i32,- #[allow(unused)] m: i32,- #[allow(unused)] act_type: u32,+ fused_up: *const T,+ hidden: *mut T,+ h: i32,+ m: i32,+ act_type: crate::backends::common::gpu_types::activation_type::ActivationType,) {- let activation_type = match ActivationType::try_from(act_type) {- Ok(a) => a,- Err(_) => return,- };-for j in 0..h {for row in 0..m {let base = row * 2 * h;unsafe {let up: T = *fused_up.add((base + j) as usize);let gate: T = *fused_up.add((base + h + j) as usize);- let g: T = activate(gate, &activation_type);+ let g: T = act_type.activate(gate);let out_float = up.to_f32().unwrap() * g.to_f32().unwrap();*hidden.add((row * h + j) as usize) = T::from(out_float).unwrap();}diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rsindex 01617af80..d00fedb10 100644--- a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs+++ b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs@@ -40,9 +40,7 @@ pub fn conv1d_decode<T: ArrayElement + Float>(#[allow(unused)] suffix_len: u32,#[allow(unused)] inner_dim: u32,#[allow(unused)] proj_dim: u32,- #[allow(unused)]- #[specialize]- activation_type: u32,+ #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,#[allow(unused)]#[specialize]has_bias: bool,@@ -72,9 +70,7 @@ pub fn conv1d_scan<T: ArrayElement + Float>(#[allow(unused)] num_channels: u32,#[allow(unused)] inner_dim: u32,#[allow(unused)] proj_dim: u32,- #[allow(unused)]- #[specialize]- activation_type: u32,+ #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,#[allow(unused)]#[specialize]has_bias: bool,diff --git a/crates/uzu/src/backends/metal/generated/activation.h b/crates/uzu/src/backends/metal/generated/activation.hnew file mode 100644index 000000000..54a579fb4--- /dev/null+++ b/crates/uzu/src/backends/metal/generated/activation.h@@ -0,0 +1,28 @@+// Auto-generated from gpu_types/activation.rs - do not edit manually+#pragma once++#ifndef UZU_ACTIVATION_H+#define UZU_ACTIVATION_H++#ifdef __METAL_VERSION__+#include <metal_stdlib>+using namespace metal;++namespace uzu {+namespace activation {+#else+#include <stdint.h>+#endif++enum ActivationType {+ SILU,+ GELU,+ IDENTITY,+};++#ifdef __METAL_VERSION__+} // namespace activation+} // namespace uzu+#endif++#endif // UZU_ACTIVATION_Hdiff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.hnew file mode 100644index 000000000..c11e87a06--- /dev/null+++ b/crates/uzu/src/backends/metal/generated/activation_type.h@@ -0,0 +1,28 @@+// Auto-generated from gpu_types/activation_type.rs - do not edit manually+#pragma once++#ifndef UZU_ACTIVATION_TYPE_H+#define UZU_ACTIVATION_TYPE_H++#ifdef __METAL_VERSION__+#include <metal_stdlib>+using namespace metal;++namespace uzu {+namespace activation_type {+#else+#include <stdint.h>+#endif++enum ActivationType {+ SILU,+ GELU,+ IDENTITY,+};++#ifdef __METAL_VERSION__+} // namespace activation_type+} // namespace uzu+#endif++#endif // UZU_ACTIVATION_TYPE_Hdiff --git a/crates/uzu/src/backends/metal/generated/quantization.h b/crates/uzu/src/backends/metal/generated/quantization.hnew file mode 100644index 000000000..1eba47c17--- /dev/null+++ b/crates/uzu/src/backends/metal/generated/quantization.h@@ -0,0 +1,28 @@+// Auto-generated from gpu_types/quantization.rs - do not edit manually+#pragma once++#ifndef UZU_QUANTIZATION_H+#define UZU_QUANTIZATION_H++#ifdef __METAL_VERSION__+#include <metal_stdlib>+using namespace metal;++namespace uzu {+namespace quantization {+#else+#include <stdint.h>+#endif++enum QuantizationMode {+ UINT4,+ INT8,+ UINT8,+};++#ifdef __METAL_VERSION__+} // namespace quantization+} // namespace uzu+#endif++#endif // UZU_QUANTIZATION_Hdiff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.h b/crates/uzu/src/backends/metal/kernel/activation/activation.hdeleted file mode 100644index 5212972b2..000000000--- a/crates/uzu/src/backends/metal/kernel/activation/activation.h+++ /dev/null@@ -1,30 +0,0 @@-#include <metal_stdlib>--enum ActivationType : uint {- ACT_SILU = 0,- ACT_GELU = 1,-};--template <typename T>-inline T silu(T x) {- float xf = float(x);- float yf = xf / (1.0f + metal::exp(-xf));- return T(yf);-}--template <typename T>-inline T gelu_approx(T x) {- constexpr float k0 = 0.044715f;- constexpr float k1 = 0.7978845608f; // sqrt(2/pi)- float xf = float(x);- float yf =- 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));- return T(yf);-}--template <typename T>-inline T activate(T x, uint act) {- if (act == ACT_SILU)- return silu(x);- return gelu_approx(x);-}diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metalindex a815a2722..681f27596 100644--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal@@ -1,7 +1,8 @@#include <metal_stdlib>#include "../common/dsl.h"+#include "activations.h"-#include "activation.h"+using namespace uzu::activation_type;template <typename T>VARIANTS(T, half, float, bfloat)@@ -9,7 +10,7 @@ PUBLIC KERNEL(Activation) (const device T* input OPTIONAL(!in_place),device T* output,const constant uint& n,- const constant uint& act_type,+ const constant ActivationType& act_type,const bool in_place SPECIALIZE,uint tid AXIS(n, 256)) {diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.hnew file mode 100644index 000000000..68ee975d5--- /dev/null+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h@@ -0,0 +1,34 @@+#include <metal_stdlib>+#include "activation_type.h"++using namespace uzu::activation_type;++template <typename T>+inline T activate_silu(T x) {+ float xf = float(x);+ float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));+ float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;+ return static_cast<T>(out);+}++template <typename T>+inline T activate_gelu(T x) {+ constexpr float k0 = 0.044715f;+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)+ float xf = float(x);+ float yf =+ 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));+ return T(yf);+}++template <typename T>+inline T activate(T x, ActivationType type) {+ switch (type) {+ case SILU:+ return activate_silu(x);+ case GELU:+ return activate_gelu(x);+ case IDENTITY:+ return x;+ }+}diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex bc7881a3b..c01cf454c 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -1,11 +1,8 @@#include <metal_stdlib>#include "../common/dsl.h"+#include "quantization.h"-enum QuantizationMode : uint {- QUANT_UINT4 = 0,- QUANT_INT8 = 1,- QUANT_UINT8 = 2-};+using namespace uzu::quantization;template <typename T>VARIANTS(T, float, half, bfloat)@@ -19,8 +16,8 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (constant uint32_t& vocab_size,constant uint32_t& model_dim,constant float& input_scale,+ const constant QuantizationMode& quant_mode,const uint32_t group_size SPECIALIZE,- const uint quant_mode SPECIALIZE,const uint dim_idx AXIS(model_dim, 16),const uint batch_idx AXIS(batch_size, 16)) {@@ -37,11 +34,12 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (const T scale = scales[token_id * num_groups + group_idx];const T bias = biases[token_id * num_groups + group_idx];- const uint packing_divisor = quant_mode == QUANT_UINT4 ? 2 : 1;+ const uint packing_divisor = quant_mode == QuantizationMode::UINT4 ? 2 : 1;const uint weights_stride = model_dim / packing_divisor;int quantized_value = 0;- if (quant_mode == QUANT_UINT4) {+ switch (quant_mode) {+ case UINT4: {const uint byte_idx = token_id * weights_stride + (dim_idx / 2);const uint8_t packed = weights[byte_idx];if ((dim_idx & 1) == 0) {@@ -49,14 +47,20 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (} else {quantized_value = int((packed >> 4) & 0x0F);}- } else if (quant_mode == QUANT_INT8) {+ break;+ };+ case INT8: {const uint elem_idx = token_id * weights_stride + dim_idx;const device int8_t* weights_i8 =reinterpret_cast<const device int8_t*>(weights);quantized_value = int(weights_i8[elem_idx]);- } else {+ break;+ };+ case UINT8: {const uint elem_idx = token_id * weights_stride + dim_idx;quantized_value = int(weights[elem_idx]);+ break;+ };}float out_f = float(scale) * float(quantized_value) + float(bias);diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metalindex 05b5f58e1..c17c6b07e 100644--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal@@ -1,7 +1,9 @@#include <metal_stdlib>-#include "../activation/activation.h"+#include "../activation/activations.h"#include "../common/dsl.h"+using namespace uzu::activation_type;+template <typename T>VARIANTS(T, float, half, bfloat)PUBLIC KERNEL(MlpGateActMul) (@@ -9,7 +11,7 @@ PUBLIC KERNEL(MlpGateActMul) (device T* hidden,const constant int& h,const constant int& m,- const constant uint& act_type,+ const constant ActivationType& act_type,uint j AXIS(h, 64),uint row AXIS(m, 1)) {diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metalindex 0c48c0d01..2fc6e8114 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal@@ -1,6 +1,8 @@#include <metal_stdlib>+#include "../activation/activations.h"#include "../common/dsl.h"-#include "ssm_common.h"++using namespace uzu::activation_type;template <typename T>VARIANTS(T, float, half, bfloat)@@ -44,7 +46,7 @@ PUBLIC KERNEL(Conv1dDecode)(constant const uint& suffix_len,constant const uint& inner_dim,constant const uint& proj_dim,- const uint activation_type SPECIALIZE,+ constant const ActivationType& activation_type,const bool has_bias SPECIALIZE,const bool state_in_place SPECIALIZE,const uint token_idx AXIS(suffix_len, 32),@@ -76,7 +78,7 @@ PUBLIC KERNEL(Conv1dDecode)(acc += float(w_row[current_tap_index]) * current;}- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);+ const T activated = activate(static_cast<T>(acc), activation_type);if (channel_idx < inner_dim) {const uint dst = token_idx * inner_dim + channel_idx;x_out[dst] = activated;@@ -120,7 +122,7 @@ PUBLIC KERNEL(Conv1dScan)(constant const uint& num_channels,constant const uint& inner_dim,constant const uint& proj_dim,- const uint activation_type SPECIALIZE,+ constant const ActivationType& activation_type,const bool has_bias SPECIALIZE,const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),const uint channel_idx AXIS(num_channels, 1)@@ -143,8 +145,7 @@ PUBLIC KERNEL(Conv1dScan)(acc += float(w_row[tap]) * sample;}- const T activated =- apply_activation_fn(static_cast<T>(acc), activation_type);+ const T activated = activate(static_cast<T>(acc), activation_type);if (channel_idx < inner_dim) {const size_t dst = size_t(token_idx) * inner_dim + channel_idx;x_out[dst] = activated;diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metalindex d9b4001f3..6330408ee 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal@@ -1,5 +1,6 @@#include <metal_stdlib>#include <metal_simdgroup>+#include "../activation/activations.h"#include "../common/dsl.h"#include "../common/thread_context.h"#include "ssm_common.h"@@ -79,7 +80,7 @@ PUBLIC KERNEL(SSDPrefill64)(const float x_val = float(x[x_idx]);const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));- const float gate = float(apply_silu(z[x_idx]));+ const float gate = float(activate_silu(z[x_idx]));const float skip = d_scalar * x_val;const float dt_scaled_input = x_val;@@ -176,7 +177,7 @@ PUBLIC KERNEL(SSDPrefill)(const float x_val = float(x[x_idx]);const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));- const float gate = float(apply_silu(z[x_idx]));+ const float gate = float(activate_silu(z[x_idx]));const float skip = d_scalar * x_val;const float dt_scaled_input = x_val;@@ -246,7 +247,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(const T this_dt = softplus(dt_raw_val);const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));const T this_D = d[h_idx];- const T this_z = apply_silu(z[x_idx]);+ const T this_z = activate_silu(z[x_idx]);const T dt_scaled_input = this_x;T acc = T(0);diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metalindex 8ddf31f79..4afcc0302 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal@@ -1,4 +1,5 @@#include <metal_stdlib>+#include "../activation/activations.h"#include "../common/dsl.h"#include "ssm_common.h"@@ -55,7 +56,7 @@ PUBLIC KERNEL(SSDUpdate)(T this_decay = static_cast<T>(fast::exp(-float(this_dt)));T this_d = d[h_idx];T this_z = z[x_idx];- this_z = apply_silu(this_z);+ this_z = activate_silu(this_z);T dt_scaled_input = this_x;T temp = T(0);diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.hindex 1e105a86e..41845ba0d 100644--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h@@ -1,14 +1,4 @@-#define ACTIVATION_IDENTITY 0-#define ACTIVATION_SILU 1-#define ACTIVATION_GELU 2--template <typename T>-inline T apply_silu(T x) {- float xf = float(x);- float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));- float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;- return static_cast<T>(out);-}+#include <metal_stdlib>template <typename T>inline T softplus(T x) {@@ -17,24 +7,4 @@ inline T softplus(T x) {return x;}return static_cast<T>(log(1.0f + fast::exp(xf)));-}--template <typename T>-inline T apply_gelu(T x) {- float xf = float(x);- return static_cast<T>(- 0.5f * xf *- (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))- );-}--template <typename T>-inline T apply_activation_fn(T x, int activation_type) {- if (activation_type == ACTIVATION_SILU) {- return apply_silu(x);- } else if (activation_type == ACTIVATION_GELU) {- return apply_gelu(x);- } else {- return x; // Identity- }}\ No newline at end of filediff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rsindex ff720dbab..7198357ed 100644--- a/crates/uzu/src/config/attention.rs+++ b/crates/uzu/src/config/attention.rs@@ -29,12 +29,10 @@ mod tests {use serde_json5::from_str;use super::{- super::{- common::{ConfigDataType, QuantizationMode},- linear::QuantizationConfig,- },+ super::{common::ConfigDataType, linear::QuantizationConfig},*,};+ use crate::backends::common::gpu_types::QuantizationMode;#[test]fn test_attention_config() {@@ -77,8 +75,8 @@ mod tests {qkv_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -87,8 +85,8 @@ mod tests {out_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,diff --git a/crates/uzu/src/config/classifier/prediction_head_config.rs b/crates/uzu/src/config/classifier/prediction_head_config.rsindex 8c45bab7c..67466d5d0 100644--- a/crates/uzu/src/config/classifier/prediction_head_config.rs+++ b/crates/uzu/src/config/classifier/prediction_head_config.rs@@ -1,11 +1,11 @@use serde::{Deserialize, Serialize};-use crate::{Activation, LinearConfig, NormalizationConfig};+use crate::{LinearConfig, NormalizationConfig, backends::common::ActivationConfig};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct PredictionHeadConfig {pub dense_config: LinearConfig,- pub activation: Activation,+ pub activation: ActivationConfig,pub normalization_config: NormalizationConfig,pub readout_config: LinearConfig,pub use_dense_bias: bool,diff --git a/crates/uzu/src/config/common.rs b/crates/uzu/src/config/common.rsindex 92bf13805..b3272ca5b 100644--- a/crates/uzu/src/config/common.rs+++ b/crates/uzu/src/config/common.rs@@ -37,69 +37,3 @@ impl From<DataType> for ConfigDataType {}}}--#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Copy, Clone)]-#[serde(rename_all = "lowercase")]-pub enum QuantizationMode {- UInt4,- Int8,- UInt8,-}--impl QuantizationMode {- pub fn packing_divisor(&self) -> usize {- match self {- QuantizationMode::UInt4 => 2,- QuantizationMode::Int8 => 1,- QuantizationMode::UInt8 => 1,- }- }-- pub fn storage_type(&self) -> DataType {- match self {- QuantizationMode::UInt4 => DataType::U8,- QuantizationMode::Int8 => DataType::I8,- QuantizationMode::UInt8 => DataType::U8,- }- }-}--impl From<QuantizationMode> for DataType {- fn from(val: QuantizationMode) -> Self {- match val {- QuantizationMode::UInt4 => DataType::U4,- QuantizationMode::Int8 => DataType::I8,- QuantizationMode::UInt8 => DataType::U8,- }- }-}--#[derive(Debug, Serialize, Deserialize, PartialEq, Copy, Clone)]-#[serde(tag = "type")]-pub enum Activation {- #[serde(rename = "SiLU")]- SiLU {- #[serde(default = "default_silu_alpha")]- alpha: f32,- },- #[serde(rename = "GELU")]- Gelu,- #[serde(rename = "Identity")]- Identity,-}--fn default_silu_alpha() -> f32 {- 1.0-}--impl Activation {- pub fn alpha(&self) -> f32 {- match self {- Activation::SiLU {- alpha,- } => *alpha,- Activation::Gelu => 1.0,- Activation::Identity => 1.0,- }- }-}diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rsindex a12e4ec80..1342f94ed 100644--- a/crates/uzu/src/config/decoder.rs+++ b/crates/uzu/src/config/decoder.rs@@ -229,7 +229,7 @@ mod tests {use super::{super::{attention::AttentionConfig,- common::{Activation, ConfigDataType, QuantizationMode},+ common::ConfigDataType,embedding::{EmbeddingConfig, EmbeddingConfigCommon},linear::{LinearConfig, QuantizationConfig},mlp::{DenseMLPConfig, MLPConfig},@@ -238,6 +238,7 @@ mod tests {},*,};+ use crate::backends::common::{ActivationConfig, gpu_types::QuantizationMode};#[test]fn test_decoder_config() {@@ -354,8 +355,8 @@ mod tests {logit_soft_cap: None,},group_size: 128,- embedding_quantization_mode: QuantizationMode::Int8,- activation_quantization_mode: Some(QuantizationMode::Int8),+ embedding_quantization_mode: QuantizationMode::INT8,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},global_rope_config: Some(RoPEConfig::Llama {@@ -391,8 +392,8 @@ mod tests {qkv_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -401,8 +402,8 @@ mod tests {out_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -425,16 +426,14 @@ mod tests {linear_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,lora_scale: 2.0,},- activation: Activation::SiLU {- alpha: 1.0,- },+ activation: ActivationConfig::silu_default(),has_up_biases: false,has_down_biases: false,gate_clipping: None,diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rsindex ccb9e2aca..a1f559a04 100644--- a/crates/uzu/src/config/decoder_layer.rs+++ b/crates/uzu/src/config/decoder_layer.rs@@ -106,13 +106,16 @@ mod tests {use super::{super::{- common::{Activation, ConfigDataType, QuantizationMode},+ common::ConfigDataType,linear::{LinearConfig, QuantizationConfig},normalization::UpcastMode,},*,};- use crate::config::mlp;+ use crate::{+ backends::common::{ActivationConfig, gpu_types::QuantizationMode},+ config::mlp,+ };#[test]fn test_decoder_layer_config() {@@ -204,8 +207,8 @@ mod tests {qkv_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -214,8 +217,8 @@ mod tests {out_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -238,16 +241,14 @@ mod tests {linear_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,lora_scale: 2.0,},- activation: Activation::SiLU {- alpha: 1.0,- },+ activation: ActivationConfig::silu_default(),has_up_biases: false,has_down_biases: false,gate_clipping: None,diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rsindex 9562a22fc..5e298212d 100644--- a/crates/uzu/src/config/embedding.rs+++ b/crates/uzu/src/config/embedding.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::{ConfigDataType, QuantizationMode};+use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct EmbeddingConfigCommon {@@ -105,7 +105,7 @@ mod tests {logit_soft_cap: None,},group_size: 128,- embedding_quantization_mode: QuantizationMode::UInt4,+ embedding_quantization_mode: QuantizationMode::UINT4,activation_quantization_mode: None,activation_precision: ConfigDataType::BFloat16,};@@ -131,7 +131,7 @@ mod tests {logit_soft_cap: None,},group_size: 128,- embedding_quantization_mode: QuantizationMode::UInt4,+ embedding_quantization_mode: QuantizationMode::UINT4,activation_quantization_mode: None,activation_precision: ConfigDataType::BFloat16,};diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rsindex db8c21645..ce68e481c 100644--- a/crates/uzu/src/config/linear.rs+++ b/crates/uzu/src/config/linear.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::{ConfigDataType, QuantizationMode};+use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct QuantizationConfig {@@ -69,8 +69,8 @@ mod tests {let ground_truth_config = LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,diff --git a/crates/uzu/src/config/mamba.rs b/crates/uzu/src/config/mamba.rsindex 055dc3c45..aaa0a8cd3 100644--- a/crates/uzu/src/config/mamba.rs+++ b/crates/uzu/src/config/mamba.rs@@ -1,9 +1,7 @@use serde::{Deserialize, Serialize};-use super::{- common::{Activation, ConfigDataType},- linear::LinearConfig,-};+use super::{common::ConfigDataType, linear::LinearConfig};+use crate::backends::common::ActivationConfig;#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct CausalConv1dConfig {@@ -16,7 +14,7 @@ pub struct Mamba2Config {pub in_projection_config: LinearConfig,pub out_projection_config: LinearConfig,pub conv_config: CausalConv1dConfig,- pub activation: Activation,+ pub activation: ActivationConfig,pub kernel_size: usize,pub num_heads: usize,pub num_groups: usize,diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rsindex 7e279ec78..49e088c91 100644--- a/crates/uzu/src/config/mlp.rs+++ b/crates/uzu/src/config/mlp.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::{Activation, LinearConfig};+use crate::{LinearConfig, backends::common::ActivationConfig};#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]#[serde(tag = "type")]@@ -14,7 +14,7 @@ pub enum MLPConfig {#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct DenseMLPConfig {pub linear_config: LinearConfig,- pub activation: Activation,+ pub activation: ActivationConfig,#[serde(default)]pub has_up_biases: bool,#[serde(default)]@@ -51,7 +51,7 @@ pub enum RoutingFunctionConfig {#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]pub struct MoeExpertConfig {pub linear_config: LinearConfig,- pub activation: Activation,+ pub activation: ActivationConfig,pub has_up_biases: bool,pub has_down_biases: bool,pub gate_clipping: [Option<f32>; 2],@@ -63,7 +63,7 @@ mod tests {use serde_json::from_str;use super::{super::linear::QuantizationConfig, *};- use crate::config::{Activation, ConfigDataType, QuantizationMode};+ use crate::{backends::common::gpu_types::QuantizationMode, config::ConfigDataType};#[test]fn test_dense_mlp_config() {@@ -87,16 +87,14 @@ mod tests {linear_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,lora_scale: 2.0,},- activation: Activation::SiLU {- alpha: 1.0,- },+ activation: ActivationConfig::silu_default(),has_up_biases: false,has_down_biases: false,gate_clipping: None,@@ -149,16 +147,14 @@ mod tests {linear_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,lora_scale: 2.0,},- activation: Activation::SiLU {- alpha: 1.0,- },+ activation: ActivationConfig::silu_default(),has_up_biases: true,has_down_biases: true,gate_clipping: [None, Some(7.0)],@@ -214,16 +210,14 @@ mod tests {linear_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,lora_scale: 2.0,},- activation: Activation::SiLU {- alpha: 1.0,- },+ activation: ActivationConfig::silu_default(),has_up_biases: true,has_down_biases: true,gate_clipping: [None, Some(7.0)],diff --git a/crates/uzu/src/config/transformer_layer.rs b/crates/uzu/src/config/transformer_layer.rsindex 791169dcb..3735ec991 100644--- a/crates/uzu/src/config/transformer_layer.rs+++ b/crates/uzu/src/config/transformer_layer.rs@@ -27,13 +27,16 @@ mod tests {use super::{super::{attention::AttentionConfig,- common::{Activation, ConfigDataType, QuantizationMode},+ common::ConfigDataType,linear::{LinearConfig, QuantizationConfig},normalization::UpcastMode,},*,};- use crate::config::mlp;+ use crate::{+ backends::common::{ActivationConfig, gpu_types::QuantizationMode},+ config::mlp,+ };#[test]fn test_transformer_layer_config() {@@ -132,8 +135,8 @@ mod tests {qkv_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -142,8 +145,8 @@ mod tests {out_projection_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,@@ -166,16 +169,14 @@ mod tests {linear_config: LinearConfig::QLoRA {quantization: QuantizationConfig {group_size: 32,- weight_quantization_mode: QuantizationMode::UInt4,- activation_quantization_mode: Some(QuantizationMode::Int8),+ weight_quantization_mode: QuantizationMode::UINT4,+ activation_quantization_mode: Some(QuantizationMode::INT8),activation_precision: ConfigDataType::BFloat16,},lora_rank: 16,lora_scale: 2.0,},- activation: Activation::SiLU {- alpha: 1.0,- },+ activation: ActivationConfig::silu_default(),has_up_biases: false,has_down_biases: false,gate_clipping: None,diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rsindex 199301464..810b64344 100644--- a/crates/uzu/src/encodable_block/activation.rs+++ b/crates/uzu/src/encodable_block/activation.rs@@ -5,16 +5,16 @@ use std::ops::DerefMut;use crate::{DataType,backends::common::{- Backend, CommandBuffer,+ ActivationConfig, Backend, CommandBuffer,+ gpu_types::ActivationType,kernel::{ActivationKernel, Kernels},},- config::Activation as ActivationConfig,forward_pass::state::{ArrayId, ForwardPassState},};pub struct Activation<B: Backend> {kernel: <B::Kernels as Kernels>::ActivationKernel,- config: ActivationConfig,+ activation: ActivationConfig,input_array_id: ArrayId,output_array_id: ArrayId,}@@ -31,7 +31,7 @@ impl<B: Backend> Activation<B> {<B::Kernels as Kernels>::ActivationKernel::new(context, data_type, input_array_id == output_array_id)?;Ok(Self {kernel,- config,+ activation: config,input_array_id,output_array_id,})@@ -47,13 +47,9 @@ impl<B: Backend> Activation<B> {let output_array = arrays[1].borrow_mut();let n = input_array.shape().iter().product::<usize>();- let act_type = match self.config {- ActivationConfig::SiLU {- ..- } => 0,- ActivationConfig::Gelu => 1,- ActivationConfig::Identity => panic!("Identity activation is not supported for kernel"),- };+ if self.activation.act_type() == ActivationType::IDENTITY {+ panic!("Identity activation is not supported for kernel");+ }let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());@@ -61,7 +57,7 @@ impl<B: Backend> Activation<B> {input_buffer_borrow.as_deref(),output_array.buffer().borrow_mut().deref_mut(),n as u32,- act_type,+ self.activation.act_type(),command_buffer,);Ok(())diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex a630ffefb..7e6a3d935 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -6,9 +6,10 @@ use std::{use thiserror::Error;use crate::{- DataType, EmbeddingConfig, QuantizationMode,+ DataType, EmbeddingConfig,backends::common::{Backend, CommandBuffer, Kernels,+ gpu_types::QuantizationMode,kernel::{FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},@@ -99,14 +100,7 @@ pub struct Embedding<B: Backend> {input_scale: f32,vocab_size: u32,model_dim: u32,-}--fn quant_mode_to_int(quant_mode: QuantizationMode) -> u32 {- match quant_mode {- QuantizationMode::UInt4 => 0,- QuantizationMode::Int8 => 1,- QuantizationMode::UInt8 => 2,- }+ quant_mode: Option<QuantizationMode>,}fn validate_tensor<'file, 'context, 'leaf, B: Backend>(@@ -138,6 +132,7 @@ impl<B: Backend> Embedding<B> {parameter_tree: &ParameterTree<B::Context>,) -> Result<Self, EmbeddingError<B>> {let common = config.common();+ let mut quant_mode: Option<QuantizationMode> = None;let tying = match config {EmbeddingConfig::Tied {@@ -225,9 +220,9 @@ impl<B: Backend> Embedding<B> {context,data_type,*group_size as u32,- quant_mode_to_int(*embedding_quantization_mode),).map_err(EmbeddingError::BackendError)?;+ quant_mode = Some(*embedding_quantization_mode);let readout = QuantizedMatmulKernelEncodable::new(context,QuantizedMatmulConfiguration {@@ -305,9 +300,9 @@ impl<B: Backend> Embedding<B> {context,data_type,*group_size as u32,- quant_mode_to_int(*embedding_quantization_mode),).map_err(EmbeddingError::BackendError)?;+ quant_mode = Some(*embedding_quantization_mode);let readout = QuantizedMatmulKernelEncodable::new(context,QuantizedMatmulConfiguration {@@ -423,6 +418,7 @@ impl<B: Backend> Embedding<B> {input_scale,vocab_size,model_dim,+ quant_mode,})}@@ -501,6 +497,7 @@ impl<B: Backend> Embedding<B> {self.vocab_size,self.model_dim,self.input_scale,+ self.quant_mode.unwrap(),command_buffer,);},diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rsindex ff2c8b42a..bc0d2ade9 100644--- a/crates/uzu/src/encodable_block/mamba_mixer.rs+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs@@ -6,7 +6,7 @@ use std::{};use crate::{- Activation, DataType,+ DataType,array::Array,backends::common::{Backend, CommandBuffer, Kernels,@@ -107,17 +107,12 @@ impl<B: Backend> MambaMixer<B> {let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type).expect("Failed to create split in-projection kernel");- let conv_scan = <B::Kernels as Kernels>::Conv1dScanKernel::new(- context,- data_type,- Self::activation_to_int(&mamba_config.activation),- mamba_config.conv_config.has_biases,- )- .expect("Failed to create conv scan kernel");+ let conv_scan =+ <B::Kernels as Kernels>::Conv1dScanKernel::new(context, data_type, mamba_config.conv_config.has_biases)+ .expect("Failed to create conv scan kernel");let conv_decode = <B::Kernels as Kernels>::Conv1dDecodeKernel::new(context,data_type,- Self::activation_to_int(&mamba_config.activation),mamba_config.conv_config.has_biases,true,)@@ -235,6 +230,7 @@ impl<B: Backend> MambaMixer<B> {suffix_length as u32,inner_dim as u32,proj_dim as u32,+ self.config.activation.act_type(),command_buffer,);}@@ -279,6 +275,7 @@ impl<B: Backend> MambaMixer<B> {conv_dim as u32,inner_dim as u32,proj_dim as u32,+ self.config.activation.act_type(),command_buffer,)}@@ -397,16 +394,6 @@ impl<B: Backend> MambaMixer<B> {);}- fn activation_to_int(activation: &Activation) -> u32 {- match activation {- Activation::Identity => 0,- Activation::SiLU {- ..- } => 1,- Activation::Gelu => 2,- }- }-pub(crate) fn encode(&self,state: &mut ForwardPassState<B>,diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rsindex 7efde157c..4339670ef 100644--- a/crates/uzu/src/encodable_block/mlp/moe.rs+++ b/crates/uzu/src/encodable_block/mlp/moe.rs@@ -9,10 +9,11 @@ use std::{use thiserror::Error;use crate::{- Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,+ DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,array::Array,backends::common::{Backend, CommandBuffer, CommandBufferEncoding, Kernels,+ gpu_types::ActivationType,kernel::{MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,MoeScatterBucketsMapKernel,@@ -175,13 +176,13 @@ impl<B: Backend> MoeBlock<B> {})}- fn gating_code_from_activation(activation: &Activation) -> u32 {+ fn gating_code_from_activation(activation: &ActivationType) -> u32 {match activation {- Activation::Gelu => 3,- Activation::SiLU {+ ActivationType::GELU => 3,+ ActivationType::SILU {..} => 2,- Activation::Identity => {+ ActivationType::IDENTITY => {panic!("Identity activation is not supported for MoE kernels")},}@@ -362,7 +363,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {},);- let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation);+ let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation.act_type());// Compute clipping values and alpha for expert kernelslet gate_clip_min = self.moe_config.expert_config.gate_clipping[0].unwrap_or(f32::NEG_INFINITY);diff --git a/crates/uzu/tests/kernel/activation_mul_test.rs b/crates/uzu/tests/kernel/activation_mul_test.rsindex 4bbbce9b9..d900a1452 100644--- a/crates/uzu/tests/kernel/activation_mul_test.rs+++ b/crates/uzu/tests/kernel/activation_mul_test.rs@@ -11,7 +11,7 @@ use uzu::{backends::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,- Context, Kernels, kernel::MlpGateActMulKernel,+ Context, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,},cpu::Cpu,},@@ -23,10 +23,10 @@ struct Input<T: ArrayElement + Float> {fused_up: Box<[T]>,h: i32,m: i32,- act_type: u32,+ act_type: ActivationType,}-fn get_test_data<T: ArrayElement + Float>(act_type: u32) -> (Input<T>, Vec<T>) {+fn get_test_data<T: ArrayElement + Float>(act_type: ActivationType) -> (Input<T>, Vec<T>) {let h = 64i32;let m = 4i32;let fused_len = (m * 2 * h) as usize;@@ -73,7 +73,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {hidden_array.as_slice().to_vec()}-fn test<T: ArrayElement + Float + Debug + Display>(act_type: u32) {+fn test<T: ArrayElement + Float + Debug + Display>(act_type: ActivationType) {let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {0.02f32} else {@@ -90,30 +90,30 @@ fn test<T: ArrayElement + Float + Debug + Display>(act_type: u32) {#[test]fn test_silu_f32() {- test::<f32>(0);+ test::<f32>(ActivationType::SILU);}#[test]fn test_silu_f16() {- test::<f16>(0);+ test::<f16>(ActivationType::SILU);}#[test]fn test_silu_bf16() {- test::<bf16>(0);+ test::<bf16>(ActivationType::SILU);}#[test]fn test_gelu_f32() {- test::<f32>(1);+ test::<f32>(ActivationType::GELU);}#[test]fn test_gelu_f16() {- test::<f16>(1);+ test::<f16>(ActivationType::GELU);}#[test]fn test_gelu_bf16() {- test::<bf16>(1);+ test::<bf16>(ActivationType::GELU);}diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rsindex 306d24b57..b3065829a 100644--- a/crates/uzu/tests/quant_matmul_test.rs+++ b/crates/uzu/tests/quant_matmul_test.rs@@ -12,6 +12,7 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context,+ gpu_types::QuantizationMode,kernel::quant_matmul::{QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,QuantizedMatmulType,@@ -19,7 +20,6 @@ use uzu::{},metal::Metal,},- config::QuantizationMode,};fn pack_u4_weights(values: &[u8]) -> Vec<u8> {@@ -489,8 +489,8 @@ fn execute_quantized_matmul(input_dim,output_dim,mode: match bits {- 4 => QuantizationMode::UInt4,- 8 => QuantizationMode::Int8,+ 4 => QuantizationMode::UINT4,+ 8 => QuantizationMode::INT8,_ => panic!("Unsupported bits: {}", bits),},quantization_type,diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rsindex c1c0ac69e..4bde961ec 100644--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs@@ -11,6 +11,7 @@ use uzu::{common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,Context, Kernels,+ gpu_types::ActivationType,kernel::{Conv1dScanKernel,ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},@@ -335,6 +336,7 @@ fn run_conv_scan_once(channels as u32,channels as u32,0u32,+ ActivationType::SILU,&mut command_buffer,);@@ -458,8 +460,7 @@ fn conv1d_scan_is_deterministic() {eprintln!("Skipping conv1d scan determinism test: no Metal device");return;};- let kernel =- <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();+ let kernel = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();let suffix_len = 192usize;let channels = 8usize;