Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Gpu types enum (#264)

Aleksandr Golokozf222b5a53d44
diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rs
index 30826ed30..e012805dd 100644
--- a/crates/uzu/build/gpu_types/mod.rs
+++ b/crates/uzu/build/gpu_types/mod.rs
@@ -2,7 +2,7 @@ use std::{collections::HashMap, env, fs, path::PathBuf};
use anyhow::Context;
use async_trait::async_trait;
-use syn::{Expr, ExprLit, Fields, Item, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};
+use syn::{Attribute, Expr, ExprLit, Fields, Item, ItemEnum, ItemStruct, Lit, Meta, Type, TypeArray, TypePath};
use tokio::task::spawn_blocking;
use crate::{
@@ -91,26 +91,42 @@ impl GpuTypesCompiler {
let syntax = syn::parse_file(&source).with_context(|| format!("cannot parse {}", src_path.display()))?;
+ let mut repr_c_enums: Vec<ItemEnum> = Vec::new();
let mut repr_c_structs: Vec<ItemStruct> = Vec::new();
for item in syntax.items {
- if let Item::Struct(item_struct) = item {
- let has_repr_c = item_struct.attrs.iter().any(|attribute| {
- if attribute.path().is_ident("repr") {
- if let Ok(ident) = attribute.parse_args::<syn::Ident>() {
- return ident == "C";
- }
- }
- false
- });
-
- if has_repr_c {
+ if let Item::Enum(item_enum) = item {
+ if have_repr_c_attr(&item_enum.attrs) {
+ repr_c_enums.push(item_enum);
+ }
+ } else if let Item::Struct(item_struct) = item {
+ if have_repr_c_attr(&item_struct.attrs) {
repr_c_structs.push(item_struct);
}
}
}
- let mut c_struct_definitions = String::new();
- for rust_struct in &repr_c_structs {
+ let c_enum_definitions = Self::get_enum_definitions(&repr_c_enums);
+ let c_struct_definitions = Self::get_struct_definitions(&repr_c_structs);
+
+ let mut content = String::new();
+ if !c_enum_definitions.is_empty() {
+ content.push_str(&c_enum_definitions);
+ }
+ if !c_struct_definitions.is_empty() {
+ if !content.is_empty() {
+ content.push_str("\n\n");
+ }
+ content.push_str(&c_struct_definitions);
+ }
+
+ self.write_header(module_name, content.as_str())?;
+
+ Ok(())
+ }
+
+ fn get_struct_definitions(c_structs: &Vec<ItemStruct>) -> String {
+ let mut definitions = String::new();
+ for (i, rust_struct) in c_structs.iter().enumerate() {
for attribute in &rust_struct.attrs {
if attribute.path().is_ident("doc") {
if let Meta::NameValue(meta_name_value) = &attribute.meta {
@@ -122,16 +138,16 @@ impl GpuTypesCompiler {
let doc_comment = literal_string.value();
let doc_comment = doc_comment.trim();
if !doc_comment.is_empty() {
- c_struct_definitions.push_str("/**");
- c_struct_definitions.push_str(doc_comment);
- c_struct_definitions.push_str(" */\n");
+ definitions.push_str("/**");
+ definitions.push_str(doc_comment);
+ definitions.push_str(" */\n");
}
}
}
}
}
- c_struct_definitions.push_str("typedef struct {\n");
+ definitions.push_str("typedef struct {\n");
if let Fields::Named(named_fields) = &rust_struct.fields {
for field in &named_fields.named {
@@ -149,9 +165,9 @@ impl GpuTypesCompiler {
let doc_comment = literal_string.value();
let doc_comment = doc_comment.trim();
if !doc_comment.is_empty() {
- c_struct_definitions.push_str(" /**");
- c_struct_definitions.push_str(doc_comment);
- c_struct_definitions.push_str(" */\n");
+ definitions.push_str(" /**");
+ definitions.push_str(doc_comment);
+ definitions.push_str(" */\n");
}
}
}
@@ -159,26 +175,49 @@ impl GpuTypesCompiler {
}
if let Some((base_type, array_size)) = c_type.split_once('[') {
- c_struct_definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));
+ definitions.push_str(&format!(" {base_type} {field_name}[{array_size};\n"));
} else {
- c_struct_definitions.push_str(&format!(" {c_type} {field_name};\n"));
+ definitions.push_str(&format!(" {c_type} {field_name};\n"));
}
}
}
let struct_name = rust_struct.ident.to_string();
- c_struct_definitions.push_str(&format!("}} {struct_name};\n\n"));
+ definitions.push_str(&format!("}} {struct_name};"));
+ if i < c_structs.len() - 1 {
+ definitions.push_str("\n\n");
+ }
}
- self.write_header(module_name, c_struct_definitions.trim())?;
+ definitions
+ }
+
+ fn get_enum_definitions(enums: &Vec<ItemEnum>) -> String {
+ let mut definitions = String::new();
+ for (i, rust_enum) in enums.iter().enumerate() {
+ let enum_name = rust_enum.ident.to_string();
+ definitions.push_str("enum ");
+ definitions.push_str(&enum_name);
+ definitions.push_str(" {\n");
+
+ for variant in &rust_enum.variants {
+ definitions.push_str(" ");
+ definitions.push_str(variant.ident.to_string().as_str());
+ definitions.push_str(",\n");
+ }
- Ok(())
+ definitions.push_str("};");
+ if i < enums.len() - 1 {
+ definitions.push_str("\n\n");
+ }
+ }
+ definitions
}
fn write_header(
&self,
module_name: &str,
- structs: &str,
+ content: &str,
) -> anyhow::Result<()> {
let include_guard = format!("UZU_{}_H", module_name.to_uppercase());
@@ -199,7 +238,7 @@ namespace {module_name} {{
#include <stdint.h>
#endif
-{structs}
+{content}
#ifdef __METAL_VERSION__
}} // namespace {module_name}
@@ -266,6 +305,17 @@ fn rust_type_to_c(rust_type: &Type) -> String {
}
}
+fn have_repr_c_attr(attrs: &Vec<Attribute>) -> bool {
+ attrs.iter().any(|attr| {
+ if attr.path().is_ident("repr") {
+ if let Ok(ident) = attr.parse_args::<syn::Ident>() {
+ return ident == "C";
+ }
+ }
+ false
+ })
+}
+
#[async_trait]
impl Compiler for GpuTypesCompiler {
async fn build(&self) -> anyhow::Result<HashMap<Box<[Box<str>]>, Box<[Kernel]>>> {
diff --git a/crates/uzu/src/backends/common/activation_config.rs b/crates/uzu/src/backends/common/activation_config.rs
new file mode 100644
index 000000000..c054a6e52
--- /dev/null
+++ b/crates/uzu/src/backends/common/activation_config.rs
@@ -0,0 +1,49 @@
+use serde::{Deserialize, Serialize};
+
+use crate::backends::common::gpu_types::ActivationType;
+
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(tag = "type")]
+pub enum ActivationConfig {
+ #[serde(rename = "SiLU")]
+ SILU {
+ #[serde(default = "default_silu_alpha")]
+ alpha: f32,
+ },
+ #[serde(rename = "GELU")]
+ GELU,
+ #[serde(rename = "Identity")]
+ IDENTITY,
+}
+
+impl ActivationConfig {
+ pub fn silu_default() -> ActivationConfig {
+ ActivationConfig::SILU {
+ alpha: 1.0,
+ }
+ }
+
+ pub fn act_type(&self) -> ActivationType {
+ match self {
+ ActivationConfig::SILU {
+ ..
+ } => ActivationType::SILU,
+ ActivationConfig::GELU => ActivationType::GELU,
+ ActivationConfig::IDENTITY => ActivationType::IDENTITY,
+ }
+ }
+
+ pub fn alpha(&self) -> f32 {
+ match self {
+ ActivationConfig::SILU {
+ alpha,
+ } => *alpha,
+ ActivationConfig::GELU => 1.0,
+ ActivationConfig::IDENTITY => 1.0,
+ }
+ }
+}
+
+fn default_silu_alpha() -> f32 {
+ 1.0
+}
diff --git a/crates/uzu/src/backends/common/gpu_types/activation_type.rs b/crates/uzu/src/backends/common/gpu_types/activation_type.rs
new file mode 100644
index 000000000..939965237
--- /dev/null
+++ b/crates/uzu/src/backends/common/gpu_types/activation_type.rs
@@ -0,0 +1,38 @@
+use num_traits::Float;
+
+const GELU_K0: f32 = 0.044715;
+const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone, PartialEq)]
+pub enum ActivationType {
+ SILU,
+ GELU,
+ IDENTITY,
+}
+
+impl ActivationType {
+ pub fn activate<T: Float>(
+ &self,
+ x: T,
+ ) -> T {
+ match self {
+ ActivationType::SILU => silu(x),
+ ActivationType::GELU => gelu(x),
+ ActivationType::IDENTITY => x,
+ }
+ }
+}
+
+fn silu<T: Float>(x: T) -> T {
+ let x_float = x.to_f32().unwrap();
+ let y_float = x_float / (1.0f32 + (-x_float).exp());
+ T::from(y_float).unwrap()
+}
+
+fn gelu<T: Float>(x: T) -> T {
+ let x_float = x.to_f32().unwrap();
+ let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);
+ let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());
+ T::from(y_float).unwrap()
+}
diff --git a/crates/uzu/src/backends/common/gpu_types/mod.rs b/crates/uzu/src/backends/common/gpu_types/mod.rs
index 8d996182d..1111d902c 100644
--- a/crates/uzu/src/backends/common/gpu_types/mod.rs
+++ b/crates/uzu/src/backends/common/gpu_types/mod.rs
@@ -3,12 +3,16 @@
//! These `#[repr(C)]` structs are the source of truth. The build system uses
//! cbindgen to generate C headers for Metal shaders.
+pub mod activation_type;
pub mod argmax;
pub mod attention;
pub mod kv_cache_update;
pub mod matmul;
+pub mod quantization;
+pub use activation_type::*;
pub use argmax::*;
pub use attention::*;
pub use kv_cache_update::*;
pub use matmul::*;
+pub use quantization::*;
diff --git a/crates/uzu/src/backends/common/gpu_types/quantization.rs b/crates/uzu/src/backends/common/gpu_types/quantization.rs
new file mode 100644
index 000000000..617afbe2b
--- /dev/null
+++ b/crates/uzu/src/backends/common/gpu_types/quantization.rs
@@ -0,0 +1,40 @@
+use serde::{Deserialize, Serialize};
+
+use crate::DataType;
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum QuantizationMode {
+ UINT4,
+ INT8,
+ UINT8,
+}
+
+impl QuantizationMode {
+ pub fn packing_divisor(&self) -> usize {
+ match self {
+ QuantizationMode::UINT4 => 2,
+ QuantizationMode::INT8 => 1,
+ QuantizationMode::UINT8 => 1,
+ }
+ }
+
+ pub fn storage_type(&self) -> DataType {
+ match self {
+ QuantizationMode::UINT4 => DataType::U8,
+ QuantizationMode::INT8 => DataType::I8,
+ QuantizationMode::UINT8 => DataType::U8,
+ }
+ }
+}
+
+impl From<QuantizationMode> for DataType {
+ fn from(val: QuantizationMode) -> Self {
+ match val {
+ QuantizationMode::UINT4 => DataType::U4,
+ QuantizationMode::INT8 => DataType::I8,
+ QuantizationMode::UINT8 => DataType::U8,
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
index 6e83fefa2..2c5f1863a 100644
--- a/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
+++ b/crates/uzu/src/backends/common/kernel/mlp_gate_act_mul.rs
@@ -1,12 +1,13 @@
use crate::{
DataType,
- backends::common::{Backend, CommandBuffer, Kernels, kernel::MlpGateActMulKernel},
- config::Activation,
+ backends::common::{
+ ActivationConfig, Backend, CommandBuffer, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
+ },
};
pub struct MlpGateActMulEncodable<B: Backend> {
kernel: <B::Kernels as Kernels>::MlpGateActMulKernel,
- activation: Activation,
+ activation: ActivationConfig,
hidden_dim: usize,
}
@@ -14,7 +15,7 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
pub fn new(
context: &B::Context,
data_type: DataType,
- activation: Activation,
+ activation: ActivationConfig,
hidden_dim: usize,
) -> Result<Self, B::Error> {
let kernel = <B::Kernels as Kernels>::MlpGateActMulKernel::new(context, data_type)?;
@@ -32,16 +33,10 @@ impl<B: Backend> MlpGateActMulEncodable<B> {
hidden: &mut B::Buffer,
m: i32,
) -> Result<(), B::Error> {
- let act_type = match self.activation {
- Activation::SiLU {
- ..
- } => 0,
- Activation::Gelu => 1,
- Activation::Identity => {
- panic!("Identity activation is not supported for kernel")
- },
- };
- self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, act_type, command_buffer);
+ if self.activation.act_type() == ActivationType::IDENTITY {
+ panic!("Identity activation is not supported for kernel")
+ }
+ self.kernel.encode(fused_up, hidden, self.hidden_dim as i32, m, self.activation.act_type(), command_buffer);
Ok(())
}
}
diff --git a/crates/uzu/src/backends/common/kernel/quant_matmul.rs b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
index 40daa27df..e63b12039 100644
--- a/crates/uzu/src/backends/common/kernel/quant_matmul.rs
+++ b/crates/uzu/src/backends/common/kernel/quant_matmul.rs
@@ -4,12 +4,12 @@ use crate::{
DataType,
backends::common::{
Backend, CommandBuffer, Kernels,
+ gpu_types::QuantizationMode,
kernel::{
QuantizedMatmulQmmKernel, QuantizedMatmulQmmTransposed64x64Kernel, QuantizedMatmulQmmTransposedKernel,
QuantizedMatmulQmvFastKernel, QuantizedMatmulQmvKernel, QuantizedMatmulQvmKernel,
},
},
- config::QuantizationMode,
};
#[derive(Debug, thiserror::Error)]
@@ -466,8 +466,8 @@ fn select_matrix_matrix_family(
fn quant_bits<B: Backend>(mode: QuantizationMode) -> Result<usize, QuantizedMatmulError<B>> {
let bits = match mode {
- QuantizationMode::UInt4 => 4,
- QuantizationMode::UInt8 | QuantizationMode::Int8 => 8,
+ QuantizationMode::UINT4 => 4,
+ QuantizationMode::INT8 | QuantizationMode::UINT8 => 8,
};
if matches!(bits, 4 | 8) {
Ok(bits)
diff --git a/crates/uzu/src/backends/common/mod.rs b/crates/uzu/src/backends/common/mod.rs
index 660d6a9f8..8557fd41d 100644
--- a/crates/uzu/src/backends/common/mod.rs
+++ b/crates/uzu/src/backends/common/mod.rs
@@ -1,3 +1,4 @@
+mod activation_config;
mod backend;
mod buffer;
mod command_buffer;
@@ -6,6 +7,7 @@ mod event;
pub mod gpu_types;
pub mod kernel;
+pub use activation_config::ActivationConfig;
pub use backend::Backend;
pub use buffer::Buffer;
pub use command_buffer::{
diff --git a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
index 66d94c2f7..2b967ff48 100644
--- a/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
+++ b/crates/uzu/src/backends/cpu/kernel/activation/activation.rs
@@ -12,7 +12,7 @@ pub fn activation<T: ArrayElement + Float>(
input: Option<*const T>,
#[allow(unused)] output: *mut T,
#[allow(unused)] n: u32,
- #[allow(unused)] act_type: u32,
+ #[allow(unused)] act_type: crate::backends::common::gpu_types::activation_type::ActivationType,
#[allow(unused)]
#[specialize]
in_place: bool,
diff --git a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
index e70a83196..3e9730314 100644
--- a/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
+++ b/crates/uzu/src/backends/cpu/kernel/activation/mod.rs
@@ -1,49 +1 @@
-use num_traits::Float;
-
-use crate::ArrayElement;
-
pub mod activation;
-
-const GELU_K0: f32 = 0.044715f32;
-const GELU_K1: f32 = 0.7978845608; // sqrt(2/pi)
-
-#[repr(u32)]
-pub enum ActivationType {
- SiLU,
- GELU,
-}
-
-impl TryFrom<u32> for ActivationType {
- type Error = ();
-
- fn try_from(value: u32) -> Result<Self, Self::Error> {
- match value {
- 0 => Ok(Self::SiLU),
- 1 => Ok(Self::GELU),
- _ => Err(()),
- }
- }
-}
-
-fn silu<T: Float>(x: T) -> T {
- let x_float = x.to_f32().unwrap();
- let y_float = x_float / (1.0f32 + (-x_float).exp());
- T::from(y_float).unwrap()
-}
-
-fn gelu<T: Float>(x: T) -> T {
- let x_float = x.to_f32().unwrap();
- let tan_arg = GELU_K1 * (x_float + GELU_K0 * x_float * x_float * x_float);
- let y_float = 0.5f32 * x_float * (1.0f32 + tan_arg.tanh());
- T::from(y_float).unwrap()
-}
-
-pub fn activate<T: Float>(
- x: T,
- activation_type: &ActivationType,
-) -> T {
- match activation_type {
- ActivationType::SiLU => silu(x),
- ActivationType::GELU => gelu(x),
- }
-}
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 7fa0e433a..43d4d79c6 100644
--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -19,9 +19,7 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[allow(unused)]
#[specialize]
group_size: u32,
- #[allow(unused)]
- #[specialize]
- quant_mode: u32,
+ #[allow(unused)] quant_mode: crate::backends::common::gpu_types::quantization::QuantizationMode,
) {
todo!()
}
diff --git a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
index f76ff38d6..477711f5c 100644
--- a/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
+++ b/crates/uzu/src/backends/cpu/kernel/mlp/activation_mul.rs
@@ -2,32 +2,24 @@ use dsl::kernel;
use half::{bf16, f16};
use num_traits::Float;
-use crate::{
- Activation, ArrayElement,
- backends::cpu::kernel::activation::{ActivationType, activate},
-};
+use crate::ArrayElement;
#[kernel(MlpGateActMul)]
#[variants(T, f32, f16, bf16)]
pub fn mlp_gate_act_mul<T: ArrayElement + Float>(
- #[allow(unused)] fused_up: *const T,
- #[allow(unused)] hidden: *mut T,
- #[allow(unused)] h: i32,
- #[allow(unused)] m: i32,
- #[allow(unused)] act_type: u32,
+ fused_up: *const T,
+ hidden: *mut T,
+ h: i32,
+ m: i32,
+ act_type: crate::backends::common::gpu_types::activation_type::ActivationType,
) {
- let activation_type = match ActivationType::try_from(act_type) {
- Ok(a) => a,
- Err(_) => return,
- };
-
for j in 0..h {
for row in 0..m {
let base = row * 2 * h;
unsafe {
let up: T = *fused_up.add((base + j) as usize);
let gate: T = *fused_up.add((base + h + j) as usize);
- let g: T = activate(gate, &activation_type);
+ let g: T = act_type.activate(gate);
let out_float = up.to_f32().unwrap() * g.to_f32().unwrap();
*hidden.add((row * h + j) as usize) = T::from(out_float).unwrap();
}
diff --git a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
index 01617af80..d00fedb10 100644
--- a/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
+++ b/crates/uzu/src/backends/cpu/kernel/ssm/conv1d.rs
@@ -40,9 +40,7 @@ pub fn conv1d_decode<T: ArrayElement + Float>(
#[allow(unused)] suffix_len: u32,
#[allow(unused)] inner_dim: u32,
#[allow(unused)] proj_dim: u32,
- #[allow(unused)]
- #[specialize]
- activation_type: u32,
+ #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
#[allow(unused)]
#[specialize]
has_bias: bool,
@@ -72,9 +70,7 @@ pub fn conv1d_scan<T: ArrayElement + Float>(
#[allow(unused)] num_channels: u32,
#[allow(unused)] inner_dim: u32,
#[allow(unused)] proj_dim: u32,
- #[allow(unused)]
- #[specialize]
- activation_type: u32,
+ #[allow(unused)] activation_type: crate::backends::common::gpu_types::activation_type::ActivationType,
#[allow(unused)]
#[specialize]
has_bias: bool,
diff --git a/crates/uzu/src/backends/metal/generated/activation.h b/crates/uzu/src/backends/metal/generated/activation.h
new file mode 100644
index 000000000..54a579fb4
--- /dev/null
+++ b/crates/uzu/src/backends/metal/generated/activation.h
@@ -0,0 +1,28 @@
+// Auto-generated from gpu_types/activation.rs - do not edit manually
+#pragma once
+
+#ifndef UZU_ACTIVATION_H
+#define UZU_ACTIVATION_H
+
+#ifdef __METAL_VERSION__
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace activation {
+#else
+#include <stdint.h>
+#endif
+
+enum ActivationType {
+ SILU,
+ GELU,
+ IDENTITY,
+};
+
+#ifdef __METAL_VERSION__
+} // namespace activation
+} // namespace uzu
+#endif
+
+#endif // UZU_ACTIVATION_H
diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.h
new file mode 100644
index 000000000..c11e87a06
--- /dev/null
+++ b/crates/uzu/src/backends/metal/generated/activation_type.h
@@ -0,0 +1,28 @@
+// Auto-generated from gpu_types/activation_type.rs - do not edit manually
+#pragma once
+
+#ifndef UZU_ACTIVATION_TYPE_H
+#define UZU_ACTIVATION_TYPE_H
+
+#ifdef __METAL_VERSION__
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace activation_type {
+#else
+#include <stdint.h>
+#endif
+
+enum ActivationType {
+ SILU,
+ GELU,
+ IDENTITY,
+};
+
+#ifdef __METAL_VERSION__
+} // namespace activation_type
+} // namespace uzu
+#endif
+
+#endif // UZU_ACTIVATION_TYPE_H
diff --git a/crates/uzu/src/backends/metal/generated/quantization.h b/crates/uzu/src/backends/metal/generated/quantization.h
new file mode 100644
index 000000000..1eba47c17
--- /dev/null
+++ b/crates/uzu/src/backends/metal/generated/quantization.h
@@ -0,0 +1,28 @@
+// Auto-generated from gpu_types/quantization.rs - do not edit manually
+#pragma once
+
+#ifndef UZU_QUANTIZATION_H
+#define UZU_QUANTIZATION_H
+
+#ifdef __METAL_VERSION__
+#include <metal_stdlib>
+using namespace metal;
+
+namespace uzu {
+namespace quantization {
+#else
+#include <stdint.h>
+#endif
+
+enum QuantizationMode {
+ UINT4,
+ INT8,
+ UINT8,
+};
+
+#ifdef __METAL_VERSION__
+} // namespace quantization
+} // namespace uzu
+#endif
+
+#endif // UZU_QUANTIZATION_H
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.h b/crates/uzu/src/backends/metal/kernel/activation/activation.h
deleted file mode 100644
index 5212972b2..000000000
--- a/crates/uzu/src/backends/metal/kernel/activation/activation.h
+++ /dev/null
@@ -1,30 +0,0 @@
-#include <metal_stdlib>
-
-enum ActivationType : uint {
- ACT_SILU = 0,
- ACT_GELU = 1,
-};
-
-template <typename T>
-inline T silu(T x) {
- float xf = float(x);
- float yf = xf / (1.0f + metal::exp(-xf));
- return T(yf);
-}
-
-template <typename T>
-inline T gelu_approx(T x) {
- constexpr float k0 = 0.044715f;
- constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
- float xf = float(x);
- float yf =
- 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));
- return T(yf);
-}
-
-template <typename T>
-inline T activate(T x, uint act) {
- if (act == ACT_SILU)
- return silu(x);
- return gelu_approx(x);
-}
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activation.metal b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
index a815a2722..681f27596 100644
--- a/crates/uzu/src/backends/metal/kernel/activation/activation.metal
+++ b/crates/uzu/src/backends/metal/kernel/activation/activation.metal
@@ -1,7 +1,8 @@
#include <metal_stdlib>
#include "../common/dsl.h"
+#include "activations.h"
-#include "activation.h"
+using namespace uzu::activation_type;
template <typename T>
VARIANTS(T, half, float, bfloat)
@@ -9,7 +10,7 @@ PUBLIC KERNEL(Activation) (
const device T* input OPTIONAL(!in_place),
device T* output,
const constant uint& n,
- const constant uint& act_type,
+ const constant ActivationType& act_type,
const bool in_place SPECIALIZE,
uint tid AXIS(n, 256)
) {
diff --git a/crates/uzu/src/backends/metal/kernel/activation/activations.h b/crates/uzu/src/backends/metal/kernel/activation/activations.h
new file mode 100644
index 000000000..68ee975d5
--- /dev/null
+++ b/crates/uzu/src/backends/metal/kernel/activation/activations.h
@@ -0,0 +1,34 @@
+#include <metal_stdlib>
+#include "activation_type.h"
+
+using namespace uzu::activation_type;
+
+template <typename T>
+inline T activate_silu(T x) {
+ float xf = float(x);
+ float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));
+ float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;
+ return static_cast<T>(out);
+}
+
+template <typename T>
+inline T activate_gelu(T x) {
+ constexpr float k0 = 0.044715f;
+ constexpr float k1 = 0.7978845608f; // sqrt(2/pi)
+ float xf = float(x);
+ float yf =
+ 0.5f * xf * (1.0f + metal::precise::tanh(k1 * (xf + k0 * xf * xf * xf)));
+ return T(yf);
+}
+
+template <typename T>
+inline T activate(T x, ActivationType type) {
+ switch (type) {
+ case SILU:
+ return activate_silu(x);
+ case GELU:
+ return activate_gelu(x);
+ case IDENTITY:
+ return x;
+ }
+}
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index bc7881a3b..c01cf454c 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -1,11 +1,8 @@
#include <metal_stdlib>
#include "../common/dsl.h"
+#include "quantization.h"
-enum QuantizationMode : uint {
- QUANT_UINT4 = 0,
- QUANT_INT8 = 1,
- QUANT_UINT8 = 2
-};
+using namespace uzu::quantization;
template <typename T>
VARIANTS(T, float, half, bfloat)
@@ -19,8 +16,8 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
constant uint32_t& vocab_size,
constant uint32_t& model_dim,
constant float& input_scale,
+ const constant QuantizationMode& quant_mode,
const uint32_t group_size SPECIALIZE,
- const uint quant_mode SPECIALIZE,
const uint dim_idx AXIS(model_dim, 16),
const uint batch_idx AXIS(batch_size, 16)
) {
@@ -37,11 +34,12 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
const T scale = scales[token_id * num_groups + group_idx];
const T bias = biases[token_id * num_groups + group_idx];
- const uint packing_divisor = quant_mode == QUANT_UINT4 ? 2 : 1;
+ const uint packing_divisor = quant_mode == QuantizationMode::UINT4 ? 2 : 1;
const uint weights_stride = model_dim / packing_divisor;
int quantized_value = 0;
- if (quant_mode == QUANT_UINT4) {
+ switch (quant_mode) {
+ case UINT4: {
const uint byte_idx = token_id * weights_stride + (dim_idx / 2);
const uint8_t packed = weights[byte_idx];
if ((dim_idx & 1) == 0) {
@@ -49,14 +47,20 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
} else {
quantized_value = int((packed >> 4) & 0x0F);
}
- } else if (quant_mode == QUANT_INT8) {
+ break;
+ };
+ case INT8: {
const uint elem_idx = token_id * weights_stride + dim_idx;
const device int8_t* weights_i8 =
reinterpret_cast<const device int8_t*>(weights);
quantized_value = int(weights_i8[elem_idx]);
- } else {
+ break;
+ };
+ case UINT8: {
const uint elem_idx = token_id * weights_stride + dim_idx;
quantized_value = int(weights[elem_idx]);
+ break;
+ };
}
float out_f = float(scale) * float(quantized_value) + float(bias);
diff --git a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
index 05b5f58e1..c17c6b07e 100644
--- a/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
+++ b/crates/uzu/src/backends/metal/kernel/mlp/activation_mul.metal
@@ -1,7 +1,9 @@
#include <metal_stdlib>
-#include "../activation/activation.h"
+#include "../activation/activations.h"
#include "../common/dsl.h"
+using namespace uzu::activation_type;
+
template <typename T>
VARIANTS(T, float, half, bfloat)
PUBLIC KERNEL(MlpGateActMul) (
@@ -9,7 +11,7 @@ PUBLIC KERNEL(MlpGateActMul) (
device T* hidden,
const constant int& h,
const constant int& m,
- const constant uint& act_type,
+ const constant ActivationType& act_type,
uint j AXIS(h, 64),
uint row AXIS(m, 1)
) {
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
index 0c48c0d01..2fc6e8114 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/conv1d.metal
@@ -1,6 +1,8 @@
#include <metal_stdlib>
+#include "../activation/activations.h"
#include "../common/dsl.h"
-#include "ssm_common.h"
+
+using namespace uzu::activation_type;
template <typename T>
VARIANTS(T, float, half, bfloat)
@@ -44,7 +46,7 @@ PUBLIC KERNEL(Conv1dDecode)(
constant const uint& suffix_len,
constant const uint& inner_dim,
constant const uint& proj_dim,
- const uint activation_type SPECIALIZE,
+ constant const ActivationType& activation_type,
const bool has_bias SPECIALIZE,
const bool state_in_place SPECIALIZE,
const uint token_idx AXIS(suffix_len, 32),
@@ -76,7 +78,7 @@ PUBLIC KERNEL(Conv1dDecode)(
acc += float(w_row[current_tap_index]) * current;
}
- const T activated = apply_activation_fn(static_cast<T>(acc), activation_type);
+ const T activated = activate(static_cast<T>(acc), activation_type);
if (channel_idx < inner_dim) {
const uint dst = token_idx * inner_dim + channel_idx;
x_out[dst] = activated;
@@ -120,7 +122,7 @@ PUBLIC KERNEL(Conv1dScan)(
constant const uint& num_channels,
constant const uint& inner_dim,
constant const uint& proj_dim,
- const uint activation_type SPECIALIZE,
+ constant const ActivationType& activation_type,
const bool has_bias SPECIALIZE,
const uint token_idx AXIS(suffix_len + kernel_size.saturating_sub(1), 32),
const uint channel_idx AXIS(num_channels, 1)
@@ -143,8 +145,7 @@ PUBLIC KERNEL(Conv1dScan)(
acc += float(w_row[tap]) * sample;
}
- const T activated =
- apply_activation_fn(static_cast<T>(acc), activation_type);
+ const T activated = activate(static_cast<T>(acc), activation_type);
if (channel_idx < inner_dim) {
const size_t dst = size_t(token_idx) * inner_dim + channel_idx;
x_out[dst] = activated;
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
index d9b4001f3..6330408ee 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_prefill.metal
@@ -1,5 +1,6 @@
#include <metal_stdlib>
#include <metal_simdgroup>
+#include "../activation/activations.h"
#include "../common/dsl.h"
#include "../common/thread_context.h"
#include "ssm_common.h"
@@ -79,7 +80,7 @@ PUBLIC KERNEL(SSDPrefill64)(
const float x_val = float(x[x_idx]);
const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
- const float gate = float(apply_silu(z[x_idx]));
+ const float gate = float(activate_silu(z[x_idx]));
const float skip = d_scalar * x_val;
const float dt_scaled_input = x_val;
@@ -176,7 +177,7 @@ PUBLIC KERNEL(SSDPrefill)(
const float x_val = float(x[x_idx]);
const float decay_val = fast::exp(-float(softplus(float(dt_raw[dt_idx]))));
- const float gate = float(apply_silu(z[x_idx]));
+ const float gate = float(activate_silu(z[x_idx]));
const float skip = d_scalar * x_val;
const float dt_scaled_input = x_val;
@@ -246,7 +247,7 @@ PUBLIC KERNEL(SSDPrefillSequential)(
const T this_dt = softplus(dt_raw_val);
const T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
const T this_D = d[h_idx];
- const T this_z = apply_silu(z[x_idx]);
+ const T this_z = activate_silu(z[x_idx]);
const T dt_scaled_input = this_x;
T acc = T(0);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
index 8ddf31f79..4afcc0302 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssd_update.metal
@@ -1,4 +1,5 @@
#include <metal_stdlib>
+#include "../activation/activations.h"
#include "../common/dsl.h"
#include "ssm_common.h"
@@ -55,7 +56,7 @@ PUBLIC KERNEL(SSDUpdate)(
T this_decay = static_cast<T>(fast::exp(-float(this_dt)));
T this_d = d[h_idx];
T this_z = z[x_idx];
- this_z = apply_silu(this_z);
+ this_z = activate_silu(this_z);
T dt_scaled_input = this_x;
T temp = T(0);
diff --git a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
index 1e105a86e..41845ba0d 100644
--- a/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
+++ b/crates/uzu/src/backends/metal/kernel/ssm/ssm_common.h
@@ -1,14 +1,4 @@
-#define ACTIVATION_IDENTITY 0
-#define ACTIVATION_SILU 1
-#define ACTIVATION_GELU 2
-
-template <typename T>
-inline T apply_silu(T x) {
- float xf = float(x);
- float y = 1.0f / (1.0f + fast::exp(-fabs(xf)));
- float out = (xf < 0.0f) ? (1.0f - y) * xf : y * xf;
- return static_cast<T>(out);
-}
+#include <metal_stdlib>
template <typename T>
inline T softplus(T x) {
@@ -17,24 +7,4 @@ inline T softplus(T x) {
return x;
}
return static_cast<T>(log(1.0f + fast::exp(xf)));
-}
-
-template <typename T>
-inline T apply_gelu(T x) {
- float xf = float(x);
- return static_cast<T>(
- 0.5f * xf *
- (1.0f + fast::tanh(0.797885f * (xf + 0.044715f * xf * xf * xf)))
- );
-}
-
-template <typename T>
-inline T apply_activation_fn(T x, int activation_type) {
- if (activation_type == ACTIVATION_SILU) {
- return apply_silu(x);
- } else if (activation_type == ACTIVATION_GELU) {
- return apply_gelu(x);
- } else {
- return x; // Identity
- }
}
\ No newline at end of file
diff --git a/crates/uzu/src/config/attention.rs b/crates/uzu/src/config/attention.rs
index ff720dbab..7198357ed 100644
--- a/crates/uzu/src/config/attention.rs
+++ b/crates/uzu/src/config/attention.rs
@@ -29,12 +29,10 @@ mod tests {
use serde_json5::from_str;
use super::{
- super::{
- common::{ConfigDataType, QuantizationMode},
- linear::QuantizationConfig,
- },
+ super::{common::ConfigDataType, linear::QuantizationConfig},
*,
};
+ use crate::backends::common::gpu_types::QuantizationMode;
#[test]
fn test_attention_config() {
@@ -77,8 +75,8 @@ mod tests {
qkv_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -87,8 +85,8 @@ mod tests {
out_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
diff --git a/crates/uzu/src/config/classifier/prediction_head_config.rs b/crates/uzu/src/config/classifier/prediction_head_config.rs
index 8c45bab7c..67466d5d0 100644
--- a/crates/uzu/src/config/classifier/prediction_head_config.rs
+++ b/crates/uzu/src/config/classifier/prediction_head_config.rs
@@ -1,11 +1,11 @@
use serde::{Deserialize, Serialize};
-use crate::{Activation, LinearConfig, NormalizationConfig};
+use crate::{LinearConfig, NormalizationConfig, backends::common::ActivationConfig};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct PredictionHeadConfig {
pub dense_config: LinearConfig,
- pub activation: Activation,
+ pub activation: ActivationConfig,
pub normalization_config: NormalizationConfig,
pub readout_config: LinearConfig,
pub use_dense_bias: bool,
diff --git a/crates/uzu/src/config/common.rs b/crates/uzu/src/config/common.rs
index 92bf13805..b3272ca5b 100644
--- a/crates/uzu/src/config/common.rs
+++ b/crates/uzu/src/config/common.rs
@@ -37,69 +37,3 @@ impl From<DataType> for ConfigDataType {
}
}
}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Copy, Clone)]
-#[serde(rename_all = "lowercase")]
-pub enum QuantizationMode {
- UInt4,
- Int8,
- UInt8,
-}
-
-impl QuantizationMode {
- pub fn packing_divisor(&self) -> usize {
- match self {
- QuantizationMode::UInt4 => 2,
- QuantizationMode::Int8 => 1,
- QuantizationMode::UInt8 => 1,
- }
- }
-
- pub fn storage_type(&self) -> DataType {
- match self {
- QuantizationMode::UInt4 => DataType::U8,
- QuantizationMode::Int8 => DataType::I8,
- QuantizationMode::UInt8 => DataType::U8,
- }
- }
-}
-
-impl From<QuantizationMode> for DataType {
- fn from(val: QuantizationMode) -> Self {
- match val {
- QuantizationMode::UInt4 => DataType::U4,
- QuantizationMode::Int8 => DataType::I8,
- QuantizationMode::UInt8 => DataType::U8,
- }
- }
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Copy, Clone)]
-#[serde(tag = "type")]
-pub enum Activation {
- #[serde(rename = "SiLU")]
- SiLU {
- #[serde(default = "default_silu_alpha")]
- alpha: f32,
- },
- #[serde(rename = "GELU")]
- Gelu,
- #[serde(rename = "Identity")]
- Identity,
-}
-
-fn default_silu_alpha() -> f32 {
- 1.0
-}
-
-impl Activation {
- pub fn alpha(&self) -> f32 {
- match self {
- Activation::SiLU {
- alpha,
- } => *alpha,
- Activation::Gelu => 1.0,
- Activation::Identity => 1.0,
- }
- }
-}
diff --git a/crates/uzu/src/config/decoder.rs b/crates/uzu/src/config/decoder.rs
index a12e4ec80..1342f94ed 100644
--- a/crates/uzu/src/config/decoder.rs
+++ b/crates/uzu/src/config/decoder.rs
@@ -229,7 +229,7 @@ mod tests {
use super::{
super::{
attention::AttentionConfig,
- common::{Activation, ConfigDataType, QuantizationMode},
+ common::ConfigDataType,
embedding::{EmbeddingConfig, EmbeddingConfigCommon},
linear::{LinearConfig, QuantizationConfig},
mlp::{DenseMLPConfig, MLPConfig},
@@ -238,6 +238,7 @@ mod tests {
},
*,
};
+ use crate::backends::common::{ActivationConfig, gpu_types::QuantizationMode};
#[test]
fn test_decoder_config() {
@@ -354,8 +355,8 @@ mod tests {
logit_soft_cap: None,
},
group_size: 128,
- embedding_quantization_mode: QuantizationMode::Int8,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ embedding_quantization_mode: QuantizationMode::INT8,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
global_rope_config: Some(RoPEConfig::Llama {
@@ -391,8 +392,8 @@ mod tests {
qkv_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -401,8 +402,8 @@ mod tests {
out_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -425,16 +426,14 @@ mod tests {
linear_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
lora_scale: 2.0,
},
- activation: Activation::SiLU {
- alpha: 1.0,
- },
+ activation: ActivationConfig::silu_default(),
has_up_biases: false,
has_down_biases: false,
gate_clipping: None,
diff --git a/crates/uzu/src/config/decoder_layer.rs b/crates/uzu/src/config/decoder_layer.rs
index ccb9e2aca..a1f559a04 100644
--- a/crates/uzu/src/config/decoder_layer.rs
+++ b/crates/uzu/src/config/decoder_layer.rs
@@ -106,13 +106,16 @@ mod tests {
use super::{
super::{
- common::{Activation, ConfigDataType, QuantizationMode},
+ common::ConfigDataType,
linear::{LinearConfig, QuantizationConfig},
normalization::UpcastMode,
},
*,
};
- use crate::config::mlp;
+ use crate::{
+ backends::common::{ActivationConfig, gpu_types::QuantizationMode},
+ config::mlp,
+ };
#[test]
fn test_decoder_layer_config() {
@@ -204,8 +207,8 @@ mod tests {
qkv_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -214,8 +217,8 @@ mod tests {
out_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -238,16 +241,14 @@ mod tests {
linear_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
lora_scale: 2.0,
},
- activation: Activation::SiLU {
- alpha: 1.0,
- },
+ activation: ActivationConfig::silu_default(),
has_up_biases: false,
has_down_biases: false,
gate_clipping: None,
diff --git a/crates/uzu/src/config/embedding.rs b/crates/uzu/src/config/embedding.rs
index 9562a22fc..5e298212d 100644
--- a/crates/uzu/src/config/embedding.rs
+++ b/crates/uzu/src/config/embedding.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::{ConfigDataType, QuantizationMode};
+use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct EmbeddingConfigCommon {
@@ -105,7 +105,7 @@ mod tests {
logit_soft_cap: None,
},
group_size: 128,
- embedding_quantization_mode: QuantizationMode::UInt4,
+ embedding_quantization_mode: QuantizationMode::UINT4,
activation_quantization_mode: None,
activation_precision: ConfigDataType::BFloat16,
};
@@ -131,7 +131,7 @@ mod tests {
logit_soft_cap: None,
},
group_size: 128,
- embedding_quantization_mode: QuantizationMode::UInt4,
+ embedding_quantization_mode: QuantizationMode::UINT4,
activation_quantization_mode: None,
activation_precision: ConfigDataType::BFloat16,
};
diff --git a/crates/uzu/src/config/linear.rs b/crates/uzu/src/config/linear.rs
index db8c21645..ce68e481c 100644
--- a/crates/uzu/src/config/linear.rs
+++ b/crates/uzu/src/config/linear.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::{ConfigDataType, QuantizationMode};
+use crate::{ConfigDataType, backends::common::gpu_types::QuantizationMode};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct QuantizationConfig {
@@ -69,8 +69,8 @@ mod tests {
let ground_truth_config = LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
diff --git a/crates/uzu/src/config/mamba.rs b/crates/uzu/src/config/mamba.rs
index 055dc3c45..aaa0a8cd3 100644
--- a/crates/uzu/src/config/mamba.rs
+++ b/crates/uzu/src/config/mamba.rs
@@ -1,9 +1,7 @@
use serde::{Deserialize, Serialize};
-use super::{
- common::{Activation, ConfigDataType},
- linear::LinearConfig,
-};
+use super::{common::ConfigDataType, linear::LinearConfig};
+use crate::backends::common::ActivationConfig;
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct CausalConv1dConfig {
@@ -16,7 +14,7 @@ pub struct Mamba2Config {
pub in_projection_config: LinearConfig,
pub out_projection_config: LinearConfig,
pub conv_config: CausalConv1dConfig,
- pub activation: Activation,
+ pub activation: ActivationConfig,
pub kernel_size: usize,
pub num_heads: usize,
pub num_groups: usize,
diff --git a/crates/uzu/src/config/mlp.rs b/crates/uzu/src/config/mlp.rs
index 7e279ec78..49e088c91 100644
--- a/crates/uzu/src/config/mlp.rs
+++ b/crates/uzu/src/config/mlp.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::{Activation, LinearConfig};
+use crate::{LinearConfig, backends::common::ActivationConfig};
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
#[serde(tag = "type")]
@@ -14,7 +14,7 @@ pub enum MLPConfig {
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct DenseMLPConfig {
pub linear_config: LinearConfig,
- pub activation: Activation,
+ pub activation: ActivationConfig,
#[serde(default)]
pub has_up_biases: bool,
#[serde(default)]
@@ -51,7 +51,7 @@ pub enum RoutingFunctionConfig {
#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
pub struct MoeExpertConfig {
pub linear_config: LinearConfig,
- pub activation: Activation,
+ pub activation: ActivationConfig,
pub has_up_biases: bool,
pub has_down_biases: bool,
pub gate_clipping: [Option<f32>; 2],
@@ -63,7 +63,7 @@ mod tests {
use serde_json::from_str;
use super::{super::linear::QuantizationConfig, *};
- use crate::config::{Activation, ConfigDataType, QuantizationMode};
+ use crate::{backends::common::gpu_types::QuantizationMode, config::ConfigDataType};
#[test]
fn test_dense_mlp_config() {
@@ -87,16 +87,14 @@ mod tests {
linear_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
lora_scale: 2.0,
},
- activation: Activation::SiLU {
- alpha: 1.0,
- },
+ activation: ActivationConfig::silu_default(),
has_up_biases: false,
has_down_biases: false,
gate_clipping: None,
@@ -149,16 +147,14 @@ mod tests {
linear_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
lora_scale: 2.0,
},
- activation: Activation::SiLU {
- alpha: 1.0,
- },
+ activation: ActivationConfig::silu_default(),
has_up_biases: true,
has_down_biases: true,
gate_clipping: [None, Some(7.0)],
@@ -214,16 +210,14 @@ mod tests {
linear_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
lora_scale: 2.0,
},
- activation: Activation::SiLU {
- alpha: 1.0,
- },
+ activation: ActivationConfig::silu_default(),
has_up_biases: true,
has_down_biases: true,
gate_clipping: [None, Some(7.0)],
diff --git a/crates/uzu/src/config/transformer_layer.rs b/crates/uzu/src/config/transformer_layer.rs
index 791169dcb..3735ec991 100644
--- a/crates/uzu/src/config/transformer_layer.rs
+++ b/crates/uzu/src/config/transformer_layer.rs
@@ -27,13 +27,16 @@ mod tests {
use super::{
super::{
attention::AttentionConfig,
- common::{Activation, ConfigDataType, QuantizationMode},
+ common::ConfigDataType,
linear::{LinearConfig, QuantizationConfig},
normalization::UpcastMode,
},
*,
};
- use crate::config::mlp;
+ use crate::{
+ backends::common::{ActivationConfig, gpu_types::QuantizationMode},
+ config::mlp,
+ };
#[test]
fn test_transformer_layer_config() {
@@ -132,8 +135,8 @@ mod tests {
qkv_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -142,8 +145,8 @@ mod tests {
out_projection_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
@@ -166,16 +169,14 @@ mod tests {
linear_config: LinearConfig::QLoRA {
quantization: QuantizationConfig {
group_size: 32,
- weight_quantization_mode: QuantizationMode::UInt4,
- activation_quantization_mode: Some(QuantizationMode::Int8),
+ weight_quantization_mode: QuantizationMode::UINT4,
+ activation_quantization_mode: Some(QuantizationMode::INT8),
activation_precision: ConfigDataType::BFloat16,
},
lora_rank: 16,
lora_scale: 2.0,
},
- activation: Activation::SiLU {
- alpha: 1.0,
- },
+ activation: ActivationConfig::silu_default(),
has_up_biases: false,
has_down_biases: false,
gate_clipping: None,
diff --git a/crates/uzu/src/encodable_block/activation.rs b/crates/uzu/src/encodable_block/activation.rs
index 199301464..810b64344 100644
--- a/crates/uzu/src/encodable_block/activation.rs
+++ b/crates/uzu/src/encodable_block/activation.rs
@@ -5,16 +5,16 @@ use std::ops::DerefMut;
use crate::{
DataType,
backends::common::{
- Backend, CommandBuffer,
+ ActivationConfig, Backend, CommandBuffer,
+ gpu_types::ActivationType,
kernel::{ActivationKernel, Kernels},
},
- config::Activation as ActivationConfig,
forward_pass::state::{ArrayId, ForwardPassState},
};
pub struct Activation<B: Backend> {
kernel: <B::Kernels as Kernels>::ActivationKernel,
- config: ActivationConfig,
+ activation: ActivationConfig,
input_array_id: ArrayId,
output_array_id: ArrayId,
}
@@ -31,7 +31,7 @@ impl<B: Backend> Activation<B> {
<B::Kernels as Kernels>::ActivationKernel::new(context, data_type, input_array_id == output_array_id)?;
Ok(Self {
kernel,
- config,
+ activation: config,
input_array_id,
output_array_id,
})
@@ -47,13 +47,9 @@ impl<B: Backend> Activation<B> {
let output_array = arrays[1].borrow_mut();
let n = input_array.shape().iter().product::<usize>();
- let act_type = match self.config {
- ActivationConfig::SiLU {
- ..
- } => 0,
- ActivationConfig::Gelu => 1,
- ActivationConfig::Identity => panic!("Identity activation is not supported for kernel"),
- };
+ if self.activation.act_type() == ActivationType::IDENTITY {
+ panic!("Identity activation is not supported for kernel");
+ }
let input_buffer = (self.input_array_id != self.output_array_id).then(|| input_array.buffer());
let input_buffer_borrow = input_buffer.as_ref().map(|b| b.borrow());
@@ -61,7 +57,7 @@ impl<B: Backend> Activation<B> {
input_buffer_borrow.as_deref(),
output_array.buffer().borrow_mut().deref_mut(),
n as u32,
- act_type,
+ self.activation.act_type(),
command_buffer,
);
Ok(())
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index a630ffefb..7e6a3d935 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -6,9 +6,10 @@ use std::{
use thiserror::Error;
use crate::{
- DataType, EmbeddingConfig, QuantizationMode,
+ DataType, EmbeddingConfig,
backends::common::{
Backend, CommandBuffer, Kernels,
+ gpu_types::QuantizationMode,
kernel::{
FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,
matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
@@ -99,14 +100,7 @@ pub struct Embedding<B: Backend> {
input_scale: f32,
vocab_size: u32,
model_dim: u32,
-}
-
-fn quant_mode_to_int(quant_mode: QuantizationMode) -> u32 {
- match quant_mode {
- QuantizationMode::UInt4 => 0,
- QuantizationMode::Int8 => 1,
- QuantizationMode::UInt8 => 2,
- }
+ quant_mode: Option<QuantizationMode>,
}
fn validate_tensor<'file, 'context, 'leaf, B: Backend>(
@@ -138,6 +132,7 @@ impl<B: Backend> Embedding<B> {
parameter_tree: &ParameterTree<B::Context>,
) -> Result<Self, EmbeddingError<B>> {
let common = config.common();
+ let mut quant_mode: Option<QuantizationMode> = None;
let tying = match config {
EmbeddingConfig::Tied {
@@ -225,9 +220,9 @@ impl<B: Backend> Embedding<B> {
context,
data_type,
*group_size as u32,
- quant_mode_to_int(*embedding_quantization_mode),
)
.map_err(EmbeddingError::BackendError)?;
+ quant_mode = Some(*embedding_quantization_mode);
let readout = QuantizedMatmulKernelEncodable::new(
context,
QuantizedMatmulConfiguration {
@@ -305,9 +300,9 @@ impl<B: Backend> Embedding<B> {
context,
data_type,
*group_size as u32,
- quant_mode_to_int(*embedding_quantization_mode),
)
.map_err(EmbeddingError::BackendError)?;
+ quant_mode = Some(*embedding_quantization_mode);
let readout = QuantizedMatmulKernelEncodable::new(
context,
QuantizedMatmulConfiguration {
@@ -423,6 +418,7 @@ impl<B: Backend> Embedding<B> {
input_scale,
vocab_size,
model_dim,
+ quant_mode,
})
}
@@ -501,6 +497,7 @@ impl<B: Backend> Embedding<B> {
self.vocab_size,
self.model_dim,
self.input_scale,
+ self.quant_mode.unwrap(),
command_buffer,
);
},
diff --git a/crates/uzu/src/encodable_block/mamba_mixer.rs b/crates/uzu/src/encodable_block/mamba_mixer.rs
index ff2c8b42a..bc0d2ade9 100644
--- a/crates/uzu/src/encodable_block/mamba_mixer.rs
+++ b/crates/uzu/src/encodable_block/mamba_mixer.rs
@@ -6,7 +6,7 @@ use std::{
};
use crate::{
- Activation, DataType,
+ DataType,
array::Array,
backends::common::{
Backend, CommandBuffer, Kernels,
@@ -107,17 +107,12 @@ impl<B: Backend> MambaMixer<B> {
let split_inproj = <B::Kernels as Kernels>::SplitInProjKernel::new(context, data_type)
.expect("Failed to create split in-projection kernel");
- let conv_scan = <B::Kernels as Kernels>::Conv1dScanKernel::new(
- context,
- data_type,
- Self::activation_to_int(&mamba_config.activation),
- mamba_config.conv_config.has_biases,
- )
- .expect("Failed to create conv scan kernel");
+ let conv_scan =
+ <B::Kernels as Kernels>::Conv1dScanKernel::new(context, data_type, mamba_config.conv_config.has_biases)
+ .expect("Failed to create conv scan kernel");
let conv_decode = <B::Kernels as Kernels>::Conv1dDecodeKernel::new(
context,
data_type,
- Self::activation_to_int(&mamba_config.activation),
mamba_config.conv_config.has_biases,
true,
)
@@ -235,6 +230,7 @@ impl<B: Backend> MambaMixer<B> {
suffix_length as u32,
inner_dim as u32,
proj_dim as u32,
+ self.config.activation.act_type(),
command_buffer,
);
}
@@ -279,6 +275,7 @@ impl<B: Backend> MambaMixer<B> {
conv_dim as u32,
inner_dim as u32,
proj_dim as u32,
+ self.config.activation.act_type(),
command_buffer,
)
}
@@ -397,16 +394,6 @@ impl<B: Backend> MambaMixer<B> {
);
}
- fn activation_to_int(activation: &Activation) -> u32 {
- match activation {
- Activation::Identity => 0,
- Activation::SiLU {
- ..
- } => 1,
- Activation::Gelu => 2,
- }
- }
-
pub(crate) fn encode(
&self,
state: &mut ForwardPassState<B>,
diff --git a/crates/uzu/src/encodable_block/mlp/moe.rs b/crates/uzu/src/encodable_block/mlp/moe.rs
index 7efde157c..4339670ef 100644
--- a/crates/uzu/src/encodable_block/mlp/moe.rs
+++ b/crates/uzu/src/encodable_block/mlp/moe.rs
@@ -9,10 +9,11 @@ use std::{
use thiserror::Error;
use crate::{
- Activation, DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,
+ DataType, LinearConfig, MixtureOfExpertsConfig, RoutingFunctionConfig,
array::Array,
backends::common::{
Backend, CommandBuffer, CommandBufferEncoding, Kernels,
+ gpu_types::ActivationType,
kernel::{
MoeBlockBasesFromPartialsKernel, MoeCountsOffsetsFusedKernel, MoeFinalizeKernel, MoeRouterTopKKernel,
MoeScatterBucketsMapKernel,
@@ -175,13 +176,13 @@ impl<B: Backend> MoeBlock<B> {
})
}
- fn gating_code_from_activation(activation: &Activation) -> u32 {
+ fn gating_code_from_activation(activation: &ActivationType) -> u32 {
match activation {
- Activation::Gelu => 3,
- Activation::SiLU {
+ ActivationType::GELU => 3,
+ ActivationType::SILU {
..
} => 2,
- Activation::Identity => {
+ ActivationType::IDENTITY => {
panic!("Identity activation is not supported for MoE kernels")
},
}
@@ -362,7 +363,7 @@ impl<B: Backend> Mlp<B> for MoeBlock<B> {
},
);
- let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation);
+ let gating_code = Self::gating_code_from_activation(&self.moe_config.expert_config.activation.act_type());
// Compute clipping values and alpha for expert kernels
let gate_clip_min = self.moe_config.expert_config.gate_clipping[0].unwrap_or(f32::NEG_INFINITY);
diff --git a/crates/uzu/tests/kernel/activation_mul_test.rs b/crates/uzu/tests/kernel/activation_mul_test.rs
index 4bbbce9b9..d900a1452 100644
--- a/crates/uzu/tests/kernel/activation_mul_test.rs
+++ b/crates/uzu/tests/kernel/activation_mul_test.rs
@@ -11,7 +11,7 @@ use uzu::{
backends::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
- Context, Kernels, kernel::MlpGateActMulKernel,
+ Context, Kernels, gpu_types::ActivationType, kernel::MlpGateActMulKernel,
},
cpu::Cpu,
},
@@ -23,10 +23,10 @@ struct Input<T: ArrayElement + Float> {
fused_up: Box<[T]>,
h: i32,
m: i32,
- act_type: u32,
+ act_type: ActivationType,
}
-fn get_test_data<T: ArrayElement + Float>(act_type: u32) -> (Input<T>, Vec<T>) {
+fn get_test_data<T: ArrayElement + Float>(act_type: ActivationType) -> (Input<T>, Vec<T>) {
let h = 64i32;
let m = 4i32;
let fused_len = (m * 2 * h) as usize;
@@ -73,7 +73,7 @@ fn get_output<T: ArrayElement + Float, B: Backend>(input: &Input<T>) -> Vec<T> {
hidden_array.as_slice().to_vec()
}
-fn test<T: ArrayElement + Float + Debug + Display>(act_type: u32) {
+fn test<T: ArrayElement + Float + Debug + Display>(act_type: ActivationType) {
let eps = if matches!(T::data_type(), DataType::F16 | DataType::BF16) {
0.02f32
} else {
@@ -90,30 +90,30 @@ fn test<T: ArrayElement + Float + Debug + Display>(act_type: u32) {
#[test]
fn test_silu_f32() {
- test::<f32>(0);
+ test::<f32>(ActivationType::SILU);
}
#[test]
fn test_silu_f16() {
- test::<f16>(0);
+ test::<f16>(ActivationType::SILU);
}
#[test]
fn test_silu_bf16() {
- test::<bf16>(0);
+ test::<bf16>(ActivationType::SILU);
}
#[test]
fn test_gelu_f32() {
- test::<f32>(1);
+ test::<f32>(ActivationType::GELU);
}
#[test]
fn test_gelu_f16() {
- test::<f16>(1);
+ test::<f16>(ActivationType::GELU);
}
#[test]
fn test_gelu_bf16() {
- test::<bf16>(1);
+ test::<bf16>(ActivationType::GELU);
}
diff --git a/crates/uzu/tests/quant_matmul_test.rs b/crates/uzu/tests/quant_matmul_test.rs
index 306d24b57..b3065829a 100644
--- a/crates/uzu/tests/quant_matmul_test.rs
+++ b/crates/uzu/tests/quant_matmul_test.rs
@@ -12,6 +12,7 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context,
+ gpu_types::QuantizationMode,
kernel::quant_matmul::{
QuantizedMatmulArguments, QuantizedMatmulConfiguration, QuantizedMatmulKernelEncodable,
QuantizedMatmulType,
@@ -19,7 +20,6 @@ use uzu::{
},
metal::Metal,
},
- config::QuantizationMode,
};
fn pack_u4_weights(values: &[u8]) -> Vec<u8> {
@@ -489,8 +489,8 @@ fn execute_quantized_matmul(
input_dim,
output_dim,
mode: match bits {
- 4 => QuantizationMode::UInt4,
- 8 => QuantizationMode::Int8,
+ 4 => QuantizationMode::UINT4,
+ 8 => QuantizationMode::INT8,
_ => panic!("Unsupported bits: {}", bits),
},
quantization_type,
diff --git a/crates/uzu/tests/ssd_prefill_kernel_test.rs b/crates/uzu/tests/ssd_prefill_kernel_test.rs
index c1c0ac69e..4bde961ec 100644
--- a/crates/uzu/tests/ssd_prefill_kernel_test.rs
+++ b/crates/uzu/tests/ssd_prefill_kernel_test.rs
@@ -11,6 +11,7 @@ use uzu::{
common::{
Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferInitial, CommandBufferPending,
Context, Kernels,
+ gpu_types::ActivationType,
kernel::{
Conv1dScanKernel,
ssd_prefill::{SSDPrefillArguments, SSDPrefillKernels, SSDPrefillMode},
@@ -335,6 +336,7 @@ fn run_conv_scan_once(
channels as u32,
channels as u32,
0u32,
+ ActivationType::SILU,
&mut command_buffer,
);
@@ -458,8 +460,7 @@ fn conv1d_scan_is_deterministic() {
eprintln!("Skipping conv1d scan determinism test: no Metal device");
return;
};
- let kernel =
- <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, 0u32, true).unwrap();
+ let kernel = <<Metal as Backend>::Kernels as Kernels>::Conv1dScanKernel::new(&ctx, DataType::F32, true).unwrap();
let suffix_len = 192usize;
let channels = 8usize;