A high-performance inference engine for AI models
Use QuantizationMode as specializable parameter (#267)
Aleksandr Golokozede4975a8811
diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rsindex e012805dd..6ffd32d19 100644--- a/crates/uzu/build/gpu_types/mod.rs+++ b/crates/uzu/build/gpu_types/mod.rs@@ -198,7 +198,7 @@ impl GpuTypesCompiler {let enum_name = rust_enum.ident.to_string();definitions.push_str("enum ");definitions.push_str(&enum_name);- definitions.push_str(" {\n");+ definitions.push_str(" : uint32_t {\n");for variant in &rust_enum.variants {definitions.push_str(" ");diff --git a/crates/uzu/src/backends/common/gpu_types/quantization.rs b/crates/uzu/src/backends/common/gpu_types/quantization.rsindex 617afbe2b..416e9370f 100644--- a/crates/uzu/src/backends/common/gpu_types/quantization.rs+++ b/crates/uzu/src/backends/common/gpu_types/quantization.rs@@ -38,3 +38,13 @@ impl From<QuantizationMode> for DataType {}}}++impl Into<u32> for QuantizationMode {+ fn into(self) -> u32 {+ match self {+ QuantizationMode::UINT4 => 0,+ QuantizationMode::INT8 => 1,+ QuantizationMode::UINT8 => 2,+ }+ }+}diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rsindex 43d4d79c6..7fa0e433a 100644--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs@@ -19,7 +19,9 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(#[allow(unused)]#[specialize]group_size: u32,- #[allow(unused)] quant_mode: crate::backends::common::gpu_types::quantization::QuantizationMode,+ #[allow(unused)]+ #[specialize]+ quant_mode: u32,) {todo!()}diff --git a/crates/uzu/src/backends/metal/generated/activation.h b/crates/uzu/src/backends/metal/generated/activation.hdeleted file mode 100644index 54a579fb4..000000000--- a/crates/uzu/src/backends/metal/generated/activation.h+++ /dev/null@@ -1,28 +0,0 @@-// Auto-generated from gpu_types/activation.rs - do not edit manually-#pragma once--#ifndef UZU_ACTIVATION_H-#define UZU_ACTIVATION_H--#ifdef __METAL_VERSION__-#include <metal_stdlib>-using namespace metal;--namespace uzu {-namespace activation {-#else-#include <stdint.h>-#endif--enum ActivationType {- SILU,- GELU,- IDENTITY,-};--#ifdef __METAL_VERSION__-} // namespace activation-} // namespace uzu-#endif--#endif // UZU_ACTIVATION_Hdiff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.hindex c11e87a06..741264acd 100644--- a/crates/uzu/src/backends/metal/generated/activation_type.h+++ b/crates/uzu/src/backends/metal/generated/activation_type.h@@ -14,7 +14,7 @@ namespace activation_type {#include <stdint.h>#endif-enum ActivationType {+enum ActivationType : uint32_t {SILU,GELU,IDENTITY,diff --git a/crates/uzu/src/backends/metal/generated/quantization.h b/crates/uzu/src/backends/metal/generated/quantization.hindex 1eba47c17..c73e771ad 100644--- a/crates/uzu/src/backends/metal/generated/quantization.h+++ b/crates/uzu/src/backends/metal/generated/quantization.h@@ -14,7 +14,7 @@ namespace quantization {#include <stdint.h>#endif-enum QuantizationMode {+enum QuantizationMode : uint32_t {UINT4,INT8,UINT8,diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metalindex c01cf454c..67b7d6c92 100644--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal@@ -16,8 +16,8 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (constant uint32_t& vocab_size,constant uint32_t& model_dim,constant float& input_scale,- const constant QuantizationMode& quant_mode,const uint32_t group_size SPECIALIZE,+ const uint32_t quant_mode SPECIALIZE,const uint dim_idx AXIS(model_dim, 16),const uint batch_idx AXIS(batch_size, 16)) {diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rsindex 7e6a3d935..775dea92f 100644--- a/crates/uzu/src/encodable_block/embedding.rs+++ b/crates/uzu/src/encodable_block/embedding.rs@@ -9,7 +9,6 @@ use crate::{DataType, EmbeddingConfig,backends::common::{Backend, CommandBuffer, Kernels,- gpu_types::QuantizationMode,kernel::{FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},@@ -100,7 +99,6 @@ pub struct Embedding<B: Backend> {input_scale: f32,vocab_size: u32,model_dim: u32,- quant_mode: Option<QuantizationMode>,}fn validate_tensor<'file, 'context, 'leaf, B: Backend>(@@ -132,7 +130,6 @@ impl<B: Backend> Embedding<B> {parameter_tree: &ParameterTree<B::Context>,) -> Result<Self, EmbeddingError<B>> {let common = config.common();- let mut quant_mode: Option<QuantizationMode> = None;let tying = match config {EmbeddingConfig::Tied {@@ -220,9 +217,9 @@ impl<B: Backend> Embedding<B> {context,data_type,*group_size as u32,+ (*embedding_quantization_mode).into(),).map_err(EmbeddingError::BackendError)?;- quant_mode = Some(*embedding_quantization_mode);let readout = QuantizedMatmulKernelEncodable::new(context,QuantizedMatmulConfiguration {@@ -300,9 +297,9 @@ impl<B: Backend> Embedding<B> {context,data_type,*group_size as u32,+ (*embedding_quantization_mode).into(),).map_err(EmbeddingError::BackendError)?;- quant_mode = Some(*embedding_quantization_mode);let readout = QuantizedMatmulKernelEncodable::new(context,QuantizedMatmulConfiguration {@@ -418,7 +415,6 @@ impl<B: Backend> Embedding<B> {input_scale,vocab_size,model_dim,- quant_mode,})}@@ -497,7 +493,6 @@ impl<B: Backend> Embedding<B> {self.vocab_size,self.model_dim,self.input_scale,- self.quant_mode.unwrap(),command_buffer,);},