Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Use QuantizationMode as specializable parameter (#267)

Aleksandr Golokozede4975a8811
diff --git a/crates/uzu/build/gpu_types/mod.rs b/crates/uzu/build/gpu_types/mod.rs
index e012805dd..6ffd32d19 100644
--- a/crates/uzu/build/gpu_types/mod.rs
+++ b/crates/uzu/build/gpu_types/mod.rs
@@ -198,7 +198,7 @@ impl GpuTypesCompiler {
let enum_name = rust_enum.ident.to_string();
definitions.push_str("enum ");
definitions.push_str(&enum_name);
- definitions.push_str(" {\n");
+ definitions.push_str(" : uint32_t {\n");
for variant in &rust_enum.variants {
definitions.push_str(" ");
diff --git a/crates/uzu/src/backends/common/gpu_types/quantization.rs b/crates/uzu/src/backends/common/gpu_types/quantization.rs
index 617afbe2b..416e9370f 100644
--- a/crates/uzu/src/backends/common/gpu_types/quantization.rs
+++ b/crates/uzu/src/backends/common/gpu_types/quantization.rs
@@ -38,3 +38,13 @@ impl From<QuantizationMode> for DataType {
}
}
}
+
+impl Into<u32> for QuantizationMode {
+ fn into(self) -> u32 {
+ match self {
+ QuantizationMode::UINT4 => 0,
+ QuantizationMode::INT8 => 1,
+ QuantizationMode::UINT8 => 2,
+ }
+ }
+}
diff --git a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
index 43d4d79c6..7fa0e433a 100644
--- a/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
+++ b/crates/uzu/src/backends/cpu/kernel/embedding/quant_embedding.rs
@@ -19,7 +19,9 @@ pub fn quantized_embedding_lookup<T: ArrayElement + Float>(
#[allow(unused)]
#[specialize]
group_size: u32,
- #[allow(unused)] quant_mode: crate::backends::common::gpu_types::quantization::QuantizationMode,
+ #[allow(unused)]
+ #[specialize]
+ quant_mode: u32,
) {
todo!()
}
diff --git a/crates/uzu/src/backends/metal/generated/activation.h b/crates/uzu/src/backends/metal/generated/activation.h
deleted file mode 100644
index 54a579fb4..000000000
--- a/crates/uzu/src/backends/metal/generated/activation.h
+++ /dev/null
@@ -1,28 +0,0 @@
-// Auto-generated from gpu_types/activation.rs - do not edit manually
-#pragma once
-
-#ifndef UZU_ACTIVATION_H
-#define UZU_ACTIVATION_H
-
-#ifdef __METAL_VERSION__
-#include <metal_stdlib>
-using namespace metal;
-
-namespace uzu {
-namespace activation {
-#else
-#include <stdint.h>
-#endif
-
-enum ActivationType {
- SILU,
- GELU,
- IDENTITY,
-};
-
-#ifdef __METAL_VERSION__
-} // namespace activation
-} // namespace uzu
-#endif
-
-#endif // UZU_ACTIVATION_H
diff --git a/crates/uzu/src/backends/metal/generated/activation_type.h b/crates/uzu/src/backends/metal/generated/activation_type.h
index c11e87a06..741264acd 100644
--- a/crates/uzu/src/backends/metal/generated/activation_type.h
+++ b/crates/uzu/src/backends/metal/generated/activation_type.h
@@ -14,7 +14,7 @@ namespace activation_type {
#include <stdint.h>
#endif
-enum ActivationType {
+enum ActivationType : uint32_t {
SILU,
GELU,
IDENTITY,
diff --git a/crates/uzu/src/backends/metal/generated/quantization.h b/crates/uzu/src/backends/metal/generated/quantization.h
index 1eba47c17..c73e771ad 100644
--- a/crates/uzu/src/backends/metal/generated/quantization.h
+++ b/crates/uzu/src/backends/metal/generated/quantization.h
@@ -14,7 +14,7 @@ namespace quantization {
#include <stdint.h>
#endif
-enum QuantizationMode {
+enum QuantizationMode : uint32_t {
UINT4,
INT8,
UINT8,
diff --git a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
index c01cf454c..67b7d6c92 100644
--- a/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
+++ b/crates/uzu/src/backends/metal/kernel/embedding/quant_embedding.metal
@@ -16,8 +16,8 @@ PUBLIC KERNEL(QuantizedEmbeddingLookup) (
constant uint32_t& vocab_size,
constant uint32_t& model_dim,
constant float& input_scale,
- const constant QuantizationMode& quant_mode,
const uint32_t group_size SPECIALIZE,
+ const uint32_t quant_mode SPECIALIZE,
const uint dim_idx AXIS(model_dim, 16),
const uint batch_idx AXIS(batch_size, 16)
) {
diff --git a/crates/uzu/src/encodable_block/embedding.rs b/crates/uzu/src/encodable_block/embedding.rs
index 7e6a3d935..775dea92f 100644
--- a/crates/uzu/src/encodable_block/embedding.rs
+++ b/crates/uzu/src/encodable_block/embedding.rs
@@ -9,7 +9,6 @@ use crate::{
DataType, EmbeddingConfig,
backends::common::{
Backend, CommandBuffer, Kernels,
- gpu_types::QuantizationMode,
kernel::{
FullPrecisionEmbeddingLookupKernel, QuantizedEmbeddingLookupKernel,
matmul::{MatmulArguments, MatmulError, MatmulKernel, MatmulKernels},
@@ -100,7 +99,6 @@ pub struct Embedding<B: Backend> {
input_scale: f32,
vocab_size: u32,
model_dim: u32,
- quant_mode: Option<QuantizationMode>,
}
fn validate_tensor<'file, 'context, 'leaf, B: Backend>(
@@ -132,7 +130,6 @@ impl<B: Backend> Embedding<B> {
parameter_tree: &ParameterTree<B::Context>,
) -> Result<Self, EmbeddingError<B>> {
let common = config.common();
- let mut quant_mode: Option<QuantizationMode> = None;
let tying = match config {
EmbeddingConfig::Tied {
@@ -220,9 +217,9 @@ impl<B: Backend> Embedding<B> {
context,
data_type,
*group_size as u32,
+ (*embedding_quantization_mode).into(),
)
.map_err(EmbeddingError::BackendError)?;
- quant_mode = Some(*embedding_quantization_mode);
let readout = QuantizedMatmulKernelEncodable::new(
context,
QuantizedMatmulConfiguration {
@@ -300,9 +297,9 @@ impl<B: Backend> Embedding<B> {
context,
data_type,
*group_size as u32,
+ (*embedding_quantization_mode).into(),
)
.map_err(EmbeddingError::BackendError)?;
- quant_mode = Some(*embedding_quantization_mode);
let readout = QuantizedMatmulKernelEncodable::new(
context,
QuantizedMatmulConfiguration {
@@ -418,7 +415,6 @@ impl<B: Backend> Embedding<B> {
input_scale,
vocab_size,
model_dim,
- quant_mode,
})
}
@@ -497,7 +493,6 @@ impl<B: Backend> Embedding<B> {
self.vocab_size,
self.model_dim,
self.input_scale,
- self.quant_mode.unwrap(),
command_buffer,
);
},