Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Add decision models support into uzu-engine (#1031)

Aleksandr Golokozd1b906462267
diff --git a/crates/legacy/shoji/src/types/model/specialization.rs b/crates/legacy/shoji/src/types/model/specialization.rs
index 0c6fa4d81..376cf001c 100644
--- a/crates/legacy/shoji/src/types/model/specialization.rs
+++ b/crates/legacy/shoji/src/types/model/specialization.rs
@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};
pub enum ModelSpecialization {
Chat {},
Classification {},
+ Decision {},
TextToSpeech {},
Translation {},
Speculation {},
@@ -16,6 +17,7 @@ impl ModelSpecialization {
match self {
ModelSpecialization::Chat {} => "Chat".to_string(),
ModelSpecialization::Classification {} => "Classification".to_string(),
+ ModelSpecialization::Decision {} => "Decision".to_string(),
ModelSpecialization::TextToSpeech {} => "Text to Speech".to_string(),
ModelSpecialization::Translation {} => "Translation".to_string(),
ModelSpecialization::Speculation {} => "Speculation".to_string(),
diff --git a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
index 9fe887149..02ea555ef 100644
--- a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
+++ b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi
@@ -1058,6 +1058,11 @@ class ModelSpecialization:
__match_args__ = ()
def __new__(cls) -> ModelSpecialization.Classification: ...
+ @typing.final
+ class Decision(ModelSpecialization):
+ __match_args__ = ()
+ def __new__(cls) -> ModelSpecialization.Decision: ...
+
@typing.final
class TextToSpeech(ModelSpecialization):
__match_args__ = ()
diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
index 6cec313b8..c3f5c6afd 100644
--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift
@@ -4553,6 +4553,7 @@ public enum ModelSpecialization: Equatable, Hashable, Codable {
case chat
case classification
+ case decision
case textToSpeech
case translation
case speculation
@@ -4581,11 +4582,13 @@ public struct FfiConverterTypeModelSpecialization: FfiConverterRustBuffer {
case 2: return .classification
- case 3: return .textToSpeech
+ case 3: return .decision
- case 4: return .translation
+ case 4: return .textToSpeech
- case 5: return .speculation
+ case 5: return .translation
+
+ case 6: return .speculation
default: throw UniffiInternalError.unexpectedEnumCase
}
@@ -4603,17 +4606,21 @@ public struct FfiConverterTypeModelSpecialization: FfiConverterRustBuffer {
writeInt(&buf, Int32(2))
- case .textToSpeech:
+ case .decision:
writeInt(&buf, Int32(3))
- case .translation:
+ case .textToSpeech:
writeInt(&buf, Int32(4))
- case .speculation:
+ case .translation:
writeInt(&buf, Int32(5))
+
+ case .speculation:
+ writeInt(&buf, Int32(6))
+
}
}
}
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
index 5b8cc1e6b..b615554b0 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts
@@ -641,6 +641,10 @@ export declare class ModelSpecializationClassification {
constructor()
}
+export declare class ModelSpecializationDecision {
+ constructor()
+}
+
export declare class ModelSpecializationSpeculation {
constructor()
}
@@ -871,7 +875,7 @@ export type ModelSource =
ModelSourceRegistry | ModelSourceFilesystem
export type ModelSpecialization =
- ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation
+ ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationDecision | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation
export type PcmBatchError =
| { type: 'UnableToSaveAsWav'; message: string }
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
index 5b8cc1e6b..b615554b0 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts
@@ -641,6 +641,10 @@ export declare class ModelSpecializationClassification {
constructor()
}
+export declare class ModelSpecializationDecision {
+ constructor()
+}
+
export declare class ModelSpecializationSpeculation {
constructor()
}
@@ -871,7 +875,7 @@ export type ModelSource =
ModelSourceRegistry | ModelSourceFilesystem
export type ModelSpecialization =
- ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation
+ ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationDecision | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation
export type PcmBatchError =
| { type: 'UnableToSaveAsWav'; message: string }
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.js b/crates/legacy/uzu/bindings/typescript/src/napi/index.js
index e6e19ce48..f4f8fda31 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.js
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.js
@@ -596,6 +596,7 @@ module.exports.ModelSourceFilesystem = nativeBinding.ModelSourceFilesystem
module.exports.ModelSourceRegistry = nativeBinding.ModelSourceRegistry
module.exports.ModelSpecializationChat = nativeBinding.ModelSpecializationChat
module.exports.ModelSpecializationClassification = nativeBinding.ModelSpecializationClassification
+module.exports.ModelSpecializationDecision = nativeBinding.ModelSpecializationDecision
module.exports.ModelSpecializationSpeculation = nativeBinding.ModelSpecializationSpeculation
module.exports.ModelSpecializationTextToSpeech = nativeBinding.ModelSpecializationTextToSpeech
module.exports.ModelSpecializationTranslation = nativeBinding.ModelSpecializationTranslation
diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
index 30424c3e3..2a451d989 100644
--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs
@@ -93,6 +93,7 @@ export const ModelSourceFilesystem = cjs.ModelSourceFilesystem;
export const ModelSourceRegistry = cjs.ModelSourceRegistry;
export const ModelSpecializationChat = cjs.ModelSpecializationChat;
export const ModelSpecializationClassification = cjs.ModelSpecializationClassification;
+export const ModelSpecializationDecision = cjs.ModelSpecializationDecision;
export const ModelSpecializationSpeculation = cjs.ModelSpecializationSpeculation;
export const ModelSpecializationTextToSpeech = cjs.ModelSpecializationTextToSpeech;
export const ModelSpecializationTranslation = cjs.ModelSpecializationTranslation;
diff --git a/crates/legacy/uzu/src/registry/local/mod.rs b/crates/legacy/uzu/src/registry/local/mod.rs
index 42757054c..def1e1d52 100644
--- a/crates/legacy/uzu/src/registry/local/mod.rs
+++ b/crates/legacy/uzu/src/registry/local/mod.rs
@@ -100,6 +100,7 @@ impl Registry {
.map(|model_type| match model_type {
ModelType::LanguageModel => ModelSpecialization::Chat {},
ModelType::Classifier => ModelSpecialization::Classification {},
+ ModelType::Decision => ModelSpecialization::Decision {},
})
.map_err(|error| RegistryError::UnableToGetModels {
message: format!("Unable to resolve specialization for {}: {error}", path.display()),
diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
index 033d4b994..e59d06470 100644
--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs
@@ -1,7 +1,7 @@
#[cfg(any(backend = "webgpu", backend = "cpu"))]
use crate::backends::common::kernel::Unsupported;
use crate::{
- backends::common::{Backend, CommandBuffer, Kernels},
+ backends::common::{Backend, CommandBuffer, GpuBuffer, Kernels},
data_type::DataType,
};
@@ -9,7 +9,7 @@ pub struct DeltaNetChunkedPrefillArgs<'a, B: Backend> {
pub in_projected: &'a B::ScratchBuffer,
pub a_log: &'a B::GlobalGpuBuffer,
pub dt_bias: &'a B::GlobalGpuBuffer,
- pub ssm_state: &'a mut B::GlobalGpuBuffer,
+ pub ssm_state: &'a mut dyn GpuBuffer<Backend = B>,
pub delta_output: &'a mut B::ScratchBuffer,
pub num_heads: u32,
pub num_groups: u32,
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
index a8f86bae1..94debb345 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs
@@ -15,7 +15,7 @@ use crate::{
context::CpuContext,
kernel::matmul::{
MatmulCpuError,
- reference::{WeightData, read_bf16, write_bf16},
+ reference::{WeightData, load, store},
},
},
},
@@ -24,6 +24,8 @@ use crate::{
};
pub struct MatmulCpuKernel {
+ weights_data_type: DataType,
+ input_data_type: DataType,
output_data_type: DataType,
output_rht: ActivationTransform<Cpu>,
bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,
@@ -40,22 +42,21 @@ impl MatmulKernel for MatmulCpuKernel {
input_data_type: DataType,
output_data_type: DataType,
) -> Result<Self, Self::Error> {
- for data_type in [weights_data_type, input_data_type] {
- if data_type != DataType::BF16 {
+ for data_type in [weights_data_type, input_data_type, output_data_type] {
+ if !matches!(data_type, DataType::BF16 | DataType::F32) {
return Err(MatmulCpuError::UnsupportedDataType(data_type));
}
}
- if !matches!(output_data_type, DataType::BF16 | DataType::F32) {
- return Err(MatmulCpuError::UnsupportedDataType(output_data_type));
- }
let output_rht = ActivationTransform::output_rht(context, output_data_type, None, true)?;
let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(
context,
output_data_type,
- DataType::BF16,
+ weights_data_type,
true,
)?;
Ok(Self {
+ weights_data_type,
+ input_data_type,
output_data_type,
output_rht,
bias_add,
@@ -112,7 +113,7 @@ impl MatmulKernel for MatmulCpuKernel {
offset,
} => {
let (values, range) = values.parts();
- let byte_offset = range.start + offset * DataType::BF16.size_in_bytes();
+ let byte_offset = range.start + offset * self.input_data_type.size_in_bytes();
AData::FullPrecision(SendPtr(
values.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(byte_offset),
))
@@ -166,6 +167,8 @@ impl MatmulKernel for MatmulCpuKernel {
let weight_data = WeightData::from_b(b, b_leading_dimension, b_transpose, k_u, n_u)?;
+ let weights_data_type = self.weights_data_type;
+ let input_data_type = self.input_data_type;
let output_data_type = self.output_data_type;
let bias_after_rht = post_rht.is_some();
command_buffer.push_command(move || {
@@ -180,7 +183,7 @@ impl MatmulKernel for MatmulCpuKernel {
let mut accumulator = 0.0f32;
for inner in 0..k_u {
let a_value = match a_data {
- AData::FullPrecision(ptr) => read_bf16(ptr.as_ptr(), row * k_u + inner),
+ AData::FullPrecision(ptr) => load(ptr.as_ptr(), input_data_type, row * k_u + inner),
AData::Int8 {
values,
scales,
@@ -206,7 +209,7 @@ impl MatmulKernel for MatmulCpuKernel {
} else {
inner * leading_dimension + b_col
};
- read_bf16(ptr.as_ptr(), index)
+ load(ptr.as_ptr(), weights_data_type, index)
},
WeightData::Quantized {
weights,
@@ -234,7 +237,7 @@ impl MatmulKernel for MatmulCpuKernel {
let group_index = inner / group_size;
let metadata_index = b_col * scale_strides.output_stride as usize
+ group_index * scale_strides.group_stride as usize;
- let scale = read_bf16(scales.as_ptr(), metadata_index);
+ let scale = load(scales.as_ptr(), weights_data_type, metadata_index);
let midpoint = (1u32 << (bits - 1)) as f32;
let zero_point = zero_points.as_ref().map(|(zp, strides)| {
let zero_point_index = b_col * strides.output_stride as usize
@@ -254,7 +257,7 @@ impl MatmulKernel for MatmulCpuKernel {
let bias_term = if let Some(zp) = zero_point {
-scale * zp
} else if let Some(b) = biases {
- read_bf16(b.as_ptr(), metadata_index)
+ load(b.as_ptr(), weights_data_type, metadata_index)
} else {
-scale * midpoint
};
@@ -267,16 +270,12 @@ impl MatmulKernel for MatmulCpuKernel {
let output_index = row * n_u + col;
let mut value = output_scale * accumulator;
if !bias_after_rht && let Some(bias) = bias_ptr {
- value += read_bf16(bias.as_ptr(), col);
+ value += load(bias.as_ptr(), weights_data_type, col);
}
if let Some(cap) = soft_cap {
value = cap * (value / cap).tanh();
}
- if output_data_type == DataType::F32 {
- *d_ptr.as_ptr().cast::<f32>().add(output_index) = value;
- } else {
- write_bf16(d_ptr.as_ptr(), output_index, value);
- }
+ store(d_ptr.as_ptr(), output_data_type, output_index, value);
}
}
}
diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
index aeb51f780..7ce7cac4e 100644
--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs
@@ -1,4 +1,4 @@
-use half::bf16;
+use num_traits::AsPrimitive;
use crate::{
backends::{
@@ -8,7 +8,8 @@ use crate::{
},
cpu::{Cpu, buffer::CpuBufferExt, kernel::matmul::MatmulCpuError},
},
- utils::pointers::SendPtr,
+ data_type::DataType,
+ utils::{dispatch_dtype::dispatch_dtype, pointers::SendPtr},
};
pub(super) enum WeightData {
@@ -76,18 +77,20 @@ impl WeightData {
}
#[inline]
-pub(super) unsafe fn read_bf16(
+pub(super) unsafe fn load(
base: *const u8,
+ data_type: DataType,
index: usize,
) -> f32 {
- unsafe { f32::from(*base.cast::<bf16>().add(index)) }
+ dispatch_dtype!(|(T: data_type)| unsafe { (*base.cast::<T>().add(index)).as_() })
}
#[inline]
-pub(super) unsafe fn write_bf16(
+pub(super) unsafe fn store(
base: *mut u8,
+ data_type: DataType,
index: usize,
value: f32,
) {
- unsafe { *base.cast::<bf16>().add(index) = bf16::from_f32(value) }
+ dispatch_dtype!(|(T: data_type)| unsafe { *base.cast::<T>().add(index) = value.as_() })
}
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
index 14e859a48..b374e8727 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal
@@ -26,9 +26,11 @@ template <
uint REDUCTION_LANES,
uint GROUP_LANES,
uint NUM_SIMDGROUPS>
-VARIANTS(AT, bfloat)
-VARIANTS(BT, bfloat)
+VARIANTS(AT, bfloat, float)
+VARIANTS(BT, bfloat, float)
VARIANTS(DT, bfloat, float)
+CONSTRAINT(AT == BT)
+CONSTRAINT(BT != "float" || (DT == "float" && B_PROLOGUE == GemmBPrologueKind::FullPrecision))
VARIANTS(
B_PROLOGUE,
GemmBPrologueKind::FullPrecision,
diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
index f58754c59..dc5becb57 100644
--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs
@@ -52,7 +52,8 @@ impl MatmulMetalKernel {
input_data_type: DataType,
output_data_type: DataType,
) -> bool {
- if shape.gathered || plan.engine != gemm::GemmEngine::Mxu {
+ // F32 is compiled for GEMV only.
+ if shape.gathered || plan.engine != gemm::GemmEngine::Mxu || weights_data_type == DataType::F32 {
return false;
}
if shape.m == 5
diff --git a/crates/uzu-engine/src/config/model/decision_model.rs b/crates/uzu-engine/src/config/model/decision_model.rs
new file mode 100644
index 000000000..5641573d7
--- /dev/null
+++ b/crates/uzu-engine/src/config/model/decision_model.rs
@@ -0,0 +1,16 @@
+use uzu_engine_macros::uzu_config;
+
+use crate::{config::decoder::DecoderConfig, engine::language_model::GenerationConfig};
+
+#[uzu_config(super::ModelConfig)]
+pub struct DecisionModelConfig {
+ pub decoder_config: DecoderConfig,
+ pub generation_config: GenerationConfig,
+ pub decision: DecisionConfig,
+}
+
+#[uzu_config]
+pub struct DecisionConfig {
+ pub codes: Box<[String]>,
+ pub temperature: f32,
+}
diff --git a/crates/uzu-engine/src/config/model/mod.rs b/crates/uzu-engine/src/config/model/mod.rs
index d30b030b3..8fd786870 100644
--- a/crates/uzu-engine/src/config/model/mod.rs
+++ b/crates/uzu-engine/src/config/model/mod.rs
@@ -3,10 +3,15 @@ use uzu_engine_macros::uzu_config_abstract;
use crate::config::token_codec::AnyTokenCodecConfig;
pub mod classifier_model;
+pub mod decision_model;
pub mod generation;
pub mod language_model;
-#[uzu_config_abstract(language_model::LanguageModelConfig, classifier_model::ClassifierModelConfig)]
+#[uzu_config_abstract(
+ language_model::LanguageModelConfig,
+ classifier_model::ClassifierModelConfig,
+ decision_model::DecisionModelConfig
+)]
pub struct ModelConfig {
pub token_codec_config: AnyTokenCodecConfig,
}
diff --git a/crates/uzu-engine/src/config/token_codec/mod.rs b/crates/uzu-engine/src/config/token_codec/mod.rs
index b839add1e..a1d48f6b7 100644
--- a/crates/uzu-engine/src/config/token_codec/mod.rs
+++ b/crates/uzu-engine/src/config/token_codec/mod.rs
@@ -1,3 +1,4 @@
+use shoji::types::session::classification::{ChatTokenCodecConfig, TokenCodecConfig as ShojiTokenCodecConfig};
use uzu_engine_macros::uzu_config_abstract;
pub mod chat_codec;
@@ -5,3 +6,22 @@ pub mod raw_text_codec;
#[uzu_config_abstract(chat_codec::ChatCodecConfig, raw_text_codec::RawTextCodecConfig)]
pub struct TokenCodecConfig;
+
+impl From<&AnyTokenCodecConfig> for ShojiTokenCodecConfig {
+ fn from(config: &AnyTokenCodecConfig) -> Self {
+ match config {
+ AnyTokenCodecConfig::ChatCodecConfig(config) => ShojiTokenCodecConfig::Chat(ChatTokenCodecConfig {
+ prompt_template: config.prompt_template.clone(),
+ output_parser_regex: config.output_parser_regex.clone(),
+ system_role_name: config.system_role_name.clone(),
+ user_role_name: config.user_role_name.clone(),
+ assistant_role_name: config.assistant_role_name.clone(),
+ eos_token: config.eos_token.clone(),
+ bos_token: config.bos_token.clone(),
+ end_of_thinking_tag: config.end_of_thinking_tag.clone(),
+ default_system_prompt: config.default_system_prompt.clone(),
+ }),
+ AnyTokenCodecConfig::RawTextCodecConfig(_) => ShojiTokenCodecConfig::RawText,
+ }
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rs
index 9304eef10..98aced9bb 100644
--- a/crates/uzu-engine/src/encodable_block/classifier.rs
+++ b/crates/uzu-engine/src/encodable_block/classifier.rs
@@ -66,6 +66,7 @@ impl<B: Backend> Classifier<B> {
let embedding_norm = Normalization::new(
config.transformer_config.model_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
PostLayerScalar::None,
@@ -76,6 +77,7 @@ impl<B: Backend> Classifier<B> {
.await?;
let transformer = Transformer::new(
+ data_type,
data_type,
None,
&config.transformer_config,
diff --git a/crates/uzu-engine/src/encodable_block/decision.rs b/crates/uzu-engine/src/encodable_block/decision.rs
new file mode 100644
index 000000000..55c7d131c
--- /dev/null
+++ b/crates/uzu-engine/src/encodable_block/decision.rs
@@ -0,0 +1,143 @@
+use derive_more::Debug;
+use thiserror::Error;
+
+use crate::{
+ backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode},
+ config::model::decision_model::DecisionModelConfig,
+ data_type::DataType,
+ encodable_block::{
+ batch_topology::BatchTopology,
+ embedding_table::{EmbeddingTable, EmbeddingTableError},
+ linear::{Linear, LinearEncodeError, LinearMatmul, LinearMatmulError},
+ transformer::{Transformer, TransformerEncodeError, TransformerNewError},
+ },
+ safetensors::{ParameterLoaderError, ParameterTree},
+};
+
+#[derive(Debug, Error)]
+pub enum DecisionError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Embedding table error: {0}")]
+ EmbeddingTable(#[from] EmbeddingTableError<B>),
+ #[error("Linear encoding error: {0}")]
+ LinearEncode(#[from] LinearEncodeError<B>),
+ #[error("Linear matmul error: {0}")]
+ LinearMatmul(#[from] LinearMatmulError<B>),
+ #[error("Parameter loading error: {0}")]
+ Parameter(#[from] ParameterLoaderError<B>),
+ #[error("Transformer error: {0}")]
+ Transformer(#[from] TransformerNewError<B>),
+ #[error("Transformer encoding error: {0}")]
+ TransformerEncode(#[from] TransformerEncodeError<B>),
+}
+
+pub struct Decision<B: Backend> {
+ input_data_type: DataType,
+ input_scale: f32,
+ input_embedding: EmbeddingTable<B>,
+ transformer: Transformer<B>,
+ readout: LinearMatmul<B>,
+}
+
+impl<B: Backend> Decision<B> {
+ pub async fn new(
+ input_data_type: DataType,
+ output_data_type: DataType,
+ config: &DecisionModelConfig,
+ parameter_tree: &ParameterTree<'_, B>,
+ context: &B::Context,
+ ) -> Result<Self, DecisionError<B>> {
+ let model_dim = config.decoder_config.transformer_config.model_dim;
+ let embedding_tree = parameter_tree.subtree("embedding");
+ let input_embedding = EmbeddingTable::load(
+ config.decoder_config.vocab_size,
+ model_dim,
+ input_data_type,
+ &embedding_tree.subtree("input_embedding"),
+ context,
+ )
+ .await?;
+
+ let transformer = Transformer::new(
+ input_data_type,
+ output_data_type,
+ None,
+ &config.decoder_config.transformer_config,
+ &parameter_tree.subtree("transformer"),
+ context,
+ )
+ .await?;
+
+ let readout_tree = embedding_tree.subtree("output_embedding");
+ let readout_spec = readout_tree.metadata("spec")?;
+ let readout = LinearMatmul::load(
+ model_dim,
+ config.decision.codes.len() as u32,
+ output_data_type,
+ output_data_type,
+ output_data_type,
+ readout_spec,
+ None,
+ &readout_tree,
+ None,
+ context,
+ )
+ .await?;
+
+ Ok(Self {
+ input_data_type,
+ input_scale: config.decoder_config.embedding_config.input_scale().unwrap_or(1.0),
+ input_embedding,
+ transformer,
+ readout,
+ })
+ }
+
+ pub fn max_context_length(&self) -> Option<u32> {
+ self.transformer.max_context_length()
+ }
+
+ /// Runs the whole prompt in one stateless pass and returns the `[1, answer_count]` logits of its last token.
+ pub fn encode(
+ &self,
+ token_ids: impl BufferRef<Backend = B>,
+ token_count: u32,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<B::ScratchBuffer, DecisionError<B>> {
+ assert!(token_count > 0, "decision needs at least one token");
+
+ let command_buffer = &mut *command_buffer.debug_group("decision");
+
+ let mut embedded = command_buffer
+ .allocate_scratch_for_shape(&[token_count, self.transformer.model_dim()], self.input_data_type)
+ .map_err(DecisionError::Backend)?;
+ self.input_embedding.encode_lookup(token_ids, &mut embedded, token_count, self.input_scale, command_buffer);
+
+ let nodes = (0..token_count)
+ .map(|index| TrieNode {
+ trie_start: index,
+ trie_end: token_count - 1,
+ height: index,
+ })
+ .collect::<Box<[TrieNode]>>();
+
+ let hidden = self
+ .transformer
+ .encode(
+ embedded,
+ None::<&B::ScratchBuffer>,
+ &BatchTopology::new(&nodes, true),
+ Some((token_count - 1..token_count).into()),
+ None,
+ None,
+ command_buffer,
+ )?
+ .output
+ .expect("transformer returns rows for an output range");
+
+ let logits = self.readout.encode(hidden, 1, command_buffer)?;
+
+ Ok(logits)
+ }
+}
diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rs
index e219213fd..73b7686e0 100644
--- a/crates/uzu-engine/src/encodable_block/decoder.rs
+++ b/crates/uzu-engine/src/encodable_block/decoder.rs
@@ -72,6 +72,7 @@ impl<B: Backend> Decoder<B> {
Normalization::new(
config.transformer_config.model_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
PostLayerScalar::None,
@@ -106,6 +107,7 @@ impl<B: Backend> Decoder<B> {
};
let transformer = Transformer::new(
+ data_type,
data_type,
readout_input_hadamard_factors,
&config.transformer_config,
diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rs
index aa2dc8731..40f8c52ef 100644
--- a/crates/uzu-engine/src/encodable_block/dflash.rs
+++ b/crates/uzu-engine/src/encodable_block/dflash.rs
@@ -111,6 +111,7 @@ impl<B: Backend> DFlash<B> {
let projected_feature_norm = Normalization::new(
config.model_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
PostLayerScalar::None,
@@ -154,6 +155,7 @@ impl<B: Backend> DFlash<B> {
let output_norm = Normalization::new(
config.model_dim,
data_type,
+ data_type,
None,
ShortcutMode::Add,
PostLayerScalar::None,
diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
index aa4b71282..cd4e1d02a 100644
--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs
@@ -3,7 +3,7 @@ use thiserror::Error;
use crate::{
backends::common::{
- Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Context, Kernels,
+ Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Context, GpuBuffer, Kernels,
kernel::{
ConvTreeScanKernel, DeltaNetConvPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,
DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,
@@ -444,89 +444,48 @@ impl<B: Backend> DeltaNet<B> {
});
Ok(output)
}
-}
-
-impl<B: Backend> Mixer<B> for DeltaNet<B> {
- fn speculation_supported(&self) -> bool {
- self.tree_verify.is_some()
- }
-
- fn max_context_length(&self) -> Option<u32> {
- None
- }
- fn create_empty_state(
- &self,
- _max_context_length: Option<u32>,
- context: &B::Context,
- ) -> Result<Box<dyn MixerState<B>>, B::Error> {
- let conv_state =
- context.create_buffer_gpu(size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE))?;
- let ssm_state = context.create_buffer_gpu(size_for_shape(
- &[self.num_heads, self.value_head_dim, self.head_dim],
- INNER_DATA_TYPE,
- ))?;
-
- let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(
- context,
- self.outer_data_type,
- self.head_dim,
- self.num_heads,
- self.num_groups,
- )?;
-
- Ok(Box::new(DeltaNetState {
- conv_state,
- ssm_state,
- suffix_status: None,
- state_advance,
- is_empty: true,
- }))
- }
-
- fn encode(
+ // Runs a fully accepted batch from zero conv and SSM states that live only for this encode.
+ fn encode_stateless(
&self,
hidden: B::ScratchBuffer,
- precalculated_rope: Option<&PrecalculatedRoPE<B>>,
batch_dim: &BatchTopology,
- state: Option<MaybeMut<dyn MixerState<B>>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
- let command_buffer = &mut *command_buffer.debug_group("delta net");
+ assert!(batch_dim.full_accept(), "stateless delta net doesn't support trie");
- assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");
-
- let state = state.expect("delta net requires state");
- let state = state.downcast::<DeltaNetState<B>>().expect("incorrect type of delta net state");
- let MaybeMut::Mut(state) = state else {
- panic!("delta net doesn't support immutable state");
- };
-
- assert!(state.suffix_status.is_none(), "delta net called with state with an unaccepted suffix");
-
- if state.is_empty {
- command_buffer.encode_fill(&mut state.conv_state, 0);
- command_buffer.encode_fill(&mut state.ssm_state, 0);
- state.is_empty = false;
- }
+ let mut conv_state = command_buffer
+ .allocate_scratch_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut conv_state, 0);
- let mut in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;
+ let mut ssm_state = command_buffer
+ .allocate_scratch_for_shape(&[self.num_heads, self.value_head_dim, self.head_dim], INNER_DATA_TYPE)
+ .map_err(MixerEncodeError::Backend)?;
+ command_buffer.encode_fill(&mut ssm_state, 0);
- if !batch_dim.full_accept() {
- let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;
+ let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;
- return Ok(output);
- }
+ self.encode_flat(in_projected, batch_dim.size(), &mut conv_state, &mut ssm_state, command_buffer)
+ }
+ fn encode_flat(
+ &self,
+ mut in_projected: B::ScratchBuffer,
+ suffix_length: u32,
+ conv_state: &mut dyn GpuBuffer<Backend = B>,
+ ssm_state: &mut dyn GpuBuffer<Backend = B>,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
let mut delta_output = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)
+ .allocate_scratch_for_shape(&[suffix_length, self.value_dim], self.outer_data_type)
.map_err(MixerEncodeError::Backend)?;
- if batch_dim.size() == 1 {
+ if suffix_length == 1 {
self.conv_update.encode(
&self.conv_weight,
self.conv_bias.as_ref(),
&mut in_projected,
- &mut state.conv_state,
+ &mut *conv_state,
self.kernel_size,
self.conv_dim,
self.kernel_size - 1,
@@ -538,7 +497,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
&self.a_log,
&self.dt_bias,
&self.norm_weight,
- &mut state.ssm_state,
+ &mut *ssm_state,
&mut delta_output,
self.num_heads,
self.num_groups,
@@ -549,16 +508,16 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
command_buffer,
);
} else {
- let conv_tile_tokens = batch_dim.size().div_ceil(CONV_TILES).max(CONV_MIN_TILE_TOKENS);
+ let conv_tile_tokens = suffix_length.div_ceil(CONV_TILES).max(CONV_MIN_TILE_TOKENS);
debug_assert!(conv_tile_tokens >= self.kernel_size - 1);
let mut packed = command_buffer
.allocate_scratch_for_shape(&[CONV_TILES, self.kernel_size - 1, self.conv_dim], INNER_DATA_TYPE)
.map_err(MixerEncodeError::Backend)?;
self.conv_pack.encode(
- &state.conv_state,
+ &*conv_state,
&in_projected,
&mut packed,
- batch_dim.size(),
+ suffix_length,
conv_tile_tokens,
self.total_proj_dim,
self.conv_dim,
@@ -569,44 +528,44 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
&self.conv_weight,
self.conv_bias.as_ref(),
&mut in_projected,
- &mut state.conv_state,
- batch_dim.size(),
+ &mut *conv_state,
+ suffix_length,
conv_tile_tokens,
self.conv_dim,
self.total_proj_dim,
command_buffer,
);
- if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {
+ if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(suffix_length)) {
chunked
.encode(
DeltaNetChunkedPrefillArgs {
in_projected: &in_projected,
a_log: &self.a_log,
dt_bias: &self.dt_bias,
- ssm_state: &mut state.ssm_state,
+ ssm_state,
delta_output: &mut delta_output,
num_heads: self.num_heads,
num_groups: self.num_groups,
value_head_dim: self.value_head_dim,
key_dim: self.key_dim,
value_dim: self.value_dim,
- suffix_len: batch_dim.size(),
+ suffix_len: suffix_length,
},
command_buffer,
)
.map_err(MixerEncodeError::Backend)?;
} else {
let mut prep_q_norm = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)
+ .allocate_scratch_for_shape(&[suffix_length, self.key_dim], INNER_DATA_TYPE)
.map_err(MixerEncodeError::Backend)?;
let mut prep_k_norm = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)
+ .allocate_scratch_for_shape(&[suffix_length, self.key_dim], INNER_DATA_TYPE)
.map_err(MixerEncodeError::Backend)?;
let mut prep_beta = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)
+ .allocate_scratch_for_shape(&[suffix_length, self.num_heads], INNER_DATA_TYPE)
.map_err(MixerEncodeError::Backend)?;
let mut prep_decay = command_buffer
- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)
+ .allocate_scratch_for_shape(&[suffix_length, self.num_heads], INNER_DATA_TYPE)
.map_err(MixerEncodeError::Backend)?;
self.delta_net_prefill_prep.encode(
&in_projected,
@@ -621,7 +580,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
self.num_groups,
self.key_dim,
self.value_dim,
- batch_dim.size(),
+ suffix_length,
command_buffer,
);
self.delta_net_prefill.encode(
@@ -630,14 +589,14 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
&prep_beta,
&prep_decay,
&in_projected,
- &mut state.ssm_state,
+ &mut *ssm_state,
&mut delta_output,
self.num_heads,
self.num_groups,
self.value_head_dim,
self.key_dim,
self.value_dim,
- batch_dim.size(),
+ suffix_length,
self.value_head_dim.div_ceil(16),
command_buffer,
);
@@ -652,17 +611,100 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {
self.conv_dim,
self.total_proj_dim,
self.norm_epsilon,
- batch_dim.size(),
+ suffix_length,
command_buffer,
);
}
+ Ok(self.out_projection.encode(delta_output, suffix_length, command_buffer)?)
+ }
+}
+
+impl<B: Backend> Mixer<B> for DeltaNet<B> {
+ fn speculation_supported(&self) -> bool {
+ self.tree_verify.is_some()
+ }
+
+ fn max_context_length(&self) -> Option<u32> {
+ None
+ }
+
+ fn create_empty_state(
+ &self,
+ _max_context_length: Option<u32>,
+ context: &B::Context,
+ ) -> Result<Box<dyn MixerState<B>>, B::Error> {
+ let conv_state =
+ context.create_buffer_gpu(size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE))?;
+ let ssm_state = context.create_buffer_gpu(size_for_shape(
+ &[self.num_heads, self.value_head_dim, self.head_dim],
+ INNER_DATA_TYPE,
+ ))?;
+
+ let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(
+ context,
+ self.outer_data_type,
+ self.head_dim,
+ self.num_heads,
+ self.num_groups,
+ )?;
+
+ Ok(Box::new(DeltaNetState {
+ conv_state,
+ ssm_state,
+ suffix_status: None,
+ state_advance,
+ is_empty: true,
+ }))
+ }
+
+ fn encode(
+ &self,
+ hidden: B::ScratchBuffer,
+ precalculated_rope: Option<&PrecalculatedRoPE<B>>,
+ batch_dim: &BatchTopology,
+ state: Option<MaybeMut<dyn MixerState<B>>>,
+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {
+ assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");
+
+ let command_buffer = &mut *command_buffer.debug_group("delta net");
+ let Some(state) = state else {
+ return self.encode_stateless(hidden, batch_dim, command_buffer);
+ };
+
+ let state = state.downcast::<DeltaNetState<B>>().expect("incorrect type of delta net state");
+ let MaybeMut::Mut(state) = state else {
+ panic!("delta net doesn't support immutable state");
+ };
+
+ assert!(state.suffix_status.is_none(), "delta net called with state with an unaccepted suffix");
+
+ if state.is_empty {
+ command_buffer.encode_fill(&mut state.conv_state, 0);
+ command_buffer.encode_fill(&mut state.ssm_state, 0);
+ state.is_empty = false;
+ }
+
+ let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;
+
+ if !batch_dim.full_accept() {
+ let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;
+ return Ok(output);
+ }
+
+ let output = self.encode_flat(
+ in_projected,
+ batch_dim.size(),
+ &mut state.conv_state,
+ &mut state.ssm_state,
+ command_buffer,
+ )?;
+
state.suffix_status = Some(DeltaNetSuffixStatus::Flat {
suffix_length: batch_dim.size(),
});
- let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;
-
Ok(output)
}
}
diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rs
index 9f6c79366..f7d7aaf35 100644
--- a/crates/uzu-engine/src/encodable_block/mod.rs
+++ b/crates/uzu-engine/src/encodable_block/mod.rs
@@ -1,6 +1,7 @@
pub mod batch_topology;
pub mod classifier;
pub mod convolution;
+pub mod decision;
pub mod decoder;
pub mod dflash;
pub mod embedding;
diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rs
index ffbe3d317..cf30952e0 100644
--- a/crates/uzu-engine/src/encodable_block/normalization.rs
+++ b/crates/uzu-engine/src/encodable_block/normalization.rs
@@ -44,14 +44,16 @@ pub struct Normalization<B: Backend> {
element_count: u32,
hadamard_factors: Option<B::GlobalGpuBuffer>,
post_layer_scalar_value: f32,
- data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
kernel: <B::Kernels as Kernels>::NormalizationKernel,
}
impl<B: Backend> Normalization<B> {
pub async fn new(
element_count: u32,
- data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
hadamard_factors: Option<B::GlobalGpuBuffer>,
shortcut_mode: ShortcutMode,
post_layer_scalar: PostLayerScalar,
@@ -84,9 +86,9 @@ impl<B: Backend> Normalization<B> {
let kernel = <B::Kernels as Kernels>::NormalizationKernel::new(
context,
- data_type,
+ input_data_type,
DataType::F32,
- data_type,
+ output_data_type,
DataType::F32,
false,
config.subtract_mean,
@@ -109,7 +111,8 @@ impl<B: Backend> Normalization<B> {
element_count,
hadamard_factors,
post_layer_scalar_value,
- data_type,
+ input_data_type,
+ output_data_type,
kernel,
})
}
@@ -122,10 +125,11 @@ impl<B: Backend> Normalization<B> {
shortcut: Option<impl BufferMut<Backend = B>>,
command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,
) -> Result<B::ScratchBuffer, B::Error> {
- let row_size = size_for_shape(&[self.element_count], self.data_type);
+ let row_size = size_for_shape(&[self.element_count], self.input_data_type);
let row_offset_bytes = row_offset as usize * row_size;
let shortcut = shortcut.map(|shortcut| shortcut.subrange_mut(row_offset_bytes..));
- let mut output = command_buffer.allocate_scratch_for_shape(&[row_count, self.element_count], self.data_type)?;
+ let mut output =
+ command_buffer.allocate_scratch_for_shape(&[row_count, self.element_count], self.output_data_type)?;
self.kernel.encode(
Some(input.subrange(row_offset_bytes..)),
self.scales.as_ref(),
diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
index 2826f5e98..b8f8afd99 100644
--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs
@@ -83,6 +83,7 @@ impl<B: Backend> PerLayerEmbedding<B> {
let projection_norm = Normalization::new(
config.ple_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
PostLayerScalar::ScaleOutput(config.input_scale),
@@ -200,6 +201,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {
let norm = Normalization::new(
model_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
PostLayerScalar::None,
diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rs
index e7d644276..3b0c8780d 100644
--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs
+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs
@@ -63,6 +63,7 @@ impl<B: Backend> PredictionHead<B> {
let normalization = Normalization::new(
hidden_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
PostLayerScalar::None,
diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rs
index 745205db7..6d93bc62c 100644
--- a/crates/uzu-engine/src/encodable_block/transformer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer.rs
@@ -108,7 +108,8 @@ pub struct Transformer<B: Backend> {
impl<B: Backend> Transformer<B> {
pub async fn new(
- data_type: DataType,
+ input_data_type: DataType,
+ output_data_type: DataType,
output_norm_hadamard_factors: Option<B::GlobalGpuBuffer>,
transformer_config: &TransformerConfig,
parameter_tree: &ParameterTree<'_, B>,
@@ -133,7 +134,7 @@ impl<B: Backend> Transformer<B> {
transformer_config.hidden_dim,
num_layers,
layer_index,
- data_type,
+ input_data_type,
layer_config,
&layer_loader,
context,
@@ -145,7 +146,8 @@ impl<B: Backend> Transformer<B> {
let output_norm = Normalization::new(
transformer_config.model_dim,
- data_type,
+ input_data_type,
+ output_data_type,
output_norm_hadamard_factors,
ShortcutMode::Add,
PostLayerScalar::None,
@@ -155,7 +157,7 @@ impl<B: Backend> Transformer<B> {
)
.await?;
- let residual_add = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, data_type, false)
+ let residual_add = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, input_data_type, false)
.map_err(TransformerNewError::Backend)?;
Ok(Self {
@@ -180,6 +182,10 @@ impl<B: Backend> Transformer<B> {
Ok(output)
}
+ pub fn model_dim(&self) -> u32 {
+ self.model_dim
+ }
+
pub fn speculation_supported(&self) -> bool {
self.layers.iter().all(|(layer, _rope)| layer.mixer.speculation_supported())
}
diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
index 1093bc694..002817d64 100644
--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs
@@ -236,6 +236,7 @@ impl<B: Backend> TransformerLayer<B> {
Normalization::new(
model_dim,
data_type,
+ data_type,
mixer_hadamard_factors,
if layer_index > 0 {
ShortcutMode::Add
@@ -261,6 +262,7 @@ impl<B: Backend> TransformerLayer<B> {
Normalization::new(
model_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
PostLayerScalar::None,
@@ -305,6 +307,7 @@ impl<B: Backend> TransformerLayer<B> {
let pre_mlp_norm = Normalization::new(
model_dim,
data_type,
+ data_type,
mlp_input_hadamard_factors,
ShortcutMode::Add,
residual_sum_scalar,
@@ -319,6 +322,7 @@ impl<B: Backend> TransformerLayer<B> {
Normalization::new(
model_dim,
data_type,
+ data_type,
None,
ShortcutMode::None,
output_scalar,
diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rs
index 694670440..0968fd8a6 100644
--- a/crates/uzu-engine/src/encodable_block/weaver.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver.rs
@@ -216,6 +216,7 @@ impl<B: Backend> Weaver<B> {
let token_embedding_norm = Normalization::new(
config.target_embedding_dim,
DATA_TYPE,
+ DATA_TYPE,
None,
ShortcutMode::None,
PostLayerScalar::None,
@@ -227,6 +228,7 @@ impl<B: Backend> Weaver<B> {
let hidden_state_norm = Normalization::new(
config.target_model_dim,
DATA_TYPE,
+ DATA_TYPE,
None,
ShortcutMode::None,
PostLayerScalar::None,
@@ -254,6 +256,7 @@ impl<B: Backend> Weaver<B> {
let readout_norm = Normalization::new(
config.model_dim,
DATA_TYPE,
+ DATA_TYPE,
None,
ShortcutMode::Add,
PostLayerScalar::None,
diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
index 8f639d48d..d05900b46 100644
--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs
+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs
@@ -98,6 +98,7 @@ impl<B: Backend> WeaverLayer<B> {
let pre_attention_norm = Normalization::new(
model_dim,
DATA_TYPE,
+ DATA_TYPE,
None,
if add_to_residual {
ShortcutMode::Add
@@ -113,6 +114,7 @@ impl<B: Backend> WeaverLayer<B> {
let pre_mlp_norm = Normalization::new(
model_dim,
DATA_TYPE,
+ DATA_TYPE,
None,
ShortcutMode::Add,
PostLayerScalar::None,
diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rs
index b606b444a..4429d8865 100644
--- a/crates/uzu-engine/src/engine/classifier_model.rs
+++ b/crates/uzu-engine/src/engine/classifier_model.rs
@@ -2,10 +2,7 @@ use std::{cmp::min, collections::HashMap, io, path::Path, sync::Arc};
use derive_more::Debug;
use half::bf16;
-use shoji::{
- traits::backend::classification::ClassifierOutput,
- types::session::classification::{ChatTokenCodecConfig, TokenCodecConfig},
-};
+use shoji::{traits::backend::classification::ClassifierOutput, types::session::classification::TokenCodecConfig};
use thiserror::Error;
use tokenizers::Tokenizer;
@@ -104,20 +101,7 @@ impl<B: Backend> ClassifierModel<B> {
}
pub fn token_codec_config(&self) -> TokenCodecConfig {
- match &self.token_codec_config {
- AnyTokenCodecConfig::ChatCodecConfig(config) => TokenCodecConfig::Chat(ChatTokenCodecConfig {
- prompt_template: config.prompt_template.clone(),
- output_parser_regex: config.output_parser_regex.clone(),
- system_role_name: config.system_role_name.clone(),
- user_role_name: config.user_role_name.clone(),
- assistant_role_name: config.assistant_role_name.clone(),
- eos_token: config.eos_token.clone(),
- bos_token: config.bos_token.clone(),
- end_of_thinking_tag: config.end_of_thinking_tag.clone(),
- default_system_prompt: config.default_system_prompt.clone(),
- }),
- AnyTokenCodecConfig::RawTextCodecConfig(_) => TokenCodecConfig::RawText,
- }
+ TokenCodecConfig::from(&self.token_codec_config)
}
pub async fn classify(
diff --git a/crates/uzu-engine/src/engine/decision_model.rs b/crates/uzu-engine/src/engine/decision_model.rs
new file mode 100644
index 000000000..ccc91d362
--- /dev/null
+++ b/crates/uzu-engine/src/engine/decision_model.rs
@@ -0,0 +1,153 @@
+use std::{io, path::Path, sync::Arc};
+
+use shoji::types::session::classification::TokenCodecConfig;
+use thiserror::Error;
+use tokenizers::Tokenizer;
+
+use crate::{
+ backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},
+ config::{
+ model::decision_model::{DecisionConfig, DecisionModelConfig},
+ token_codec::AnyTokenCodecConfig,
+ },
+ data_type::DataType,
+ encodable_block::decision::{Decision, DecisionError},
+ engine::Engine,
+ safetensors::{ParameterLoader, ParameterLoaderError},
+ utils::file,
+};
+
+#[derive(Debug, Error)]
+pub enum DecisionModelError<B: Backend> {
+ #[error("Backend error: {0}")]
+ Backend(#[source] B::Error),
+ #[error("Input larger than model context size")]
+ ContextOverflow,
+ #[error("Decision error: {0}")]
+ Decision(#[from] DecisionError<B>),
+ #[error("Input is empty")]
+ EmptyInput,
+ #[error("Option count {option_count} is outside 1..={answer_count}")]
+ InvalidOptionCount {
+ option_count: usize,
+ answer_count: usize,
+ },
+ #[error("I/O error: {0}")]
+ IO(#[from] io::Error),
+ #[error("ParameterLoader error: {0}")]
+ ParameterLoader(#[from] ParameterLoaderError<B>),
+ #[error("Serde error: {0}")]
+ Serde(#[from] serde_json::Error),
+ #[error("Tokenizer error: {0}")]
+ Tokenizer(#[from] tokenizers::Error),
+}
+
+pub struct DecisionModel<B: Backend> {
+ context: Arc<B::Context>,
+ decision: Decision<B>,
+ decision_config: DecisionConfig,
+ tokenizer: Arc<Tokenizer>,
+ token_codec_config: AnyTokenCodecConfig,
+}
+
+impl<B: Backend> DecisionModel<B> {
+ pub fn decision_config(&self) -> &DecisionConfig {
+ &self.decision_config
+ }
+
+ pub fn tokenizer(&self) -> &Arc<Tokenizer> {
+ &self.tokenizer
+ }
+
+ pub fn token_codec_config(&self) -> TokenCodecConfig {
+ TokenCodecConfig::from(&self.token_codec_config)
+ }
+
+ pub async fn probabilities(
+ &self,
+ input: &[u64],
+ option_count: usize,
+ ) -> Result<Vec<f32>, DecisionModelError<B>> {
+ if input.is_empty() {
+ return Err(DecisionModelError::EmptyInput);
+ }
+
+ if self
+ .decision
+ .max_context_length()
+ .is_some_and(|max_context_length| input.len() > max_context_length as usize)
+ {
+ return Err(DecisionModelError::ContextOverflow);
+ }
+
+ let answer_count = self.decision_config.codes.len();
+ if !(1..=answer_count).contains(&option_count) {
+ return Err(DecisionModelError::InvalidOptionCount {
+ option_count,
+ answer_count,
+ });
+ }
+
+ let mut command_buffer = self.context.create_command_buffer(None, None).map_err(DecisionModelError::Backend)?;
+
+ let token_ids = command_buffer
+ .allocate_constant_from_slice(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>())
+ .map_err(DecisionModelError::Backend)?;
+ let logits = self.decision.encode(&token_ids, input.len() as u32, &mut command_buffer)?;
+
+ let mut output_buffer = self
+ .context
+ .create_buffer_gpu(answer_count * DataType::F32.size_in_bytes())
+ .map_err(DecisionModelError::Backend)?;
+ command_buffer.encode_copy(&logits, &mut output_buffer);
+ drop(logits);
+ drop(token_ids);
+
+ command_buffer.end_encoding().submit().wait_until_completed().await.map_err(DecisionModelError::Backend)?;
+
+ let temperature = self.decision_config.temperature;
+ let logits = self.context.copyout::<f32>(&output_buffer).await.map_err(DecisionModelError::Backend)?;
+ let scaled = logits[..option_count].iter().map(|logit| logit / temperature).collect::<Vec<_>>();
+ let max = scaled.iter().copied().fold(f32::NEG_INFINITY, f32::max);
+ let exps = scaled.iter().map(|logit| (logit - max).exp()).collect::<Vec<_>>();
+ let sum = exps.iter().sum::<f32>();
+ let result = exps.into_iter().map(|value| value / sum).collect();
+ Ok(result)
+ }
+}
+
+impl<B: Backend> Engine<B> {
+ pub async fn load_decision_model(
+ &self,
+ model_path: &Path,
+ ) -> Result<DecisionModel<B>, DecisionModelError<B>> {
+ let context = self.context.clone();
+
+ let config_file_path = model_path.join("config.json");
+ let config_file_content = file::read(config_file_path).await?;
+ let config = serde_json::from_slice::<DecisionModelConfig>(&config_file_content)?;
+
+ let tokenizer_file_path = model_path.join("tokenizer.json");
+ let tokenizer_file_content = file::read(tokenizer_file_path).await?;
+ let tokenizer = Tokenizer::from_bytes(tokenizer_file_content)?;
+
+ let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), context.as_ref()).await?;
+ let decision = Decision::new(
+ DataType::BF16,
+ DataType::F32,
+ &config,
+ &weight_loader.tree().subtree("decoder"),
+ context.as_ref(),
+ )
+ .await?;
+ weight_loader.assert_all_tensors_read()?;
+
+ Ok(DecisionModel {
+ context,
+ decision,
+ decision_config: config.decision,
+ tokenizer: Arc::new(tokenizer),
+ token_codec_config: config.token_codec_config,
+ })
+ }
+}
diff --git a/crates/uzu-engine/src/engine/mod.rs b/crates/uzu-engine/src/engine/mod.rs
index 95781931d..e9593328e 100644
--- a/crates/uzu-engine/src/engine/mod.rs
+++ b/crates/uzu-engine/src/engine/mod.rs
@@ -13,6 +13,7 @@ use crate::{
};
pub mod classifier_model;
+pub mod decision_model;
pub mod language_model;
pub struct Engine<B: Backend> {
@@ -48,6 +49,7 @@ impl<B: Backend> Engine<B> {
pub enum ModelType {
LanguageModel,
Classifier,
+ Decision,
}
#[derive(Debug, Error)]
@@ -62,6 +64,7 @@ pub async fn resolve_model_type(model_path: &Path) -> Result<ModelType, ResolveM
Ok(match read_model_config(model_path).await? {
AnyModelConfig::LanguageModelConfig(_) => ModelType::LanguageModel,
AnyModelConfig::ClassifierModelConfig(_) => ModelType::Classifier,
+ AnyModelConfig::DecisionModelConfig(_) => ModelType::Decision,
})
}
@@ -69,6 +72,7 @@ pub async fn resolve_model_sampling_method(model_path: &Path) -> Result<Option<S
Ok(match read_model_config(model_path).await? {
AnyModelConfig::LanguageModelConfig(config) => Some(config.generation_config.default_sampling_method()),
AnyModelConfig::ClassifierModelConfig(_) => None,
+ AnyModelConfig::DecisionModelConfig(config) => Some(config.generation_config.default_sampling_method()),
})
}
@@ -76,6 +80,7 @@ pub async fn resolve_model_context_length(model_path: &Path) -> Result<Option<u3
let transformer = match read_model_config(model_path).await? {
AnyModelConfig::LanguageModelConfig(config) => config.decoder_config.transformer_config,
AnyModelConfig::ClassifierModelConfig(_) => return Ok(None),
+ AnyModelConfig::DecisionModelConfig(config) => config.decoder_config.transformer_config,
};
Ok(transformer
.layer_configs
diff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
index 711d76e2e..9da7c011e 100644
--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs
@@ -182,6 +182,22 @@ fn group_major_gemv_bf16(
assert_eq_float(&reference, &actual, 0.05, &format!("GroupOutput GEMV W{bits} {method:?}"));
}
+#[rstest]
+#[test_attr(uzu_test)]
+#[case::m1(1, 128, 64)]
+#[case::batched(4, 128, 64)]
+#[case::max_batch(8, 128, 64)]
+#[case::unaligned_k(1, 33, 64)]
+#[case::unaligned_n(1, 128, 11)]
+#[case::large(1, 4096, 2048)]
+fn gemv_f32(
+ #[case] m: usize,
+ #[case] k: usize,
+ #[case] n: usize,
+) {
+ test::<f32>(m, k, n, 0.01);
+}
+
fn assert_gather<T: DataTypeRepr + Float + Debug + Display>(
dense: &[T],
gather: &[T],
@@ -264,9 +280,10 @@ fn quant_gather_case(
#[uzu_test]
fn gemv_gather() {
- // Full-precision BF16 GEMV.
+ // Full-precision GEMV.
for soft_cap in [None, Some(15.0)] {
fp_gather_case::<bf16>(soft_cap, 0.1);
+ fp_gather_case::<f32>(soft_cap, 0.01);
}
// Quantized (bf16, per bits/method) — inline, since it isn't type-generic.
for (bits, method, signed_codes) in [
diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
index f8b5e6c3f..e53936048 100644
--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs
@@ -195,6 +195,7 @@ fn gemv_gemm_route_boundaries_are_preserved() {
for (shape, data_type, prefer_gemm) in [
(shape(4, 4096, 8192), DataType::BF16, true),
(shape(4, 8192, 4096), DataType::BF16, false),
+ (shape(4, 4096, 8192), DataType::F32, false),
(shape(3, 4096, 8192), DataType::BF16, false),
(shape(5, 4096, 8192), DataType::BF16, false),
] {