A high-performance inference engine for AI models
Add decision models support into uzu-engine (#1031)
Aleksandr Golokozd1b906462267
diff --git a/crates/legacy/shoji/src/types/model/specialization.rs b/crates/legacy/shoji/src/types/model/specialization.rsindex 0c6fa4d81..376cf001c 100644--- a/crates/legacy/shoji/src/types/model/specialization.rs+++ b/crates/legacy/shoji/src/types/model/specialization.rs@@ -6,6 +6,7 @@ use serde::{Deserialize, Serialize};pub enum ModelSpecialization {Chat {},Classification {},+ Decision {},TextToSpeech {},Translation {},Speculation {},@@ -16,6 +17,7 @@ impl ModelSpecialization {match self {ModelSpecialization::Chat {} => "Chat".to_string(),ModelSpecialization::Classification {} => "Classification".to_string(),+ ModelSpecialization::Decision {} => "Decision".to_string(),ModelSpecialization::TextToSpeech {} => "Text to Speech".to_string(),ModelSpecialization::Translation {} => "Translation".to_string(),ModelSpecialization::Speculation {} => "Speculation".to_string(),diff --git a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi b/crates/legacy/uzu/bindings/python/uzu/__init__.pyiindex 9fe887149..02ea555ef 100644--- a/crates/legacy/uzu/bindings/python/uzu/__init__.pyi+++ b/crates/legacy/uzu/bindings/python/uzu/__init__.pyi@@ -1058,6 +1058,11 @@ class ModelSpecialization:__match_args__ = ()def __new__(cls) -> ModelSpecialization.Classification: ...+ @typing.final+ class Decision(ModelSpecialization):+ __match_args__ = ()+ def __new__(cls) -> ModelSpecialization.Decision: ...+@typing.finalclass TextToSpeech(ModelSpecialization):__match_args__ = ()diff --git a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swiftindex 6cec313b8..c3f5c6afd 100644--- a/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift+++ b/crates/legacy/uzu/bindings/swift/Sources/Uzu/Generated/uzu/shoji.swift@@ -4553,6 +4553,7 @@ public enum ModelSpecialization: Equatable, Hashable, Codable {case chatcase classification+ case decisioncase textToSpeechcase translationcase speculation@@ -4581,11 +4582,13 @@ public struct FfiConverterTypeModelSpecialization: FfiConverterRustBuffer {case 2: return .classification- case 3: return .textToSpeech+ case 3: return .decision- case 4: return .translation+ case 4: return .textToSpeech- case 5: return .speculation+ case 5: return .translation++ case 6: return .speculationdefault: throw UniffiInternalError.unexpectedEnumCase}@@ -4603,17 +4606,21 @@ public struct FfiConverterTypeModelSpecialization: FfiConverterRustBuffer {writeInt(&buf, Int32(2))- case .textToSpeech:+ case .decision:writeInt(&buf, Int32(3))- case .translation:+ case .textToSpeech:writeInt(&buf, Int32(4))- case .speculation:+ case .translation:writeInt(&buf, Int32(5))++ case .speculation:+ writeInt(&buf, Int32(6))+}}}diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mtsindex 5b8cc1e6b..b615554b0 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.mts@@ -641,6 +641,10 @@ export declare class ModelSpecializationClassification {constructor()}+export declare class ModelSpecializationDecision {+ constructor()+}+export declare class ModelSpecializationSpeculation {constructor()}@@ -871,7 +875,7 @@ export type ModelSource =ModelSourceRegistry | ModelSourceFilesystemexport type ModelSpecialization =- ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation+ ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationDecision | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculationexport type PcmBatchError =| { type: 'UnableToSaveAsWav'; message: string }diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.tsindex 5b8cc1e6b..b615554b0 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.d.ts@@ -641,6 +641,10 @@ export declare class ModelSpecializationClassification {constructor()}+export declare class ModelSpecializationDecision {+ constructor()+}+export declare class ModelSpecializationSpeculation {constructor()}@@ -871,7 +875,7 @@ export type ModelSource =ModelSourceRegistry | ModelSourceFilesystemexport type ModelSpecialization =- ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculation+ ModelSpecializationChat | ModelSpecializationClassification | ModelSpecializationDecision | ModelSpecializationTextToSpeech | ModelSpecializationTranslation | ModelSpecializationSpeculationexport type PcmBatchError =| { type: 'UnableToSaveAsWav'; message: string }diff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.js b/crates/legacy/uzu/bindings/typescript/src/napi/index.jsindex e6e19ce48..f4f8fda31 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.js+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.js@@ -596,6 +596,7 @@ module.exports.ModelSourceFilesystem = nativeBinding.ModelSourceFilesystemmodule.exports.ModelSourceRegistry = nativeBinding.ModelSourceRegistrymodule.exports.ModelSpecializationChat = nativeBinding.ModelSpecializationChatmodule.exports.ModelSpecializationClassification = nativeBinding.ModelSpecializationClassification+module.exports.ModelSpecializationDecision = nativeBinding.ModelSpecializationDecisionmodule.exports.ModelSpecializationSpeculation = nativeBinding.ModelSpecializationSpeculationmodule.exports.ModelSpecializationTextToSpeech = nativeBinding.ModelSpecializationTextToSpeechmodule.exports.ModelSpecializationTranslation = nativeBinding.ModelSpecializationTranslationdiff --git a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjsindex 30424c3e3..2a451d989 100644--- a/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs+++ b/crates/legacy/uzu/bindings/typescript/src/napi/index.mjs@@ -93,6 +93,7 @@ export const ModelSourceFilesystem = cjs.ModelSourceFilesystem;export const ModelSourceRegistry = cjs.ModelSourceRegistry;export const ModelSpecializationChat = cjs.ModelSpecializationChat;export const ModelSpecializationClassification = cjs.ModelSpecializationClassification;+export const ModelSpecializationDecision = cjs.ModelSpecializationDecision;export const ModelSpecializationSpeculation = cjs.ModelSpecializationSpeculation;export const ModelSpecializationTextToSpeech = cjs.ModelSpecializationTextToSpeech;export const ModelSpecializationTranslation = cjs.ModelSpecializationTranslation;diff --git a/crates/legacy/uzu/src/registry/local/mod.rs b/crates/legacy/uzu/src/registry/local/mod.rsindex 42757054c..def1e1d52 100644--- a/crates/legacy/uzu/src/registry/local/mod.rs+++ b/crates/legacy/uzu/src/registry/local/mod.rs@@ -100,6 +100,7 @@ impl Registry {.map(|model_type| match model_type {ModelType::LanguageModel => ModelSpecialization::Chat {},ModelType::Classifier => ModelSpecialization::Classification {},+ ModelType::Decision => ModelSpecialization::Decision {},}).map_err(|error| RegistryError::UnableToGetModels {message: format!("Unable to resolve specialization for {}: {error}", path.display()),diff --git a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rsindex 033d4b994..e59d06470 100644--- a/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs+++ b/crates/uzu-engine/src/backends/common/kernel/delta_net_chunked_prefill.rs@@ -1,7 +1,7 @@#[cfg(any(backend = "webgpu", backend = "cpu"))]use crate::backends::common::kernel::Unsupported;use crate::{- backends::common::{Backend, CommandBuffer, Kernels},+ backends::common::{Backend, CommandBuffer, GpuBuffer, Kernels},data_type::DataType,};@@ -9,7 +9,7 @@ pub struct DeltaNetChunkedPrefillArgs<'a, B: Backend> {pub in_projected: &'a B::ScratchBuffer,pub a_log: &'a B::GlobalGpuBuffer,pub dt_bias: &'a B::GlobalGpuBuffer,- pub ssm_state: &'a mut B::GlobalGpuBuffer,+ pub ssm_state: &'a mut dyn GpuBuffer<Backend = B>,pub delta_output: &'a mut B::ScratchBuffer,pub num_heads: u32,pub num_groups: u32,diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rsindex a8f86bae1..94debb345 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/kernel.rs@@ -15,7 +15,7 @@ use crate::{context::CpuContext,kernel::matmul::{MatmulCpuError,- reference::{WeightData, read_bf16, write_bf16},+ reference::{WeightData, load, store},},},},@@ -24,6 +24,8 @@ use crate::{};pub struct MatmulCpuKernel {+ weights_data_type: DataType,+ input_data_type: DataType,output_data_type: DataType,output_rht: ActivationTransform<Cpu>,bias_add: <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel,@@ -40,22 +42,21 @@ impl MatmulKernel for MatmulCpuKernel {input_data_type: DataType,output_data_type: DataType,) -> Result<Self, Self::Error> {- for data_type in [weights_data_type, input_data_type] {- if data_type != DataType::BF16 {+ for data_type in [weights_data_type, input_data_type, output_data_type] {+ if !matches!(data_type, DataType::BF16 | DataType::F32) {return Err(MatmulCpuError::UnsupportedDataType(data_type));}}- if !matches!(output_data_type, DataType::BF16 | DataType::F32) {- return Err(MatmulCpuError::UnsupportedDataType(output_data_type));- }let output_rht = ActivationTransform::output_rht(context, output_data_type, None, true)?;let bias_add = <<Cpu as Backend>::Kernels as Kernels>::TensorAddBiasKernel::new(context,output_data_type,- DataType::BF16,+ weights_data_type,true,)?;Ok(Self {+ weights_data_type,+ input_data_type,output_data_type,output_rht,bias_add,@@ -112,7 +113,7 @@ impl MatmulKernel for MatmulCpuKernel {offset,} => {let (values, range) = values.parts();- let byte_offset = range.start + offset * DataType::BF16.size_in_bytes();+ let byte_offset = range.start + offset * self.input_data_type.size_in_bytes();AData::FullPrecision(SendPtr(values.cpu_address().as_ptr().cast::<u8>().cast_const().wrapping_byte_add(byte_offset),))@@ -166,6 +167,8 @@ impl MatmulKernel for MatmulCpuKernel {let weight_data = WeightData::from_b(b, b_leading_dimension, b_transpose, k_u, n_u)?;+ let weights_data_type = self.weights_data_type;+ let input_data_type = self.input_data_type;let output_data_type = self.output_data_type;let bias_after_rht = post_rht.is_some();command_buffer.push_command(move || {@@ -180,7 +183,7 @@ impl MatmulKernel for MatmulCpuKernel {let mut accumulator = 0.0f32;for inner in 0..k_u {let a_value = match a_data {- AData::FullPrecision(ptr) => read_bf16(ptr.as_ptr(), row * k_u + inner),+ AData::FullPrecision(ptr) => load(ptr.as_ptr(), input_data_type, row * k_u + inner),AData::Int8 {values,scales,@@ -206,7 +209,7 @@ impl MatmulKernel for MatmulCpuKernel {} else {inner * leading_dimension + b_col};- read_bf16(ptr.as_ptr(), index)+ load(ptr.as_ptr(), weights_data_type, index)},WeightData::Quantized {weights,@@ -234,7 +237,7 @@ impl MatmulKernel for MatmulCpuKernel {let group_index = inner / group_size;let metadata_index = b_col * scale_strides.output_stride as usize+ group_index * scale_strides.group_stride as usize;- let scale = read_bf16(scales.as_ptr(), metadata_index);+ let scale = load(scales.as_ptr(), weights_data_type, metadata_index);let midpoint = (1u32 << (bits - 1)) as f32;let zero_point = zero_points.as_ref().map(|(zp, strides)| {let zero_point_index = b_col * strides.output_stride as usize@@ -254,7 +257,7 @@ impl MatmulKernel for MatmulCpuKernel {let bias_term = if let Some(zp) = zero_point {-scale * zp} else if let Some(b) = biases {- read_bf16(b.as_ptr(), metadata_index)+ load(b.as_ptr(), weights_data_type, metadata_index)} else {-scale * midpoint};@@ -267,16 +270,12 @@ impl MatmulKernel for MatmulCpuKernel {let output_index = row * n_u + col;let mut value = output_scale * accumulator;if !bias_after_rht && let Some(bias) = bias_ptr {- value += read_bf16(bias.as_ptr(), col);+ value += load(bias.as_ptr(), weights_data_type, col);}if let Some(cap) = soft_cap {value = cap * (value / cap).tanh();}- if output_data_type == DataType::F32 {- *d_ptr.as_ptr().cast::<f32>().add(output_index) = value;- } else {- write_bf16(d_ptr.as_ptr(), output_index, value);- }+ store(d_ptr.as_ptr(), output_data_type, output_index, value);}}}diff --git a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rsindex aeb51f780..7ce7cac4e 100644--- a/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs+++ b/crates/uzu-engine/src/backends/cpu/kernel/matmul/reference.rs@@ -1,4 +1,4 @@-use half::bf16;+use num_traits::AsPrimitive;use crate::{backends::{@@ -8,7 +8,8 @@ use crate::{},cpu::{Cpu, buffer::CpuBufferExt, kernel::matmul::MatmulCpuError},},- utils::pointers::SendPtr,+ data_type::DataType,+ utils::{dispatch_dtype::dispatch_dtype, pointers::SendPtr},};pub(super) enum WeightData {@@ -76,18 +77,20 @@ impl WeightData {}#[inline]-pub(super) unsafe fn read_bf16(+pub(super) unsafe fn load(base: *const u8,+ data_type: DataType,index: usize,) -> f32 {- unsafe { f32::from(*base.cast::<bf16>().add(index)) }+ dispatch_dtype!(|(T: data_type)| unsafe { (*base.cast::<T>().add(index)).as_() })}#[inline]-pub(super) unsafe fn write_bf16(+pub(super) unsafe fn store(base: *mut u8,+ data_type: DataType,index: usize,value: f32,) {- unsafe { *base.cast::<bf16>().add(index) = bf16::from_f32(value) }+ dispatch_dtype!(|(T: data_type)| unsafe { *base.cast::<T>().add(index) = value.as_() })}diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metalindex 14e859a48..b374e8727 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/gemv/gemv.metal@@ -26,9 +26,11 @@ template <uint REDUCTION_LANES,uint GROUP_LANES,uint NUM_SIMDGROUPS>-VARIANTS(AT, bfloat)-VARIANTS(BT, bfloat)+VARIANTS(AT, bfloat, float)+VARIANTS(BT, bfloat, float)VARIANTS(DT, bfloat, float)+CONSTRAINT(AT == BT)+CONSTRAINT(BT != "float" || (DT == "float" && B_PROLOGUE == GemmBPrologueKind::FullPrecision))VARIANTS(B_PROLOGUE,GemmBPrologueKind::FullPrecision,diff --git a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rsindex f58754c59..dc5becb57 100644--- a/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs+++ b/crates/uzu-engine/src/backends/metal/kernel/matmul/matmul_metal_kernel.rs@@ -52,7 +52,8 @@ impl MatmulMetalKernel {input_data_type: DataType,output_data_type: DataType,) -> bool {- if shape.gathered || plan.engine != gemm::GemmEngine::Mxu {+ // F32 is compiled for GEMV only.+ if shape.gathered || plan.engine != gemm::GemmEngine::Mxu || weights_data_type == DataType::F32 {return false;}if shape.m == 5diff --git a/crates/uzu-engine/src/config/model/decision_model.rs b/crates/uzu-engine/src/config/model/decision_model.rsnew file mode 100644index 000000000..5641573d7--- /dev/null+++ b/crates/uzu-engine/src/config/model/decision_model.rs@@ -0,0 +1,16 @@+use uzu_engine_macros::uzu_config;++use crate::{config::decoder::DecoderConfig, engine::language_model::GenerationConfig};++#[uzu_config(super::ModelConfig)]+pub struct DecisionModelConfig {+ pub decoder_config: DecoderConfig,+ pub generation_config: GenerationConfig,+ pub decision: DecisionConfig,+}++#[uzu_config]+pub struct DecisionConfig {+ pub codes: Box<[String]>,+ pub temperature: f32,+}diff --git a/crates/uzu-engine/src/config/model/mod.rs b/crates/uzu-engine/src/config/model/mod.rsindex d30b030b3..8fd786870 100644--- a/crates/uzu-engine/src/config/model/mod.rs+++ b/crates/uzu-engine/src/config/model/mod.rs@@ -3,10 +3,15 @@ use uzu_engine_macros::uzu_config_abstract;use crate::config::token_codec::AnyTokenCodecConfig;pub mod classifier_model;+pub mod decision_model;pub mod generation;pub mod language_model;-#[uzu_config_abstract(language_model::LanguageModelConfig, classifier_model::ClassifierModelConfig)]+#[uzu_config_abstract(+ language_model::LanguageModelConfig,+ classifier_model::ClassifierModelConfig,+ decision_model::DecisionModelConfig+)]pub struct ModelConfig {pub token_codec_config: AnyTokenCodecConfig,}diff --git a/crates/uzu-engine/src/config/token_codec/mod.rs b/crates/uzu-engine/src/config/token_codec/mod.rsindex b839add1e..a1d48f6b7 100644--- a/crates/uzu-engine/src/config/token_codec/mod.rs+++ b/crates/uzu-engine/src/config/token_codec/mod.rs@@ -1,3 +1,4 @@+use shoji::types::session::classification::{ChatTokenCodecConfig, TokenCodecConfig as ShojiTokenCodecConfig};use uzu_engine_macros::uzu_config_abstract;pub mod chat_codec;@@ -5,3 +6,22 @@ pub mod raw_text_codec;#[uzu_config_abstract(chat_codec::ChatCodecConfig, raw_text_codec::RawTextCodecConfig)]pub struct TokenCodecConfig;++impl From<&AnyTokenCodecConfig> for ShojiTokenCodecConfig {+ fn from(config: &AnyTokenCodecConfig) -> Self {+ match config {+ AnyTokenCodecConfig::ChatCodecConfig(config) => ShojiTokenCodecConfig::Chat(ChatTokenCodecConfig {+ prompt_template: config.prompt_template.clone(),+ output_parser_regex: config.output_parser_regex.clone(),+ system_role_name: config.system_role_name.clone(),+ user_role_name: config.user_role_name.clone(),+ assistant_role_name: config.assistant_role_name.clone(),+ eos_token: config.eos_token.clone(),+ bos_token: config.bos_token.clone(),+ end_of_thinking_tag: config.end_of_thinking_tag.clone(),+ default_system_prompt: config.default_system_prompt.clone(),+ }),+ AnyTokenCodecConfig::RawTextCodecConfig(_) => ShojiTokenCodecConfig::RawText,+ }+ }+}diff --git a/crates/uzu-engine/src/encodable_block/classifier.rs b/crates/uzu-engine/src/encodable_block/classifier.rsindex 9304eef10..98aced9bb 100644--- a/crates/uzu-engine/src/encodable_block/classifier.rs+++ b/crates/uzu-engine/src/encodable_block/classifier.rs@@ -66,6 +66,7 @@ impl<B: Backend> Classifier<B> {let embedding_norm = Normalization::new(config.transformer_config.model_dim,data_type,+ data_type,None,ShortcutMode::None,PostLayerScalar::None,@@ -76,6 +77,7 @@ impl<B: Backend> Classifier<B> {.await?;let transformer = Transformer::new(+ data_type,data_type,None,&config.transformer_config,diff --git a/crates/uzu-engine/src/encodable_block/decision.rs b/crates/uzu-engine/src/encodable_block/decision.rsnew file mode 100644index 000000000..55c7d131c--- /dev/null+++ b/crates/uzu-engine/src/encodable_block/decision.rs@@ -0,0 +1,143 @@+use derive_more::Debug;+use thiserror::Error;++use crate::{+ backends::common::{Backend, BufferRef, CommandBuffer, CommandBufferEncoding, gpu_types::trie::TrieNode},+ config::model::decision_model::DecisionModelConfig,+ data_type::DataType,+ encodable_block::{+ batch_topology::BatchTopology,+ embedding_table::{EmbeddingTable, EmbeddingTableError},+ linear::{Linear, LinearEncodeError, LinearMatmul, LinearMatmulError},+ transformer::{Transformer, TransformerEncodeError, TransformerNewError},+ },+ safetensors::{ParameterLoaderError, ParameterTree},+};++#[derive(Debug, Error)]+pub enum DecisionError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Embedding table error: {0}")]+ EmbeddingTable(#[from] EmbeddingTableError<B>),+ #[error("Linear encoding error: {0}")]+ LinearEncode(#[from] LinearEncodeError<B>),+ #[error("Linear matmul error: {0}")]+ LinearMatmul(#[from] LinearMatmulError<B>),+ #[error("Parameter loading error: {0}")]+ Parameter(#[from] ParameterLoaderError<B>),+ #[error("Transformer error: {0}")]+ Transformer(#[from] TransformerNewError<B>),+ #[error("Transformer encoding error: {0}")]+ TransformerEncode(#[from] TransformerEncodeError<B>),+}++pub struct Decision<B: Backend> {+ input_data_type: DataType,+ input_scale: f32,+ input_embedding: EmbeddingTable<B>,+ transformer: Transformer<B>,+ readout: LinearMatmul<B>,+}++impl<B: Backend> Decision<B> {+ pub async fn new(+ input_data_type: DataType,+ output_data_type: DataType,+ config: &DecisionModelConfig,+ parameter_tree: &ParameterTree<'_, B>,+ context: &B::Context,+ ) -> Result<Self, DecisionError<B>> {+ let model_dim = config.decoder_config.transformer_config.model_dim;+ let embedding_tree = parameter_tree.subtree("embedding");+ let input_embedding = EmbeddingTable::load(+ config.decoder_config.vocab_size,+ model_dim,+ input_data_type,+ &embedding_tree.subtree("input_embedding"),+ context,+ )+ .await?;++ let transformer = Transformer::new(+ input_data_type,+ output_data_type,+ None,+ &config.decoder_config.transformer_config,+ ¶meter_tree.subtree("transformer"),+ context,+ )+ .await?;++ let readout_tree = embedding_tree.subtree("output_embedding");+ let readout_spec = readout_tree.metadata("spec")?;+ let readout = LinearMatmul::load(+ model_dim,+ config.decision.codes.len() as u32,+ output_data_type,+ output_data_type,+ output_data_type,+ readout_spec,+ None,+ &readout_tree,+ None,+ context,+ )+ .await?;++ Ok(Self {+ input_data_type,+ input_scale: config.decoder_config.embedding_config.input_scale().unwrap_or(1.0),+ input_embedding,+ transformer,+ readout,+ })+ }++ pub fn max_context_length(&self) -> Option<u32> {+ self.transformer.max_context_length()+ }++ /// Runs the whole prompt in one stateless pass and returns the `[1, answer_count]` logits of its last token.+ pub fn encode(+ &self,+ token_ids: impl BufferRef<Backend = B>,+ token_count: u32,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<B::ScratchBuffer, DecisionError<B>> {+ assert!(token_count > 0, "decision needs at least one token");++ let command_buffer = &mut *command_buffer.debug_group("decision");++ let mut embedded = command_buffer+ .allocate_scratch_for_shape(&[token_count, self.transformer.model_dim()], self.input_data_type)+ .map_err(DecisionError::Backend)?;+ self.input_embedding.encode_lookup(token_ids, &mut embedded, token_count, self.input_scale, command_buffer);++ let nodes = (0..token_count)+ .map(|index| TrieNode {+ trie_start: index,+ trie_end: token_count - 1,+ height: index,+ })+ .collect::<Box<[TrieNode]>>();++ let hidden = self+ .transformer+ .encode(+ embedded,+ None::<&B::ScratchBuffer>,+ &BatchTopology::new(&nodes, true),+ Some((token_count - 1..token_count).into()),+ None,+ None,+ command_buffer,+ )?+ .output+ .expect("transformer returns rows for an output range");++ let logits = self.readout.encode(hidden, 1, command_buffer)?;++ Ok(logits)+ }+}diff --git a/crates/uzu-engine/src/encodable_block/decoder.rs b/crates/uzu-engine/src/encodable_block/decoder.rsindex e219213fd..73b7686e0 100644--- a/crates/uzu-engine/src/encodable_block/decoder.rs+++ b/crates/uzu-engine/src/encodable_block/decoder.rs@@ -72,6 +72,7 @@ impl<B: Backend> Decoder<B> {Normalization::new(config.transformer_config.model_dim,data_type,+ data_type,None,ShortcutMode::None,PostLayerScalar::None,@@ -106,6 +107,7 @@ impl<B: Backend> Decoder<B> {};let transformer = Transformer::new(+ data_type,data_type,readout_input_hadamard_factors,&config.transformer_config,diff --git a/crates/uzu-engine/src/encodable_block/dflash.rs b/crates/uzu-engine/src/encodable_block/dflash.rsindex aa2dc8731..40f8c52ef 100644--- a/crates/uzu-engine/src/encodable_block/dflash.rs+++ b/crates/uzu-engine/src/encodable_block/dflash.rs@@ -111,6 +111,7 @@ impl<B: Backend> DFlash<B> {let projected_feature_norm = Normalization::new(config.model_dim,data_type,+ data_type,None,ShortcutMode::None,PostLayerScalar::None,@@ -154,6 +155,7 @@ impl<B: Backend> DFlash<B> {let output_norm = Normalization::new(config.model_dim,data_type,+ data_type,None,ShortcutMode::Add,PostLayerScalar::None,diff --git a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rsindex aa4b71282..cd4e1d02a 100644--- a/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs+++ b/crates/uzu-engine/src/encodable_block/mixer/delta_net.rs@@ -3,7 +3,7 @@ use thiserror::Error;use crate::{backends::common::{- Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Context, Kernels,+ Backend, Buffer, BufferRef, CommandBuffer, CommandBufferEncoding, Context, GpuBuffer, Kernels,kernel::{ConvTreeScanKernel, DeltaNetConvPackKernel, DeltaNetConvScanKernel, DeltaNetConvUpdateKernel,DeltaNetNormGateKernel, DeltaNetPrefillKernel, DeltaNetPrefillPrepKernel, DeltaNetUpdateKernel,@@ -444,89 +444,48 @@ impl<B: Backend> DeltaNet<B> {});Ok(output)}-}--impl<B: Backend> Mixer<B> for DeltaNet<B> {- fn speculation_supported(&self) -> bool {- self.tree_verify.is_some()- }-- fn max_context_length(&self) -> Option<u32> {- None- }- fn create_empty_state(- &self,- _max_context_length: Option<u32>,- context: &B::Context,- ) -> Result<Box<dyn MixerState<B>>, B::Error> {- let conv_state =- context.create_buffer_gpu(size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE))?;- let ssm_state = context.create_buffer_gpu(size_for_shape(- &[self.num_heads, self.value_head_dim, self.head_dim],- INNER_DATA_TYPE,- ))?;-- let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(- context,- self.outer_data_type,- self.head_dim,- self.num_heads,- self.num_groups,- )?;-- Ok(Box::new(DeltaNetState {- conv_state,- ssm_state,- suffix_status: None,- state_advance,- is_empty: true,- }))- }-- fn encode(+ // Runs a fully accepted batch from zero conv and SSM states that live only for this encode.+ fn encode_stateless(&self,hidden: B::ScratchBuffer,- precalculated_rope: Option<&PrecalculatedRoPE<B>>,batch_dim: &BatchTopology,- state: Option<MaybeMut<dyn MixerState<B>>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {- let command_buffer = &mut *command_buffer.debug_group("delta net");+ assert!(batch_dim.full_accept(), "stateless delta net doesn't support trie");- assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");-- let state = state.expect("delta net requires state");- let state = state.downcast::<DeltaNetState<B>>().expect("incorrect type of delta net state");- let MaybeMut::Mut(state) = state else {- panic!("delta net doesn't support immutable state");- };-- assert!(state.suffix_status.is_none(), "delta net called with state with an unaccepted suffix");-- if state.is_empty {- command_buffer.encode_fill(&mut state.conv_state, 0);- command_buffer.encode_fill(&mut state.ssm_state, 0);- state.is_empty = false;- }+ let mut conv_state = command_buffer+ .allocate_scratch_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ command_buffer.encode_fill(&mut conv_state, 0);- let mut in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;+ let mut ssm_state = command_buffer+ .allocate_scratch_for_shape(&[self.num_heads, self.value_head_dim, self.head_dim], INNER_DATA_TYPE)+ .map_err(MixerEncodeError::Backend)?;+ command_buffer.encode_fill(&mut ssm_state, 0);- if !batch_dim.full_accept() {- let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;+ let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;- return Ok(output);- }+ self.encode_flat(in_projected, batch_dim.size(), &mut conv_state, &mut ssm_state, command_buffer)+ }+ fn encode_flat(+ &self,+ mut in_projected: B::ScratchBuffer,+ suffix_length: u32,+ conv_state: &mut dyn GpuBuffer<Backend = B>,+ ssm_state: &mut dyn GpuBuffer<Backend = B>,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {let mut delta_output = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.value_dim], self.outer_data_type)+ .allocate_scratch_for_shape(&[suffix_length, self.value_dim], self.outer_data_type).map_err(MixerEncodeError::Backend)?;- if batch_dim.size() == 1 {+ if suffix_length == 1 {self.conv_update.encode(&self.conv_weight,self.conv_bias.as_ref(),&mut in_projected,- &mut state.conv_state,+ &mut *conv_state,self.kernel_size,self.conv_dim,self.kernel_size - 1,@@ -538,7 +497,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {&self.a_log,&self.dt_bias,&self.norm_weight,- &mut state.ssm_state,+ &mut *ssm_state,&mut delta_output,self.num_heads,self.num_groups,@@ -549,16 +508,16 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {command_buffer,);} else {- let conv_tile_tokens = batch_dim.size().div_ceil(CONV_TILES).max(CONV_MIN_TILE_TOKENS);+ let conv_tile_tokens = suffix_length.div_ceil(CONV_TILES).max(CONV_MIN_TILE_TOKENS);debug_assert!(conv_tile_tokens >= self.kernel_size - 1);let mut packed = command_buffer.allocate_scratch_for_shape(&[CONV_TILES, self.kernel_size - 1, self.conv_dim], INNER_DATA_TYPE).map_err(MixerEncodeError::Backend)?;self.conv_pack.encode(- &state.conv_state,+ &*conv_state,&in_projected,&mut packed,- batch_dim.size(),+ suffix_length,conv_tile_tokens,self.total_proj_dim,self.conv_dim,@@ -569,44 +528,44 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {&self.conv_weight,self.conv_bias.as_ref(),&mut in_projected,- &mut state.conv_state,- batch_dim.size(),+ &mut *conv_state,+ suffix_length,conv_tile_tokens,self.conv_dim,self.total_proj_dim,command_buffer,);- if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(batch_dim.size())) {+ if let Some(chunked) = self.chunked.as_ref().filter(|chunked| chunked.should_use(suffix_length)) {chunked.encode(DeltaNetChunkedPrefillArgs {in_projected: &in_projected,a_log: &self.a_log,dt_bias: &self.dt_bias,- ssm_state: &mut state.ssm_state,+ ssm_state,delta_output: &mut delta_output,num_heads: self.num_heads,num_groups: self.num_groups,value_head_dim: self.value_head_dim,key_dim: self.key_dim,value_dim: self.value_dim,- suffix_len: batch_dim.size(),+ suffix_len: suffix_length,},command_buffer,).map_err(MixerEncodeError::Backend)?;} else {let mut prep_q_norm = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)+ .allocate_scratch_for_shape(&[suffix_length, self.key_dim], INNER_DATA_TYPE).map_err(MixerEncodeError::Backend)?;let mut prep_k_norm = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.key_dim], INNER_DATA_TYPE)+ .allocate_scratch_for_shape(&[suffix_length, self.key_dim], INNER_DATA_TYPE).map_err(MixerEncodeError::Backend)?;let mut prep_beta = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)+ .allocate_scratch_for_shape(&[suffix_length, self.num_heads], INNER_DATA_TYPE).map_err(MixerEncodeError::Backend)?;let mut prep_decay = command_buffer- .allocate_scratch_for_shape(&[batch_dim.size(), self.num_heads], INNER_DATA_TYPE)+ .allocate_scratch_for_shape(&[suffix_length, self.num_heads], INNER_DATA_TYPE).map_err(MixerEncodeError::Backend)?;self.delta_net_prefill_prep.encode(&in_projected,@@ -621,7 +580,7 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {self.num_groups,self.key_dim,self.value_dim,- batch_dim.size(),+ suffix_length,command_buffer,);self.delta_net_prefill.encode(@@ -630,14 +589,14 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {&prep_beta,&prep_decay,&in_projected,- &mut state.ssm_state,+ &mut *ssm_state,&mut delta_output,self.num_heads,self.num_groups,self.value_head_dim,self.key_dim,self.value_dim,- batch_dim.size(),+ suffix_length,self.value_head_dim.div_ceil(16),command_buffer,);@@ -652,17 +611,100 @@ impl<B: Backend> Mixer<B> for DeltaNet<B> {self.conv_dim,self.total_proj_dim,self.norm_epsilon,- batch_dim.size(),+ suffix_length,command_buffer,);}+ Ok(self.out_projection.encode(delta_output, suffix_length, command_buffer)?)+ }+}++impl<B: Backend> Mixer<B> for DeltaNet<B> {+ fn speculation_supported(&self) -> bool {+ self.tree_verify.is_some()+ }++ fn max_context_length(&self) -> Option<u32> {+ None+ }++ fn create_empty_state(+ &self,+ _max_context_length: Option<u32>,+ context: &B::Context,+ ) -> Result<Box<dyn MixerState<B>>, B::Error> {+ let conv_state =+ context.create_buffer_gpu(size_for_shape(&[self.conv_dim, self.kernel_size - 1], INNER_DATA_TYPE))?;+ let ssm_state = context.create_buffer_gpu(size_for_shape(+ &[self.num_heads, self.value_head_dim, self.head_dim],+ INNER_DATA_TYPE,+ ))?;++ let state_advance = <B::Kernels as Kernels>::StateAdvanceKernel::new(+ context,+ self.outer_data_type,+ self.head_dim,+ self.num_heads,+ self.num_groups,+ )?;++ Ok(Box::new(DeltaNetState {+ conv_state,+ ssm_state,+ suffix_status: None,+ state_advance,+ is_empty: true,+ }))+ }++ fn encode(+ &self,+ hidden: B::ScratchBuffer,+ precalculated_rope: Option<&PrecalculatedRoPE<B>>,+ batch_dim: &BatchTopology,+ state: Option<MaybeMut<dyn MixerState<B>>>,+ command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,+ ) -> Result<B::ScratchBuffer, MixerEncodeError<B>> {+ assert!(precalculated_rope.is_none(), "unexpected rope for delta net mixer");++ let command_buffer = &mut *command_buffer.debug_group("delta net");+ let Some(state) = state else {+ return self.encode_stateless(hidden, batch_dim, command_buffer);+ };++ let state = state.downcast::<DeltaNetState<B>>().expect("incorrect type of delta net state");+ let MaybeMut::Mut(state) = state else {+ panic!("delta net doesn't support immutable state");+ };++ assert!(state.suffix_status.is_none(), "delta net called with state with an unaccepted suffix");++ if state.is_empty {+ command_buffer.encode_fill(&mut state.conv_state, 0);+ command_buffer.encode_fill(&mut state.ssm_state, 0);+ state.is_empty = false;+ }++ let in_projected = self.in_projection.encode(hidden, batch_dim.size(), command_buffer)?;++ if !batch_dim.full_accept() {+ let output = self.encode_tree_verify(&in_projected, batch_dim, state, command_buffer)?;+ return Ok(output);+ }++ let output = self.encode_flat(+ in_projected,+ batch_dim.size(),+ &mut state.conv_state,+ &mut state.ssm_state,+ command_buffer,+ )?;+state.suffix_status = Some(DeltaNetSuffixStatus::Flat {suffix_length: batch_dim.size(),});- let output = self.out_projection.encode(delta_output, batch_dim.size(), command_buffer)?;-Ok(output)}}diff --git a/crates/uzu-engine/src/encodable_block/mod.rs b/crates/uzu-engine/src/encodable_block/mod.rsindex 9f6c79366..f7d7aaf35 100644--- a/crates/uzu-engine/src/encodable_block/mod.rs+++ b/crates/uzu-engine/src/encodable_block/mod.rs@@ -1,6 +1,7 @@pub mod batch_topology;pub mod classifier;pub mod convolution;+pub mod decision;pub mod decoder;pub mod dflash;pub mod embedding;diff --git a/crates/uzu-engine/src/encodable_block/normalization.rs b/crates/uzu-engine/src/encodable_block/normalization.rsindex ffbe3d317..cf30952e0 100644--- a/crates/uzu-engine/src/encodable_block/normalization.rs+++ b/crates/uzu-engine/src/encodable_block/normalization.rs@@ -44,14 +44,16 @@ pub struct Normalization<B: Backend> {element_count: u32,hadamard_factors: Option<B::GlobalGpuBuffer>,post_layer_scalar_value: f32,- data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,kernel: <B::Kernels as Kernels>::NormalizationKernel,}impl<B: Backend> Normalization<B> {pub async fn new(element_count: u32,- data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,hadamard_factors: Option<B::GlobalGpuBuffer>,shortcut_mode: ShortcutMode,post_layer_scalar: PostLayerScalar,@@ -84,9 +86,9 @@ impl<B: Backend> Normalization<B> {let kernel = <B::Kernels as Kernels>::NormalizationKernel::new(context,- data_type,+ input_data_type,DataType::F32,- data_type,+ output_data_type,DataType::F32,false,config.subtract_mean,@@ -109,7 +111,8 @@ impl<B: Backend> Normalization<B> {element_count,hadamard_factors,post_layer_scalar_value,- data_type,+ input_data_type,+ output_data_type,kernel,})}@@ -122,10 +125,11 @@ impl<B: Backend> Normalization<B> {shortcut: Option<impl BufferMut<Backend = B>>,command_buffer: &mut <B::CommandBuffer as CommandBuffer>::Encoding,) -> Result<B::ScratchBuffer, B::Error> {- let row_size = size_for_shape(&[self.element_count], self.data_type);+ let row_size = size_for_shape(&[self.element_count], self.input_data_type);let row_offset_bytes = row_offset as usize * row_size;let shortcut = shortcut.map(|shortcut| shortcut.subrange_mut(row_offset_bytes..));- let mut output = command_buffer.allocate_scratch_for_shape(&[row_count, self.element_count], self.data_type)?;+ let mut output =+ command_buffer.allocate_scratch_for_shape(&[row_count, self.element_count], self.output_data_type)?;self.kernel.encode(Some(input.subrange(row_offset_bytes..)),self.scales.as_ref(),diff --git a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rsindex 2826f5e98..b8f8afd99 100644--- a/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs+++ b/crates/uzu-engine/src/encodable_block/per_layer_embedding.rs@@ -83,6 +83,7 @@ impl<B: Backend> PerLayerEmbedding<B> {let projection_norm = Normalization::new(config.ple_dim,data_type,+ data_type,None,ShortcutMode::None,PostLayerScalar::ScaleOutput(config.input_scale),@@ -200,6 +201,7 @@ impl<B: Backend> PerLayerEmbeddingProjection<B> {let norm = Normalization::new(model_dim,data_type,+ data_type,None,ShortcutMode::None,PostLayerScalar::None,diff --git a/crates/uzu-engine/src/encodable_block/prediction_head.rs b/crates/uzu-engine/src/encodable_block/prediction_head.rsindex e7d644276..3b0c8780d 100644--- a/crates/uzu-engine/src/encodable_block/prediction_head.rs+++ b/crates/uzu-engine/src/encodable_block/prediction_head.rs@@ -63,6 +63,7 @@ impl<B: Backend> PredictionHead<B> {let normalization = Normalization::new(hidden_dim,data_type,+ data_type,None,ShortcutMode::None,PostLayerScalar::None,diff --git a/crates/uzu-engine/src/encodable_block/transformer.rs b/crates/uzu-engine/src/encodable_block/transformer.rsindex 745205db7..6d93bc62c 100644--- a/crates/uzu-engine/src/encodable_block/transformer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer.rs@@ -108,7 +108,8 @@ pub struct Transformer<B: Backend> {impl<B: Backend> Transformer<B> {pub async fn new(- data_type: DataType,+ input_data_type: DataType,+ output_data_type: DataType,output_norm_hadamard_factors: Option<B::GlobalGpuBuffer>,transformer_config: &TransformerConfig,parameter_tree: &ParameterTree<'_, B>,@@ -133,7 +134,7 @@ impl<B: Backend> Transformer<B> {transformer_config.hidden_dim,num_layers,layer_index,- data_type,+ input_data_type,layer_config,&layer_loader,context,@@ -145,7 +146,8 @@ impl<B: Backend> Transformer<B> {let output_norm = Normalization::new(transformer_config.model_dim,- data_type,+ input_data_type,+ output_data_type,output_norm_hadamard_factors,ShortcutMode::Add,PostLayerScalar::None,@@ -155,7 +157,7 @@ impl<B: Backend> Transformer<B> {).await?;- let residual_add = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, data_type, false)+ let residual_add = <B::Kernels as Kernels>::TensorAddScaleKernel::new(context, input_data_type, false).map_err(TransformerNewError::Backend)?;Ok(Self {@@ -180,6 +182,10 @@ impl<B: Backend> Transformer<B> {Ok(output)}+ pub fn model_dim(&self) -> u32 {+ self.model_dim+ }+pub fn speculation_supported(&self) -> bool {self.layers.iter().all(|(layer, _rope)| layer.mixer.speculation_supported())}diff --git a/crates/uzu-engine/src/encodable_block/transformer_layer.rs b/crates/uzu-engine/src/encodable_block/transformer_layer.rsindex 1093bc694..002817d64 100644--- a/crates/uzu-engine/src/encodable_block/transformer_layer.rs+++ b/crates/uzu-engine/src/encodable_block/transformer_layer.rs@@ -236,6 +236,7 @@ impl<B: Backend> TransformerLayer<B> {Normalization::new(model_dim,data_type,+ data_type,mixer_hadamard_factors,if layer_index > 0 {ShortcutMode::Add@@ -261,6 +262,7 @@ impl<B: Backend> TransformerLayer<B> {Normalization::new(model_dim,data_type,+ data_type,None,ShortcutMode::None,PostLayerScalar::None,@@ -305,6 +307,7 @@ impl<B: Backend> TransformerLayer<B> {let pre_mlp_norm = Normalization::new(model_dim,data_type,+ data_type,mlp_input_hadamard_factors,ShortcutMode::Add,residual_sum_scalar,@@ -319,6 +322,7 @@ impl<B: Backend> TransformerLayer<B> {Normalization::new(model_dim,data_type,+ data_type,None,ShortcutMode::None,output_scalar,diff --git a/crates/uzu-engine/src/encodable_block/weaver.rs b/crates/uzu-engine/src/encodable_block/weaver.rsindex 694670440..0968fd8a6 100644--- a/crates/uzu-engine/src/encodable_block/weaver.rs+++ b/crates/uzu-engine/src/encodable_block/weaver.rs@@ -216,6 +216,7 @@ impl<B: Backend> Weaver<B> {let token_embedding_norm = Normalization::new(config.target_embedding_dim,DATA_TYPE,+ DATA_TYPE,None,ShortcutMode::None,PostLayerScalar::None,@@ -227,6 +228,7 @@ impl<B: Backend> Weaver<B> {let hidden_state_norm = Normalization::new(config.target_model_dim,DATA_TYPE,+ DATA_TYPE,None,ShortcutMode::None,PostLayerScalar::None,@@ -254,6 +256,7 @@ impl<B: Backend> Weaver<B> {let readout_norm = Normalization::new(config.model_dim,DATA_TYPE,+ DATA_TYPE,None,ShortcutMode::Add,PostLayerScalar::None,diff --git a/crates/uzu-engine/src/encodable_block/weaver_layer.rs b/crates/uzu-engine/src/encodable_block/weaver_layer.rsindex 8f639d48d..d05900b46 100644--- a/crates/uzu-engine/src/encodable_block/weaver_layer.rs+++ b/crates/uzu-engine/src/encodable_block/weaver_layer.rs@@ -98,6 +98,7 @@ impl<B: Backend> WeaverLayer<B> {let pre_attention_norm = Normalization::new(model_dim,DATA_TYPE,+ DATA_TYPE,None,if add_to_residual {ShortcutMode::Add@@ -113,6 +114,7 @@ impl<B: Backend> WeaverLayer<B> {let pre_mlp_norm = Normalization::new(model_dim,DATA_TYPE,+ DATA_TYPE,None,ShortcutMode::Add,PostLayerScalar::None,diff --git a/crates/uzu-engine/src/engine/classifier_model.rs b/crates/uzu-engine/src/engine/classifier_model.rsindex b606b444a..4429d8865 100644--- a/crates/uzu-engine/src/engine/classifier_model.rs+++ b/crates/uzu-engine/src/engine/classifier_model.rs@@ -2,10 +2,7 @@ use std::{cmp::min, collections::HashMap, io, path::Path, sync::Arc};use derive_more::Debug;use half::bf16;-use shoji::{- traits::backend::classification::ClassifierOutput,- types::session::classification::{ChatTokenCodecConfig, TokenCodecConfig},-};+use shoji::{traits::backend::classification::ClassifierOutput, types::session::classification::TokenCodecConfig};use thiserror::Error;use tokenizers::Tokenizer;@@ -104,20 +101,7 @@ impl<B: Backend> ClassifierModel<B> {}pub fn token_codec_config(&self) -> TokenCodecConfig {- match &self.token_codec_config {- AnyTokenCodecConfig::ChatCodecConfig(config) => TokenCodecConfig::Chat(ChatTokenCodecConfig {- prompt_template: config.prompt_template.clone(),- output_parser_regex: config.output_parser_regex.clone(),- system_role_name: config.system_role_name.clone(),- user_role_name: config.user_role_name.clone(),- assistant_role_name: config.assistant_role_name.clone(),- eos_token: config.eos_token.clone(),- bos_token: config.bos_token.clone(),- end_of_thinking_tag: config.end_of_thinking_tag.clone(),- default_system_prompt: config.default_system_prompt.clone(),- }),- AnyTokenCodecConfig::RawTextCodecConfig(_) => TokenCodecConfig::RawText,- }+ TokenCodecConfig::from(&self.token_codec_config)}pub async fn classify(diff --git a/crates/uzu-engine/src/engine/decision_model.rs b/crates/uzu-engine/src/engine/decision_model.rsnew file mode 100644index 000000000..ccc91d362--- /dev/null+++ b/crates/uzu-engine/src/engine/decision_model.rs@@ -0,0 +1,153 @@+use std::{io, path::Path, sync::Arc};++use shoji::types::session::classification::TokenCodecConfig;+use thiserror::Error;+use tokenizers::Tokenizer;++use crate::{+ backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},+ config::{+ model::decision_model::{DecisionConfig, DecisionModelConfig},+ token_codec::AnyTokenCodecConfig,+ },+ data_type::DataType,+ encodable_block::decision::{Decision, DecisionError},+ engine::Engine,+ safetensors::{ParameterLoader, ParameterLoaderError},+ utils::file,+};++#[derive(Debug, Error)]+pub enum DecisionModelError<B: Backend> {+ #[error("Backend error: {0}")]+ Backend(#[source] B::Error),+ #[error("Input larger than model context size")]+ ContextOverflow,+ #[error("Decision error: {0}")]+ Decision(#[from] DecisionError<B>),+ #[error("Input is empty")]+ EmptyInput,+ #[error("Option count {option_count} is outside 1..={answer_count}")]+ InvalidOptionCount {+ option_count: usize,+ answer_count: usize,+ },+ #[error("I/O error: {0}")]+ IO(#[from] io::Error),+ #[error("ParameterLoader error: {0}")]+ ParameterLoader(#[from] ParameterLoaderError<B>),+ #[error("Serde error: {0}")]+ Serde(#[from] serde_json::Error),+ #[error("Tokenizer error: {0}")]+ Tokenizer(#[from] tokenizers::Error),+}++pub struct DecisionModel<B: Backend> {+ context: Arc<B::Context>,+ decision: Decision<B>,+ decision_config: DecisionConfig,+ tokenizer: Arc<Tokenizer>,+ token_codec_config: AnyTokenCodecConfig,+}++impl<B: Backend> DecisionModel<B> {+ pub fn decision_config(&self) -> &DecisionConfig {+ &self.decision_config+ }++ pub fn tokenizer(&self) -> &Arc<Tokenizer> {+ &self.tokenizer+ }++ pub fn token_codec_config(&self) -> TokenCodecConfig {+ TokenCodecConfig::from(&self.token_codec_config)+ }++ pub async fn probabilities(+ &self,+ input: &[u64],+ option_count: usize,+ ) -> Result<Vec<f32>, DecisionModelError<B>> {+ if input.is_empty() {+ return Err(DecisionModelError::EmptyInput);+ }++ if self+ .decision+ .max_context_length()+ .is_some_and(|max_context_length| input.len() > max_context_length as usize)+ {+ return Err(DecisionModelError::ContextOverflow);+ }++ let answer_count = self.decision_config.codes.len();+ if !(1..=answer_count).contains(&option_count) {+ return Err(DecisionModelError::InvalidOptionCount {+ option_count,+ answer_count,+ });+ }++ let mut command_buffer = self.context.create_command_buffer(None, None).map_err(DecisionModelError::Backend)?;++ let token_ids = command_buffer+ .allocate_constant_from_slice(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>())+ .map_err(DecisionModelError::Backend)?;+ let logits = self.decision.encode(&token_ids, input.len() as u32, &mut command_buffer)?;++ let mut output_buffer = self+ .context+ .create_buffer_gpu(answer_count * DataType::F32.size_in_bytes())+ .map_err(DecisionModelError::Backend)?;+ command_buffer.encode_copy(&logits, &mut output_buffer);+ drop(logits);+ drop(token_ids);++ command_buffer.end_encoding().submit().wait_until_completed().await.map_err(DecisionModelError::Backend)?;++ let temperature = self.decision_config.temperature;+ let logits = self.context.copyout::<f32>(&output_buffer).await.map_err(DecisionModelError::Backend)?;+ let scaled = logits[..option_count].iter().map(|logit| logit / temperature).collect::<Vec<_>>();+ let max = scaled.iter().copied().fold(f32::NEG_INFINITY, f32::max);+ let exps = scaled.iter().map(|logit| (logit - max).exp()).collect::<Vec<_>>();+ let sum = exps.iter().sum::<f32>();+ let result = exps.into_iter().map(|value| value / sum).collect();+ Ok(result)+ }+}++impl<B: Backend> Engine<B> {+ pub async fn load_decision_model(+ &self,+ model_path: &Path,+ ) -> Result<DecisionModel<B>, DecisionModelError<B>> {+ let context = self.context.clone();++ let config_file_path = model_path.join("config.json");+ let config_file_content = file::read(config_file_path).await?;+ let config = serde_json::from_slice::<DecisionModelConfig>(&config_file_content)?;++ let tokenizer_file_path = model_path.join("tokenizer.json");+ let tokenizer_file_content = file::read(tokenizer_file_path).await?;+ let tokenizer = Tokenizer::from_bytes(tokenizer_file_content)?;++ let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), context.as_ref()).await?;+ let decision = Decision::new(+ DataType::BF16,+ DataType::F32,+ &config,+ &weight_loader.tree().subtree("decoder"),+ context.as_ref(),+ )+ .await?;+ weight_loader.assert_all_tensors_read()?;++ Ok(DecisionModel {+ context,+ decision,+ decision_config: config.decision,+ tokenizer: Arc::new(tokenizer),+ token_codec_config: config.token_codec_config,+ })+ }+}diff --git a/crates/uzu-engine/src/engine/mod.rs b/crates/uzu-engine/src/engine/mod.rsindex 95781931d..e9593328e 100644--- a/crates/uzu-engine/src/engine/mod.rs+++ b/crates/uzu-engine/src/engine/mod.rs@@ -13,6 +13,7 @@ use crate::{};pub mod classifier_model;+pub mod decision_model;pub mod language_model;pub struct Engine<B: Backend> {@@ -48,6 +49,7 @@ impl<B: Backend> Engine<B> {pub enum ModelType {LanguageModel,Classifier,+ Decision,}#[derive(Debug, Error)]@@ -62,6 +64,7 @@ pub async fn resolve_model_type(model_path: &Path) -> Result<ModelType, ResolveMOk(match read_model_config(model_path).await? {AnyModelConfig::LanguageModelConfig(_) => ModelType::LanguageModel,AnyModelConfig::ClassifierModelConfig(_) => ModelType::Classifier,+ AnyModelConfig::DecisionModelConfig(_) => ModelType::Decision,})}@@ -69,6 +72,7 @@ pub async fn resolve_model_sampling_method(model_path: &Path) -> Result<Option<SOk(match read_model_config(model_path).await? {AnyModelConfig::LanguageModelConfig(config) => Some(config.generation_config.default_sampling_method()),AnyModelConfig::ClassifierModelConfig(_) => None,+ AnyModelConfig::DecisionModelConfig(config) => Some(config.generation_config.default_sampling_method()),})}@@ -76,6 +80,7 @@ pub async fn resolve_model_context_length(model_path: &Path) -> Result<Option<u3let transformer = match read_model_config(model_path).await? {AnyModelConfig::LanguageModelConfig(config) => config.decoder_config.transformer_config,AnyModelConfig::ClassifierModelConfig(_) => return Ok(None),+ AnyModelConfig::DecisionModelConfig(config) => config.decoder_config.transformer_config,};Ok(transformer.layer_configsdiff --git a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rsindex 711d76e2e..9da7c011e 100644--- a/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs+++ b/crates/uzu-engine/unit/backends/common/kernel/matmul/gemv_test.rs@@ -182,6 +182,22 @@ fn group_major_gemv_bf16(assert_eq_float(&reference, &actual, 0.05, &format!("GroupOutput GEMV W{bits} {method:?}"));}+#[rstest]+#[test_attr(uzu_test)]+#[case::m1(1, 128, 64)]+#[case::batched(4, 128, 64)]+#[case::max_batch(8, 128, 64)]+#[case::unaligned_k(1, 33, 64)]+#[case::unaligned_n(1, 128, 11)]+#[case::large(1, 4096, 2048)]+fn gemv_f32(+ #[case] m: usize,+ #[case] k: usize,+ #[case] n: usize,+) {+ test::<f32>(m, k, n, 0.01);+}+fn assert_gather<T: DataTypeRepr + Float + Debug + Display>(dense: &[T],gather: &[T],@@ -264,9 +280,10 @@ fn quant_gather_case(#[uzu_test]fn gemv_gather() {- // Full-precision BF16 GEMV.+ // Full-precision GEMV.for soft_cap in [None, Some(15.0)] {fp_gather_case::<bf16>(soft_cap, 0.1);+ fp_gather_case::<f32>(soft_cap, 0.01);}// Quantized (bf16, per bits/method) — inline, since it isn't type-generic.for (bits, method, signed_codes) in [diff --git a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rsindex f8b5e6c3f..e53936048 100644--- a/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs+++ b/crates/uzu-engine/unit/backends/metal/kernel/matmul/gemm/selection_test.rs@@ -195,6 +195,7 @@ fn gemv_gemm_route_boundaries_are_preserved() {for (shape, data_type, prefer_gemm) in [(shape(4, 4096, 8192), DataType::BF16, true),(shape(4, 8192, 4096), DataType::BF16, false),+ (shape(4, 4096, 8192), DataType::F32, false),(shape(3, 4096, 8192), DataType::BF16, false),(shape(5, 4096, 8192), DataType::BF16, false),] {