A high-performance inference engine for AI models
use std::{io, path::Path, sync::Arc};
use derive_more::Debug;
use thiserror::Error;
use tokenizers::Tokenizer;
use crate::{
backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
config::model::language_model::LanguageModelConfig,
data_type::DataType,
encodable_block::{
decoder::{Decoder, DecoderError},
sampling::{Sampling, SamplingMethod},
},
engine::Engine,
safetensors::{ParameterLoader, ParameterLoaderError},
speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},
utils::file,
};
pub use crate::{config::model::generation::GenerationConfig, utils::strict_serde::Unsupported};
pub mod state;
pub mod stream;
pub mod grammar;
pub struct LanguageModel<B: Backend> {
engine: Arc<Engine<B>>,
decoder: Decoder<B>,
speculator: Option<DFlashTfmSpeculator<B>>,
sampling: Sampling<B>,
context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
generation_config: GenerationConfig,
tokenizer: Arc<Tokenizer>,
vocab_size: usize,
}
#[derive(Debug, Error)]
pub enum EngineLoadLanguageModelError<B: Backend> {
#[error("I/O error: {0}")]
IO(#[from] io::Error),
#[error("Serde error: {0}")]
Serde(#[from] serde_json::Error),
#[error("ParameterLoader error: {0}")]
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
#[error("Decoder error: {0}")]
Decoder(#[from] DecoderError<B>),
#[error("Speculator error: {0}")]
Speculator(#[from] DFlashSpeculatorLoadError<B>),
#[error("Tokenizer error: {0}")]
Tokenizer(#[from] tokenizers::Error),
}
impl<B: Backend> Engine<B> {
pub async fn load_language_model(
self: &Arc<Self>,
model_path: &Path,
) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {
let config: LanguageModelConfig = serde_json::from_slice(&file::read(model_path.join("config.json")).await?)?;
let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), &*self.context).await?;
// TODO
let speculator_path = model_path.join("speculator");
let speculator_path = file::exists(&speculator_path).await.then_some(speculator_path);
let tokenizer = Arc::new(Tokenizer::from_bytes(file::read(model_path.join("tokenizer.json")).await?)?);
let data_type = DataType::BF16;
let decoder = Decoder::new(
data_type,
&config.decoder_config,
&weight_loader.tree().subtree("decoder"),
self.context.as_ref(),
)
.await?;
assert!(
speculator_path.is_none() || decoder.speculation_supported(),
"attempted to load speculator for a model that doesn't support one"
);
let speculator = if let Some(speculator_path) = speculator_path.as_deref() {
DFlashTfmSpeculator::new(speculator_path, decoder.embedding(), self.context.clone()).await?
} else {
None
};
let sampling = Sampling::new(config.decoder_config.vocab_size, data_type);
let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)
.map_err(EngineLoadLanguageModelError::Backend)?;
weight_loader.assert_all_tensors_read()?;
let generation_config = config.generation_config;
let vocab_size = config.decoder_config.vocab_size as usize;
Ok(LanguageModel {
engine: self.clone(),
decoder,
speculator,
sampling,
context_ring_update,
generation_config,
tokenizer,
vocab_size,
})
}
}
impl<B: Backend> LanguageModel<B> {
pub fn max_context_length(&self) -> Option<u32> {
self.decoder.max_context_length()
}
pub fn recommended_context_length(&self) -> Option<u32> {
let max_context_length = self.max_context_length();
// TODO: This is not the correct way to do it, there should be a real memory model
if self.engine.context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS) {
// We just assume that all mixers use sparse if it's available to make max context free until it's actually used
// Currenlty true for all mixers in uzu:
// - full attention uses sparse if it's available to make max context free until it's actually used
// - sliding window attention is bound, usually well below the recommended max context size on non-sparse (but can be made to use sparse if we care about it enough)
// - short conv/delta net are constant state size
max_context_length
} else if let Some(max_context_length) = max_context_length {
// If sparse buffers aren't supported and model has finite maximum context length we assume that kv cache is expensive enough that we should probably clamp it to
// something reasonable-ish for the platform. This is very primitive but works I guess...
let platform_recommended_context_length = if cfg!(target_os = "ios") {
8192
} else {
16384
};
Some(u32::min(max_context_length, platform_recommended_context_length))
} else {
// We just assume that unlimited context means constant state size on all mixers and is thus free
None
}
}
pub fn speculation_supported(&self) -> bool {
self.decoder.speculation_supported()
}
pub fn default_sampling_method(&self) -> SamplingMethod {
self.generation_config.default_sampling_method()
}
pub fn generation_config(&self) -> &GenerationConfig {
&self.generation_config
}
pub fn tokenizer(&self) -> &Arc<Tokenizer> {
&self.tokenizer
}
}