A high-performance inference engine for AI models
use std::{path::Path, sync::Arc};
use derive_more::Debug;
use thiserror::Error;
use crate::{
backends::common::{
Backend, Context,
capture_matcher::{CAPTURE_MATCHER, CaptureError, CaptureMatcherSpan},
},
config::{model::AnyModelConfig, token_mixer::AnyTokenMixerConfig},
encodable_block::sampling::SamplingMethod,
};
pub mod classifier_model;
pub mod decision_model;
pub mod language_model;
pub struct Engine<B: Backend> {
_capture_span: Option<CaptureMatcherSpan>,
context: Arc<B::Context>,
}
#[derive(Debug, Error)]
pub enum EngineNewError<B: Backend> {
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
#[error("Capture error: {0}")]
Capture(#[from] CaptureError<B>),
}
impl<B: Backend> Engine<B> {
pub async fn new() -> Result<Arc<Self>, EngineNewError<B>> {
let context = <B::Context as Context>::new().await.map_err(EngineNewError::Backend)?;
let capture_span = CAPTURE_MATCHER.capture::<B>(&context, "")?;
Ok(Arc::new(Self {
_capture_span: capture_span,
context,
}))
}
pub fn peak_memory_usage(&self) -> Option<usize> {
self.context.peak_memory_usage()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ModelType {
LanguageModel,
Classifier,
Decision,
}
#[derive(Debug, Error)]
pub enum ResolveModelTypeError {
#[error("Unable to open model configuration: {0}")]
UnableToOpenConfig(#[from] std::io::Error),
#[error("Unable to deserialize model configuration: {0}")]
UnableToDeserializeConfig(#[from] serde_json::Error),
}
pub async fn resolve_model_type(model_path: &Path) -> Result<ModelType, ResolveModelTypeError> {
Ok(match read_model_config(model_path).await? {
AnyModelConfig::LanguageModelConfig(_) => ModelType::LanguageModel,
AnyModelConfig::ClassifierModelConfig(_) => ModelType::Classifier,
AnyModelConfig::DecisionModelConfig(_) => ModelType::Decision,
})
}
pub async fn resolve_model_sampling_method(model_path: &Path) -> Result<Option<SamplingMethod>, ResolveModelTypeError> {
Ok(match read_model_config(model_path).await? {
AnyModelConfig::LanguageModelConfig(config) => Some(config.generation_config.default_sampling_method()),
AnyModelConfig::ClassifierModelConfig(_) => None,
AnyModelConfig::DecisionModelConfig(config) => Some(config.generation_config.default_sampling_method()),
})
}
pub async fn resolve_model_context_length(model_path: &Path) -> Result<Option<u32>, ResolveModelTypeError> {
let transformer = match read_model_config(model_path).await? {
AnyModelConfig::LanguageModelConfig(config) => config.decoder_config.transformer_config,
AnyModelConfig::ClassifierModelConfig(_) => return Ok(None),
AnyModelConfig::DecisionModelConfig(config) => config.decoder_config.transformer_config,
};
Ok(transformer
.layer_configs
.iter()
.filter(|layer| {
matches!(
layer.mixer.body_config,
AnyTokenMixerConfig::AttentionConfig(_) | AnyTokenMixerConfig::SparseAttentionConfig(_)
)
})
.filter_map(|layer| layer.rope_config.as_ref().map(|rope| *rope.max_sequence_length()))
.min())
}
async fn read_model_config(model_path: &Path) -> Result<AnyModelConfig, ResolveModelTypeError> {
let bytes = crate::utils::file::read(model_path.join("config.json")).await?;
Ok(serde_json::from_slice(&bytes)?)
}