A high-performance inference engine for AI models
use std::{cmp::min, collections::HashMap, io, path::Path, sync::Arc};
use derive_more::Debug;
use half::bf16;
use shoji::{traits::backend::classification::ClassifierOutput, types::session::classification::TokenCodecConfig};
use thiserror::Error;
use tokenizers::Tokenizer;
use crate::{
backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},
config::{model::classifier_model::ClassifierModelConfig, token_codec::AnyTokenCodecConfig},
data_type::DataType,
encodable_block::classifier::{Classifier as ClassifierEncodable, ClassifierError as ClassifierEncodableError},
engine::Engine,
safetensors::{ParameterLoader, ParameterLoaderError},
utils::file,
};
pub struct ClassifierModel<B: Backend> {
context: Arc<B::Context>,
classifier: ClassifierEncodable<B>,
output_labels: Box<[String]>,
data_type: DataType,
tokenizer: Arc<Tokenizer>,
token_codec_config: AnyTokenCodecConfig,
}
#[derive(Debug, Error)]
pub enum EngineLoadClassifierError<B: Backend> {
#[error("I/O error: {0}")]
IO(#[from] io::Error),
#[error("Serde error: {0}")]
Serde(#[from] serde_json::Error),
#[error("ParameterLoader error: {0}")]
ParameterLoader(#[from] ParameterLoaderError<B>),
#[error("Classifier error: {0}")]
Classifier(#[from] ClassifierEncodableError<B>),
#[error("Tokenizer error: {0}")]
Tokenizer(#[from] tokenizers::Error),
}
impl<B: Backend> Engine<B> {
pub async fn load_classifier_model(
&self,
model_path: &Path,
) -> Result<ClassifierModel<B>, EngineLoadClassifierError<B>> {
let context = self.context.clone();
let config: ClassifierModelConfig = serde_json::from_slice(&file::read(model_path.join("config.json")).await?)?;
let data_type = DataType::BF16;
let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), context.as_ref()).await?;
let classifier = ClassifierEncodable::new(
data_type,
&config.classifier_config,
&weight_loader.tree().subtree("classifier"),
context.as_ref(),
)
.await?;
let output_labels = if let Some(output_labels) = config.classifier_config.output_labels {
assert!(output_labels.len() as u32 == config.classifier_config.num_labels);
output_labels
} else {
(0..config.classifier_config.num_labels).map(|index| format!("class_{index}")).collect()
};
weight_loader.assert_all_tensors_read()?;
let tokenizer = Arc::new(Tokenizer::from_bytes(file::read(model_path.join("tokenizer.json")).await?)?);
let token_codec_config = config.token_codec_config;
Ok(ClassifierModel {
context,
classifier,
output_labels,
data_type,
tokenizer,
token_codec_config,
})
}
}
#[derive(Debug, Error)]
pub enum ClassifierModelClassifyError<B: Backend> {
#[error("Backend error: {0}")]
Backend(#[source] B::Error),
#[error("Classifier error: {0}")]
Classifier(#[from] ClassifierEncodableError<B>),
#[error("Input is empty")]
EmptyInput,
#[error("Input larger than model context size")]
ContextOverflow,
}
impl<B: Backend> ClassifierModel<B> {
pub fn tokenizer(&self) -> &Arc<Tokenizer> {
&self.tokenizer
}
pub fn token_codec_config(&self) -> TokenCodecConfig {
TokenCodecConfig::from(&self.token_codec_config)
}
pub async fn classify(
&self,
input: &[u64],
) -> Result<ClassifierOutput, ClassifierModelClassifyError<B>> {
if input.is_empty() {
return Err(ClassifierModelClassifyError::EmptyInput);
}
if self
.classifier
.max_context_length()
.is_some_and(|max_context_length| input.len() > max_context_length as usize)
{
return Err(ClassifierModelClassifyError::ContextOverflow);
}
let mut command_buffer =
self.context.create_command_buffer(None, None).map_err(ClassifierModelClassifyError::Backend)?;
let token_ids = command_buffer
.allocate_constant_from_slice(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>())
.map_err(ClassifierModelClassifyError::Backend)?;
let logits = self.classifier.encode(&token_ids, input.len() as u32, &mut command_buffer)?;
let mut output_buffer = self
.context
.create_buffer_gpu(self.output_labels.len() * self.data_type.size_in_bytes())
.map_err(ClassifierModelClassifyError::Backend)?;
command_buffer.encode_copy(&logits, &mut output_buffer);
drop(logits);
drop(token_ids);
command_buffer
.end_encoding()
.submit()
.wait_until_completed()
.await
.map_err(ClassifierModelClassifyError::Backend)?;
assert!(self.data_type == DataType::BF16);
let logits_bf16 =
self.context.copyout::<bf16>(&output_buffer).await.map_err(ClassifierModelClassifyError::Backend)?;
let count = min(self.output_labels.len(), logits_bf16.len());
let mut logits: Vec<f32> = vec![0.0f32; count];
let mut probabilities = HashMap::new();
for i in 0..count {
logits[i] = logits_bf16[i].to_f32();
let prob = 1.0 / (1.0 + (-logits[i]).exp());
probabilities.insert(self.output_labels[i].clone(), prob);
}
Ok(ClassifierOutput {
logits,
probabilities,
})
}
}