Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

3.4 KB
Raw Blame
use std::{path::Path, sync::Arc};

use derive_more::Debug;
use thiserror::Error;

use crate::{
    backends::common::{
        Backend, Context,
        capture_matcher::{CAPTURE_MATCHER, CaptureError, CaptureMatcherSpan},
    },
    config::{model::AnyModelConfig, token_mixer::AnyTokenMixerConfig},
    encodable_block::sampling::SamplingMethod,
};

pub mod classifier_model;
pub mod decision_model;
pub mod language_model;

pub struct Engine<B: Backend> {
    _capture_span: Option<CaptureMatcherSpan>,
    context: Arc<B::Context>,
}

#[derive(Debug, Error)]
pub enum EngineNewError<B: Backend> {
    #[error("Backend error: {0}")]
    Backend(#[source] B::Error),
    #[error("Capture error: {0}")]
    Capture(#[from] CaptureError<B>),
}

impl<B: Backend> Engine<B> {
    pub async fn new() -> Result<Arc<Self>, EngineNewError<B>> {
        let context = <B::Context as Context>::new().await.map_err(EngineNewError::Backend)?;
        let capture_span = CAPTURE_MATCHER.capture::<B>(&context, "")?;

        Ok(Arc::new(Self {
            _capture_span: capture_span,
            context,
        }))
    }

    pub fn peak_memory_usage(&self) -> Option<usize> {
        self.context.peak_memory_usage()
    }
}

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ModelType {
    LanguageModel,
    Classifier,
    Decision,
}

#[derive(Debug, Error)]
pub enum ResolveModelTypeError {
    #[error("Unable to open model configuration: {0}")]
    UnableToOpenConfig(#[from] std::io::Error),
    #[error("Unable to deserialize model configuration: {0}")]
    UnableToDeserializeConfig(#[from] serde_json::Error),
}

pub async fn resolve_model_type(model_path: &Path) -> Result<ModelType, ResolveModelTypeError> {
    Ok(match read_model_config(model_path).await? {
        AnyModelConfig::LanguageModelConfig(_) => ModelType::LanguageModel,
        AnyModelConfig::ClassifierModelConfig(_) => ModelType::Classifier,
        AnyModelConfig::DecisionModelConfig(_) => ModelType::Decision,
    })
}

pub async fn resolve_model_sampling_method(model_path: &Path) -> Result<Option<SamplingMethod>, ResolveModelTypeError> {
    Ok(match read_model_config(model_path).await? {
        AnyModelConfig::LanguageModelConfig(config) => Some(config.generation_config.default_sampling_method()),
        AnyModelConfig::ClassifierModelConfig(_) => None,
        AnyModelConfig::DecisionModelConfig(config) => Some(config.generation_config.default_sampling_method()),
    })
}

pub async fn resolve_model_context_length(model_path: &Path) -> Result<Option<u32>, ResolveModelTypeError> {
    let transformer = match read_model_config(model_path).await? {
        AnyModelConfig::LanguageModelConfig(config) => config.decoder_config.transformer_config,
        AnyModelConfig::ClassifierModelConfig(_) => return Ok(None),
        AnyModelConfig::DecisionModelConfig(config) => config.decoder_config.transformer_config,
    };
    Ok(transformer
        .layer_configs
        .iter()
        .filter(|layer| {
            matches!(
                layer.mixer.body_config,
                AnyTokenMixerConfig::AttentionConfig(_) | AnyTokenMixerConfig::SparseAttentionConfig(_)
            )
        })
        .filter_map(|layer| layer.rope_config.as_ref().map(|rope| *rope.max_sequence_length()))
        .min())
}

async fn read_model_config(model_path: &Path) -> Result<AnyModelConfig, ResolveModelTypeError> {
    let bytes = crate::utils::file::read(model_path.join("config.json")).await?;
    Ok(serde_json::from_slice(&bytes)?)
}