Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

5.6 KB
Raw Blame
use std::{io, path::Path, sync::Arc, time::Instant};

use shoji::{traits::backend::decision::DecisionSessionOutput, types::session::classification::TokenCodecConfig};
use thiserror::Error;
use tokenizers::Tokenizer;

use crate::{
    backends::common::{Backend, CommandBufferEncoding, CommandBufferExecutable, CommandBufferPending, Context},
    config::{
        model::decision_model::{DecisionConfig, DecisionModelConfig},
        token_codec::AnyTokenCodecConfig,
    },
    data_type::DataType,
    encodable_block::decision::{Decision, DecisionError},
    engine::Engine,
    safetensors::{ParameterLoader, ParameterLoaderError},
    utils::file,
};

#[derive(Debug, Error)]
pub enum DecisionModelError<B: Backend> {
    #[error("Backend error: {0}")]
    Backend(#[source] B::Error),
    #[error("Input larger than model context size")]
    ContextOverflow,
    #[error("Decision error: {0}")]
    Decision(#[from] DecisionError<B>),
    #[error("Input is empty")]
    EmptyInput,
    #[error("Option count {option_count} is outside 1..={answer_count}")]
    InvalidOptionCount {
        option_count: usize,
        answer_count: usize,
    },
    #[error("I/O error: {0}")]
    IO(#[from] io::Error),
    #[error("ParameterLoader error: {0}")]
    ParameterLoader(#[from] ParameterLoaderError<B>),
    #[error("Serde error: {0}")]
    Serde(#[from] serde_json::Error),
    #[error("Tokenizer error: {0}")]
    Tokenizer(#[from] tokenizers::Error),
}

pub struct DecisionModel<B: Backend> {
    context: Arc<B::Context>,
    decision: Decision<B>,
    decision_config: DecisionConfig,
    tokenizer: Arc<Tokenizer>,
    token_codec_config: AnyTokenCodecConfig,
}

impl<B: Backend> DecisionModel<B> {
    pub fn decision_config(&self) -> &DecisionConfig {
        &self.decision_config
    }

    pub fn tokenizer(&self) -> &Arc<Tokenizer> {
        &self.tokenizer
    }

    pub fn token_codec_config(&self) -> TokenCodecConfig {
        TokenCodecConfig::from(&self.token_codec_config)
    }

    pub async fn probabilities(
        &self,
        input: &[u64],
        option_count: usize,
    ) -> Result<DecisionSessionOutput, DecisionModelError<B>> {
        if input.is_empty() {
            return Err(DecisionModelError::EmptyInput);
        }

        if self
            .decision
            .max_context_length()
            .is_some_and(|max_context_length| input.len() > max_context_length as usize)
        {
            return Err(DecisionModelError::ContextOverflow);
        }

        let answer_count = self.decision_config.codes.len();
        if !(1..=answer_count).contains(&option_count) {
            return Err(DecisionModelError::InvalidOptionCount {
                option_count,
                answer_count,
            });
        }

        let prefill_start = Instant::now();

        let mut command_buffer = self.context.create_command_buffer(None, None).map_err(DecisionModelError::Backend)?;
        let token_ids = command_buffer
            .allocate_constant_from_slice(&input.iter().map(|token_id| *token_id as u32).collect::<Box<[u32]>>())
            .map_err(DecisionModelError::Backend)?;

        let logits = self.decision.encode(&token_ids, input.len() as u32, &mut command_buffer)?;
        let mut output_buffer = self
            .context
            .create_buffer_gpu(answer_count * DataType::F32.size_in_bytes())
            .map_err(DecisionModelError::Backend)?;
        command_buffer.encode_copy(&logits, &mut output_buffer);
        drop(logits);
        drop(token_ids);

        command_buffer.end_encoding().submit().wait_until_completed().await.map_err(DecisionModelError::Backend)?;

        let prefill_duration = prefill_start.elapsed();

        let temperature = self.decision_config.temperature;
        let logits = self.context.copyout::<f32>(&output_buffer).await.map_err(DecisionModelError::Backend)?;
        let scaled = logits[..option_count].iter().map(|logit| logit / temperature).collect::<Vec<_>>();
        let max = scaled.iter().copied().fold(f32::NEG_INFINITY, f32::max);
        let exps = scaled.iter().map(|logit| (logit - max).exp()).collect::<Vec<_>>();
        let sum = exps.iter().sum::<f32>();
        let probabilities = exps.into_iter().map(|value| value / sum).collect();
        Ok(DecisionSessionOutput {
            probabilities,
            prefill_duration,
            memory_used_bytes: self.context.peak_memory_usage(),
        })
    }
}

impl<B: Backend> Engine<B> {
    pub async fn load_decision_model(
        &self,
        model_path: &Path,
    ) -> Result<DecisionModel<B>, DecisionModelError<B>> {
        let context = self.context.clone();

        let config_file_path = model_path.join("config.json");
        let config_file_content = file::read(config_file_path).await?;
        let config = serde_json::from_slice::<DecisionModelConfig>(&config_file_content)?;

        let tokenizer_file_path = model_path.join("tokenizer.json");
        let tokenizer_file_content = file::read(tokenizer_file_path).await?;
        let tokenizer = Tokenizer::from_bytes(tokenizer_file_content)?;

        let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), context.as_ref()).await?;
        let decision = Decision::new(
            DataType::BF16,
            DataType::F32,
            &config,
            &weight_loader.tree().subtree("decoder"),
            context.as_ref(),
        )
        .await?;
        weight_loader.assert_all_tensors_read()?;

        Ok(DecisionModel {
            context,
            decision,
            decision_config: config.decision,
            tokenizer: Arc::new(tokenizer),
            token_codec_config: config.token_codec_config,
        })
    }
}