Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

1.9 KB
Raw Blame
use derive_more::Debug;
use thiserror::Error;

use crate::{
    backends::common::Backend,
    encodable_block::{dflash::DFlashState, sampling::PRng, transformer::TransformerState},
    engine::language_model::LanguageModel,
};

pub struct LanguageModelState<B: Backend> {
    pub(super) tokens: Vec<u64>,
    pub(super) prng: PRng,
    pub(super) transformer_state: TransformerState<B>,
    pub(super) speculator_state: Option<DFlashState<B>>,
    pub(super) max_context_length: Option<u32>,
}

impl<B: Backend> LanguageModelState<B> {
    pub fn tokens(&self) -> &[u64] {
        &self.tokens
    }

    pub fn set_sampling_seed(
        &mut self,
        sampling_seed: u64,
    ) {
        self.prng = PRng::new(sampling_seed);
    }
}

#[derive(Debug, Error)]
pub enum LanguageModelCreateEmptyStateError<B: Backend> {
    #[error("Backend error: {0}")]
    Backend(#[source] B::Error),
}

impl<B: Backend> LanguageModel<B> {
    pub fn create_empty_state(
        &self,
        max_context_length: Option<u32>,
        sampling_seed: u64,
    ) -> Result<LanguageModelState<B>, LanguageModelCreateEmptyStateError<B>> {
        let tokens = Vec::new();

        let prng = PRng::new(sampling_seed);

        let transformer_state = self
            .decoder
            .create_empty_state(max_context_length, &self.engine.context)
            .map_err(LanguageModelCreateEmptyStateError::Backend)?;

        let speculator_state = self
            .speculator
            .as_ref()
            .map(|speculator| {
                speculator.empty_state(max_context_length.expect("speculator doesn't support unlimited state capacity"))
            })
            .transpose()
            .map_err(LanguageModelCreateEmptyStateError::Backend)?;

        Ok(LanguageModelState {
            tokens,
            prng,
            transformer_state,
            speculator_state,
            max_context_length,
        })
    }
}