Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

5.8 KB
Raw Blame
use std::{io, path::Path, sync::Arc};

use derive_more::Debug;
use thiserror::Error;
use tokenizers::Tokenizer;

use crate::{
    backends::common::{Backend, Context, DeviceCapabilities, Kernels, kernel::ContextRingUpdateKernel},
    config::model::language_model::LanguageModelConfig,
    data_type::DataType,
    encodable_block::{
        decoder::{Decoder, DecoderError},
        sampling::{Sampling, SamplingMethod},
    },
    engine::Engine,
    safetensors::{ParameterLoader, ParameterLoaderError},
    speculators::dflash_tfm::{DFlashSpeculatorLoadError, DFlashTfmSpeculator},
    utils::file,
};
pub use crate::{config::model::generation::GenerationConfig, utils::strict_serde::Unsupported};

pub mod state;
pub mod stream;

pub mod grammar;

pub struct LanguageModel<B: Backend> {
    engine: Arc<Engine<B>>,
    decoder: Decoder<B>,
    speculator: Option<DFlashTfmSpeculator<B>>,
    sampling: Sampling<B>,
    context_ring_update: <B::Kernels as Kernels>::ContextRingUpdateKernel,
    generation_config: GenerationConfig,
    tokenizer: Arc<Tokenizer>,
    vocab_size: usize,
}

#[derive(Debug, Error)]
pub enum EngineLoadLanguageModelError<B: Backend> {
    #[error("I/O error: {0}")]
    IO(#[from] io::Error),
    #[error("Serde error: {0}")]
    Serde(#[from] serde_json::Error),
    #[error("ParameterLoader error: {0}")]
    ParameterLoader(#[from] ParameterLoaderError<B>),
    #[error("Backend error: {0}")]
    Backend(#[source] B::Error),
    #[error("Decoder error: {0}")]
    Decoder(#[from] DecoderError<B>),
    #[error("Speculator error: {0}")]
    Speculator(#[from] DFlashSpeculatorLoadError<B>),
    #[error("Tokenizer error: {0}")]
    Tokenizer(#[from] tokenizers::Error),
}

impl<B: Backend> Engine<B> {
    pub async fn load_language_model(
        self: &Arc<Self>,
        model_path: &Path,
    ) -> Result<LanguageModel<B>, EngineLoadLanguageModelError<B>> {
        let config: LanguageModelConfig = serde_json::from_slice(&file::read(model_path.join("config.json")).await?)?;

        let weight_loader = ParameterLoader::new(model_path.join("model.safetensors"), &*self.context).await?;

        // TODO
        let speculator_path = model_path.join("speculator");
        let speculator_path = file::exists(&speculator_path).await.then_some(speculator_path);

        let tokenizer = Arc::new(Tokenizer::from_bytes(file::read(model_path.join("tokenizer.json")).await?)?);

        let data_type = DataType::BF16;

        let decoder = Decoder::new(
            data_type,
            &config.decoder_config,
            &weight_loader.tree().subtree("decoder"),
            self.context.as_ref(),
        )
        .await?;

        assert!(
            speculator_path.is_none() || decoder.speculation_supported(),
            "attempted to load speculator for a model that doesn't support one"
        );

        let speculator = if let Some(speculator_path) = speculator_path.as_deref() {
            DFlashTfmSpeculator::new(speculator_path, decoder.embedding(), self.context.clone()).await?
        } else {
            None
        };

        let sampling = Sampling::new(config.decoder_config.vocab_size, data_type);

        let context_ring_update = <B::Kernels as Kernels>::ContextRingUpdateKernel::new(&self.context)
            .map_err(EngineLoadLanguageModelError::Backend)?;

        weight_loader.assert_all_tensors_read()?;

        let generation_config = config.generation_config;
        let vocab_size = config.decoder_config.vocab_size as usize;

        Ok(LanguageModel {
            engine: self.clone(),
            decoder,
            speculator,
            sampling,
            context_ring_update,
            generation_config,
            tokenizer,
            vocab_size,
        })
    }
}

impl<B: Backend> LanguageModel<B> {
    pub fn max_context_length(&self) -> Option<u32> {
        self.decoder.max_context_length()
    }

    pub fn recommended_context_length(&self) -> Option<u32> {
        let max_context_length = self.max_context_length();

        // TODO: This is not the correct way to do it, there should be a real memory model
        if self.engine.context.device_capabilities().contains(DeviceCapabilities::SPARSE_BUFFERS) {
            // We just assume that all mixers use sparse if it's available to make max context free until it's actually used
            // Currenlty true for all mixers in uzu:
            // - full attention uses sparse if it's available to make max context free until it's actually used
            // - sliding window attention is bound, usually well below the recommended max context size on non-sparse (but can be made to use sparse if we care about it enough)
            // - short conv/delta net are constant state size
            max_context_length
        } else if let Some(max_context_length) = max_context_length {
            // If sparse buffers aren't supported and model has finite maximum context length we assume that kv cache is expensive enough that we should probably clamp it to
            // something reasonable-ish for the platform. This is very primitive but works I guess...
            let platform_recommended_context_length = if cfg!(target_os = "ios") {
                8192
            } else {
                16384
            };

            Some(u32::min(max_context_length, platform_recommended_context_length))
        } else {
            // We just assume that unlimited context means constant state size on all mixers and is thus free
            None
        }
    }

    pub fn speculation_supported(&self) -> bool {
        self.decoder.speculation_supported()
    }

    pub fn default_sampling_method(&self) -> SamplingMethod {
        self.generation_config.default_sampling_method()
    }

    pub fn generation_config(&self) -> &GenerationConfig {
        &self.generation_config
    }

    pub fn tokenizer(&self) -> &Arc<Tokenizer> {
        &self.tokenizer
    }
}