Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

5.3 KB
Raw Blame
use std::{net::IpAddr, path::PathBuf, sync::Arc, time::Duration};

use anyhow::{Context, Result, bail};
use indicatif::{ProgressBar, ProgressStyle};
use rocket::{Config, config::LogLevel, routes};
use tokio::sync::Mutex;
use uzu::{
    engine::{Engine, EngineConfig},
    types::{basic::SamplingSeed, model::ModelSpecialization, session::chat::ChatConfig},
};

use crate::{
    common::thinking::ThinkingSupport,
    server::{
        ChatState, DecisionState, ServerState, handle_chat_completions, handle_decisions, handle_models,
        handle_system_one,
        log::{self, FileLogger, RequestLoggingFairing},
    },
};

pub async fn run_server(
    model: String,
    host: String,
    port: u16,
    prefix_cache: bool,
    verbose_file_logs: bool,
    seed: Option<i64>,
) -> Result<()> {
    let file_logger = if verbose_file_logs {
        let cache_dir = std::env::var_os("XDG_CACHE_HOME")
            .map(PathBuf::from)
            .filter(|path| path.is_absolute())
            .or_else(|| dirs::home_dir().map(|path| path.join(".cache")))
            .context("Failed to resolve cache directory")?;
        let logs_dir_path = cache_dir.join("mirai").join("server").join("logs");
        Some(FileLogger::new(logs_dir_path)?)
    } else {
        None
    };
    log::init(tracing::Level::INFO, file_logger.clone(), tracing::Level::DEBUG)?;

    let engine_config = EngineConfig::default();
    let engine = Engine::new(engine_config).await.context("Failed to create engine")?;

    let resolved = engine.model(model.clone()).await?.with_context(|| format!("Model not found: {model}"))?;

    let spinner = ProgressBar::new_spinner();
    spinner.enable_steady_tick(Duration::from_millis(100));
    spinner.set_style(ProgressStyle::default_spinner().template("{spinner:.green} {msg}").unwrap());
    spinner.set_message(format!("Preparing model: {}", resolved.identifier));
    let downloader = engine.download(&resolved).await.context("Failed to start model download")?;
    while let Some(update) = downloader.next().await {
        spinner.set_message(format!("Downloading {}: {:.0}%", resolved.identifier, update.progress() * 100.0));
    }
    spinner.finish_with_message(format!("Loaded: {}", resolved.identifier));

    let model_name = resolved.identifier.clone();
    let address: IpAddr = host.parse().with_context(|| format!("Invalid host: {host}"))?;
    let config = Config {
        port,
        address,
        log_level: LogLevel::Off,
        ..Config::default()
    };

    let rocket = rocket::custom(config)
        .manage(ServerState {
            model_name: model_name.clone(),
        })
        .attach(RequestLoggingFairing);

    let (rocket, title, details) = if resolved.specializations.contains(&ModelSpecialization::Decision {}) {
        let session = engine.decision(resolved).await.context("Failed to create decision session")?;
        let state = DecisionState {
            session: Arc::new(Mutex::new(session)),
        };
        let rocket = rocket
            .manage(state)
            .mount("/", routes![handle_system_one, handle_decisions, handle_models])
            .mount("/v1", routes![handle_system_one, handle_decisions, handle_models]);
        let details = concat!(
            "πŸ“ Endpoints:\n",
            "   POST /v1/systemone        (or /v1/decisions, /systemone, /decisions)\n",
            "        Docs: https://github.com/trymirai/uzu#decision-models\n",
        );
        (rocket, "System One server", details.to_string())
    } else {
        let thinking_support = ThinkingSupport::for_model(&resolved);
        let chat_config = seed.map_or_else(ChatConfig::default, |seed| {
            ChatConfig::default().with_sampling_seed(SamplingSeed::Custom {
                seed,
            })
        });
        let session = engine.chat(resolved, chat_config).await.context("Failed to create chat session")?;
        let state = ChatState {
            session: Arc::new(Mutex::new(session)),
            thinking_support,
            prefix_cache,
        };
        let rocket = rocket
            .manage(state)
            .mount("/", routes![handle_chat_completions, handle_models])
            .mount("/v1", routes![handle_chat_completions, handle_models]);
        let prefix_cache_enabled = if prefix_cache {
            "enabled"
        } else {
            "disabled"
        };
        let details = format!(
            concat!(
                "πŸ—„οΈ  Prefix cache: {prefix_cache_enabled}\n",
                "πŸ“ Endpoints:\n",
                "   POST /v1/chat/completions (or /chat/completions)\n",
            ),
            prefix_cache_enabled = prefix_cache_enabled,
        );
        (rocket, "OpenAI-compatible server", details)
    };

    if let Some(ref file_logger) = file_logger {
        tracing::info!("πŸ—ƒοΈ  Logs will be written to file: {}", file_logger.file_path().display())
    }
    tracing::info!(
        concat!(
            "πŸš€ {title} v{version} for model: {model_name}\n",
            "🌐 Available at: http://{host}:{port}\n",
            "{details}",
            "   GET  /v1/models           (or /models)\n",
        ),
        title = title,
        version = Engine::version(),
        model_name = model_name,
        host = host,
        port = port,
        details = details,
    );

    if let Err(error) = rocket.launch().await {
        bail!("Server error: {error}");
    }

    Ok(())
}