Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

3.8 KB
Raw Blame
use anyhow::Result;
use clap::{Parser, Subcommand};
use shoji::types::basic::ReasoningEffort;

mod bench;
mod common;
mod interactive;
mod server;
mod storage;

#[derive(Parser)]
#[command(name = "cli", bin_name = "cli", version, args_conflicts_with_subcommands = true)]
struct Cli {
    /// Model identifier, repository ID, or local model directory.
    #[arg(long, value_name = "MODEL")]
    model: Option<String>,
    /// Send one chat message, stream the reply, and exit.
    #[arg(short, long, value_name = "TEXT", requires = "model")]
    message: Option<String>,
    /// Reasoning effort: disabled, default, low, medium or high.
    /// Overrides the saved preference for this run only; never persisted.
    #[arg(long, value_name = "EFFORT")]
    reasoning_effort: Option<ReasoningEffort>,
    /// Sampling seed for chat sessions. In server mode, the seed for requests that don't set `seed`.
    #[arg(long, value_name = "SEED", allow_negative_numbers = true, global = true)]
    seed: Option<i64>,
    /// Disable built-in tools in interactive chat sessions.
    #[arg(long)]
    no_tools: bool,
    #[command(subcommand)]
    command: Option<Commands>,
}

#[derive(Subcommand)]
enum Commands {
    Bench {
        model_path: String,
        task_path: String,
        output_path: String,
    },
    ListCheckpoints {
        /// Model ID shown by `list-models`.
        #[arg(value_name = "MODEL_ID")]
        model_id: String,
    },
    ListModels,
    Server {
        /// Model identifier, repository ID, or local model directory.
        #[arg(long, value_name = "MODEL")]
        model: String,
        #[arg(long, default_value_t = 8000)]
        port: u16,
        #[arg(long, default_value = "127.0.0.1")]
        host: String,
        /// Reuse the previous request's state when the new prompt extends it (enabled by default).
        #[arg(long = "prefix-cache", action = clap::ArgAction::SetTrue, conflicts_with = "no_prefix_cache")]
        prefix_cache: bool,
        /// Reset the session on every request, disabling prefix cache reuse.
        #[arg(long = "no-prefix-cache", action = clap::ArgAction::SetTrue)]
        no_prefix_cache: bool,
        /// Print verbose information into the file log
        #[arg(long)]
        verbose_file_log: bool,
    },
    Storage {
        #[arg(long, value_enum, default_value_t = storage::DownloadManagerCliType::default())]
        download_manager: storage::DownloadManagerCliType,
    },
}

#[tokio::main]
async fn main() -> Result<()> {
    let cli = Cli::parse();

    match cli.command {
        Some(Commands::Bench {
            model_path,
            task_path,
            output_path,
        }) => bench::run_bench(model_path, task_path, output_path).await?,
        Some(Commands::ListCheckpoints {
            model_id,
        }) => interactive::run_list_checkpoints(model_id).await?,
        Some(Commands::ListModels) => interactive::run_list_models().await?,
        Some(Commands::Server {
            model,
            port,
            host,
            prefix_cache,
            no_prefix_cache,
            verbose_file_log,
        }) => {
            server::run_server(model, host, port, prefix_cache || !no_prefix_cache, verbose_file_log, cli.seed).await?
        },
        Some(Commands::Storage {
            download_manager,
        }) => storage::run(download_manager).await?,
        None => match cli.message {
            Some(message) => {
                interactive::run_non_interactive(
                    cli.model.expect("--message requires --model"),
                    message,
                    cli.reasoning_effort,
                    cli.seed,
                )
                .await?;
            },
            None => interactive::run_interactive(cli.model, cli.reasoning_effort, cli.seed, cli.no_tools).await?,
        },
    }

    Ok(())
}