A high-performance inference engine for AI models
CLI: prefill step size
Aleksei Savin01acdb74c412
diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rsindex 4f0ed45a9..21882966a 100644--- a/crates/cli/src/handlers/run.rs+++ b/crates/cli/src/handlers/run.rs@@ -36,8 +36,9 @@ fn format_output(output: Output) -> String {pub fn handle_run(model_path: String,tokens_limit: usize,+ prefill_step_size: Option<usize>,) {- let mut session = load_session(model_path);+ let mut session = load_session(model_path, prefill_step_size);let is_model_running = Arc::new(AtomicBool::new(false));let is_model_running_for_ctrlc = is_model_running.clone();diff --git a/crates/cli/src/handlers/serve.rs b/crates/cli/src/handlers/serve.rsindex 4e71ef49b..23a4ebcff 100644--- a/crates/cli/src/handlers/serve.rs+++ b/crates/cli/src/handlers/serve.rs@@ -2,7 +2,10 @@ use tokio::runtime::Runtime;use crate::server::main::run_server;-pub fn handle_serve(model_path: String) {+pub fn handle_serve(+ model_path: String,+ prefill_step_size: Option<usize>,+) {let runtime = Runtime::new().unwrap();- runtime.block_on(run_server(model_path));+ runtime.block_on(run_server(model_path, prefill_step_size));}diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rsindex 6ce5f1a19..9e74854ed 100644--- a/crates/cli/src/main.rs+++ b/crates/cli/src/main.rs@@ -13,11 +13,15 @@ enum Commands {Run {/// Folder with model's filesmodel_path: String,+ /// Prefill step size+ prefill_step_size: Option<usize>,},/// Start a server with the specified model pathServe {/// Folder with model's filesmodel_path: String,+ /// Prefill step size+ prefill_step_size: Option<usize>,},}@@ -27,13 +31,15 @@ fn main() {match cli.command {Some(Commands::Run {model_path,+ prefill_step_size,}) => {- handle_run(model_path, 2048);+ handle_run(model_path, 2048, prefill_step_size);},Some(Commands::Serve {model_path,+ prefill_step_size,}) => {- handle_serve(model_path);+ handle_serve(model_path, prefill_step_size);},None => {let mut cmd = Cli::command();diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rsindex 356109563..f0d46c414 100644--- a/crates/cli/src/server/main.rs+++ b/crates/cli/src/server/main.rs@@ -25,7 +25,10 @@ impl log::Log for SilentLogger {}// --------------------------------------------------------------------------------pub async fn run_server(model_path: String) {+pub async fn run_server(+ model_path: String,+ prefill_step_size: Option<usize>,+) {// Install the silent logger **before** Rocket initializes its own logger.let _ = log::set_logger(&SILENT_LOGGER).map(|_| log::set_max_level(LevelFilter::Off));@@ -53,7 +56,7 @@ pub async fn run_server(model_path: String) {"📝 Endpoints:\n POST /chat/completions - Chat completions API\n");- let session = load_session(model_path);+ let session = load_session(model_path, prefill_step_size);let state = SessionState {model_name,session_wrapper: SessionWrapper::new(session),diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rsindex b9b5baafc..c31980ef7 100644--- a/crates/cli/src/server/state.rs+++ b/crates/cli/src/server/state.rs@@ -2,7 +2,9 @@ use std::{path::PathBuf, sync::Mutex};use console::Style;use indicatif::{ProgressBar, ProgressStyle};-use uzu::session::{Session, config::DecodingConfig};+use uzu::session::{+ Session, config::DecodingConfig, parameter::PrefillStepSize,+};pub struct SessionWrapper(Mutex<Session>);unsafe impl Send for SessionWrapper {}@@ -25,7 +27,10 @@ pub struct SessionState {unsafe impl Send for SessionState {}unsafe impl Sync for SessionState {}-pub fn load_session(model_path: String) -> Session {+pub fn load_session(+ model_path: String,+ prefill_step_size: Option<usize>,+) -> Session {let style_bold = Style::new().bold();let model_path_buf = PathBuf::from(model_path);@@ -44,7 +49,16 @@ pub fn load_session(model_path: String) -> Session {);progress_bar.set_message(model_name.clone());- let session = Session::new(model_path_buf, DecodingConfig::default())+ let prefill_step_size_config: PrefillStepSize;+ if let Some(value) = prefill_step_size {+ prefill_step_size_config = PrefillStepSize::Custom(value);+ } else {+ prefill_step_size_config = PrefillStepSize::Default;+ }++ let decoding_config = DecodingConfig::default()+ .with_prefill_step_size(prefill_step_size_config);+ let session = Session::new(model_path_buf, decoding_config).expect("Failed to create session");progress_bar.set_style(