Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

CLI: prefill step size

Aleksei Savin01acdb74c412
diff --git a/crates/cli/src/handlers/run.rs b/crates/cli/src/handlers/run.rs
index 4f0ed45a9..21882966a 100644
--- a/crates/cli/src/handlers/run.rs
+++ b/crates/cli/src/handlers/run.rs
@@ -36,8 +36,9 @@ fn format_output(output: Output) -> String {
pub fn handle_run(
model_path: String,
tokens_limit: usize,
+ prefill_step_size: Option<usize>,
) {
- let mut session = load_session(model_path);
+ let mut session = load_session(model_path, prefill_step_size);
let is_model_running = Arc::new(AtomicBool::new(false));
let is_model_running_for_ctrlc = is_model_running.clone();
diff --git a/crates/cli/src/handlers/serve.rs b/crates/cli/src/handlers/serve.rs
index 4e71ef49b..23a4ebcff 100644
--- a/crates/cli/src/handlers/serve.rs
+++ b/crates/cli/src/handlers/serve.rs
@@ -2,7 +2,10 @@ use tokio::runtime::Runtime;
use crate::server::main::run_server;
-pub fn handle_serve(model_path: String) {
+pub fn handle_serve(
+ model_path: String,
+ prefill_step_size: Option<usize>,
+) {
let runtime = Runtime::new().unwrap();
- runtime.block_on(run_server(model_path));
+ runtime.block_on(run_server(model_path, prefill_step_size));
}
diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs
index 6ce5f1a19..9e74854ed 100644
--- a/crates/cli/src/main.rs
+++ b/crates/cli/src/main.rs
@@ -13,11 +13,15 @@ enum Commands {
Run {
/// Folder with model's files
model_path: String,
+ /// Prefill step size
+ prefill_step_size: Option<usize>,
},
/// Start a server with the specified model path
Serve {
/// Folder with model's files
model_path: String,
+ /// Prefill step size
+ prefill_step_size: Option<usize>,
},
}
@@ -27,13 +31,15 @@ fn main() {
match cli.command {
Some(Commands::Run {
model_path,
+ prefill_step_size,
}) => {
- handle_run(model_path, 2048);
+ handle_run(model_path, 2048, prefill_step_size);
},
Some(Commands::Serve {
model_path,
+ prefill_step_size,
}) => {
- handle_serve(model_path);
+ handle_serve(model_path, prefill_step_size);
},
None => {
let mut cmd = Cli::command();
diff --git a/crates/cli/src/server/main.rs b/crates/cli/src/server/main.rs
index 356109563..f0d46c414 100644
--- a/crates/cli/src/server/main.rs
+++ b/crates/cli/src/server/main.rs
@@ -25,7 +25,10 @@ impl log::Log for SilentLogger {
}
// -------------------------------------------------------------------------------
-pub async fn run_server(model_path: String) {
+pub async fn run_server(
+ model_path: String,
+ prefill_step_size: Option<usize>,
+) {
// Install the silent logger **before** Rocket initializes its own logger.
let _ = log::set_logger(&SILENT_LOGGER)
.map(|_| log::set_max_level(LevelFilter::Off));
@@ -53,7 +56,7 @@ pub async fn run_server(model_path: String) {
"📝 Endpoints:\n POST /chat/completions - Chat completions API\n"
);
- let session = load_session(model_path);
+ let session = load_session(model_path, prefill_step_size);
let state = SessionState {
model_name,
session_wrapper: SessionWrapper::new(session),
diff --git a/crates/cli/src/server/state.rs b/crates/cli/src/server/state.rs
index b9b5baafc..c31980ef7 100644
--- a/crates/cli/src/server/state.rs
+++ b/crates/cli/src/server/state.rs
@@ -2,7 +2,9 @@ use std::{path::PathBuf, sync::Mutex};
use console::Style;
use indicatif::{ProgressBar, ProgressStyle};
-use uzu::session::{Session, config::DecodingConfig};
+use uzu::session::{
+ Session, config::DecodingConfig, parameter::PrefillStepSize,
+};
pub struct SessionWrapper(Mutex<Session>);
unsafe impl Send for SessionWrapper {}
@@ -25,7 +27,10 @@ pub struct SessionState {
unsafe impl Send for SessionState {}
unsafe impl Sync for SessionState {}
-pub fn load_session(model_path: String) -> Session {
+pub fn load_session(
+ model_path: String,
+ prefill_step_size: Option<usize>,
+) -> Session {
let style_bold = Style::new().bold();
let model_path_buf = PathBuf::from(model_path);
@@ -44,7 +49,16 @@ pub fn load_session(model_path: String) -> Session {
);
progress_bar.set_message(model_name.clone());
- let session = Session::new(model_path_buf, DecodingConfig::default())
+ let prefill_step_size_config: PrefillStepSize;
+ if let Some(value) = prefill_step_size {
+ prefill_step_size_config = PrefillStepSize::Custom(value);
+ } else {
+ prefill_step_size_config = PrefillStepSize::Default;
+ }
+
+ let decoding_config = DecodingConfig::default()
+ .with_prefill_step_size(prefill_step_size_config);
+ let session = Session::new(model_path_buf, decoding_config)
.expect("Failed to create session");
progress_bar.set_style(