Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

change default prefill step size

Aleksei Savineba9b8be1325
diff --git a/crates/uzu/src/session/parameter/prefill_step_size.rs b/crates/uzu/src/session/parameter/prefill_step_size.rs
index ef861b38d..7904eff5b 100644
--- a/crates/uzu/src/session/parameter/prefill_step_size.rs
+++ b/crates/uzu/src/session/parameter/prefill_step_size.rs
@@ -60,9 +60,9 @@ impl ConfigResolvableValue<LanguageModelConfig, usize> for PrefillStepSize {
if let Some(v) = env_prefill_step_size_default_override() {
v
} else if cfg!(target_os = "ios") {
- 64
+ 512
} else {
- 256
+ 2048
}
},
PrefillStepSize::Maximal => maximal_value,