A high-performance inference engine for AI models
weaverless dflash 16 batch size (#711)
uuuvnc36dc5b13c66
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rsindex 6d0fb2de5..355c530e7 100644--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs@@ -545,7 +545,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {None};- let full_batch_size = 32;+ let full_batch_size = self.model.speculator.as_ref().map_or(1, |speculator| {+ if speculator.has_weaver() {+ 32+ } else {+ 16+ }+ });let speculation_batch = self.model_state.max_context_length