Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

weaverless dflash 16 batch size (#711)

uuuvnc36dc5b13c66
diff --git a/crates/backend-uzu/src/engine/language_model/stream/stream.rs b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
index 6d0fb2de5..355c530e7 100644
--- a/crates/backend-uzu/src/engine/language_model/stream/stream.rs
+++ b/crates/backend-uzu/src/engine/language_model/stream/stream.rs
@@ -545,7 +545,13 @@ impl<'a, B: Backend> LanguageModelStream<'a, B> {
None
};
- let full_batch_size = 32;
+ let full_batch_size = self.model.speculator.as_ref().map_or(1, |speculator| {
+ if speculator.has_weaver() {
+ 32
+ } else {
+ 16
+ }
+ });
let speculation_batch = self
.model_state
.max_context_length