A high-performance inference engine for AI models
Disale speculation for SSM models
Aleksei Savin0ebdda0e0d6a
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rsindex 4990a8cc2..69f3ffc1d 100644--- a/crates/uzu/src/session/session.rs+++ b/crates/uzu/src/session/session.rs@@ -82,6 +82,11 @@ impl Session {return Err(Error::UnsupportedContextModeForModel);},}++ if decoding_config.speculator_config.number_of_speculated_tokens > 0+ {+ return Err(Error::UnsupportedSpeculatorConfigForModel);+ }}let tokenizer_path = model_path.join("tokenizer.json");diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rsindex a3905a09c..2d58fcf82 100644--- a/crates/uzu/src/session/types/error.rs+++ b/crates/uzu/src/session/types/error.rs@@ -14,6 +14,8 @@ pub enum Error {NotEnoughMemory,#[error("Unsupported context mode for model")]UnsupportedContextModeForModel,+ #[error("Unsupported speculator config for model")]+ UnsupportedSpeculatorConfigForModel,#[error("Generator not loaded")]GeneratorNotLoaded,#[error("Unable to load prompt template")]