Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

Disale speculation for SSM models

Aleksei Savin0ebdda0e0d6a
diff --git a/crates/uzu/src/session/session.rs b/crates/uzu/src/session/session.rs
index 4990a8cc2..69f3ffc1d 100644
--- a/crates/uzu/src/session/session.rs
+++ b/crates/uzu/src/session/session.rs
@@ -82,6 +82,11 @@ impl Session {
return Err(Error::UnsupportedContextModeForModel);
},
}
+
+ if decoding_config.speculator_config.number_of_speculated_tokens > 0
+ {
+ return Err(Error::UnsupportedSpeculatorConfigForModel);
+ }
}
let tokenizer_path = model_path.join("tokenizer.json");
diff --git a/crates/uzu/src/session/types/error.rs b/crates/uzu/src/session/types/error.rs
index a3905a09c..2d58fcf82 100644
--- a/crates/uzu/src/session/types/error.rs
+++ b/crates/uzu/src/session/types/error.rs
@@ -14,6 +14,8 @@ pub enum Error {
NotEnoughMemory,
#[error("Unsupported context mode for model")]
UnsupportedContextModeForModel,
+ #[error("Unsupported speculator config for model")]
+ UnsupportedSpeculatorConfigForModel,
#[error("Generator not loaded")]
GeneratorNotLoaded,
#[error("Unable to load prompt template")]