A high-performance inference engine for AI models
wire sampling seed (#753)
CCYehcefdf50e5654
diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rsindex 328ad404d..c25f56714 100644--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs@@ -19,7 +19,10 @@ use shoji::{},},},- types::session::chat::{ChatConfig, ChatReplyConfig},+ types::{+ basic::SamplingSeed,+ session::chat::{ChatConfig, ChatReplyConfig},+ },};use tokenizers::Tokenizer;use tokio_util::sync::CancellationToken;@@ -81,8 +84,14 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn State>, BackendError>> + Send + '_>> {Box::pin(async move {let max_context_length = get_max_context_length(&self.model, self.config.context_length.clone());+ let sampling_seed = match &self.config.sampling_seed {+ SamplingSeed::Default {} => rand::random(),+ SamplingSeed::Custom {+ seed,+ } => *seed as u64,+ };self.model- .create_empty_state(max_context_length)+ .create_empty_state(max_context_length, sampling_seed).map_err(|err| BackendError::from(err.to_string())).map(|state| Box::new(UzuChatTokenBackendInstanceState::new(state)) as Box<dyn State>)})diff --git a/crates/backend-uzu/src/engine/language_model/state.rs b/crates/backend-uzu/src/engine/language_model/state.rsindex 189b43f6e..65c11a3b5 100644--- a/crates/backend-uzu/src/engine/language_model/state.rs+++ b/crates/backend-uzu/src/engine/language_model/state.rs@@ -31,11 +31,12 @@ impl<B: Backend> LanguageModel<B> {pub fn create_empty_state(&self,max_context_length: Option<u32>,+ sampling_seed: u64,) -> Result<LanguageModelState<B>, LanguageModelCreateEmptyStateError<B>> {let tokens = Vec::new();let last_output_token = None;- let prng = PRng::new(rand::random());+ let prng = PRng::new(sampling_seed);let transformer_state = self.decoder