Skip to content

armgit / uzu

Public Imported from GitHub
0

A high-performance inference engine for AI models

wire sampling seed (#753)

CCYehcefdf50e5654
diff --git a/crates/backend-uzu/src/bridge/chat_token_backend.rs b/crates/backend-uzu/src/bridge/chat_token_backend.rs
index 328ad404d..c25f56714 100644
--- a/crates/backend-uzu/src/bridge/chat_token_backend.rs
+++ b/crates/backend-uzu/src/bridge/chat_token_backend.rs
@@ -19,7 +19,10 @@ use shoji::{
},
},
},
- types::session::chat::{ChatConfig, ChatReplyConfig},
+ types::{
+ basic::SamplingSeed,
+ session::chat::{ChatConfig, ChatReplyConfig},
+ },
};
use tokenizers::Tokenizer;
use tokio_util::sync::CancellationToken;
@@ -81,8 +84,14 @@ impl<B: Backend> BackendInstance for UzuChatTokenBackendInstance<B> {
fn state(&self) -> Pin<Box<dyn Future<Output = Result<Box<dyn State>, BackendError>> + Send + '_>> {
Box::pin(async move {
let max_context_length = get_max_context_length(&self.model, self.config.context_length.clone());
+ let sampling_seed = match &self.config.sampling_seed {
+ SamplingSeed::Default {} => rand::random(),
+ SamplingSeed::Custom {
+ seed,
+ } => *seed as u64,
+ };
self.model
- .create_empty_state(max_context_length)
+ .create_empty_state(max_context_length, sampling_seed)
.map_err(|err| BackendError::from(err.to_string()))
.map(|state| Box::new(UzuChatTokenBackendInstanceState::new(state)) as Box<dyn State>)
})
diff --git a/crates/backend-uzu/src/engine/language_model/state.rs b/crates/backend-uzu/src/engine/language_model/state.rs
index 189b43f6e..65c11a3b5 100644
--- a/crates/backend-uzu/src/engine/language_model/state.rs
+++ b/crates/backend-uzu/src/engine/language_model/state.rs
@@ -31,11 +31,12 @@ impl<B: Backend> LanguageModel<B> {
pub fn create_empty_state(
&self,
max_context_length: Option<u32>,
+ sampling_seed: u64,
) -> Result<LanguageModelState<B>, LanguageModelCreateEmptyStateError<B>> {
let tokens = Vec::new();
let last_output_token = None;
- let prng = PRng::new(rand::random());
+ let prng = PRng::new(sampling_seed);
let transformer_state = self
.decoder