Skip to content

web search done right

if Alice stops using <|endoftext|> token, go back to prev state and load ‘\n\n’

Mikkel Denkerc04cef7de76b
diff --git a/core/src/alice/generate.rs b/core/src/alice/generate.rs
index 4a4e466f..7663ec48 100644
--- a/core/src/alice/generate.rs
+++ b/core/src/alice/generate.rs
@@ -29,8 +29,11 @@ use super::{
ModelState, ModelWebsite, Result, Searcher, SimplifiedWebsite, Tokenizer, TransitionValidator,
};
-const TAU: f64 = 0.8;
+// const TAU: f64 = 0.3;
const TEMP: f64 = 0.4;
+const TOP_P: f64 = 0.5;
+
+const MAX_NUM_QUERIES: usize = 3;
pub enum TokenGeneratorState {
InProgress,
@@ -112,7 +115,7 @@ impl AliceTokenGenerator {
.squeeze()
.to(tch::Device::Cpu);
- let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
+ let next_token = llm_utils::sample_nucleus(probs, TEMP, TOP_P);
self.states.push(ModelState {
state: ClonableTensor(new_state),
next_token,
@@ -157,6 +160,13 @@ impl AliceTokenGenerator {
self.banned_tokens = tokens.to_vec();
}
}
+
+ fn previous_state(&mut self) {
+ if let Some(f) = self.states.pop() {
+ self.states.push(f.clone());
+ self.states.push(f);
+ }
+ }
}
impl Iterator for AliceTokenGenerator {
@@ -209,7 +219,7 @@ impl Iterator for AliceTokenGenerator {
.copy_(&Tensor::from_slice(&[0.0]).squeeze());
}
- let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
+ let next_token = llm_utils::sample_nucleus(probs, TEMP, TOP_P);
token = next_token;
state = ClonableTensor(new_state);
}
@@ -228,7 +238,7 @@ impl Iterator for AliceTokenGenerator {
.copy_(&Tensor::from_slice(&[0.0]).squeeze());
}
- let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
+ let next_token = llm_utils::sample_nucleus(probs, TEMP, TOP_P);
self.states.push(ModelState {
state: ClonableTensor(state),
next_token,
@@ -305,11 +315,48 @@ enum State {
},
Speaking {
banned_tokens: TTLBannedTokens,
- end: AnyTransitionValidator,
+ single_end: AnyTransitionValidator,
+ double_newline: TransitionValidator,
max_new_tokens: Option<usize>,
},
}
+impl State {
+ fn new_thought(tokenizer: &Tokenizer) -> Self {
+ let search = TransitionValidator::new(tokenizer.encode("Search:".to_string()).unwrap());
+ let speaking = TransitionValidator::new(tokenizer.encode("Alice:".to_string()).unwrap());
+
+ Self::Thought { search, speaking }
+ }
+
+ fn new_query_build(tokenizer: &Tokenizer) -> Self {
+ let new_line = TransitionValidator::new(tokenizer.encode("\n".to_string()).unwrap());
+ let end_of_text =
+ TransitionValidator::new(tokenizer.encode("<|endoftext|>".to_string()).unwrap());
+ let end = AnyTransitionValidator::new(vec![new_line, end_of_text]);
+
+ Self::QueryBuild {
+ tokens: Vec::new(),
+ end,
+ }
+ }
+
+ fn new_speaking(initially_banned_tokens: Vec<i64>) -> Self {
+ let end = TransitionValidator::new(vec![0]);
+ let double_newline_single_tok = TransitionValidator::new(vec![535]);
+ let single_end = AnyTransitionValidator::new(vec![end, double_newline_single_tok]);
+
+ let double_newline = TransitionValidator::new(vec![187, 187]);
+
+ Self::Speaking {
+ single_end,
+ double_newline,
+ max_new_tokens: Some(1024),
+ banned_tokens: TTLBannedTokens::new(initially_banned_tokens, 2), // the tokens should only be banned at the start of generation
+ }
+ }
+}
+
struct TTLBannedTokens {
tokens: Vec<i64>,
orig_ttl: usize,
@@ -344,41 +391,6 @@ impl TTLBannedTokens {
}
}
-impl State {
- fn new_thought(tokenizer: &Tokenizer) -> Self {
- let search = TransitionValidator::new(tokenizer.encode("Search:".to_string()).unwrap());
- let speaking = TransitionValidator::new(tokenizer.encode("Alice:".to_string()).unwrap());
-
- Self::Thought { search, speaking }
- }
-
- fn new_query_build(tokenizer: &Tokenizer) -> Self {
- let new_line = TransitionValidator::new(tokenizer.encode("\n".to_string()).unwrap());
- let end_of_text =
- TransitionValidator::new(tokenizer.encode("<|endoftext|>".to_string()).unwrap());
- let end = AnyTransitionValidator::new(vec![new_line, end_of_text]);
-
- Self::QueryBuild {
- tokens: Vec::new(),
- end,
- }
- }
-
- fn new_speaking(initially_banned_tokens: Vec<i64>) -> Self {
- let end = TransitionValidator::new(vec![0]);
- let double_newline = TransitionValidator::new(vec![187, 187]);
- let double_newline_single_tok = TransitionValidator::new(vec![535]);
-
- let end = AnyTransitionValidator::new(vec![end, double_newline, double_newline_single_tok]);
-
- Self::Speaking {
- end,
- max_new_tokens: Some(1024),
- banned_tokens: TTLBannedTokens::new(initially_banned_tokens, 2), // the tokens should only be banned at the start of generation
- }
- }
-}
-
pub struct RawActionGenerator {
state: State,
tokenizer: Arc<Tokenizer>,
@@ -399,7 +411,7 @@ impl RawActionGenerator {
tokenizer,
token_generator,
queries_performed: 0,
- max_num_queries: 3,
+ max_num_queries: MAX_NUM_QUERIES,
}
}
@@ -457,7 +469,8 @@ impl Iterator for RawActionGenerator {
}
}
State::Speaking {
- end,
+ single_end,
+ double_newline,
max_new_tokens,
banned_tokens,
} => {
@@ -466,9 +479,16 @@ impl Iterator for RawActionGenerator {
let token = self.token_generator.next()?;
- if end.validate(token) {
+ if double_newline.validate(token) {
+ banned_tokens.reset_ttl();
+ self.token_generator.reset_tokens_counter();
+ self.state = State::new_thought(&self.tokenizer);
+ return None;
+ } else if single_end.validate(token) {
banned_tokens.reset_ttl();
self.token_generator.reset_tokens_counter();
+ self.token_generator.previous_state();
+ self.token_generator.load_tokens(&[187, 187]);
self.state = State::new_thought(&self.tokenizer);
return None;
} else {
@@ -634,7 +654,7 @@ impl ActionExecutor {
.generator
.raw
.tokenizer
- .encode(format!("Result: {json}<|endoftext|>\n\n"))
+ .encode(format!("Result: {json}\n\n"))
.unwrap();
self.generator
.raw
diff --git a/core/src/llm_utils.rs b/core/src/llm_utils.rs
index 9c1d0046..ae90083d 100644
--- a/core/src/llm_utils.rs
+++ b/core/src/llm_utils.rs
@@ -39,6 +39,30 @@ pub fn sample_typical(probs: Tensor, temp: f64, tau: f64) -> i64 {
probs.multinomial(1, true).int64_value(&[0])
}
+pub fn sample_nucleus(probs: Tensor, temp: f64, top_p: f64) -> i64 {
+ debug_assert!(probs.dim() == 1, "batch support not implemented");
+ let mut probs = probs;
+
+ let sorted_ids = probs.argsort(-1, true);
+ let sorted_probs = probs.index_select(-1, &sorted_ids);
+ let cumulative_probs = sorted_probs.cumsum(-1, Kind::Float);
+
+ let ids_to_remove = cumulative_probs.ge(top_p).to_kind(Kind::Int64);
+
+ // let cutoff = sorted_probs
+ // .index_select(-1, &cumulative_probs.gt(top_p).argmax(-1, true))
+ // .double_value(&[]);
+
+ probs = probs.index_fill_(-1, &ids_to_remove, 0.0);
+
+ if temp != 1.0 {
+ let t = Tensor::from_slice(&[1.0 / temp]);
+ probs = probs.pow(&t);
+ }
+
+ probs.multinomial(1, true).int64_value(&[0])
+}
+
pub struct ClonableTensor(pub Tensor);
impl Clone for ClonableTensor {