Skip to content

web search done right

refactor alice into multiple files

Mikkel Denker44baa1931fb3
diff --git a/core/examples/alice.rs b/core/examples/alice.rs
index 5a846702..cac805d0 100644
--- a/core/examples/alice.rs
+++ b/core/examples/alice.rs
@@ -1,6 +1,6 @@
use std::io::Write;
-use stract::alice::{AcceleratorConfig, Alice, ExecutionState};
+use stract::alice::{generate::ExecutionState, AcceleratorConfig, Alice};
#[tokio::main]
async fn main() {
diff --git a/core/src/alice/generate.rs b/core/src/alice/generate.rs
new file mode 100644
index 00000000..4a4e466f
--- /dev/null
+++ b/core/src/alice/generate.rs
@@ -0,0 +1,701 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use aes_gcm::{Aes256Gcm, Key};
+use tch::{IndexOp, Kind, Tensor};
+
+use crate::{
+ leaky_queue::LeakyQueue,
+ llm_utils::{self, ClonableTensor},
+};
+
+use super::{
+ raw_model::RawModel, AnyTransitionValidator, EncodedEncryptedState, EncryptedState, Error,
+ ModelState, ModelWebsite, Result, Searcher, SimplifiedWebsite, Tokenizer, TransitionValidator,
+};
+
+const TAU: f64 = 0.8;
+const TEMP: f64 = 0.4;
+
+pub enum TokenGeneratorState {
+ InProgress,
+ Finished,
+}
+
+pub struct AliceTokenGenerator {
+ model: Arc<RawModel>,
+ // state needs to be a queue of size 2 to have the ability
+ // to generate a new token if the current one is banned
+ states: LeakyQueue<ModelState>,
+ // the state of the model after each search result has been
+ // loaded. This allows us to go back to the end of the previous search
+ // and force speak if the model wants us to search for the same query again.
+ end_search_states: Vec<LeakyQueue<ModelState>>,
+ end_tokens: Vec<i64>,
+ generation_state: TokenGeneratorState,
+ max_new_tokens: Option<usize>,
+ num_generated_tokens: usize,
+ banned_tokens: Vec<i64>,
+}
+
+impl AliceTokenGenerator {
+ pub fn new(
+ model: Arc<RawModel>,
+ end_tokens: Vec<i64>,
+ state: Tensor,
+ tokens: &[i64],
+ max_new_tokens: Option<usize>,
+ ) -> Result<Self> {
+ if tokens.is_empty() {
+ return Err(Error::EmptyInput);
+ }
+
+ // this must be 2, otherwise `.front()` and `.back()` calls are wrong throughout the generation code
+ let num_states = 2;
+
+ let mut states = LeakyQueue::new(num_states);
+
+ let state = ClonableTensor(state);
+
+ for _ in 0..num_states {
+ states.push(ModelState {
+ state: state.clone(),
+ next_token: -1,
+ });
+ }
+
+ let mut generator = Self {
+ model,
+ states,
+ end_tokens,
+ generation_state: TokenGeneratorState::InProgress,
+ max_new_tokens,
+ num_generated_tokens: 0,
+ end_search_states: Vec::new(),
+ banned_tokens: Vec::new(),
+ };
+ generator.load_tokens(tokens);
+
+ Ok(generator)
+ }
+
+ pub fn load_tokens(&mut self, tokens: &[i64]) {
+ for token in tokens {
+ let (logits, new_state) = self.model.forward(
+ *token,
+ Some(
+ &self
+ .states
+ .back()
+ .expect("there should be 2 states at all times")
+ .state
+ .0,
+ ),
+ );
+ let probs = logits
+ .softmax(-1, Kind::Float)
+ .squeeze()
+ .to(tch::Device::Cpu);
+
+ let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
+ self.states.push(ModelState {
+ state: ClonableTensor(new_state),
+ next_token,
+ });
+
+ if let Some(state) = self.states.front_mut() {
+ state.next_token = *token;
+ }
+ }
+ }
+
+ fn load_search_result(&mut self, tokens: &[i64]) {
+ self.load_tokens(tokens);
+ self.end_search_states.push(self.states.clone());
+ }
+
+ fn go_to_last_search(&mut self) {
+ if let Some(last_search) = self.end_search_states.pop() {
+ self.states = last_search;
+ }
+ }
+
+ pub fn set_end_tokens(&mut self, tokens: &[i64]) {
+ if self.end_tokens != tokens {
+ self.end_tokens = tokens.to_vec();
+ }
+ }
+
+ pub fn set_max_new_tokens(&mut self, max_new_tokens: Option<usize>) {
+ if self.max_new_tokens != max_new_tokens {
+ self.num_generated_tokens = 0;
+ self.max_new_tokens = max_new_tokens;
+ }
+ }
+
+ pub fn reset_tokens_counter(&mut self) {
+ self.num_generated_tokens = 0;
+ }
+
+ pub fn set_banned_tokens(&mut self, tokens: &[i64]) {
+ if self.banned_tokens != tokens {
+ self.banned_tokens = tokens.to_vec();
+ }
+ }
+}
+
+impl Iterator for AliceTokenGenerator {
+ type Item = i64;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ match self.generation_state {
+ TokenGeneratorState::InProgress => {
+ let s = self
+ .states
+ .back()
+ .expect("there should be 2 states at all times");
+ let mut token = s.next_token;
+ let mut state = s.state.clone();
+
+ if self.end_tokens.contains(&token) {
+ self.generation_state = TokenGeneratorState::Finished;
+ return None;
+ }
+
+ if let Some(max_new_tokens) = self.max_new_tokens {
+ if self.num_generated_tokens >= max_new_tokens {
+ if let Some(tok) = self.end_tokens.first() {
+ token = *tok;
+ }
+
+ self.generation_state = TokenGeneratorState::Finished;
+ }
+ }
+
+ if self.banned_tokens.contains(&token) {
+ let (output, new_state) = {
+ let s = self
+ .states
+ .front()
+ .expect("there should be 2 states at all times");
+ self.model.forward(s.next_token, Some(&s.state.0))
+ };
+
+ let output = output.squeeze();
+
+ let probs = output
+ .softmax(-1, Kind::Float)
+ .squeeze()
+ .to(tch::Device::Cpu);
+
+ for banned_token in &self.banned_tokens {
+ probs
+ .i(*banned_token)
+ .copy_(&Tensor::from_slice(&[0.0]).squeeze());
+ }
+
+ let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
+ token = next_token;
+ state = ClonableTensor(new_state);
+ }
+
+ let (output, state) = self.model.forward(token, Some(&state.0));
+ let output = output.squeeze();
+
+ let probs = output
+ .softmax(-1, Kind::Float)
+ .squeeze()
+ .to(tch::Device::Cpu);
+
+ for banned_token in &self.banned_tokens {
+ probs
+ .i(*banned_token)
+ .copy_(&Tensor::from_slice(&[0.0]).squeeze());
+ }
+
+ let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
+ self.states.push(ModelState {
+ state: ClonableTensor(state),
+ next_token,
+ });
+
+ self.num_generated_tokens += 1;
+
+ Some(token)
+ }
+ TokenGeneratorState::Finished => None,
+ }
+ }
+}
+
+pub struct AliceStringGenerator {
+ token_generator: AliceTokenGenerator,
+ tokenizer: Arc<Tokenizer>,
+ tokens: Vec<i64>,
+}
+
+impl Iterator for AliceStringGenerator {
+ type Item = String;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ loop {
+ match self.token_generator.next() {
+ Some(token) => {
+ self.tokens.push(token);
+ if let Some(s) = self.tokenizer.decode(&self.tokens).ok().and_then(|s| {
+ if !s.contains('\u{fffd}') {
+ // valid utf-8 string
+ Some(s)
+ } else {
+ None
+ }
+ }) {
+ self.tokens.clear();
+ return Some(s);
+ }
+ }
+ None if !self.tokens.is_empty() => {
+ let s = self.tokenizer.decode(&self.tokens).ok().and_then(|s| {
+ if !s.contains('\u{fffd}') {
+ Some(s)
+ } else {
+ None
+ }
+ });
+
+ self.tokens.clear();
+
+ return s;
+ }
+ None => return None,
+ }
+ }
+ }
+}
+
+#[derive(Debug)]
+pub enum RawAction {
+ Search { query: Vec<i64> },
+ Speak { token: i64 },
+}
+
+enum State {
+ Thought {
+ search: TransitionValidator,
+ speaking: TransitionValidator,
+ },
+ QueryBuild {
+ tokens: Vec<i64>,
+ end: AnyTransitionValidator,
+ },
+ Speaking {
+ banned_tokens: TTLBannedTokens,
+ end: AnyTransitionValidator,
+ max_new_tokens: Option<usize>,
+ },
+}
+
+struct TTLBannedTokens {
+ tokens: Vec<i64>,
+ orig_ttl: usize,
+ ttl: usize,
+}
+
+impl TTLBannedTokens {
+ fn new(tokens: Vec<i64>, ttl: usize) -> Self {
+ Self {
+ tokens,
+ ttl,
+ orig_ttl: ttl,
+ }
+ }
+
+ fn reset_ttl(&mut self) {
+ self.ttl = self.orig_ttl;
+ }
+
+ fn tick(&mut self) {
+ if self.ttl > 0 {
+ self.ttl -= 1;
+ }
+ }
+
+ fn set_banned_tokens(&self, token_generator: &mut AliceTokenGenerator) {
+ if self.ttl > 0 {
+ token_generator.set_banned_tokens(&self.tokens);
+ } else {
+ token_generator.set_banned_tokens(&[]);
+ }
+ }
+}
+
+impl State {
+ fn new_thought(tokenizer: &Tokenizer) -> Self {
+ let search = TransitionValidator::new(tokenizer.encode("Search:".to_string()).unwrap());
+ let speaking = TransitionValidator::new(tokenizer.encode("Alice:".to_string()).unwrap());
+
+ Self::Thought { search, speaking }
+ }
+
+ fn new_query_build(tokenizer: &Tokenizer) -> Self {
+ let new_line = TransitionValidator::new(tokenizer.encode("\n".to_string()).unwrap());
+ let end_of_text =
+ TransitionValidator::new(tokenizer.encode("<|endoftext|>".to_string()).unwrap());
+ let end = AnyTransitionValidator::new(vec![new_line, end_of_text]);
+
+ Self::QueryBuild {
+ tokens: Vec::new(),
+ end,
+ }
+ }
+
+ fn new_speaking(initially_banned_tokens: Vec<i64>) -> Self {
+ let end = TransitionValidator::new(vec![0]);
+ let double_newline = TransitionValidator::new(vec![187, 187]);
+ let double_newline_single_tok = TransitionValidator::new(vec![535]);
+
+ let end = AnyTransitionValidator::new(vec![end, double_newline, double_newline_single_tok]);
+
+ Self::Speaking {
+ end,
+ max_new_tokens: Some(1024),
+ banned_tokens: TTLBannedTokens::new(initially_banned_tokens, 2), // the tokens should only be banned at the start of generation
+ }
+ }
+}
+
+pub struct RawActionGenerator {
+ state: State,
+ tokenizer: Arc<Tokenizer>,
+ token_generator: AliceTokenGenerator,
+ queries_performed: usize,
+ max_num_queries: usize,
+}
+
+impl RawActionGenerator {
+ pub fn new(token_generator: AliceTokenGenerator, tokenizer: Arc<Tokenizer>) -> Self {
+ let state = State::new_thought(&tokenizer);
+
+ let mut token_generator = token_generator;
+ token_generator.set_end_tokens(&[]);
+
+ Self {
+ state,
+ tokenizer,
+ token_generator,
+ queries_performed: 0,
+ max_num_queries: 3,
+ }
+ }
+
+ fn force_speaking(&mut self) {
+ self.token_generator.load_tokens(
+ &self
+ .tokenizer
+ .encode("Alice[thought]: I now know the final answer.\n\n".to_string())
+ .unwrap(),
+ );
+
+ self.state = State::new_speaking(vec![
+ self.tokenizer.encode(" [thought]:".to_string()).unwrap()[0],
+ self.tokenizer.encode("[thought]:".to_string()).unwrap()[0],
+ ]);
+
+ self.token_generator
+ .load_tokens(&self.tokenizer.encode("Alice:".to_string()).unwrap());
+ }
+}
+
+impl Iterator for RawActionGenerator {
+ type Item = RawAction;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ loop {
+ match &mut self.state {
+ State::Thought { search, speaking } => {
+ let token = self.token_generator.next()?;
+ self.token_generator.set_max_new_tokens(Some(1024));
+
+ if self.queries_performed >= self.max_num_queries {
+ self.token_generator.reset_tokens_counter();
+ self.force_speaking();
+ } else if search.validate(token) {
+ self.token_generator.reset_tokens_counter();
+ self.state = State::new_query_build(&self.tokenizer);
+ } else if speaking.validate(token) {
+ self.token_generator.reset_tokens_counter();
+ self.state = State::new_speaking(vec![]);
+ }
+ }
+ State::QueryBuild { tokens, end } => {
+ let token = self.token_generator.next()?;
+ self.token_generator.set_max_new_tokens(Some(1024));
+ tokens.push(token);
+
+ if end.validate(token) {
+ let query = tokens.clone();
+ *tokens = Vec::new();
+ self.token_generator.reset_tokens_counter();
+ self.state = State::new_thought(&self.tokenizer);
+ self.queries_performed += 1;
+ return Some(RawAction::Search { query });
+ }
+ }
+ State::Speaking {
+ end,
+ max_new_tokens,
+ banned_tokens,
+ } => {
+ self.token_generator.set_max_new_tokens(*max_new_tokens);
+ banned_tokens.set_banned_tokens(&mut self.token_generator);
+
+ let token = self.token_generator.next()?;
+
+ if end.validate(token) {
+ banned_tokens.reset_ttl();
+ self.token_generator.reset_tokens_counter();
+ self.state = State::new_thought(&self.tokenizer);
+ return None;
+ } else {
+ banned_tokens.tick();
+ return Some(RawAction::Speak { token });
+ }
+ }
+ }
+ }
+ }
+}
+
+pub enum Action {
+ Search { query: String },
+ Speak { text: String },
+}
+
+pub struct ActionGenerator {
+ raw: RawActionGenerator,
+ tokens_to_speak: Vec<i64>,
+}
+
+impl ActionGenerator {
+ pub fn new(raw_action_gen: RawActionGenerator) -> ActionGenerator {
+ Self {
+ raw: raw_action_gen,
+ tokens_to_speak: Vec::new(),
+ }
+ }
+}
+
+impl Iterator for ActionGenerator {
+ type Item = Action;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ loop {
+ let action = self.raw.next();
+
+ match action {
+ Some(RawAction::Search { query }) => {
+ return Some(Action::Search {
+ query: self
+ .raw
+ .tokenizer
+ .decode(&query)
+ .unwrap_or_default()
+ .trim()
+ .to_ascii_lowercase(),
+ });
+ }
+ Some(RawAction::Speak { token }) => {
+ self.tokens_to_speak.push(token);
+
+ if let Some(s) = self
+ .raw
+ .tokenizer
+ .decode(&self.tokens_to_speak)
+ .ok()
+ .and_then(|s| {
+ if !s.contains('\u{fffd}') {
+ // valid utf-8 string
+ Some(s)
+ } else {
+ None
+ }
+ })
+ {
+ self.tokens_to_speak.clear();
+ return Some(Action::Speak { text: s });
+ }
+ }
+ None if !self.tokens_to_speak.is_empty() => {
+ let s = self
+ .raw
+ .tokenizer
+ .decode(&self.tokens_to_speak)
+ .ok()
+ .and_then(|s| {
+ if !s.contains('\u{fffd}') {
+ Some(s)
+ } else {
+ None
+ }
+ });
+
+ self.tokens_to_speak.clear();
+
+ return s.map(|s| Action::Speak { text: s });
+ }
+ None => {
+ return None;
+ }
+ }
+ }
+ }
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+#[serde(tag = "@type", rename_all = "camelCase")]
+pub enum ExecutionState {
+ BeginSearch {
+ query: String,
+ },
+ SearchResult {
+ query: String,
+ result: Vec<SimplifiedWebsite>,
+ },
+ Speaking {
+ text: String,
+ },
+ Done {
+ state: EncodedEncryptedState,
+ },
+}
+
+pub struct ActionExecutor {
+ generator: ActionGenerator,
+ searcher: Searcher,
+ query_to_search: Option<String>,
+ queries_performed: Vec<String>,
+ has_finished: bool,
+ encryption_key: Key<Aes256Gcm>,
+}
+
+unsafe impl Send for ActionExecutor {}
+
+impl ActionExecutor {
+ pub fn new(
+ action_gen: ActionGenerator,
+ searcher: Searcher,
+ encryption_key: Key<Aes256Gcm>,
+ ) -> Self {
+ ActionExecutor {
+ generator: action_gen,
+ searcher,
+ query_to_search: None,
+ queries_performed: Vec::new(),
+ has_finished: false,
+ encryption_key,
+ }
+ }
+ pub fn state(&self) -> Tensor {
+ self.generator
+ .raw
+ .token_generator
+ .states
+ .back()
+ .expect("there should be 2 states at all times")
+ .state
+ .clone()
+ .0
+ }
+
+ fn load_search_result(&mut self, result: &[SimplifiedWebsite]) {
+ let result = result
+ .iter()
+ .map(|r| ModelWebsite::from(r.clone()))
+ .collect::<Vec<_>>();
+
+ let json = serde_json::to_string(&result).unwrap();
+
+ let tokens = self
+ .generator
+ .raw
+ .tokenizer
+ .encode(format!("Result: {json}<|endoftext|>\n\n"))
+ .unwrap();
+ self.generator
+ .raw
+ .token_generator
+ .load_search_result(&tokens);
+
+ let tokens = self
+ .generator
+ .raw
+ .tokenizer
+ .encode("Alice[thought]:".to_string())
+ .unwrap();
+ self.generator.raw.token_generator.load_tokens(&tokens);
+ }
+}
+
+impl Iterator for ActionExecutor {
+ type Item = ExecutionState;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if self.has_finished {
+ return None;
+ }
+
+ if let Some(query) = self.query_to_search.take() {
+ if self.queries_performed.contains(&query) {
+ self.generator.raw.token_generator.go_to_last_search();
+ self.generator.raw.force_speaking();
+ } else {
+ let res = self.searcher.search(&query).unwrap_or_default();
+
+ tracing::debug!("loading search results");
+ self.load_search_result(&res);
+ tracing::debug!("done loading search results");
+ self.queries_performed.push(query.clone());
+
+ return Some(ExecutionState::SearchResult { query, result: res });
+ }
+ }
+
+ let action = self.generator.next();
+
+ match action {
+ Some(Action::Search { query }) => {
+ self.query_to_search = Some(query.clone());
+ Some(ExecutionState::BeginSearch { query })
+ }
+ Some(Action::Speak { text }) => Some(ExecutionState::Speaking { text }),
+ None if !self.has_finished => {
+ self.has_finished = true;
+
+ let state: Vec<Vec<f32>> =
+ self.state().to_device(tch::Device::Cpu).try_into().unwrap();
+
+ let encrypted = EncryptedState::encrypt(state, &self.encryption_key);
+
+ Some(ExecutionState::Done {
+ state: EncodedEncryptedState::encode(encrypted),
+ })
+ }
+ None => None,
+ }
+ }
+}
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index 48cf8334..1e05a89c 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{
- fs::OpenOptions,
io::{Read, Write},
path::Path,
sync::Arc,
@@ -27,16 +26,11 @@ use aes_gcm::{
};
use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};
use half::bf16;
-use safetensors::SafeTensors;
-use tch::{
- nn::{embedding, layer_norm, Embedding, LayerNorm, LayerNormConfig, Linear, ModuleT, VarStore},
- IndexOp, Kind, Tensor,
-};
+use tch::Tensor;
use crate::{
api::search::ApiSearchQuery,
- leaky_queue::LeakyQueue,
- llm_utils::{self, ClonableTensor},
+ llm_utils::ClonableTensor,
search_prettifier::DisplayedWebpage,
searcher::{SearchResult, WebsitesResult},
summarizer::{self, ExtractiveSummarizer},
@@ -44,10 +38,13 @@ use crate::{
AcceleratorDevice, AcceleratorDtype, AliceAcceleratorConfig,
};
-const NUM_TOKENS: i64 = 50277;
+use self::{
+ generate::{ActionExecutor, ActionGenerator, AliceTokenGenerator, RawActionGenerator},
+ raw_model::RawModel,
+};
-const TAU: f64 = 0.8;
-const TEMP: f64 = 0.4;
+pub mod generate;
+mod raw_model;
const PROMPT_PREFIX: &str = r#"System: Your name is Alice, and you are an AI assistant trained by Stract. Below is a conversation between you and a user. Help the user as best you can.
You can lookup information on the web in the following format:
@@ -100,440 +97,6 @@ pub enum Error {
Cluster(#[from] crate::distributed::cluster::Error),
}
-fn load_linear(weights: &SafeTensors, prefix: &str) -> Result<Linear> {
- let ws = weights.tensor(&format!("{prefix}.weight"))?.try_into()?;
-
- let mut linear = Linear { ws, bs: None };
-
- if let Ok(bias_tensor) = weights.tensor(&format!("{prefix}.bias")) {
- linear.bs = Some(bias_tensor.try_into()?);
- }
-
- if let Some(bs) = &mut linear.bs {
- *bs = bs.to_kind(Kind::Float);
- }
-
- linear.ws = linear.ws.to_kind(Kind::Float);
-
- Ok(linear)
-}
-
-fn load_emb(weights: &SafeTensors, prefix: &str, emb_size: i64) -> Result<Embedding> {
- let vars = VarStore::new(tch::Device::Cpu);
-
- let ws = weights.tensor(&format!("{prefix}.weight"))?;
- let ws: Tensor = ws.try_into()?;
-
- vars.variables_
- .lock()
- .unwrap()
- .named_variables
- .insert("weight".to_string(), ws.shallow_clone());
-
- let config = tch::nn::EmbeddingConfig::default();
-
- let mut emb = embedding(vars.root(), NUM_TOKENS, emb_size, config);
-
- emb.ws = ws.to_kind(Kind::Float);
-
- Ok(emb)
-}
-
-fn load_ln(weights: &SafeTensors, prefix: &str, emb_size: i64) -> Result<LayerNorm> {
- let vars = VarStore::new(tch::Device::Cpu);
-
- if let Ok(ws) = weights.tensor(&format!("{prefix}.weight")) {
- vars.variables_
- .lock()
- .unwrap()
- .named_variables
- .insert("weight".to_string(), ws.try_into()?);
- }
-
- if let Ok(bs) = weights.tensor(&format!("{prefix}.bias")) {
- vars.variables_
- .lock()
- .unwrap()
- .named_variables
- .insert("bias".to_string(), bs.try_into()?);
- }
-
- let config = LayerNormConfig::default();
-
- let mut ln = layer_norm(vars.root(), vec![emb_size], config);
-
- if let Ok(ws) = weights.tensor(&format!("{prefix}.weight")) {
- let ws: Tensor = ws.try_into()?;
- ln.ws = Some(ws.to_kind(Kind::Float));
- }
-
- if let Ok(bs) = weights.tensor(&format!("{prefix}.bias")) {
- let bs: Tensor = bs.try_into()?;
- ln.bs = Some(bs.to_kind(Kind::Float));
- }
-
- ln.normalized_shape = vec![emb_size];
-
- Ok(ln)
-}
-
-struct TimeMix {
- time_decay: Tensor,
- time_first: Tensor,
-
- time_mix_k: Tensor,
- time_mix_v: Tensor,
- time_mix_r: Tensor,
-
- key: Linear,
- value: Linear,
- receptance: Linear,
- output: Linear,
-
- block_idx: i64,
-}
-impl TimeMix {
- fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {
- let x = x.squeeze();
- let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_k);
- let xv = &x * &self.time_mix_v + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_v);
- let xr = &x * &self.time_mix_r + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_r);
-
- state.i(5 * self.block_idx + 1).copy_(&x);
-
- let r = self.receptance.forward_t(&xr, false).sigmoid();
- let k = self.key.forward_t(&xk, false);
- let v = self.value.forward_t(&xv, false);
-
- let aa = state.i(5 * self.block_idx + 2);
- let bb = state.i(5 * self.block_idx + 3);
- let pp = state.i(5 * self.block_idx + 4);
-
- let ww = &self.time_first + &k;
- let qq = pp.maximum(&ww);
- let e1 = (&pp - &qq).exp();
- let e2 = (ww - &qq).exp();
-
- let a = &e1 * &aa + &e2 * &v;
- let b = &e1 * &bb + &e2;
- let wkv = a / b;
- let ww = pp - self.time_decay.exp();
- let qq = ww.maximum(&k);
- let e1 = (ww - &qq).exp();
- let e2 = (k - &qq).exp();
-
- state
- .i(5 * self.block_idx + 2)
- .copy_(&(&e1 * &aa + &e2 * &v).squeeze());
- state
- .i(5 * self.block_idx + 3)
- .copy_(&(&e1 * &bb + &e2).squeeze());
- state.i(5 * self.block_idx + 4).copy_(&qq.squeeze());
-
- self.output.forward_t(&(wkv * &r), false)
- }
-
- fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {
- self.time_decay = self.time_decay.to_kind(kind).to(device);
- self.time_first = self.time_first.to_kind(kind).to(device);
-
- self.time_mix_k = self.time_mix_k.to_kind(kind).to(device);
- self.time_mix_v = self.time_mix_v.to_kind(kind).to(device);
- self.time_mix_r = self.time_mix_r.to_kind(kind).to(device);
-
- self.key.ws = self.key.ws.to_kind(kind).to(device);
- self.key.bs = self.key.bs.as_ref().map(|t| t.to_kind(kind).to(device));
-
- self.value.ws = self.value.ws.to_kind(kind).to(device);
- self.value.bs = self.value.bs.as_ref().map(|t| t.to_kind(kind).to(device));
-
- self.receptance.ws = self.receptance.ws.to_kind(kind).to(device);
- self.receptance.bs = self
- .receptance
- .bs
- .as_ref()
- .map(|t| t.to_kind(kind).to(device));
-
- self.output.ws = self.output.ws.to_kind(kind).to(device);
- self.output.bs = self.output.bs.as_ref().map(|t| t.to_kind(kind).to(device));
- }
-}
-
-struct ChannelMix {
- time_mix_k: Tensor,
- time_mix_r: Tensor,
-
- key: Linear,
- value: Linear,
-
- receptance: Linear,
-
- block_idx: i64,
-}
-impl ChannelMix {
- fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {
- let x = x.squeeze();
- let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx) * (1 - &self.time_mix_k);
- let xr = &x * &self.time_mix_r + state.i(5 * self.block_idx) * (1 - &self.time_mix_r);
-
- state.i(5 * self.block_idx).copy_(&x);
-
- let r = self.receptance.forward_t(&xr, false).sigmoid();
- let k = self.key.forward_t(&xk, false).relu().square();
-
- r * self.value.forward_t(&k, false)
- }
-
- fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {
- self.time_mix_k = self.time_mix_k.to_kind(kind).to(device);
- self.time_mix_r = self.time_mix_r.to_kind(kind).to(device);
-
- self.key.ws = self.key.ws.to_kind(kind).to(device);
- self.key.bs = self.key.bs.as_ref().map(|t| t.to_kind(kind).to(device));
-
- self.value.ws = self.value.ws.to_kind(kind).to(device);
- self.value.bs = self.value.bs.as_ref().map(|t| t.to_kind(kind).to(device));
-
- self.receptance.ws = self.receptance.ws.to_kind(kind).to(device);
- self.receptance.bs = self
- .receptance
- .bs
- .as_ref()
- .map(|t| t.to_kind(kind).to(device));
- }
-}
-
-struct Block {
- ln0: Option<LayerNorm>,
- ln1: LayerNorm,
- ln2: LayerNorm,
-
- att: TimeMix,
- ffn: ChannelMix,
-
- device: tch::Device,
-}
-
-impl Block {
- fn load(weights: &SafeTensors, prefix: &str, emb_size: i64, block_idx: i64) -> Result<Self> {
- let mut ln0 = None;
-
- let names = weights.names();
-
- if names.contains(&&format!("{prefix}.ln0.weight")) {
- ln0 = Some(load_ln(weights, &format!("{prefix}.ln0"), emb_size)?);
- }
-
- let ln1 = load_ln(weights, &format!("{prefix}.ln1"), emb_size)?;
- let ln2 = load_ln(weights, &format!("{prefix}.ln2"), emb_size)?;
-
- let att = TimeMix {
- time_decay: weights
- .tensor(&format!("{prefix}.att.time_decay"))?
- .try_into()?,
- time_first: weights
- .tensor(&format!("{prefix}.att.time_first"))?
- .try_into()?,
- time_mix_k: weights
- .tensor(&format!("{prefix}.att.time_mix_k"))?
- .try_into()?,
- time_mix_v: weights
- .tensor(&format!("{prefix}.att.time_mix_v"))?
- .try_into()?,
- time_mix_r: weights
- .tensor(&format!("{prefix}.att.time_mix_r"))?
- .try_into()?,
- key: load_linear(weights, &format!("{prefix}.att.key"))?,
- value: load_linear(weights, &format!("{prefix}.att.value"))?,
- receptance: load_linear(weights, &format!("{prefix}.att.receptance"))?,
- output: load_linear(weights, &format!("{prefix}.att.output"))?,
- block_idx,
- };
-
- let ffn = ChannelMix {
- time_mix_k: weights
- .tensor(&format!("{prefix}.ffn.time_mix_k"))?
- .try_into()?,
- time_mix_r: weights
- .tensor(&format!("{prefix}.ffn.time_mix_r"))?
- .try_into()?,
- key: load_linear(weights, &format!("{prefix}.ffn.key"))?,
- value: load_linear(weights, &format!("{prefix}.ffn.value"))?,
- receptance: load_linear(weights, &format!("{prefix}.ffn.receptance"))?,
- block_idx,
- };
-
- Ok(Self {
- ln0,
- ln1,
- ln2,
- att,
- ffn,
- device: tch::Device::Cpu,
- })
- }
-
- fn forward(&self, mut x: Tensor, mut state: Tensor) -> (Tensor, Tensor) {
- if x.device() != self.device {
- x = x.to(self.device);
- }
-
- if state.device() != self.device {
- state = state.to(self.device);
- }
-
- x = match &self.ln0 {
- Some(ln) => ln.forward_t(&x, false),
- None => x,
- };
-
- x += self
- .att
- .forward(&self.ln1.forward_t(&x, false), &mut state)
- .reshape([1, -1]);
- x += self
- .ffn
- .forward(&self.ln2.forward_t(&x, false), &mut state)
- .reshape([1, -1]);
-
- (x, state)
- }
-
- fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {
- self.device = device;
-
- if let Some(ln) = &mut self.ln0 {
- ln.ws = ln.ws.as_ref().map(|t| t.to_kind(kind).to(device));
- ln.bs = ln.bs.as_ref().map(|t| t.to_kind(kind).to(device));
- }
-
- self.att.load_to_device(device, kind);
- self.ffn.load_to_device(device, kind);
-
- self.ln1.ws = self.ln1.ws.as_ref().map(|t| t.to_kind(kind).to(device));
- self.ln1.bs = self.ln1.bs.as_ref().map(|t| t.to_kind(kind).to(device));
-
- self.ln2.ws = self.ln2.ws.as_ref().map(|t| t.to_kind(kind).to(device));
- self.ln2.bs = self.ln2.bs.as_ref().map(|t| t.to_kind(kind).to(device));
- }
-}
-
-struct RawModel {
- emb: Embedding,
- blocks: Vec<Block>,
- ln_out: LayerNorm,
- head: Linear,
-}
-
-impl RawModel {
- fn open<P: AsRef<Path>>(path: P) -> Result<RawModel> {
- // SAFETY
- // broadcasting: please don't modify the file while we're reading it.
- let mmap = unsafe {
- let file = OpenOptions::new()
- .read(true)
- .write(true)
- .create(true)
- .open(path)
- .map_err(Error::Io)?;
-
- memmap2::MmapOptions::new()
- .map_mut(&file)
- .map_err(Error::Io)?
- };
-
- let weights = SafeTensors::deserialize(&mmap)?;
- let names = weights.names();
-
- let num_layers = names
- .iter()
- .filter(|name| name.starts_with("blocks"))
- .map(|name| name.split('.').nth(1).unwrap().parse::<usize>().unwrap())
- .max()
- .unwrap()
- + 1;
-
- let emb = weights.tensor("emb.weight")?;
- let emb_size = emb.shape()[1] as i64;
-
- let emb = load_emb(&weights, "emb", emb_size)?;
-
- let mut blocks = Vec::with_capacity(num_layers);
- for layer in 0..num_layers {
- let prefix = format!("blocks.{}", layer);
- blocks.push(Block::load(&weights, &prefix, emb_size, layer as i64)?);
- }
-
- Ok(Self {
- emb,
- blocks,
- ln_out: load_ln(&weights, "ln_out", emb_size)?,
- head: load_linear(&weights, "head")?,
- })
- }
-
- fn load_to_device(&mut self, layer_fraction: f64, device: tch::Device, kind: tch::Kind) {
- let layer_fraction = layer_fraction.max(0.0).min(1.0);
-
- self.emb.ws = self.emb.ws.to_kind(kind).to(device);
-
- let layers_to_move = (self.blocks.len() as f64 * layer_fraction).ceil() as usize;
-
- for block in self.blocks.iter_mut().take(layers_to_move) {
- block.load_to_device(device, kind);
- }
-
- if layer_fraction == 1.0 {
- self.ln_out.ws = self.ln_out.ws.as_ref().map(|t| t.to_kind(kind).to(device));
- self.ln_out.bs = self.ln_out.bs.as_ref().map(|t| t.to_kind(kind).to(device));
-
- self.head.ws = self.head.ws.to_kind(kind).to(device);
- self.head.bs = self.head.bs.as_ref().map(|t| t.to_kind(kind).to(device));
- }
- }
-
- fn init_state(&self) -> Tensor {
- let t = Tensor::zeros(
- [self.blocks.len() as i64 * 5, self.emb.ws.size()[1]],
- (tch::Kind::Float, tch::Device::Cpu),
- );
-
- for i in 0..self.blocks.len() as i64 {
- t.i(5 * i + 4).copy_(&Tensor::from_slice(&[-1e30f32]))
- }
-
- t
- }
-
- fn forward(&self, token: i64, state: Option<&Tensor>) -> (Tensor, Tensor) {
- tch::no_grad(|| self.forward_grad(token, state))
- }
-
- fn forward_grad(&self, token: i64, state: Option<&Tensor>) -> (Tensor, Tensor) {
- let mut tokens = Tensor::from_slice(&[token]);
-
- if self.emb.ws.device() != tokens.device() {
- tokens = tokens.to(self.emb.ws.device());
- }
-
- let mut x = self.emb.forward_t(&tokens, false);
-
- let mut state = match state {
- Some(state) => state.shallow_clone(),
- None => self.init_state(),
- };
-
- for block in &self.blocks {
- (x, state) = block.forward(x, state);
- }
-
- x = self.ln_out.forward_t(&x, false);
- x = self.head.forward_t(&x, false);
-
- (x, state)
- }
-}
-
pub struct Alice {
inner: Arc<RawModel>,
tokenizer: Arc<Tokenizer>,
@@ -691,253 +254,13 @@ impl Alice {
}
}
-pub enum TokenGeneratorState {
- InProgress,
- Finished,
-}
-
#[derive(Clone)]
-struct ModelState {
+pub struct ModelState {
state: ClonableTensor,
next_token: i64,
}
-pub struct AliceTokenGenerator {
- model: Arc<RawModel>,
- // state needs to be a queue of size 2 to have the ability
- // to generate a new token if the current one is banned
- states: LeakyQueue<ModelState>,
- // the state of the model after each search result has been
- // loaded. This allows us to go back to the end of the previous search
- // and force speak if the model wants us to search for the same query again.
- end_search_states: Vec<LeakyQueue<ModelState>>,
- end_tokens: Vec<i64>,
- generation_state: TokenGeneratorState,
- max_new_tokens: Option<usize>,
- num_generated_tokens: usize,
- banned_tokens: Vec<i64>,
-}
-
-impl AliceTokenGenerator {
- fn new(
- model: Arc<RawModel>,
- end_tokens: Vec<i64>,
- state: Tensor,
- tokens: &[i64],
- max_new_tokens: Option<usize>,
- ) -> Result<Self> {
- if tokens.is_empty() {
- return Err(Error::EmptyInput);
- }
-
- // this must be 2, otherwise `.front()` and `.back()` calls are wrong throughout the generation code
- let num_states = 2;
-
- let mut states = LeakyQueue::new(num_states);
-
- let state = ClonableTensor(state);
-
- for _ in 0..num_states {
- states.push(ModelState {
- state: state.clone(),
- next_token: -1,
- });
- }
-
- let mut generator = Self {
- model,
- states,
- end_tokens,
- generation_state: TokenGeneratorState::InProgress,
- max_new_tokens,
- num_generated_tokens: 0,
- end_search_states: Vec::new(),
- banned_tokens: Vec::new(),
- };
- generator.load_tokens(tokens);
-
- Ok(generator)
- }
-
- pub fn load_tokens(&mut self, tokens: &[i64]) {
- for token in tokens {
- let (logits, new_state) = self
- .model
- .forward(*token, Some(&self.states.back().unwrap().state.0));
- let probs = logits
- .softmax(-1, Kind::Float)
- .squeeze()
- .to(tch::Device::Cpu);
-
- let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
- self.states.push(ModelState {
- state: ClonableTensor(new_state),
- next_token,
- });
-
- self.states.front_mut().unwrap().next_token = *token;
- }
- }
-
- fn load_search_result(&mut self, tokens: &[i64]) {
- self.load_tokens(tokens);
- self.end_search_states.push(self.states.clone());
- }
-
- fn go_to_last_search(&mut self) {
- if let Some(last_search) = self.end_search_states.pop() {
- self.states = last_search;
- }
- }
-
- pub fn set_end_tokens(&mut self, tokens: &[i64]) {
- if self.end_tokens != tokens {
- self.end_tokens = tokens.to_vec();
- }
- }
-
- pub fn set_max_new_tokens(&mut self, max_new_tokens: Option<usize>) {
- if self.max_new_tokens != max_new_tokens {
- self.num_generated_tokens = 0;
- self.max_new_tokens = max_new_tokens;
- }
- }
-
- pub fn reset_tokens_counter(&mut self) {
- self.num_generated_tokens = 0;
- }
-
- pub fn set_banned_tokens(&mut self, tokens: &[i64]) {
- if self.banned_tokens != tokens {
- self.banned_tokens = tokens.to_vec();
- }
- }
-}
-
-impl Iterator for AliceTokenGenerator {
- type Item = i64;
-
- fn next(&mut self) -> Option<Self::Item> {
- match self.generation_state {
- TokenGeneratorState::InProgress => {
- let s = self.states.back().unwrap();
- let mut token = s.next_token;
- let mut state = s.state.clone();
-
- if self.end_tokens.contains(&token) {
- self.generation_state = TokenGeneratorState::Finished;
- return None;
- }
-
- if let Some(max_new_tokens) = self.max_new_tokens {
- if self.num_generated_tokens >= max_new_tokens {
- if let Some(tok) = self.end_tokens.first() {
- token = *tok;
- }
-
- self.generation_state = TokenGeneratorState::Finished;
- }
- }
-
- if self.banned_tokens.contains(&token) {
- let (output, new_state) = {
- let s = self.states.front().unwrap();
- self.model.forward(s.next_token, Some(&s.state.0))
- };
-
- let output = output.squeeze();
-
- let probs = output
- .softmax(-1, Kind::Float)
- .squeeze()
- .to(tch::Device::Cpu);
-
- for banned_token in &self.banned_tokens {
- probs
- .i(*banned_token)
- .copy_(&Tensor::from_slice(&[0.0]).squeeze());
- }
-
- let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
- token = next_token;
- state = ClonableTensor(new_state);
- }
-
- let (output, state) = self.model.forward(token, Some(&state.0));
- let output = output.squeeze();
-
- let probs = output
- .softmax(-1, Kind::Float)
- .squeeze()
- .to(tch::Device::Cpu);
-
- for banned_token in &self.banned_tokens {
- probs
- .i(*banned_token)
- .copy_(&Tensor::from_slice(&[0.0]).squeeze());
- }
-
- let next_token = llm_utils::sample_typical(probs, TEMP, TAU);
- self.states.push(ModelState {
- state: ClonableTensor(state),
- next_token,
- });
-
- self.num_generated_tokens += 1;
-
- Some(token)
- }
- TokenGeneratorState::Finished => None,
- }
- }
-}
-
-pub struct AliceStringGenerator {
- token_generator: AliceTokenGenerator,
- tokenizer: Arc<Tokenizer>,
- tokens: Vec<i64>,
-}
-
-impl Iterator for AliceStringGenerator {
- type Item = String;
-
- fn next(&mut self) -> Option<Self::Item> {
- loop {
- match self.token_generator.next() {
- Some(token) => {
- self.tokens.push(token);
- if let Some(s) = self.tokenizer.decode(&self.tokens).ok().and_then(|s| {
- if !s.contains('\u{fffd}') {
- // valid utf-8 string
- Some(s)
- } else {
- None
- }
- }) {
- self.tokens.clear();
- return Some(s);
- }
- }
- None if !self.tokens.is_empty() => {
- let s = self.tokenizer.decode(&self.tokens).ok().and_then(|s| {
- if !s.contains('\u{fffd}') {
- Some(s)
- } else {
- None
- }
- });
-
- self.tokens.clear();
-
- return s;
- }
- None => return None,
- }
- }
- }
-}
-
-struct Tokenizer {
+pub struct Tokenizer {
tokenizer: tokenizers::Tokenizer,
}
@@ -969,199 +292,6 @@ impl Tokenizer {
}
}
-#[derive(Debug)]
-pub enum RawAction {
- Search { query: Vec<i64> },
- Speak { token: i64 },
-}
-
-enum State {
- Thought {
- search: TransitionValidator,
- speaking: TransitionValidator,
- },
- QueryBuild {
- tokens: Vec<i64>,
- end: AnyTransitionValidator,
- },
- Speaking {
- banned_tokens: TTLBannedTokens,
- end: AnyTransitionValidator,
- max_new_tokens: Option<usize>,
- },
-}
-
-struct TTLBannedTokens {
- tokens: Vec<i64>,
- orig_ttl: usize,
- ttl: usize,
-}
-
-impl TTLBannedTokens {
- fn new(tokens: Vec<i64>, ttl: usize) -> Self {
- Self {
- tokens,
- ttl,
- orig_ttl: ttl,
- }
- }
-
- fn reset_ttl(&mut self) {
- self.ttl = self.orig_ttl;
- }
-
- fn tick(&mut self) {
- if self.ttl > 0 {
- self.ttl -= 1;
- }
- }
-
- fn set_banned_tokens(&self, token_generator: &mut AliceTokenGenerator) {
- if self.ttl > 0 {
- token_generator.set_banned_tokens(&self.tokens);
- } else {
- token_generator.set_banned_tokens(&[]);
- }
- }
-}
-
-impl State {
- fn new_thought(tokenizer: &Tokenizer) -> Self {
- let search = TransitionValidator::new(tokenizer.encode("Search:".to_string()).unwrap());
- let speaking = TransitionValidator::new(tokenizer.encode("Alice:".to_string()).unwrap());
-
- Self::Thought { search, speaking }
- }
-
- fn new_query_build(tokenizer: &Tokenizer) -> Self {
- let new_line = TransitionValidator::new(tokenizer.encode("\n".to_string()).unwrap());
- let end_of_text =
- TransitionValidator::new(tokenizer.encode("<|endoftext|>".to_string()).unwrap());
- let end = AnyTransitionValidator::new(vec![new_line, end_of_text]);
-
- Self::QueryBuild {
- tokens: Vec::new(),
- end,
- }
- }
-
- fn new_speaking(initially_banned_tokens: Vec<i64>) -> Self {
- let end = TransitionValidator::new(vec![0]);
- let double_newline = TransitionValidator::new(vec![187, 187]);
- let double_newline_single_tok = TransitionValidator::new(vec![535]);
-
- let end = AnyTransitionValidator::new(vec![end, double_newline, double_newline_single_tok]);
-
- Self::Speaking {
- end,
- max_new_tokens: Some(1024),
- banned_tokens: TTLBannedTokens::new(initially_banned_tokens, 2), // the tokens should only be banned at the start of generation
- }
- }
-}
-
-pub struct RawActionGenerator {
- state: State,
- tokenizer: Arc<Tokenizer>,
- token_generator: AliceTokenGenerator,
- queries_performed: usize,
- max_num_queries: usize,
-}
-
-impl RawActionGenerator {
- fn new(token_generator: AliceTokenGenerator, tokenizer: Arc<Tokenizer>) -> Self {
- let state = State::new_thought(&tokenizer);
-
- let mut token_generator = token_generator;
- token_generator.set_end_tokens(&[]);
-
- Self {
- state,
- tokenizer,
- token_generator,
- queries_performed: 0,
- max_num_queries: 3,
- }
- }
-
- fn force_speaking(&mut self) {
- self.token_generator.load_tokens(
- &self
- .tokenizer
- .encode("Alice[thought]: I now know the final answer.\n\n".to_string())
- .unwrap(),
- );
-
- self.state = State::new_speaking(vec![
- self.tokenizer.encode(" [thought]:".to_string()).unwrap()[0],
- self.tokenizer.encode("[thought]:".to_string()).unwrap()[0],
- ]);
-
- self.token_generator
- .load_tokens(&self.tokenizer.encode("Alice:".to_string()).unwrap());
- }
-}
-
-impl Iterator for RawActionGenerator {
- type Item = RawAction;
-
- fn next(&mut self) -> Option<Self::Item> {
- loop {
- match &mut self.state {
- State::Thought { search, speaking } => {
- let token = self.token_generator.next()?;
- self.token_generator.set_max_new_tokens(Some(1024));
-
- if self.queries_performed >= self.max_num_queries {
- self.token_generator.reset_tokens_counter();
- self.force_speaking();
- } else if search.validate(token) {
- self.token_generator.reset_tokens_counter();
- self.state = State::new_query_build(&self.tokenizer);
- } else if speaking.validate(token) {
- self.token_generator.reset_tokens_counter();
- self.state = State::new_speaking(vec![]);
- }
- }
- State::QueryBuild { tokens, end } => {
- let token = self.token_generator.next()?;
- self.token_generator.set_max_new_tokens(Some(1024));
- tokens.push(token);
-
- if end.validate(token) {
- let query = tokens.clone();
- *tokens = Vec::new();
- self.token_generator.reset_tokens_counter();
- self.state = State::new_thought(&self.tokenizer);
- self.queries_performed += 1;
- return Some(RawAction::Search { query });
- }
- }
- State::Speaking {
- end,
- max_new_tokens,
- banned_tokens,
- } => {
- self.token_generator.set_max_new_tokens(*max_new_tokens);
- banned_tokens.set_banned_tokens(&mut self.token_generator);
-
- let token = self.token_generator.next()?;
-
- if end.validate(token) {
- banned_tokens.reset_ttl();
- self.token_generator.reset_tokens_counter();
- self.state = State::new_thought(&self.tokenizer);
- return None;
- } else {
- banned_tokens.tick();
- return Some(RawAction::Speak { token });
- }
- }
- }
- }
- }
-}
-
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct SimplifiedWebsite {
pub title: String,
@@ -1201,7 +331,7 @@ impl From<SimplifiedWebsite> for ModelWebsite {
}
}
-struct Searcher {
+pub struct Searcher {
url: String,
optic_url: Option<String>,
summarizer: Arc<ExtractiveSummarizer>,
@@ -1250,91 +380,6 @@ impl Searcher {
}
}
-enum Action {
- Search { query: String },
- Speak { text: String },
-}
-
-struct ActionGenerator {
- raw: RawActionGenerator,
- tokens_to_speak: Vec<i64>,
-}
-
-impl ActionGenerator {
- fn new(raw_action_gen: RawActionGenerator) -> ActionGenerator {
- Self {
- raw: raw_action_gen,
- tokens_to_speak: Vec::new(),
- }
- }
-}
-
-impl Iterator for ActionGenerator {
- type Item = Action;
-
- fn next(&mut self) -> Option<Self::Item> {
- loop {
- let action = self.raw.next();
-
- match action {
- Some(RawAction::Search { query }) => {
- return Some(Action::Search {
- query: self
- .raw
- .tokenizer
- .decode(&query)
- .unwrap()
- .trim()
- .to_ascii_lowercase(),
- });
- }
- Some(RawAction::Speak { token }) => {
- self.tokens_to_speak.push(token);
-
- if let Some(s) = self
- .raw
- .tokenizer
- .decode(&self.tokens_to_speak)
- .ok()
- .and_then(|s| {
- if !s.contains('\u{fffd}') {
- // valid utf-8 string
- Some(s)
- } else {
- None
- }
- })
- {
- self.tokens_to_speak.clear();
- return Some(Action::Speak { text: s });
- }
- }
- None if !self.tokens_to_speak.is_empty() => {
- let s = self
- .raw
- .tokenizer
- .decode(&self.tokens_to_speak)
- .ok()
- .and_then(|s| {
- if !s.contains('\u{fffd}') {
- Some(s)
- } else {
- None
- }
- });
-
- self.tokens_to_speak.clear();
-
- return s.map(|s| Action::Speak { text: s });
- }
- None => {
- return None;
- }
- }
- }
- }
-}
-
#[derive(Debug, serde::Serialize, serde::Deserialize)]
/// base64 encoded `EncryptedState`
pub struct EncodedEncryptedState(String);
@@ -1355,141 +400,6 @@ impl EncodedEncryptedState {
}
}
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
-#[serde(tag = "@type", rename_all = "camelCase")]
-pub enum ExecutionState {
- BeginSearch {
- query: String,
- },
- SearchResult {
- query: String,
- result: Vec<SimplifiedWebsite>,
- },
- Speaking {
- text: String,
- },
- Done {
- state: EncodedEncryptedState,
- },
-}
-
-pub struct ActionExecutor {
- generator: ActionGenerator,
- searcher: Searcher,
- query_to_search: Option<String>,
- queries_performed: Vec<String>,
- has_finished: bool,
- encryption_key: Key<Aes256Gcm>,
-}
-
-unsafe impl Send for ActionExecutor {}
-
-impl ActionExecutor {
- fn new(
- action_gen: ActionGenerator,
- searcher: Searcher,
- encryption_key: Key<Aes256Gcm>,
- ) -> Self {
- ActionExecutor {
- generator: action_gen,
- searcher,
- query_to_search: None,
- queries_performed: Vec::new(),
- has_finished: false,
- encryption_key,
- }
- }
- pub fn state(&self) -> Option<Tensor> {
- self.generator
- .raw
- .token_generator
- .states
- .back()
- .map(|s| s.state.clone().0)
- }
-
- fn load_search_result(&mut self, result: &[SimplifiedWebsite]) {
- let result = result
- .iter()
- .map(|r| ModelWebsite::from(r.clone()))
- .collect::<Vec<_>>();
-
- let json = serde_json::to_string(&result).unwrap();
-
- let tokens = self
- .generator
- .raw
- .tokenizer
- .encode(format!("Result: {json}<|endoftext|>\n\n"))
- .unwrap();
- self.generator
- .raw
- .token_generator
- .load_search_result(&tokens);
-
- let tokens = self
- .generator
- .raw
- .tokenizer
- .encode("Alice[thought]:".to_string())
- .unwrap();
- self.generator.raw.token_generator.load_tokens(&tokens);
- }
-}
-
-impl Iterator for ActionExecutor {
- type Item = ExecutionState;
-
- fn next(&mut self) -> Option<Self::Item> {
- if self.has_finished {
- return None;
- }
-
- if let Some(query) = self.query_to_search.take() {
- if self.queries_performed.contains(&query) {
- self.generator.raw.token_generator.go_to_last_search();
- self.generator.raw.force_speaking();
- } else {
- let res = self.searcher.search(&query).unwrap_or_default();
-
- tracing::debug!("loading search results");
- self.load_search_result(&res);
- tracing::debug!("done loading search results");
- self.queries_performed.push(query.clone());
-
- return Some(ExecutionState::SearchResult { query, result: res });
- }
- }
-
- let action = self.generator.next();
-
- match action {
- Some(Action::Search { query }) => {
- self.query_to_search = Some(query.clone());
- Some(ExecutionState::BeginSearch { query })
- }
- Some(Action::Speak { text }) => Some(ExecutionState::Speaking { text }),
- None if !self.has_finished => {
- self.has_finished = true;
-
- let state: Vec<Vec<f32>> = self
- .state()
- .unwrap()
- .to_device(tch::Device::Cpu)
- .try_into()
- .unwrap();
-
- let encrypted = EncryptedState::encrypt(state, &self.encryption_key).unwrap();
-
- Some(ExecutionState::Done {
- state: EncodedEncryptedState::encode(encrypted),
- })
- }
- None => None,
- }
- }
-}
-
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct EncryptedState {
nonce: Vec<u8>,
@@ -1498,7 +408,7 @@ pub struct EncryptedState {
}
impl EncryptedState {
- fn encrypt(state: Vec<Vec<f32>>, key: &Key<Aes256Gcm>) -> Result<Self> {
+ fn encrypt(state: Vec<Vec<f32>>, key: &Key<Aes256Gcm>) -> Self {
let compressed = compress_state(state);
let cipher = Aes256Gcm::new(key);
@@ -1508,10 +418,10 @@ impl EncryptedState {
let ciphertext = cipher.encrypt(&nonce, bytes.as_slice()).unwrap();
- Ok(Self {
+ Self {
nonce: nonce.to_vec(),
ciphertext,
- })
+ }
}
fn decrypt(&self, key: &Key<Aes256Gcm>) -> Result<Vec<Vec<f32>>> {
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/raw_model.rs
new file mode 100644
index 00000000..e86ebf3c
--- /dev/null
+++ b/core/src/alice/raw_model.rs
@@ -0,0 +1,459 @@
+use std::{fs::OpenOptions, path::Path};
+
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use super::{Error, Result};
+use safetensors::SafeTensors;
+use tch::{
+ nn::{embedding, layer_norm, Embedding, LayerNorm, LayerNormConfig, Linear, ModuleT, VarStore},
+ IndexOp, Kind, Tensor,
+};
+
+const NUM_TOKENS: i64 = 50277;
+
+fn load_linear(weights: &SafeTensors, prefix: &str) -> Result<Linear> {
+ let ws = weights.tensor(&format!("{prefix}.weight"))?.try_into()?;
+
+ let mut linear = Linear { ws, bs: None };
+
+ if let Ok(bias_tensor) = weights.tensor(&format!("{prefix}.bias")) {
+ linear.bs = Some(bias_tensor.try_into()?);
+ }
+
+ if let Some(bs) = &mut linear.bs {
+ *bs = bs.to_kind(Kind::Float);
+ }
+
+ linear.ws = linear.ws.to_kind(Kind::Float);
+
+ Ok(linear)
+}
+
+fn load_emb(weights: &SafeTensors, prefix: &str, emb_size: i64) -> Result<Embedding> {
+ let vars = VarStore::new(tch::Device::Cpu);
+
+ let ws = weights.tensor(&format!("{prefix}.weight"))?;
+ let ws: Tensor = ws.try_into()?;
+
+ vars.variables_
+ .lock()
+ .unwrap()
+ .named_variables
+ .insert("weight".to_string(), ws.shallow_clone());
+
+ let config = tch::nn::EmbeddingConfig::default();
+
+ let mut emb = embedding(vars.root(), NUM_TOKENS, emb_size, config);
+
+ emb.ws = ws.to_kind(Kind::Float);
+
+ Ok(emb)
+}
+
+fn load_ln(weights: &SafeTensors, prefix: &str, emb_size: i64) -> Result<LayerNorm> {
+ let vars = VarStore::new(tch::Device::Cpu);
+
+ if let Ok(ws) = weights.tensor(&format!("{prefix}.weight")) {
+ vars.variables_
+ .lock()
+ .unwrap()
+ .named_variables
+ .insert("weight".to_string(), ws.try_into()?);
+ }
+
+ if let Ok(bs) = weights.tensor(&format!("{prefix}.bias")) {
+ vars.variables_
+ .lock()
+ .unwrap()
+ .named_variables
+ .insert("bias".to_string(), bs.try_into()?);
+ }
+
+ let config = LayerNormConfig::default();
+
+ let mut ln = layer_norm(vars.root(), vec![emb_size], config);
+
+ if let Ok(ws) = weights.tensor(&format!("{prefix}.weight")) {
+ let ws: Tensor = ws.try_into()?;
+ ln.ws = Some(ws.to_kind(Kind::Float));
+ }
+
+ if let Ok(bs) = weights.tensor(&format!("{prefix}.bias")) {
+ let bs: Tensor = bs.try_into()?;
+ ln.bs = Some(bs.to_kind(Kind::Float));
+ }
+
+ ln.normalized_shape = vec![emb_size];
+
+ Ok(ln)
+}
+
+struct TimeMix {
+ time_decay: Tensor,
+ time_first: Tensor,
+
+ time_mix_k: Tensor,
+ time_mix_v: Tensor,
+ time_mix_r: Tensor,
+
+ key: Linear,
+ value: Linear,
+ receptance: Linear,
+ output: Linear,
+
+ block_idx: i64,
+}
+impl TimeMix {
+ fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {
+ let x = x.squeeze();
+ let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_k);
+ let xv = &x * &self.time_mix_v + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_v);
+ let xr = &x * &self.time_mix_r + state.i(5 * self.block_idx + 1) * (1 - &self.time_mix_r);
+
+ state.i(5 * self.block_idx + 1).copy_(&x);
+
+ let r = self.receptance.forward_t(&xr, false).sigmoid();
+ let k = self.key.forward_t(&xk, false);
+ let v = self.value.forward_t(&xv, false);
+
+ let aa = state.i(5 * self.block_idx + 2);
+ let bb = state.i(5 * self.block_idx + 3);
+ let pp = state.i(5 * self.block_idx + 4);
+
+ let ww = &self.time_first + &k;
+ let qq = pp.maximum(&ww);
+ let e1 = (&pp - &qq).exp();
+ let e2 = (ww - &qq).exp();
+
+ let a = &e1 * &aa + &e2 * &v;
+ let b = &e1 * &bb + &e2;
+ let wkv = a / b;
+ let ww = pp - self.time_decay.exp();
+ let qq = ww.maximum(&k);
+ let e1 = (ww - &qq).exp();
+ let e2 = (k - &qq).exp();
+
+ state
+ .i(5 * self.block_idx + 2)
+ .copy_(&(&e1 * &aa + &e2 * &v).squeeze());
+ state
+ .i(5 * self.block_idx + 3)
+ .copy_(&(&e1 * &bb + &e2).squeeze());
+ state.i(5 * self.block_idx + 4).copy_(&qq.squeeze());
+
+ self.output.forward_t(&(wkv * &r), false)
+ }
+
+ fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {
+ self.time_decay = self.time_decay.to_kind(kind).to(device);
+ self.time_first = self.time_first.to_kind(kind).to(device);
+
+ self.time_mix_k = self.time_mix_k.to_kind(kind).to(device);
+ self.time_mix_v = self.time_mix_v.to_kind(kind).to(device);
+ self.time_mix_r = self.time_mix_r.to_kind(kind).to(device);
+
+ self.key.ws = self.key.ws.to_kind(kind).to(device);
+ self.key.bs = self.key.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+
+ self.value.ws = self.value.ws.to_kind(kind).to(device);
+ self.value.bs = self.value.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+
+ self.receptance.ws = self.receptance.ws.to_kind(kind).to(device);
+ self.receptance.bs = self
+ .receptance
+ .bs
+ .as_ref()
+ .map(|t| t.to_kind(kind).to(device));
+
+ self.output.ws = self.output.ws.to_kind(kind).to(device);
+ self.output.bs = self.output.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ }
+}
+
+struct ChannelMix {
+ time_mix_k: Tensor,
+ time_mix_r: Tensor,
+
+ key: Linear,
+ value: Linear,
+
+ receptance: Linear,
+
+ block_idx: i64,
+}
+impl ChannelMix {
+ fn forward(&self, x: &Tensor, state: &mut Tensor) -> Tensor {
+ let x = x.squeeze();
+ let xk = &x * &self.time_mix_k + state.i(5 * self.block_idx) * (1 - &self.time_mix_k);
+ let xr = &x * &self.time_mix_r + state.i(5 * self.block_idx) * (1 - &self.time_mix_r);
+
+ state.i(5 * self.block_idx).copy_(&x);
+
+ let r = self.receptance.forward_t(&xr, false).sigmoid();
+ let k = self.key.forward_t(&xk, false).relu().square();
+
+ r * self.value.forward_t(&k, false)
+ }
+
+ fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {
+ self.time_mix_k = self.time_mix_k.to_kind(kind).to(device);
+ self.time_mix_r = self.time_mix_r.to_kind(kind).to(device);
+
+ self.key.ws = self.key.ws.to_kind(kind).to(device);
+ self.key.bs = self.key.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+
+ self.value.ws = self.value.ws.to_kind(kind).to(device);
+ self.value.bs = self.value.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+
+ self.receptance.ws = self.receptance.ws.to_kind(kind).to(device);
+ self.receptance.bs = self
+ .receptance
+ .bs
+ .as_ref()
+ .map(|t| t.to_kind(kind).to(device));
+ }
+}
+
+struct Block {
+ ln0: Option<LayerNorm>,
+ ln1: LayerNorm,
+ ln2: LayerNorm,
+
+ att: TimeMix,
+ ffn: ChannelMix,
+
+ device: tch::Device,
+}
+
+impl Block {
+ fn load(weights: &SafeTensors, prefix: &str, emb_size: i64, block_idx: i64) -> Result<Self> {
+ let mut ln0 = None;
+
+ let names = weights.names();
+
+ if names.contains(&&format!("{prefix}.ln0.weight")) {
+ ln0 = Some(load_ln(weights, &format!("{prefix}.ln0"), emb_size)?);
+ }
+
+ let ln1 = load_ln(weights, &format!("{prefix}.ln1"), emb_size)?;
+ let ln2 = load_ln(weights, &format!("{prefix}.ln2"), emb_size)?;
+
+ let att = TimeMix {
+ time_decay: weights
+ .tensor(&format!("{prefix}.att.time_decay"))?
+ .try_into()?,
+ time_first: weights
+ .tensor(&format!("{prefix}.att.time_first"))?
+ .try_into()?,
+ time_mix_k: weights
+ .tensor(&format!("{prefix}.att.time_mix_k"))?
+ .try_into()?,
+ time_mix_v: weights
+ .tensor(&format!("{prefix}.att.time_mix_v"))?
+ .try_into()?,
+ time_mix_r: weights
+ .tensor(&format!("{prefix}.att.time_mix_r"))?
+ .try_into()?,
+ key: load_linear(weights, &format!("{prefix}.att.key"))?,
+ value: load_linear(weights, &format!("{prefix}.att.value"))?,
+ receptance: load_linear(weights, &format!("{prefix}.att.receptance"))?,
+ output: load_linear(weights, &format!("{prefix}.att.output"))?,
+ block_idx,
+ };
+
+ let ffn = ChannelMix {
+ time_mix_k: weights
+ .tensor(&format!("{prefix}.ffn.time_mix_k"))?
+ .try_into()?,
+ time_mix_r: weights
+ .tensor(&format!("{prefix}.ffn.time_mix_r"))?
+ .try_into()?,
+ key: load_linear(weights, &format!("{prefix}.ffn.key"))?,
+ value: load_linear(weights, &format!("{prefix}.ffn.value"))?,
+ receptance: load_linear(weights, &format!("{prefix}.ffn.receptance"))?,
+ block_idx,
+ };
+
+ Ok(Self {
+ ln0,
+ ln1,
+ ln2,
+ att,
+ ffn,
+ device: tch::Device::Cpu,
+ })
+ }
+
+ fn forward(&self, mut x: Tensor, mut state: Tensor) -> (Tensor, Tensor) {
+ if x.device() != self.device {
+ x = x.to(self.device);
+ }
+
+ if state.device() != self.device {
+ state = state.to(self.device);
+ }
+
+ x = match &self.ln0 {
+ Some(ln) => ln.forward_t(&x, false),
+ None => x,
+ };
+
+ x += self
+ .att
+ .forward(&self.ln1.forward_t(&x, false), &mut state)
+ .reshape([1, -1]);
+ x += self
+ .ffn
+ .forward(&self.ln2.forward_t(&x, false), &mut state)
+ .reshape([1, -1]);
+
+ (x, state)
+ }
+
+ fn load_to_device(&mut self, device: tch::Device, kind: tch::Kind) {
+ self.device = device;
+
+ if let Some(ln) = &mut self.ln0 {
+ ln.ws = ln.ws.as_ref().map(|t| t.to_kind(kind).to(device));
+ ln.bs = ln.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ }
+
+ self.att.load_to_device(device, kind);
+ self.ffn.load_to_device(device, kind);
+
+ self.ln1.ws = self.ln1.ws.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.ln1.bs = self.ln1.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+
+ self.ln2.ws = self.ln2.ws.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.ln2.bs = self.ln2.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ }
+}
+
+pub struct RawModel {
+ emb: Embedding,
+ blocks: Vec<Block>,
+ ln_out: LayerNorm,
+ head: Linear,
+}
+
+impl RawModel {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<RawModel> {
+ // SAFETY
+ // broadcasting: please don't modify the file while we're reading it.
+ let mmap = unsafe {
+ let file = OpenOptions::new()
+ .read(true)
+ .write(true)
+ .create(true)
+ .open(path)
+ .map_err(Error::Io)?;
+
+ memmap2::MmapOptions::new()
+ .map_mut(&file)
+ .map_err(Error::Io)?
+ };
+
+ let weights = SafeTensors::deserialize(&mmap)?;
+ let names = weights.names();
+
+ let num_layers = names
+ .iter()
+ .filter(|name| name.starts_with("blocks"))
+ .map(|name| name.split('.').nth(1).unwrap().parse::<usize>().unwrap())
+ .max()
+ .unwrap()
+ + 1;
+
+ let emb = weights.tensor("emb.weight")?;
+ let emb_size = emb.shape()[1] as i64;
+
+ let emb = load_emb(&weights, "emb", emb_size)?;
+
+ let mut blocks = Vec::with_capacity(num_layers);
+ for layer in 0..num_layers {
+ let prefix = format!("blocks.{}", layer);
+ blocks.push(Block::load(&weights, &prefix, emb_size, layer as i64)?);
+ }
+
+ Ok(Self {
+ emb,
+ blocks,
+ ln_out: load_ln(&weights, "ln_out", emb_size)?,
+ head: load_linear(&weights, "head")?,
+ })
+ }
+
+ pub fn load_to_device(&mut self, layer_fraction: f64, device: tch::Device, kind: tch::Kind) {
+ let layer_fraction = layer_fraction.max(0.0).min(1.0);
+
+ self.emb.ws = self.emb.ws.to_kind(kind).to(device);
+
+ let layers_to_move = (self.blocks.len() as f64 * layer_fraction).ceil() as usize;
+
+ for block in self.blocks.iter_mut().take(layers_to_move) {
+ block.load_to_device(device, kind);
+ }
+
+ if layer_fraction == 1.0 {
+ self.ln_out.ws = self.ln_out.ws.as_ref().map(|t| t.to_kind(kind).to(device));
+ self.ln_out.bs = self.ln_out.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+
+ self.head.ws = self.head.ws.to_kind(kind).to(device);
+ self.head.bs = self.head.bs.as_ref().map(|t| t.to_kind(kind).to(device));
+ }
+ }
+
+ pub fn init_state(&self) -> Tensor {
+ let t = Tensor::zeros(
+ [self.blocks.len() as i64 * 5, self.emb.ws.size()[1]],
+ (tch::Kind::Float, tch::Device::Cpu),
+ );
+
+ for i in 0..self.blocks.len() as i64 {
+ t.i(5 * i + 4).copy_(&Tensor::from_slice(&[-1e30f32]))
+ }
+
+ t
+ }
+
+ pub fn forward(&self, token: i64, state: Option<&Tensor>) -> (Tensor, Tensor) {
+ tch::no_grad(|| self.forward_grad(token, state))
+ }
+
+ pub fn forward_grad(&self, token: i64, state: Option<&Tensor>) -> (Tensor, Tensor) {
+ let mut tokens = Tensor::from_slice(&[token]);
+
+ if self.emb.ws.device() != tokens.device() {
+ tokens = tokens.to(self.emb.ws.device());
+ }
+
+ let mut x = self.emb.forward_t(&tokens, false);
+
+ let mut state = match state {
+ Some(state) => state.shallow_clone(),
+ None => self.init_state(),
+ };
+
+ for block in &self.blocks {
+ (x, state) = block.forward(x, state);
+ }
+
+ x = self.ln_out.forward_t(&x, false);
+ x = self.head.forward_t(&x, false);
+
+ (x, state)
+ }
+}