web search done right
openai alice
Mikkel Denker8f3853d2d12b
diff --git a/Cargo.lock b/Cargo.lockindex 92248fe0..b29cfedc 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1525,6 +1525,12 @@ version = "0.3.28"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"+[[package]]+name = "futures-timer"+version = "3.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"+[[package]]name = "futures-util"version = "0.3.28"@@ -3346,6 +3352,22 @@ dependencies = ["winreg",]+[[package]]+name = "reqwest-eventsource"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f03f570355882dd8d15acc3a313841e6e90eddbc76a93c748fd82cc13ba9f51"+dependencies = [+ "eventsource-stream",+ "futures-core",+ "futures-timer",+ "mime",+ "nom",+ "pin-project-lite",+ "reqwest",+ "thiserror",+]+[[package]]name = "rkyv"version = "0.7.41"@@ -3851,6 +3873,7 @@ dependencies = ["rayon","regex","reqwest",+ "reqwest-eventsource","rkyv","rocksdb","rust-stemmers",diff --git a/Cargo.toml b/Cargo.tomlindex c61062a9..b9ebf59c 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -22,6 +22,7 @@ itertools = "0.10.3"rayon = "1.5.3"half = { version = "2.2.1", features = ["serde"] }eventsource-stream = "0.2.3"+reqwest-eventsource = "0.4.0"reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }serde = { version = "1.0.137", features = ["rc", "derive"] }serde_json = "1.0.81"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 85232e4c..93b29dd0 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -86,6 +86,7 @@ eventsource-stream = { workspace = true }rand = { workspace = true }half = { workspace = true }aes-gcm = { workspace = true }+reqwest-eventsource = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/core/examples/alice.rs b/core/examples/alice.rsindex cac805d0..5e73bb48 100644--- a/core/examples/alice.rs+++ b/core/examples/alice.rs@@ -1,6 +1,9 @@use std::io::Write;-use stract::alice::{generate::ExecutionState, AcceleratorConfig, Alice};+use stract::alice::{+ local::{AcceleratorConfig, Alice},+ ExecutionState,+};#[tokio::main]async fn main() {diff --git a/core/src/alice/generate.rs b/core/src/alice/local/generate.rssimilarity index 97%rename from core/src/alice/generate.rsrename to core/src/alice/local/generate.rsindex 7663ec48..8fa675bf 100644--- a/core/src/alice/generate.rs+++ b/core/src/alice/local/generate.rs@@ -20,14 +20,16 @@ use aes_gcm::{Aes256Gcm, Key};use tch::{IndexOp, Kind, Tensor};use crate::{+ alice::ExecutionState,leaky_queue::LeakyQueue,llm_utils::{self, ClonableTensor},};use super::{raw_model::RawModel, AnyTransitionValidator, EncodedEncryptedState, EncryptedState, Error,- ModelState, ModelWebsite, Result, Searcher, SimplifiedWebsite, Tokenizer, TransitionValidator,+ ModelState, Result, Searcher, Tokenizer, TransitionValidator,};+use crate::alice::{ModelWebsite, SimplifiedWebsite};// const TAU: f64 = 0.3;const TEMP: f64 = 0.4;@@ -586,24 +588,6 @@ impl Iterator for ActionGenerator {}}-#[derive(Debug, serde::Serialize, serde::Deserialize)]-#[serde(tag = "@type", rename_all = "camelCase")]-pub enum ExecutionState {- BeginSearch {- query: String,- },- SearchResult {- query: String,- result: Vec<SimplifiedWebsite>,- },- Speaking {- text: String,- },- Done {- state: EncodedEncryptedState,- },-}-pub struct ActionExecutor {generator: ActionGenerator,searcher: Searcher,diff --git a/core/src/alice/local/mod.rs b/core/src/alice/local/mod.rsnew file mode 100644index 00000000..a7eb8a33--- /dev/null+++ b/core/src/alice/local/mod.rs@@ -0,0 +1,393 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::io::{Read, Write};+use std::{path::Path, sync::Arc};++use aes_gcm::aead::{Aead, OsRng};+use aes_gcm::{AeadCore, Aes256Gcm, Key, KeyInit, Nonce};+use flate2::bufread::GzDecoder;+use flate2::write::GzEncoder;+use flate2::Compression;+use half::bf16;+use tch::Tensor;++use crate::{+ llm_utils::ClonableTensor, summarizer::ExtractiveSummarizer, AcceleratorDevice,+ AcceleratorDtype, AliceAcceleratorConfig,+};++use crate::alice::Result;++use self::generate::{ActionExecutor, ActionGenerator, AliceTokenGenerator, RawActionGenerator};+use self::raw_model::RawModel;++use super::{Error, Searcher};++pub mod generate;+mod raw_model;++const PROMPT_PREFIX: &str = r#"System: Your name is Alice, and you are an AI assistant trained by Stract. Below is a conversation between you and a user. Help the user as best you can.+You can lookup information on the web in the following format:+Alice[thought]: I should look up "<keyword>" on the web.+Search: <query><|endoftext|>+Result: <search result>++This Thought/Search/Result can repeat N times.+When you are ready to answer the user, use the following format:+Alice: <answer><|endoftext|>"#;++pub struct Alice {+ inner: Arc<RawModel>,+ tokenizer: Arc<Tokenizer>,+ summarizer: Arc<ExtractiveSummarizer>,+ end_tokens: Vec<i64>,+ initial_state: ClonableTensor,+ encryption_key: Key<Aes256Gcm>,+}++/// SAFETY:+/// Alice is thread-safe because it never mutates it's internal state.+/// It only ever spawns executors that keeps track of the generation state. These are not threadsafe.+unsafe impl Send for Alice {}+unsafe impl Sync for Alice {}++pub struct AcceleratorConfig {+ pub layer_fraction: f64,+ pub device: tch::Device,+ pub kind: tch::Kind,+}++impl From<AcceleratorDevice> for tch::Device {+ fn from(value: AcceleratorDevice) -> Self {+ match value {+ AcceleratorDevice::Cpu => tch::Device::Cpu,+ AcceleratorDevice::Cuda(d) => tch::Device::Cuda(d),+ AcceleratorDevice::Mps => tch::Device::Mps,+ }+ }+}++impl From<AcceleratorDtype> for tch::Kind {+ fn from(value: AcceleratorDtype) -> Self {+ match value {+ AcceleratorDtype::Float => tch::Kind::Float,+ AcceleratorDtype::Bf16 => tch::Kind::BFloat16,+ }+ }+}++impl From<AliceAcceleratorConfig> for AcceleratorConfig {+ fn from(value: AliceAcceleratorConfig) -> Self {+ Self {+ layer_fraction: value.layer_fraction,+ device: value.device.into(),+ kind: value.dtype.into(),+ }+ }+}++impl Alice {+ pub fn open<P: AsRef<Path>>(+ folder: P,+ summarizer_path: P,+ accelerator: Option<AcceleratorConfig>,+ encryption_key: &[u8],+ ) -> Result<Alice> {+ let encryption_key = *Key::<Aes256Gcm>::from_slice(encryption_key);+ let mut model = RawModel::open(folder.as_ref().join("model.safetensors"))?;++ if let Some(accelerator) = accelerator {+ model.load_to_device(+ accelerator.layer_fraction,+ accelerator.device,+ accelerator.kind,+ );+ }++ let mut summarizer = ExtractiveSummarizer::open(summarizer_path, 1)?;+ summarizer.set_window_size(30);++ let summarizer = Arc::new(summarizer);++ let inner = Arc::new(model);+ let tokenizer = Arc::new(Tokenizer::open(folder.as_ref().join("tokenizer.json"))?);++ let end_tokens = vec![tokenizer.tokenizer.token_to_id("<|endoftext|>").unwrap() as i64];++ let initial_state = Alice::prepare_initial_state(&inner, &tokenizer)?;++ Ok(Self {+ inner,+ tokenizer,+ end_tokens,+ summarizer,+ initial_state,+ encryption_key,+ })+ }++ fn prepare_initial_state(model: &RawModel, tokenizer: &Tokenizer) -> Result<ClonableTensor> {+ let tokens = tokenizer.encode(PROMPT_PREFIX.to_string())?;++ let mut state = model.init_state();++ for token in tokens {+ let (_, new_state) = model.forward(token, Some(&state));+ state = new_state;+ }++ Ok(ClonableTensor(state))+ }++ pub fn new_executor(+ &self,+ user_question: &str,+ last_state: Option<EncryptedState>,+ search_url: String,+ optic_url: Option<String>,+ ) -> Result<ActionExecutor> {+ let mut state = None;++ if let Some(s) = last_state {+ let decrypted_state = s.decrypt(&self.encryption_key)?;+ state = Some(Tensor::from_slice2(&decrypted_state));+ }++ let (tokens, state) = match state {+ Some(state) => {+ let input = format!("User: {user_question}\n\nAlice[thought]:");+ let tokens = self.tokenizer.encode(input).unwrap();++ (tokens, state)+ }+ None => {+ let input = format!("\n\nUser: {user_question}\n\nAlice[thought]:");+ let tokens = self.tokenizer.encode(input).unwrap();++ (tokens, self.initial_state.clone().0)+ }+ };++ let token_generator = AliceTokenGenerator::new(+ self.inner.clone(),+ self.end_tokens.clone(),+ state,+ &tokens,+ None,+ )?;++ let raw_action_gen = RawActionGenerator::new(token_generator, self.tokenizer.clone());+ let action_gen = ActionGenerator::new(raw_action_gen);++ let searcher = Searcher {+ url: search_url,+ optic_url,+ summarizer: Arc::clone(&self.summarizer),+ };++ Ok(ActionExecutor::new(+ action_gen,+ searcher,+ self.encryption_key,+ ))+ }+}++#[derive(Clone)]+pub struct ModelState {+ state: ClonableTensor,+ next_token: i64,+}++pub struct Tokenizer {+ tokenizer: tokenizers::Tokenizer,+}++impl Tokenizer {+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Tokenizer> {+ let tokenizer = tokenizers::Tokenizer::from_file(path)?;++ Ok(Self { tokenizer })+ }++ pub fn encode(&self, input: String) -> Result<Vec<i64>> {+ let encoding = self.tokenizer.encode(input, false)?;++ let ids = encoding+ .get_ids()+ .iter()+ .map(|&id| id as i64)+ .collect::<Vec<_>>();++ Ok(ids)+ }++ pub fn decode(&self, tokens: &[i64]) -> Result<String> {+ let tokens = tokens.iter().map(|&id| id as u32).collect::<Vec<_>>();++ let output = self.tokenizer.decode(tokens, true)?;++ Ok(output)+ }+}++#[derive(Debug, serde::Serialize, serde::Deserialize)]+/// base64 encoded `EncryptedState`+pub struct EncodedEncryptedState(String);++impl EncodedEncryptedState {+ pub fn encode(state: EncryptedState) -> Self {+ let bytes = bincode::serialize(&state).unwrap();+ let encoded = base64::encode(bytes);++ Self(encoded)+ }++ pub fn decode(self) -> Result<EncryptedState> {+ let bytes = base64::decode(self.0)?;+ let state = bincode::deserialize(&bytes)?;++ Ok(state)+ }+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+pub struct EncryptedState {+ nonce: Vec<u8>,+ // ciphertext is bincoded `CompressedState` encrypted with `nonce` and a key+ ciphertext: Vec<u8>,+}++impl EncryptedState {+ fn encrypt(state: Vec<Vec<f32>>, key: &Key<Aes256Gcm>) -> Self {+ let compressed = compress_state(state);++ let cipher = Aes256Gcm::new(key);+ let nonce = Aes256Gcm::generate_nonce(&mut OsRng);++ let bytes = bincode::serialize(&compressed).unwrap();++ let ciphertext = cipher.encrypt(&nonce, bytes.as_slice()).unwrap();++ Self {+ nonce: nonce.to_vec(),+ ciphertext,+ }+ }++ fn decrypt(&self, key: &Key<Aes256Gcm>) -> Result<Vec<Vec<f32>>> {+ let cipher = Aes256Gcm::new(key);+ let nonce = Nonce::from_slice(&self.nonce);++ let decrypted = cipher+ .decrypt(nonce, self.ciphertext.as_slice())+ .map_err(|_| Error::DecryptionFailed)?;++ let compressed: CompressedState = bincode::deserialize(&decrypted)?;++ decompress_state(compressed)+ }+}++/// base64 encoded gzipped state+#[derive(Debug, serde::Serialize, serde::Deserialize)]+pub struct CompressedState(String);++pub fn compress_state(state: Vec<Vec<f32>>) -> CompressedState {+ let state: Vec<Vec<bf16>> = state+ .into_iter()+ .map(|v| v.into_iter().map(bf16::from_f32).collect())+ .collect();++ let mut encoder = GzEncoder::new(Vec::new(), Compression::best());++ encoder+ .write_all(&bincode::serialize(&state).unwrap())+ .unwrap();++ let compressed = encoder.finish().unwrap();++ // base64 encode the compressed state+ CompressedState(base64::encode(compressed))+}++pub fn decompress_state(state: CompressedState) -> Result<Vec<Vec<f32>>> {+ let state = base64::decode(state.0)?;++ let mut decoder = GzDecoder::new(&state[..]);++ let mut state = Vec::new();+ decoder.read_to_end(&mut state)?;++ let state: Vec<Vec<bf16>> = bincode::deserialize(&state)?;++ Ok(state+ .into_iter()+ .map(|v| v.into_iter().map(f32::from).collect())+ .collect())+}++#[derive(Debug)]+struct AnyTransitionValidator {+ validators: Vec<TransitionValidator>,+}++impl AnyTransitionValidator {+ fn new(validators: Vec<TransitionValidator>) -> Self {+ Self { validators }+ }++ fn validate(&mut self, token: i64) -> bool {+ for validator in &mut self.validators {+ if validator.validate(token) {+ return true;+ }+ }++ false+ }+}++#[derive(Debug)]+struct TransitionValidator {+ tokens: Vec<i64>,+ next_match: usize,+}++impl TransitionValidator {+ fn new(tokens: Vec<i64>) -> Self {+ Self {+ tokens,+ next_match: 0,+ }+ }++ fn validate(&mut self, token: i64) -> bool {+ if self.tokens[self.next_match] == token {+ self.next_match += 1;+ if self.next_match == self.tokens.len() {+ self.next_match = 0;+ true+ } else {+ false+ }+ } else {+ self.next_match = 0;+ false+ }+ }+}diff --git a/core/src/alice/raw_model.rs b/core/src/alice/local/raw_model.rssimilarity index 99%rename from core/src/alice/raw_model.rsrename to core/src/alice/local/raw_model.rsindex e86ebf3c..44d96404 100644--- a/core/src/alice/raw_model.rs+++ b/core/src/alice/local/raw_model.rs@@ -15,7 +15,7 @@ use std::{fs::OpenOptions, path::Path};//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use super::{Error, Result};+use crate::alice::{Error, Result};use safetensors::SafeTensors;use tch::{nn::{embedding, layer_norm, Embedding, LayerNorm, LayerNormConfig, Linear, ModuleT, VarStore},diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex 1e05a89c..19dae59b 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -14,47 +14,18 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{- io::{Read, Write},- path::Path,- sync::Arc,-};--use aes_gcm::{- aead::{Aead, OsRng},- AeadCore, Aes256Gcm, Key, KeyInit, Nonce,-};-use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};-use half::bf16;-use tch::Tensor;+use std::sync::Arc;use crate::{api::search::ApiSearchQuery,- llm_utils::ClonableTensor,search_prettifier::DisplayedWebpage,searcher::{SearchResult, WebsitesResult},summarizer::{self, ExtractiveSummarizer},webpage::Url,- AcceleratorDevice, AcceleratorDtype, AliceAcceleratorConfig,-};--use self::{- generate::{ActionExecutor, ActionGenerator, AliceTokenGenerator, RawActionGenerator},- raw_model::RawModel,};-pub mod generate;-mod raw_model;--const PROMPT_PREFIX: &str = r#"System: Your name is Alice, and you are an AI assistant trained by Stract. Below is a conversation between you and a user. Help the user as best you can.-You can lookup information on the web in the following format:-Alice[thought]: I should look up "<keyword>" on the web.-Search: <query><|endoftext|>-Result: <search result>--This Thought/Search/Result can repeat N times.-When you are ready to answer the user, use the following format:-Alice: <answer><|endoftext|>"#;+pub mod local;+pub mod openai;type Result<T> = std::result::Result<T, Error>;@@ -95,201 +66,21 @@ pub enum Error {#[error("Cluster")]Cluster(#[from] crate::distributed::cluster::Error),-}--pub struct Alice {- inner: Arc<RawModel>,- tokenizer: Arc<Tokenizer>,- summarizer: Arc<ExtractiveSummarizer>,- end_tokens: Vec<i64>,- initial_state: ClonableTensor,- encryption_key: Key<Aes256Gcm>,-}--/// SAFETY:-/// Alice is thread-safe because it never mutates it's internal state.-/// It only ever spawns executors that keeps track of the generation state. These are not threadsafe.-unsafe impl Send for Alice {}-unsafe impl Sync for Alice {}--pub struct AcceleratorConfig {- pub layer_fraction: f64,- pub device: tch::Device,- pub kind: tch::Kind,-}--impl From<AcceleratorDevice> for tch::Device {- fn from(value: AcceleratorDevice) -> Self {- match value {- AcceleratorDevice::Cpu => tch::Device::Cpu,- AcceleratorDevice::Cuda(d) => tch::Device::Cuda(d),- AcceleratorDevice::Mps => tch::Device::Mps,- }- }-}--impl From<AcceleratorDtype> for tch::Kind {- fn from(value: AcceleratorDtype) -> Self {- match value {- AcceleratorDtype::Float => tch::Kind::Float,- AcceleratorDtype::Bf16 => tch::Kind::BFloat16,- }- }-}--impl From<AliceAcceleratorConfig> for AcceleratorConfig {- fn from(value: AliceAcceleratorConfig) -> Self {- Self {- layer_fraction: value.layer_fraction,- device: value.device.into(),- kind: value.dtype.into(),- }- }-}--impl Alice {- pub fn open<P: AsRef<Path>>(- folder: P,- summarizer_path: P,- accelerator: Option<AcceleratorConfig>,- encryption_key: &[u8],- ) -> Result<Alice> {- let encryption_key = *Key::<Aes256Gcm>::from_slice(encryption_key);- let mut model = RawModel::open(folder.as_ref().join("model.safetensors"))?;-- if let Some(accelerator) = accelerator {- model.load_to_device(- accelerator.layer_fraction,- accelerator.device,- accelerator.kind,- );- }-- let mut summarizer = ExtractiveSummarizer::open(summarizer_path, 1)?;- summarizer.set_window_size(30);-- let summarizer = Arc::new(summarizer);-- let inner = Arc::new(model);- let tokenizer = Arc::new(Tokenizer::open(folder.as_ref().join("tokenizer.json"))?);-- let end_tokens = vec![tokenizer.tokenizer.token_to_id("<|endoftext|>").unwrap() as i64];-- let initial_state = Alice::prepare_initial_state(&inner, &tokenizer)?;- Ok(Self {- inner,- tokenizer,- end_tokens,- summarizer,- initial_state,- encryption_key,- })- }-- fn prepare_initial_state(model: &RawModel, tokenizer: &Tokenizer) -> Result<ClonableTensor> {- let tokens = tokenizer.encode(PROMPT_PREFIX.to_string())?;-- let mut state = model.init_state();-- for token in tokens {- let (_, new_state) = model.forward(token, Some(&state));- state = new_state;- }-- Ok(ClonableTensor(state))- }-- pub fn new_executor(- &self,- user_question: &str,- last_state: Option<EncryptedState>,- search_url: String,- optic_url: Option<String>,- ) -> Result<ActionExecutor> {- let mut state = None;-- if let Some(s) = last_state {- let decrypted_state = s.decrypt(&self.encryption_key)?;- state = Some(Tensor::from_slice2(&decrypted_state));- }-- let (tokens, state) = match state {- Some(state) => {- let input = format!("User: {user_question}\n\nAlice[thought]:");- let tokens = self.tokenizer.encode(input).unwrap();-- (tokens, state)- }- None => {- let input = format!("\n\nUser: {user_question}\n\nAlice[thought]:");- let tokens = self.tokenizer.encode(input).unwrap();-- (tokens, self.initial_state.clone().0)- }- };-- let token_generator = AliceTokenGenerator::new(- self.inner.clone(),- self.end_tokens.clone(),- state,- &tokens,- None,- )?;-- let raw_action_gen = RawActionGenerator::new(token_generator, self.tokenizer.clone());- let action_gen = ActionGenerator::new(raw_action_gen);-- let searcher = Searcher {- url: search_url,- optic_url,- summarizer: Arc::clone(&self.summarizer),- };+ #[error("Unexpected completion")]+ UnexpectedCompletion,- Ok(ActionExecutor::new(- action_gen,- searcher,- self.encryption_key,- ))- }-}--#[derive(Clone)]-pub struct ModelState {- state: ClonableTensor,- next_token: i64,-}--pub struct Tokenizer {- tokenizer: tokenizers::Tokenizer,-}--impl Tokenizer {- pub fn open<P: AsRef<Path>>(path: P) -> Result<Tokenizer> {- let tokenizer = tokenizers::Tokenizer::from_file(path)?;-- Ok(Self { tokenizer })- }+ #[error("Event source cannot clone")]+ EventSourceCannotClone(#[from] reqwest_eventsource::CannotCloneRequestError),- pub fn encode(&self, input: String) -> Result<Vec<i64>> {- let encoding = self.tokenizer.encode(input, false)?;+ #[error("Event source error")]+ EventSource(#[from] reqwest_eventsource::Error),- let ids = encoding- .get_ids()- .iter()- .map(|&id| id as i64)- .collect::<Vec<_>>();+ #[error("Serde JSON error")]+ SerdeJson(#[from] serde_json::Error),- Ok(ids)- }-- pub fn decode(&self, tokens: &[i64]) -> Result<String> {- let tokens = tokens.iter().map(|&id| id as u32).collect::<Vec<_>>();-- let output = self.tokenizer.decode(tokens, true)?;-- Ok(output)- }+ #[error("Last message should be from user")]+ LastMessageNotUser,}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]@@ -314,6 +105,24 @@ impl SimplifiedWebsite {}}+#[derive(Debug, serde::Serialize, serde::Deserialize)]+#[serde(tag = "@type", rename_all = "camelCase")]+pub enum ExecutionState {+ BeginSearch {+ query: String,+ },+ SearchResult {+ query: String,+ result: Vec<SimplifiedWebsite>,+ },+ Speaking {+ text: String,+ },+ Done {+ state: local::EncodedEncryptedState,+ },+}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]struct ModelWebsite {pub title: String,@@ -378,151 +187,56 @@ impl Searcher {Ok(websites)}-}--#[derive(Debug, serde::Serialize, serde::Deserialize)]-/// base64 encoded `EncryptedState`-pub struct EncodedEncryptedState(String);-impl EncodedEncryptedState {- pub fn encode(state: EncryptedState) -> Self {- let bytes = bincode::serialize(&state).unwrap();- let encoded = base64::encode(bytes);-- Self(encoded)- }-- pub fn decode(self) -> Result<EncryptedState> {- let bytes = base64::decode(self.0)?;- let state = bincode::deserialize(&bytes)?;-- Ok(state)- }-}+ async fn raw_search_async(&self, query: &str) -> Result<WebsitesResult> {+ let mut optic = None;-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]-pub struct EncryptedState {- nonce: Vec<u8>,- // ciphertext is bincoded `CompressedState` encrypted with `nonce` and a key- ciphertext: Vec<u8>,-}--impl EncryptedState {- fn encrypt(state: Vec<Vec<f32>>, key: &Key<Aes256Gcm>) -> Self {- let compressed = compress_state(state);-- let cipher = Aes256Gcm::new(key);- let nonce = Aes256Gcm::generate_nonce(&mut OsRng);-- let bytes = bincode::serialize(&compressed).unwrap();-- let ciphertext = cipher.encrypt(&nonce, bytes.as_slice()).unwrap();-- Self {- nonce: nonce.to_vec(),- ciphertext,+ if let Some(url) = &self.optic_url {+ if let Ok(r) = reqwest::get(url).await {+ if let Ok(text) = r.text().await {+ optic = Some(text);+ }+ }}- }-- fn decrypt(&self, key: &Key<Aes256Gcm>) -> Result<Vec<Vec<f32>>> {- let cipher = Aes256Gcm::new(key);- let nonce = Nonce::from_slice(&self.nonce);- let decrypted = cipher- .decrypt(nonce, self.ciphertext.as_slice())- .map_err(|_| Error::DecryptionFailed)?;+ let client = reqwest::Client::new();+ let query = ApiSearchQuery {+ query: query.trim().to_string(),+ num_results: Some(3),+ optic,+ page: None,+ selected_region: None,+ site_rankings: None,+ return_ranking_signals: None,+ flatten_response: Some(false),+ };+ tracing::debug!("searching at {:?}: {:#?}", self.url, query);- let compressed: CompressedState = bincode::deserialize(&decrypted)?;+ let res: SearchResult = client+ .post(&self.url)+ .json(&query)+ .send()+ .await?+ .json()+ .await?;- decompress_state(compressed)+ match res {+ SearchResult::Websites(res) => Ok(res),+ SearchResult::Bang(_) => Err(Error::UnexpectedSearchResult),+ }}-}--/// base64 encoded gzipped state-#[derive(Debug, serde::Serialize, serde::Deserialize)]-pub struct CompressedState(String);--pub fn compress_state(state: Vec<Vec<f32>>) -> CompressedState {- let state: Vec<Vec<bf16>> = state- .into_iter()- .map(|v| v.into_iter().map(bf16::from_f32).collect())- .collect();- let mut encoder = GzEncoder::new(Vec::new(), Compression::best());-- encoder- .write_all(&bincode::serialize(&state).unwrap())- .unwrap();-- let compressed = encoder.finish().unwrap();-- // base64 encode the compressed state- CompressedState(base64::encode(compressed))-}--pub fn decompress_state(state: CompressedState) -> Result<Vec<Vec<f32>>> {- let state = base64::decode(state.0)?;-- let mut decoder = GzDecoder::new(&state[..]);-- let mut state = Vec::new();- decoder.read_to_end(&mut state)?;-- let state: Vec<Vec<bf16>> = bincode::deserialize(&state)?;-- Ok(state- .into_iter()- .map(|v| v.into_iter().map(f32::from).collect())- .collect())-}--#[derive(Debug)]-struct AnyTransitionValidator {- validators: Vec<TransitionValidator>,-}+ async fn search_async(&self, query: &str) -> Result<Vec<SimplifiedWebsite>> {+ let res = self.raw_search_async(query).await?;-impl AnyTransitionValidator {- fn new(validators: Vec<TransitionValidator>) -> Self {- Self { validators }- }+ let mut websites = Vec::new();- fn validate(&mut self, token: i64) -> bool {- for validator in &mut self.validators {- if validator.validate(token) {- return true;- }+ for website in res.webpages {+ websites.push(SimplifiedWebsite::new(website, query, &self.summarizer));}- false- }-}--#[derive(Debug)]-struct TransitionValidator {- tokens: Vec<i64>,- next_match: usize,-}--impl TransitionValidator {- fn new(tokens: Vec<i64>) -> Self {- Self {- tokens,- next_match: 0,- }- }+ tracing::debug!("search result: {:#?}", websites);- fn validate(&mut self, token: i64) -> bool {- if self.tokens[self.next_match] == token {- self.next_match += 1;- if self.next_match == self.tokens.len() {- self.next_match = 0;- true- } else {- false- }- } else {- self.next_match = 0;- false- }+ Ok(websites)}}diff --git a/core/src/alice/openai.rs b/core/src/alice/openai.rsnew file mode 100644index 00000000..ccec6388--- /dev/null+++ b/core/src/alice/openai.rs@@ -0,0 +1,385 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::path::Path;+use std::sync::Arc;++use reqwest_eventsource::EventSource;+use tokio_stream::StreamExt;++use crate::summarizer::ExtractiveSummarizer;++use super::Error;+use super::ExecutionState;+use super::ModelWebsite;+use super::Result;+use super::Searcher;+use super::SimplifiedWebsite;++const MAX_NUM_SEARCHES: usize = 3;++#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+#[serde(tag = "@type", rename_all = "camelCase")]+pub enum Message {+ User {+ message: String,+ },+ Alice {+ message: String,+ queries: Vec<String>,+ },+}++fn prompt(user_question: &str, conv: &[Message]) -> String {+ let mut prompt = r#"System: Your name is Alice, and you are an AI assistant developed by Stract. Below is a conversation between you and a user. Help the user as best you can.+You can lookup information on the web in the following format:+Alice[thought]: I should look up "<keywords>" on the web.+Search i: <query>+Result: <search result>++This Thought/Search/Result can repeat N times.+You should always cite the source of your statements using the format [search i source j] at the end of the statement.++The search results from your previous messages have been truncated.++When you are ready to answer the user, use the following format:+Alice[thought]: I now know enough to answer the user.+Alice: <answer>++You should only use the information you find on the web. If you couldn't find a good answer to the user, you should tell them and ask for clarification.++Begin!"#.to_string();++ // truncate conv+ let conv = if conv.len() > 4 {+ &conv[conv.len() - 4..]+ } else {+ conv+ };++ for msg in conv {+ match msg {+ Message::User { message } => {+ prompt.push_str("\n\n");+ prompt.push_str(format!("User: {}", message).as_str());+ }+ Message::Alice { message, queries } => {+ prompt.push('\n');+ for query in queries {+ prompt.push('\n');+ prompt.push_str(&format!(+ "Alice[thought]: I should look up \"{query}\" on the web."+ ));+ prompt.push('\n');+ prompt.push_str(&format!("Search i: {query}\n"));+ prompt.push_str("Result: <truncated>");+ }+ prompt.push('\n');+ prompt.push_str("Alice[thought]: I now know enough to answer the user.");+ prompt.push('\n');+ prompt.push_str(&format!("Alice: {}", message));+ }+ }+ }++ prompt.push_str("\n\n");+ prompt.push_str(format!("User: {}", user_question).as_str());++ prompt+}++#[derive(serde::Serialize, serde::Deserialize)]+struct OpenaiParams {+ model: String,+ prompt: String,+ max_tokens: usize,+ temperature: f32,+ top_p: f32,+ stream: bool,+ stop: Vec<String>,+}++async fn thoughts(prompt: &str, api_key: &str) -> Result<String> {+ let client = reqwest::Client::new();+ let res: serde_json::Value = client+ .post("https://api.openai.com/v1/completions")+ .bearer_auth(api_key)+ .json(&OpenaiParams {+ model: "text-davinci-003".to_string(),+ prompt: prompt.to_string(),+ max_tokens: 100,+ temperature: 0.7,+ top_p: 0.7,+ stream: false,+ stop: vec!["Result:".to_string(), "Alice:".to_string()],+ })+ .send()+ .await?+ .json()+ .await?;++ let text = res["choices"][0]["text"].as_str().unwrap().to_string();++ Ok(dbg!(text))+}++async fn speak(prompt: &str, api_key: &str) -> Result<EventSource> {+ let client = reqwest::Client::new();+ let req = client+ .post("https://api.openai.com/v1/completions")+ .bearer_auth(api_key)+ .json(&OpenaiParams {+ model: "text-davinci-003".to_string(),+ prompt: prompt.to_string(),+ max_tokens: 512,+ temperature: 0.7,+ top_p: 0.7,+ stream: true,+ stop: vec!["<|endoftext|>".to_string()],+ });++ let res = EventSource::new(req)?;++ Ok(res)+}++#[derive(Debug)]+enum State {+ BeginSearch { query: String },+ PerformSearch { query: String },++ Thinking,+ Speaking,+}++fn next_state(completion: &str) -> Result<State> {+ let completion = completion.trim();++ if completion.ends_with("Alice:")+ || completion.contains("I now know enough to answer the user.")+ {+ Ok(State::Speaking)+ } else if completion.contains("Search") {+ // find last "Search \d: <query>" and extract query+ let mut query = None;+ for line in completion.lines().rev() {+ if line.starts_with("Search") {+ query = Some(line.split(':').nth(1).unwrap().trim());+ break;+ }+ }++ if let Some(query) = query {+ Ok(State::BeginSearch {+ query: query.to_string(),+ })+ } else {+ Err(Error::UnexpectedCompletion)+ }+ } else {+ Err(Error::UnexpectedCompletion)+ }+}++pub struct Alice {+ api_key: String,+ summarizer: Arc<ExtractiveSummarizer>,+}++impl Alice {+ pub fn open<P: AsRef<Path>>(summarizer_path: P, api_key: &str) -> Result<Self> {+ let mut summarizer = ExtractiveSummarizer::open(summarizer_path, 1)?;+ summarizer.set_window_size(30);++ let summarizer = Arc::new(summarizer);++ Ok(Self {+ api_key: api_key.to_string(),+ summarizer,+ })+ }++ pub fn new_executor(+ &self,+ conversation: Vec<Message>,+ optic_url: Option<String>,+ search_url: String,+ ) -> Result<Executor> {+ let user_question = match conversation.last() {+ Some(Message::User { message }) => message.clone(),+ _ => return Err(Error::LastMessageNotUser),+ };++ let prev_conv = conversation[..conversation.len() - 1].to_vec();++ let searcher = Searcher {+ url: search_url,+ optic_url,+ summarizer: self.summarizer.clone(),+ };++ Ok(Executor::new(+ prev_conv,+ self.api_key.clone(),+ user_question,+ searcher,+ ))+ }+}++#[derive(Clone)]+struct Search {+ query: String,+ results: Vec<SimplifiedWebsite>,+}++pub struct Executor {+ api_key: String,+ prev_conv: Vec<Message>,+ current_searches: Vec<Search>,+ user_question: String,+ state: State,+ searcher: Searcher,+ speaking_stream: Option<EventSource>,+ performed_searches: usize,+}++impl Executor {+ fn new(+ prev_conv: Vec<Message>,+ api_key: String,+ user_question: String,+ searcher: Searcher,+ ) -> Self {+ Self {+ api_key,+ state: State::Thinking,+ prev_conv,+ user_question,+ current_searches: Vec::new(),+ speaking_stream: None,+ searcher,+ performed_searches: 0,+ }+ }++ fn build_prompt(&self, speaking: bool) -> String {+ let mut prompt = prompt(&self.user_question, &self.prev_conv);++ for (i, search) in self.current_searches.clone().into_iter().enumerate() {+ prompt.push_str("\n\n");+ prompt.push_str(+ format!(+ "Alice[thought]: I should look up \"{}\" on the web.",+ search.query+ )+ .as_str(),+ );+ prompt.push_str(format!("\nSearch {}: {}", i + 1, search.query).as_str());+ let res = search+ .results+ .into_iter()+ .take(3)+ .map(ModelWebsite::from)+ .collect::<Vec<_>>();++ prompt+ .push_str(format!("\nResponse: {}", serde_json::to_string(&res).unwrap()).as_str());+ }++ prompt.push_str("\n\n");++ if speaking {+ prompt.push_str("Alice[thought]: I now know enough to answer the user.");+ prompt.push_str("\n\n");+ prompt.push_str("Alice:");+ } else {+ prompt.push_str("Alice[thought]:");+ }++ dbg!(prompt)+ }++ pub async fn next(&mut self) -> Result<Option<ExecutionState>> {+ loop {+ match &mut self.state {+ State::BeginSearch { query } => {+ let query = query.clone();++ if self.current_searches.iter().any(|s| s.query == query) {+ self.state = State::Speaking;+ } else {+ self.state = State::PerformSearch {+ query: query.clone(),+ };++ return Ok(Some(ExecutionState::BeginSearch { query }));+ }+ }+ State::PerformSearch { query } => {+ let query = query.to_string();+ let res = self.searcher.search_async(&query).await?;++ self.current_searches.push(Search {+ query: query.clone(),+ results: res.clone(),+ });++ self.performed_searches += 1;++ if self.performed_searches >= MAX_NUM_SEARCHES {+ self.state = State::Speaking;+ } else {+ self.state = State::Thinking;+ }++ return Ok(Some(ExecutionState::SearchResult { query, result: res }));+ }+ State::Thinking => {+ let prompt = self.build_prompt(false);+ let cur_thoughts = thoughts(&prompt, &self.api_key).await?;+ let next_state = dbg!(next_state(&cur_thoughts))?;+ self.state = next_state;+ }+ State::Speaking => {+ if self.speaking_stream.is_none() {+ let prompt = self.build_prompt(true);+ self.speaking_stream = Some(speak(&prompt, &self.api_key).await?);+ }++ let stream = self.speaking_stream.as_mut().unwrap();+ match stream.next().await {+ Some(event) => {+ let event = event?;++ match event {+ reqwest_eventsource::Event::Open => {}+ reqwest_eventsource::Event::Message(msg) => {+ let res: serde_json::Value = serde_json::from_str(&msg.data)?;+ let text =+ res["choices"][0]["text"].as_str().unwrap().to_string();++ return Ok(Some(ExecutionState::Speaking { text }));+ }+ }+ }+ None => return Ok(None),+ }+ }+ }+ }+ }+}diff --git a/core/src/api/alice.rs b/core/src/api/alice.rsindex eaead227..efc5308b 100644--- a/core/src/api/alice.rs+++ b/core/src/api/alice.rs@@ -27,7 +27,7 @@ use tokio_stream::{Stream, StreamExt};use crate::{distributed::member::Service,- entrypoint::{self, alice::SaveStateParams},+ entrypoint::{self, alice::openai::SaveStateParams},};#[derive(Debug, serde::Serialize, serde::Deserialize)]@@ -164,7 +164,7 @@ pub async fn route(.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?};- let params = entrypoint::alice::Params {+ let params = entrypoint::alice::local::Params {message: params.message,prev_state: saved_state.map(|s| s.uuid),optic: params.optic,@@ -187,3 +187,70 @@ pub async fn route(Ok(Sse::new(stream))}++// for openai alice+// #[derive(serde::Serialize, serde::Deserialize, Debug)]+// #[serde(rename_all = "camelCase")]+// pub struct Params {+// optic: Option<String>,+// conv: EncodedSavedState,+// }+//+// pub async fn route(+// extract::State(state): extract::State<Arc<super::State>>,+// extract::Query(params): extract::Query<Params>,+// ) -> std::result::Result<+// Sse<impl Stream<Item = std::result::Result<axum::response::sse::Event, Infallible>>>,+// StatusCode,+// > {+// let saved_state: SavedState = params.conv.decode().map_err(|e| {+// tracing::error!("Error decoding saved state: {}", e);+// StatusCode::BAD_REQUEST+// })?;+//+// let client = reqwest::ClientBuilder::default()+// .build()+// .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;+//+// let mut members = state.cluster.members().await;+//+// members.shuffle(&mut rand::thread_rng());+//+// let id = saved_state.id.clone();+// let alice_addr = members+// .into_iter()+// .find_map(|m| {+// if m.id == id {+// if let Service::Alice { host } = m.service {+// Some(host)+// } else {+// None+// }+// } else {+// None+// }+// })+// .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;+//+// let params = entrypoint::alice::openai::Params {+// conversation: saved_state.uuid,+// optic: params.optic,+// };+//+// let mut events = client+// .get(format!("http://{}", alice_addr))+// .query(¶ms)+// .send()+// .await+// .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?+// .bytes_stream()+// .eventsource();+//+// let stream = async_stream::stream! {+// while let Some(Ok(item)) = events.next().await {+// yield Ok(Event::default().data(item.data));+// }+// };+//+// Ok(Sse::new(stream))+// }diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice/local.rssimilarity index 94%rename from core/src/entrypoint/alice.rsrename to core/src/entrypoint/alice/local.rsindex c6744e0b..a7032932 100644--- a/core/src/entrypoint/alice.rs+++ b/core/src/entrypoint/alice/local.rs@@ -16,7 +16,6 @@use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};-use aes_gcm::{aead::OsRng, Aes256Gcm, KeyInit};use axum::{extract,response::{@@ -32,13 +31,13 @@ use tokio_stream::StreamExt as _;use tracing::info;use crate::{- alice::{Alice, EncodedEncryptedState, EncryptedState},+ alice::local::{Alice, EncodedEncryptedState, EncryptedState},distributed::{cluster::Cluster,member::{Member, Service},},ttl_cache::TTLCache,- AliceConfig,+ AliceLocalConfig,};pub struct State {@@ -167,7 +166,7 @@ pub async fn route()}-pub async fn run(config: AliceConfig) -> Result<(), crate::alice::Error> {+pub async fn run(config: AliceLocalConfig) -> Result<(), crate::alice::Error> {let addr: SocketAddr = config.host;let key = base64::decode(config.encryption_key)?;@@ -198,8 +197,3 @@ pub async fn run(config: AliceConfig) -> Result<(), crate::alice::Error> {Ok(())}--pub fn generate_key() {- let key = Aes256Gcm::generate_key(OsRng);- println!("{}", base64::encode(key.as_slice()));-}diff --git a/core/src/entrypoint/alice/mod.rs b/core/src/entrypoint/alice/mod.rsnew file mode 100644index 00000000..bad2db74--- /dev/null+++ b/core/src/entrypoint/alice/mod.rs@@ -0,0 +1,28 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use aes_gcm::{aead::OsRng, Aes256Gcm, KeyInit};++pub mod local;++/// This is currently not used, but let's keep it around for now+/// in case we want to use it later.+pub mod openai;++pub fn generate_key() {+ let key = Aes256Gcm::generate_key(OsRng);+ println!("{}", base64::encode(key.as_slice()));+}diff --git a/core/src/entrypoint/alice/openai.rs b/core/src/entrypoint/alice/openai.rsnew file mode 100644index 00000000..d70bef54--- /dev/null+++ b/core/src/entrypoint/alice/openai.rs@@ -0,0 +1,186 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};++use axum::{+ extract,+ response::{+ sse::{Event, KeepAlive},+ IntoResponse, Sse,+ },+ routing::{get, post},+ Router,+};+use tokio::sync::Mutex;+use tokio_stream::{Stream, StreamExt};+use tracing::info;++use crate::{+ alice::openai::{Alice, Message},+ distributed::{+ cluster::Cluster,+ member::{Member, Service},+ },+ ttl_cache::TTLCache,+ AliceOpenaiConfig,+};++pub struct State {+ pub alice: Alice,+ pub cluster: Cluster,+ pub conv_states: Arc<Mutex<TTLCache<uuid::Uuid, Vec<Message>>>>,+}++fn router(alice: Alice, cluster: Cluster) -> Router {+ let state = Arc::new(State {+ alice,+ cluster,+ conv_states: Arc::new(Mutex::new(TTLCache::with_ttl(Duration::from_secs(10)))),+ });++ Router::new()+ .route("/", get(route))+ .route("/save_state", post(save_state))+ .with_state(state)+}++#[derive(serde::Serialize, serde::Deserialize)]+pub struct SaveStateParams {+ pub conversation: Vec<Message>,+}++pub async fn save_state(+ extract::State(state): extract::State<Arc<State>>,+ extract::Json(params): extract::Json<SaveStateParams>,+) -> impl IntoResponse {+ let uuid = uuid::Uuid::new_v4();++ state+ .conv_states+ .lock()+ .await+ .insert(uuid, params.conversation);++ uuid.to_string()+}++#[derive(serde::Serialize, serde::Deserialize, Debug)]+#[serde(rename_all = "camelCase")]+pub struct Params {+ pub optic: Option<String>,+ pub conversation: uuid::Uuid,+}++pub async fn route(+ extract::State(state): extract::State<Arc<State>>,+ extract::Query(params): extract::Query<Params>,+) -> std::result::Result<+ Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>,+ http::StatusCode,+> {+ let conversation = state+ .conv_states+ .lock()+ .await+ .get(¶ms.conversation)+ .cloned()+ .ok_or(http::StatusCode::BAD_REQUEST)?;++ let search_addr = state+ .cluster+ .members()+ .await+ .into_iter()+ .find_map(|m| {+ if let Service::Frontend { host } = m.service {+ Some(host)+ } else {+ None+ }+ })+ .ok_or_else(|| {+ info!("no frontend found");++ http::StatusCode::INTERNAL_SERVER_ERROR+ })?;++ let mut executor = state+ .alice+ .new_executor(+ conversation,+ params.optic,+ format!("http://{}/beta/api/search", search_addr),+ )+ .map_err(|e| {+ info!("error creating executor: {}", e);+ http::StatusCode::BAD_REQUEST+ })?;++ let (tx, mut rx) = tokio::sync::mpsc::channel(1);++ tokio::task::spawn(async move {+ while let Ok(Some(msg)) = executor.next().await {+ let msg = serde_json::to_string(&msg)+ .map_err(|e| {+ info!("error serializing message: {}", e);+ e+ })+ .unwrap();++ tx.send(msg).await.ok();+ }+ });++ let stream = async_stream::stream! {+ while let Some(item) = rx.recv().await {+ yield item;+ }+ };++ Ok(+ Sse::new(stream.map(|term| Event::default().data(term)).map(Ok))+ .keep_alive(KeepAlive::default()),+ )+}++/// This is currently not used, but let's keep it around for now+/// in case we want to use it later.+pub async fn run(config: AliceOpenaiConfig) -> Result<(), crate::alice::Error> {+ let addr: SocketAddr = config.host;++ let alice = Alice::open(&config.summarizer_path, &config.api_key)?;++ let cluster = Cluster::join(+ Member {+ id: config.cluster_id,+ service: Service::Alice { host: config.host },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.unwrap_or_default(),+ )+ .await?;++ let app = router(alice, cluster);++ info!("alice is ready to accept requests on {}", addr);+ axum::Server::bind(&addr)+ .serve(app.into_make_service())+ .await+ .unwrap();++ Ok(())+}diff --git a/core/src/lib.rs b/core/src/lib.rsindex 2d3f665a..cbd6b691 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -215,19 +215,30 @@ pub struct AliceAcceleratorConfig {}#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct AliceConfig {+pub struct AliceLocalConfig {pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,pub host: SocketAddr,+ pub summarizer_path: String,pub alice_path: String,- pub summarizer_path: String,pub accelerator: Option<AliceAcceleratorConfig>,/// base64 encodedpub encryption_key: String,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct AliceOpenaiConfig {+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,+ pub host: SocketAddr,+ pub summarizer_path: String,++ pub api_key: String,+}+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct WebgraphServerConfig {pub host: SocketAddr,diff --git a/core/src/main.rs b/core/src/main.rsindex 975a737c..43f92acf 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -24,7 +24,7 @@ use stract::entrypoint::configure;use stract::entrypoint::indexer::IndexPointer;use stract::entrypoint::{self, frontend, search_server, webgraph_server};use stract::webgraph::WebgraphBuilder;-use stract::{AliceConfig, FrontendConfig, SearchServerConfig, WebgraphServerConfig};+use stract::{AliceLocalConfig, FrontendConfig, SearchServerConfig, WebgraphServerConfig};use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -317,12 +317,12 @@ fn main() -> Result<()> {),Commands::Alice { options } => match options {AliceOptions::Serve { config_path } => {- let config: AliceConfig = load_toml_config(config_path);+ let config: AliceLocalConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(entrypoint::alice::run(config))?+ .block_on(entrypoint::alice::local::run(config))?}AliceOptions::GenerateKey => entrypoint::alice::generate_key(),},diff --git a/frontend/src/pages/chat.astro b/frontend/src/pages/chat.astroindex 25b5d186..751b66f2 100644--- a/frontend/src/pages/chat.astro+++ b/frontend/src/pages/chat.astro@@ -18,7 +18,7 @@ import OpticSelector from "../components/OpticSelector.astro";<div id="message-container" class="flex px-5 flex-col gap-y-3 h-full"><div class="flex flex-col h-full items-center gap-y-3"><h1 class="text-2xl font-bold">Chat with Alice</h1>- <p class="text-sm">Alice is an AI that tries to answer your questions by searhing for information on the internet. Alice is <b class="font-bold">highly experimental</b> and might produce wrong answers. As always, you should verify the information from multiple sources and make sure the sources supports the claims.</p>+ <p class="text-sm">Alice is an AI that tries to answer your questions by searhing for information on the internet. Alice is <b class="font-bold">highly experimental</b> and might produce bad or downright wrong answers. As always, you should verify the information from multiple sources and make sure the sources supports the claims.</p></div></div><div class="w-full gap-y-2">