Skip to content

web search done right

openai alice

Mikkel Denker8f3853d2d12b
diff --git a/Cargo.lock b/Cargo.lock
index 92248fe0..b29cfedc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1525,6 +1525,12 @@ version = "0.3.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
+[[package]]
+name = "futures-timer"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
+
[[package]]
name = "futures-util"
version = "0.3.28"
@@ -3346,6 +3352,22 @@ dependencies = [
"winreg",
]
+[[package]]
+name = "reqwest-eventsource"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f03f570355882dd8d15acc3a313841e6e90eddbc76a93c748fd82cc13ba9f51"
+dependencies = [
+ "eventsource-stream",
+ "futures-core",
+ "futures-timer",
+ "mime",
+ "nom",
+ "pin-project-lite",
+ "reqwest",
+ "thiserror",
+]
+
[[package]]
name = "rkyv"
version = "0.7.41"
@@ -3851,6 +3873,7 @@ dependencies = [
"rayon",
"regex",
"reqwest",
+ "reqwest-eventsource",
"rkyv",
"rocksdb",
"rust-stemmers",
diff --git a/Cargo.toml b/Cargo.toml
index c61062a9..b9ebf59c 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -22,6 +22,7 @@ itertools = "0.10.3"
rayon = "1.5.3"
half = { version = "2.2.1", features = ["serde"] }
eventsource-stream = "0.2.3"
+reqwest-eventsource = "0.4.0"
reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }
serde = { version = "1.0.137", features = ["rc", "derive"] }
serde_json = "1.0.81"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 85232e4c..93b29dd0 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -86,6 +86,7 @@ eventsource-stream = { workspace = true }
rand = { workspace = true }
half = { workspace = true }
aes-gcm = { workspace = true }
+reqwest-eventsource = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/examples/alice.rs b/core/examples/alice.rs
index cac805d0..5e73bb48 100644
--- a/core/examples/alice.rs
+++ b/core/examples/alice.rs
@@ -1,6 +1,9 @@
use std::io::Write;
-use stract::alice::{generate::ExecutionState, AcceleratorConfig, Alice};
+use stract::alice::{
+ local::{AcceleratorConfig, Alice},
+ ExecutionState,
+};
#[tokio::main]
async fn main() {
diff --git a/core/src/alice/generate.rs b/core/src/alice/local/generate.rs
similarity index 97%
rename from core/src/alice/generate.rs
rename to core/src/alice/local/generate.rs
index 7663ec48..8fa675bf 100644
--- a/core/src/alice/generate.rs
+++ b/core/src/alice/local/generate.rs
@@ -20,14 +20,16 @@ use aes_gcm::{Aes256Gcm, Key};
use tch::{IndexOp, Kind, Tensor};
use crate::{
+ alice::ExecutionState,
leaky_queue::LeakyQueue,
llm_utils::{self, ClonableTensor},
};
use super::{
raw_model::RawModel, AnyTransitionValidator, EncodedEncryptedState, EncryptedState, Error,
- ModelState, ModelWebsite, Result, Searcher, SimplifiedWebsite, Tokenizer, TransitionValidator,
+ ModelState, Result, Searcher, Tokenizer, TransitionValidator,
};
+use crate::alice::{ModelWebsite, SimplifiedWebsite};
// const TAU: f64 = 0.3;
const TEMP: f64 = 0.4;
@@ -586,24 +588,6 @@ impl Iterator for ActionGenerator {
}
}
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
-#[serde(tag = "@type", rename_all = "camelCase")]
-pub enum ExecutionState {
- BeginSearch {
- query: String,
- },
- SearchResult {
- query: String,
- result: Vec<SimplifiedWebsite>,
- },
- Speaking {
- text: String,
- },
- Done {
- state: EncodedEncryptedState,
- },
-}
-
pub struct ActionExecutor {
generator: ActionGenerator,
searcher: Searcher,
diff --git a/core/src/alice/local/mod.rs b/core/src/alice/local/mod.rs
new file mode 100644
index 00000000..a7eb8a33
--- /dev/null
+++ b/core/src/alice/local/mod.rs
@@ -0,0 +1,393 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::io::{Read, Write};
+use std::{path::Path, sync::Arc};
+
+use aes_gcm::aead::{Aead, OsRng};
+use aes_gcm::{AeadCore, Aes256Gcm, Key, KeyInit, Nonce};
+use flate2::bufread::GzDecoder;
+use flate2::write::GzEncoder;
+use flate2::Compression;
+use half::bf16;
+use tch::Tensor;
+
+use crate::{
+ llm_utils::ClonableTensor, summarizer::ExtractiveSummarizer, AcceleratorDevice,
+ AcceleratorDtype, AliceAcceleratorConfig,
+};
+
+use crate::alice::Result;
+
+use self::generate::{ActionExecutor, ActionGenerator, AliceTokenGenerator, RawActionGenerator};
+use self::raw_model::RawModel;
+
+use super::{Error, Searcher};
+
+pub mod generate;
+mod raw_model;
+
+const PROMPT_PREFIX: &str = r#"System: Your name is Alice, and you are an AI assistant trained by Stract. Below is a conversation between you and a user. Help the user as best you can.
+You can lookup information on the web in the following format:
+Alice[thought]: I should look up "<keyword>" on the web.
+Search: <query><|endoftext|>
+Result: <search result>
+
+This Thought/Search/Result can repeat N times.
+When you are ready to answer the user, use the following format:
+Alice: <answer><|endoftext|>"#;
+
+pub struct Alice {
+ inner: Arc<RawModel>,
+ tokenizer: Arc<Tokenizer>,
+ summarizer: Arc<ExtractiveSummarizer>,
+ end_tokens: Vec<i64>,
+ initial_state: ClonableTensor,
+ encryption_key: Key<Aes256Gcm>,
+}
+
+/// SAFETY:
+/// Alice is thread-safe because it never mutates it's internal state.
+/// It only ever spawns executors that keeps track of the generation state. These are not threadsafe.
+unsafe impl Send for Alice {}
+unsafe impl Sync for Alice {}
+
+pub struct AcceleratorConfig {
+ pub layer_fraction: f64,
+ pub device: tch::Device,
+ pub kind: tch::Kind,
+}
+
+impl From<AcceleratorDevice> for tch::Device {
+ fn from(value: AcceleratorDevice) -> Self {
+ match value {
+ AcceleratorDevice::Cpu => tch::Device::Cpu,
+ AcceleratorDevice::Cuda(d) => tch::Device::Cuda(d),
+ AcceleratorDevice::Mps => tch::Device::Mps,
+ }
+ }
+}
+
+impl From<AcceleratorDtype> for tch::Kind {
+ fn from(value: AcceleratorDtype) -> Self {
+ match value {
+ AcceleratorDtype::Float => tch::Kind::Float,
+ AcceleratorDtype::Bf16 => tch::Kind::BFloat16,
+ }
+ }
+}
+
+impl From<AliceAcceleratorConfig> for AcceleratorConfig {
+ fn from(value: AliceAcceleratorConfig) -> Self {
+ Self {
+ layer_fraction: value.layer_fraction,
+ device: value.device.into(),
+ kind: value.dtype.into(),
+ }
+ }
+}
+
+impl Alice {
+ pub fn open<P: AsRef<Path>>(
+ folder: P,
+ summarizer_path: P,
+ accelerator: Option<AcceleratorConfig>,
+ encryption_key: &[u8],
+ ) -> Result<Alice> {
+ let encryption_key = *Key::<Aes256Gcm>::from_slice(encryption_key);
+ let mut model = RawModel::open(folder.as_ref().join("model.safetensors"))?;
+
+ if let Some(accelerator) = accelerator {
+ model.load_to_device(
+ accelerator.layer_fraction,
+ accelerator.device,
+ accelerator.kind,
+ );
+ }
+
+ let mut summarizer = ExtractiveSummarizer::open(summarizer_path, 1)?;
+ summarizer.set_window_size(30);
+
+ let summarizer = Arc::new(summarizer);
+
+ let inner = Arc::new(model);
+ let tokenizer = Arc::new(Tokenizer::open(folder.as_ref().join("tokenizer.json"))?);
+
+ let end_tokens = vec![tokenizer.tokenizer.token_to_id("<|endoftext|>").unwrap() as i64];
+
+ let initial_state = Alice::prepare_initial_state(&inner, &tokenizer)?;
+
+ Ok(Self {
+ inner,
+ tokenizer,
+ end_tokens,
+ summarizer,
+ initial_state,
+ encryption_key,
+ })
+ }
+
+ fn prepare_initial_state(model: &RawModel, tokenizer: &Tokenizer) -> Result<ClonableTensor> {
+ let tokens = tokenizer.encode(PROMPT_PREFIX.to_string())?;
+
+ let mut state = model.init_state();
+
+ for token in tokens {
+ let (_, new_state) = model.forward(token, Some(&state));
+ state = new_state;
+ }
+
+ Ok(ClonableTensor(state))
+ }
+
+ pub fn new_executor(
+ &self,
+ user_question: &str,
+ last_state: Option<EncryptedState>,
+ search_url: String,
+ optic_url: Option<String>,
+ ) -> Result<ActionExecutor> {
+ let mut state = None;
+
+ if let Some(s) = last_state {
+ let decrypted_state = s.decrypt(&self.encryption_key)?;
+ state = Some(Tensor::from_slice2(&decrypted_state));
+ }
+
+ let (tokens, state) = match state {
+ Some(state) => {
+ let input = format!("User: {user_question}\n\nAlice[thought]:");
+ let tokens = self.tokenizer.encode(input).unwrap();
+
+ (tokens, state)
+ }
+ None => {
+ let input = format!("\n\nUser: {user_question}\n\nAlice[thought]:");
+ let tokens = self.tokenizer.encode(input).unwrap();
+
+ (tokens, self.initial_state.clone().0)
+ }
+ };
+
+ let token_generator = AliceTokenGenerator::new(
+ self.inner.clone(),
+ self.end_tokens.clone(),
+ state,
+ &tokens,
+ None,
+ )?;
+
+ let raw_action_gen = RawActionGenerator::new(token_generator, self.tokenizer.clone());
+ let action_gen = ActionGenerator::new(raw_action_gen);
+
+ let searcher = Searcher {
+ url: search_url,
+ optic_url,
+ summarizer: Arc::clone(&self.summarizer),
+ };
+
+ Ok(ActionExecutor::new(
+ action_gen,
+ searcher,
+ self.encryption_key,
+ ))
+ }
+}
+
+#[derive(Clone)]
+pub struct ModelState {
+ state: ClonableTensor,
+ next_token: i64,
+}
+
+pub struct Tokenizer {
+ tokenizer: tokenizers::Tokenizer,
+}
+
+impl Tokenizer {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Tokenizer> {
+ let tokenizer = tokenizers::Tokenizer::from_file(path)?;
+
+ Ok(Self { tokenizer })
+ }
+
+ pub fn encode(&self, input: String) -> Result<Vec<i64>> {
+ let encoding = self.tokenizer.encode(input, false)?;
+
+ let ids = encoding
+ .get_ids()
+ .iter()
+ .map(|&id| id as i64)
+ .collect::<Vec<_>>();
+
+ Ok(ids)
+ }
+
+ pub fn decode(&self, tokens: &[i64]) -> Result<String> {
+ let tokens = tokens.iter().map(|&id| id as u32).collect::<Vec<_>>();
+
+ let output = self.tokenizer.decode(tokens, true)?;
+
+ Ok(output)
+ }
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+/// base64 encoded `EncryptedState`
+pub struct EncodedEncryptedState(String);
+
+impl EncodedEncryptedState {
+ pub fn encode(state: EncryptedState) -> Self {
+ let bytes = bincode::serialize(&state).unwrap();
+ let encoded = base64::encode(bytes);
+
+ Self(encoded)
+ }
+
+ pub fn decode(self) -> Result<EncryptedState> {
+ let bytes = base64::decode(self.0)?;
+ let state = bincode::deserialize(&bytes)?;
+
+ Ok(state)
+ }
+}
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct EncryptedState {
+ nonce: Vec<u8>,
+ // ciphertext is bincoded `CompressedState` encrypted with `nonce` and a key
+ ciphertext: Vec<u8>,
+}
+
+impl EncryptedState {
+ fn encrypt(state: Vec<Vec<f32>>, key: &Key<Aes256Gcm>) -> Self {
+ let compressed = compress_state(state);
+
+ let cipher = Aes256Gcm::new(key);
+ let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
+
+ let bytes = bincode::serialize(&compressed).unwrap();
+
+ let ciphertext = cipher.encrypt(&nonce, bytes.as_slice()).unwrap();
+
+ Self {
+ nonce: nonce.to_vec(),
+ ciphertext,
+ }
+ }
+
+ fn decrypt(&self, key: &Key<Aes256Gcm>) -> Result<Vec<Vec<f32>>> {
+ let cipher = Aes256Gcm::new(key);
+ let nonce = Nonce::from_slice(&self.nonce);
+
+ let decrypted = cipher
+ .decrypt(nonce, self.ciphertext.as_slice())
+ .map_err(|_| Error::DecryptionFailed)?;
+
+ let compressed: CompressedState = bincode::deserialize(&decrypted)?;
+
+ decompress_state(compressed)
+ }
+}
+
+/// base64 encoded gzipped state
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct CompressedState(String);
+
+pub fn compress_state(state: Vec<Vec<f32>>) -> CompressedState {
+ let state: Vec<Vec<bf16>> = state
+ .into_iter()
+ .map(|v| v.into_iter().map(bf16::from_f32).collect())
+ .collect();
+
+ let mut encoder = GzEncoder::new(Vec::new(), Compression::best());
+
+ encoder
+ .write_all(&bincode::serialize(&state).unwrap())
+ .unwrap();
+
+ let compressed = encoder.finish().unwrap();
+
+ // base64 encode the compressed state
+ CompressedState(base64::encode(compressed))
+}
+
+pub fn decompress_state(state: CompressedState) -> Result<Vec<Vec<f32>>> {
+ let state = base64::decode(state.0)?;
+
+ let mut decoder = GzDecoder::new(&state[..]);
+
+ let mut state = Vec::new();
+ decoder.read_to_end(&mut state)?;
+
+ let state: Vec<Vec<bf16>> = bincode::deserialize(&state)?;
+
+ Ok(state
+ .into_iter()
+ .map(|v| v.into_iter().map(f32::from).collect())
+ .collect())
+}
+
+#[derive(Debug)]
+struct AnyTransitionValidator {
+ validators: Vec<TransitionValidator>,
+}
+
+impl AnyTransitionValidator {
+ fn new(validators: Vec<TransitionValidator>) -> Self {
+ Self { validators }
+ }
+
+ fn validate(&mut self, token: i64) -> bool {
+ for validator in &mut self.validators {
+ if validator.validate(token) {
+ return true;
+ }
+ }
+
+ false
+ }
+}
+
+#[derive(Debug)]
+struct TransitionValidator {
+ tokens: Vec<i64>,
+ next_match: usize,
+}
+
+impl TransitionValidator {
+ fn new(tokens: Vec<i64>) -> Self {
+ Self {
+ tokens,
+ next_match: 0,
+ }
+ }
+
+ fn validate(&mut self, token: i64) -> bool {
+ if self.tokens[self.next_match] == token {
+ self.next_match += 1;
+ if self.next_match == self.tokens.len() {
+ self.next_match = 0;
+ true
+ } else {
+ false
+ }
+ } else {
+ self.next_match = 0;
+ false
+ }
+ }
+}
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/local/raw_model.rs
similarity index 99%
rename from core/src/alice/raw_model.rs
rename to core/src/alice/local/raw_model.rs
index e86ebf3c..44d96404 100644
--- a/core/src/alice/raw_model.rs
+++ b/core/src/alice/local/raw_model.rs
@@ -15,7 +15,7 @@ use std::{fs::OpenOptions, path::Path};
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use super::{Error, Result};
+use crate::alice::{Error, Result};
use safetensors::SafeTensors;
use tch::{
nn::{embedding, layer_norm, Embedding, LayerNorm, LayerNormConfig, Linear, ModuleT, VarStore},
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index 1e05a89c..19dae59b 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -14,47 +14,18 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{
- io::{Read, Write},
- path::Path,
- sync::Arc,
-};
-
-use aes_gcm::{
- aead::{Aead, OsRng},
- AeadCore, Aes256Gcm, Key, KeyInit, Nonce,
-};
-use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};
-use half::bf16;
-use tch::Tensor;
+use std::sync::Arc;
use crate::{
api::search::ApiSearchQuery,
- llm_utils::ClonableTensor,
search_prettifier::DisplayedWebpage,
searcher::{SearchResult, WebsitesResult},
summarizer::{self, ExtractiveSummarizer},
webpage::Url,
- AcceleratorDevice, AcceleratorDtype, AliceAcceleratorConfig,
-};
-
-use self::{
- generate::{ActionExecutor, ActionGenerator, AliceTokenGenerator, RawActionGenerator},
- raw_model::RawModel,
};
-pub mod generate;
-mod raw_model;
-
-const PROMPT_PREFIX: &str = r#"System: Your name is Alice, and you are an AI assistant trained by Stract. Below is a conversation between you and a user. Help the user as best you can.
-You can lookup information on the web in the following format:
-Alice[thought]: I should look up "<keyword>" on the web.
-Search: <query><|endoftext|>
-Result: <search result>
-
-This Thought/Search/Result can repeat N times.
-When you are ready to answer the user, use the following format:
-Alice: <answer><|endoftext|>"#;
+pub mod local;
+pub mod openai;
type Result<T> = std::result::Result<T, Error>;
@@ -95,201 +66,21 @@ pub enum Error {
#[error("Cluster")]
Cluster(#[from] crate::distributed::cluster::Error),
-}
-
-pub struct Alice {
- inner: Arc<RawModel>,
- tokenizer: Arc<Tokenizer>,
- summarizer: Arc<ExtractiveSummarizer>,
- end_tokens: Vec<i64>,
- initial_state: ClonableTensor,
- encryption_key: Key<Aes256Gcm>,
-}
-
-/// SAFETY:
-/// Alice is thread-safe because it never mutates it's internal state.
-/// It only ever spawns executors that keeps track of the generation state. These are not threadsafe.
-unsafe impl Send for Alice {}
-unsafe impl Sync for Alice {}
-
-pub struct AcceleratorConfig {
- pub layer_fraction: f64,
- pub device: tch::Device,
- pub kind: tch::Kind,
-}
-
-impl From<AcceleratorDevice> for tch::Device {
- fn from(value: AcceleratorDevice) -> Self {
- match value {
- AcceleratorDevice::Cpu => tch::Device::Cpu,
- AcceleratorDevice::Cuda(d) => tch::Device::Cuda(d),
- AcceleratorDevice::Mps => tch::Device::Mps,
- }
- }
-}
-
-impl From<AcceleratorDtype> for tch::Kind {
- fn from(value: AcceleratorDtype) -> Self {
- match value {
- AcceleratorDtype::Float => tch::Kind::Float,
- AcceleratorDtype::Bf16 => tch::Kind::BFloat16,
- }
- }
-}
-
-impl From<AliceAcceleratorConfig> for AcceleratorConfig {
- fn from(value: AliceAcceleratorConfig) -> Self {
- Self {
- layer_fraction: value.layer_fraction,
- device: value.device.into(),
- kind: value.dtype.into(),
- }
- }
-}
-
-impl Alice {
- pub fn open<P: AsRef<Path>>(
- folder: P,
- summarizer_path: P,
- accelerator: Option<AcceleratorConfig>,
- encryption_key: &[u8],
- ) -> Result<Alice> {
- let encryption_key = *Key::<Aes256Gcm>::from_slice(encryption_key);
- let mut model = RawModel::open(folder.as_ref().join("model.safetensors"))?;
-
- if let Some(accelerator) = accelerator {
- model.load_to_device(
- accelerator.layer_fraction,
- accelerator.device,
- accelerator.kind,
- );
- }
-
- let mut summarizer = ExtractiveSummarizer::open(summarizer_path, 1)?;
- summarizer.set_window_size(30);
-
- let summarizer = Arc::new(summarizer);
-
- let inner = Arc::new(model);
- let tokenizer = Arc::new(Tokenizer::open(folder.as_ref().join("tokenizer.json"))?);
-
- let end_tokens = vec![tokenizer.tokenizer.token_to_id("<|endoftext|>").unwrap() as i64];
-
- let initial_state = Alice::prepare_initial_state(&inner, &tokenizer)?;
- Ok(Self {
- inner,
- tokenizer,
- end_tokens,
- summarizer,
- initial_state,
- encryption_key,
- })
- }
-
- fn prepare_initial_state(model: &RawModel, tokenizer: &Tokenizer) -> Result<ClonableTensor> {
- let tokens = tokenizer.encode(PROMPT_PREFIX.to_string())?;
-
- let mut state = model.init_state();
-
- for token in tokens {
- let (_, new_state) = model.forward(token, Some(&state));
- state = new_state;
- }
-
- Ok(ClonableTensor(state))
- }
-
- pub fn new_executor(
- &self,
- user_question: &str,
- last_state: Option<EncryptedState>,
- search_url: String,
- optic_url: Option<String>,
- ) -> Result<ActionExecutor> {
- let mut state = None;
-
- if let Some(s) = last_state {
- let decrypted_state = s.decrypt(&self.encryption_key)?;
- state = Some(Tensor::from_slice2(&decrypted_state));
- }
-
- let (tokens, state) = match state {
- Some(state) => {
- let input = format!("User: {user_question}\n\nAlice[thought]:");
- let tokens = self.tokenizer.encode(input).unwrap();
-
- (tokens, state)
- }
- None => {
- let input = format!("\n\nUser: {user_question}\n\nAlice[thought]:");
- let tokens = self.tokenizer.encode(input).unwrap();
-
- (tokens, self.initial_state.clone().0)
- }
- };
-
- let token_generator = AliceTokenGenerator::new(
- self.inner.clone(),
- self.end_tokens.clone(),
- state,
- &tokens,
- None,
- )?;
-
- let raw_action_gen = RawActionGenerator::new(token_generator, self.tokenizer.clone());
- let action_gen = ActionGenerator::new(raw_action_gen);
-
- let searcher = Searcher {
- url: search_url,
- optic_url,
- summarizer: Arc::clone(&self.summarizer),
- };
+ #[error("Unexpected completion")]
+ UnexpectedCompletion,
- Ok(ActionExecutor::new(
- action_gen,
- searcher,
- self.encryption_key,
- ))
- }
-}
-
-#[derive(Clone)]
-pub struct ModelState {
- state: ClonableTensor,
- next_token: i64,
-}
-
-pub struct Tokenizer {
- tokenizer: tokenizers::Tokenizer,
-}
-
-impl Tokenizer {
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Tokenizer> {
- let tokenizer = tokenizers::Tokenizer::from_file(path)?;
-
- Ok(Self { tokenizer })
- }
+ #[error("Event source cannot clone")]
+ EventSourceCannotClone(#[from] reqwest_eventsource::CannotCloneRequestError),
- pub fn encode(&self, input: String) -> Result<Vec<i64>> {
- let encoding = self.tokenizer.encode(input, false)?;
+ #[error("Event source error")]
+ EventSource(#[from] reqwest_eventsource::Error),
- let ids = encoding
- .get_ids()
- .iter()
- .map(|&id| id as i64)
- .collect::<Vec<_>>();
+ #[error("Serde JSON error")]
+ SerdeJson(#[from] serde_json::Error),
- Ok(ids)
- }
-
- pub fn decode(&self, tokens: &[i64]) -> Result<String> {
- let tokens = tokens.iter().map(|&id| id as u32).collect::<Vec<_>>();
-
- let output = self.tokenizer.decode(tokens, true)?;
-
- Ok(output)
- }
+ #[error("Last message should be from user")]
+ LastMessageNotUser,
}
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
@@ -314,6 +105,24 @@ impl SimplifiedWebsite {
}
}
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+#[serde(tag = "@type", rename_all = "camelCase")]
+pub enum ExecutionState {
+ BeginSearch {
+ query: String,
+ },
+ SearchResult {
+ query: String,
+ result: Vec<SimplifiedWebsite>,
+ },
+ Speaking {
+ text: String,
+ },
+ Done {
+ state: local::EncodedEncryptedState,
+ },
+}
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
struct ModelWebsite {
pub title: String,
@@ -378,151 +187,56 @@ impl Searcher {
Ok(websites)
}
-}
-
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
-/// base64 encoded `EncryptedState`
-pub struct EncodedEncryptedState(String);
-impl EncodedEncryptedState {
- pub fn encode(state: EncryptedState) -> Self {
- let bytes = bincode::serialize(&state).unwrap();
- let encoded = base64::encode(bytes);
-
- Self(encoded)
- }
-
- pub fn decode(self) -> Result<EncryptedState> {
- let bytes = base64::decode(self.0)?;
- let state = bincode::deserialize(&bytes)?;
-
- Ok(state)
- }
-}
+ async fn raw_search_async(&self, query: &str) -> Result<WebsitesResult> {
+ let mut optic = None;
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-pub struct EncryptedState {
- nonce: Vec<u8>,
- // ciphertext is bincoded `CompressedState` encrypted with `nonce` and a key
- ciphertext: Vec<u8>,
-}
-
-impl EncryptedState {
- fn encrypt(state: Vec<Vec<f32>>, key: &Key<Aes256Gcm>) -> Self {
- let compressed = compress_state(state);
-
- let cipher = Aes256Gcm::new(key);
- let nonce = Aes256Gcm::generate_nonce(&mut OsRng);
-
- let bytes = bincode::serialize(&compressed).unwrap();
-
- let ciphertext = cipher.encrypt(&nonce, bytes.as_slice()).unwrap();
-
- Self {
- nonce: nonce.to_vec(),
- ciphertext,
+ if let Some(url) = &self.optic_url {
+ if let Ok(r) = reqwest::get(url).await {
+ if let Ok(text) = r.text().await {
+ optic = Some(text);
+ }
+ }
}
- }
-
- fn decrypt(&self, key: &Key<Aes256Gcm>) -> Result<Vec<Vec<f32>>> {
- let cipher = Aes256Gcm::new(key);
- let nonce = Nonce::from_slice(&self.nonce);
- let decrypted = cipher
- .decrypt(nonce, self.ciphertext.as_slice())
- .map_err(|_| Error::DecryptionFailed)?;
+ let client = reqwest::Client::new();
+ let query = ApiSearchQuery {
+ query: query.trim().to_string(),
+ num_results: Some(3),
+ optic,
+ page: None,
+ selected_region: None,
+ site_rankings: None,
+ return_ranking_signals: None,
+ flatten_response: Some(false),
+ };
+ tracing::debug!("searching at {:?}: {:#?}", self.url, query);
- let compressed: CompressedState = bincode::deserialize(&decrypted)?;
+ let res: SearchResult = client
+ .post(&self.url)
+ .json(&query)
+ .send()
+ .await?
+ .json()
+ .await?;
- decompress_state(compressed)
+ match res {
+ SearchResult::Websites(res) => Ok(res),
+ SearchResult::Bang(_) => Err(Error::UnexpectedSearchResult),
+ }
}
-}
-
-/// base64 encoded gzipped state
-#[derive(Debug, serde::Serialize, serde::Deserialize)]
-pub struct CompressedState(String);
-
-pub fn compress_state(state: Vec<Vec<f32>>) -> CompressedState {
- let state: Vec<Vec<bf16>> = state
- .into_iter()
- .map(|v| v.into_iter().map(bf16::from_f32).collect())
- .collect();
- let mut encoder = GzEncoder::new(Vec::new(), Compression::best());
-
- encoder
- .write_all(&bincode::serialize(&state).unwrap())
- .unwrap();
-
- let compressed = encoder.finish().unwrap();
-
- // base64 encode the compressed state
- CompressedState(base64::encode(compressed))
-}
-
-pub fn decompress_state(state: CompressedState) -> Result<Vec<Vec<f32>>> {
- let state = base64::decode(state.0)?;
-
- let mut decoder = GzDecoder::new(&state[..]);
-
- let mut state = Vec::new();
- decoder.read_to_end(&mut state)?;
-
- let state: Vec<Vec<bf16>> = bincode::deserialize(&state)?;
-
- Ok(state
- .into_iter()
- .map(|v| v.into_iter().map(f32::from).collect())
- .collect())
-}
-
-#[derive(Debug)]
-struct AnyTransitionValidator {
- validators: Vec<TransitionValidator>,
-}
+ async fn search_async(&self, query: &str) -> Result<Vec<SimplifiedWebsite>> {
+ let res = self.raw_search_async(query).await?;
-impl AnyTransitionValidator {
- fn new(validators: Vec<TransitionValidator>) -> Self {
- Self { validators }
- }
+ let mut websites = Vec::new();
- fn validate(&mut self, token: i64) -> bool {
- for validator in &mut self.validators {
- if validator.validate(token) {
- return true;
- }
+ for website in res.webpages {
+ websites.push(SimplifiedWebsite::new(website, query, &self.summarizer));
}
- false
- }
-}
-
-#[derive(Debug)]
-struct TransitionValidator {
- tokens: Vec<i64>,
- next_match: usize,
-}
-
-impl TransitionValidator {
- fn new(tokens: Vec<i64>) -> Self {
- Self {
- tokens,
- next_match: 0,
- }
- }
+ tracing::debug!("search result: {:#?}", websites);
- fn validate(&mut self, token: i64) -> bool {
- if self.tokens[self.next_match] == token {
- self.next_match += 1;
- if self.next_match == self.tokens.len() {
- self.next_match = 0;
- true
- } else {
- false
- }
- } else {
- self.next_match = 0;
- false
- }
+ Ok(websites)
}
}
diff --git a/core/src/alice/openai.rs b/core/src/alice/openai.rs
new file mode 100644
index 00000000..ccec6388
--- /dev/null
+++ b/core/src/alice/openai.rs
@@ -0,0 +1,385 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::path::Path;
+use std::sync::Arc;
+
+use reqwest_eventsource::EventSource;
+use tokio_stream::StreamExt;
+
+use crate::summarizer::ExtractiveSummarizer;
+
+use super::Error;
+use super::ExecutionState;
+use super::ModelWebsite;
+use super::Result;
+use super::Searcher;
+use super::SimplifiedWebsite;
+
+const MAX_NUM_SEARCHES: usize = 3;
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+#[serde(tag = "@type", rename_all = "camelCase")]
+pub enum Message {
+ User {
+ message: String,
+ },
+ Alice {
+ message: String,
+ queries: Vec<String>,
+ },
+}
+
+fn prompt(user_question: &str, conv: &[Message]) -> String {
+ let mut prompt = r#"System: Your name is Alice, and you are an AI assistant developed by Stract. Below is a conversation between you and a user. Help the user as best you can.
+You can lookup information on the web in the following format:
+Alice[thought]: I should look up "<keywords>" on the web.
+Search i: <query>
+Result: <search result>
+
+This Thought/Search/Result can repeat N times.
+You should always cite the source of your statements using the format [search i source j] at the end of the statement.
+
+The search results from your previous messages have been truncated.
+
+When you are ready to answer the user, use the following format:
+Alice[thought]: I now know enough to answer the user.
+Alice: <answer>
+
+You should only use the information you find on the web. If you couldn't find a good answer to the user, you should tell them and ask for clarification.
+
+Begin!"#.to_string();
+
+ // truncate conv
+ let conv = if conv.len() > 4 {
+ &conv[conv.len() - 4..]
+ } else {
+ conv
+ };
+
+ for msg in conv {
+ match msg {
+ Message::User { message } => {
+ prompt.push_str("\n\n");
+ prompt.push_str(format!("User: {}", message).as_str());
+ }
+ Message::Alice { message, queries } => {
+ prompt.push('\n');
+ for query in queries {
+ prompt.push('\n');
+ prompt.push_str(&format!(
+ "Alice[thought]: I should look up \"{query}\" on the web."
+ ));
+ prompt.push('\n');
+ prompt.push_str(&format!("Search i: {query}\n"));
+ prompt.push_str("Result: <truncated>");
+ }
+ prompt.push('\n');
+ prompt.push_str("Alice[thought]: I now know enough to answer the user.");
+ prompt.push('\n');
+ prompt.push_str(&format!("Alice: {}", message));
+ }
+ }
+ }
+
+ prompt.push_str("\n\n");
+ prompt.push_str(format!("User: {}", user_question).as_str());
+
+ prompt
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+struct OpenaiParams {
+ model: String,
+ prompt: String,
+ max_tokens: usize,
+ temperature: f32,
+ top_p: f32,
+ stream: bool,
+ stop: Vec<String>,
+}
+
+async fn thoughts(prompt: &str, api_key: &str) -> Result<String> {
+ let client = reqwest::Client::new();
+ let res: serde_json::Value = client
+ .post("https://api.openai.com/v1/completions")
+ .bearer_auth(api_key)
+ .json(&OpenaiParams {
+ model: "text-davinci-003".to_string(),
+ prompt: prompt.to_string(),
+ max_tokens: 100,
+ temperature: 0.7,
+ top_p: 0.7,
+ stream: false,
+ stop: vec!["Result:".to_string(), "Alice:".to_string()],
+ })
+ .send()
+ .await?
+ .json()
+ .await?;
+
+ let text = res["choices"][0]["text"].as_str().unwrap().to_string();
+
+ Ok(dbg!(text))
+}
+
+async fn speak(prompt: &str, api_key: &str) -> Result<EventSource> {
+ let client = reqwest::Client::new();
+ let req = client
+ .post("https://api.openai.com/v1/completions")
+ .bearer_auth(api_key)
+ .json(&OpenaiParams {
+ model: "text-davinci-003".to_string(),
+ prompt: prompt.to_string(),
+ max_tokens: 512,
+ temperature: 0.7,
+ top_p: 0.7,
+ stream: true,
+ stop: vec!["<|endoftext|>".to_string()],
+ });
+
+ let res = EventSource::new(req)?;
+
+ Ok(res)
+}
+
+#[derive(Debug)]
+enum State {
+ BeginSearch { query: String },
+ PerformSearch { query: String },
+
+ Thinking,
+ Speaking,
+}
+
+fn next_state(completion: &str) -> Result<State> {
+ let completion = completion.trim();
+
+ if completion.ends_with("Alice:")
+ || completion.contains("I now know enough to answer the user.")
+ {
+ Ok(State::Speaking)
+ } else if completion.contains("Search") {
+ // find last "Search \d: <query>" and extract query
+ let mut query = None;
+ for line in completion.lines().rev() {
+ if line.starts_with("Search") {
+ query = Some(line.split(':').nth(1).unwrap().trim());
+ break;
+ }
+ }
+
+ if let Some(query) = query {
+ Ok(State::BeginSearch {
+ query: query.to_string(),
+ })
+ } else {
+ Err(Error::UnexpectedCompletion)
+ }
+ } else {
+ Err(Error::UnexpectedCompletion)
+ }
+}
+
+pub struct Alice {
+ api_key: String,
+ summarizer: Arc<ExtractiveSummarizer>,
+}
+
+impl Alice {
+ pub fn open<P: AsRef<Path>>(summarizer_path: P, api_key: &str) -> Result<Self> {
+ let mut summarizer = ExtractiveSummarizer::open(summarizer_path, 1)?;
+ summarizer.set_window_size(30);
+
+ let summarizer = Arc::new(summarizer);
+
+ Ok(Self {
+ api_key: api_key.to_string(),
+ summarizer,
+ })
+ }
+
+ pub fn new_executor(
+ &self,
+ conversation: Vec<Message>,
+ optic_url: Option<String>,
+ search_url: String,
+ ) -> Result<Executor> {
+ let user_question = match conversation.last() {
+ Some(Message::User { message }) => message.clone(),
+ _ => return Err(Error::LastMessageNotUser),
+ };
+
+ let prev_conv = conversation[..conversation.len() - 1].to_vec();
+
+ let searcher = Searcher {
+ url: search_url,
+ optic_url,
+ summarizer: self.summarizer.clone(),
+ };
+
+ Ok(Executor::new(
+ prev_conv,
+ self.api_key.clone(),
+ user_question,
+ searcher,
+ ))
+ }
+}
+
+#[derive(Clone)]
+struct Search {
+ query: String,
+ results: Vec<SimplifiedWebsite>,
+}
+
+pub struct Executor {
+ api_key: String,
+ prev_conv: Vec<Message>,
+ current_searches: Vec<Search>,
+ user_question: String,
+ state: State,
+ searcher: Searcher,
+ speaking_stream: Option<EventSource>,
+ performed_searches: usize,
+}
+
+impl Executor {
+ fn new(
+ prev_conv: Vec<Message>,
+ api_key: String,
+ user_question: String,
+ searcher: Searcher,
+ ) -> Self {
+ Self {
+ api_key,
+ state: State::Thinking,
+ prev_conv,
+ user_question,
+ current_searches: Vec::new(),
+ speaking_stream: None,
+ searcher,
+ performed_searches: 0,
+ }
+ }
+
+ fn build_prompt(&self, speaking: bool) -> String {
+ let mut prompt = prompt(&self.user_question, &self.prev_conv);
+
+ for (i, search) in self.current_searches.clone().into_iter().enumerate() {
+ prompt.push_str("\n\n");
+ prompt.push_str(
+ format!(
+ "Alice[thought]: I should look up \"{}\" on the web.",
+ search.query
+ )
+ .as_str(),
+ );
+ prompt.push_str(format!("\nSearch {}: {}", i + 1, search.query).as_str());
+ let res = search
+ .results
+ .into_iter()
+ .take(3)
+ .map(ModelWebsite::from)
+ .collect::<Vec<_>>();
+
+ prompt
+ .push_str(format!("\nResponse: {}", serde_json::to_string(&res).unwrap()).as_str());
+ }
+
+ prompt.push_str("\n\n");
+
+ if speaking {
+ prompt.push_str("Alice[thought]: I now know enough to answer the user.");
+ prompt.push_str("\n\n");
+ prompt.push_str("Alice:");
+ } else {
+ prompt.push_str("Alice[thought]:");
+ }
+
+ dbg!(prompt)
+ }
+
+ pub async fn next(&mut self) -> Result<Option<ExecutionState>> {
+ loop {
+ match &mut self.state {
+ State::BeginSearch { query } => {
+ let query = query.clone();
+
+ if self.current_searches.iter().any(|s| s.query == query) {
+ self.state = State::Speaking;
+ } else {
+ self.state = State::PerformSearch {
+ query: query.clone(),
+ };
+
+ return Ok(Some(ExecutionState::BeginSearch { query }));
+ }
+ }
+ State::PerformSearch { query } => {
+ let query = query.to_string();
+ let res = self.searcher.search_async(&query).await?;
+
+ self.current_searches.push(Search {
+ query: query.clone(),
+ results: res.clone(),
+ });
+
+ self.performed_searches += 1;
+
+ if self.performed_searches >= MAX_NUM_SEARCHES {
+ self.state = State::Speaking;
+ } else {
+ self.state = State::Thinking;
+ }
+
+ return Ok(Some(ExecutionState::SearchResult { query, result: res }));
+ }
+ State::Thinking => {
+ let prompt = self.build_prompt(false);
+ let cur_thoughts = thoughts(&prompt, &self.api_key).await?;
+ let next_state = dbg!(next_state(&cur_thoughts))?;
+ self.state = next_state;
+ }
+ State::Speaking => {
+ if self.speaking_stream.is_none() {
+ let prompt = self.build_prompt(true);
+ self.speaking_stream = Some(speak(&prompt, &self.api_key).await?);
+ }
+
+ let stream = self.speaking_stream.as_mut().unwrap();
+ match stream.next().await {
+ Some(event) => {
+ let event = event?;
+
+ match event {
+ reqwest_eventsource::Event::Open => {}
+ reqwest_eventsource::Event::Message(msg) => {
+ let res: serde_json::Value = serde_json::from_str(&msg.data)?;
+ let text =
+ res["choices"][0]["text"].as_str().unwrap().to_string();
+
+ return Ok(Some(ExecutionState::Speaking { text }));
+ }
+ }
+ }
+ None => return Ok(None),
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/core/src/api/alice.rs b/core/src/api/alice.rs
index eaead227..efc5308b 100644
--- a/core/src/api/alice.rs
+++ b/core/src/api/alice.rs
@@ -27,7 +27,7 @@ use tokio_stream::{Stream, StreamExt};
use crate::{
distributed::member::Service,
- entrypoint::{self, alice::SaveStateParams},
+ entrypoint::{self, alice::openai::SaveStateParams},
};
#[derive(Debug, serde::Serialize, serde::Deserialize)]
@@ -164,7 +164,7 @@ pub async fn route(
.ok_or(StatusCode::INTERNAL_SERVER_ERROR)?
};
- let params = entrypoint::alice::Params {
+ let params = entrypoint::alice::local::Params {
message: params.message,
prev_state: saved_state.map(|s| s.uuid),
optic: params.optic,
@@ -187,3 +187,70 @@ pub async fn route(
Ok(Sse::new(stream))
}
+
+// for openai alice
+// #[derive(serde::Serialize, serde::Deserialize, Debug)]
+// #[serde(rename_all = "camelCase")]
+// pub struct Params {
+// optic: Option<String>,
+// conv: EncodedSavedState,
+// }
+//
+// pub async fn route(
+// extract::State(state): extract::State<Arc<super::State>>,
+// extract::Query(params): extract::Query<Params>,
+// ) -> std::result::Result<
+// Sse<impl Stream<Item = std::result::Result<axum::response::sse::Event, Infallible>>>,
+// StatusCode,
+// > {
+// let saved_state: SavedState = params.conv.decode().map_err(|e| {
+// tracing::error!("Error decoding saved state: {}", e);
+// StatusCode::BAD_REQUEST
+// })?;
+//
+// let client = reqwest::ClientBuilder::default()
+// .build()
+// .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+//
+// let mut members = state.cluster.members().await;
+//
+// members.shuffle(&mut rand::thread_rng());
+//
+// let id = saved_state.id.clone();
+// let alice_addr = members
+// .into_iter()
+// .find_map(|m| {
+// if m.id == id {
+// if let Service::Alice { host } = m.service {
+// Some(host)
+// } else {
+// None
+// }
+// } else {
+// None
+// }
+// })
+// .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;
+//
+// let params = entrypoint::alice::openai::Params {
+// conversation: saved_state.uuid,
+// optic: params.optic,
+// };
+//
+// let mut events = client
+// .get(format!("http://{}", alice_addr))
+// .query(&params)
+// .send()
+// .await
+// .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?
+// .bytes_stream()
+// .eventsource();
+//
+// let stream = async_stream::stream! {
+// while let Some(Ok(item)) = events.next().await {
+// yield Ok(Event::default().data(item.data));
+// }
+// };
+//
+// Ok(Sse::new(stream))
+// }
diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice/local.rs
similarity index 94%
rename from core/src/entrypoint/alice.rs
rename to core/src/entrypoint/alice/local.rs
index c6744e0b..a7032932 100644
--- a/core/src/entrypoint/alice.rs
+++ b/core/src/entrypoint/alice/local.rs
@@ -16,7 +16,6 @@
use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};
-use aes_gcm::{aead::OsRng, Aes256Gcm, KeyInit};
use axum::{
extract,
response::{
@@ -32,13 +31,13 @@ use tokio_stream::StreamExt as _;
use tracing::info;
use crate::{
- alice::{Alice, EncodedEncryptedState, EncryptedState},
+ alice::local::{Alice, EncodedEncryptedState, EncryptedState},
distributed::{
cluster::Cluster,
member::{Member, Service},
},
ttl_cache::TTLCache,
- AliceConfig,
+ AliceLocalConfig,
};
pub struct State {
@@ -167,7 +166,7 @@ pub async fn route(
)
}
-pub async fn run(config: AliceConfig) -> Result<(), crate::alice::Error> {
+pub async fn run(config: AliceLocalConfig) -> Result<(), crate::alice::Error> {
let addr: SocketAddr = config.host;
let key = base64::decode(config.encryption_key)?;
@@ -198,8 +197,3 @@ pub async fn run(config: AliceConfig) -> Result<(), crate::alice::Error> {
Ok(())
}
-
-pub fn generate_key() {
- let key = Aes256Gcm::generate_key(OsRng);
- println!("{}", base64::encode(key.as_slice()));
-}
diff --git a/core/src/entrypoint/alice/mod.rs b/core/src/entrypoint/alice/mod.rs
new file mode 100644
index 00000000..bad2db74
--- /dev/null
+++ b/core/src/entrypoint/alice/mod.rs
@@ -0,0 +1,28 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use aes_gcm::{aead::OsRng, Aes256Gcm, KeyInit};
+
+pub mod local;
+
+/// This is currently not used, but let's keep it around for now
+/// in case we want to use it later.
+pub mod openai;
+
+pub fn generate_key() {
+ let key = Aes256Gcm::generate_key(OsRng);
+ println!("{}", base64::encode(key.as_slice()));
+}
diff --git a/core/src/entrypoint/alice/openai.rs b/core/src/entrypoint/alice/openai.rs
new file mode 100644
index 00000000..d70bef54
--- /dev/null
+++ b/core/src/entrypoint/alice/openai.rs
@@ -0,0 +1,186 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};
+
+use axum::{
+ extract,
+ response::{
+ sse::{Event, KeepAlive},
+ IntoResponse, Sse,
+ },
+ routing::{get, post},
+ Router,
+};
+use tokio::sync::Mutex;
+use tokio_stream::{Stream, StreamExt};
+use tracing::info;
+
+use crate::{
+ alice::openai::{Alice, Message},
+ distributed::{
+ cluster::Cluster,
+ member::{Member, Service},
+ },
+ ttl_cache::TTLCache,
+ AliceOpenaiConfig,
+};
+
+pub struct State {
+ pub alice: Alice,
+ pub cluster: Cluster,
+ pub conv_states: Arc<Mutex<TTLCache<uuid::Uuid, Vec<Message>>>>,
+}
+
+fn router(alice: Alice, cluster: Cluster) -> Router {
+ let state = Arc::new(State {
+ alice,
+ cluster,
+ conv_states: Arc::new(Mutex::new(TTLCache::with_ttl(Duration::from_secs(10)))),
+ });
+
+ Router::new()
+ .route("/", get(route))
+ .route("/save_state", post(save_state))
+ .with_state(state)
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+pub struct SaveStateParams {
+ pub conversation: Vec<Message>,
+}
+
+pub async fn save_state(
+ extract::State(state): extract::State<Arc<State>>,
+ extract::Json(params): extract::Json<SaveStateParams>,
+) -> impl IntoResponse {
+ let uuid = uuid::Uuid::new_v4();
+
+ state
+ .conv_states
+ .lock()
+ .await
+ .insert(uuid, params.conversation);
+
+ uuid.to_string()
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Params {
+ pub optic: Option<String>,
+ pub conversation: uuid::Uuid,
+}
+
+pub async fn route(
+ extract::State(state): extract::State<Arc<State>>,
+ extract::Query(params): extract::Query<Params>,
+) -> std::result::Result<
+ Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>,
+ http::StatusCode,
+> {
+ let conversation = state
+ .conv_states
+ .lock()
+ .await
+ .get(&params.conversation)
+ .cloned()
+ .ok_or(http::StatusCode::BAD_REQUEST)?;
+
+ let search_addr = state
+ .cluster
+ .members()
+ .await
+ .into_iter()
+ .find_map(|m| {
+ if let Service::Frontend { host } = m.service {
+ Some(host)
+ } else {
+ None
+ }
+ })
+ .ok_or_else(|| {
+ info!("no frontend found");
+
+ http::StatusCode::INTERNAL_SERVER_ERROR
+ })?;
+
+ let mut executor = state
+ .alice
+ .new_executor(
+ conversation,
+ params.optic,
+ format!("http://{}/beta/api/search", search_addr),
+ )
+ .map_err(|e| {
+ info!("error creating executor: {}", e);
+ http::StatusCode::BAD_REQUEST
+ })?;
+
+ let (tx, mut rx) = tokio::sync::mpsc::channel(1);
+
+ tokio::task::spawn(async move {
+ while let Ok(Some(msg)) = executor.next().await {
+ let msg = serde_json::to_string(&msg)
+ .map_err(|e| {
+ info!("error serializing message: {}", e);
+ e
+ })
+ .unwrap();
+
+ tx.send(msg).await.ok();
+ }
+ });
+
+ let stream = async_stream::stream! {
+ while let Some(item) = rx.recv().await {
+ yield item;
+ }
+ };
+
+ Ok(
+ Sse::new(stream.map(|term| Event::default().data(term)).map(Ok))
+ .keep_alive(KeepAlive::default()),
+ )
+}
+
+/// This is currently not used, but let's keep it around for now
+/// in case we want to use it later.
+pub async fn run(config: AliceOpenaiConfig) -> Result<(), crate::alice::Error> {
+ let addr: SocketAddr = config.host;
+
+ let alice = Alice::open(&config.summarizer_path, &config.api_key)?;
+
+ let cluster = Cluster::join(
+ Member {
+ id: config.cluster_id,
+ service: Service::Alice { host: config.host },
+ },
+ config.gossip_addr,
+ config.gossip_seed_nodes.unwrap_or_default(),
+ )
+ .await?;
+
+ let app = router(alice, cluster);
+
+ info!("alice is ready to accept requests on {}", addr);
+ axum::Server::bind(&addr)
+ .serve(app.into_make_service())
+ .await
+ .unwrap();
+
+ Ok(())
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 2d3f665a..cbd6b691 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -215,19 +215,30 @@ pub struct AliceAcceleratorConfig {
}
#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct AliceConfig {
+pub struct AliceLocalConfig {
pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub host: SocketAddr,
+ pub summarizer_path: String,
pub alice_path: String,
- pub summarizer_path: String,
pub accelerator: Option<AliceAcceleratorConfig>,
/// base64 encoded
pub encryption_key: String,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AliceOpenaiConfig {
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+ pub host: SocketAddr,
+ pub summarizer_path: String,
+
+ pub api_key: String,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct WebgraphServerConfig {
pub host: SocketAddr,
diff --git a/core/src/main.rs b/core/src/main.rs
index 975a737c..43f92acf 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -24,7 +24,7 @@ use stract::entrypoint::configure;
use stract::entrypoint::indexer::IndexPointer;
use stract::entrypoint::{self, frontend, search_server, webgraph_server};
use stract::webgraph::WebgraphBuilder;
-use stract::{AliceConfig, FrontendConfig, SearchServerConfig, WebgraphServerConfig};
+use stract::{AliceLocalConfig, FrontendConfig, SearchServerConfig, WebgraphServerConfig};
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -317,12 +317,12 @@ fn main() -> Result<()> {
),
Commands::Alice { options } => match options {
AliceOptions::Serve { config_path } => {
- let config: AliceConfig = load_toml_config(config_path);
+ let config: AliceLocalConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
- .block_on(entrypoint::alice::run(config))?
+ .block_on(entrypoint::alice::local::run(config))?
}
AliceOptions::GenerateKey => entrypoint::alice::generate_key(),
},
diff --git a/frontend/src/pages/chat.astro b/frontend/src/pages/chat.astro
index 25b5d186..751b66f2 100644
--- a/frontend/src/pages/chat.astro
+++ b/frontend/src/pages/chat.astro
@@ -18,7 +18,7 @@ import OpticSelector from "../components/OpticSelector.astro";
<div id="message-container" class="flex px-5 flex-col gap-y-3 h-full">
<div class="flex flex-col h-full items-center gap-y-3">
<h1 class="text-2xl font-bold">Chat with Alice</h1>
- <p class="text-sm">Alice is an AI that tries to answer your questions by searhing for information on the internet. Alice is <b class="font-bold">highly experimental</b> and might produce wrong answers. As always, you should verify the information from multiple sources and make sure the sources supports the claims.</p>
+ <p class="text-sm">Alice is an AI that tries to answer your questions by searhing for information on the internet. Alice is <b class="font-bold">highly experimental</b> and might produce bad or downright wrong answers. As always, you should verify the information from multiple sources and make sure the sources supports the claims.</p>
</div>
</div>
<div class="w-full gap-y-2">