Skip to content

web search done right

summarization using mistral and 'chain-of-density' approach. the summarization becomes much better if we allow the model to first generate a candidate summarization and then improving on it. doing the improvement step just once seems to significantly improve the summary. we also now use an llm (mistral 7b) for the summarisations, as we can then use the same model for multiple tasks and serve it using gpus, thus significantly decreasing the latency.

Mikkel Denkerfbc01ad865fb
diff --git a/.git-hooks/pre-commit b/.git-hooks/pre-commit
index 121094ee..42ea6c92 100755
--- a/.git-hooks/pre-commit
+++ b/.git-hooks/pre-commit
@@ -1,4 +1,5 @@
#!/bin/bash
+set -e
just cargo check
just cargo check --no-default-features
diff --git a/Cargo.lock b/Cargo.lock
index e549001c..894b8bf4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4346,6 +4346,7 @@ dependencies = [
"lalrpop",
"lalrpop-util",
"libc",
+ "log",
"logos",
"lz-str",
"lz4_flex",
diff --git a/Cargo.toml b/Cargo.toml
index a61614f1..0391fce0 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -47,6 +47,7 @@ itertools = "0.11.0"
lalrpop = {version = "0.20.0", features = ["lexer"]}
lalrpop-util = {version = "0.20.0", features = ["lexer"]}
libc = "0.2.142"
+log = {version = "0.4", features = ["release_max_level_info"]}
logos = "0.13.0"
lz-str = "0.2.1"
lz4_flex = "0.11.1"
@@ -98,7 +99,7 @@ tokio = {version = "1.23.1", features = ["full"]}
tokio-stream = "0.1.11"
toml = "0.8.2"
tower-http = {version = "0.5.0", features = ["compression-gzip", "cors"]}
-tracing = "0.1.34"
+tracing = {version = "0.1.34", features = ["release_max_level_info"]}
tracing-subscriber = {version = "0.3.11", features = ["env-filter"]}
url = {version = "2.4.0", features = ["serde"]}
urlencoding = "2.1.2"
diff --git a/configs/api.toml b/configs/api.toml
index d7432de2..7ea05df9 100644
--- a/configs/api.toml
+++ b/configs/api.toml
@@ -9,6 +9,9 @@ queries_csv_path = "data/queries_us.csv"
spell_checker_path = "data/web_spell/checker"
# qa_model_path = "data/qa_model"
bangs_path = "data/bangs.json"
+# openai_api_model = "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf"
+openai_api_model = "TheBloke/Mistral-7B-Instruct-v0.2-AWQ"
+openai_api_uri = "http://localhost:4000"
summarizer_path = "data/summarizer"
[thresholds]
diff --git a/core/Cargo.toml b/core/Cargo.toml
index d7e8a760..5c1c5dff 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -56,6 +56,7 @@ itertools = {workspace = true}
kuchiki = {path = "../kuchiki"}
lalrpop-util = {workspace = true}
libc = {workspace = true}
+log = {workspace = true}
logos = {workspace = true}
lz-str = {workspace = true}
lz4_flex = {workspace = true}
@@ -129,11 +130,6 @@ name = "search_preindexed_optic"
harness = false
name = "harmonic_centrality"
-[[bench]]
-harness = false
-name = "abstractive_summarizer"
-required-features = ["libtorch"]
-
[[bench]]
harness = false
name = "build_similarity"
diff --git a/core/benches/abstractive_summarizer.rs b/core/benches/abstractive_summarizer.rs
deleted file mode 100644
index 3ce01c74..00000000
--- a/core/benches/abstractive_summarizer.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-use criterion::{criterion_group, criterion_main, Criterion};
-use stract::summarizer::{AbstractiveModel, AbstractiveSummarizer};
-
-pub fn criterion_benchmark(c: &mut Criterion) {
- let model =
- AbstractiveSummarizer::new(AbstractiveModel::open("data/abstractive_summary").unwrap());
-
- let text = r#"Aristotle (/ˈærɪstɒtəl/;[1] Greek: Ἀριστοτέλης Aristotélēs, pronounced [aristotélɛːs]; 384–322 BC) was an Ancient Greek philosopher and polymath. His writings cover a broad range of subjects including physics, biology, zoology, metaphysics, logic, ethics, aesthetics, poetry, drama, music, rhetoric, psychology, linguistics, economics, politics, meteorology, geology, and government. As the founder of the Peripatetic school of philosophy in the Lyceum in Athens, he began the wider Aristotelian tradition that followed, which set the groundwork for the development of modern science.
- Little is known about Aristotle's life. He was born in the city of Stagira in Northern Greece during the Classical period. His father, Nicomachus, died when Aristotle was a child, and he was brought up by a guardian. At seventeen or eighteen years of age he joined Plato's Academy in Athens and remained there until the age of thirty-seven (c. 347 BC). Shortly after Plato died, Aristotle left Athens and, at the request of Philip II of Macedon, tutored his son Alexander the Great beginning in 343 BC. He established a library in the Lyceum which helped him to produce many of his hundreds of books on papyrus scrolls.
- Though Aristotle wrote many elegant treatises and dialogues for publication, only around a third of his original output has survived, none of it intended for publication. Aristotle provided a complex synthesis of the various philosophies existing prior to him. It was above all from his teachings that the West inherited its intellectual lexicon, as well as problems and methods of inquiry. As a result, his philosophy has exerted a unique influence on almost every form of knowledge in the West and it continues to be a subject of contemporary philosophical discussion.
- Aristotle's views profoundly shaped medieval scholarship. The influence of physical science extended from Late Antiquity and the Early Middle Ages into the Renaissance, and were not replaced systematically until the Enlightenment and theories such as classical mechanics were developed. Some of Aristotle's zoological observations found in his biology, such as on the hectocotyl (reproductive) arm of the octopus, were disbelieved until the 19th century. He also influenced Judeo-Islamic philosophies during the Middle Ages, as well as Christian theology, especially the Neoplatonism of the Early Church and the scholastic tradition of the Catholic Church. Aristotle was revered among medieval Muslim scholars as "The First Teacher", and among medieval Christians like Thomas Aquinas as simply "The Philosopher", while the poet Dante called him "the master of those who know". His works contain the earliest known formal study of logic, and were studied by medieval scholars such as Peter Abelard and John Buridan. Aristotle's influence on logic continued well into the 19th century. In addition, his ethics, though always influential, gained renewed interest with the modern advent of virtue ethics."#;
-
- c.bench_function("Summarize Aristotles wiki text", |b| {
- b.iter(|| model.summarize(text))
- });
-}
-
-criterion_group!(benches, criterion_benchmark);
-criterion_main!(benches);
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 9b49db18..8ba562fd 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -167,7 +167,11 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
autosuggest,
counters,
remote_webgraph,
- summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),
+ summarizer: Arc::new(Summarizer::new(
+ &config.summarizer_path,
+ config.openai_api_uri.clone(),
+ config.openai_api_model.clone(),
+ )?),
improvement_queue: query_store_queue,
cluster,
})
diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rs
index 1742b5f7..1c9faed8 100644
--- a/core/src/api/summarize.rs
+++ b/core/src/api/summarize.rs
@@ -23,7 +23,6 @@ use axum::response::{sse::Event, Sse};
use futures::stream::Stream;
use http::StatusCode;
use serde::Deserialize;
-use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
use tokio_stream::StreamExt as _;
use utoipa::IntoParams;
@@ -36,14 +35,6 @@ pub struct SummarizeParams {
pub query: String,
}
-fn summarize_blocking(iter: impl Iterator<Item = String>, tx: UnboundedSender<String>) {
- for tok in iter {
- if tx.send(tok).is_err() {
- break;
- }
- }
-}
-
#[cfg(feature = "libtorch")]
async fn summarize(
params: SummarizeParams,
@@ -55,18 +46,10 @@ async fn summarize(
.await?
.ok_or_else(|| anyhow::anyhow!("Webpage not found: {}", params.url))?;
- let (tx, mut rx) = unbounded_channel();
-
- let summarizer = Arc::clone(&state.summarizer);
- let it = summarizer.summarize_iter(&params.query, &webpage.body)?;
-
- tokio::task::spawn_blocking(move || summarize_blocking(it, tx));
-
- let stream = async_stream::stream! {
- while let Some(item) = rx.recv().await {
- yield item;
- }
- };
+ let stream = state
+ .summarizer
+ .summarize(&params.query, &webpage.body)
+ .await?;
Ok(
Sse::new(stream.map(|term| Event::default().data(term)).map(Ok))
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 8e365610..005242cf 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -201,6 +201,8 @@ pub struct ApiConfig {
pub spell_checker_path: Option<String>,
pub bangs_path: String,
pub summarizer_path: String,
+ pub openai_api_uri: String,
+ pub openai_api_model: String,
pub query_store_db_host: Option<String>,
pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 7e3ed07d..5fbab8df 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -45,6 +45,7 @@ fn download_files() {
"english-wordnet-2022-subset.ttl",
"lambdamart.txt",
"test.zim",
+ "mistral-7b-instruct-v0.2.Q4_K_M.gguf",
] {
let path = Path::new(DATA_PATH).join(name);
diff --git a/core/src/llm_utils.rs b/core/src/llm_utils.rs
index ae90083d..ddc191b7 100644
--- a/core/src/llm_utils.rs
+++ b/core/src/llm_utils.rs
@@ -14,30 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::Result;
+use anyhow::anyhow;
+use eventsource_stream::Eventsource;
+use futures::stream::Stream;
use tch::{Kind, Tensor};
-
-pub fn sample_typical(probs: Tensor, temp: f64, tau: f64) -> i64 {
- debug_assert!(probs.dim() == 1, "batch support not implemented");
- let mut probs = probs;
-
- let logits = -probs.log();
- let ent = (&logits * &probs).nansum(-1, true, Kind::Float);
- let shifted_logits = (&logits - ent).abs();
- let sorted_ids = shifted_logits.argsort(-1, true);
- let sorted_logits = shifted_logits.index_select(-1, &sorted_ids);
- let sorted_probs = probs.index_select(-1, &sorted_ids);
- let cumulative_probs = sorted_probs.cumsum(-1, Kind::Float);
- let cutoff = cumulative_probs.lt(tau).sum(Kind::Int64).int64_value(&[]);
-
- probs = probs.index_fill_(-1, &sorted_logits.gt(cutoff).to_kind(Kind::Int64), 0.0);
-
- if temp != 1.0 {
- let t = Tensor::from_slice(&[1.0 / temp]);
- probs = probs.pow(&t);
- }
-
- probs.multinomial(1, true).int64_value(&[0])
-}
+use tokio_stream::StreamExt;
pub fn sample_nucleus(probs: Tensor, temp: f64, top_p: f64) -> i64 {
debug_assert!(probs.dim() == 1, "batch support not implemented");
@@ -71,3 +53,143 @@ impl Clone for ClonableTensor {
ClonableTensor(self.0.clone(&out))
}
}
+
+pub struct OpenAiApi {
+ api: String,
+ top_p: f64,
+ temp: f64,
+ model: String,
+ max_tokens: Option<u64>,
+ stop: Vec<String>,
+}
+
+impl OpenAiApi {
+ pub fn builder(api: String, model: String) -> OpenAiApiBuilder {
+ OpenAiApiBuilder::new(api, model)
+ }
+
+ fn payload(&self, prompt: &str) -> Result<serde_json::Value> {
+ let mut payload = serde_json::json!({
+ "prompt": prompt,
+ "temperature": self.temp,
+ "top_p": self.top_p,
+ "stop": self.stop.clone(),
+ "model": self.model.clone(),
+ });
+
+ if let Some(max_tokens) = self.max_tokens {
+ payload["max_tokens"] = serde_json::json!(max_tokens);
+ }
+
+ Ok(payload)
+ }
+
+ pub async fn generate(&self, prompt: &str) -> Result<String> {
+ let client = reqwest::Client::new();
+ let res = client
+ .post(format!("{}/v1/completions", &self.api))
+ .json(&self.payload(prompt)?)
+ .send()
+ .await?;
+
+ let res = res.text().await?;
+
+ let res: serde_json::Value = serde_json::from_str(&res)?;
+
+ let res = res
+ .get("choices")
+ .ok_or(anyhow!("unexepected response format"))?
+ .get(0)
+ .ok_or(anyhow!("unexpected response format"))?
+ .get("text")
+ .ok_or(anyhow!("unexpected response format"))?
+ .as_str()
+ .ok_or(anyhow!("unexpected response format"))?;
+
+ Ok(String::from(res))
+ }
+
+ pub async fn stream(&self, prompt: &str) -> Result<impl Stream<Item = Result<String>>> {
+ let client = reqwest::Client::new();
+
+ let mut payload = self.payload(prompt)?;
+ payload["stream"] = serde_json::json!(true);
+
+ Ok(client
+ .post(format!("{}/v1/completions", &self.api))
+ .json(&payload)
+ .send()
+ .await?
+ .bytes_stream()
+ .eventsource()
+ .map(|event| {
+ let event = event?;
+ let data: serde_json::Value = serde_json::from_str(&event.data)?;
+
+ let text = data
+ .get("choices")
+ .ok_or(anyhow!("unexpected response format"))?
+ .get(0)
+ .ok_or(anyhow!("unexpected response format"))?
+ .get("text")
+ .ok_or(anyhow!("unexpected response format"))?
+ .as_str()
+ .ok_or(anyhow!("unexpected response format"))?;
+
+ Ok(text.to_string())
+ }))
+ }
+}
+
+pub struct OpenAiApiBuilder {
+ api: String,
+ model: String,
+ top_p: f64,
+ temp: f64,
+ max_tokens: Option<u64>,
+ stop: Vec<String>,
+}
+
+impl OpenAiApiBuilder {
+ pub fn new(api: String, model: String) -> Self {
+ Self {
+ api,
+ model,
+ top_p: 0.9,
+ temp: 1.0,
+ max_tokens: None,
+ stop: vec![],
+ }
+ }
+
+ pub fn top_p(mut self, top_p: f64) -> Self {
+ self.top_p = top_p;
+ self
+ }
+
+ pub fn temp(mut self, temp: f64) -> Self {
+ self.temp = temp;
+ self
+ }
+
+ pub fn max_tokens(mut self, max_tokens: u64) -> Self {
+ self.max_tokens = Some(max_tokens);
+ self
+ }
+
+ pub fn stop(mut self, stop: Vec<&str>) -> Self {
+ self.stop = stop.iter().map(|s| s.to_string()).collect();
+ self
+ }
+
+ pub fn build(self) -> OpenAiApi {
+ OpenAiApi {
+ api: self.api,
+ top_p: self.top_p,
+ temp: self.temp,
+ max_tokens: self.max_tokens,
+ stop: self.stop,
+ model: self.model,
+ }
+ }
+}
diff --git a/core/src/summarizer.rs b/core/src/summarizer.rs
index 16bd3f89..df82bfec 100644
--- a/core/src/summarizer.rs
+++ b/core/src/summarizer.rs
@@ -14,39 +14,21 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use futures::stream::Stream;
use std::{
cmp::Reverse,
collections::{BinaryHeap, VecDeque},
ops::Range,
path::Path,
- sync::Arc,
};
+use tokio_stream::StreamExt;
+use crate::{llm_utils::OpenAiApi, Result};
use itertools::{intersperse, Itertools};
-use tch::{IValue, Kind, Tensor};
+use tch::Tensor;
use tokenizers::{PaddingParams, TruncationParams};
-use crate::{
- ceil_char_boundary,
- llm_utils::{self, ClonableTensor},
-};
-
-#[derive(thiserror::Error, Debug)]
-pub enum Error {
- #[error("Torch")]
- Tch(#[from] tch::TchError),
-
- #[error("Tokenizer")]
- Tokenizer(#[from] tokenizers::Error),
-
- #[error("IO")]
- Io(#[from] std::io::Error),
-
- #[error("Unexpected output type")]
- UnexpectedOutputType,
-}
-
-type Result<T> = std::result::Result<T, Error>;
+use crate::ceil_char_boundary;
#[derive(Clone)]
struct CandidatePassage<'a> {
@@ -186,8 +168,8 @@ impl ExtractiveSummarizer {
Ok(Self {
passage_scorer: DualEncoder::open(path)?,
top_n_passages,
- window_size: 100,
- overlap: 10,
+ window_size: 64,
+ overlap: 0,
})
}
@@ -274,34 +256,29 @@ pub struct Summarizer {
}
impl Summarizer {
- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ pub fn new<P: AsRef<Path>>(
+ path: P,
+ openai_api_uri: String,
+ model_name: String,
+ ) -> Result<Self> {
Ok(Self {
extractive: ExtractiveSummarizer::open(
path.as_ref().join("dual_encoder").as_path(),
- 50,
+ 16,
)?,
- abstractive: AbstractiveSummarizer {
- model: Arc::new(AbstractiveModel::open(
- path.as_ref().join("abstractive").as_path(),
- )?),
- },
+ abstractive: AbstractiveSummarizer::new(openai_api_uri, model_name),
})
}
- pub fn summarize(&self, query: &str, text: &str) -> String {
- let summary = self.extractive.summarize(query, text);
- match self.abstractive.summarize(summary.as_str()) {
- Ok(stream) => stream.collect(),
- Err(err) => {
- tracing::error!("Abstractive summarization failed: {}", err);
- summary
- }
- }
- }
+ pub async fn summarize(&self, query: &str, text: &str) -> Result<impl Stream<Item = String>> {
+ let words = text.split_whitespace().count();
- pub fn summarize_iter(&self, query: &str, text: &str) -> Result<impl Iterator<Item = String>> {
- let summary = self.extractive.summarize(query, text);
- self.abstractive.summarize(&summary)
+ if words < 2000 {
+ self.abstractive.summarize(text).await
+ } else {
+ let summary = self.extractive.summarize(query, text);
+ self.abstractive.summarize(&summary).await
+ }
}
}
@@ -322,9 +299,9 @@ impl DualEncoder {
};
let mut tokenizer =
- tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;
+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json")).unwrap();
- tokenizer.with_truncation(Some(truncation))?;
+ tokenizer.with_truncation(Some(truncation)).unwrap();
tokenizer.with_padding(Some(padding));
let model = tch::CModule::load(folder.as_ref().join("model.pt"))?;
@@ -380,376 +357,90 @@ impl PassageScorer for DualEncoder {
}
}
-const TRUNCATE_INPUT_ABSTRACTIVE: usize = 1024;
-pub struct AbstractiveModel {
- encoder: tch::CModule,
- decoder: tch::CModule,
- decoder_with_past: tch::CModule,
- tokenizer: tokenizers::Tokenizer,
-
- bos_token_id: usize,
- eos_token_id: usize,
- begin_decoder_token: usize,
-}
-
-impl AbstractiveModel {
- pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {
- let truncation = TruncationParams {
- max_length: TRUNCATE_INPUT_ABSTRACTIVE,
- ..Default::default()
- };
-
- let padding = PaddingParams {
- ..Default::default()
- };
-
- let mut tokenizer =
- tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json"))?;
-
- tokenizer.with_truncation(Some(truncation))?;
- tokenizer.with_padding(Some(padding));
-
- let encoder = tch::CModule::load(folder.as_ref().join("traced_encoder.pt"))?;
- let decoder = tch::CModule::load(folder.as_ref().join("traced_decoder.pt"))?;
- let decoder_with_past = tch::CModule::load(folder.as_ref().join("traced_decoder_wp.pt"))?;
-
- Ok(Self {
- tokenizer,
- encoder,
- decoder,
- decoder_with_past,
-
- bos_token_id: 0,
- eos_token_id: 2,
- begin_decoder_token: 2,
- })
- }
-
- fn parse_decoder_output(
- &self,
- output: IValue,
- ) -> std::result::Result<(Tensor, Vec<Vec<ClonableTensor>>), Error> {
- let mut output = if let IValue::Tuple(tup) = output {
- Ok(tup)
- } else {
- Err(Error::UnexpectedOutputType)
- }?;
-
- if output.len() != 2 {
- return Err(Error::UnexpectedOutputType);
- }
-
- let logits = if let IValue::Tensor(t) = output.remove(0) {
- Ok(t)
- } else {
- Err(Error::UnexpectedOutputType)
- }?;
-
- let caches = if let IValue::Tuple(caches) = output.remove(0) {
- Ok(caches)
- } else {
- Err(Error::UnexpectedOutputType)
- }?;
-
- let past_key_values = if caches.len() == 12 {
- let mut new_caches = Vec::with_capacity(caches.len());
-
- for cache in caches {
- if let IValue::Tuple(cache) = cache {
- let mut c = Vec::with_capacity(2);
- for cache in cache.into_iter().take(2) {
- if let IValue::Tensor(cache) = cache {
- c.push(ClonableTensor(cache));
- } else {
- return Err(Error::UnexpectedOutputType);
- }
- }
-
- new_caches.push(c);
- } else {
- return Err(Error::UnexpectedOutputType);
- }
- }
-
- Ok(new_caches)
- } else {
- Err(Error::UnexpectedOutputType)
- }?;
-
- Ok((logits, past_key_values))
- }
-}
+const TRUNCATE_WORDS_ABSTRACTIVE: usize = 1024;
pub struct AbstractiveSummarizer {
- model: Arc<AbstractiveModel>,
+ api: String,
+ model: String,
}
impl AbstractiveSummarizer {
- pub fn new(model: AbstractiveModel) -> Self {
- Self {
- model: Arc::new(model),
- }
- }
-
- pub fn summarize(&self, text: &str) -> Result<impl Iterator<Item = String>> {
- self.summarize_with_max(text, Some(128))
- }
-
- pub fn summarize_with_max(
- &self,
- text: &str,
- max_summary_tokens: Option<usize>,
- ) -> Result<impl Iterator<Item = String>> {
- let ids = self
- .model
- .tokenizer
- .encode(text, false)
- .unwrap()
- .get_ids()
- .iter()
- .map(|id| *id as i64)
- .collect_vec();
-
- let ids = Tensor::from_slice(&ids).reshape([1, -1]);
-
- let decoder_tokens = Tensor::from_slice(&[
- self.model.begin_decoder_token as i64,
- self.model.bos_token_id as i64,
- ])
- .reshape([1, -1]);
-
- self.generate(&ids, &decoder_tokens, max_summary_tokens)
+ pub fn new(api: String, model: String) -> Self {
+ Self { api, model }
}
- fn generate(
- &self,
- encoder_ids: &Tensor,
- decoder_start_tokens: &Tensor,
- max_new_tokens: Option<usize>,
- ) -> Result<impl Iterator<Item = String>> {
- let tau = 0.8;
- let temp = 1.0;
-
- let encoded = self.model.initial(encoder_ids, decoder_start_tokens)?;
-
- let token_it = TokenStreamingGenerator {
- model: Arc::clone(&self.model),
- tau,
- temp,
- banned_tokens: vec![self.model.bos_token_id as i64],
- end_tokens: vec![self.model.eos_token_id as i64],
- next_token_logits: Some(encoded.next_token_logits),
- memory: encoded.memory,
- num_tokens_generated: 0,
- max_new_tokens,
- };
+ fn client(&self, max_tokens: Option<u64>) -> OpenAiApi {
+ let mut builder = OpenAiApi::builder(self.api.clone(), self.model.clone())
+ .top_p(0.9)
+ .temp(0.0)
+ .stop(vec!["</s>", "<|endoftext|>"]);
- let it = StringStreamingGenerator {
- token_streamer: token_it,
- tokens: Vec::new(),
- };
+ if let Some(max_tokens) = max_tokens {
+ builder = builder.max_tokens(max_tokens);
+ }
- Ok(it)
+ builder.build()
}
-}
-#[derive(Clone)]
-pub struct BartMemory {
- encoder_hidden_states: ClonableTensor,
- past_key_values: Vec<Vec<ClonableTensor>>,
-}
+ fn first_prompt(&self, text: &str) -> String {
+ let wc = 100;
+ format!(
+ r##"[INST] I will provide you with piece of content (e.g. articles, papers, documentation, etc.)
-impl AbstractiveModel {
- fn initial(
- &self,
- encoder_ids: &Tensor,
- decoder_ids: &Tensor,
- ) -> std::result::Result<DecoderOutput, Error> {
- let encoder_hidden_states = self.encoder.forward_ts(&[encoder_ids])?;
-
- let decoder_output = self.decoder.forward_is(&[
- IValue::Tensor(decoder_ids.shallow_clone()),
- IValue::Tensor(encoder_hidden_states.shallow_clone()),
- ])?;
-
- let (logits, past_key_values) = self.parse_decoder_output(decoder_output)?;
-
- // logits is [batch_size, seq_len, vocab_size]
- // get last token logits
- let next_token_logits = logits.select(1, logits.size()[1] - 1);
-
- Ok(DecoderOutput {
- next_token_logits,
- memory: BartMemory {
- encoder_hidden_states: ClonableTensor(encoder_hidden_states),
- past_key_values,
- },
- })
- }
+You will generate summaries of the content. Refer to the content using words as "this article discusses" etc.
+The summaries should be -{wc} words.
- fn step(
- &self,
- last_token: i64,
- memory: &BartMemory,
- ) -> std::result::Result<DecoderOutput, Error> {
- let ids = Tensor::from_slice(&[last_token]).reshape([1, 1]);
-
- let past_key_value: IValue = IValue::Tuple(
- memory
- .past_key_values
- .iter()
- .map(|c| {
- IValue::Tuple(
- c.iter()
- .map(|c| IValue::Tensor(c.0.shallow_clone()))
- .collect_vec(),
- )
- })
- .collect_vec(),
- );
+After you have generated the summary, identify 1-3 informative entities from the content which are missing from the summary that can be used to make a more concise summary.
- let decoder_output = self.decoder_with_past.forward_is(&[
- IValue::Tensor(ids),
- IValue::Tensor(memory.encoder_hidden_states.clone().0),
- past_key_value,
- ])?;
+A Missing Entity is:
- let (logits, past_key_values) = self.parse_decoder_output(decoder_output)?;
+Relevant: to the main story.
+Specific: descriptive yet concise (5 words or fewer).
+Novel: not in the previous summary.
+Faithful: present in the content piece.
+Anywhere: located anywhere in the Article.
- // logits is [batch_size, seq_len, vocab_size]
- // get last token logits
- let next_token_logits = logits.select(1, logits.size()[1] - 1).squeeze();
+Finaly, give guidance on how to improve the summary.
- Ok(DecoderOutput {
- next_token_logits,
- memory: BartMemory {
- encoder_hidden_states: memory.encoder_hidden_states.clone(),
- past_key_values,
- },
- })
+Content to summarize:
+{text}
+[/INST]"##,
+ )
}
-}
-
-pub struct TokenStreamingGenerator {
- model: Arc<AbstractiveModel>,
- tau: f64,
- temp: f64,
- banned_tokens: Vec<i64>,
- end_tokens: Vec<i64>,
- next_token_logits: Option<Tensor>,
- memory: BartMemory,
- num_tokens_generated: usize,
- max_new_tokens: Option<usize>,
-}
-
-impl Iterator for TokenStreamingGenerator {
- type Item = i64;
-
- fn next(&mut self) -> Option<Self::Item> {
- match &self.next_token_logits {
- Some(next_token_logits) => {
- if let Some(max_new_tokens) = self.max_new_tokens {
- if self.num_tokens_generated >= max_new_tokens {
- self.next_token_logits = None;
- return None;
- }
- }
-
- let mut probs = next_token_logits.softmax(-1, Kind::Float).squeeze();
- // remove banned tokens
- for token in &self.banned_tokens {
- let _ = probs.index_fill_(0, &Tensor::from_slice(&[*token]), 0.0);
- }
-
- let next_token = llm_utils::sample_typical(probs, self.temp, self.tau);
-
- if self.end_tokens.contains(&next_token) {
- self.next_token_logits = None;
- return None;
- }
+ fn final_prompt(&self, text: &str, prev_summary: &str) -> String {
+ let wc = 100;
+ format!(
+ r##"[INST] I will provide you with piece of content (e.g. articles, papers, documentation, etc.) and a summary of the content.
- let encoded = self.model.step(next_token, &self.memory).ok()?;
+You will improve upon the previous summary to generate a highly consice summary of the content. The summary should be -{wc} words.
- self.next_token_logits = Some(encoded.next_token_logits);
- self.memory = encoded.memory;
+Content to summarize:
+{text}
- self.num_tokens_generated += 1;
-
- Some(next_token)
- }
- None => None,
- }
+Previous summary:
+{prev_summary}
+[/INST] Summary:"##,
+ )
}
-}
-pub struct StringStreamingGenerator {
- token_streamer: TokenStreamingGenerator,
- tokens: Vec<u32>,
-}
+ pub async fn summarize(&self, text: &str) -> Result<impl Stream<Item = String>> {
+ let text = text
+ .split_ascii_whitespace()
+ .take(TRUNCATE_WORDS_ABSTRACTIVE)
+ .join(" ");
-impl Iterator for StringStreamingGenerator {
- type Item = String;
+ let prompt = self.first_prompt(&text);
+ let first_summary = self.client(Some(2048)).generate(&prompt).await?;
- fn next(&mut self) -> Option<Self::Item> {
- loop {
- match self.token_streamer.next() {
- Some(token) => {
- self.tokens.push(token as u32);
-
- if let Some(s) = self
- .token_streamer
- .model
- .tokenizer
- .decode(&self.tokens, true)
- .ok()
- .and_then(|s| {
- if !s.contains('\u{fffd}') {
- // valid utf-8 string
- Some(s)
- } else {
- None
- }
- })
- {
- self.tokens.clear();
- return Some(s);
- }
- }
- None => {
- if self.tokens.is_empty() {
- return None;
- } else {
- match self
- .token_streamer
- .model
- .tokenizer
- .decode(&self.tokens, true)
- {
- Ok(s) => {
- self.tokens.clear();
-
- if !s.contains('\u{fffd}') {
- return Some(s);
- } else {
- return None;
- }
- }
- Err(_) => {
- self.tokens.clear();
- return None;
- }
- }
- }
- }
- }
- }
- }
-}
+ let prompt = self.final_prompt(&text, &first_summary);
-struct DecoderOutput {
- next_token_logits: Tensor,
- memory: BartMemory,
+ Ok(self
+ .client(Some(128))
+ .stream(&prompt)
+ .await?
+ .filter_map(|tok| tok.ok()))
+ }
}
#[cfg(test)]
@@ -796,30 +487,4 @@ mod tests {
assert_eq!(p, &text[range]);
}
}
-
- #[test]
- fn abstractive_summary() {
- let summarizer = AbstractiveSummarizer {
- model: Arc::new(
- AbstractiveModel::open("../data/summarizer/abstractive")
- .expect("abstractive summary model not found"),
- ),
- };
-
- let text = r#"Aristotle (/ˈærɪstɒtəl/;[1] Greek: Ἀριστοτέλης Aristotélēs, pronounced [aristotélɛːs]; 384–322 BC) was an Ancient Greek philosopher and polymath. His writings cover a broad range of subjects including physics, biology, zoology, metaphysics, logic, ethics, aesthetics, poetry, drama, music, rhetoric, psychology, linguistics, economics, politics, meteorology, geology, and government. As the founder of the Peripatetic school of philosophy in the Lyceum in Athens, he began the wider Aristotelian tradition that followed, which set the groundwork for the development of modern science.
- Little is known about Aristotle's life. He was born in the city of Stagira in Northern Greece during the Classical period. His father, Nicomachus, died when Aristotle was a child, and he was brought up by a guardian. At seventeen or eighteen years of age he joined Plato's Academy in Athens and remained there until the age of thirty-seven (c. 347 BC). Shortly after Plato died, Aristotle left Athens and, at the request of Philip II of Macedon, tutored his son Alexander the Great beginning in 343 BC. He established a library in the Lyceum which helped him to produce many of his hundreds of books on papyrus scrolls.
- Though Aristotle wrote many elegant treatises and dialogues for publication, only around a third of his original output has survived, none of it intended for publication. Aristotle provided a complex synthesis of the various philosophies existing prior to him. It was above all from his teachings that the West inherited its intellectual lexicon, as well as problems and methods of inquiry. As a result, his philosophy has exerted a unique influence on almost every form of knowledge in the West and it continues to be a subject of contemporary philosophical discussion.
- Aristotle's views profoundly shaped medieval scholarship. The influence of physical science extended from Late Antiquity and the Early Middle Ages into the Renaissance, and were not replaced systematically until the Enlightenment and theories such as classical mechanics were developed. Some of Aristotle's zoological observations found in his biology, such as on the hectocotyl (reproductive) arm of the octopus, were disbelieved until the 19th century. He also influenced Judeo-Islamic philosophies during the Middle Ages, as well as Christian theology, especially the Neoplatonism of the Early Church and the scholastic tradition of the Catholic Church. Aristotle was revered among medieval Muslim scholars as "The First Teacher", and among medieval Christians like Thomas Aquinas as simply "The Philosopher", while the poet Dante called him "the master of those who know". His works contain the earliest known formal study of logic, and were studied by medieval scholars such as Peter Abelard and John Buridan. Aristotle's influence on logic continued well into the 19th century. In addition, his ethics, though always influential, gained renewed interest with the modern advent of virtue ethics."#;
-
- let start = std::time::Instant::now();
- let summary = summarizer
- .summarize_with_max(text, Some(16))
- .unwrap()
- .collect::<String>();
-
- println!("Elapsed: {:?}", start.elapsed());
- println!("{:?}", &summary);
-
- assert!(summary.len() > 50);
- }
}
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index dd1729a5..3d7436bf 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -317,7 +317,7 @@ impl EdgeStore {
block_options.set_ribbon_filter(10.0);
options.set_block_based_table_factory(&block_options);
- options.set_compression_type(rocksdb::DBCompressionType::Lz4);
+ options.set_compression_type(rocksdb::DBCompressionType::None);
options.optimize_for_point_lookup(512);
let ranges = match rocksdb::DB::open_cf_with_opts(
diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts
index 35d5c31e..730be1f2 100644
--- a/frontend/src/lib/stores.ts
+++ b/frontend/src/lib/stores.ts
@@ -86,7 +86,7 @@ if (browser)
document.documentElement.className = `${c} ${theme}`.trim();
});
-type SummaryState = { inProgress: boolean; data: string } | undefined;
+type SummaryState = { inProgress: boolean; tokens: string[]} | undefined;
export const summariesStore = writable<Record<string, SummaryState>>({});
// Actions
@@ -99,7 +99,7 @@ export const summarize = (query: string, site: DisplayedWebpage) => {
}));
};
- updateSummary(() => ({ inProgress: true, data: '' }));
+ updateSummary(() => ({ inProgress: true, tokens: [] }));
const { listen, cancel } = api.summarize({ query, url: site.url });
@@ -108,14 +108,14 @@ export const summarize = (query: string, site: DisplayedWebpage) => {
.with({ type: 'message' }, ({ data }) =>
updateSummary((summary) => ({
inProgress: true,
- data: (summary?.data ?? '') + data,
+ tokens: [...(summary?.tokens ?? []), data]
})),
)
.with({ type: 'error' }, () => {
cancel();
updateSummary((summary) => ({
inProgress: false,
- data: summary?.data ?? '',
+ tokens: [...(summary?.tokens ?? [])]
}));
})
.exhaustive();
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte
index 7ecf34f3..f1b8ebc3 100644
--- a/frontend/src/routes/search/+page.svelte
+++ b/frontend/src/routes/search/+page.svelte
@@ -1,20 +1,13 @@
<script lang="ts">
- import HandThumbDown from '~icons/heroicons/hand-thumb-down-20-solid';
- import HandThumbUp from '~icons/heroicons/hand-thumb-up-20-solid';
- import NoSymbol from '~icons/heroicons/no-symbol-20-solid';
import ChevronLeft from '~icons/heroicons/chevron-left-20-solid';
import ChevronRight from '~icons/heroicons/chevron-right-20-solid';
import OpticSelector from '$lib/components/OpticSelector.svelte';
import Searchbar from '$lib/components/Searchbar.svelte';
import type { PageData } from './$types';
- import { scale } from 'svelte/transition';
import RegionSelect from '$lib/components/RegionSelect.svelte';
import type { DisplayedWebpage } from '$lib/api';
import { onMount } from 'svelte';
- import { twJoin } from 'tailwind-merge';
- import Button from '$lib/components/Button.svelte';
- import { searchQueryStore, hostRankingsStore, summarize } from '$lib/stores';
- import type { Ranking } from '$lib/rankings';
+ import { searchQueryStore } from '$lib/stores';
import { flip } from 'svelte/animate';
import Result from './Result.svelte';
import Sidebar from './Sidebar.svelte';
@@ -24,6 +17,7 @@
import { page } from '$app/stores';
import { updateQueryId } from '$lib/improvements';
import { browser } from '$app/environment';
+ import Modal from './Modal.svelte';
export let data: PageData;
$: results = data.results;
@@ -55,7 +49,7 @@
// | x <--
// +---+
modal = {
- top: window.scrollY + rect.top + rect.height / 2,
+ top: window.scrollY + rect.top + rect.height/2,
left: window.scrollX + rect.right,
site,
};
@@ -77,68 +71,10 @@
if (browser && results.type == 'websites') updateQueryId({ query, webpages: results.webpages });
}
- const rankingChoices = [
- {
- ranking: 'liked',
- kind: 'success',
- Icon: HandThumbUp,
- },
- {
- ranking: 'disliked',
- kind: 'warning',
- Icon: HandThumbDown,
- },
- {
- ranking: 'blocked',
- kind: 'error',
- Icon: NoSymbol,
- },
- ] as const;
-
- const rankSite = (site: DisplayedWebpage, ranking: Ranking) => () => {
- hostRankingsStore?.update(($rankings) => ({
- ...$rankings,
- [site.site]: $rankings[site.site] == ranking ? void 0 : ranking,
- }));
- };
-
- const summarizeSite = (site: DisplayedWebpage) => () => summarize(query, site);
</script>
{#if modal}
- <!-- svelte-ignore a11y-click-events-have-key-events -->
- <!-- svelte-ignore a11y-no-static-element-interactions -->
- <div
- class={twJoin(
- 'absolute -translate-y-1/2 transition-all',
- 'h-fit w-52 flex-col items-center overflow-hidden rounded-lg border bg-base-100 px-2 py-5 text-sm drop-shadow-md',
- )}
- style="top: {modal.top}px; left: calc({modal.left}px + 1rem)"
- transition:scale={{ duration: 150 }}
- on:click|stopPropagation={() => {}}
- >
- <div>
- <h2 class="w-fit text-center">
- Do you like results from {modal.site.site}?
- </h2>
- <div class="flex justify-center space-x-1.5 pt-2">
- {#each rankingChoices as { ranking, kind, Icon }}
- <Button
- {kind}
- pale={$hostRankingsStore[modal.site.site] != ranking}
- padding={false}
- form="searchbar-form"
- on:click={rankSite(modal.site, ranking)}
- >
- <Icon class="w-4" />
- </Button>
- {/each}
- </div>
- <div class="mt-4 flex justify-center">
- <Button pale on:click={summarizeSite(modal.site)}>Summarize Result</Button>
- </div>
- </div>
- </div>
+ <Modal {query} {modal} />
{/if}
{#if results.type == 'websites'}
diff --git a/frontend/src/routes/search/+page.ts b/frontend/src/routes/search/+page.ts
index 721565ad..4469dd15 100644
--- a/frontend/src/routes/search/+page.ts
+++ b/frontend/src/routes/search/+page.ts
@@ -13,18 +13,18 @@ export const load: PageLoad = async (req) => {
if (form) {
params = form;
} else {
- redirect(300, '/');
+ redirect(301, '/');
}
}
if (!params.query.trim()) {
- redirect(300, '/');
+ redirect(301, '/');
}
const results = req.data['results'];
if (results.type == 'bang') {
- redirect(300, results.redirectTo);
+ redirect(301, results.redirectTo);
}
const prevPageSearchParams = match(params.currentPage > 1)
diff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelte
new file mode 100644
index 00000000..62ac4672
--- /dev/null
+++ b/frontend/src/routes/search/Modal.svelte
@@ -0,0 +1,95 @@
+<script lang="ts">
+ import type { Ranking } from '$lib/rankings';
+ import { hostRankingsStore, summarize } from '$lib/stores';
+ import type { DisplayedWebpage } from '$lib/api';
+ import { twJoin } from 'tailwind-merge';
+ import Button from '$lib/components/Button.svelte';
+ import HandThumbDown from '~icons/heroicons/hand-thumb-down-20-solid';
+ import HandThumbUp from '~icons/heroicons/hand-thumb-up-20-solid';
+ import NoSymbol from '~icons/heroicons/no-symbol-20-solid';
+ import { scale } from 'svelte/transition';
+
+ export let query: string;
+ export let modal: { top: number; left: number; site: DisplayedWebpage };
+
+ const mediumWidthCutoff = 768;
+
+ const widthPixels = 208;
+
+ let innerWidth = 0;
+
+ let left = modal.left;
+ let top = modal.top + 32;
+
+ $: {
+ if (innerWidth < mediumWidthCutoff) {
+ left = modal.left - (widthPixels + 16);
+ } else {
+ left = modal.left + 16;
+ }
+ }
+
+ const rankingChoices = [
+ {
+ ranking: 'liked',
+ kind: 'success',
+ Icon: HandThumbUp,
+ },
+ {
+ ranking: 'disliked',
+ kind: 'warning',
+ Icon: HandThumbDown,
+ },
+ {
+ ranking: 'blocked',
+ kind: 'error',
+ Icon: NoSymbol,
+ },
+ ] as const;
+
+ const rankSite = (site: DisplayedWebpage, ranking: Ranking) => () => {
+ hostRankingsStore?.update(($rankings) => ({
+ ...$rankings,
+ [site.site]: $rankings[site.site] == ranking ? void 0 : ranking,
+ }));
+ };
+
+ const summarizeSite = (site: DisplayedWebpage) => () => summarize(query, site);
+</script>
+
+<svelte:window bind:innerWidth />
+
+
+<!-- svelte-ignore a11y-click-events-have-key-events -->
+<!-- svelte-ignore a11y-no-static-element-interactions -->
+<div
+class={twJoin(
+ 'absolute -translate-y-1/2 transition-all',
+ 'h-fit flex-col items-center overflow-hidden rounded-lg border bg-base-100 px-2 py-5 text-sm drop-shadow-md',
+)}
+style="top: {top}px; left: calc({left}px); width: {widthPixels}px;"
+transition:scale={{ duration: 150 }}
+on:click|stopPropagation={() => {}}
+>
+ <div>
+ <h2 class="w-fit text-center">
+ Do you like results from {modal.site.site}?
+ </h2>
+ <div class="flex justify-center space-x-1.5 pt-2">
+ {#each rankingChoices as { ranking, kind, Icon }}
+ <Button
+ {kind}
+ pale={$hostRankingsStore[modal.site.site] != ranking}
+ padding={false}
+ form="searchbar-form"
+ on:click={rankSite(modal.site, ranking)}
+ >
+ <Icon class="w-4" />
+ </Button>
+ {/each}
+ </div>
+ <div class="mt-4 flex justify-center">
+ <Button pale on:click={summarizeSite(modal.site)}>Summarize Result</Button>
+ </div>
+ </div>
+</div>
\ No newline at end of file
diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelte
index 3cfdb326..e6fe395d 100644
--- a/frontend/src/routes/search/Result.svelte
+++ b/frontend/src/routes/search/Result.svelte
@@ -41,7 +41,7 @@
</a>
</div>
<button
- class="noscript:hidden hidden w-5 min-w-fit items-center justify-center bg-transparent text-neutral hover:cursor-pointer hover:text-neutral-focus md:flex"
+ class="noscript:hidden flex w-5 min-w-fit items-center justify-center bg-transparent text-neutral hover:cursor-pointer hover:text-neutral-focus"
bind:this={button}
on:click|stopPropagation={() => dispatch('modal', button)}
>
@@ -50,7 +50,7 @@
</div>
<div class="text-sm font-normal text-neutral-focus [&>b]:font-bold">
{#if $summary}
- <Summary {...$summary} on:hide={() => clearSummary(webpage)} />
+ <Summary url={webpage.url} on:hide={() => clearSummary(webpage)} />
{:else if webpage.snippet.type == 'normal'}
<div class="snippet">
<div class="line-clamp-3">
diff --git a/frontend/src/routes/search/Summary.svelte b/frontend/src/routes/search/Summary.svelte
index ae44dd43..eeea8b34 100644
--- a/frontend/src/routes/search/Summary.svelte
+++ b/frontend/src/routes/search/Summary.svelte
@@ -2,9 +2,12 @@
import XMark from '~icons/heroicons/x-mark-20-solid';
import Callout from '$lib/components/Callout.svelte';
import { createEventDispatcher } from 'svelte';
+ import { summariesStore } from '$lib/stores';
+ import { fade } from 'svelte/transition';
- export let inProgress: boolean;
- export let data: string;
+ export let url: string;
+
+ $: tokens = ($summariesStore[url]?.tokens ?? []);
const dispatch = createEventDispatcher<{ hide: null }>();
</script>
@@ -15,11 +18,10 @@
</button>
<p class="line-clamp-3">
- {#if data}
- {data}
- {#if inProgress}
- <span class="inline-block -translate-y-0.5 animate-blink font-thin">|</span>
- {/if}
+ {#if tokens.length > 0}
+ {#each tokens as tok}
+ <span transition:fade={{duration: 500}}>{tok}</span>
+ {/each}
{:else}
<span class="flex">
<span class="inline-block animate-bounce [animation-delay:000ms]">.</span>
diff --git a/justfile b/justfile
index d29ea7be..bc9d3fd2 100644
--- a/justfile
+++ b/justfile
@@ -12,6 +12,8 @@ export STRACT_CARGO_ARGS := env_var_or_default("STRACT_CARGO_ARGS", "")
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- search-server configs/search_server.toml"
@dev-webgraph:
cargo watch -i frontend -x "run $STRACT_CARGO_ARGS -- webgraph server configs/webgraph/server.toml"
+@dev-llm:
+ ./.venv/bin/python3 -m llama_cpp.server --model data/mistral-7b-instruct-v0.2.Q4_K_M.gguf --port 4000
@dev-frontend:
cd frontend && npm run dev
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index b834663d..eabeb7e7 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -4,4 +4,6 @@ torch>=2.0.0
datasets
gensim
safetensors
-pandas
\ No newline at end of file
+pandas
+llama-cpp-python[server]
+llama-cpp-python
\ No newline at end of file
diff --git a/scripts/run_dev.py b/scripts/run_dev.py
index 42813287..24727eb9 100755
--- a/scripts/run_dev.py
+++ b/scripts/run_dev.py
@@ -22,6 +22,7 @@ processes.append(subprocess.Popen(["just", "dev-api"]))
processes.append(subprocess.Popen(["just", "dev-search-server"]))
processes.append(subprocess.Popen(["just", "dev-webgraph"]))
processes.append(subprocess.Popen(["just", "dev-frontend"]))
+processes.append(subprocess.Popen(["just", "dev-llm"]))
# kill processes on ctrl-c
import time