Skip to content

web search done right

alice improvement endpoint

Mikkel Denker25f4b282eb71
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rs
index 776d3332..c0f3d844 100644
--- a/core/src/api/improvement.rs
+++ b/core/src/api/improvement.rs
@@ -21,7 +21,7 @@ use axum::{extract, response::IntoResponse};
use serde::Deserialize;
use uuid::Uuid;
-use crate::query_store::{ImprovementEvent, StoredQuery};
+use crate::improvement::{AliceConversation, ImprovementEvent, StoredQuery};
use super::{HtmlTemplate, State};
@@ -76,6 +76,21 @@ pub async fn store(
}
}
+#[allow(clippy::unused_async)]
+pub async fn new_chat_id() -> impl IntoResponse {
+ Uuid::new_v4().to_string()
+}
+
+#[allow(clippy::unused_async)]
+pub async fn store_chat(
+ extract::State(state): extract::State<Arc<State>>,
+ extract::Json(params): extract::Json<AliceConversation>,
+) {
+ if let Some(q) = state.improvement_queue.as_ref() {
+ q.lock().await.push(ImprovementEvent::Chat { chat: params })
+ }
+}
+
#[allow(clippy::unused_async)]
pub async fn settings() -> impl IntoResponse {
let template = SettingsTemplate {};
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 5b7fb696..e0814e92 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -25,9 +25,9 @@ use crate::{
cluster::Cluster,
member::{Member, Service},
},
+ improvement::{store_improvements_loop, ImprovementEvent},
leaky_queue::LeakyQueue,
qa_model::QaModel,
- query_store::{self, ImprovementEvent},
ranking::models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},
searcher::frontend::FrontendSearcher,
summarizer::Summarizer,
@@ -51,7 +51,7 @@ mod alice;
mod autosuggest;
mod chat;
mod explore;
-mod improvement;
+pub mod improvement;
mod index;
mod metrics;
mod opensearch;
@@ -121,10 +121,7 @@ pub async fn router(
let query_store_queue = config.query_store_db_host.clone().map(|db_host| {
let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));
- tokio::spawn(query_store::store_queries_loop(
- query_store_queue.clone(),
- db_host,
- ));
+ tokio::spawn(store_improvements_loop(query_store_queue.clone(), db_host));
query_store_queue
});
@@ -179,6 +176,14 @@ pub async fn router(
.route("/opensearch.xml", get(opensearch::route))
.route("/improvement/click", post(improvement::click))
.route("/improvement/store", post(improvement::store))
+ .route(
+ "/improvement/alice/new_chat_id",
+ post(improvement::new_chat_id),
+ )
+ .route(
+ "/improvement/alice/store_chat",
+ post(improvement::store_chat),
+ )
.fallback(get_service(ServeDir::new("frontend/dist/")))
.layer(CompressionLayer::new())
.merge(
diff --git a/core/src/query_store.rs b/core/src/improvement.rs
similarity index 77%
rename from core/src/query_store.rs
rename to core/src/improvement.rs
index 6f7bd0aa..98b49de3 100644
--- a/core/src/query_store.rs
+++ b/core/src/improvement.rs
@@ -43,10 +43,30 @@ pub struct StoredQuery {
timestamp: Option<DateTime<Utc>>, // it is extremely important that we strip minutes, seconds and nanoseconds here for privacy
}
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+pub struct AliceConversation {
+ pub id: Uuid,
+ messages: Vec<AliceMessage>,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+struct AliceMessage {
+ from: String,
+ message: String,
+ queries: Vec<AliceQuery>,
+}
+
+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]
+struct AliceQuery {
+ query: String,
+ results: Vec<crate::alice::SimplifiedWebsite>,
+}
+
#[derive(Clone)]
pub enum ImprovementEvent {
StoreQuery(StoredQuery),
Click { qid: Uuid, idx: usize },
+ Chat { chat: AliceConversation },
}
impl StoredQuery {
@@ -82,7 +102,7 @@ async fn dump_queue(queue: &Mutex<LeakyQueue<ImprovementEvent>>) -> Vec<Improvem
res
}
-pub async fn store_queries_loop(
+pub async fn store_improvements_loop(
queue: Arc<Mutex<LeakyQueue<ImprovementEvent>>>,
scylla_host: String,
) {
@@ -95,12 +115,13 @@ pub async fn store_queries_loop(
for event in events {
match event {
- ImprovementEvent::StoreQuery(query) => scylla.insert(query).await,
+ ImprovementEvent::StoreQuery(query) => scylla.store_query(query).await,
ImprovementEvent::Click { qid, idx } => {
if let Ok(idx) = idx.try_into() {
scylla.store_click(qid, idx).await
}
}
+ ImprovementEvent::Chat { chat } => scylla.store_chat(chat).await,
}
}
}
@@ -110,6 +131,7 @@ struct ScyllaConn {
session: scylla::Session,
prepared_insert: PreparedStatement,
prepared_click: PreparedStatement,
+ prepared_chat: PreparedStatement,
}
impl ScyllaConn {
@@ -123,6 +145,7 @@ impl ScyllaConn {
&[],
)
.await?;
+
session
.query(
"CREATE TABLE IF NOT EXISTS ks.clicks (qid uuid, click tinyint, primary key (qid))",
@@ -130,21 +153,34 @@ impl ScyllaConn {
)
.await?;
+ session
+ .query(
+ "CREATE TABLE IF NOT EXISTS ks.chats (id uuid, json_encoded text, primary key (qid))",
+ &[],
+ )
+ .await?;
+
let prepared_insert: PreparedStatement = session
.prepare("INSERT INTO ks.queries (qid, query, urls, timestamp) VALUES(?, ?, ?, ?)")
.await?;
+
let prepared_click: PreparedStatement = session
.prepare("INSERT INTO ks.clicks (qid, click) VALUES(?, ?)")
.await?;
+ let prepared_chat: PreparedStatement = session
+ .prepare("INSERT INTO ks.chats (id, json_encoded) VALUES(?, ?)")
+ .await?;
+
Ok(Self {
session,
prepared_insert,
prepared_click,
+ prepared_chat,
})
}
- async fn insert(&self, query: StoredQuery) {
+ async fn store_query(&self, query: StoredQuery) {
let urls = serde_json::to_string(&query.result_urls).unwrap();
let timestamp = query
.timestamp
@@ -177,4 +213,17 @@ impl ScyllaConn {
tracing::error!("scylla store_click error: {err}");
}
}
+
+ async fn store_chat(&self, chat: AliceConversation) {
+ if let Ok(json_encoded) = serde_json::to_string(&chat) {
+ let res = self
+ .session
+ .execute(&self.prepared_chat, (chat.id, json_encoded))
+ .await;
+
+ if let Err(err) = res {
+ tracing::error!("scylla store_chat error: {err}");
+ }
+ }
+ }
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 1945ac4e..2d3f665a 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -48,6 +48,7 @@ mod human_website_annotations;
mod hyperloglog;
mod image_downloader;
mod image_store;
+mod improvement;
pub mod index;
mod intmap;
mod kahan_sum;
@@ -58,7 +59,6 @@ mod metrics;
pub mod prehashed;
mod qa_model;
mod query;
-mod query_store;
pub mod ranking;
mod schema;
mod search_ctx;
diff --git a/core/src/main.rs b/core/src/main.rs
index 8e77de82..975a737c 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -88,6 +88,7 @@ enum Commands {
#[clap(long, takes_value = false)]
skip_download: bool,
+ #[clap(long, takes_value = false)]
alice: bool,
},
}
diff --git a/frontend/src/pages/chat.astro b/frontend/src/pages/chat.astro
index 32bb1a4b..25b5d186 100644
--- a/frontend/src/pages/chat.astro
+++ b/frontend/src/pages/chat.astro
@@ -58,6 +58,7 @@ import OpticSelector from "../components/OpticSelector.astro";
border-color: rgba(11, 123, 255, 0.3);
width: fit-content;
height: fit-content;
+ max-width: 100%;
border-width: 1px;
border-radius: 0.75rem;
border-bottom-right-radius: 0;
@@ -69,6 +70,7 @@ import OpticSelector from "../components/OpticSelector.astro";
border-color: rgba(0, 0, 0, 0.15);
width: fit-content;
height: fit-content;
+ max-width: 100%;
border-width: 1px;
border-radius: 0.75rem;
border-bottom-left-radius: 0;
@@ -95,17 +97,20 @@ import OpticSelector from "../components/OpticSelector.astro";
const statusTemplate: HTMLDivElement = document.querySelector(".status-template")!;
const opticSelector: HTMLSelectElement = document.querySelector("#optics-selector")!;
+ const allowStats = window.localStorage.getItem("allowStats") === "true";
+
// let prevState: number[][] | null = null;
let prevState: any | null = null;
+ let chatId: string | null = null;
let originalSendButtonColor = sendButton.style.color;
let numMessageLines = 1;
let messageInputHeight = 24;
- let messages: {from: string, message: string, queries: {query: string, results: {text: string, url: string, domain: string}[]}[]}[] = [];
+ let messages: {from: string, message: string, queries: {query: string, results: {text: string, url: string, title: string}[]}[]}[] = [];
let aliceResponding = false;
let aliceStreamingMessage = "";
- let currentQueries: {query: string, results: {text: string, url: string, domain: string}[]}[] = [];
+ let currentQueries: {query: string, results: {text: string, url: string, title: string}[]}[] = [];
function updateMessageInputHeight() {
const lines = messageInput.value.split("\n");
@@ -220,6 +225,8 @@ import OpticSelector from "../components/OpticSelector.astro";
messageInput.disabled = false;
sendButton.style.color = originalSendButtonColor;
messageInput.focus();
+
+ improvements();
}
async function sendMessageToAlice(message: string) {
@@ -256,7 +263,7 @@ import OpticSelector from "../components/OpticSelector.astro";
};
}
- function addMessage(from: string, message: string, queries: {query: string, results: {text: string, url: string, domain: string}[]}[] = []) {
+ function addMessage(from: string, message: string, queries: {query: string, results: {text: string, url: string, title: string}[]}[] = []) {
messages.push({
from: from,
message: message,
@@ -317,4 +324,48 @@ import OpticSelector from "../components/OpticSelector.astro";
codeBlock.style.background = "none";
}
}
+
+ // improvements
+ async function newChatId(): Promise<string> {
+ let url = "/improvement/alice/new_chat_id";
+
+ // post request using fetch api
+ let response = await fetch(url, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json'
+ }
+ });
+
+ return await response.text();
+ }
+
+ async function storeChat() {
+ if (chatId == null) {
+ chatId = await newChatId();
+ }
+
+ let chat = {
+ messages: messages,
+ id: chatId
+ };
+
+ let url = "/improvement/alice/store_chat";
+
+ await fetch(url, {
+ method: 'POST',
+ body: JSON.stringify(chat),
+ headers: {
+ 'Content-Type': 'application/json'
+ }
+ });
+ }
+
+ function improvements() {
+ if (allowStats) {
+ storeChat();
+ }
+ }
+
+
</script>
\ No newline at end of file
diff --git a/frontend/src/pages/settings/privacy.astro b/frontend/src/pages/settings/privacy.astro
index ef28eb1d..4ba7c67b 100644
--- a/frontend/src/pages/settings/privacy.astro
+++ b/frontend/src/pages/settings/privacy.astro
@@ -11,7 +11,7 @@ import SettingsLayout from "../../layouts/Settings.astro";
<div class="text-sm">
It is important to note, that <b class="font-extrabold">no</b> personal information is stored about your search. We don't store your IP, browser fingerprint or even a precise timestamp.
- In fact, you can see exactly what gets stored <a href="https://github.com/StractOrg/Stract/blob/main/core/src/query_store.rs#L36-#L50">here</a>.
+ In fact, you can see exactly what gets stored <a href="https://github.com/StractOrg/Stract/blob/main/core/src/improvement.rs#L36-#L70">here</a>.
</div>
<div class="text-sm">