web search done right
alice improvement endpoint
Mikkel Denker25f4b282eb71
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rsindex 776d3332..c0f3d844 100644--- a/core/src/api/improvement.rs+++ b/core/src/api/improvement.rs@@ -21,7 +21,7 @@ use axum::{extract, response::IntoResponse};use serde::Deserialize;use uuid::Uuid;-use crate::query_store::{ImprovementEvent, StoredQuery};+use crate::improvement::{AliceConversation, ImprovementEvent, StoredQuery};use super::{HtmlTemplate, State};@@ -76,6 +76,21 @@ pub async fn store(}}+#[allow(clippy::unused_async)]+pub async fn new_chat_id() -> impl IntoResponse {+ Uuid::new_v4().to_string()+}++#[allow(clippy::unused_async)]+pub async fn store_chat(+ extract::State(state): extract::State<Arc<State>>,+ extract::Json(params): extract::Json<AliceConversation>,+) {+ if let Some(q) = state.improvement_queue.as_ref() {+ q.lock().await.push(ImprovementEvent::Chat { chat: params })+ }+}+#[allow(clippy::unused_async)]pub async fn settings() -> impl IntoResponse {let template = SettingsTemplate {};diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 5b7fb696..e0814e92 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -25,9 +25,9 @@ use crate::{cluster::Cluster,member::{Member, Service},},+ improvement::{store_improvements_loop, ImprovementEvent},leaky_queue::LeakyQueue,qa_model::QaModel,- query_store::{self, ImprovementEvent},ranking::models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},searcher::frontend::FrontendSearcher,summarizer::Summarizer,@@ -51,7 +51,7 @@ mod alice;mod autosuggest;mod chat;mod explore;-mod improvement;+pub mod improvement;mod index;mod metrics;mod opensearch;@@ -121,10 +121,7 @@ pub async fn router(let query_store_queue = config.query_store_db_host.clone().map(|db_host| {let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));- tokio::spawn(query_store::store_queries_loop(- query_store_queue.clone(),- db_host,- ));+ tokio::spawn(store_improvements_loop(query_store_queue.clone(), db_host));query_store_queue});@@ -179,6 +176,14 @@ pub async fn router(.route("/opensearch.xml", get(opensearch::route)).route("/improvement/click", post(improvement::click)).route("/improvement/store", post(improvement::store))+ .route(+ "/improvement/alice/new_chat_id",+ post(improvement::new_chat_id),+ )+ .route(+ "/improvement/alice/store_chat",+ post(improvement::store_chat),+ ).fallback(get_service(ServeDir::new("frontend/dist/"))).layer(CompressionLayer::new()).merge(diff --git a/core/src/query_store.rs b/core/src/improvement.rssimilarity index 77%rename from core/src/query_store.rsrename to core/src/improvement.rsindex 6f7bd0aa..98b49de3 100644--- a/core/src/query_store.rs+++ b/core/src/improvement.rs@@ -43,10 +43,30 @@ pub struct StoredQuery {timestamp: Option<DateTime<Utc>>, // it is extremely important that we strip minutes, seconds and nanoseconds here for privacy}+#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+pub struct AliceConversation {+ pub id: Uuid,+ messages: Vec<AliceMessage>,+}++#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+struct AliceMessage {+ from: String,+ message: String,+ queries: Vec<AliceQuery>,+}++#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+struct AliceQuery {+ query: String,+ results: Vec<crate::alice::SimplifiedWebsite>,+}+#[derive(Clone)]pub enum ImprovementEvent {StoreQuery(StoredQuery),Click { qid: Uuid, idx: usize },+ Chat { chat: AliceConversation },}impl StoredQuery {@@ -82,7 +102,7 @@ async fn dump_queue(queue: &Mutex<LeakyQueue<ImprovementEvent>>) -> Vec<Improvemres}-pub async fn store_queries_loop(+pub async fn store_improvements_loop(queue: Arc<Mutex<LeakyQueue<ImprovementEvent>>>,scylla_host: String,) {@@ -95,12 +115,13 @@ pub async fn store_queries_loop(for event in events {match event {- ImprovementEvent::StoreQuery(query) => scylla.insert(query).await,+ ImprovementEvent::StoreQuery(query) => scylla.store_query(query).await,ImprovementEvent::Click { qid, idx } => {if let Ok(idx) = idx.try_into() {scylla.store_click(qid, idx).await}}+ ImprovementEvent::Chat { chat } => scylla.store_chat(chat).await,}}}@@ -110,6 +131,7 @@ struct ScyllaConn {session: scylla::Session,prepared_insert: PreparedStatement,prepared_click: PreparedStatement,+ prepared_chat: PreparedStatement,}impl ScyllaConn {@@ -123,6 +145,7 @@ impl ScyllaConn {&[],).await?;+session.query("CREATE TABLE IF NOT EXISTS ks.clicks (qid uuid, click tinyint, primary key (qid))",@@ -130,21 +153,34 @@ impl ScyllaConn {).await?;+ session+ .query(+ "CREATE TABLE IF NOT EXISTS ks.chats (id uuid, json_encoded text, primary key (qid))",+ &[],+ )+ .await?;+let prepared_insert: PreparedStatement = session.prepare("INSERT INTO ks.queries (qid, query, urls, timestamp) VALUES(?, ?, ?, ?)").await?;+let prepared_click: PreparedStatement = session.prepare("INSERT INTO ks.clicks (qid, click) VALUES(?, ?)").await?;+ let prepared_chat: PreparedStatement = session+ .prepare("INSERT INTO ks.chats (id, json_encoded) VALUES(?, ?)")+ .await?;+Ok(Self {session,prepared_insert,prepared_click,+ prepared_chat,})}- async fn insert(&self, query: StoredQuery) {+ async fn store_query(&self, query: StoredQuery) {let urls = serde_json::to_string(&query.result_urls).unwrap();let timestamp = query.timestamp@@ -177,4 +213,17 @@ impl ScyllaConn {tracing::error!("scylla store_click error: {err}");}}++ async fn store_chat(&self, chat: AliceConversation) {+ if let Ok(json_encoded) = serde_json::to_string(&chat) {+ let res = self+ .session+ .execute(&self.prepared_chat, (chat.id, json_encoded))+ .await;++ if let Err(err) = res {+ tracing::error!("scylla store_chat error: {err}");+ }+ }+ }}diff --git a/core/src/lib.rs b/core/src/lib.rsindex 1945ac4e..2d3f665a 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -48,6 +48,7 @@ mod human_website_annotations;mod hyperloglog;mod image_downloader;mod image_store;+mod improvement;pub mod index;mod intmap;mod kahan_sum;@@ -58,7 +59,6 @@ mod metrics;pub mod prehashed;mod qa_model;mod query;-mod query_store;pub mod ranking;mod schema;mod search_ctx;diff --git a/core/src/main.rs b/core/src/main.rsindex 8e77de82..975a737c 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -88,6 +88,7 @@ enum Commands {#[clap(long, takes_value = false)]skip_download: bool,+ #[clap(long, takes_value = false)]alice: bool,},}diff --git a/frontend/src/pages/chat.astro b/frontend/src/pages/chat.astroindex 32bb1a4b..25b5d186 100644--- a/frontend/src/pages/chat.astro+++ b/frontend/src/pages/chat.astro@@ -58,6 +58,7 @@ import OpticSelector from "../components/OpticSelector.astro";border-color: rgba(11, 123, 255, 0.3);width: fit-content;height: fit-content;+ max-width: 100%;border-width: 1px;border-radius: 0.75rem;border-bottom-right-radius: 0;@@ -69,6 +70,7 @@ import OpticSelector from "../components/OpticSelector.astro";border-color: rgba(0, 0, 0, 0.15);width: fit-content;height: fit-content;+ max-width: 100%;border-width: 1px;border-radius: 0.75rem;border-bottom-left-radius: 0;@@ -95,17 +97,20 @@ import OpticSelector from "../components/OpticSelector.astro";const statusTemplate: HTMLDivElement = document.querySelector(".status-template")!;const opticSelector: HTMLSelectElement = document.querySelector("#optics-selector")!;+ const allowStats = window.localStorage.getItem("allowStats") === "true";+// let prevState: number[][] | null = null;let prevState: any | null = null;+ let chatId: string | null = null;let originalSendButtonColor = sendButton.style.color;let numMessageLines = 1;let messageInputHeight = 24;- let messages: {from: string, message: string, queries: {query: string, results: {text: string, url: string, domain: string}[]}[]}[] = [];+ let messages: {from: string, message: string, queries: {query: string, results: {text: string, url: string, title: string}[]}[]}[] = [];let aliceResponding = false;let aliceStreamingMessage = "";- let currentQueries: {query: string, results: {text: string, url: string, domain: string}[]}[] = [];+ let currentQueries: {query: string, results: {text: string, url: string, title: string}[]}[] = [];function updateMessageInputHeight() {const lines = messageInput.value.split("\n");@@ -220,6 +225,8 @@ import OpticSelector from "../components/OpticSelector.astro";messageInput.disabled = false;sendButton.style.color = originalSendButtonColor;messageInput.focus();++ improvements();}async function sendMessageToAlice(message: string) {@@ -256,7 +263,7 @@ import OpticSelector from "../components/OpticSelector.astro";};}- function addMessage(from: string, message: string, queries: {query: string, results: {text: string, url: string, domain: string}[]}[] = []) {+ function addMessage(from: string, message: string, queries: {query: string, results: {text: string, url: string, title: string}[]}[] = []) {messages.push({from: from,message: message,@@ -317,4 +324,48 @@ import OpticSelector from "../components/OpticSelector.astro";codeBlock.style.background = "none";}}++ // improvements+ async function newChatId(): Promise<string> {+ let url = "/improvement/alice/new_chat_id";++ // post request using fetch api+ let response = await fetch(url, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json'+ }+ });++ return await response.text();+ }++ async function storeChat() {+ if (chatId == null) {+ chatId = await newChatId();+ }++ let chat = {+ messages: messages,+ id: chatId+ };++ let url = "/improvement/alice/store_chat";++ await fetch(url, {+ method: 'POST',+ body: JSON.stringify(chat),+ headers: {+ 'Content-Type': 'application/json'+ }+ });+ }++ function improvements() {+ if (allowStats) {+ storeChat();+ }+ }++</script>\ No newline at end of filediff --git a/frontend/src/pages/settings/privacy.astro b/frontend/src/pages/settings/privacy.astroindex ef28eb1d..4ba7c67b 100644--- a/frontend/src/pages/settings/privacy.astro+++ b/frontend/src/pages/settings/privacy.astro@@ -11,7 +11,7 @@ import SettingsLayout from "../../layouts/Settings.astro";<div class="text-sm">It is important to note, that <b class="font-extrabold">no</b> personal information is stored about your search. We don't store your IP, browser fingerprint or even a precise timestamp.- In fact, you can see exactly what gets stored <a href="https://github.com/StractOrg/Stract/blob/main/core/src/query_store.rs#L36-#L50">here</a>.+ In fact, you can see exactly what gets stored <a href="https://github.com/StractOrg/Stract/blob/main/core/src/improvement.rs#L36-#L70">here</a>.</div><div class="text-sm">