web search done right
Usage statistics
Mikkel Denker2714d9fcc374
diff --git a/Cargo.lock b/Cargo.lockindex 10066b31..7ca744fe 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -249,7 +249,7 @@ version = "0.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6293dae2ec708e679da6736e857cf8532886ef258e92930f38279c12641628b8"dependencies = [- "heck",+ "heck 0.4.1","proc-macro2","quote","syn",@@ -273,6 +273,17 @@ version = "0.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"+[[package]]+name = "bigdecimal"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256"+dependencies = [+ "num-bigint",+ "num-integer",+ "num-traits",+]+[[package]]name = "bincode"version = "1.3.3"@@ -540,7 +551,7 @@ version = "3.2.18"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65"dependencies = [- "heck",+ "heck 0.4.1","proc-macro-error","proc-macro2","quote",@@ -828,6 +839,7 @@ dependencies = ["reqwest","rocksdb","rust-stemmers",+ "scylla","serde","serde_json","serde_urlencoded",@@ -924,6 +936,19 @@ dependencies = ["syn",]+[[package]]+name = "dashmap"+version = "5.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"+dependencies = [+ "cfg-if",+ "hashbrown",+ "lock_api",+ "once_cell",+ "parking_lot_core",+]+[[package]]name = "derive_builder"version = "0.9.0"@@ -1444,6 +1469,15 @@ dependencies = ["ahash",]+[[package]]+name = "heck"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"+dependencies = [+ "unicode-segmentation",+]+[[package]]name = "heck"version = "0.4.1"@@ -1468,6 +1502,12 @@ dependencies = ["libc",]+[[package]]+name = "histogram"+version = "0.6.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669"+[[package]]name = "html-escape"version = "0.2.13"@@ -1947,6 +1987,9 @@ name = "lz4_flex"version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3"+dependencies = [+ "twox-hash",+][[package]]name = "mac"@@ -2200,6 +2243,17 @@ dependencies = ["winapi",]+[[package]]+name = "num-bigint"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3"+dependencies = [+ "autocfg",+ "num-integer",+ "num-traits",+]+[[package]]name = "num-complex"version = "0.4.3"@@ -2249,6 +2303,27 @@ dependencies = ["libc",]+[[package]]+name = "num_enum"+version = "0.5.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9"+dependencies = [+ "num_enum_derive",+]++[[package]]+name = "num_enum_derive"+version = "0.5.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799"+dependencies = [+ "proc-macro-crate",+ "proc-macro2",+ "quote",+ "syn",+]+[[package]]name = "number_prefix"version = "0.3.0"@@ -2631,6 +2706,16 @@ version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"+[[package]]+name = "proc-macro-crate"+version = "1.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919"+dependencies = [+ "once_cell",+ "toml_edit",+]+[[package]]name = "proc-macro-error"version = "1.0.4"@@ -3034,6 +3119,69 @@ dependencies = ["untrusted",]+[[package]]+name = "scylla"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4b9d4ef7fb24d95d30c4a8da782bb2afead3a8b66f32c805bb82a03722d7be33"+dependencies = [+ "arc-swap",+ "async-trait",+ "bigdecimal",+ "byteorder",+ "bytes",+ "chrono",+ "dashmap",+ "futures",+ "histogram",+ "itertools 0.10.5",+ "lz4_flex",+ "num-bigint",+ "num_enum",+ "rand 0.8.5",+ "scylla-cql",+ "scylla-macros",+ "smallvec",+ "snap",+ "strum",+ "strum_macros",+ "thiserror",+ "tokio",+ "tracing",+ "uuid",+]++[[package]]+name = "scylla-cql"+version = "0.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6972061cbcc83754b4243d007ae51c1a1345a950b368cbdaad0186eac8799203"+dependencies = [+ "async-trait",+ "bigdecimal",+ "byteorder",+ "bytes",+ "chrono",+ "lz4_flex",+ "num-bigint",+ "num_enum",+ "scylla-macros",+ "snap",+ "thiserror",+ "tokio",+ "uuid",+]++[[package]]+name = "scylla-macros"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e03b3a19daa79085439113c746d2946e5e6effd2d9039bf092bb08df915487b2"+dependencies = [+ "quote",+ "syn",+]+[[package]]name = "security-framework"version = "2.8.2"@@ -3208,6 +3356,12 @@ version = "1.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"+[[package]]+name = "snap"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831"+[[package]]name = "socket2"version = "0.4.7"@@ -3251,6 +3405,12 @@ version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"+[[package]]+name = "static_assertions"+version = "1.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"+[[package]]name = "string_cache"version = "0.8.4"@@ -3295,6 +3455,25 @@ version = "0.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"+[[package]]+name = "strum"+version = "0.23.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"++[[package]]+name = "strum_macros"+version = "0.23.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"+dependencies = [+ "heck 0.3.3",+ "proc-macro2",+ "quote",+ "rustversion",+ "syn",+]+[[package]]name = "syn"version = "1.0.107"@@ -3712,6 +3891,23 @@ dependencies = ["serde",]+[[package]]+name = "toml_datetime"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622"++[[package]]+name = "toml_edit"+version = "0.19.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825"+dependencies = [+ "indexmap",+ "toml_datetime",+ "winnow",+]+[[package]]name = "tower"version = "0.4.13"@@ -3835,6 +4031,16 @@ version = "0.2.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"+[[package]]+name = "twox-hash"+version = "1.6.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"+dependencies = [+ "cfg-if",+ "static_assertions",+]+[[package]]name = "typenum"version = "1.16.0"@@ -4313,6 +4519,15 @@ version = "0.42.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"+[[package]]+name = "winnow"+version = "0.3.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658"+dependencies = [+ "memchr",+]+[[package]]name = "winreg"version = "0.10.1"diff --git a/Cargo.toml b/Cargo.tomlindex 376624d8..93c501f4 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -6,6 +6,7 @@ members = [[profile.release]debug = true+lto = "thin"[workspace.dependencies]bincode = "1.3.3"@@ -67,4 +68,5 @@ onnxruntime = { git = "https://github.com/Cuely/onnxruntime-rs" }tokenizers = "0.13.2"byteorder = "1.4.3"async-stream = "0.3.3"-tokio-stream = "0.1.11"\ No newline at end of file+tokio-stream = "0.1.11"+scylla = "0.7.0"\ No newline at end of filediff --git a/core/Cargo.toml b/core/Cargo.tomlindex 00af4f19..8abd892b 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -73,6 +73,7 @@ lalrpop-util = { workspace = true }byteorder = { workspace = true }async-stream = { workspace = true }tokio-stream = { workspace = true }+scylla = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rsnew file mode 100644index 00000000..2163856c--- /dev/null+++ b/core/src/api/improvement.rs@@ -0,0 +1,88 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use askama::Template;+use axum::{extract, response::IntoResponse, Extension};+use serde::Deserialize;+use uuid::Uuid;++use crate::query_store::{ImprovementEvent, StoredQuery};++use super::{HtmlTemplate, State};++#[derive(Deserialize, Debug)]+pub struct ClickParams {+ pub qid: Uuid,+ pub click: usize,+}++#[derive(Deserialize, Debug)]+pub struct StoreParams {+ pub query: String,+ pub urls: Vec<String>,+}++#[allow(clippy::unused_async)]+pub async fn click(+ extract::Query(params): extract::Query<ClickParams>,+ Extension(state): Extension<Arc<State>>,+) {+ if let Some(q) = state.improvement_queue.as_ref() {+ q.lock().await.push(ImprovementEvent::Click {+ qid: params.qid,+ idx: params.click,+ })+ }+}++impl From<StoreParams> for StoredQuery {+ fn from(params: StoreParams) -> Self {+ StoredQuery::new(+ params.query,+ params.urls.into_iter().map(|s| s.into()).collect(),+ )+ }+}++#[allow(clippy::unused_async)]+pub async fn store(+ extract::Json(params): extract::Json<StoreParams>,+ Extension(state): Extension<Arc<State>>,+) -> impl IntoResponse {+ match state.improvement_queue.as_ref() {+ Some(q) => {+ let query: StoredQuery = params.into();+ let qid = *query.qid();+ q.lock().await.push(ImprovementEvent::StoreQuery(query));++ qid.to_string()+ }+ None => String::new(),+ }+}++#[allow(clippy::unused_async)]+pub async fn settings() -> impl IntoResponse {+ let template = SettingsTemplate {};++ HtmlTemplate(template)+}++#[derive(Template)]+#[template(path = "settings/privacy/index.html")]+struct SettingsTemplate {}diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex a3fd4a3c..03c9b289 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -15,12 +15,15 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use axum::{body::Body, routing::get_service, Extension, Router};+use tokio::sync::Mutex;use tower_http::{compression::CompressionLayer, services::ServeDir};use crate::{autosuggest::Autosuggest,bangs::Bangs,+ leaky_queue::LeakyQueue,qa_model::QaModel,+ query_store::{self, ImprovementEvent},ranking::models::cross_encoder::CrossEncoderModel,searcher::{DistributedSearcher, Shard},summarizer::Summarizer,@@ -39,6 +42,7 @@ use axum::{mod about;mod autosuggest;+mod improvement;mod index;mod metrics;mod opensearch;@@ -55,6 +59,7 @@ pub struct State {pub autosuggest: Autosuggest,pub search_counter: crate::metrics::Counter,pub summarizer: Arc<Summarizer>,+ pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,}impl<T> IntoResponse for HtmlTemplate<T>@@ -100,6 +105,15 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)None => None,};+ let query_store_queue = config.query_store_db_host.clone().map(|db_host| {+ let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));+ tokio::spawn(query_store::store_queries_loop(+ query_store_queue.clone(),+ db_host,+ ));+ query_store_queue+ });+let bangs = Bangs::from_path(&config.bangs_path);let searcher = DistributedSearcher::new(shards, crossencoder, qa_model, bangs);@@ -108,6 +122,7 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)autosuggest,search_counter,summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),+ improvement_queue: query_store_queue,});Ok(Router::new()@@ -121,8 +136,11 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter).route("/settings", get(optics::route)).route("/settings/optics", get(optics::route)).route("/settings/sites", get(sites::route))+ .route("/settings/privacy", get(improvement::settings)).route("/privacy-and-happy-lawyers", get(privacy::route)).route("/opensearch.xml", get(opensearch::route))+ .route("/improvement/click", post(improvement::click))+ .route("/improvement/store", post(improvement::store)).fallback(get_service(ServeDir::new("frontend/dist/")).handle_error(|error: std::io::Error| async move {(diff --git a/core/src/leaky_queue.rs b/core/src/leaky_queue.rsnew file mode 100644index 00000000..d5389c6a--- /dev/null+++ b/core/src/leaky_queue.rs@@ -0,0 +1,68 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::collections::VecDeque;++pub struct LeakyQueue<T> {+ inner: VecDeque<T>,+ max_size: usize,+}++impl<T> LeakyQueue<T> {+ pub fn new(max_size: usize) -> Self {+ Self {+ inner: VecDeque::with_capacity(max_size),+ max_size,+ }+ }++ pub fn push(&mut self, item: T) {+ while self.inner.len() >= self.max_size {+ self.inner.pop_front();+ }++ self.inner.push_back(item);+ }++ pub fn pop(&mut self) -> Option<T> {+ self.inner.pop_front()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple() {+ let mut q = LeakyQueue::new(3);++ q.push(1);++ assert_eq!(q.pop(), Some(1));+ assert_eq!(q.pop(), None);++ q.push(1);+ q.push(2);+ q.push(3);+ q.push(4);++ assert_eq!(q.pop(), Some(2));+ assert_eq!(q.pop(), Some(3));+ assert_eq!(q.pop(), Some(4));+ assert_eq!(q.pop(), None);+ }+}diff --git a/core/src/lib.rs b/core/src/lib.rsindex 7b00c810..33229cef 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -49,10 +49,12 @@ pub mod index;mod intmap;mod kahan_sum;mod kv;+mod leaky_queue;mod metrics;pub mod prehashed;mod qa_model;mod query;+mod query_store;pub mod ranking;mod schema;mod search_ctx;@@ -168,6 +170,7 @@ pub struct FrontendConfig {pub bangs_path: String,pub summarizer_path: String,pub search_servers: Vec<Vec<String>>,+ pub query_store_db_host: Option<String>,}#[derive(Debug, Serialize, Deserialize, Clone)]diff --git a/core/src/query_store.rs b/core/src/query_store.rsnew file mode 100644index 00000000..2448d1b8--- /dev/null+++ b/core/src/query_store.rs@@ -0,0 +1,180 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{sync::Arc, time::Duration};++use chrono::{DateTime, Timelike, Utc};+use scylla::{prepared_statement::PreparedStatement, SessionBuilder};+use thiserror::Error;+use tokio::{sync::Mutex, time};+use uuid::Uuid;++use crate::{leaky_queue::LeakyQueue, webpage::Url};++#[derive(Debug, Error)]+enum Error {+ #[error("scylla query")]+ ScyllaQuery(#[from] scylla::transport::errors::QueryError),++ #[error("scylla new session")]+ ScyllaNewSess(#[from] scylla::transport::errors::NewSessionError),+}++/// Note that we don't store any information that can be used to link+/// the query and result back to the user performing the query. This is extremely important!+#[derive(Debug)]+pub struct StoredQuery {+ qid: Uuid,+ query: String,+ result_urls: Vec<Url>,+ timestamp: Option<DateTime<Utc>>, // it is extremely important that we strip hours, minutes, seconds and nanoseconds here for privacy+}++impl StoredQuery {+ pub fn new(query: String, urls: Vec<Url>) -> Self {+ let timestamp = Utc::now()+ .with_minute(0)+ .and_then(|t| t.with_minute(0))+ .and_then(|t| t.with_second(0))+ .and_then(|t| t.with_nanosecond(0));++ let qid = Uuid::new_v4();++ Self {+ qid,+ query,+ result_urls: urls,+ timestamp,+ }+ }++ pub fn qid(&self) -> &Uuid {+ &self.qid+ }+}++pub enum ImprovementEvent {+ StoreQuery(StoredQuery),+ Click { qid: Uuid, idx: usize },+}++async fn dump_queue(queue: &Mutex<LeakyQueue<ImprovementEvent>>) -> Vec<ImprovementEvent> {+ let mut res = Vec::new();+ let mut lock = queue.lock().await;++ while let Some(query) = lock.pop() {+ res.push(query);+ }++ res+}++pub async fn store_queries_loop(+ queue: Arc<Mutex<LeakyQueue<ImprovementEvent>>>,+ scylla_host: String,+) {+ let scylla = ScyllaConn::new(scylla_host.as_str()).await.unwrap();+ let mut interval = time::interval(Duration::from_secs(30));++ loop {+ interval.tick().await;+ let events = dump_queue(&queue).await;++ for event in events {+ match event {+ ImprovementEvent::StoreQuery(query) => scylla.insert(query).await,+ ImprovementEvent::Click { qid, idx } => {+ if let Ok(idx) = idx.try_into() {+ scylla.store_click(qid, idx).await+ }+ }+ }+ }+ }+}++struct ScyllaConn {+ session: scylla::Session,+ prepared_insert: PreparedStatement,+ prepared_click: PreparedStatement,+}++impl ScyllaConn {+ async fn new(seed_node: &str) -> Result<Self, Error> {+ let session = SessionBuilder::new().known_node(seed_node).build().await?;++ session.query("CREATE KEYSPACE IF NOT EXISTS ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}", &[]).await?;+ session+ .query(+ "CREATE TABLE IF NOT EXISTS ks.queries (qid uuid, query text, urls text, timestamp timestamp, primary key (qid, timestamp))",+ &[],+ )+ .await?;+ session+ .query(+ "CREATE TABLE IF NOT EXISTS ks.clicks (qid uuid, click tinyint, primary key (qid))",+ &[],+ )+ .await?;++ let prepared_insert: PreparedStatement = session+ .prepare("INSERT INTO ks.queries (qid, query, urls, timestamp) VALUES(?, ?, ?, ?)")+ .await?;+ let prepared_click: PreparedStatement = session+ .prepare("INSERT INTO ks.clicks (qid, click) VALUES(?, ?)")+ .await?;++ Ok(Self {+ session,+ prepared_insert,+ prepared_click,+ })+ }++ async fn insert(&self, query: StoredQuery) {+ let urls = serde_json::to_string(&query.result_urls).unwrap();+ let timestamp = query+ .timestamp+ .map(|timestamp| chrono::Duration::seconds(timestamp.timestamp()))+ .unwrap_or_else(|| chrono::Duration::seconds(0));+ let qid = query.qid;++ let res = self+ .session+ .execute(+ &self.prepared_insert,+ (+ qid,+ query.query,+ urls,+ scylla::frame::value::Timestamp(timestamp),+ ),+ )+ .await;++ if let Err(err) = res {+ tracing::error!("scylla insert error: {err}");+ }+ }++ async fn store_click(&self, qid: Uuid, idx: i8) {+ let res = self.session.execute(&self.prepared_click, (qid, idx)).await;++ if let Err(err) = res {+ tracing::error!("scylla store_click error: {err}");+ }+ }+}diff --git a/frontend/src/components/SettingsMenu.astro b/frontend/src/components/SettingsMenu.astroindex 92f29afc..8cc051fa 100644--- a/frontend/src/components/SettingsMenu.astro+++ b/frontend/src/components/SettingsMenu.astro@@ -2,6 +2,7 @@const links = [{ url: "/settings", title: "Manage Optics" },{ url: "/settings/sites", title: "Site Rankings" },+ { url: "/settings/privacy", title: "Privacy" },];---diff --git a/frontend/src/layouts/Settings.astro b/frontend/src/layouts/Settings.astronew file mode 100644index 00000000..564810ef--- /dev/null+++ b/frontend/src/layouts/Settings.astro@@ -0,0 +1,20 @@+---+import Footer from "../components/Footer.astro";+import Layout from "./Layout.astro";+import Header from "../components/Header.astro";+import SettingsMenu from "../components/SettingsMenu.astro";+---+<Layout title="Cuely">+ <div class="flex h-full w-full flex-col">+ <Header />++ <div class="flex h-fit w-full justify-center pt-10">+ <SettingsMenu />+ <div class="flex w-full max-w-2xl flex-col">+ <slot />+ </div>+ </div>+ <div class="flex flex-grow"></div>+ <Footer />+ </div>+</Layout>\ No newline at end of filediff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex 9e242278..713f35af 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -97,14 +97,16 @@ import { askama } from "../askama";<aclass="url truncate text-gray-800 hover:no-underline"href="{{ item.url }}"+ onclick="improvement({{ loop.index0 }})">{askama`item.pretty_url $ {{internet.url}}`}</a></div><a- class="title-link truncate text-xl font-medium"+ class="title-link sr-title-link truncate text-xl font-medium"title="{{ item.title }}"href="{{ item.url }}"+ onclick="improvement({{ loop.index0 }})">{askama`item.title $ {{lorem.sentence}}`}</a>@@ -223,8 +225,39 @@ import { askama } from "../askama";<script is:inline src="/js/highlight.min.js"></script><script is:inline>var rankingAdjustButtons = document.querySelectorAll(".adjust-btn");+ var titleLinks = document.querySelectorAll(".sr-title-link");var modal = document.getElementById("modal");+ var searchParams = new URLSearchParams(window.location.search);+ var query = searchParams.get("q");++ const allowStats = window.localStorage.getItem("allowStats") === "true";++ var qid = null;++ var improvementSent = new Set();++ var urls = [];+ for (const elem of titleLinks) {+ urls.push(elem.href);+ }++ if (allowStats) {+ fetch("/improvement/store", {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ },+ body: JSON.stringify({+ "query": query,+ "urls": urls+ })+ }).then((response) => response.text()).then((data) => {+ console.log(data);+ qid = data;+ });+ }+document.addEventListener("click", (elem) => {modal.classList.remove("modal-open");modal.classList.add("modal-closed");@@ -255,6 +288,13 @@ import { askama } from "../askama";}, 0);});});++ function improvement(idx) {+ if (allowStats && qid && !improvementSent.has(idx)) {+ window.navigator.sendBeacon("/improvement/click?qid=" + qid + "&click=" + idx);+ improvementSent.add(idx);+ }+ }</script><script is:inline>hljs.highlightAll();diff --git a/frontend/src/pages/settings/index.astro b/frontend/src/pages/settings/index.astroindex 3a0dc059..5f9cc1d6 100644--- a/frontend/src/pages/settings/index.astro+++ b/frontend/src/pages/settings/index.astro@@ -1,91 +1,79 @@----import Footer from "../../components/Footer.astro";-import Layout from "../../layouts/Layout.astro";-import Header from "../../components/Header.astro";-import SettingsMenu from "../../components/SettingsMenu.astro";+import SettingsLayout from "../../layouts/Settings.astro";import { askama } from "../../askama";----<Layout title="Cuely">- <div class="flex h-full w-full flex-col">- <Header />-- <div class="flex h-fit w-full justify-center pt-10">- <SettingsMenu />- <div class="flex max-w-2xl flex-col">- <h1 class="text-2xl font-medium">Manage Optics</h1>- <div class="mt-5 text-sm">- Optics lets you control almost everything about which search results- that gets returned to you. You can discard results from specific- sites, boost results from other sites and much more.- </div>- <div class="mt-3 text-sm">- See our <a- href="https://github.com/Cuely/sample-optics/blob/main/quickstart.optic"- >quickstart</a- > for how to get started. Once you have developed your optic, you can add- it here to be used during your search.- </div>- <div class="mt-3 text-sm">- Simply host the optic on a url that returns a plain-text HTTP response- with the optic. We use raw.githubusercontent.com, but you are free to- host them elsewhere.- </div>- <div- class="mt-16"- x-data="{ items: $persist([]).as('optics'), name: '', url: '' }"+<SettingsLayout>+ <div>+ <h1 class="text-2xl font-medium">Manage Optics</h1>+ <div class="mt-5 text-sm">+ Optics lets you control almost everything about which search results+ that gets returned to you. You can discard results from specific+ sites, boost results from other sites and much more.+ </div>+ <div class="mt-3 text-sm">+ See our <a+ href="https://github.com/Cuely/sample-optics/blob/main/quickstart.optic"+ >quickstart</a+ > for how to get started. Once you have developed your optic, you can add+ it here to be used during your search.+ </div>+ <div class="mt-3 text-sm">+ Simply host the optic on a url that returns a plain-text HTTP response+ with the optic. We use raw.githubusercontent.com, but you are free to+ host them elsewhere.+ </div>+ <div+ class="mt-16"+ x-data="{ items: $persist([]).as('optics'), name: '', url: '' }"+ >+ <div class="flex w-full justify-between pl-5 pr-5">+ <input id="name" type="text" placeholder="Name" x-model="name" />+ <input id="url" type="text" placeholder="Url" x-model="url" />+ <button+ x-on:click="if (name != '' && url != '') {items.push({ name, url }); name = url = ''}"+ class="h-10 w-20 rounded-full border-0 bg-brand text-sm text-white"+ >Add</button>- <div class="flex w-full justify-between pl-5 pr-5">- <input id="name" type="text" placeholder="Name" x-model="name" />- <input id="url" type="text" placeholder="Url" x-model="url" />- <button- x-on:click="items.push({ name, url }); name = url = ''"- class="h-10 w-20 rounded-full border-0 bg-brand text-sm text-white"- >Add</button- >+ </div>+ <div class="mt-5">+ <div class="grid w-full space-y-5" id="optics-list">+ <div class="flex">+ <div class="w-10"></div>+ <div class="flex-1 font-medium">Name</div>+ <div class="flex-1 font-medium">Url</div></div>- <div class="mt-5">- <div class="grid w-full space-y-5" id="optics-list">+ {+ askama.for_("optic in default_optics", () => (<div class="flex">- <div class="w-10"></div>- <div class="flex-1 font-medium">Name</div>- <div class="flex-1 font-medium">Url</div>- </div>- {- askama.for_("optic in default_optics", () => (- <div class="flex">- <div class="mr-5 w-5">- <img src="/images/disabled-delete.svg" class="h-5 w-5" />- </div>- <div class="flex-1 text-sm">{askama`optic.name $ {{random.words}}`}</div>- <div class="flex-1 text-sm">{askama`optic.url $ {{internet.url}}`}</div>- </div>- ))- }- <template x-for="(item, index) in items">- <div class="flex">- <div class="w-10">- <img- src="/images/delete.svg"- class="h-5 w-5 hover:cursor-pointer"- x-on:click="items.splice(index, 1)"- />- </div>- <div class="user-optic flex-1 text-sm" x-text="item.name">- </div>- <div class="user-optic flex-1 text-sm" x-text="item.url">- </div>+ <div class="mr-5 w-5">+ <img src="/images/disabled-delete.svg" class="h-5 w-5" /></div>- </template>+ <div class="flex-1 text-sm">{askama`optic.name $ {{random.words}}`}</div>+ <div class="flex-1 text-sm">{askama`optic.url $ {{internet.url}}`}</div>+ </div>+ ))+ }+ <template x-for="(item, index) in items">+ <div class="flex">+ <div class="w-10">+ <img+ src="/images/delete.svg"+ class="h-5 w-5 hover:cursor-pointer"+ x-on:click="items.splice(index, 1)"+ />+ </div>+ <div class="user-optic flex-1 text-sm" x-text="item.name">+ </div>+ <div class="user-optic flex-1 text-sm" x-text="item.url">+ </div></div>- </div>+ </template></div></div></div>- <div class="flex flex-grow"></div>- <Footer /></div>-</Layout>+</SettingsLayout><script>import "$/alpine";diff --git a/frontend/src/pages/settings/privacy.astro b/frontend/src/pages/settings/privacy.astronew file mode 100644index 00000000..69b939a2--- /dev/null+++ b/frontend/src/pages/settings/privacy.astro@@ -0,0 +1,37 @@+---+import SettingsLayout from "../../layouts/Settings.astro";+---+<SettingsLayout>+ <div class="flex flex-col space-y-2">+ <h1 class="text-2xl font-medium">Privacy</h1>+ <div class="text-sm">+ By default, some data about your searches are stored on our servers. We primarily store the text you used for the search, and which results (if any) you clicked on.+ This data immensely helps us improve everyones search experience by training our ranking models.+ </div>++ <div class="text-sm">+ It is important to note, that <b class="font-extrabold">no</b> personal information is stored about your search. We don't store your IP, browser fingerprint or even a precise timestamp.+ In fact, you can see exactly what gets stored <a href="#">here</a>.+ </div>++ <div class="text-sm">+ We hope you feel comfortable sharing this data with us as this makes the results much better for everyone, but we also respect if you don't.+ You can opt-out at any point by toggling the checkbox below and we wont store anything at all about your searches.+ </div>++ <div class="flex space-x-1 pl-1 text-sm" x-data="{ allowStats: $persist(true).as('allowStats') }">+ <input type="checkbox" id="allowStats" x-model="allowStats">+ <label for="allowStats">Allow usage statistics</label>+ </div>+ </div>+</SettingsLayout>++<style>+ a {+ @apply text-blue-600;+ }+</style>++<script>+ import "$/alpine";+</script>\ No newline at end of filediff --git a/frontend/src/pages/settings/sites.astro b/frontend/src/pages/settings/sites.astroindex 71aed401..a23828cb 100644--- a/frontend/src/pages/settings/sites.astro+++ b/frontend/src/pages/settings/sites.astro@@ -1,73 +1,58 @@----import Footer from "../../components/Footer.astro";-import Layout from "../../layouts/Layout.astro";-import Header from "../../components/Header.astro";-import SettingsMenu from "../../components/SettingsMenu.astro";+import SettingsLayout from "../../layouts/Settings.astro";---+<SettingsLayout>+ <div>+ <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>+ <div class="text-sm">+ These sites receive a boost during search. Results from these and similar sites+ are more likely to appear in your search results.+ </div>+ <div class="mt-5">+ <div+ class="grid w-full grid-cols-[auto_1fr] space-y-5"+ id="liked-list"+ >+ <div class="w-10"></div>+ <div class="font-medium">Site Name</div>+ </div>+ </div>+ </div>++ <div>+ <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>+ <div class="text-sm">+ These sites get de-prioritized during search. Results from these and similar+ sites are less likely to appear in your search results.+ </div>+ <div class="mt-5">+ <div+ class="grid w-full grid-cols-[auto_1fr] space-y-5"+ id="disliked-list"+ >+ <div class="w-10"></div>+ <div class="font-medium">Site Name</div>+ </div>+ </div>+ </div>-<Layout title="Cuely">- <div class="flex h-full w-full flex-col">- <Header />-- <div class="flex h-fit w-full justify-center pt-10">- <SettingsMenu />- <div class="flex w-full max-w-2xl flex-col space-y-10">- <div>- <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>- <div class="text-sm">- These sites receive a boost during search. Results from these sites- are more likely to appear in your search results.- </div>- <div class="mt-5">- <div- class="grid w-full grid-cols-[auto_1fr] space-y-5"- id="liked-list"- >- <div class="w-10"></div>- <div class="font-medium">Site Name</div>- </div>- </div>- </div>-- <div>- <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>- <div class="text-sm">- These sites get de-prioritized during search. Results from these- sites are less likely to appear in your search results.- </div>- <div class="mt-5">- <div- class="grid w-full grid-cols-[auto_1fr] space-y-5"- id="disliked-list"- >- <div class="w-10"></div>- <div class="font-medium">Site Name</div>- </div>- </div>- </div>-- <div>- <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>- <div class="text-sm">- These are the sites you have blocked. They won't appear in any of- your searches.- </div>- <div class="mt-5">- <div- class="grid w-full grid-cols-[auto_1fr] space-y-5"- id="blocked-list"- >- <div class="w-10"></div>- <div class="font-medium">Site Name</div>- </div>- </div>- </div>+ <div>+ <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>+ <div class="text-sm">+ These are the sites you have blocked. They won't appear in any of+ your searches.+ </div>+ <div class="mt-5">+ <div+ class="grid w-full grid-cols-[auto_1fr] space-y-5"+ id="blocked-list"+ >+ <div class="w-10"></div>+ <div class="font-medium">Site Name</div></div></div>- <div class="flex flex-grow"></div>- <Footer /></div>-</Layout>+</SettingsLayout><script is:inline>let sites = JSON.parse(