Skip to content

web search done right

Usage statistics

Mikkel Denker2714d9fcc374
diff --git a/Cargo.lock b/Cargo.lock
index 10066b31..7ca744fe 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -249,7 +249,7 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6293dae2ec708e679da6736e857cf8532886ef258e92930f38279c12641628b8"
dependencies = [
- "heck",
+ "heck 0.4.1",
"proc-macro2",
"quote",
"syn",
@@ -273,6 +273,17 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"
+[[package]]
+name = "bigdecimal"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
[[package]]
name = "bincode"
version = "1.3.3"
@@ -540,7 +551,7 @@ version = "3.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65"
dependencies = [
- "heck",
+ "heck 0.4.1",
"proc-macro-error",
"proc-macro2",
"quote",
@@ -828,6 +839,7 @@ dependencies = [
"reqwest",
"rocksdb",
"rust-stemmers",
+ "scylla",
"serde",
"serde_json",
"serde_urlencoded",
@@ -924,6 +936,19 @@ dependencies = [
"syn",
]
+[[package]]
+name = "dashmap"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc"
+dependencies = [
+ "cfg-if",
+ "hashbrown",
+ "lock_api",
+ "once_cell",
+ "parking_lot_core",
+]
+
[[package]]
name = "derive_builder"
version = "0.9.0"
@@ -1444,6 +1469,15 @@ dependencies = [
"ahash",
]
+[[package]]
+name = "heck"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c"
+dependencies = [
+ "unicode-segmentation",
+]
+
[[package]]
name = "heck"
version = "0.4.1"
@@ -1468,6 +1502,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "histogram"
+version = "0.6.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669"
+
[[package]]
name = "html-escape"
version = "0.2.13"
@@ -1947,6 +1987,9 @@ name = "lz4_flex"
version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3"
+dependencies = [
+ "twox-hash",
+]
[[package]]
name = "mac"
@@ -2200,6 +2243,17 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "num-bigint"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
[[package]]
name = "num-complex"
version = "0.4.3"
@@ -2249,6 +2303,27 @@ dependencies = [
"libc",
]
+[[package]]
+name = "num_enum"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9"
+dependencies = [
+ "num_enum_derive",
+]
+
+[[package]]
+name = "num_enum_derive"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "number_prefix"
version = "0.3.0"
@@ -2631,6 +2706,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+[[package]]
+name = "proc-macro-crate"
+version = "1.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919"
+dependencies = [
+ "once_cell",
+ "toml_edit",
+]
+
[[package]]
name = "proc-macro-error"
version = "1.0.4"
@@ -3034,6 +3119,69 @@ dependencies = [
"untrusted",
]
+[[package]]
+name = "scylla"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b9d4ef7fb24d95d30c4a8da782bb2afead3a8b66f32c805bb82a03722d7be33"
+dependencies = [
+ "arc-swap",
+ "async-trait",
+ "bigdecimal",
+ "byteorder",
+ "bytes",
+ "chrono",
+ "dashmap",
+ "futures",
+ "histogram",
+ "itertools 0.10.5",
+ "lz4_flex",
+ "num-bigint",
+ "num_enum",
+ "rand 0.8.5",
+ "scylla-cql",
+ "scylla-macros",
+ "smallvec",
+ "snap",
+ "strum",
+ "strum_macros",
+ "thiserror",
+ "tokio",
+ "tracing",
+ "uuid",
+]
+
+[[package]]
+name = "scylla-cql"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6972061cbcc83754b4243d007ae51c1a1345a950b368cbdaad0186eac8799203"
+dependencies = [
+ "async-trait",
+ "bigdecimal",
+ "byteorder",
+ "bytes",
+ "chrono",
+ "lz4_flex",
+ "num-bigint",
+ "num_enum",
+ "scylla-macros",
+ "snap",
+ "thiserror",
+ "tokio",
+ "uuid",
+]
+
+[[package]]
+name = "scylla-macros"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e03b3a19daa79085439113c746d2946e5e6effd2d9039bf092bb08df915487b2"
+dependencies = [
+ "quote",
+ "syn",
+]
+
[[package]]
name = "security-framework"
version = "2.8.2"
@@ -3208,6 +3356,12 @@ version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
+[[package]]
+name = "snap"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831"
+
[[package]]
name = "socket2"
version = "0.4.7"
@@ -3251,6 +3405,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
[[package]]
name = "string_cache"
version = "0.8.4"
@@ -3295,6 +3455,25 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+[[package]]
+name = "strum"
+version = "0.23.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"
+
+[[package]]
+name = "strum_macros"
+version = "0.23.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"
+dependencies = [
+ "heck 0.3.3",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
[[package]]
name = "syn"
version = "1.0.107"
@@ -3712,6 +3891,23 @@ dependencies = [
"serde",
]
+[[package]]
+name = "toml_datetime"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622"
+
+[[package]]
+name = "toml_edit"
+version = "0.19.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825"
+dependencies = [
+ "indexmap",
+ "toml_datetime",
+ "winnow",
+]
+
[[package]]
name = "tower"
version = "0.4.13"
@@ -3835,6 +4031,16 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
+[[package]]
+name = "twox-hash"
+version = "1.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675"
+dependencies = [
+ "cfg-if",
+ "static_assertions",
+]
+
[[package]]
name = "typenum"
version = "1.16.0"
@@ -4313,6 +4519,15 @@ version = "0.42.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
+[[package]]
+name = "winnow"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faf09497b8f8b5ac5d3bb4d05c0a99be20f26fd3d5f2db7b0716e946d5103658"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "winreg"
version = "0.10.1"
diff --git a/Cargo.toml b/Cargo.toml
index 376624d8..93c501f4 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,6 +6,7 @@ members = [
[profile.release]
debug = true
+lto = "thin"
[workspace.dependencies]
bincode = "1.3.3"
@@ -67,4 +68,5 @@ onnxruntime = { git = "https://github.com/Cuely/onnxruntime-rs" }
tokenizers = "0.13.2"
byteorder = "1.4.3"
async-stream = "0.3.3"
-tokio-stream = "0.1.11"
\ No newline at end of file
+tokio-stream = "0.1.11"
+scylla = "0.7.0"
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 00af4f19..8abd892b 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -73,6 +73,7 @@ lalrpop-util = { workspace = true }
byteorder = { workspace = true }
async-stream = { workspace = true }
tokio-stream = { workspace = true }
+scylla = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rs
new file mode 100644
index 00000000..2163856c
--- /dev/null
+++ b/core/src/api/improvement.rs
@@ -0,0 +1,88 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use askama::Template;
+use axum::{extract, response::IntoResponse, Extension};
+use serde::Deserialize;
+use uuid::Uuid;
+
+use crate::query_store::{ImprovementEvent, StoredQuery};
+
+use super::{HtmlTemplate, State};
+
+#[derive(Deserialize, Debug)]
+pub struct ClickParams {
+ pub qid: Uuid,
+ pub click: usize,
+}
+
+#[derive(Deserialize, Debug)]
+pub struct StoreParams {
+ pub query: String,
+ pub urls: Vec<String>,
+}
+
+#[allow(clippy::unused_async)]
+pub async fn click(
+ extract::Query(params): extract::Query<ClickParams>,
+ Extension(state): Extension<Arc<State>>,
+) {
+ if let Some(q) = state.improvement_queue.as_ref() {
+ q.lock().await.push(ImprovementEvent::Click {
+ qid: params.qid,
+ idx: params.click,
+ })
+ }
+}
+
+impl From<StoreParams> for StoredQuery {
+ fn from(params: StoreParams) -> Self {
+ StoredQuery::new(
+ params.query,
+ params.urls.into_iter().map(|s| s.into()).collect(),
+ )
+ }
+}
+
+#[allow(clippy::unused_async)]
+pub async fn store(
+ extract::Json(params): extract::Json<StoreParams>,
+ Extension(state): Extension<Arc<State>>,
+) -> impl IntoResponse {
+ match state.improvement_queue.as_ref() {
+ Some(q) => {
+ let query: StoredQuery = params.into();
+ let qid = *query.qid();
+ q.lock().await.push(ImprovementEvent::StoreQuery(query));
+
+ qid.to_string()
+ }
+ None => String::new(),
+ }
+}
+
+#[allow(clippy::unused_async)]
+pub async fn settings() -> impl IntoResponse {
+ let template = SettingsTemplate {};
+
+ HtmlTemplate(template)
+}
+
+#[derive(Template)]
+#[template(path = "settings/privacy/index.html")]
+struct SettingsTemplate {}
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index a3fd4a3c..03c9b289 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -15,12 +15,15 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use axum::{body::Body, routing::get_service, Extension, Router};
+use tokio::sync::Mutex;
use tower_http::{compression::CompressionLayer, services::ServeDir};
use crate::{
autosuggest::Autosuggest,
bangs::Bangs,
+ leaky_queue::LeakyQueue,
qa_model::QaModel,
+ query_store::{self, ImprovementEvent},
ranking::models::cross_encoder::CrossEncoderModel,
searcher::{DistributedSearcher, Shard},
summarizer::Summarizer,
@@ -39,6 +42,7 @@ use axum::{
mod about;
mod autosuggest;
+mod improvement;
mod index;
mod metrics;
mod opensearch;
@@ -55,6 +59,7 @@ pub struct State {
pub autosuggest: Autosuggest,
pub search_counter: crate::metrics::Counter,
pub summarizer: Arc<Summarizer>,
+ pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,
}
impl<T> IntoResponse for HtmlTemplate<T>
@@ -100,6 +105,15 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)
None => None,
};
+ let query_store_queue = config.query_store_db_host.clone().map(|db_host| {
+ let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));
+ tokio::spawn(query_store::store_queries_loop(
+ query_store_queue.clone(),
+ db_host,
+ ));
+ query_store_queue
+ });
+
let bangs = Bangs::from_path(&config.bangs_path);
let searcher = DistributedSearcher::new(shards, crossencoder, qa_model, bangs);
@@ -108,6 +122,7 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)
autosuggest,
search_counter,
summarizer: Arc::new(Summarizer::open(&config.summarizer_path)?),
+ improvement_queue: query_store_queue,
});
Ok(Router::new()
@@ -121,8 +136,11 @@ pub fn router(config: &FrontendConfig, search_counter: crate::metrics::Counter)
.route("/settings", get(optics::route))
.route("/settings/optics", get(optics::route))
.route("/settings/sites", get(sites::route))
+ .route("/settings/privacy", get(improvement::settings))
.route("/privacy-and-happy-lawyers", get(privacy::route))
.route("/opensearch.xml", get(opensearch::route))
+ .route("/improvement/click", post(improvement::click))
+ .route("/improvement/store", post(improvement::store))
.fallback(get_service(ServeDir::new("frontend/dist/")).handle_error(
|error: std::io::Error| async move {
(
diff --git a/core/src/leaky_queue.rs b/core/src/leaky_queue.rs
new file mode 100644
index 00000000..d5389c6a
--- /dev/null
+++ b/core/src/leaky_queue.rs
@@ -0,0 +1,68 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::VecDeque;
+
+pub struct LeakyQueue<T> {
+ inner: VecDeque<T>,
+ max_size: usize,
+}
+
+impl<T> LeakyQueue<T> {
+ pub fn new(max_size: usize) -> Self {
+ Self {
+ inner: VecDeque::with_capacity(max_size),
+ max_size,
+ }
+ }
+
+ pub fn push(&mut self, item: T) {
+ while self.inner.len() >= self.max_size {
+ self.inner.pop_front();
+ }
+
+ self.inner.push_back(item);
+ }
+
+ pub fn pop(&mut self) -> Option<T> {
+ self.inner.pop_front()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple() {
+ let mut q = LeakyQueue::new(3);
+
+ q.push(1);
+
+ assert_eq!(q.pop(), Some(1));
+ assert_eq!(q.pop(), None);
+
+ q.push(1);
+ q.push(2);
+ q.push(3);
+ q.push(4);
+
+ assert_eq!(q.pop(), Some(2));
+ assert_eq!(q.pop(), Some(3));
+ assert_eq!(q.pop(), Some(4));
+ assert_eq!(q.pop(), None);
+ }
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 7b00c810..33229cef 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -49,10 +49,12 @@ pub mod index;
mod intmap;
mod kahan_sum;
mod kv;
+mod leaky_queue;
mod metrics;
pub mod prehashed;
mod qa_model;
mod query;
+mod query_store;
pub mod ranking;
mod schema;
mod search_ctx;
@@ -168,6 +170,7 @@ pub struct FrontendConfig {
pub bangs_path: String,
pub summarizer_path: String,
pub search_servers: Vec<Vec<String>>,
+ pub query_store_db_host: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/core/src/query_store.rs b/core/src/query_store.rs
new file mode 100644
index 00000000..2448d1b8
--- /dev/null
+++ b/core/src/query_store.rs
@@ -0,0 +1,180 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{sync::Arc, time::Duration};
+
+use chrono::{DateTime, Timelike, Utc};
+use scylla::{prepared_statement::PreparedStatement, SessionBuilder};
+use thiserror::Error;
+use tokio::{sync::Mutex, time};
+use uuid::Uuid;
+
+use crate::{leaky_queue::LeakyQueue, webpage::Url};
+
+#[derive(Debug, Error)]
+enum Error {
+ #[error("scylla query")]
+ ScyllaQuery(#[from] scylla::transport::errors::QueryError),
+
+ #[error("scylla new session")]
+ ScyllaNewSess(#[from] scylla::transport::errors::NewSessionError),
+}
+
+/// Note that we don't store any information that can be used to link
+/// the query and result back to the user performing the query. This is extremely important!
+#[derive(Debug)]
+pub struct StoredQuery {
+ qid: Uuid,
+ query: String,
+ result_urls: Vec<Url>,
+ timestamp: Option<DateTime<Utc>>, // it is extremely important that we strip hours, minutes, seconds and nanoseconds here for privacy
+}
+
+impl StoredQuery {
+ pub fn new(query: String, urls: Vec<Url>) -> Self {
+ let timestamp = Utc::now()
+ .with_minute(0)
+ .and_then(|t| t.with_minute(0))
+ .and_then(|t| t.with_second(0))
+ .and_then(|t| t.with_nanosecond(0));
+
+ let qid = Uuid::new_v4();
+
+ Self {
+ qid,
+ query,
+ result_urls: urls,
+ timestamp,
+ }
+ }
+
+ pub fn qid(&self) -> &Uuid {
+ &self.qid
+ }
+}
+
+pub enum ImprovementEvent {
+ StoreQuery(StoredQuery),
+ Click { qid: Uuid, idx: usize },
+}
+
+async fn dump_queue(queue: &Mutex<LeakyQueue<ImprovementEvent>>) -> Vec<ImprovementEvent> {
+ let mut res = Vec::new();
+ let mut lock = queue.lock().await;
+
+ while let Some(query) = lock.pop() {
+ res.push(query);
+ }
+
+ res
+}
+
+pub async fn store_queries_loop(
+ queue: Arc<Mutex<LeakyQueue<ImprovementEvent>>>,
+ scylla_host: String,
+) {
+ let scylla = ScyllaConn::new(scylla_host.as_str()).await.unwrap();
+ let mut interval = time::interval(Duration::from_secs(30));
+
+ loop {
+ interval.tick().await;
+ let events = dump_queue(&queue).await;
+
+ for event in events {
+ match event {
+ ImprovementEvent::StoreQuery(query) => scylla.insert(query).await,
+ ImprovementEvent::Click { qid, idx } => {
+ if let Ok(idx) = idx.try_into() {
+ scylla.store_click(qid, idx).await
+ }
+ }
+ }
+ }
+ }
+}
+
+struct ScyllaConn {
+ session: scylla::Session,
+ prepared_insert: PreparedStatement,
+ prepared_click: PreparedStatement,
+}
+
+impl ScyllaConn {
+ async fn new(seed_node: &str) -> Result<Self, Error> {
+ let session = SessionBuilder::new().known_node(seed_node).build().await?;
+
+ session.query("CREATE KEYSPACE IF NOT EXISTS ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}", &[]).await?;
+ session
+ .query(
+ "CREATE TABLE IF NOT EXISTS ks.queries (qid uuid, query text, urls text, timestamp timestamp, primary key (qid, timestamp))",
+ &[],
+ )
+ .await?;
+ session
+ .query(
+ "CREATE TABLE IF NOT EXISTS ks.clicks (qid uuid, click tinyint, primary key (qid))",
+ &[],
+ )
+ .await?;
+
+ let prepared_insert: PreparedStatement = session
+ .prepare("INSERT INTO ks.queries (qid, query, urls, timestamp) VALUES(?, ?, ?, ?)")
+ .await?;
+ let prepared_click: PreparedStatement = session
+ .prepare("INSERT INTO ks.clicks (qid, click) VALUES(?, ?)")
+ .await?;
+
+ Ok(Self {
+ session,
+ prepared_insert,
+ prepared_click,
+ })
+ }
+
+ async fn insert(&self, query: StoredQuery) {
+ let urls = serde_json::to_string(&query.result_urls).unwrap();
+ let timestamp = query
+ .timestamp
+ .map(|timestamp| chrono::Duration::seconds(timestamp.timestamp()))
+ .unwrap_or_else(|| chrono::Duration::seconds(0));
+ let qid = query.qid;
+
+ let res = self
+ .session
+ .execute(
+ &self.prepared_insert,
+ (
+ qid,
+ query.query,
+ urls,
+ scylla::frame::value::Timestamp(timestamp),
+ ),
+ )
+ .await;
+
+ if let Err(err) = res {
+ tracing::error!("scylla insert error: {err}");
+ }
+ }
+
+ async fn store_click(&self, qid: Uuid, idx: i8) {
+ let res = self.session.execute(&self.prepared_click, (qid, idx)).await;
+
+ if let Err(err) = res {
+ tracing::error!("scylla store_click error: {err}");
+ }
+ }
+}
diff --git a/frontend/src/components/SettingsMenu.astro b/frontend/src/components/SettingsMenu.astro
index 92f29afc..8cc051fa 100644
--- a/frontend/src/components/SettingsMenu.astro
+++ b/frontend/src/components/SettingsMenu.astro
@@ -2,6 +2,7 @@
const links = [
{ url: "/settings", title: "Manage Optics" },
{ url: "/settings/sites", title: "Site Rankings" },
+ { url: "/settings/privacy", title: "Privacy" },
];
---
diff --git a/frontend/src/layouts/Settings.astro b/frontend/src/layouts/Settings.astro
new file mode 100644
index 00000000..564810ef
--- /dev/null
+++ b/frontend/src/layouts/Settings.astro
@@ -0,0 +1,20 @@
+---
+import Footer from "../components/Footer.astro";
+import Layout from "./Layout.astro";
+import Header from "../components/Header.astro";
+import SettingsMenu from "../components/SettingsMenu.astro";
+---
+<Layout title="Cuely">
+ <div class="flex h-full w-full flex-col">
+ <Header />
+
+ <div class="flex h-fit w-full justify-center pt-10">
+ <SettingsMenu />
+ <div class="flex w-full max-w-2xl flex-col">
+ <slot />
+ </div>
+ </div>
+ <div class="flex flex-grow"></div>
+ <Footer />
+ </div>
+</Layout>
\ No newline at end of file
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index 9e242278..713f35af 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -97,14 +97,16 @@ import { askama } from "../askama";
<a
class="url truncate text-gray-800 hover:no-underline"
href="{{ item.url }}"
+ onclick="improvement({{ loop.index0 }})"
>
{askama`item.pretty_url $ {{internet.url}}`}
</a>
</div>
<a
- class="title-link truncate text-xl font-medium"
+ class="title-link sr-title-link truncate text-xl font-medium"
title="{{ item.title }}"
href="{{ item.url }}"
+ onclick="improvement({{ loop.index0 }})"
>
{askama`item.title $ {{lorem.sentence}}`}
</a>
@@ -223,8 +225,39 @@ import { askama } from "../askama";
<script is:inline src="/js/highlight.min.js"></script>
<script is:inline>
var rankingAdjustButtons = document.querySelectorAll(".adjust-btn");
+ var titleLinks = document.querySelectorAll(".sr-title-link");
var modal = document.getElementById("modal");
+ var searchParams = new URLSearchParams(window.location.search);
+ var query = searchParams.get("q");
+
+ const allowStats = window.localStorage.getItem("allowStats") === "true";
+
+ var qid = null;
+
+ var improvementSent = new Set();
+
+ var urls = [];
+ for (const elem of titleLinks) {
+ urls.push(elem.href);
+ }
+
+ if (allowStats) {
+ fetch("/improvement/store", {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ },
+ body: JSON.stringify({
+ "query": query,
+ "urls": urls
+ })
+ }).then((response) => response.text()).then((data) => {
+ console.log(data);
+ qid = data;
+ });
+ }
+
document.addEventListener("click", (elem) => {
modal.classList.remove("modal-open");
modal.classList.add("modal-closed");
@@ -255,6 +288,13 @@ import { askama } from "../askama";
}, 0);
});
});
+
+ function improvement(idx) {
+ if (allowStats && qid && !improvementSent.has(idx)) {
+ window.navigator.sendBeacon("/improvement/click?qid=" + qid + "&click=" + idx);
+ improvementSent.add(idx);
+ }
+ }
</script>
<script is:inline>
hljs.highlightAll();
diff --git a/frontend/src/pages/settings/index.astro b/frontend/src/pages/settings/index.astro
index 3a0dc059..5f9cc1d6 100644
--- a/frontend/src/pages/settings/index.astro
+++ b/frontend/src/pages/settings/index.astro
@@ -1,91 +1,79 @@
---
-import Footer from "../../components/Footer.astro";
-import Layout from "../../layouts/Layout.astro";
-import Header from "../../components/Header.astro";
-import SettingsMenu from "../../components/SettingsMenu.astro";
+import SettingsLayout from "../../layouts/Settings.astro";
import { askama } from "../../askama";
---
-<Layout title="Cuely">
- <div class="flex h-full w-full flex-col">
- <Header />
-
- <div class="flex h-fit w-full justify-center pt-10">
- <SettingsMenu />
- <div class="flex max-w-2xl flex-col">
- <h1 class="text-2xl font-medium">Manage Optics</h1>
- <div class="mt-5 text-sm">
- Optics lets you control almost everything about which search results
- that gets returned to you. You can discard results from specific
- sites, boost results from other sites and much more.
- </div>
- <div class="mt-3 text-sm">
- See our <a
- href="https://github.com/Cuely/sample-optics/blob/main/quickstart.optic"
- >quickstart</a
- > for how to get started. Once you have developed your optic, you can add
- it here to be used during your search.
- </div>
- <div class="mt-3 text-sm">
- Simply host the optic on a url that returns a plain-text HTTP response
- with the optic. We use raw.githubusercontent.com, but you are free to
- host them elsewhere.
- </div>
- <div
- class="mt-16"
- x-data="{ items: $persist([]).as('optics'), name: '', url: '' }"
+<SettingsLayout>
+ <div>
+ <h1 class="text-2xl font-medium">Manage Optics</h1>
+ <div class="mt-5 text-sm">
+ Optics lets you control almost everything about which search results
+ that gets returned to you. You can discard results from specific
+ sites, boost results from other sites and much more.
+ </div>
+ <div class="mt-3 text-sm">
+ See our <a
+ href="https://github.com/Cuely/sample-optics/blob/main/quickstart.optic"
+ >quickstart</a
+ > for how to get started. Once you have developed your optic, you can add
+ it here to be used during your search.
+ </div>
+ <div class="mt-3 text-sm">
+ Simply host the optic on a url that returns a plain-text HTTP response
+ with the optic. We use raw.githubusercontent.com, but you are free to
+ host them elsewhere.
+ </div>
+ <div
+ class="mt-16"
+ x-data="{ items: $persist([]).as('optics'), name: '', url: '' }"
+ >
+ <div class="flex w-full justify-between pl-5 pr-5">
+ <input id="name" type="text" placeholder="Name" x-model="name" />
+ <input id="url" type="text" placeholder="Url" x-model="url" />
+ <button
+ x-on:click="if (name != '' && url != '') {items.push({ name, url }); name = url = ''}"
+ class="h-10 w-20 rounded-full border-0 bg-brand text-sm text-white"
+ >Add</button
>
- <div class="flex w-full justify-between pl-5 pr-5">
- <input id="name" type="text" placeholder="Name" x-model="name" />
- <input id="url" type="text" placeholder="Url" x-model="url" />
- <button
- x-on:click="items.push({ name, url }); name = url = ''"
- class="h-10 w-20 rounded-full border-0 bg-brand text-sm text-white"
- >Add</button
- >
+ </div>
+ <div class="mt-5">
+ <div class="grid w-full space-y-5" id="optics-list">
+ <div class="flex">
+ <div class="w-10"></div>
+ <div class="flex-1 font-medium">Name</div>
+ <div class="flex-1 font-medium">Url</div>
</div>
- <div class="mt-5">
- <div class="grid w-full space-y-5" id="optics-list">
+ {
+ askama.for_("optic in default_optics", () => (
<div class="flex">
- <div class="w-10"></div>
- <div class="flex-1 font-medium">Name</div>
- <div class="flex-1 font-medium">Url</div>
- </div>
- {
- askama.for_("optic in default_optics", () => (
- <div class="flex">
- <div class="mr-5 w-5">
- <img src="/images/disabled-delete.svg" class="h-5 w-5" />
- </div>
- <div class="flex-1 text-sm">{askama`optic.name $ {{random.words}}`}</div>
- <div class="flex-1 text-sm">{askama`optic.url $ {{internet.url}}`}</div>
- </div>
- ))
- }
- <template x-for="(item, index) in items">
- <div class="flex">
- <div class="w-10">
- <img
- src="/images/delete.svg"
- class="h-5 w-5 hover:cursor-pointer"
- x-on:click="items.splice(index, 1)"
- />
- </div>
- <div class="user-optic flex-1 text-sm" x-text="item.name">
- </div>
- <div class="user-optic flex-1 text-sm" x-text="item.url">
- </div>
+ <div class="mr-5 w-5">
+ <img src="/images/disabled-delete.svg" class="h-5 w-5" />
</div>
- </template>
+ <div class="flex-1 text-sm">{askama`optic.name $ {{random.words}}`}</div>
+ <div class="flex-1 text-sm">{askama`optic.url $ {{internet.url}}`}</div>
+ </div>
+ ))
+ }
+ <template x-for="(item, index) in items">
+ <div class="flex">
+ <div class="w-10">
+ <img
+ src="/images/delete.svg"
+ class="h-5 w-5 hover:cursor-pointer"
+ x-on:click="items.splice(index, 1)"
+ />
+ </div>
+ <div class="user-optic flex-1 text-sm" x-text="item.name">
+ </div>
+ <div class="user-optic flex-1 text-sm" x-text="item.url">
+ </div>
</div>
- </div>
+ </template>
</div>
</div>
</div>
- <div class="flex flex-grow"></div>
- <Footer />
</div>
-</Layout>
+</SettingsLayout>
<script>
import "$/alpine";
diff --git a/frontend/src/pages/settings/privacy.astro b/frontend/src/pages/settings/privacy.astro
new file mode 100644
index 00000000..69b939a2
--- /dev/null
+++ b/frontend/src/pages/settings/privacy.astro
@@ -0,0 +1,37 @@
+---
+import SettingsLayout from "../../layouts/Settings.astro";
+---
+<SettingsLayout>
+ <div class="flex flex-col space-y-2">
+ <h1 class="text-2xl font-medium">Privacy</h1>
+ <div class="text-sm">
+ By default, some data about your searches are stored on our servers. We primarily store the text you used for the search, and which results (if any) you clicked on.
+ This data immensely helps us improve everyones search experience by training our ranking models.
+ </div>
+
+ <div class="text-sm">
+ It is important to note, that <b class="font-extrabold">no</b> personal information is stored about your search. We don't store your IP, browser fingerprint or even a precise timestamp.
+ In fact, you can see exactly what gets stored <a href="#">here</a>.
+ </div>
+
+ <div class="text-sm">
+ We hope you feel comfortable sharing this data with us as this makes the results much better for everyone, but we also respect if you don't.
+ You can opt-out at any point by toggling the checkbox below and we wont store anything at all about your searches.
+ </div>
+
+ <div class="flex space-x-1 pl-1 text-sm" x-data="{ allowStats: $persist(true).as('allowStats') }">
+ <input type="checkbox" id="allowStats" x-model="allowStats">
+ <label for="allowStats">Allow usage statistics</label>
+ </div>
+ </div>
+</SettingsLayout>
+
+<style>
+ a {
+ @apply text-blue-600;
+ }
+</style>
+
+<script>
+ import "$/alpine";
+</script>
\ No newline at end of file
diff --git a/frontend/src/pages/settings/sites.astro b/frontend/src/pages/settings/sites.astro
index 71aed401..a23828cb 100644
--- a/frontend/src/pages/settings/sites.astro
+++ b/frontend/src/pages/settings/sites.astro
@@ -1,73 +1,58 @@
---
-import Footer from "../../components/Footer.astro";
-import Layout from "../../layouts/Layout.astro";
-import Header from "../../components/Header.astro";
-import SettingsMenu from "../../components/SettingsMenu.astro";
+import SettingsLayout from "../../layouts/Settings.astro";
---
+<SettingsLayout>
+ <div>
+ <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>
+ <div class="text-sm">
+ These sites receive a boost during search. Results from these and similar sites
+ are more likely to appear in your search results.
+ </div>
+ <div class="mt-5">
+ <div
+ class="grid w-full grid-cols-[auto_1fr] space-y-5"
+ id="liked-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
+ </div>
+ </div>
+
+ <div>
+ <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>
+ <div class="text-sm">
+ These sites get de-prioritized during search. Results from these and similar
+ sites are less likely to appear in your search results.
+ </div>
+ <div class="mt-5">
+ <div
+ class="grid w-full grid-cols-[auto_1fr] space-y-5"
+ id="disliked-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
+ </div>
+ </div>
+ </div>
-<Layout title="Cuely">
- <div class="flex h-full w-full flex-col">
- <Header />
-
- <div class="flex h-fit w-full justify-center pt-10">
- <SettingsMenu />
- <div class="flex w-full max-w-2xl flex-col space-y-10">
- <div>
- <h1 class="mb-2 text-2xl font-medium">Liked Sites</h1>
- <div class="text-sm">
- These sites receive a boost during search. Results from these sites
- are more likely to appear in your search results.
- </div>
- <div class="mt-5">
- <div
- class="grid w-full grid-cols-[auto_1fr] space-y-5"
- id="liked-list"
- >
- <div class="w-10"></div>
- <div class="font-medium">Site Name</div>
- </div>
- </div>
- </div>
-
- <div>
- <h1 class="mb-2 text-2xl font-medium">Disliked Sites</h1>
- <div class="text-sm">
- These sites get de-prioritized during search. Results from these
- sites are less likely to appear in your search results.
- </div>
- <div class="mt-5">
- <div
- class="grid w-full grid-cols-[auto_1fr] space-y-5"
- id="disliked-list"
- >
- <div class="w-10"></div>
- <div class="font-medium">Site Name</div>
- </div>
- </div>
- </div>
-
- <div>
- <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>
- <div class="text-sm">
- These are the sites you have blocked. They won't appear in any of
- your searches.
- </div>
- <div class="mt-5">
- <div
- class="grid w-full grid-cols-[auto_1fr] space-y-5"
- id="blocked-list"
- >
- <div class="w-10"></div>
- <div class="font-medium">Site Name</div>
- </div>
- </div>
- </div>
+ <div>
+ <h1 class="mb-2 text-2xl font-medium">Blocked Sites</h1>
+ <div class="text-sm">
+ These are the sites you have blocked. They won't appear in any of
+ your searches.
+ </div>
+ <div class="mt-5">
+ <div
+ class="grid w-full grid-cols-[auto_1fr] space-y-5"
+ id="blocked-list"
+ >
+ <div class="w-10"></div>
+ <div class="font-medium">Site Name</div>
</div>
</div>
- <div class="flex flex-grow"></div>
- <Footer />
</div>
-</Layout>
+</SettingsLayout>
<script is:inline>
let sites = JSON.parse(