Skip to content

web search done right

metric to probabilistically estimate number of daily unique users

Mikkel Denkere7fa85f523d1
diff --git a/Cargo.lock b/Cargo.lock
index ef422264..9cd8bbed 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3620,6 +3620,20 @@ dependencies = [
"thiserror",
]
+[[package]]
+name = "ring"
+version = "0.17.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9babe80d5c16becf6594aa32ad2be8fe08498e7ae60b77de8df700e67f191d7e"
+dependencies = [
+ "cc",
+ "getrandom 0.2.10",
+ "libc",
+ "spin",
+ "untrusted",
+ "windows-sys 0.48.0",
+]
+
[[package]]
name = "rio_api"
version = "0.8.4"
@@ -4314,6 +4328,7 @@ dependencies = [
"regex",
"reqwest",
"reqwest-eventsource",
+ "ring",
"rio_api",
"rio_turtle",
"rkyv",
@@ -5179,6 +5194,12 @@ dependencies = [
"subtle",
]
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
[[package]]
name = "url"
version = "2.4.0"
diff --git a/Cargo.toml b/Cargo.toml
index 4ae57dfb..3db2bda9 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -69,6 +69,7 @@ rayon = "1.5.3"
regex = "1.6.0"
reqwest = {version = "0.11.16", features = ["blocking", "stream", "json"]}
reqwest-eventsource = "0.4.0"
+ring = "0.17.3"
rio_api = "0.8.4"
rio_turtle = "0.8.4"
rkyv = {version = "0.7.42", default_features = false, features = [
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 1dc0e1e4..40ebdc5b 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -75,6 +75,7 @@ rayon = {workspace = true}
regex = {workspace = true}
reqwest = {workspace = true}
reqwest-eventsource = {workspace = true}
+ring = {workspace = true}
rio_api = {workspace = true}
rio_turtle = {workspace = true}
rkyv = {workspace = true}
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index c5f6c008..a8c98575 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -38,7 +38,7 @@ use crate::{
summarizer::Summarizer,
};
use anyhow::Result;
-use std::sync::Arc;
+use std::{net::SocketAddr, sync::Arc};
use axum::{
http::StatusCode,
@@ -59,12 +59,14 @@ mod metrics;
pub mod search;
mod sites;
mod summarize;
+pub mod user_count;
mod webgraph;
pub struct Counters {
pub search_counter_success: crate::metrics::Counter,
pub search_counter_fail: crate::metrics::Counter,
pub explore_counter: crate::metrics::Counter,
+ pub daily_active_users: user_count::UserCount<user_count::Daily>,
}
pub struct State {
@@ -215,9 +217,25 @@ pub fn metrics_router(registry: crate::metrics::PrometheusRegistry) -> Router {
async fn search_metric<B>(
extract::State(state): extract::State<Arc<State>>,
+ extract::ConnectInfo(addr): extract::ConnectInfo<SocketAddr>,
request: axum::http::Request<B>,
next: middleware::Next<B>,
) -> Response {
+ // It is very important that the ip address is not stored. It is only used
+ // for a probabilistic estimate of the number of unique users using a hyperloglog datastructure.
+ let mut ip = None;
+ if let Some(forwarded_for) = request.headers().get("x-forwarded-for") {
+ let forwarded_for = forwarded_for.to_str().unwrap_or_default();
+ if let Some(client_ip) = forwarded_for.split(',').next() {
+ if let Ok(client_ip) = client_ip.trim().parse::<SocketAddr>() {
+ ip = Some(client_ip.ip());
+ }
+ }
+ }
+
+ let ip = ip.unwrap_or_else(|| addr.ip());
+ state.counters.daily_active_users.inc(&ip).ok();
+
let response = next.run(request).await;
if response.status().is_success() {
diff --git a/core/src/api/user_count.rs b/core/src/api/user_count.rs
new file mode 100644
index 00000000..c602c7f0
--- /dev/null
+++ b/core/src/api/user_count.rs
@@ -0,0 +1,140 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use anyhow::{anyhow, Result};
+use chrono::{NaiveDateTime, Utc};
+use parking_lot::Mutex;
+use ring::rand::SecureRandom;
+use ring::{digest, pbkdf2, rand};
+
+use crate::hyperloglog::HyperLogLog;
+use crate::metrics::Counter;
+
+pub trait Frequency: Clone + Copy + Default {
+ fn next_reset(&self) -> NaiveDateTime;
+}
+
+#[derive(Clone, Copy, Default)]
+pub struct Daily;
+#[derive(Clone, Copy, Default)]
+pub struct Monthly;
+
+impl Frequency for Daily {
+ fn next_reset(&self) -> NaiveDateTime {
+ Utc::now()
+ .date_naive()
+ .succ_opt()
+ .unwrap()
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ }
+}
+
+impl Frequency for Monthly {
+ fn next_reset(&self) -> NaiveDateTime {
+ (Utc::now().date_naive() + chrono::Duration::days(31))
+ .and_hms_opt(0, 0, 0)
+ .unwrap()
+ }
+}
+
+struct InnerUserCount<F: Frequency> {
+ frequency: F,
+ metric: Counter,
+ next_reset: NaiveDateTime,
+ counter: HyperLogLog<131_072>, // 2^17
+ salt: [u8; digest::SHA512_OUTPUT_LEN],
+}
+
+impl<F: Frequency> InnerUserCount<F> {
+ fn new() -> Result<InnerUserCount<F>> {
+ let frequency = F::default();
+ let rng = rand::SystemRandom::new();
+ let mut salt = [0u8; digest::SHA512_OUTPUT_LEN];
+ rng.fill(&mut salt)
+ .map_err(|_| anyhow!("failed to generate salt"))?;
+
+ Ok(Self {
+ frequency,
+ next_reset: frequency.next_reset(),
+ counter: HyperLogLog::default(),
+ metric: Counter::default(),
+ salt,
+ })
+ }
+
+ fn inc<T: serde::Serialize>(&mut self, user_id: &T) -> Result<()> {
+ // It is important that we do not store the user_id as it might be sensitive.
+ // It is first hashed with a salt for good measure, and then we only use
+ // the id for a probabilistic count using hyperloglog.
+ self.maybe_reset();
+ let bytes = bincode::serialize(user_id)?;
+ let mut hash = [0u8; digest::SHA512_OUTPUT_LEN];
+
+ pbkdf2::derive(
+ pbkdf2::PBKDF2_HMAC_SHA512,
+ 100.try_into().unwrap(),
+ &self.salt,
+ &bytes,
+ &mut hash,
+ );
+
+ let mut user_id = [0u8; 8];
+ user_id.copy_from_slice(&hash[..8]);
+
+ self.counter.add(u64::from_le_bytes(user_id));
+ self.metric.store(self.counter.size() as u64);
+
+ Ok(())
+ }
+
+ fn maybe_reset(&mut self) {
+ if Utc::now().naive_utc() >= self.next_reset {
+ self.reset();
+ }
+ }
+
+ fn reset(&mut self) {
+ self.counter.clear();
+ self.next_reset = self.frequency.next_reset();
+ }
+
+ fn metric(&self) -> Counter {
+ self.metric.clone()
+ }
+}
+
+pub struct UserCount<F: Frequency> {
+ inner: Arc<Mutex<InnerUserCount<F>>>,
+}
+
+impl<F: Frequency> UserCount<F> {
+ pub fn new() -> Result<UserCount<F>> {
+ Ok(Self {
+ inner: Arc::new(Mutex::new(InnerUserCount::new()?)),
+ })
+ }
+
+ pub fn inc<T: serde::Serialize>(&self, user_id: &T) -> Result<()> {
+ self.inner.lock().inc(user_id)
+ }
+
+ pub fn metric(&self) -> Counter {
+ self.inner.lock().metric()
+ }
+}
diff --git a/core/src/entrypoint/api.rs b/core/src/entrypoint/api.rs
index e0c03864..9a6231fb 100644
--- a/core/src/entrypoint/api.rs
+++ b/core/src/entrypoint/api.rs
@@ -14,10 +14,12 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use std::net::SocketAddr;
+
use anyhow::Result;
use crate::{
- api::{metrics_router, router, Counters},
+ api::{metrics_router, router, user_count, Counters},
config,
metrics::Label,
};
@@ -26,6 +28,7 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {
let search_counter_success = crate::metrics::Counter::default();
let search_counter_fail = crate::metrics::Counter::default();
let explore_counter = crate::metrics::Counter::default();
+ let daily_active_users = user_count::UserCount::new()?;
let mut registry = crate::metrics::PrometheusRegistry::default();
@@ -59,10 +62,19 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {
.unwrap();
group.register(explore_counter.clone(), vec![]);
+ let group = registry
+ .new_group(
+ "daily_active_users".to_string(),
+ Some("Approximate number of unique daily active users.".to_string()),
+ )
+ .unwrap();
+ group.register(daily_active_users.metric(), vec![]);
+
let counters = Counters {
search_counter_success,
search_counter_fail,
explore_counter,
+ daily_active_users,
};
let app = router(&config, counters).await?;
@@ -70,7 +82,8 @@ pub async fn run(config: config::ApiConfig) -> Result<()> {
let addr = config.host;
tracing::info!("api server listening on {}", addr);
- let server = axum::Server::bind(&addr).serve(app.into_make_service());
+ let server =
+ axum::Server::bind(&addr).serve(app.into_make_service_with_connect_info::<SocketAddr>());
let addr = config.prometheus_host;
tracing::info!("prometheus exporter listening on {}", addr);
diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rs
index c7a14e93..0984dc45 100644
--- a/core/src/hyperloglog.rs
+++ b/core/src/hyperloglog.rs
@@ -4345,6 +4345,10 @@ impl<const N: usize> HyperLogLog<N> {
self.registers[j] = self.registers[j].max(p as u8);
}
+ pub fn clear(&mut self) {
+ self.registers.iter_mut().for_each(|r| *r = 0);
+ }
+
pub fn estimate_bias(&self, e: f64, b: usize) -> f64 {
// binary search first nearest neighbor
let lookup_array = RAW_ESTIMATE_DATA_VEC[b - 1 - RAW_ESTIMATE_DATA_OFFSET];
@@ -4525,4 +4529,17 @@ mod tests {
assert_eq!(a.registers, without_merge.registers);
}
+
+ #[test]
+ fn accurate_counts() {
+ let mut set: HyperLogLog<65_536> = HyperLogLog::default();
+ let mut counter = 0;
+
+ for item in 0..1_000 {
+ set.add(item);
+ counter += 1;
+
+ assert!((set.size() as f64 - counter as f64).abs() <= 10.0);
+ }
+ }
}
diff --git a/core/src/metrics.rs b/core/src/metrics.rs
index dab806e8..dd25551e 100644
--- a/core/src/metrics.rs
+++ b/core/src/metrics.rs
@@ -39,6 +39,10 @@ impl Counter {
pub fn inc(&self) {
self.0.fetch_add(1, Ordering::SeqCst);
}
+
+ pub fn store(&self, val: u64) {
+ self.0.store(val, Ordering::SeqCst);
+ }
}
pub enum PrometheusMetric {
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index d9a404d5..980d6396 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -251,6 +251,10 @@ fn score_region(webpage_region: Region, aggregator: &SignalAggregator) -> f64 {
}
fn bm25(field: &mut TextFieldData, doc: DocId) -> f64 {
+ if field.postings.is_empty() {
+ return 0.0;
+ }
+
let mut term_freq = 0;
for posting in &mut field.postings {
if posting.doc() == doc || (posting.doc() < doc && posting.seek(doc) == doc) {
@@ -294,12 +298,12 @@ impl Signal {
Signal::Bm25DomainIfHomepageNoTokenizer => 0.007,
Signal::Bm25TitleIfHomepage => 0.00002,
Signal::Bm25BacklinkText => 0.003,
- Signal::Bm25Description => 0.00001,
- Signal::ProximitySlop0 => 0.01,
- Signal::ProximitySlop1 => 0.01,
- Signal::ProximitySlop2 => 0.01,
- Signal::ProximitySlop4 => 0.01,
- Signal::ProximitySlop8 => 0.01,
+ Signal::Bm25Description => 0.0,
+ Signal::ProximitySlop0 => 0.00,
+ Signal::ProximitySlop1 => 0.00,
+ Signal::ProximitySlop2 => 0.00,
+ Signal::ProximitySlop4 => 0.00,
+ Signal::ProximitySlop8 => 0.00,
Signal::CrossEncoder => 0.17,
Signal::HostCentrality => 1.0,
Signal::PageCentrality => 4.0,