Skip to content

web search done right

Speedup bitvec similarity calculation using hyperloglog. I also tried estimating the intersection directly using a minhash sketch, but I found that the space requirements were quite substancial to get adequate accuracy. It seems a much better trade-off to just filter highly unsimilar vecs using hyperloglog and calculating the remaining manually using the postings.

Mikkel Denker0c463e1f6adb
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 2714ce28..66593336 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -43,12 +43,19 @@ fn download_files() {
"content.rdf.u8.gz",
"english-wordnet-2022-subset.ttl",
] {
+ let path = Path::new(DATA_PATH).join(name);
+
+ if path.exists() {
+ info!("Skipping {}", name);
+ continue;
+ }
+
info!("Downloading {}", name);
let body = reqwest::get(format!("http://s3.trystract.com/{BUCKET_NAME}/{name}"))
.await
.unwrap();
- let mut file = File::create(Path::new(DATA_PATH).join(name)).await.unwrap();
+ let mut file = File::create(path).await.unwrap();
let mut bytes = body.bytes_stream();
while let Some(item) = bytes.next().await {
diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rs
index ad80de1d..6349fc41 100644
--- a/core/src/hyperloglog.rs
+++ b/core/src/hyperloglog.rs
@@ -4295,7 +4295,7 @@ const ONE_OVER_POWER_OF_TWO: [f64; 256] = [
1.727233711018889e-77,
];
-#[derive(Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HyperLogLog<const N: usize> {
registers: Vec<u8>,
}
@@ -4319,7 +4319,7 @@ impl<const N: usize> HyperLogLog<N> {
let m = self.registers.len();
if m >= 128 {
- 0.7213 / (1. + 1.079 / (m as f64))
+ 0.7213 / (1. + (1.079 / (m as f64)))
} else if m >= 64 {
0.709
} else if m >= 32 {
@@ -4338,11 +4338,11 @@ impl<const N: usize> HyperLogLog<N> {
let b = self.b();
let hash = Self::hash(item) as usize;
- let w = hash >> b;
- let j = hash - (w << b);
+ let j = hash >> (64 - b);
+ let w = hash << b;
// p = leftmost bit (1-based count)
- let p = w.leading_zeros() + 1 - (b as u32);
+ let p = w.leading_zeros() + 1;
self.registers[j] = self.registers[j].max(p as u8);
}
@@ -4460,9 +4460,15 @@ impl<const N: usize> HyperLogLog<N> {
}
}
- #[cfg(test)]
pub fn relative_error(&self) -> f64 {
- (3f64 * 2f64.ln() - 1f64).sqrt() / (self.registers.len() as f64).sqrt()
+ 1.04 / (self.registers.len() as f64).sqrt()
+ }
+
+ pub fn size_bounds(&self) -> (usize, usize) {
+ let size = self.size();
+ let delta = (self.relative_error() * 2.0 * (size as f64)) as usize;
+
+ (size - delta, size + delta)
}
pub fn merge(&mut self, other: &Self) {
@@ -4471,6 +4477,12 @@ impl<const N: usize> HyperLogLog<N> {
}
}
+ pub fn merge_into(&self, other: &Self, out: &mut Self) {
+ for i in 0..N {
+ out.registers[i] = self.registers[i].max(other.registers[i]);
+ }
+ }
+
pub fn registers(&self) -> &[u8] {
self.registers.as_ref()
}
@@ -4488,9 +4500,7 @@ mod tests {
set.add(item);
}
- let delta = (set.relative_error() * (set.size() as f64)) as usize;
- let lower_bound = set.size() - delta;
- let upper_bound = set.size() + delta;
+ let (lower_bound, upper_bound) = set.size_bounds();
assert!(set.size() > lower_bound && set.size() < upper_bound);
}
@@ -4500,12 +4510,9 @@ mod tests {
for item in 0..10_000 {
set.add(item);
- set.size();
}
- let delta = (set.relative_error() * (set.size() as f64)) as usize;
- let lower_bound = set.size() - delta;
- let upper_bound = set.size() + delta;
+ let (lower_bound, upper_bound) = set.size_bounds();
assert!(set.size() > lower_bound && set.size() < upper_bound);
}
diff --git a/core/src/intmap.rs b/core/src/intmap.rs
index 9d24af28..71dd3750 100644
--- a/core/src/intmap.rs
+++ b/core/src/intmap.rs
@@ -101,7 +101,7 @@ impl<K: Key, V> IntMap<K, V> {
fn grow(&mut self) {
let mut bins = Vec::new();
- for _ in 0..self.bins.len() * 2 {
+ for _ in 0..(self.bins.len() as f64 * 1.5) as usize {
bins.push(Vec::new());
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index 04cf11bc..c9a478f5 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -268,7 +268,7 @@ impl Term {
let processed_terms = Term::process_tantivy_term(term, *field);
if processed_terms.len() == 1 {
- let term = processed_terms.get(0).unwrap().clone();
+ let term = processed_terms.first().unwrap().clone();
(
Occur::Should,
Box::new(TermQuery::new(
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
index 6918d40e..f859db60 100644
--- a/core/src/ranking/bitvec_similarity.rs
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -17,9 +17,15 @@
use itertools::Itertools;
use serde::{Deserialize, Serialize};
+use crate::hyperloglog::HyperLogLog;
+
+const THRESHOLD_SIM_ESTIMATE: f64 = 0.1;
+
+const HYPERLOGLOG_REGISTERS: usize = 128;
+
#[derive(Serialize, Deserialize, Debug, Clone)]
struct VeryJankyBloomFilter {
- data: Vec<u128>,
+ data: Vec<u64>,
}
impl VeryJankyBloomFilter {
@@ -29,15 +35,14 @@ impl VeryJankyBloomFilter {
}
}
- fn hash(&self, item: &u128) -> (usize, u128) {
+ fn hash(&self, item: &u64) -> (usize, u64) {
(
- (item.wrapping_mul(180811355057196146399512320062616506473) % self.data.len() as u128)
- as usize,
- item.wrapping_mul(258349730118558892798889897730042154643) % 128,
+ (item.wrapping_mul(11400714819323198549) % self.data.len() as u64) as usize,
+ item.wrapping_mul(11400714819323198549) % 64,
)
}
- fn insert(&mut self, item: u128) {
+ fn insert(&mut self, item: u64) {
let (a, b) = self.hash(&item);
self.data[a] |= 1 << b;
}
@@ -51,15 +56,20 @@ impl VeryJankyBloomFilter {
}
}
+#[derive(Clone)]
+struct ScratchSpace {
+ hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,
+}
+
#[derive(Serialize, Deserialize, Debug, Clone)]
struct Posting {
- ranks: Vec<u128>,
- skip_pointers: Vec<u128>,
+ ranks: Vec<u64>,
+ skip_pointers: Vec<u64>,
skip_size: usize,
}
impl Posting {
- fn new(ranks: Vec<u128>) -> Self {
+ fn new(ranks: Vec<u64>) -> Self {
let skip_size = (ranks.len() as f64).sqrt() as usize;
let skip_pointers = ranks
@@ -130,33 +140,37 @@ impl Posting {
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct BitVec {
bloom: VeryJankyBloomFilter,
+ hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,
posting: Posting,
sqrt_len: f64,
}
impl BitVec {
- pub fn new(mut ranks: Vec<u128>) -> Self {
+ pub fn new(mut ranks: Vec<u64>) -> Self {
ranks.sort();
ranks.dedup();
ranks.shrink_to_fit();
let len = ranks.len();
let mut bloom = VeryJankyBloomFilter::new(2);
+ let mut hyperloglog = HyperLogLog::default();
for rank in &ranks {
bloom.insert(*rank);
+ hyperloglog.add(*rank);
}
let posting = Posting::new(ranks);
Self {
bloom,
+ hyperloglog,
posting,
sqrt_len: (len as f64).sqrt(),
}
}
- pub fn sim(&self, other: &Self) -> f64 {
+ fn sim(&self, other: &Self, scratchspace: &mut ScratchSpace) -> f64 {
if self.sqrt_len == 0.0 || other.sqrt_len == 0.0 {
return 0.0;
}
@@ -165,14 +179,49 @@ impl BitVec {
return 0.0;
}
- let intersect = self.posting.intersection_size(&other.posting);
+ self.hyperloglog
+ .merge_into(&other.hyperloglog, &mut scratchspace.hyperloglog);
+
+ let union_est = scratchspace.hyperloglog.size();
+
+ let intersect_est = (self.posting.ranks.len() + other.posting.ranks.len())
+ .checked_sub(union_est)
+ .unwrap_or_default();
+
+ let sim_est = (intersect_est as f64) / (self.sqrt_len * other.sqrt_len);
+
+ if sim_est < THRESHOLD_SIM_ESTIMATE {
+ return 0.0;
+ }
+
+ let intersect = self.posting.intersection_size(&other.posting) as f64;
+ intersect / (self.sqrt_len * other.sqrt_len)
+ }
+}
+
+#[derive(Clone)]
+pub struct BitVecSimilarity {
+ scratchspace: ScratchSpace,
+}
- if intersect == 0 {
- 0.0
- } else {
- intersect as f64 / (self.sqrt_len * other.sqrt_len)
+impl Default for BitVecSimilarity {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl BitVecSimilarity {
+ pub fn new() -> Self {
+ Self {
+ scratchspace: ScratchSpace {
+ hyperloglog: HyperLogLog::default(),
+ },
}
}
+
+ pub fn sim(&mut self, a: &BitVec, b: &BitVec) -> f64 {
+ a.sim(b, &mut self.scratchspace)
+ }
}
#[cfg(test)]
@@ -181,11 +230,11 @@ mod tests {
use itertools::Itertools;
use std::iter::repeat;
- fn into_ranks(a: &[bool]) -> Vec<u128> {
+ fn into_ranks(a: &[bool]) -> Vec<u64> {
a.iter()
.enumerate()
.filter(|(_, b)| **b)
- .map(|(i, _)| i as u128)
+ .map(|(i, _)| i as u64)
.collect()
}
@@ -220,7 +269,9 @@ mod tests {
let a = BitVec::new(into_ranks(&a));
let b = BitVec::new(into_ranks(&b));
- assert!((expected - a.sim(&b)).abs() < 0.001);
+ let sim = BitVecSimilarity::default().sim(&a, &b);
+
+ assert!((expected - sim).abs() < 0.1);
}
#[test]
@@ -232,7 +283,9 @@ mod tests {
let a = BitVec::new(into_ranks(&a));
let b = BitVec::new(into_ranks(&b));
- assert_eq!(a.sim(&b), 0.0);
+ let sim = BitVecSimilarity::default().sim(&a, &b);
+
+ assert_eq!(sim, 0.0);
}
#[test]
@@ -243,7 +296,9 @@ mod tests {
let a = BitVec::new(into_ranks(&a));
let b = BitVec::new(into_ranks(&b));
- assert_eq!(a.sim(&b), 0.0);
+ let sim = BitVecSimilarity::default().sim(&a, &b);
+
+ assert_eq!(sim, 0.0);
}
#[test]
@@ -266,6 +321,8 @@ mod tests {
let a = BitVec::new(into_ranks(&a));
let b = BitVec::new(into_ranks(&b));
- assert!((expected - a.sim(&b)).abs() < 0.01);
+ let sim = BitVecSimilarity::default().sim(&a, &b);
+
+ assert!((expected - sim).abs() < 0.1);
}
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 6d070a49..dd49abea 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -33,15 +33,17 @@ use crate::{
use super::bitvec_similarity;
+#[derive(Clone)]
pub struct Scorer {
+ similarity: bitvec_similarity::BitVecSimilarity,
liked: Vec<NodeScorer>,
disliked: Vec<NodeScorer>,
vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,
- cache: DashMap<NodeID, f64>,
+ cache: IntMap<NodeID, f64>,
normalized: bool,
}
-#[derive(Debug)]
+#[derive(Debug, Clone)]
struct NodeScorer {
node: NodeID,
inbound: bitvec_similarity::BitVec,
@@ -61,29 +63,34 @@ impl NodeScorer {
self.self_score = self_score;
}
- fn sim(&self, other: &NodeID, other_inbound: &bitvec_similarity::BitVec) -> f64 {
+ fn sim(
+ &self,
+ other: &NodeID,
+ other_inbound: &bitvec_similarity::BitVec,
+ similarity: &mut bitvec_similarity::BitVecSimilarity,
+ ) -> f64 {
if self.node == *other {
self.self_score
} else {
- self.inbound.sim(other_inbound)
+ similarity.sim(&self.inbound, other_inbound)
}
}
}
impl Scorer {
- fn calculate_score(&self, node: &NodeID) -> f64 {
+ fn calculate_score(&mut self, node: &NodeID) -> f64 {
let s = match self.vectors.get(node) {
Some(vec) => {
(self.disliked.len() as f64)
+ (self
.liked
.iter()
- .map(|liked| liked.sim(node, vec))
+ .map(|liked| liked.sim(node, vec, &mut self.similarity))
.sum::<f64>()
- self
.disliked
.iter()
- .map(|disliked| disliked.sim(node, vec))
+ .map(|disliked| disliked.sim(node, vec, &mut self.similarity))
.sum::<f64>())
}
None => 0.0,
@@ -96,7 +103,7 @@ impl Scorer {
}
.max(0.0)
}
- pub fn score(&self, node: &NodeID) -> f64 {
+ pub fn score(&mut self, node: &NodeID) -> f64 {
if let Some(cached) = self.cache.get(node) {
return *cached;
}
@@ -135,7 +142,7 @@ impl InboundSimilarity {
for (node_id, inbound) in adjacency {
vectors.insert(
node_id,
- bitvec_similarity::BitVec::new(inbound.into_iter().map(|n| n.bit_128()).collect()),
+ bitvec_similarity::BitVec::new(inbound.into_iter().map(|n| n.bit_64()).collect()),
);
}
@@ -163,10 +170,11 @@ impl InboundSimilarity {
.collect();
Scorer {
+ similarity: bitvec_similarity::BitVecSimilarity::default(),
liked,
disliked,
vectors: self.vectors.clone(),
- cache: DashMap::new(),
+ cache: IntMap::new(),
normalized,
}
}
@@ -241,7 +249,7 @@ mod tests {
let inbound = InboundSimilarity::build(&graph);
- let scorer = inbound.scorer(&[Node::from("b.com").id()], &[], false);
+ let mut scorer = inbound.scorer(&[Node::from("b.com").id()], &[], false);
let e = Node::from("e.com").id();
let d = Node::from("d.com").id();
diff --git a/core/src/ranking/query_centrality.rs b/core/src/ranking/query_centrality.rs
index 15e5623a..1551b826 100644
--- a/core/src/ranking/query_centrality.rs
+++ b/core/src/ranking/query_centrality.rs
@@ -18,6 +18,7 @@ use crate::webgraph::NodeID;
use super::inbound_similarity;
+#[derive(Clone)]
pub struct Scorer {
inbound_centrality: inbound_similarity::Scorer,
}
@@ -30,7 +31,7 @@ impl Scorer {
Self { inbound_centrality }
}
- pub fn score(&self, node: NodeID) -> f64 {
+ pub fn score(&mut self, node: NodeID) -> f64 {
self.inbound_centrality.score(&node)
}
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 08e282c4..047e5b44 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -28,6 +28,7 @@ use crate::{
use optics::ast::RankingTarget;
use optics::Optic;
use serde::{Deserialize, Serialize};
+use std::cell::RefCell;
use std::str::FromStr;
use std::sync::Arc;
use tantivy::fieldnorm::FieldNormReader;
@@ -267,7 +268,7 @@ impl Signal {
Signal::UpdateTimestamp => 0.001,
Signal::TrackerScore => 0.02,
Signal::Region => 0.01,
- Signal::InboundSimilarity => 1.0,
+ Signal::InboundSimilarity => 0.25,
Signal::LambdaMART => 10.0,
Signal::UrlSlashes => 0.01,
Signal::UrlDigits => 0.01,
@@ -667,10 +668,10 @@ pub struct SignalAggregator {
query: Option<QueryData>,
query_signal_coefficients: Option<SignalCoefficient>,
segment_reader: Option<SegmentReader>,
- inbound_similarity: Option<Arc<inbound_similarity::Scorer>>,
+ inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,
fetch_time_ms_cache: Vec<f64>,
update_time_cache: Vec<f64>,
- query_centrality: Option<Arc<query_centrality::Scorer>>,
+ query_centrality: Option<RefCell<query_centrality::Scorer>>,
region_count: Option<Arc<RegionCount>>,
selected_region: Option<Region>,
current_timestamp: Option<usize>,
@@ -679,14 +680,24 @@ pub struct SignalAggregator {
impl Clone for SignalAggregator {
fn clone(&self) -> Self {
+ let inbound_similarity = self
+ .inbound_similarity
+ .as_ref()
+ .map(|scorer| RefCell::new(scorer.borrow().clone()));
+
+ let query_centrality = self
+ .query_centrality
+ .as_ref()
+ .map(|scorer| RefCell::new(scorer.borrow().clone()));
+
Self {
query: self.query.clone(),
query_signal_coefficients: self.query_signal_coefficients.clone(),
segment_reader: None,
- inbound_similarity: self.inbound_similarity.clone(),
+ inbound_similarity,
fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),
update_time_cache: self.update_time_cache.clone(),
- query_centrality: self.query_centrality.clone(),
+ query_centrality,
region_count: self.region_count.clone(),
selected_region: self.selected_region,
current_timestamp: self.current_timestamp,
@@ -866,11 +877,11 @@ impl SignalAggregator {
}
pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {
- self.query_centrality = Some(Arc::new(query_centrality));
+ self.query_centrality = Some(RefCell::new(query_centrality));
}
pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {
- self.inbound_similarity = Some(Arc::new(scorer));
+ self.inbound_similarity = Some(RefCell::new(scorer));
}
pub fn set_region_count(&mut self, region_count: RegionCount) {
@@ -892,13 +903,13 @@ impl SignalAggregator {
pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {
self.query_centrality
.as_ref()
- .map(|scorer| scorer.score(host_id))
+ .map(|scorer| scorer.borrow_mut().score(host_id))
}
pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {
self.inbound_similarity
.as_ref()
- .map(|scorer| scorer.score(&host_id))
+ .map(|scorer| scorer.borrow_mut().score(&host_id))
.unwrap_or_default()
}
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index a9f32ecb..7b4568a4 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -97,7 +97,7 @@ impl SimilarSitesFinder {
let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();
- let scorer = self.inbound_similarity.scorer(&nodes, &[], true);
+ let mut scorer = self.inbound_similarity.scorer(&nodes, &[], true);
let mut backlinks: IntMap<NodeID, f64> = IntMap::new();
diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rs
index b40bdb7f..2fcea1f0 100644
--- a/core/src/widgets/mod.rs
+++ b/core/src/widgets/mod.rs
@@ -44,7 +44,7 @@ impl Widgets {
return Err(anyhow!("Only one thesaurus path is supported for now"));
}
- let thesaurus = if let Some(path) = config.thesaurus_paths.get(0) {
+ let thesaurus = if let Some(path) = config.thesaurus_paths.first() {
Some(thesaurus::Dictionary::build(path)?)
} else {
None