web search done right
Speedup bitvec similarity calculation using hyperloglog. I also tried estimating the intersection directly using a minhash sketch, but I found that the space requirements were quite substancial to get adequate accuracy. It seems a much better trade-off to just filter highly unsimilar vecs using hyperloglog and calculating the remaining manually using the postings.
Mikkel Denker0c463e1f6adb
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 2714ce28..66593336 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -43,12 +43,19 @@ fn download_files() {"content.rdf.u8.gz","english-wordnet-2022-subset.ttl",] {+ let path = Path::new(DATA_PATH).join(name);++ if path.exists() {+ info!("Skipping {}", name);+ continue;+ }+info!("Downloading {}", name);let body = reqwest::get(format!("http://s3.trystract.com/{BUCKET_NAME}/{name}")).await.unwrap();- let mut file = File::create(Path::new(DATA_PATH).join(name)).await.unwrap();+ let mut file = File::create(path).await.unwrap();let mut bytes = body.bytes_stream();while let Some(item) = bytes.next().await {diff --git a/core/src/hyperloglog.rs b/core/src/hyperloglog.rsindex ad80de1d..6349fc41 100644--- a/core/src/hyperloglog.rs+++ b/core/src/hyperloglog.rs@@ -4295,7 +4295,7 @@ const ONE_OVER_POWER_OF_TWO: [f64; 256] = [1.727233711018889e-77,];-#[derive(Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct HyperLogLog<const N: usize> {registers: Vec<u8>,}@@ -4319,7 +4319,7 @@ impl<const N: usize> HyperLogLog<N> {let m = self.registers.len();if m >= 128 {- 0.7213 / (1. + 1.079 / (m as f64))+ 0.7213 / (1. + (1.079 / (m as f64)))} else if m >= 64 {0.709} else if m >= 32 {@@ -4338,11 +4338,11 @@ impl<const N: usize> HyperLogLog<N> {let b = self.b();let hash = Self::hash(item) as usize;- let w = hash >> b;- let j = hash - (w << b);+ let j = hash >> (64 - b);+ let w = hash << b;// p = leftmost bit (1-based count)- let p = w.leading_zeros() + 1 - (b as u32);+ let p = w.leading_zeros() + 1;self.registers[j] = self.registers[j].max(p as u8);}@@ -4460,9 +4460,15 @@ impl<const N: usize> HyperLogLog<N> {}}- #[cfg(test)]pub fn relative_error(&self) -> f64 {- (3f64 * 2f64.ln() - 1f64).sqrt() / (self.registers.len() as f64).sqrt()+ 1.04 / (self.registers.len() as f64).sqrt()+ }++ pub fn size_bounds(&self) -> (usize, usize) {+ let size = self.size();+ let delta = (self.relative_error() * 2.0 * (size as f64)) as usize;++ (size - delta, size + delta)}pub fn merge(&mut self, other: &Self) {@@ -4471,6 +4477,12 @@ impl<const N: usize> HyperLogLog<N> {}}+ pub fn merge_into(&self, other: &Self, out: &mut Self) {+ for i in 0..N {+ out.registers[i] = self.registers[i].max(other.registers[i]);+ }+ }+pub fn registers(&self) -> &[u8] {self.registers.as_ref()}@@ -4488,9 +4500,7 @@ mod tests {set.add(item);}- let delta = (set.relative_error() * (set.size() as f64)) as usize;- let lower_bound = set.size() - delta;- let upper_bound = set.size() + delta;+ let (lower_bound, upper_bound) = set.size_bounds();assert!(set.size() > lower_bound && set.size() < upper_bound);}@@ -4500,12 +4510,9 @@ mod tests {for item in 0..10_000 {set.add(item);- set.size();}- let delta = (set.relative_error() * (set.size() as f64)) as usize;- let lower_bound = set.size() - delta;- let upper_bound = set.size() + delta;+ let (lower_bound, upper_bound) = set.size_bounds();assert!(set.size() > lower_bound && set.size() < upper_bound);}diff --git a/core/src/intmap.rs b/core/src/intmap.rsindex 9d24af28..71dd3750 100644--- a/core/src/intmap.rs+++ b/core/src/intmap.rs@@ -101,7 +101,7 @@ impl<K: Key, V> IntMap<K, V> {fn grow(&mut self) {let mut bins = Vec::new();- for _ in 0..self.bins.len() * 2 {+ for _ in 0..(self.bins.len() as f64 * 1.5) as usize {bins.push(Vec::new());}diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex 04cf11bc..c9a478f5 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -268,7 +268,7 @@ impl Term {let processed_terms = Term::process_tantivy_term(term, *field);if processed_terms.len() == 1 {- let term = processed_terms.get(0).unwrap().clone();+ let term = processed_terms.first().unwrap().clone();(Occur::Should,Box::new(TermQuery::new(diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rsindex 6918d40e..f859db60 100644--- a/core/src/ranking/bitvec_similarity.rs+++ b/core/src/ranking/bitvec_similarity.rs@@ -17,9 +17,15 @@use itertools::Itertools;use serde::{Deserialize, Serialize};+use crate::hyperloglog::HyperLogLog;++const THRESHOLD_SIM_ESTIMATE: f64 = 0.1;++const HYPERLOGLOG_REGISTERS: usize = 128;+#[derive(Serialize, Deserialize, Debug, Clone)]struct VeryJankyBloomFilter {- data: Vec<u128>,+ data: Vec<u64>,}impl VeryJankyBloomFilter {@@ -29,15 +35,14 @@ impl VeryJankyBloomFilter {}}- fn hash(&self, item: &u128) -> (usize, u128) {+ fn hash(&self, item: &u64) -> (usize, u64) {(- (item.wrapping_mul(180811355057196146399512320062616506473) % self.data.len() as u128)- as usize,- item.wrapping_mul(258349730118558892798889897730042154643) % 128,+ (item.wrapping_mul(11400714819323198549) % self.data.len() as u64) as usize,+ item.wrapping_mul(11400714819323198549) % 64,)}- fn insert(&mut self, item: u128) {+ fn insert(&mut self, item: u64) {let (a, b) = self.hash(&item);self.data[a] |= 1 << b;}@@ -51,15 +56,20 @@ impl VeryJankyBloomFilter {}}+#[derive(Clone)]+struct ScratchSpace {+ hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,+}+#[derive(Serialize, Deserialize, Debug, Clone)]struct Posting {- ranks: Vec<u128>,- skip_pointers: Vec<u128>,+ ranks: Vec<u64>,+ skip_pointers: Vec<u64>,skip_size: usize,}impl Posting {- fn new(ranks: Vec<u128>) -> Self {+ fn new(ranks: Vec<u64>) -> Self {let skip_size = (ranks.len() as f64).sqrt() as usize;let skip_pointers = ranks@@ -130,33 +140,37 @@ impl Posting {#[derive(Serialize, Deserialize, Debug, Clone)]pub struct BitVec {bloom: VeryJankyBloomFilter,+ hyperloglog: HyperLogLog<HYPERLOGLOG_REGISTERS>,posting: Posting,sqrt_len: f64,}impl BitVec {- pub fn new(mut ranks: Vec<u128>) -> Self {+ pub fn new(mut ranks: Vec<u64>) -> Self {ranks.sort();ranks.dedup();ranks.shrink_to_fit();let len = ranks.len();let mut bloom = VeryJankyBloomFilter::new(2);+ let mut hyperloglog = HyperLogLog::default();for rank in &ranks {bloom.insert(*rank);+ hyperloglog.add(*rank);}let posting = Posting::new(ranks);Self {bloom,+ hyperloglog,posting,sqrt_len: (len as f64).sqrt(),}}- pub fn sim(&self, other: &Self) -> f64 {+ fn sim(&self, other: &Self, scratchspace: &mut ScratchSpace) -> f64 {if self.sqrt_len == 0.0 || other.sqrt_len == 0.0 {return 0.0;}@@ -165,14 +179,49 @@ impl BitVec {return 0.0;}- let intersect = self.posting.intersection_size(&other.posting);+ self.hyperloglog+ .merge_into(&other.hyperloglog, &mut scratchspace.hyperloglog);++ let union_est = scratchspace.hyperloglog.size();++ let intersect_est = (self.posting.ranks.len() + other.posting.ranks.len())+ .checked_sub(union_est)+ .unwrap_or_default();++ let sim_est = (intersect_est as f64) / (self.sqrt_len * other.sqrt_len);++ if sim_est < THRESHOLD_SIM_ESTIMATE {+ return 0.0;+ }++ let intersect = self.posting.intersection_size(&other.posting) as f64;+ intersect / (self.sqrt_len * other.sqrt_len)+ }+}++#[derive(Clone)]+pub struct BitVecSimilarity {+ scratchspace: ScratchSpace,+}- if intersect == 0 {- 0.0- } else {- intersect as f64 / (self.sqrt_len * other.sqrt_len)+impl Default for BitVecSimilarity {+ fn default() -> Self {+ Self::new()+ }+}++impl BitVecSimilarity {+ pub fn new() -> Self {+ Self {+ scratchspace: ScratchSpace {+ hyperloglog: HyperLogLog::default(),+ },}}++ pub fn sim(&mut self, a: &BitVec, b: &BitVec) -> f64 {+ a.sim(b, &mut self.scratchspace)+ }}#[cfg(test)]@@ -181,11 +230,11 @@ mod tests {use itertools::Itertools;use std::iter::repeat;- fn into_ranks(a: &[bool]) -> Vec<u128> {+ fn into_ranks(a: &[bool]) -> Vec<u64> {a.iter().enumerate().filter(|(_, b)| **b)- .map(|(i, _)| i as u128)+ .map(|(i, _)| i as u64).collect()}@@ -220,7 +269,9 @@ mod tests {let a = BitVec::new(into_ranks(&a));let b = BitVec::new(into_ranks(&b));- assert!((expected - a.sim(&b)).abs() < 0.001);+ let sim = BitVecSimilarity::default().sim(&a, &b);++ assert!((expected - sim).abs() < 0.1);}#[test]@@ -232,7 +283,9 @@ mod tests {let a = BitVec::new(into_ranks(&a));let b = BitVec::new(into_ranks(&b));- assert_eq!(a.sim(&b), 0.0);+ let sim = BitVecSimilarity::default().sim(&a, &b);++ assert_eq!(sim, 0.0);}#[test]@@ -243,7 +296,9 @@ mod tests {let a = BitVec::new(into_ranks(&a));let b = BitVec::new(into_ranks(&b));- assert_eq!(a.sim(&b), 0.0);+ let sim = BitVecSimilarity::default().sim(&a, &b);++ assert_eq!(sim, 0.0);}#[test]@@ -266,6 +321,8 @@ mod tests {let a = BitVec::new(into_ranks(&a));let b = BitVec::new(into_ranks(&b));- assert!((expected - a.sim(&b)).abs() < 0.01);+ let sim = BitVecSimilarity::default().sim(&a, &b);++ assert!((expected - sim).abs() < 0.1);}}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 6d070a49..dd49abea 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -33,15 +33,17 @@ use crate::{use super::bitvec_similarity;+#[derive(Clone)]pub struct Scorer {+ similarity: bitvec_similarity::BitVecSimilarity,liked: Vec<NodeScorer>,disliked: Vec<NodeScorer>,vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,- cache: DashMap<NodeID, f64>,+ cache: IntMap<NodeID, f64>,normalized: bool,}-#[derive(Debug)]+#[derive(Debug, Clone)]struct NodeScorer {node: NodeID,inbound: bitvec_similarity::BitVec,@@ -61,29 +63,34 @@ impl NodeScorer {self.self_score = self_score;}- fn sim(&self, other: &NodeID, other_inbound: &bitvec_similarity::BitVec) -> f64 {+ fn sim(+ &self,+ other: &NodeID,+ other_inbound: &bitvec_similarity::BitVec,+ similarity: &mut bitvec_similarity::BitVecSimilarity,+ ) -> f64 {if self.node == *other {self.self_score} else {- self.inbound.sim(other_inbound)+ similarity.sim(&self.inbound, other_inbound)}}}impl Scorer {- fn calculate_score(&self, node: &NodeID) -> f64 {+ fn calculate_score(&mut self, node: &NodeID) -> f64 {let s = match self.vectors.get(node) {Some(vec) => {(self.disliked.len() as f64)+ (self.liked.iter()- .map(|liked| liked.sim(node, vec))+ .map(|liked| liked.sim(node, vec, &mut self.similarity)).sum::<f64>()- self.disliked.iter()- .map(|disliked| disliked.sim(node, vec))+ .map(|disliked| disliked.sim(node, vec, &mut self.similarity)).sum::<f64>())}None => 0.0,@@ -96,7 +103,7 @@ impl Scorer {}.max(0.0)}- pub fn score(&self, node: &NodeID) -> f64 {+ pub fn score(&mut self, node: &NodeID) -> f64 {if let Some(cached) = self.cache.get(node) {return *cached;}@@ -135,7 +142,7 @@ impl InboundSimilarity {for (node_id, inbound) in adjacency {vectors.insert(node_id,- bitvec_similarity::BitVec::new(inbound.into_iter().map(|n| n.bit_128()).collect()),+ bitvec_similarity::BitVec::new(inbound.into_iter().map(|n| n.bit_64()).collect()),);}@@ -163,10 +170,11 @@ impl InboundSimilarity {.collect();Scorer {+ similarity: bitvec_similarity::BitVecSimilarity::default(),liked,disliked,vectors: self.vectors.clone(),- cache: DashMap::new(),+ cache: IntMap::new(),normalized,}}@@ -241,7 +249,7 @@ mod tests {let inbound = InboundSimilarity::build(&graph);- let scorer = inbound.scorer(&[Node::from("b.com").id()], &[], false);+ let mut scorer = inbound.scorer(&[Node::from("b.com").id()], &[], false);let e = Node::from("e.com").id();let d = Node::from("d.com").id();diff --git a/core/src/ranking/query_centrality.rs b/core/src/ranking/query_centrality.rsindex 15e5623a..1551b826 100644--- a/core/src/ranking/query_centrality.rs+++ b/core/src/ranking/query_centrality.rs@@ -18,6 +18,7 @@ use crate::webgraph::NodeID;use super::inbound_similarity;+#[derive(Clone)]pub struct Scorer {inbound_centrality: inbound_similarity::Scorer,}@@ -30,7 +31,7 @@ impl Scorer {Self { inbound_centrality }}- pub fn score(&self, node: NodeID) -> f64 {+ pub fn score(&mut self, node: NodeID) -> f64 {self.inbound_centrality.score(&node)}}diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 08e282c4..047e5b44 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -28,6 +28,7 @@ use crate::{use optics::ast::RankingTarget;use optics::Optic;use serde::{Deserialize, Serialize};+use std::cell::RefCell;use std::str::FromStr;use std::sync::Arc;use tantivy::fieldnorm::FieldNormReader;@@ -267,7 +268,7 @@ impl Signal {Signal::UpdateTimestamp => 0.001,Signal::TrackerScore => 0.02,Signal::Region => 0.01,- Signal::InboundSimilarity => 1.0,+ Signal::InboundSimilarity => 0.25,Signal::LambdaMART => 10.0,Signal::UrlSlashes => 0.01,Signal::UrlDigits => 0.01,@@ -667,10 +668,10 @@ pub struct SignalAggregator {query: Option<QueryData>,query_signal_coefficients: Option<SignalCoefficient>,segment_reader: Option<SegmentReader>,- inbound_similarity: Option<Arc<inbound_similarity::Scorer>>,+ inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,- query_centrality: Option<Arc<query_centrality::Scorer>>,+ query_centrality: Option<RefCell<query_centrality::Scorer>>,region_count: Option<Arc<RegionCount>>,selected_region: Option<Region>,current_timestamp: Option<usize>,@@ -679,14 +680,24 @@ pub struct SignalAggregator {impl Clone for SignalAggregator {fn clone(&self) -> Self {+ let inbound_similarity = self+ .inbound_similarity+ .as_ref()+ .map(|scorer| RefCell::new(scorer.borrow().clone()));++ let query_centrality = self+ .query_centrality+ .as_ref()+ .map(|scorer| RefCell::new(scorer.borrow().clone()));+Self {query: self.query.clone(),query_signal_coefficients: self.query_signal_coefficients.clone(),segment_reader: None,- inbound_similarity: self.inbound_similarity.clone(),+ inbound_similarity,fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),update_time_cache: self.update_time_cache.clone(),- query_centrality: self.query_centrality.clone(),+ query_centrality,region_count: self.region_count.clone(),selected_region: self.selected_region,current_timestamp: self.current_timestamp,@@ -866,11 +877,11 @@ impl SignalAggregator {}pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {- self.query_centrality = Some(Arc::new(query_centrality));+ self.query_centrality = Some(RefCell::new(query_centrality));}pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {- self.inbound_similarity = Some(Arc::new(scorer));+ self.inbound_similarity = Some(RefCell::new(scorer));}pub fn set_region_count(&mut self, region_count: RegionCount) {@@ -892,13 +903,13 @@ impl SignalAggregator {pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {self.query_centrality.as_ref()- .map(|scorer| scorer.score(host_id))+ .map(|scorer| scorer.borrow_mut().score(host_id))}pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {self.inbound_similarity.as_ref()- .map(|scorer| scorer.score(&host_id))+ .map(|scorer| scorer.borrow_mut().score(&host_id)).unwrap_or_default()}diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex a9f32ecb..7b4568a4 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -97,7 +97,7 @@ impl SimilarSitesFinder {let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();- let scorer = self.inbound_similarity.scorer(&nodes, &[], true);+ let mut scorer = self.inbound_similarity.scorer(&nodes, &[], true);let mut backlinks: IntMap<NodeID, f64> = IntMap::new();diff --git a/core/src/widgets/mod.rs b/core/src/widgets/mod.rsindex b40bdb7f..2fcea1f0 100644--- a/core/src/widgets/mod.rs+++ b/core/src/widgets/mod.rs@@ -44,7 +44,7 @@ impl Widgets {return Err(anyhow!("Only one thesaurus path is supported for now"));}- let thesaurus = if let Some(path) = config.thesaurus_paths.get(0) {+ let thesaurus = if let Some(path) = config.thesaurus_paths.first() {Some(thesaurus::Dictionary::build(path)?)} else {None