Skip to content

web search done right

Build inbound similarity index

Mikkel Denker8138f5e6f80f
diff --git a/core/src/intmap.rs b/core/src/intmap.rs
index d198fe36..319d0cd4 100644
--- a/core/src/intmap.rs
+++ b/core/src/intmap.rs
@@ -35,7 +35,15 @@ impl<V: Clone> Clone for IntMap<V> {
impl<V> IntMap<V> {
pub fn new() -> Self {
- let bins = vec![Vec::new(), Vec::new()];
+ Self::with_capacity(2)
+ }
+
+ pub fn with_capacity(cap: usize) -> Self {
+ let mut bins = Vec::with_capacity(cap);
+
+ for _ in 0..cap {
+ bins.push(Vec::new());
+ }
Self { bins, len: 0 }
}
@@ -47,7 +55,7 @@ impl<V> IntMap<V> {
}
pub fn insert(&mut self, key: Key, value: V) {
- if self.len >= (self.bins.len() as f64 * 0.8) as usize {
+ if self.len >= (self.bins.len() as f64 * 1.2) as usize {
self.grow();
}
@@ -118,7 +126,15 @@ impl<V> IntMap<V> {
impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {
fn from_iter<T: IntoIterator<Item = (u64, V)>>(iter: T) -> Self {
- let mut map = Self::new();
+ let iter = iter.into_iter();
+
+ let (_, upper) = iter.size_hint();
+
+ let mut map = if let Some(cap) = upper {
+ Self::with_capacity(cap)
+ } else {
+ Self::new()
+ };
for (key, val) in iter {
map.insert(key, val);
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
new file mode 100644
index 00000000..bf7467f7
--- /dev/null
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -0,0 +1,172 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use itertools::Itertools;
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize)]
+struct Block {
+ data: u64,
+ offset: usize,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct BitVec {
+ blocks: Vec<Block>,
+ sqrt_len: f64,
+}
+
+impl BitVec {
+ pub fn new(data: Vec<bool>) -> Self {
+ let mut blocks = Vec::new();
+ let mut len = 0;
+
+ for (offset, chunk) in data.into_iter().chunks(64).into_iter().enumerate() {
+ let mut data = 0u64;
+ for (i, b) in chunk.into_iter().enumerate() {
+ if b {
+ data |= 1u64 << (63 - i);
+ len += 1;
+ }
+ }
+
+ if data == 0 {
+ continue;
+ }
+
+ blocks.push(Block { data, offset })
+ }
+
+ Self {
+ blocks,
+ sqrt_len: (len as f64).sqrt(),
+ }
+ }
+
+ fn sim(&self, other: &Self) -> f64 {
+ let mut i = 0;
+ let mut j = 0;
+
+ let mut dot: u64 = 0;
+
+ while i < self.blocks.len() && j < other.blocks.len() {
+ match self.blocks[i].offset.cmp(&other.blocks[j].offset) {
+ std::cmp::Ordering::Less => i += 1,
+ std::cmp::Ordering::Greater => j += 1,
+ std::cmp::Ordering::Equal => {
+ dot += (self.blocks[i].data & other.blocks[j].data).count_ones() as u64;
+ i += 1;
+ j += 1;
+ }
+ }
+ }
+
+ if dot == 0 {
+ 0.0
+ } else {
+ dot as f64 / (self.sqrt_len * other.sqrt_len)
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::iter::repeat;
+
+ fn naive_sim(a: &[bool], b: &[bool]) -> f64 {
+ let dot = a
+ .iter()
+ .zip_eq(b.iter())
+ .filter(|(a, b)| **a && **b)
+ .count();
+
+ let len_a = a.iter().filter(|a| **a).count();
+ let len_b = b.iter().filter(|b| **b).count();
+
+ dot as f64 / ((len_a as f64).sqrt() * (len_b as f64).sqrt())
+ }
+
+ #[test]
+ fn simple() {
+ let a: Vec<_> = repeat(false)
+ .take(1000)
+ .chain(repeat(true).take(10))
+ .collect();
+
+ let b: Vec<_> = repeat(false)
+ .take(1000)
+ .chain(repeat(true).take(8))
+ .chain(repeat(false).take(2))
+ .collect();
+
+ let expected = naive_sim(&a, &b);
+
+ assert!(expected > 0.894);
+
+ let a = BitVec::new(a);
+ let b = BitVec::new(b);
+
+ assert!((expected - a.sim(&b)).abs() < 0.00001);
+ }
+
+ #[test]
+ fn zero_sim() {
+ let a: Vec<_> = repeat(false).take(1000).collect();
+
+ let b: Vec<_> = repeat(true).take(1000).collect();
+
+ let a = BitVec::new(a);
+ let b = BitVec::new(b);
+
+ assert_eq!(a.sim(&b), 0.0);
+ }
+
+ #[test]
+ fn empty_sim() {
+ let a: Vec<_> = Vec::new();
+ let b: Vec<_> = Vec::new();
+
+ let a = BitVec::new(a);
+ let b = BitVec::new(b);
+
+ assert_eq!(a.sim(&b), 0.0);
+ }
+
+ #[test]
+ fn low_sim() {
+ let a: Vec<_> = repeat(false)
+ .take(100000)
+ .chain(repeat(true).take(10))
+ .collect();
+
+ let b: Vec<_> = repeat(true)
+ .take(100000)
+ .chain(repeat(true).take(8))
+ .chain(repeat(false).take(2))
+ .collect();
+
+ let expected = naive_sim(&a, &b);
+
+ dbg!(expected);
+ assert!(expected < 0.01);
+
+ let a = BitVec::new(a);
+ let b = BitVec::new(b);
+
+ assert!((expected - a.sim(&b)).abs() < 0.00001);
+ }
+}
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 805f3ee9..b6ca7365 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -24,6 +24,8 @@ use crate::{
},
};
+use super::inbound_similarity::InboundSimilarity;
+
pub struct HarmonicCentralityStore {
pub host: Box<dyn Kv<String, f64>>,
pub full: Box<dyn Kv<String, f64>>,
@@ -46,6 +48,7 @@ impl HarmonicCentralityStore {
pub struct CentralityStore {
pub harmonic: HarmonicCentralityStore,
pub online_harmonic: OnlineHarmonicCentrality,
+ pub inbound_similarity: InboundSimilarity,
pub base_path: String,
}
@@ -56,6 +59,9 @@ impl CentralityStore {
online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic"))
.ok()
.unwrap_or_default(),
+ inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))
+ .ok()
+ .unwrap_or_default(),
base_path: path.as_ref().to_str().unwrap().to_string(),
}
}
@@ -90,6 +96,7 @@ impl CentralityStore {
Self::store_host(&output_path, &mut store, &harmonic_centrality);
store.online_harmonic = OnlineHarmonicCentrality::new(graph, &harmonic_centrality);
+ store.inbound_similarity = InboundSimilarity::build(graph, &harmonic_centrality);
store.flush();
@@ -102,5 +109,9 @@ impl CentralityStore {
self.online_harmonic
.save(Path::new(&self.base_path).join("online_harmonic"))
.unwrap();
+
+ self.inbound_similarity
+ .save(Path::new(&self.base_path).join("inbound_similarity"))
+ .unwrap();
}
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
new file mode 100644
index 00000000..ea6ca6f7
--- /dev/null
+++ b/core/src/ranking/inbound_similarity.rs
@@ -0,0 +1,92 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ fs::File,
+ io::{BufReader, BufWriter, Read},
+ path::Path,
+};
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ intmap::IntMap,
+ webgraph::{centrality::harmonic::HarmonicCentrality, Webgraph},
+ Result,
+};
+
+use super::bitvec_similarity;
+const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.025;
+
+#[derive(Serialize, Deserialize, Default)]
+pub struct InboundSimilarity {
+ vectors: IntMap<bitvec_similarity::BitVec>,
+}
+
+impl InboundSimilarity {
+ pub fn build(graph: &Webgraph, harmonic: &HarmonicCentrality) -> Self {
+ let mut vectors = IntMap::new();
+ let nodes: Vec<_> = graph.nodes().collect();
+
+ if let Some(max_node) = nodes.iter().max().copied() {
+ for node_id in nodes {
+ let mut buf = vec![false; max_node.0 as usize];
+
+ for edge in graph
+ .raw_ingoing_edges(&node_id)
+ .into_iter()
+ .filter(|edge| match graph.id2node(&edge.from) {
+ Some(node) => {
+ let score = *harmonic.host.get(&node.into_host()).unwrap_or(&0.0);
+ score >= HARMONIC_CENTRALITY_THRESHOLD
+ }
+ None => false,
+ })
+ {
+ buf[edge.from.0 as usize] = true;
+ }
+
+ vectors.insert(node_id.0, bitvec_similarity::BitVec::new(buf));
+ }
+ }
+
+ Self { vectors }
+ }
+
+ pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {
+ let mut file = BufWriter::new(
+ File::options()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(path)?,
+ );
+
+ bincode::serialize_into(&mut file, &self)?;
+
+ Ok(())
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let file = File::open(path)?;
+ let mut reader = BufReader::new(file);
+
+ let mut buf = Vec::new();
+ reader.read_to_end(&mut buf)?;
+
+ Ok(bincode::deserialize(&buf)?)
+ }
+}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 1ae5f243..21daf289 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -14,8 +14,10 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod bitvec_similarity;
pub mod bm25;
pub mod centrality_store;
+pub mod inbound_similarity;
pub mod initial;
pub mod models;
pub mod optics;
@@ -175,7 +177,10 @@ mod tests {
index::Index,
searcher::{LocalSearcher, SearchQuery},
webgraph::{
- centrality::{online_harmonic::OnlineHarmonicCentrality, topic::TopicCentrality},
+ centrality::{
+ harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality,
+ topic::TopicCentrality,
+ },
Node, WebgraphBuilder,
},
webpage::{Html, Link, Webpage},
@@ -1235,7 +1240,8 @@ mod tests {
webgraph.commit();
- let harmonic = OnlineHarmonicCentrality::new(&webgraph);
+ let centrality = HarmonicCentrality::calculate(&webgraph);
+ let harmonic = OnlineHarmonicCentrality::new(&webgraph, &centrality);
let topic_centrality = TopicCentrality::build(&index, topics, webgraph, harmonic);