web search done right
Build inbound similarity index
Mikkel Denker8138f5e6f80f
diff --git a/core/src/intmap.rs b/core/src/intmap.rsindex d198fe36..319d0cd4 100644--- a/core/src/intmap.rs+++ b/core/src/intmap.rs@@ -35,7 +35,15 @@ impl<V: Clone> Clone for IntMap<V> {impl<V> IntMap<V> {pub fn new() -> Self {- let bins = vec![Vec::new(), Vec::new()];+ Self::with_capacity(2)+ }++ pub fn with_capacity(cap: usize) -> Self {+ let mut bins = Vec::with_capacity(cap);++ for _ in 0..cap {+ bins.push(Vec::new());+ }Self { bins, len: 0 }}@@ -47,7 +55,7 @@ impl<V> IntMap<V> {}pub fn insert(&mut self, key: Key, value: V) {- if self.len >= (self.bins.len() as f64 * 0.8) as usize {+ if self.len >= (self.bins.len() as f64 * 1.2) as usize {self.grow();}@@ -118,7 +126,15 @@ impl<V> IntMap<V> {impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {fn from_iter<T: IntoIterator<Item = (u64, V)>>(iter: T) -> Self {- let mut map = Self::new();+ let iter = iter.into_iter();++ let (_, upper) = iter.size_hint();++ let mut map = if let Some(cap) = upper {+ Self::with_capacity(cap)+ } else {+ Self::new()+ };for (key, val) in iter {map.insert(key, val);diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rsnew file mode 100644index 00000000..bf7467f7--- /dev/null+++ b/core/src/ranking/bitvec_similarity.rs@@ -0,0 +1,172 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use itertools::Itertools;+use serde::{Deserialize, Serialize};++#[derive(Serialize, Deserialize)]+struct Block {+ data: u64,+ offset: usize,+}++#[derive(Serialize, Deserialize)]+pub struct BitVec {+ blocks: Vec<Block>,+ sqrt_len: f64,+}++impl BitVec {+ pub fn new(data: Vec<bool>) -> Self {+ let mut blocks = Vec::new();+ let mut len = 0;++ for (offset, chunk) in data.into_iter().chunks(64).into_iter().enumerate() {+ let mut data = 0u64;+ for (i, b) in chunk.into_iter().enumerate() {+ if b {+ data |= 1u64 << (63 - i);+ len += 1;+ }+ }++ if data == 0 {+ continue;+ }++ blocks.push(Block { data, offset })+ }++ Self {+ blocks,+ sqrt_len: (len as f64).sqrt(),+ }+ }++ fn sim(&self, other: &Self) -> f64 {+ let mut i = 0;+ let mut j = 0;++ let mut dot: u64 = 0;++ while i < self.blocks.len() && j < other.blocks.len() {+ match self.blocks[i].offset.cmp(&other.blocks[j].offset) {+ std::cmp::Ordering::Less => i += 1,+ std::cmp::Ordering::Greater => j += 1,+ std::cmp::Ordering::Equal => {+ dot += (self.blocks[i].data & other.blocks[j].data).count_ones() as u64;+ i += 1;+ j += 1;+ }+ }+ }++ if dot == 0 {+ 0.0+ } else {+ dot as f64 / (self.sqrt_len * other.sqrt_len)+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::iter::repeat;++ fn naive_sim(a: &[bool], b: &[bool]) -> f64 {+ let dot = a+ .iter()+ .zip_eq(b.iter())+ .filter(|(a, b)| **a && **b)+ .count();++ let len_a = a.iter().filter(|a| **a).count();+ let len_b = b.iter().filter(|b| **b).count();++ dot as f64 / ((len_a as f64).sqrt() * (len_b as f64).sqrt())+ }++ #[test]+ fn simple() {+ let a: Vec<_> = repeat(false)+ .take(1000)+ .chain(repeat(true).take(10))+ .collect();++ let b: Vec<_> = repeat(false)+ .take(1000)+ .chain(repeat(true).take(8))+ .chain(repeat(false).take(2))+ .collect();++ let expected = naive_sim(&a, &b);++ assert!(expected > 0.894);++ let a = BitVec::new(a);+ let b = BitVec::new(b);++ assert!((expected - a.sim(&b)).abs() < 0.00001);+ }++ #[test]+ fn zero_sim() {+ let a: Vec<_> = repeat(false).take(1000).collect();++ let b: Vec<_> = repeat(true).take(1000).collect();++ let a = BitVec::new(a);+ let b = BitVec::new(b);++ assert_eq!(a.sim(&b), 0.0);+ }++ #[test]+ fn empty_sim() {+ let a: Vec<_> = Vec::new();+ let b: Vec<_> = Vec::new();++ let a = BitVec::new(a);+ let b = BitVec::new(b);++ assert_eq!(a.sim(&b), 0.0);+ }++ #[test]+ fn low_sim() {+ let a: Vec<_> = repeat(false)+ .take(100000)+ .chain(repeat(true).take(10))+ .collect();++ let b: Vec<_> = repeat(true)+ .take(100000)+ .chain(repeat(true).take(8))+ .chain(repeat(false).take(2))+ .collect();++ let expected = naive_sim(&a, &b);++ dbg!(expected);+ assert!(expected < 0.01);++ let a = BitVec::new(a);+ let b = BitVec::new(b);++ assert!((expected - a.sim(&b)).abs() < 0.00001);+ }+}diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 805f3ee9..b6ca7365 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -24,6 +24,8 @@ use crate::{},};+use super::inbound_similarity::InboundSimilarity;+pub struct HarmonicCentralityStore {pub host: Box<dyn Kv<String, f64>>,pub full: Box<dyn Kv<String, f64>>,@@ -46,6 +48,7 @@ impl HarmonicCentralityStore {pub struct CentralityStore {pub harmonic: HarmonicCentralityStore,pub online_harmonic: OnlineHarmonicCentrality,+ pub inbound_similarity: InboundSimilarity,pub base_path: String,}@@ -56,6 +59,9 @@ impl CentralityStore {online_harmonic: OnlineHarmonicCentrality::open(path.as_ref().join("online_harmonic")).ok().unwrap_or_default(),+ inbound_similarity: InboundSimilarity::open(path.as_ref().join("inbound_similarity"))+ .ok()+ .unwrap_or_default(),base_path: path.as_ref().to_str().unwrap().to_string(),}}@@ -90,6 +96,7 @@ impl CentralityStore {Self::store_host(&output_path, &mut store, &harmonic_centrality);store.online_harmonic = OnlineHarmonicCentrality::new(graph, &harmonic_centrality);+ store.inbound_similarity = InboundSimilarity::build(graph, &harmonic_centrality);store.flush();@@ -102,5 +109,9 @@ impl CentralityStore {self.online_harmonic.save(Path::new(&self.base_path).join("online_harmonic")).unwrap();++ self.inbound_similarity+ .save(Path::new(&self.base_path).join("inbound_similarity"))+ .unwrap();}}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsnew file mode 100644index 00000000..ea6ca6f7--- /dev/null+++ b/core/src/ranking/inbound_similarity.rs@@ -0,0 +1,92 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ fs::File,+ io::{BufReader, BufWriter, Read},+ path::Path,+};++use serde::{Deserialize, Serialize};++use crate::{+ intmap::IntMap,+ webgraph::{centrality::harmonic::HarmonicCentrality, Webgraph},+ Result,+};++use super::bitvec_similarity;+const HARMONIC_CENTRALITY_THRESHOLD: f64 = 0.025;++#[derive(Serialize, Deserialize, Default)]+pub struct InboundSimilarity {+ vectors: IntMap<bitvec_similarity::BitVec>,+}++impl InboundSimilarity {+ pub fn build(graph: &Webgraph, harmonic: &HarmonicCentrality) -> Self {+ let mut vectors = IntMap::new();+ let nodes: Vec<_> = graph.nodes().collect();++ if let Some(max_node) = nodes.iter().max().copied() {+ for node_id in nodes {+ let mut buf = vec![false; max_node.0 as usize];++ for edge in graph+ .raw_ingoing_edges(&node_id)+ .into_iter()+ .filter(|edge| match graph.id2node(&edge.from) {+ Some(node) => {+ let score = *harmonic.host.get(&node.into_host()).unwrap_or(&0.0);+ score >= HARMONIC_CENTRALITY_THRESHOLD+ }+ None => false,+ })+ {+ buf[edge.from.0 as usize] = true;+ }++ vectors.insert(node_id.0, bitvec_similarity::BitVec::new(buf));+ }+ }++ Self { vectors }+ }++ pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {+ let mut file = BufWriter::new(+ File::options()+ .create(true)+ .truncate(true)+ .write(true)+ .open(path)?,+ );++ bincode::serialize_into(&mut file, &self)?;++ Ok(())+ }++ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {+ let file = File::open(path)?;+ let mut reader = BufReader::new(file);++ let mut buf = Vec::new();+ reader.read_to_end(&mut buf)?;++ Ok(bincode::deserialize(&buf)?)+ }+}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 1ae5f243..21daf289 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -14,8 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+mod bitvec_similarity;pub mod bm25;pub mod centrality_store;+pub mod inbound_similarity;pub mod initial;pub mod models;pub mod optics;@@ -175,7 +177,10 @@ mod tests {index::Index,searcher::{LocalSearcher, SearchQuery},webgraph::{- centrality::{online_harmonic::OnlineHarmonicCentrality, topic::TopicCentrality},+ centrality::{+ harmonic::HarmonicCentrality, online_harmonic::OnlineHarmonicCentrality,+ topic::TopicCentrality,+ },Node, WebgraphBuilder,},webpage::{Html, Link, Webpage},@@ -1235,7 +1240,8 @@ mod tests {webgraph.commit();- let harmonic = OnlineHarmonicCentrality::new(&webgraph);+ let centrality = HarmonicCentrality::calculate(&webgraph);+ let harmonic = OnlineHarmonicCentrality::new(&webgraph, ¢rality);let topic_centrality = TopicCentrality::build(&index, topics, webgraph, harmonic);