web search done right
specialized integer hashmap implementation for speedup in apprxomate harmonic calculations
Mikkel Denkercfe03f99925c
diff --git a/Cargo.lock b/Cargo.lockindex 2246aff5..7acb7745 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -315,19 +315,6 @@ dependencies = ["generic-array",]-[[package]]-name = "boomphf"-version = "0.5.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4e8fb7f38fef59c32549861151d63a6190865e60cf690340c13e3d7178b42a2f"-dependencies = [- "crossbeam-utils",- "log",- "rayon",- "serde",- "wyhash",-]-[[package]]name = "bstr"version = "0.2.17"@@ -733,7 +720,6 @@ dependencies = ["axum-macros","base64","bincode",- "boomphf","bzip2","chrono","clap 3.1.18",@@ -3859,15 +3845,6 @@ dependencies = ["winapi",]-[[package]]-name = "wyhash"-version = "0.5.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295"-dependencies = [- "rand_core 0.6.3",-]-[[package]]name = "xml-rs"version = "0.8.4"diff --git a/Cargo.toml b/Cargo.tomlindex 4a4d7c1b..1ce3fccd 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -61,7 +61,6 @@ rust-stemmers = "1.2.0"min-max-heap = "1.3.0"base64 = "0.13.0"async-trait = "0.1.57"-boomphf = { version = "0.5.9", features = ["serde"] }axum-macros = "0.2.3"urlencoding = "2.1.2"rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false, optional = true }diff --git a/src/hyperloglog.rs b/src/hyperloglog.rsnew file mode 100644index 00000000..811ff243--- /dev/null+++ b/src/hyperloglog.rs@@ -0,0 +1,19 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++struct HyperLogLog<const N: usize = 16> {+ registers: [u8; N],+}diff --git a/src/intmap.rs b/src/intmap.rsnew file mode 100644index 00000000..4b9cb967--- /dev/null+++ b/src/intmap.rs@@ -0,0 +1,127 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use serde::{Deserialize, Serialize};++type Key = u64;++#[derive(Serialize, Deserialize)]+pub struct IntMap<V> {+ bins: Vec<Vec<(Key, V)>>,+ len: usize,+}++impl<V> IntMap<V> {+ pub fn new() -> Self {+ let bins = vec![Vec::new(), Vec::new()];++ Self { bins, len: 0 }+ }++ fn bin_idx(&self, key: &Key) -> usize {+ let mask = (self.bins.len() - 1) as Key;+ let key = key.wrapping_mul(11400714819323198549 as Key);+ (key & mask) as usize+ }++ pub fn insert(&mut self, key: Key, value: V) {+ if self.len >= (self.bins.len() as f64 * 0.8) as usize {+ self.grow();+ }++ let bin_idx = self.bin_idx(&key);+ let bin = &mut self.bins[bin_idx];++ if let Some(idx) = bin.iter().position(|(k, _)| *k == key) {+ bin[idx] = (key, value);+ } else {+ bin.push((key, value));+ self.len += 1;+ }+ }++ fn grow(&mut self) {+ let mut bins = Vec::new();++ for _ in 0..self.bins.len() * 2 {+ bins.push(Vec::new());+ }++ std::mem::swap(&mut self.bins, &mut bins);++ for bin in bins {+ for (key, value) in bin {+ let bin_idx = self.bin_idx(&key);+ self.bins[bin_idx].push((key, value));+ }+ }+ }++ pub fn get(&self, key: &Key) -> Option<&V> {+ let bin = self.bin_idx(key);+ self.bins[bin]+ .iter()+ .find(|(stored_key, _)| stored_key == key)+ .map(|(_, val)| val)+ }+}++impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {+ fn from_iter<T: IntoIterator<Item = (u64, V)>>(iter: T) -> Self {+ let mut map = Self::new();++ for (key, val) in iter {+ map.insert(key, val);+ }++ map+ }+}++impl<V> Default for IntMap<V> {+ fn default() -> Self {+ Self::new()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple() {+ let mut map = IntMap::new();++ assert_eq!(map.len, 0);++ map.insert(42, "test".to_string());++ assert_eq!(map.len, 1);+ assert_eq!(map.get(&42), Some(&"test".to_string()));+ assert_eq!(map.get(&43), None);++ map.insert(43, "kage".to_string());+ assert_eq!(map.get(&43), Some(&"kage".to_string()));++ for key in 0..1000 {+ map.insert(key, key.to_string());++ assert_eq!(map.get(&key), Some(&key.to_string()));+ }++ assert_eq!(map.len, 1000);+ }+}diff --git a/src/lib.rs b/src/lib.rsindex c4a9f897..a61d624f 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -40,9 +40,11 @@ mod exponential_backoff;mod fastfield_cache;mod frontend;mod human_website_annotations;+mod hyperloglog;mod image_downloader;mod image_store;pub mod index;+mod intmap;mod kv;pub mod prehashed;mod query;diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rsindex 2aba833e..8f61dde9 100644--- a/src/webgraph/centrality/approximate_harmonic.rs+++ b/src/webgraph/centrality/approximate_harmonic.rs@@ -29,11 +29,11 @@ use std::io::{BufReader, Read, Write};use std::path::Path;use std::sync::{Arc, Mutex};-use boomphf::hashmap::BoomHashMap;use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};+use crate::intmap::IntMap;use crate::webgraph::Node;use crate::webgraph::{centrality::betweenness::Betweenness, NodeID, Webgraph};use crate::Result;@@ -46,8 +46,8 @@ pub const SHIFT: f64 = 1.0;#[derive(Serialize, Deserialize)]pub struct ProxyNode {pub id: NodeID,- dist_from_node: BoomHashMap<NodeID, usize>, // from node to proxy node- dist_to_node: BoomHashMap<NodeID, usize>, // from proxy node to other node+ dist_from_node: IntMap<usize>, // from node to proxy node+ dist_to_node: IntMap<usize>, // from proxy node to other node}impl ProxyNode {@@ -274,16 +274,16 @@ impl ApproximatedHarmonicCentrality {.map(|(node, _)| node).filter(|node| node2id.contains_key(node)).map(|node| {- let (ids, dist) = graph.raw_host_distances(node.clone()).into_iter().unzip();- let (rev_ids, rev_dist) = graph+ let dist_to_node = graph.raw_host_distances(node.clone()).into_iter().collect();+ let dist_from_node = graph.raw_host_reversed_distances(node.clone()).into_iter()- .unzip();+ .collect();Arc::new(ProxyNode {id: *node2id.get(node).unwrap(),- dist_to_node: BoomHashMap::new(ids, dist),- dist_from_node: BoomHashMap::new(rev_ids, rev_dist),+ dist_to_node,+ dist_from_node,})}).collect();