Skip to content

web search done right

specialized integer hashmap implementation for speedup in apprxomate harmonic calculations

Mikkel Denkercfe03f99925c
diff --git a/Cargo.lock b/Cargo.lock
index 2246aff5..7acb7745 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -315,19 +315,6 @@ dependencies = [
"generic-array",
]
-[[package]]
-name = "boomphf"
-version = "0.5.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e8fb7f38fef59c32549861151d63a6190865e60cf690340c13e3d7178b42a2f"
-dependencies = [
- "crossbeam-utils",
- "log",
- "rayon",
- "serde",
- "wyhash",
-]
-
[[package]]
name = "bstr"
version = "0.2.17"
@@ -733,7 +720,6 @@ dependencies = [
"axum-macros",
"base64",
"bincode",
- "boomphf",
"bzip2",
"chrono",
"clap 3.1.18",
@@ -3859,15 +3845,6 @@ dependencies = [
"winapi",
]
-[[package]]
-name = "wyhash"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf6e163c25e3fac820b4b453185ea2dea3b6a3e0a721d4d23d75bd33734c295"
-dependencies = [
- "rand_core 0.6.3",
-]
-
[[package]]
name = "xml-rs"
version = "0.8.4"
diff --git a/Cargo.toml b/Cargo.toml
index 4a4d7c1b..1ce3fccd 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -61,7 +61,6 @@ rust-stemmers = "1.2.0"
min-max-heap = "1.3.0"
base64 = "0.13.0"
async-trait = "0.1.57"
-boomphf = { version = "0.5.9", features = ["serde"] }
axum-macros = "0.2.3"
urlencoding = "2.1.2"
rusoto_s3 = { version = "0.48.0", features = ["rustls"], default-features = false, optional = true }
diff --git a/src/hyperloglog.rs b/src/hyperloglog.rs
new file mode 100644
index 00000000..811ff243
--- /dev/null
+++ b/src/hyperloglog.rs
@@ -0,0 +1,19 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+struct HyperLogLog<const N: usize = 16> {
+ registers: [u8; N],
+}
diff --git a/src/intmap.rs b/src/intmap.rs
new file mode 100644
index 00000000..4b9cb967
--- /dev/null
+++ b/src/intmap.rs
@@ -0,0 +1,127 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{Deserialize, Serialize};
+
+type Key = u64;
+
+#[derive(Serialize, Deserialize)]
+pub struct IntMap<V> {
+ bins: Vec<Vec<(Key, V)>>,
+ len: usize,
+}
+
+impl<V> IntMap<V> {
+ pub fn new() -> Self {
+ let bins = vec![Vec::new(), Vec::new()];
+
+ Self { bins, len: 0 }
+ }
+
+ fn bin_idx(&self, key: &Key) -> usize {
+ let mask = (self.bins.len() - 1) as Key;
+ let key = key.wrapping_mul(11400714819323198549 as Key);
+ (key & mask) as usize
+ }
+
+ pub fn insert(&mut self, key: Key, value: V) {
+ if self.len >= (self.bins.len() as f64 * 0.8) as usize {
+ self.grow();
+ }
+
+ let bin_idx = self.bin_idx(&key);
+ let bin = &mut self.bins[bin_idx];
+
+ if let Some(idx) = bin.iter().position(|(k, _)| *k == key) {
+ bin[idx] = (key, value);
+ } else {
+ bin.push((key, value));
+ self.len += 1;
+ }
+ }
+
+ fn grow(&mut self) {
+ let mut bins = Vec::new();
+
+ for _ in 0..self.bins.len() * 2 {
+ bins.push(Vec::new());
+ }
+
+ std::mem::swap(&mut self.bins, &mut bins);
+
+ for bin in bins {
+ for (key, value) in bin {
+ let bin_idx = self.bin_idx(&key);
+ self.bins[bin_idx].push((key, value));
+ }
+ }
+ }
+
+ pub fn get(&self, key: &Key) -> Option<&V> {
+ let bin = self.bin_idx(key);
+ self.bins[bin]
+ .iter()
+ .find(|(stored_key, _)| stored_key == key)
+ .map(|(_, val)| val)
+ }
+}
+
+impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {
+ fn from_iter<T: IntoIterator<Item = (u64, V)>>(iter: T) -> Self {
+ let mut map = Self::new();
+
+ for (key, val) in iter {
+ map.insert(key, val);
+ }
+
+ map
+ }
+}
+
+impl<V> Default for IntMap<V> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple() {
+ let mut map = IntMap::new();
+
+ assert_eq!(map.len, 0);
+
+ map.insert(42, "test".to_string());
+
+ assert_eq!(map.len, 1);
+ assert_eq!(map.get(&42), Some(&"test".to_string()));
+ assert_eq!(map.get(&43), None);
+
+ map.insert(43, "kage".to_string());
+ assert_eq!(map.get(&43), Some(&"kage".to_string()));
+
+ for key in 0..1000 {
+ map.insert(key, key.to_string());
+
+ assert_eq!(map.get(&key), Some(&key.to_string()));
+ }
+
+ assert_eq!(map.len, 1000);
+ }
+}
diff --git a/src/lib.rs b/src/lib.rs
index c4a9f897..a61d624f 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -40,9 +40,11 @@ mod exponential_backoff;
mod fastfield_cache;
mod frontend;
mod human_website_annotations;
+mod hyperloglog;
mod image_downloader;
mod image_store;
pub mod index;
+mod intmap;
mod kv;
pub mod prehashed;
mod query;
diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rs
index 2aba833e..8f61dde9 100644
--- a/src/webgraph/centrality/approximate_harmonic.rs
+++ b/src/webgraph/centrality/approximate_harmonic.rs
@@ -29,11 +29,11 @@ use std::io::{BufReader, Read, Write};
use std::path::Path;
use std::sync::{Arc, Mutex};
-use boomphf::hashmap::BoomHashMap;
use itertools::Itertools;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
+use crate::intmap::IntMap;
use crate::webgraph::Node;
use crate::webgraph::{centrality::betweenness::Betweenness, NodeID, Webgraph};
use crate::Result;
@@ -46,8 +46,8 @@ pub const SHIFT: f64 = 1.0;
#[derive(Serialize, Deserialize)]
pub struct ProxyNode {
pub id: NodeID,
- dist_from_node: BoomHashMap<NodeID, usize>, // from node to proxy node
- dist_to_node: BoomHashMap<NodeID, usize>, // from proxy node to other node
+ dist_from_node: IntMap<usize>, // from node to proxy node
+ dist_to_node: IntMap<usize>, // from proxy node to other node
}
impl ProxyNode {
@@ -274,16 +274,16 @@ impl ApproximatedHarmonicCentrality {
.map(|(node, _)| node)
.filter(|node| node2id.contains_key(node))
.map(|node| {
- let (ids, dist) = graph.raw_host_distances(node.clone()).into_iter().unzip();
- let (rev_ids, rev_dist) = graph
+ let dist_to_node = graph.raw_host_distances(node.clone()).into_iter().collect();
+ let dist_from_node = graph
.raw_host_reversed_distances(node.clone())
.into_iter()
- .unzip();
+ .collect();
Arc::new(ProxyNode {
id: *node2id.get(node).unwrap(),
- dist_to_node: BoomHashMap::new(ids, dist),
- dist_from_node: BoomHashMap::new(rev_ids, rev_dist),
+ dist_to_node,
+ dist_from_node,
})
})
.collect();