Skip to content

web search done right

remove some files from git lfs

Mikkel Denkerc3e51d23e870
diff --git a/.gitignore b/.gitignore
index 30896fa4..27a4ffd2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,3 +15,5 @@ notes
/data/**/OPTIONS*
/data/**/CURRENT
/data/**/LOG
+/data/webgraph
+/data/warc_files
diff --git a/.ignore b/.ignore
new file mode 100644
index 00000000..93dba0ca
--- /dev/null
+++ b/.ignore
@@ -0,0 +1 @@
+data/**/*
diff --git a/configs/webgraph/local/master.toml b/configs/webgraph/local/master.toml
new file mode 100644
index 00000000..a7192ced
--- /dev/null
+++ b/configs/webgraph/local/master.toml
@@ -0,0 +1,6 @@
+workers = ["0.0.0.0:1338"]
+
+[warc_source]
+type = "local"
+folder = "./warc_files"
+names = ["small1.warc.gz", "small2.warc.gz"]
diff --git a/configs/webgraph/local/profile.toml b/configs/webgraph/local/profile.toml
new file mode 100644
index 00000000..e9f56997
--- /dev/null
+++ b/configs/webgraph/local/profile.toml
@@ -0,0 +1,4 @@
+[warc_source]
+type = "Local"
+folder = "./warc_files"
+names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]
diff --git a/configs/webgraph/master.toml b/configs/webgraph/master.toml
new file mode 100644
index 00000000..69e3e024
--- /dev/null
+++ b/configs/webgraph/master.toml
@@ -0,0 +1,8 @@
+limit_warc_files = 10000
+workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]
+graph_base_path = "/mnt/webgraph"
+
+[warc_source]
+type = "HTTP"
+base_url = "https://data.commoncrawl.org/"
+warc_paths_file = "warc.paths"
diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rs
new file mode 100644
index 00000000..7ecd931a
--- /dev/null
+++ b/src/webgraph/graph_store.rs
@@ -0,0 +1,457 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::{
+ cell::RefCell, collections::HashMap, hash::Hash, marker::PhantomData, ops::Div, path::Path,
+};
+
+use lru::LruCache;
+use serde::{de::DeserializeOwned, Serialize};
+
+use super::{Edge, EdgeIterator, Node, NodeID, Store, StoredEdge};
+use crate::kv::{rocksdb_store::RocksDbStore, sled_store::SledStore, Kv};
+pub(crate) struct Adjacency {
+ pub(crate) tree: BlockedCachedTree<NodeID, Vec<StoredEdge>>,
+}
+
+impl Adjacency {
+ fn insert(&mut self, from: NodeID, to: NodeID, label: String) {
+ self.tree.insert(from, &mut |block| {
+ block.entry(from).or_default().push(StoredEdge {
+ other: to,
+ label: label.clone(),
+ })
+ });
+ }
+
+ fn edges(&mut self, node: NodeID) -> Vec<StoredEdge> {
+ self.tree.get(&node).cloned().unwrap_or_default()
+ }
+}
+
+pub(crate) struct BlockedCachedTree<K, V>
+where
+ K: Hash + Eq + Serialize + Clone + Div<u64, Output = u64> + DeserializeOwned,
+ V: Serialize + DeserializeOwned + Clone,
+{
+ pub(crate) inner: CachedTree<u64, HashMap<K, V>>,
+ pub(crate) block_size: u64,
+}
+
+impl<K, V> BlockedCachedTree<K, V>
+where
+ K: Hash + Eq + Serialize + Clone + Div<u64, Output = u64> + DeserializeOwned,
+ V: Serialize + DeserializeOwned + Clone,
+{
+ fn insert<B>(&mut self, key: K, mutate_block: &mut B)
+ where
+ B: FnMut(&mut HashMap<K, V>),
+ {
+ let block_id = key / self.block_size;
+
+ if let Some(block) = self.inner.get_mut(&block_id) {
+ // block.entry(key).or_default().push(value);
+ // block.insert(key, value);
+ mutate_block(block);
+ return;
+ }
+
+ let mut new_block = HashMap::new();
+ mutate_block(&mut new_block);
+
+ self.inner.insert(block_id, new_block);
+ }
+
+ fn get(&mut self, key: &K) -> Option<&V> {
+ let block_id = key.clone() / self.block_size;
+ self.inner.get(&block_id).and_then(|block| block.get(key))
+ }
+}
+
+pub(crate) struct CachedTree<K, V>
+where
+ K: Hash + Eq + Serialize + DeserializeOwned + Clone,
+ V: Serialize + DeserializeOwned + Clone,
+{
+ pub(crate) store: Box<dyn Kv<K, V> + Send + Sync>,
+ pub(crate) cache: LruCache<K, V>,
+}
+
+impl<K, V> CachedTree<K, V>
+where
+ K: Hash + Eq + Serialize + DeserializeOwned + Clone,
+ V: Serialize + DeserializeOwned + Clone,
+{
+ pub(crate) fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {
+ Self {
+ store,
+ cache: LruCache::new(cache_size),
+ }
+ }
+
+ fn update_cache(&mut self, key: &K) {
+ if !self.cache.contains(key) {
+ let val = self.store.get(key);
+
+ if let Some(val) = val {
+ self.cache.put(key.clone(), val);
+ }
+ }
+ }
+
+ pub(crate) fn get(&mut self, key: &K) -> Option<&V> {
+ self.update_cache(key);
+ self.cache.get(key)
+ }
+
+ fn get_mut(&mut self, key: &K) -> Option<&mut V> {
+ self.update_cache(key);
+ self.cache.get_mut(key)
+ }
+
+ fn insert(&mut self, key: K, value: V) {
+ if self.cache.len() == self.cache.cap() {
+ if let Some((key, value)) = self.cache.pop_lru() {
+ self.store.insert(key, value);
+ }
+ }
+
+ self.cache.push(key, value);
+ }
+
+ fn flush(&mut self) {
+ for (key, value) in self.cache.iter() {
+ self.store.insert(key.clone(), value.clone());
+ }
+
+ self.store.flush();
+ }
+
+ fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {
+ self.store.iter()
+ }
+}
+
+pub struct GraphStore<S> {
+ pub(crate) adjacency: RefCell<Adjacency>,
+ pub(crate) reversed_adjacency: RefCell<Adjacency>,
+ pub(crate) node2id: RefCell<CachedTree<Node, NodeID>>,
+ pub(crate) id2node: RefCell<BlockedCachedTree<NodeID, Node>>,
+ pub(crate) meta: RefCell<CachedTree<String, u64>>,
+ pub(crate) store: PhantomData<S>,
+}
+
+impl<S: Store> GraphStore<S> {
+ #[cfg(test)]
+ pub(crate) fn temporary() -> GraphStore<S> {
+ S::temporary()
+ }
+
+ pub fn open<P: AsRef<Path>>(path: P) -> Self {
+ S::open(path)
+ }
+
+ fn next_id(&self) -> NodeID {
+ self.meta
+ .borrow_mut()
+ .get(&"next_id".to_string())
+ .cloned()
+ .unwrap_or(0)
+ }
+
+ fn increment_next_id(&self) {
+ let current_id = self.next_id();
+ let next_id = current_id + 1;
+ self.meta
+ .borrow_mut()
+ .insert("next_id".to_string(), next_id);
+ }
+
+ fn id_and_increment(&self) -> NodeID {
+ let id = self.next_id();
+ self.increment_next_id();
+ id
+ }
+
+ fn assign_id(&self, node: Node, id: NodeID) {
+ self.node2id.borrow_mut().insert(node.clone(), id);
+ self.id2node.borrow_mut().insert(id, &mut |block| {
+ block.insert(id, node.clone());
+ });
+ }
+
+ fn id_or_assign(&self, node: Node) -> NodeID {
+ if let Some(id) = self.node2id.borrow_mut().get(&node) {
+ return *id;
+ }
+ let id = self.id_and_increment();
+ self.assign_id(node, id);
+ id
+ }
+ pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {
+ self.adjacency
+ .borrow_mut()
+ .edges(node)
+ .into_iter()
+ .map(|edge| Edge {
+ from: node,
+ to: edge.other,
+ label: edge.label,
+ })
+ .collect()
+ }
+
+ pub fn ingoing_edges(&self, node: NodeID) -> Vec<Edge> {
+ self.reversed_adjacency
+ .borrow_mut()
+ .edges(node)
+ .into_iter()
+ .map(|edge| Edge {
+ from: edge.other,
+ to: node,
+ label: edge.label,
+ })
+ .collect()
+ }
+
+ pub fn nodes(&self) -> impl Iterator<Item = NodeID> {
+ self.node2id
+ .borrow_mut()
+ .iter()
+ .map(|(_, id)| id)
+ .collect::<Vec<u64>>()
+ .into_iter()
+ }
+
+ pub fn insert(&mut self, from: Node, to: Node, label: String) {
+ let from_id = self.id_or_assign(from);
+ let to_id = self.id_or_assign(to);
+
+ self.adjacency
+ .borrow_mut()
+ .insert(from_id, to_id, label.clone());
+ self.reversed_adjacency
+ .borrow_mut()
+ .insert(to_id, from_id, label);
+ }
+
+ pub fn node2id(&self, node: &Node) -> Option<NodeID> {
+ self.node2id.borrow_mut().get(node).cloned()
+ }
+
+ pub fn id2node(&self, id: &NodeID) -> Option<Node> {
+ self.id2node.borrow_mut().get(id).cloned()
+ }
+
+ pub fn flush(&self) {
+ self.adjacency.borrow_mut().tree.inner.flush();
+ self.reversed_adjacency.borrow_mut().tree.inner.flush();
+ self.node2id.borrow_mut().flush();
+ self.id2node.borrow_mut().inner.flush();
+ self.meta.borrow_mut().flush();
+ }
+
+ pub fn edges(&self) -> EdgeIterator<'_> {
+ self.flush();
+
+ EdgeIterator::new(&self.adjacency)
+ }
+
+ pub fn append(&mut self, other: GraphStore<S>) {
+ for edge in other.edges() {
+ let from = other.id2node(&edge.from).expect("node not found");
+ let to = other.id2node(&edge.to).expect("node not found");
+
+ self.insert(from, to, edge.label);
+ }
+ }
+}
+
+impl Store for RocksDbStore {
+ fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {
+ let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));
+ let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));
+ let node2id = RocksDbStore::open(path.as_ref().join("node2id"));
+ let id2node = RocksDbStore::open(path.as_ref().join("id2node"));
+ let meta = RocksDbStore::open(path.as_ref().join("meta"));
+
+ GraphStore {
+ adjacency: RefCell::new(Adjacency {
+ tree: BlockedCachedTree {
+ inner: CachedTree::new(adjacency, 10_000),
+ block_size: 1_024,
+ },
+ }),
+ reversed_adjacency: RefCell::new(Adjacency {
+ tree: BlockedCachedTree {
+ inner: CachedTree::new(reversed_adjacency, 10_000),
+ block_size: 1_024,
+ },
+ }),
+ node2id: RefCell::new(CachedTree::new(node2id, 100_000)),
+ id2node: RefCell::new(BlockedCachedTree {
+ inner: CachedTree::new(id2node, 100_000),
+ block_size: 1_024,
+ }),
+ meta: RefCell::new(CachedTree::new(meta, 1_000)),
+ store: Default::default(),
+ }
+ }
+}
+
+impl Store for SledStore {
+ fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {
+ let db = sled::Config::default()
+ .path(path)
+ .use_compression(true)
+ .mode(sled::Mode::LowSpace)
+ .open()
+ .expect("Failed to open database");
+
+ let adjacency = Box::new(db.open_tree("adjacency").expect("unable to open sled tree"));
+ let reversed_adjacency = Box::new(
+ db.open_tree("reversed_adjacency")
+ .expect("unable to open sled tree"),
+ );
+ let node2id = Box::new(db.open_tree("node2id").expect("unable to open sled tree"));
+ let id2node = Box::new(db.open_tree("id2node").expect("unable to open sled tree"));
+ let meta = Box::new(db.open_tree("meta").expect("unable to open sled tree"));
+
+ GraphStore {
+ adjacency: RefCell::new(Adjacency {
+ tree: BlockedCachedTree {
+ inner: CachedTree::new(adjacency, 10_000),
+ block_size: 1_024,
+ },
+ }),
+ reversed_adjacency: RefCell::new(Adjacency {
+ tree: BlockedCachedTree {
+ inner: CachedTree::new(reversed_adjacency, 10_000),
+ block_size: 1_024,
+ },
+ }),
+ node2id: RefCell::new(CachedTree::new(node2id, 100_000)),
+ id2node: RefCell::new(BlockedCachedTree {
+ inner: CachedTree::new(id2node, 100_000),
+ block_size: 1_024,
+ }),
+ meta: RefCell::new(CachedTree::new(meta, 1_000)),
+ store: Default::default(),
+ }
+ }
+}
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ #[test]
+ fn simple_triangle_graph() {
+ // ┌────┐
+ // │ │
+ // ┌───A◄─┐ │
+ // │ │ │
+ // ▼ │ │
+ // B─────►C◄┘
+
+ let mut store: GraphStore<RocksDbStore> = GraphStore::temporary();
+
+ let a = Node {
+ name: "A".to_string(),
+ };
+ let b = Node {
+ name: "B".to_string(),
+ };
+ let c = Node {
+ name: "C".to_string(),
+ };
+
+ store.insert(a.clone(), b.clone(), String::new());
+ store.insert(b.clone(), c.clone(), String::new());
+ store.insert(c.clone(), a.clone(), String::new());
+ store.insert(a.clone(), c.clone(), String::new());
+
+ store.flush();
+
+ let nodes: Vec<Node> = store
+ .nodes()
+ .map(|id| store.id2node.borrow_mut().get(&id).unwrap().clone())
+ .collect();
+ assert_eq!(nodes, vec![a.clone(), b.clone(), c.clone()]);
+
+ let a_id = store.node2id(&a).unwrap();
+ let b_id = store.node2id(&b).unwrap();
+ let c_id = store.node2id(&c).unwrap();
+
+ assert_eq!(
+ store.outgoing_edges(a_id),
+ vec![
+ Edge {
+ from: a_id,
+ to: b_id,
+ label: String::new()
+ },
+ Edge {
+ from: a_id,
+ to: c_id,
+ label: String::new()
+ },
+ ]
+ );
+
+ assert_eq!(
+ store.outgoing_edges(b_id),
+ vec![Edge {
+ from: b_id,
+ to: c_id,
+ label: String::new()
+ },]
+ );
+
+ assert_eq!(
+ store.ingoing_edges(c_id),
+ vec![
+ Edge {
+ from: b_id,
+ to: c_id,
+ label: String::new()
+ },
+ Edge {
+ from: a_id,
+ to: c_id,
+ label: String::new()
+ },
+ ]
+ );
+
+ assert_eq!(
+ store.ingoing_edges(a_id),
+ vec![Edge {
+ from: c_id,
+ to: a_id,
+ label: String::new()
+ },]
+ );
+
+ assert_eq!(
+ store.ingoing_edges(b_id),
+ vec![Edge {
+ from: a_id,
+ to: b_id,
+ label: String::new()
+ },]
+ );
+ }
+}
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
new file mode 100644
index 00000000..41c28779
--- /dev/null
+++ b/src/webgraph/mod.rs
@@ -0,0 +1,653 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod graph_store;
+
+use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};
+use serde::{Deserialize, Serialize};
+use std::cell::RefCell;
+use std::collections::{BinaryHeap, HashMap};
+use std::path::Path;
+use std::{cmp, fs};
+
+use graph_store::GraphStore;
+
+use crate::directory::{self, DirEntry};
+use crate::webpage::Url;
+
+use self::graph_store::Adjacency;
+use crate::kv::rocksdb_store::RocksDbStore;
+
+type NodeID = u64;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub(crate) struct StoredEdge {
+ other: NodeID,
+ label: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct Node {
+ pub name: String,
+}
+
+impl Node {
+ fn into_host(self) -> Node {
+ Node {
+ name: Url::from(self.name).host().to_string(),
+ }
+ }
+}
+
+impl From<String> for Node {
+ fn from(name: String) -> Self {
+ Self { name }
+ }
+}
+
+impl From<&str> for Node {
+ fn from(name: &str) -> Self {
+ Self::from(name.to_string())
+ }
+}
+
+impl From<Url> for Node {
+ fn from(url: Url) -> Self {
+ Self {
+ name: url.to_string(),
+ }
+ }
+}
+
+pub struct EdgeIterator<'a> {
+ current_block_idx: usize,
+ blocks: Vec<u64>,
+ adjacency: &'a RefCell<Adjacency>,
+ current_block: Option<Box<dyn Iterator<Item = Edge>>>,
+}
+
+impl<'a> EdgeIterator<'a> {
+ fn new(adjacency: &'a RefCell<Adjacency>) -> EdgeIterator<'a> {
+ let blocks: Vec<_> = adjacency
+ .borrow()
+ .tree
+ .inner
+ .store
+ .iter()
+ .map(|(key, _)| key)
+ .collect();
+
+ EdgeIterator {
+ current_block_idx: 0,
+ blocks,
+ adjacency,
+ current_block: None,
+ }
+ }
+
+ fn load_next_block(&mut self) {
+ if self.current_block_idx < self.blocks.len() {
+ let block_id = self.blocks[self.current_block_idx];
+ let block = self
+ .adjacency
+ .borrow_mut()
+ .tree
+ .inner
+ .get(&block_id)
+ .unwrap()
+ .clone();
+
+ self.current_block = Some(Box::new(block.into_iter().flat_map(|(node_id, edges)| {
+ edges.into_iter().map(move |edge| Edge {
+ from: node_id,
+ to: edge.other,
+ label: edge.label,
+ })
+ })));
+
+ self.current_block_idx += 1;
+ }
+ }
+}
+
+impl<'a> Iterator for EdgeIterator<'a> {
+ type Item = Edge;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ if let Some(res) = self.current_block.as_mut().and_then(|it| it.next()) {
+ return Some(res);
+ }
+
+ if self.current_block.is_none() {
+ self.load_next_block();
+ }
+
+ self.current_block.as_mut().and_then(|it| it.next())
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Edge {
+ from: NodeID,
+ to: NodeID,
+ label: String,
+}
+
+pub struct WebgraphBuilder {
+ path: Box<Path>,
+ full_graph_path: Option<Box<Path>>,
+ host_graph_path: Option<Box<Path>>,
+}
+
+impl WebgraphBuilder {
+ #[cfg(test)]
+ pub fn new_memory() -> Self {
+ use crate::gen_temp_path;
+
+ let path = gen_temp_path();
+ Self::new(path)
+ }
+
+ pub fn new<P: AsRef<Path>>(path: P) -> Self {
+ Self {
+ path: path.as_ref().into(),
+ full_graph_path: None,
+ host_graph_path: None,
+ }
+ }
+
+ pub fn with_full_graph(mut self) -> Self {
+ self.full_graph_path = Some(self.path.join("full").into());
+ self
+ }
+
+ pub fn with_host_graph(mut self) -> Self {
+ self.host_graph_path = Some(self.path.join("host").into());
+ self
+ }
+
+ pub fn open(self) -> Webgraph {
+ Webgraph {
+ full_graph: self.full_graph_path.map(GraphStore::open),
+ host_graph: self.host_graph_path.map(GraphStore::open),
+ path: self.path.to_str().unwrap().to_string(),
+ }
+ }
+}
+
+pub trait Store
+where
+ Self: Sized,
+{
+ fn open<P: AsRef<Path>>(path: P) -> GraphStore<Self>;
+
+ fn temporary() -> GraphStore<Self> {
+ Self::open(crate::gen_temp_path())
+ }
+}
+
+pub struct Webgraph<S: Store = RocksDbStore> {
+ pub path: String,
+ full_graph: Option<GraphStore<S>>,
+ host_graph: Option<GraphStore<S>>,
+}
+
+impl<S: Store> Webgraph<S> {
+ pub fn insert(&mut self, from: Node, to: Node, label: String) {
+ if let Some(full_graph) = &mut self.full_graph {
+ full_graph.insert(from.clone(), to.clone(), label.clone());
+ }
+
+ if let Some(host_graph) = &mut self.host_graph {
+ host_graph.insert(from.into_host(), to.into_host(), label);
+ }
+ }
+
+ pub fn merge(&mut self, other: Webgraph<S>) {
+ match (&mut self.full_graph, other.full_graph) {
+ (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),
+ (None, Some(other_graph)) => self.full_graph = Some(other_graph),
+ (Some(_), None) | (None, None) => {}
+ }
+
+ match (&mut self.host_graph, other.host_graph) {
+ (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),
+ (None, Some(other_graph)) => self.host_graph = Some(other_graph),
+ (Some(_), None) | (None, None) => {}
+ }
+
+ self.flush();
+ }
+
+ fn dijkstra<F1, F2>(
+ source: Node,
+ node_edges: F1,
+ edge_node: F2,
+ store: &GraphStore<S>,
+ ) -> HashMap<NodeID, usize>
+ where
+ F1: Fn(NodeID) -> Vec<Edge>,
+ F2: Fn(&Edge) -> NodeID,
+ {
+ let source_id = store.node2id(&source);
+ if source_id.is_none() {
+ return HashMap::new();
+ }
+
+ let source_id = source_id.unwrap();
+ let mut distances: HashMap<NodeID, usize> = HashMap::default();
+
+ let mut queue = BinaryHeap::new();
+
+ queue.push(cmp::Reverse((0_usize, source_id)));
+ distances.insert(source_id, 0);
+
+ while let Some(state) = queue.pop() {
+ let (cost, v) = state.0;
+ let current_dist = distances.get(&v).unwrap_or(&usize::MAX);
+
+ if cost > *current_dist {
+ continue;
+ }
+
+ for edge in node_edges(v) {
+ if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {
+ let next = cmp::Reverse((cost + 1, edge_node(&edge)));
+ queue.push(next);
+ distances.insert(edge_node(&edge), cost + 1);
+ }
+ }
+ }
+
+ distances
+ }
+
+ pub fn distances(&self, source: Node) -> HashMap<Node, usize> {
+ self.full_graph
+ .as_ref()
+ .map(|full_graph| {
+ let distances = Webgraph::dijkstra(
+ source,
+ |node_id| full_graph.outgoing_edges(node_id),
+ |edge| edge.to,
+ full_graph,
+ );
+
+ distances
+ .into_iter()
+ .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))
+ .collect()
+ })
+ .unwrap_or_default()
+ }
+
+ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {
+ self.full_graph
+ .as_ref()
+ .map(|full_graph| {
+ Webgraph::dijkstra(
+ source,
+ |node| full_graph.ingoing_edges(node),
+ |edge| edge.from,
+ full_graph,
+ )
+ })
+ .unwrap_or_default()
+ }
+
+ pub fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
+ self.full_graph
+ .as_ref()
+ .map(|full_graph| {
+ self.raw_reversed_distances(source)
+ .into_iter()
+ .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))
+ .collect()
+ })
+ .unwrap_or_default()
+ }
+
+ pub fn host_distances(&self, source: Node) -> HashMap<Node, usize> {
+ self.host_graph
+ .as_ref()
+ .map(|host_graph| {
+ let distances = Webgraph::dijkstra(
+ source,
+ |node| host_graph.outgoing_edges(node),
+ |edge| edge.to,
+ host_graph,
+ );
+
+ distances
+ .into_iter()
+ .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))
+ .collect()
+ })
+ .unwrap_or_default()
+ }
+
+ fn raw_host_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {
+ self.host_graph
+ .as_ref()
+ .map(|host_graph| {
+ Webgraph::dijkstra(
+ source,
+ |node| host_graph.ingoing_edges(node),
+ |edge| edge.from,
+ host_graph,
+ )
+ })
+ .unwrap_or_default()
+ }
+
+ pub fn host_reversed_distances(&self, source: Node) -> HashMap<Node, usize> {
+ self.host_graph
+ .as_ref()
+ .map(|host_graph| {
+ self.raw_host_reversed_distances(source)
+ .into_iter()
+ .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))
+ .collect()
+ })
+ .unwrap_or_default()
+ }
+
+ fn calculate_centrality<F>(graph: &GraphStore<S>, node_distances: F) -> HashMap<Node, f64>
+ where
+ F: Fn(Node) -> HashMap<NodeID, usize>,
+ {
+ let nodes: Vec<_> = graph.nodes().collect();
+ dbg!(&nodes);
+ let pb = ProgressBar::new(nodes.len() as u64);
+ pb.set_style(
+ ProgressStyle::default_bar()
+ .template(
+ "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",
+ )
+ .progress_chars("#>-"),
+ );
+ let norm_factor = (nodes.len() - 1) as f64;
+ nodes
+ .iter()
+ .progress_with(pb)
+ .map(|node_id| {
+ let node = graph.id2node(node_id).expect("unknown node");
+ let centrality_values: HashMap<NodeID, f64> = node_distances(node.clone())
+ .into_iter()
+ .filter(|(other_id, _)| *other_id != *node_id)
+ .map(|(other_node, dist)| (other_node, 1f64 / dist as f64))
+ .collect();
+
+ let centrality = centrality_values
+ .into_iter()
+ .map(|(_, val)| val)
+ .sum::<f64>()
+ / norm_factor;
+
+ (node, centrality)
+ })
+ .collect()
+ }
+
+ pub fn harmonic_centrality(&self) -> HashMap<Node, f64> {
+ self.full_graph
+ .as_ref()
+ .map(|full_graph| {
+ Webgraph::calculate_centrality(full_graph, |node| self.raw_reversed_distances(node))
+ })
+ .unwrap_or_default()
+ }
+
+ pub fn host_harmonic_centrality(&self) -> HashMap<Node, f64> {
+ self.host_graph
+ .as_ref()
+ .map(|host_graph| {
+ Webgraph::calculate_centrality(host_graph, |node| {
+ self.raw_host_reversed_distances(node)
+ })
+ })
+ .unwrap_or_default()
+ }
+
+ pub fn flush(&self) {
+ if let Some(full_graph) = &self.full_graph {
+ full_graph.flush();
+ }
+ if let Some(host_graph) = &self.host_graph {
+ host_graph.flush();
+ }
+ }
+}
+
+impl From<FrozenWebgraph> for Webgraph {
+ fn from(frozen: FrozenWebgraph) -> Self {
+ let path = match &frozen.root {
+ DirEntry::Folder { name, entries: _ } => name.clone(),
+ DirEntry::File {
+ name: _,
+ content: _,
+ } => {
+ panic!("Cannot open webgraph from a file - must be directory.")
+ }
+ };
+
+ if Path::new(&path).exists() {
+ fs::remove_dir_all(&path).unwrap();
+ }
+
+ directory::recreate_folder(&frozen.root).unwrap();
+
+ let mut builder = WebgraphBuilder::new(path);
+
+ if frozen.has_full {
+ builder = builder.with_full_graph();
+ }
+
+ if frozen.has_host {
+ builder = builder.with_host_graph();
+ }
+
+ builder.open()
+ }
+}
+
+impl From<Webgraph> for FrozenWebgraph {
+ fn from(graph: Webgraph) -> Self {
+ graph.flush();
+ let path = graph.path.clone();
+ let has_full = graph.full_graph.is_some();
+ let has_host = graph.host_graph.is_some();
+ drop(graph);
+ let root = directory::scan_folder(path).unwrap();
+
+ Self {
+ root,
+ has_full,
+ has_host,
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FrozenWebgraph {
+ pub root: DirEntry,
+ has_full: bool,
+ has_host: bool,
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ fn test_graph() -> Webgraph {
+ // ┌────┐
+ // │ │
+ // ┌───A◄─┐ │
+ // │ │ │
+ // ▼ │ │
+ // B─────►C◄┘
+ // ▲
+ // │
+ // │
+ // D
+
+ let mut graph = WebgraphBuilder::new_memory()
+ .with_full_graph()
+ .with_host_graph()
+ .open();
+
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+ graph.insert(Node::from("B"), Node::from("C"), String::new());
+ graph.insert(Node::from("A"), Node::from("C"), String::new());
+ graph.insert(Node::from("C"), Node::from("A"), String::new());
+ graph.insert(Node::from("D"), Node::from("C"), String::new());
+
+ graph.flush();
+
+ graph
+ }
+
+ #[test]
+ fn distance_calculation() {
+ let graph = test_graph();
+
+ let distances = graph.distances(Node::from("D"));
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("A")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&3));
+ }
+
+ #[test]
+ fn nonexisting_node() {
+ let graph = test_graph();
+ assert_eq!(graph.distances(Node::from("E")).len(), 0);
+ assert_eq!(graph.reversed_distances(Node::from("E")).len(), 0);
+ }
+
+ #[test]
+ fn reversed_distance_calculation() {
+ let graph = test_graph();
+
+ let distances = graph.reversed_distances(Node::from("D"));
+
+ assert_eq!(distances.get(&Node::from("C")), None);
+ assert_eq!(distances.get(&Node::from("A")), None);
+ assert_eq!(distances.get(&Node::from("B")), None);
+
+ let distances = graph.reversed_distances(Node::from("A"));
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("D")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&2));
+ }
+
+ #[test]
+ fn harmonic_centrality() {
+ let graph = test_graph();
+
+ let centrality = graph.harmonic_centrality();
+
+ assert_eq!(centrality.get(&Node::from("C")).unwrap(), &1.0);
+ assert_eq!(centrality.get(&Node::from("D")).unwrap(), &0.0);
+ assert_eq!(
+ (*centrality.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,
+ 0.67
+ );
+ assert_eq!(
+ (*centrality.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,
+ 0.61
+ );
+ }
+
+ #[test]
+ fn host_harmonic_centrality() {
+ let mut graph = WebgraphBuilder::new_memory()
+ .with_full_graph()
+ .with_host_graph()
+ .open();
+
+ graph.insert(Node::from("A.com/1"), Node::from("A.com/2"), String::new());
+ graph.insert(Node::from("A.com/1"), Node::from("A.com/3"), String::new());
+ graph.insert(Node::from("A.com/1"), Node::from("A.com/4"), String::new());
+ graph.insert(Node::from("A.com/2"), Node::from("A.com/1"), String::new());
+ graph.insert(Node::from("A.com/2"), Node::from("A.com/3"), String::new());
+ graph.insert(Node::from("A.com/2"), Node::from("A.com/4"), String::new());
+ graph.insert(Node::from("A.com/3"), Node::from("A.com/1"), String::new());
+ graph.insert(Node::from("A.com/3"), Node::from("A.com/2"), String::new());
+ graph.insert(Node::from("A.com/3"), Node::from("A.com/4"), String::new());
+ graph.insert(Node::from("A.com/4"), Node::from("A.com/1"), String::new());
+ graph.insert(Node::from("A.com/4"), Node::from("A.com/2"), String::new());
+ graph.insert(Node::from("A.com/4"), Node::from("A.com/3"), String::new());
+ graph.insert(Node::from("C.com"), Node::from("B.com"), String::new());
+ graph.insert(Node::from("D.com"), Node::from("B.com"), String::new());
+
+ graph.flush();
+
+ let centrality = graph.harmonic_centrality();
+ assert!(
+ centrality.get(&Node::from("A.com/1")).unwrap()
+ > centrality.get(&Node::from("B.com")).unwrap()
+ );
+
+ let host_centrality = graph.host_harmonic_centrality();
+ assert!(
+ host_centrality.get(&Node::from("B.com")).unwrap()
+ > host_centrality.get(&Node::from("A.com")).unwrap()
+ );
+ }
+
+ #[test]
+ fn merge() {
+ let mut graph1 = WebgraphBuilder::new_memory()
+ .with_full_graph()
+ .with_host_graph()
+ .open();
+
+ graph1.insert(Node::from("A"), Node::from("B"), String::new());
+
+ let mut graph2 = WebgraphBuilder::new_memory()
+ .with_full_graph()
+ .with_host_graph()
+ .open();
+ graph2.insert(Node::from("B"), Node::from("C"), String::new());
+
+ graph1.merge(graph2);
+
+ assert_eq!(
+ graph1.distances(Node::from("A")).get(&Node::from("C")),
+ Some(&2)
+ )
+ }
+
+ #[test]
+ fn serialize_deserialize_bincode() {
+ let graph = test_graph();
+ let path = graph.path.clone();
+ let frozen: FrozenWebgraph = graph.into();
+ let bytes = bincode::serialize(&frozen).unwrap();
+
+ std::fs::remove_dir_all(path).unwrap();
+
+ let deserialized_frozen: FrozenWebgraph = bincode::deserialize(&bytes).unwrap();
+ let graph: Webgraph = deserialized_frozen.into();
+
+ let distances = graph.distances(Node::from("D"));
+
+ assert_eq!(distances.get(&Node::from("C")), Some(&1));
+ assert_eq!(distances.get(&Node::from("A")), Some(&2));
+ assert_eq!(distances.get(&Node::from("B")), Some(&3));
+ }
+}
diff --git a/templates/search.html b/templates/search.html
index cc0313ee..112fcc94 100644
--- a/templates/search.html
+++ b/templates/search.html
@@ -17,7 +17,7 @@
<div class="flex pt-8">
<div class="search-result flex flex-col space-y-3 pb-8 pl-40">
{% for item in search_result %}
- <div class="flex max-w-3xl w-full p-3 h-36">
+ <div class="flex max-w-3xl w-full p-3">
<div class="flex flex-col w-full">
<div class="text-sm mb-1 flex">
<div class="flex items-center mr-2">