web search done right
remove some files from git lfs
Mikkel Denkerc3e51d23e870
diff --git a/.gitignore b/.gitignoreindex 30896fa4..27a4ffd2 100644--- a/.gitignore+++ b/.gitignore@@ -15,3 +15,5 @@ notes/data/**/OPTIONS*/data/**/CURRENT/data/**/LOG+/data/webgraph+/data/warc_filesdiff --git a/.ignore b/.ignorenew file mode 100644index 00000000..93dba0ca--- /dev/null+++ b/.ignore@@ -0,0 +1 @@+data/**/*diff --git a/configs/webgraph/local/master.toml b/configs/webgraph/local/master.tomlnew file mode 100644index 00000000..a7192ced--- /dev/null+++ b/configs/webgraph/local/master.toml@@ -0,0 +1,6 @@+workers = ["0.0.0.0:1338"]++[warc_source]+type = "local"+folder = "./warc_files"+names = ["small1.warc.gz", "small2.warc.gz"]diff --git a/configs/webgraph/local/profile.toml b/configs/webgraph/local/profile.tomlnew file mode 100644index 00000000..e9f56997--- /dev/null+++ b/configs/webgraph/local/profile.toml@@ -0,0 +1,4 @@+[warc_source]+type = "Local"+folder = "./warc_files"+names = ["CC-MAIN-20220116093137-20220116123137-00000.warc.gz", "CC-MAIN-20220116093137-20220116123137-00001.warc.gz", "CC-MAIN-20220116093137-20220116123137-00003.warc.gz"]diff --git a/configs/webgraph/master.toml b/configs/webgraph/master.tomlnew file mode 100644index 00000000..69e3e024--- /dev/null+++ b/configs/webgraph/master.toml@@ -0,0 +1,8 @@+limit_warc_files = 10000+workers = ["0.0.0.0:1337", "0.0.0.0:1338", "0.0.0.0:1339", "0.0.0.0:1340", "0.0.0.0:1341", "0.0.0.0:1342", "0.0.0.0:1343"]+graph_base_path = "/mnt/webgraph"++[warc_source]+type = "HTTP"+base_url = "https://data.commoncrawl.org/"+warc_paths_file = "warc.paths"diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rsnew file mode 100644index 00000000..7ecd931a--- /dev/null+++ b/src/webgraph/graph_store.rs@@ -0,0 +1,457 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{+ cell::RefCell, collections::HashMap, hash::Hash, marker::PhantomData, ops::Div, path::Path,+};++use lru::LruCache;+use serde::{de::DeserializeOwned, Serialize};++use super::{Edge, EdgeIterator, Node, NodeID, Store, StoredEdge};+use crate::kv::{rocksdb_store::RocksDbStore, sled_store::SledStore, Kv};+pub(crate) struct Adjacency {+ pub(crate) tree: BlockedCachedTree<NodeID, Vec<StoredEdge>>,+}++impl Adjacency {+ fn insert(&mut self, from: NodeID, to: NodeID, label: String) {+ self.tree.insert(from, &mut |block| {+ block.entry(from).or_default().push(StoredEdge {+ other: to,+ label: label.clone(),+ })+ });+ }++ fn edges(&mut self, node: NodeID) -> Vec<StoredEdge> {+ self.tree.get(&node).cloned().unwrap_or_default()+ }+}++pub(crate) struct BlockedCachedTree<K, V>+where+ K: Hash + Eq + Serialize + Clone + Div<u64, Output = u64> + DeserializeOwned,+ V: Serialize + DeserializeOwned + Clone,+{+ pub(crate) inner: CachedTree<u64, HashMap<K, V>>,+ pub(crate) block_size: u64,+}++impl<K, V> BlockedCachedTree<K, V>+where+ K: Hash + Eq + Serialize + Clone + Div<u64, Output = u64> + DeserializeOwned,+ V: Serialize + DeserializeOwned + Clone,+{+ fn insert<B>(&mut self, key: K, mutate_block: &mut B)+ where+ B: FnMut(&mut HashMap<K, V>),+ {+ let block_id = key / self.block_size;++ if let Some(block) = self.inner.get_mut(&block_id) {+ // block.entry(key).or_default().push(value);+ // block.insert(key, value);+ mutate_block(block);+ return;+ }++ let mut new_block = HashMap::new();+ mutate_block(&mut new_block);++ self.inner.insert(block_id, new_block);+ }++ fn get(&mut self, key: &K) -> Option<&V> {+ let block_id = key.clone() / self.block_size;+ self.inner.get(&block_id).and_then(|block| block.get(key))+ }+}++pub(crate) struct CachedTree<K, V>+where+ K: Hash + Eq + Serialize + DeserializeOwned + Clone,+ V: Serialize + DeserializeOwned + Clone,+{+ pub(crate) store: Box<dyn Kv<K, V> + Send + Sync>,+ pub(crate) cache: LruCache<K, V>,+}++impl<K, V> CachedTree<K, V>+where+ K: Hash + Eq + Serialize + DeserializeOwned + Clone,+ V: Serialize + DeserializeOwned + Clone,+{+ pub(crate) fn new(store: Box<dyn Kv<K, V> + Send + Sync>, cache_size: usize) -> Self {+ Self {+ store,+ cache: LruCache::new(cache_size),+ }+ }++ fn update_cache(&mut self, key: &K) {+ if !self.cache.contains(key) {+ let val = self.store.get(key);++ if let Some(val) = val {+ self.cache.put(key.clone(), val);+ }+ }+ }++ pub(crate) fn get(&mut self, key: &K) -> Option<&V> {+ self.update_cache(key);+ self.cache.get(key)+ }++ fn get_mut(&mut self, key: &K) -> Option<&mut V> {+ self.update_cache(key);+ self.cache.get_mut(key)+ }++ fn insert(&mut self, key: K, value: V) {+ if self.cache.len() == self.cache.cap() {+ if let Some((key, value)) = self.cache.pop_lru() {+ self.store.insert(key, value);+ }+ }++ self.cache.push(key, value);+ }++ fn flush(&mut self) {+ for (key, value) in self.cache.iter() {+ self.store.insert(key.clone(), value.clone());+ }++ self.store.flush();+ }++ fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {+ self.store.iter()+ }+}++pub struct GraphStore<S> {+ pub(crate) adjacency: RefCell<Adjacency>,+ pub(crate) reversed_adjacency: RefCell<Adjacency>,+ pub(crate) node2id: RefCell<CachedTree<Node, NodeID>>,+ pub(crate) id2node: RefCell<BlockedCachedTree<NodeID, Node>>,+ pub(crate) meta: RefCell<CachedTree<String, u64>>,+ pub(crate) store: PhantomData<S>,+}++impl<S: Store> GraphStore<S> {+ #[cfg(test)]+ pub(crate) fn temporary() -> GraphStore<S> {+ S::temporary()+ }++ pub fn open<P: AsRef<Path>>(path: P) -> Self {+ S::open(path)+ }++ fn next_id(&self) -> NodeID {+ self.meta+ .borrow_mut()+ .get(&"next_id".to_string())+ .cloned()+ .unwrap_or(0)+ }++ fn increment_next_id(&self) {+ let current_id = self.next_id();+ let next_id = current_id + 1;+ self.meta+ .borrow_mut()+ .insert("next_id".to_string(), next_id);+ }++ fn id_and_increment(&self) -> NodeID {+ let id = self.next_id();+ self.increment_next_id();+ id+ }++ fn assign_id(&self, node: Node, id: NodeID) {+ self.node2id.borrow_mut().insert(node.clone(), id);+ self.id2node.borrow_mut().insert(id, &mut |block| {+ block.insert(id, node.clone());+ });+ }++ fn id_or_assign(&self, node: Node) -> NodeID {+ if let Some(id) = self.node2id.borrow_mut().get(&node) {+ return *id;+ }+ let id = self.id_and_increment();+ self.assign_id(node, id);+ id+ }+ pub fn outgoing_edges(&self, node: NodeID) -> Vec<Edge> {+ self.adjacency+ .borrow_mut()+ .edges(node)+ .into_iter()+ .map(|edge| Edge {+ from: node,+ to: edge.other,+ label: edge.label,+ })+ .collect()+ }++ pub fn ingoing_edges(&self, node: NodeID) -> Vec<Edge> {+ self.reversed_adjacency+ .borrow_mut()+ .edges(node)+ .into_iter()+ .map(|edge| Edge {+ from: edge.other,+ to: node,+ label: edge.label,+ })+ .collect()+ }++ pub fn nodes(&self) -> impl Iterator<Item = NodeID> {+ self.node2id+ .borrow_mut()+ .iter()+ .map(|(_, id)| id)+ .collect::<Vec<u64>>()+ .into_iter()+ }++ pub fn insert(&mut self, from: Node, to: Node, label: String) {+ let from_id = self.id_or_assign(from);+ let to_id = self.id_or_assign(to);++ self.adjacency+ .borrow_mut()+ .insert(from_id, to_id, label.clone());+ self.reversed_adjacency+ .borrow_mut()+ .insert(to_id, from_id, label);+ }++ pub fn node2id(&self, node: &Node) -> Option<NodeID> {+ self.node2id.borrow_mut().get(node).cloned()+ }++ pub fn id2node(&self, id: &NodeID) -> Option<Node> {+ self.id2node.borrow_mut().get(id).cloned()+ }++ pub fn flush(&self) {+ self.adjacency.borrow_mut().tree.inner.flush();+ self.reversed_adjacency.borrow_mut().tree.inner.flush();+ self.node2id.borrow_mut().flush();+ self.id2node.borrow_mut().inner.flush();+ self.meta.borrow_mut().flush();+ }++ pub fn edges(&self) -> EdgeIterator<'_> {+ self.flush();++ EdgeIterator::new(&self.adjacency)+ }++ pub fn append(&mut self, other: GraphStore<S>) {+ for edge in other.edges() {+ let from = other.id2node(&edge.from).expect("node not found");+ let to = other.id2node(&edge.to).expect("node not found");++ self.insert(from, to, edge.label);+ }+ }+}++impl Store for RocksDbStore {+ fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {+ let adjacency = RocksDbStore::open(path.as_ref().join("adjacency"));+ let reversed_adjacency = RocksDbStore::open(path.as_ref().join("reversed_adjacency"));+ let node2id = RocksDbStore::open(path.as_ref().join("node2id"));+ let id2node = RocksDbStore::open(path.as_ref().join("id2node"));+ let meta = RocksDbStore::open(path.as_ref().join("meta"));++ GraphStore {+ adjacency: RefCell::new(Adjacency {+ tree: BlockedCachedTree {+ inner: CachedTree::new(adjacency, 10_000),+ block_size: 1_024,+ },+ }),+ reversed_adjacency: RefCell::new(Adjacency {+ tree: BlockedCachedTree {+ inner: CachedTree::new(reversed_adjacency, 10_000),+ block_size: 1_024,+ },+ }),+ node2id: RefCell::new(CachedTree::new(node2id, 100_000)),+ id2node: RefCell::new(BlockedCachedTree {+ inner: CachedTree::new(id2node, 100_000),+ block_size: 1_024,+ }),+ meta: RefCell::new(CachedTree::new(meta, 1_000)),+ store: Default::default(),+ }+ }+}++impl Store for SledStore {+ fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {+ let db = sled::Config::default()+ .path(path)+ .use_compression(true)+ .mode(sled::Mode::LowSpace)+ .open()+ .expect("Failed to open database");++ let adjacency = Box::new(db.open_tree("adjacency").expect("unable to open sled tree"));+ let reversed_adjacency = Box::new(+ db.open_tree("reversed_adjacency")+ .expect("unable to open sled tree"),+ );+ let node2id = Box::new(db.open_tree("node2id").expect("unable to open sled tree"));+ let id2node = Box::new(db.open_tree("id2node").expect("unable to open sled tree"));+ let meta = Box::new(db.open_tree("meta").expect("unable to open sled tree"));++ GraphStore {+ adjacency: RefCell::new(Adjacency {+ tree: BlockedCachedTree {+ inner: CachedTree::new(adjacency, 10_000),+ block_size: 1_024,+ },+ }),+ reversed_adjacency: RefCell::new(Adjacency {+ tree: BlockedCachedTree {+ inner: CachedTree::new(reversed_adjacency, 10_000),+ block_size: 1_024,+ },+ }),+ node2id: RefCell::new(CachedTree::new(node2id, 100_000)),+ id2node: RefCell::new(BlockedCachedTree {+ inner: CachedTree::new(id2node, 100_000),+ block_size: 1_024,+ }),+ meta: RefCell::new(CachedTree::new(meta, 1_000)),+ store: Default::default(),+ }+ }+}+#[cfg(test)]+mod test {+ use super::*;++ #[test]+ fn simple_triangle_graph() {+ // ┌────┐+ // │ │+ // ┌───A◄─┐ │+ // │ │ │+ // ▼ │ │+ // B─────►C◄┘++ let mut store: GraphStore<RocksDbStore> = GraphStore::temporary();++ let a = Node {+ name: "A".to_string(),+ };+ let b = Node {+ name: "B".to_string(),+ };+ let c = Node {+ name: "C".to_string(),+ };++ store.insert(a.clone(), b.clone(), String::new());+ store.insert(b.clone(), c.clone(), String::new());+ store.insert(c.clone(), a.clone(), String::new());+ store.insert(a.clone(), c.clone(), String::new());++ store.flush();++ let nodes: Vec<Node> = store+ .nodes()+ .map(|id| store.id2node.borrow_mut().get(&id).unwrap().clone())+ .collect();+ assert_eq!(nodes, vec![a.clone(), b.clone(), c.clone()]);++ let a_id = store.node2id(&a).unwrap();+ let b_id = store.node2id(&b).unwrap();+ let c_id = store.node2id(&c).unwrap();++ assert_eq!(+ store.outgoing_edges(a_id),+ vec![+ Edge {+ from: a_id,+ to: b_id,+ label: String::new()+ },+ Edge {+ from: a_id,+ to: c_id,+ label: String::new()+ },+ ]+ );++ assert_eq!(+ store.outgoing_edges(b_id),+ vec![Edge {+ from: b_id,+ to: c_id,+ label: String::new()+ },]+ );++ assert_eq!(+ store.ingoing_edges(c_id),+ vec![+ Edge {+ from: b_id,+ to: c_id,+ label: String::new()+ },+ Edge {+ from: a_id,+ to: c_id,+ label: String::new()+ },+ ]+ );++ assert_eq!(+ store.ingoing_edges(a_id),+ vec![Edge {+ from: c_id,+ to: a_id,+ label: String::new()+ },]+ );++ assert_eq!(+ store.ingoing_edges(b_id),+ vec![Edge {+ from: a_id,+ to: b_id,+ label: String::new()+ },]+ );+ }+}diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsnew file mode 100644index 00000000..41c28779--- /dev/null+++ b/src/webgraph/mod.rs@@ -0,0 +1,653 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+mod graph_store;++use indicatif::{ProgressBar, ProgressIterator, ProgressStyle};+use serde::{Deserialize, Serialize};+use std::cell::RefCell;+use std::collections::{BinaryHeap, HashMap};+use std::path::Path;+use std::{cmp, fs};++use graph_store::GraphStore;++use crate::directory::{self, DirEntry};+use crate::webpage::Url;++use self::graph_store::Adjacency;+use crate::kv::rocksdb_store::RocksDbStore;++type NodeID = u64;++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub(crate) struct StoredEdge {+ other: NodeID,+ label: String,+}++#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+pub struct Node {+ pub name: String,+}++impl Node {+ fn into_host(self) -> Node {+ Node {+ name: Url::from(self.name).host().to_string(),+ }+ }+}++impl From<String> for Node {+ fn from(name: String) -> Self {+ Self { name }+ }+}++impl From<&str> for Node {+ fn from(name: &str) -> Self {+ Self::from(name.to_string())+ }+}++impl From<Url> for Node {+ fn from(url: Url) -> Self {+ Self {+ name: url.to_string(),+ }+ }+}++pub struct EdgeIterator<'a> {+ current_block_idx: usize,+ blocks: Vec<u64>,+ adjacency: &'a RefCell<Adjacency>,+ current_block: Option<Box<dyn Iterator<Item = Edge>>>,+}++impl<'a> EdgeIterator<'a> {+ fn new(adjacency: &'a RefCell<Adjacency>) -> EdgeIterator<'a> {+ let blocks: Vec<_> = adjacency+ .borrow()+ .tree+ .inner+ .store+ .iter()+ .map(|(key, _)| key)+ .collect();++ EdgeIterator {+ current_block_idx: 0,+ blocks,+ adjacency,+ current_block: None,+ }+ }++ fn load_next_block(&mut self) {+ if self.current_block_idx < self.blocks.len() {+ let block_id = self.blocks[self.current_block_idx];+ let block = self+ .adjacency+ .borrow_mut()+ .tree+ .inner+ .get(&block_id)+ .unwrap()+ .clone();++ self.current_block = Some(Box::new(block.into_iter().flat_map(|(node_id, edges)| {+ edges.into_iter().map(move |edge| Edge {+ from: node_id,+ to: edge.other,+ label: edge.label,+ })+ })));++ self.current_block_idx += 1;+ }+ }+}++impl<'a> Iterator for EdgeIterator<'a> {+ type Item = Edge;++ fn next(&mut self) -> Option<Self::Item> {+ if let Some(res) = self.current_block.as_mut().and_then(|it| it.next()) {+ return Some(res);+ }++ if self.current_block.is_none() {+ self.load_next_block();+ }++ self.current_block.as_mut().and_then(|it| it.next())+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct Edge {+ from: NodeID,+ to: NodeID,+ label: String,+}++pub struct WebgraphBuilder {+ path: Box<Path>,+ full_graph_path: Option<Box<Path>>,+ host_graph_path: Option<Box<Path>>,+}++impl WebgraphBuilder {+ #[cfg(test)]+ pub fn new_memory() -> Self {+ use crate::gen_temp_path;++ let path = gen_temp_path();+ Self::new(path)+ }++ pub fn new<P: AsRef<Path>>(path: P) -> Self {+ Self {+ path: path.as_ref().into(),+ full_graph_path: None,+ host_graph_path: None,+ }+ }++ pub fn with_full_graph(mut self) -> Self {+ self.full_graph_path = Some(self.path.join("full").into());+ self+ }++ pub fn with_host_graph(mut self) -> Self {+ self.host_graph_path = Some(self.path.join("host").into());+ self+ }++ pub fn open(self) -> Webgraph {+ Webgraph {+ full_graph: self.full_graph_path.map(GraphStore::open),+ host_graph: self.host_graph_path.map(GraphStore::open),+ path: self.path.to_str().unwrap().to_string(),+ }+ }+}++pub trait Store+where+ Self: Sized,+{+ fn open<P: AsRef<Path>>(path: P) -> GraphStore<Self>;++ fn temporary() -> GraphStore<Self> {+ Self::open(crate::gen_temp_path())+ }+}++pub struct Webgraph<S: Store = RocksDbStore> {+ pub path: String,+ full_graph: Option<GraphStore<S>>,+ host_graph: Option<GraphStore<S>>,+}++impl<S: Store> Webgraph<S> {+ pub fn insert(&mut self, from: Node, to: Node, label: String) {+ if let Some(full_graph) = &mut self.full_graph {+ full_graph.insert(from.clone(), to.clone(), label.clone());+ }++ if let Some(host_graph) = &mut self.host_graph {+ host_graph.insert(from.into_host(), to.into_host(), label);+ }+ }++ pub fn merge(&mut self, other: Webgraph<S>) {+ match (&mut self.full_graph, other.full_graph) {+ (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),+ (None, Some(other_graph)) => self.full_graph = Some(other_graph),+ (Some(_), None) | (None, None) => {}+ }++ match (&mut self.host_graph, other.host_graph) {+ (Some(self_graph), Some(other_graph)) => self_graph.append(other_graph),+ (None, Some(other_graph)) => self.host_graph = Some(other_graph),+ (Some(_), None) | (None, None) => {}+ }++ self.flush();+ }++ fn dijkstra<F1, F2>(+ source: Node,+ node_edges: F1,+ edge_node: F2,+ store: &GraphStore<S>,+ ) -> HashMap<NodeID, usize>+ where+ F1: Fn(NodeID) -> Vec<Edge>,+ F2: Fn(&Edge) -> NodeID,+ {+ let source_id = store.node2id(&source);+ if source_id.is_none() {+ return HashMap::new();+ }++ let source_id = source_id.unwrap();+ let mut distances: HashMap<NodeID, usize> = HashMap::default();++ let mut queue = BinaryHeap::new();++ queue.push(cmp::Reverse((0_usize, source_id)));+ distances.insert(source_id, 0);++ while let Some(state) = queue.pop() {+ let (cost, v) = state.0;+ let current_dist = distances.get(&v).unwrap_or(&usize::MAX);++ if cost > *current_dist {+ continue;+ }++ for edge in node_edges(v) {+ if cost + 1 < *distances.get(&edge_node(&edge)).unwrap_or(&usize::MAX) {+ let next = cmp::Reverse((cost + 1, edge_node(&edge)));+ queue.push(next);+ distances.insert(edge_node(&edge), cost + 1);+ }+ }+ }++ distances+ }++ pub fn distances(&self, source: Node) -> HashMap<Node, usize> {+ self.full_graph+ .as_ref()+ .map(|full_graph| {+ let distances = Webgraph::dijkstra(+ source,+ |node_id| full_graph.outgoing_edges(node_id),+ |edge| edge.to,+ full_graph,+ );++ distances+ .into_iter()+ .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))+ .collect()+ })+ .unwrap_or_default()+ }++ fn raw_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ self.full_graph+ .as_ref()+ .map(|full_graph| {+ Webgraph::dijkstra(+ source,+ |node| full_graph.ingoing_edges(node),+ |edge| edge.from,+ full_graph,+ )+ })+ .unwrap_or_default()+ }++ pub fn reversed_distances(&self, source: Node) -> HashMap<Node, usize> {+ self.full_graph+ .as_ref()+ .map(|full_graph| {+ self.raw_reversed_distances(source)+ .into_iter()+ .map(|(id, dist)| (full_graph.id2node(&id).expect("unknown node"), dist))+ .collect()+ })+ .unwrap_or_default()+ }++ pub fn host_distances(&self, source: Node) -> HashMap<Node, usize> {+ self.host_graph+ .as_ref()+ .map(|host_graph| {+ let distances = Webgraph::dijkstra(+ source,+ |node| host_graph.outgoing_edges(node),+ |edge| edge.to,+ host_graph,+ );++ distances+ .into_iter()+ .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))+ .collect()+ })+ .unwrap_or_default()+ }++ fn raw_host_reversed_distances(&self, source: Node) -> HashMap<NodeID, usize> {+ self.host_graph+ .as_ref()+ .map(|host_graph| {+ Webgraph::dijkstra(+ source,+ |node| host_graph.ingoing_edges(node),+ |edge| edge.from,+ host_graph,+ )+ })+ .unwrap_or_default()+ }++ pub fn host_reversed_distances(&self, source: Node) -> HashMap<Node, usize> {+ self.host_graph+ .as_ref()+ .map(|host_graph| {+ self.raw_host_reversed_distances(source)+ .into_iter()+ .map(|(id, dist)| (host_graph.id2node(&id).expect("unknown node"), dist))+ .collect()+ })+ .unwrap_or_default()+ }++ fn calculate_centrality<F>(graph: &GraphStore<S>, node_distances: F) -> HashMap<Node, f64>+ where+ F: Fn(Node) -> HashMap<NodeID, usize>,+ {+ let nodes: Vec<_> = graph.nodes().collect();+ dbg!(&nodes);+ let pb = ProgressBar::new(nodes.len() as u64);+ pb.set_style(+ ProgressStyle::default_bar()+ .template(+ "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({eta})",+ )+ .progress_chars("#>-"),+ );+ let norm_factor = (nodes.len() - 1) as f64;+ nodes+ .iter()+ .progress_with(pb)+ .map(|node_id| {+ let node = graph.id2node(node_id).expect("unknown node");+ let centrality_values: HashMap<NodeID, f64> = node_distances(node.clone())+ .into_iter()+ .filter(|(other_id, _)| *other_id != *node_id)+ .map(|(other_node, dist)| (other_node, 1f64 / dist as f64))+ .collect();++ let centrality = centrality_values+ .into_iter()+ .map(|(_, val)| val)+ .sum::<f64>()+ / norm_factor;++ (node, centrality)+ })+ .collect()+ }++ pub fn harmonic_centrality(&self) -> HashMap<Node, f64> {+ self.full_graph+ .as_ref()+ .map(|full_graph| {+ Webgraph::calculate_centrality(full_graph, |node| self.raw_reversed_distances(node))+ })+ .unwrap_or_default()+ }++ pub fn host_harmonic_centrality(&self) -> HashMap<Node, f64> {+ self.host_graph+ .as_ref()+ .map(|host_graph| {+ Webgraph::calculate_centrality(host_graph, |node| {+ self.raw_host_reversed_distances(node)+ })+ })+ .unwrap_or_default()+ }++ pub fn flush(&self) {+ if let Some(full_graph) = &self.full_graph {+ full_graph.flush();+ }+ if let Some(host_graph) = &self.host_graph {+ host_graph.flush();+ }+ }+}++impl From<FrozenWebgraph> for Webgraph {+ fn from(frozen: FrozenWebgraph) -> Self {+ let path = match &frozen.root {+ DirEntry::Folder { name, entries: _ } => name.clone(),+ DirEntry::File {+ name: _,+ content: _,+ } => {+ panic!("Cannot open webgraph from a file - must be directory.")+ }+ };++ if Path::new(&path).exists() {+ fs::remove_dir_all(&path).unwrap();+ }++ directory::recreate_folder(&frozen.root).unwrap();++ let mut builder = WebgraphBuilder::new(path);++ if frozen.has_full {+ builder = builder.with_full_graph();+ }++ if frozen.has_host {+ builder = builder.with_host_graph();+ }++ builder.open()+ }+}++impl From<Webgraph> for FrozenWebgraph {+ fn from(graph: Webgraph) -> Self {+ graph.flush();+ let path = graph.path.clone();+ let has_full = graph.full_graph.is_some();+ let has_host = graph.host_graph.is_some();+ drop(graph);+ let root = directory::scan_folder(path).unwrap();++ Self {+ root,+ has_full,+ has_host,+ }+ }+}++#[derive(Debug, Serialize, Deserialize)]+pub struct FrozenWebgraph {+ pub root: DirEntry,+ has_full: bool,+ has_host: bool,+}++#[cfg(test)]+mod test {+ use super::*;++ fn test_graph() -> Webgraph {+ // ┌────┐+ // │ │+ // ┌───A◄─┐ │+ // │ │ │+ // ▼ │ │+ // B─────►C◄┘+ // ▲+ // │+ // │+ // D++ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("B"), Node::from("C"), String::new());+ graph.insert(Node::from("A"), Node::from("C"), String::new());+ graph.insert(Node::from("C"), Node::from("A"), String::new());+ graph.insert(Node::from("D"), Node::from("C"), String::new());++ graph.flush();++ graph+ }++ #[test]+ fn distance_calculation() {+ let graph = test_graph();++ let distances = graph.distances(Node::from("D"));++ assert_eq!(distances.get(&Node::from("C")), Some(&1));+ assert_eq!(distances.get(&Node::from("A")), Some(&2));+ assert_eq!(distances.get(&Node::from("B")), Some(&3));+ }++ #[test]+ fn nonexisting_node() {+ let graph = test_graph();+ assert_eq!(graph.distances(Node::from("E")).len(), 0);+ assert_eq!(graph.reversed_distances(Node::from("E")).len(), 0);+ }++ #[test]+ fn reversed_distance_calculation() {+ let graph = test_graph();++ let distances = graph.reversed_distances(Node::from("D"));++ assert_eq!(distances.get(&Node::from("C")), None);+ assert_eq!(distances.get(&Node::from("A")), None);+ assert_eq!(distances.get(&Node::from("B")), None);++ let distances = graph.reversed_distances(Node::from("A"));++ assert_eq!(distances.get(&Node::from("C")), Some(&1));+ assert_eq!(distances.get(&Node::from("D")), Some(&2));+ assert_eq!(distances.get(&Node::from("B")), Some(&2));+ }++ #[test]+ fn harmonic_centrality() {+ let graph = test_graph();++ let centrality = graph.harmonic_centrality();++ assert_eq!(centrality.get(&Node::from("C")).unwrap(), &1.0);+ assert_eq!(centrality.get(&Node::from("D")).unwrap(), &0.0);+ assert_eq!(+ (*centrality.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,+ 0.67+ );+ assert_eq!(+ (*centrality.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,+ 0.61+ );+ }++ #[test]+ fn host_harmonic_centrality() {+ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(Node::from("A.com/1"), Node::from("A.com/2"), String::new());+ graph.insert(Node::from("A.com/1"), Node::from("A.com/3"), String::new());+ graph.insert(Node::from("A.com/1"), Node::from("A.com/4"), String::new());+ graph.insert(Node::from("A.com/2"), Node::from("A.com/1"), String::new());+ graph.insert(Node::from("A.com/2"), Node::from("A.com/3"), String::new());+ graph.insert(Node::from("A.com/2"), Node::from("A.com/4"), String::new());+ graph.insert(Node::from("A.com/3"), Node::from("A.com/1"), String::new());+ graph.insert(Node::from("A.com/3"), Node::from("A.com/2"), String::new());+ graph.insert(Node::from("A.com/3"), Node::from("A.com/4"), String::new());+ graph.insert(Node::from("A.com/4"), Node::from("A.com/1"), String::new());+ graph.insert(Node::from("A.com/4"), Node::from("A.com/2"), String::new());+ graph.insert(Node::from("A.com/4"), Node::from("A.com/3"), String::new());+ graph.insert(Node::from("C.com"), Node::from("B.com"), String::new());+ graph.insert(Node::from("D.com"), Node::from("B.com"), String::new());++ graph.flush();++ let centrality = graph.harmonic_centrality();+ assert!(+ centrality.get(&Node::from("A.com/1")).unwrap()+ > centrality.get(&Node::from("B.com")).unwrap()+ );++ let host_centrality = graph.host_harmonic_centrality();+ assert!(+ host_centrality.get(&Node::from("B.com")).unwrap()+ > host_centrality.get(&Node::from("A.com")).unwrap()+ );+ }++ #[test]+ fn merge() {+ let mut graph1 = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph1.insert(Node::from("A"), Node::from("B"), String::new());++ let mut graph2 = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();+ graph2.insert(Node::from("B"), Node::from("C"), String::new());++ graph1.merge(graph2);++ assert_eq!(+ graph1.distances(Node::from("A")).get(&Node::from("C")),+ Some(&2)+ )+ }++ #[test]+ fn serialize_deserialize_bincode() {+ let graph = test_graph();+ let path = graph.path.clone();+ let frozen: FrozenWebgraph = graph.into();+ let bytes = bincode::serialize(&frozen).unwrap();++ std::fs::remove_dir_all(path).unwrap();++ let deserialized_frozen: FrozenWebgraph = bincode::deserialize(&bytes).unwrap();+ let graph: Webgraph = deserialized_frozen.into();++ let distances = graph.distances(Node::from("D"));++ assert_eq!(distances.get(&Node::from("C")), Some(&1));+ assert_eq!(distances.get(&Node::from("A")), Some(&2));+ assert_eq!(distances.get(&Node::from("B")), Some(&3));+ }+}diff --git a/templates/search.html b/templates/search.htmlindex cc0313ee..112fcc94 100644--- a/templates/search.html+++ b/templates/search.html@@ -17,7 +17,7 @@<div class="flex pt-8"><div class="search-result flex flex-col space-y-3 pb-8 pl-40">{% for item in search_result %}- <div class="flex max-w-3xl w-full p-3 h-36">+ <div class="flex max-w-3xl w-full p-3"><div class="flex flex-col w-full"><div class="text-sm mb-1 flex"><div class="flex items-center mr-2">