web search done right
remove protocol from full-centrality
Mikkel Denker01326f4310ea
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex 3151375f..d8ff7c58 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -58,6 +58,11 @@ impl Node {pub fn from_url(url: &Url) -> Self {Node::from(url.full())}++ fn remove_protocol(&mut self) {+ let url = Url::from(self.name.clone());+ self.name = url.without_protocol().to_string();+ }}impl From<String> for Node {@@ -248,7 +253,10 @@ pub struct Webgraph<S: Store = RocksDbStore> {}impl<S: Store> Webgraph<S> {- pub fn insert(&mut self, from: Node, to: Node, label: String) {+ pub fn insert(&mut self, mut from: Node, mut to: Node, label: String) {+ from.remove_protocol();+ to.remove_protocol();+if let Some(full_graph) = &mut self.full {full_graph.insert(from.clone(), to.clone(), label.clone());}@@ -645,4 +653,31 @@ mod test {let n = Node::from("https://www.example.com?test").into_host();assert_eq!(&n.name, "example.com");}++ #[test]+ fn remove_protocol() {+ let mut n = Node::from("https://www.example.com?test");+ n.remove_protocol();++ assert_eq!(&n.name, "www.example.com?test");++ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(+ Node::from("http://A"),+ Node::from("https://B"),+ String::new(),+ );++ graph.flush();++ let distances = graph.distances(Node::from("A"));+ assert_eq!(distances.get(&Node::from("B")), Some(&1));++ let distances = graph.distances(Node::from("http://A"));+ assert!(distances.is_empty());+ }}diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex dd59dedf..9342d5ae 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -226,7 +226,7 @@ impl Url {pub fn without_protocol(&self) -> &str {self.0[self.find_protocol_end()..].strip_prefix("://")- .unwrap_or_default()+ .unwrap_or(&self.0)}}@@ -332,7 +332,9 @@ mod tests {#[test]fn url_without_protocol() {let url: Url = "https://test.example.com/test/test".to_string().into();+ assert_eq!(url.without_protocol(), "test.example.com/test/test");+ let url: Url = "test.example.com/test/test".to_string().into();assert_eq!(url.without_protocol(), "test.example.com/test/test");}}