web search done right
ensure that additional links from A -> B does not artificially increase the centrality of B
Mikkel Denker596b58586e1d
diff --git a/examples/print_inbound_links.rs b/examples/print_inbound_links.rsnew file mode 100644index 00000000..0a1a4f6a--- /dev/null+++ b/examples/print_inbound_links.rs@@ -0,0 +1,37 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use cuely::webgraph::{Node, WebgraphBuilder};++pub fn main() {+ let graph = WebgraphBuilder::new("data/webgraph")+ .with_host_graph()+ .read_only(true)+ .open();++ for host in [+ "plapcatesq.ga",+ "lfsni36qvn.ga",+ "onlinester.ga",+ "GroomersNew.ga",+ ] {+ println!("{}:", host);+ for edge in graph.ingoing_edges_host(Node::from(host)) {+ println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);+ }+ println!();+ }+}diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rsindex 9de385ca..b880ab93 100644--- a/src/entrypoint/search_server.rs+++ b/src/entrypoint/search_server.rs@@ -36,9 +36,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {.entity_index_path.map(|path| EntityIndex::open(path).unwrap());let bangs = config.bangs_path.map(Bangs::from_path);- let centrality_store = config- .centrality_store_path- .map(|path| CentralityStore::open(path));+ let centrality_store = config.centrality_store_path.map(CentralityStore::open);let search_index = Index::open(config.index_path)?;let mut local_searcher = LocalSearcher::new(search_index);diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex a31ca8e7..5ae7f73f 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -83,7 +83,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {.into_iter().filter(|link| matches!(link.destination.protocol(), "http" | "https")).filter(|link| link.source.domain() != link.destination.domain())- .filter(|link| !link.matches_url_regex())+ .filter(|link| link.matches_url_regex()){trace!("inserting link {:?}", link);graph.insert(diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rsindex c446a577..a484f5e3 100644--- a/src/webgraph/centrality/harmonic.rs+++ b/src/webgraph/centrality/harmonic.rs@@ -195,4 +195,31 @@ mod tests {assert_eq!(centrality.host.get(&Node::from("A.com")), Some(&1.0));assert_eq!(centrality.host.get(&Node::from("www.A.com")), None);}++ #[test]+ fn additional_edges_ignored() {+ let mut graph = test_graph();++ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("A"), Node::from("B"), String::new());+ graph.insert(Node::from("A"), Node::from("B"), String::new());++ graph.flush();+ let centrality = HarmonicCentrality::calculate(&graph);++ assert_eq!(centrality.full.get(&Node::from("C")).unwrap(), &1.0);+ assert_eq!(centrality.full.get(&Node::from("D")), None);+ assert_eq!(+ (*centrality.full.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,+ 0.67+ );+ assert_eq!(+ (*centrality.full.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,+ 0.61+ );+ }}diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rsindex 4140382f..c3a2338a 100644--- a/src/webgraph/graph_store.rs+++ b/src/webgraph/graph_store.rs@@ -63,8 +63,6 @@ wherelet block_id = key / self.block_size;if let Some(block) = self.inner.get_mut(&block_id) {- // block.entry(key).or_default().push(value);- // block.insert(key, value);mutate_block(block);return;}diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex 08b2ff6a..3151375f 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -447,6 +447,26 @@ impl<S: Store> Webgraph<S> {Vec::new()}}++ pub fn ingoing_edges_host(&self, node: Node) -> Vec<FullEdge> {+ if let Some(graph) = &self.host {+ if let Some(node_id) = graph.node2id(&node) {+ graph+ .ingoing_edges(node_id)+ .into_iter()+ .map(|edge| FullEdge {+ from: graph.id2node(&edge.from).unwrap(),+ to: graph.id2node(&edge.to).unwrap(),+ label: edge.label,+ })+ .collect()+ } else {+ Vec::new()+ }+ } else {+ Vec::new()+ }+ }}impl From<FrozenWebgraph> for Webgraph {