Skip to content

web search done right

ensure that additional links from A -> B does not artificially increase the centrality of B

Mikkel Denker596b58586e1d
diff --git a/examples/print_inbound_links.rs b/examples/print_inbound_links.rs
new file mode 100644
index 00000000..0a1a4f6a
--- /dev/null
+++ b/examples/print_inbound_links.rs
@@ -0,0 +1,37 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use cuely::webgraph::{Node, WebgraphBuilder};
+
+pub fn main() {
+ let graph = WebgraphBuilder::new("data/webgraph")
+ .with_host_graph()
+ .read_only(true)
+ .open();
+
+ for host in [
+ "plapcatesq.ga",
+ "lfsni36qvn.ga",
+ "onlinester.ga",
+ "GroomersNew.ga",
+ ] {
+ println!("{}:", host);
+ for edge in graph.ingoing_edges_host(Node::from(host)) {
+ println!("{} -> {} ({})", edge.from.name, edge.to.name, edge.label);
+ }
+ println!();
+ }
+}
diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rs
index 9de385ca..b880ab93 100644
--- a/src/entrypoint/search_server.rs
+++ b/src/entrypoint/search_server.rs
@@ -36,9 +36,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
.entity_index_path
.map(|path| EntityIndex::open(path).unwrap());
let bangs = config.bangs_path.map(Bangs::from_path);
- let centrality_store = config
- .centrality_store_path
- .map(|path| CentralityStore::open(path));
+ let centrality_store = config.centrality_store_path.map(CentralityStore::open);
let search_index = Index::open(config.index_path)?;
let mut local_searcher = LocalSearcher::new(search_index);
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index a31ca8e7..5ae7f73f 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -83,7 +83,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
.into_iter()
.filter(|link| matches!(link.destination.protocol(), "http" | "https"))
.filter(|link| link.source.domain() != link.destination.domain())
- .filter(|link| !link.matches_url_regex())
+ .filter(|link| link.matches_url_regex())
{
trace!("inserting link {:?}", link);
graph.insert(
diff --git a/src/webgraph/centrality/harmonic.rs b/src/webgraph/centrality/harmonic.rs
index c446a577..a484f5e3 100644
--- a/src/webgraph/centrality/harmonic.rs
+++ b/src/webgraph/centrality/harmonic.rs
@@ -195,4 +195,31 @@ mod tests {
assert_eq!(centrality.host.get(&Node::from("A.com")), Some(&1.0));
assert_eq!(centrality.host.get(&Node::from("www.A.com")), None);
}
+
+ #[test]
+ fn additional_edges_ignored() {
+ let mut graph = test_graph();
+
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+ graph.insert(Node::from("A"), Node::from("B"), String::new());
+
+ graph.flush();
+ let centrality = HarmonicCentrality::calculate(&graph);
+
+ assert_eq!(centrality.full.get(&Node::from("C")).unwrap(), &1.0);
+ assert_eq!(centrality.full.get(&Node::from("D")), None);
+ assert_eq!(
+ (*centrality.full.get(&Node::from("A")).unwrap() * 100.0).round() / 100.0,
+ 0.67
+ );
+ assert_eq!(
+ (*centrality.full.get(&Node::from("B")).unwrap() * 100.0).round() / 100.0,
+ 0.61
+ );
+ }
}
diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rs
index 4140382f..c3a2338a 100644
--- a/src/webgraph/graph_store.rs
+++ b/src/webgraph/graph_store.rs
@@ -63,8 +63,6 @@ where
let block_id = key / self.block_size;
if let Some(block) = self.inner.get_mut(&block_id) {
- // block.entry(key).or_default().push(value);
- // block.insert(key, value);
mutate_block(block);
return;
}
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index 08b2ff6a..3151375f 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -447,6 +447,26 @@ impl<S: Store> Webgraph<S> {
Vec::new()
}
}
+
+ pub fn ingoing_edges_host(&self, node: Node) -> Vec<FullEdge> {
+ if let Some(graph) = &self.host {
+ if let Some(node_id) = graph.node2id(&node) {
+ graph
+ .ingoing_edges(node_id)
+ .into_iter()
+ .map(|edge| FullEdge {
+ from: graph.id2node(&edge.from).unwrap(),
+ to: graph.id2node(&edge.to).unwrap(),
+ label: edge.label,
+ })
+ .collect()
+ } else {
+ Vec::new()
+ }
+ } else {
+ Vec::new()
+ }
+ }
}
impl From<FrozenWebgraph> for Webgraph {