Skip to content

web search done right

unit test personal centrality from goggles

Mikkel Denker76ed6624c512
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 567998e5..5fbe6a1f 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -401,9 +401,12 @@ impl Instruction {
#[cfg(test)]
mod tests {
use crate::{
+ gen_temp_path,
index::Index,
+ ranking::centrality_store::CentralityStore,
schema::create_schema,
searcher::{LocalSearcher, SearchQuery},
+ webgraph::{Node, WebgraphBuilder},
webpage::{Html, Webpage},
};
@@ -763,4 +766,163 @@ mod tests {
assert_eq!(res.len(), 2);
assert_eq!(res[0].url, "https://www.a.com/this/is/a/pattern");
}
+
+ #[test]
+ fn liked_sites() {
+ let mut index = Index::temporary().expect("Unable to open index");
+
+ let mut graph = WebgraphBuilder::new_memory()
+ .with_full_graph()
+ .with_host_graph()
+ .open();
+
+ graph.insert(
+ Node::from("https://www.a.com").into_host(),
+ Node::from("https://www.b.com").into_host(),
+ String::new(),
+ );
+
+ graph.insert(
+ Node::from("https://www.c.com").into_host(),
+ Node::from("https://www.c.com").into_host(),
+ String::new(),
+ );
+
+ graph.flush();
+
+ let centrality_store = CentralityStore::build(&graph, gen_temp_path());
+
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website A</title>
+ </head>
+ <body>
+ {CONTENT}
+ example example example
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.a.com/this/is/a/pattern",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0,
+ page_centrality: 0.0,
+ fetch_time_ms: 500,
+ pre_computed_score: 0.0,
+ primary_image: None,
+ node_id: Some(
+ *centrality_store
+ .approx_harmonic
+ .node2id
+ .get(&Node::from("www.a.com").into_host())
+ .unwrap(),
+ ),
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.b.com/this/is/b/pattern",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0001,
+ page_centrality: 0.0,
+ primary_image: None,
+ pre_computed_score: 0.0,
+ fetch_time_ms: 500,
+ node_id: Some(
+ *centrality_store
+ .approx_harmonic
+ .node2id
+ .get(&Node::from("www.b.com").into_host())
+ .unwrap(),
+ ),
+ })
+ .expect("failed to insert webpage");
+ index
+ .insert(Webpage {
+ html: Html::parse(
+ &format!(
+ r#"
+ <html>
+ <head>
+ <title>Website B</title>
+ </head>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ ),
+ "https://www.c.com/this/is/c/pattern",
+ ),
+ backlinks: vec![],
+ host_centrality: 0.0002,
+ page_centrality: 0.0,
+ primary_image: None,
+ pre_computed_score: 0.0,
+ fetch_time_ms: 500,
+ node_id: Some(
+ *centrality_store
+ .approx_harmonic
+ .node2id
+ .get(&Node::from("www.c.com").into_host())
+ .unwrap(),
+ ),
+ })
+ .expect("failed to insert webpage");
+
+ index.commit().expect("failed to commit index");
+ let mut searcher = LocalSearcher::from(index);
+
+ searcher.set_centrality_store(centrality_store);
+
+ let res = searcher
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: Some(
+ r#"
+ liked = [
+ www.a.com,
+ www.b.com,
+ ]
+ disliked = [
+ www.c.com,
+ ]
+ "#
+ .to_string(),
+ ),
+ skip_pages: None,
+ site_rankings: None,
+ })
+ .unwrap()
+ .into_websites()
+ .unwrap()
+ .webpages
+ .documents;
+
+ assert_eq!(res.len(), 3);
+ assert_eq!(res[0].url, "https://www.b.com/this/is/b/pattern");
+ assert_eq!(res[1].url, "https://www.a.com/this/is/a/pattern");
+ assert_eq!(res[2].url, "https://www.c.com/this/is/c/pattern");
+ }
}
diff --git a/src/ranking/signal.rs b/src/ranking/signal.rs
index b38dd886..fe38f808 100644
--- a/src/ranking/signal.rs
+++ b/src/ranking/signal.rs
@@ -139,7 +139,7 @@ impl Signal {
Signal::UpdateTimestamp => 80.0,
Signal::NumTrackers => 20.0,
Signal::Region => 60.0,
- Signal::PersonalCentrality => 1024.0,
+ Signal::PersonalCentrality => 2048.0,
}
}
diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rs
index e9ba5b75..4eb302fb 100644
--- a/src/webgraph/centrality/approximate_harmonic.rs
+++ b/src/webgraph/centrality/approximate_harmonic.rs
@@ -153,23 +153,26 @@ impl Scorer {
return *cached;
}
- let res = ((self
- .liked_nodes
- .iter()
- .filter_map(|liked_node| liked_node.best_dist(&node).map(|dist| (dist, liked_node)))
- .map(|(dist, liked_node)| liked_node.weight as f64 / dist as f64)
- .sum::<f64>()
- - self
- .disliked_nodes
+ let res = (1.0
+ + (self
+ .liked_nodes
.iter()
- .filter_map(|disliked_node| {
- disliked_node
- .best_dist(&node)
- .map(|dist| (dist, disliked_node))
+ .filter_map(|liked_node| {
+ liked_node.best_dist(&node).map(|dist| (dist, liked_node))
})
- .map(|(dist, disliked_node)| disliked_node.weight as f64 / dist as f64)
- .sum::<f64>())
- / self.num_liked_nodes as f64)
+ .map(|(dist, liked_node)| liked_node.weight as f64 / dist as f64)
+ .sum::<f64>()
+ - self
+ .disliked_nodes
+ .iter()
+ .filter_map(|disliked_node| {
+ disliked_node
+ .best_dist(&node)
+ .map(|dist| (dist, disliked_node))
+ })
+ .map(|(dist, disliked_node)| disliked_node.weight as f64 / dist as f64)
+ .sum::<f64>())
+ / self.num_liked_nodes as f64)
.max(0.0);
self.cache.lock().unwrap().insert(node, res);
@@ -308,7 +311,7 @@ impl ApproximatedHarmonicCentrality {
let fixed_scores: HashMap<_, _> = liked_nodes
.iter()
- .map(|node| (*node, 1.0))
+ .map(|node| (*node, 2.0))
.chain(disliked_nodes.iter().map(|node| (*node, 0.0)))
.collect();
@@ -422,7 +425,7 @@ mod tests {
let scorer = centrality.scorer(&liked_nodes, &[]);
for node in &liked_nodes {
- assert_eq!(scorer.score(*centrality.node2id.get(node).unwrap()), 1.0);
+ assert_eq!(scorer.score(*centrality.node2id.get(node).unwrap()), 2.0);
}
}