web search done right
unit test personal centrality from goggles
Mikkel Denker76ed6624c512
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex 567998e5..5fbe6a1f 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -401,9 +401,12 @@ impl Instruction {#[cfg(test)]mod tests {use crate::{+ gen_temp_path,index::Index,+ ranking::centrality_store::CentralityStore,schema::create_schema,searcher::{LocalSearcher, SearchQuery},+ webgraph::{Node, WebgraphBuilder},webpage::{Html, Webpage},};@@ -763,4 +766,163 @@ mod tests {assert_eq!(res.len(), 2);assert_eq!(res[0].url, "https://www.a.com/this/is/a/pattern");}++ #[test]+ fn liked_sites() {+ let mut index = Index::temporary().expect("Unable to open index");++ let mut graph = WebgraphBuilder::new_memory()+ .with_full_graph()+ .with_host_graph()+ .open();++ graph.insert(+ Node::from("https://www.a.com").into_host(),+ Node::from("https://www.b.com").into_host(),+ String::new(),+ );++ graph.insert(+ Node::from("https://www.c.com").into_host(),+ Node::from("https://www.c.com").into_host(),+ String::new(),+ );++ graph.flush();++ let centrality_store = CentralityStore::build(&graph, gen_temp_path());++ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Website A</title>+ </head>+ <body>+ {CONTENT}+ example example example+ </body>+ </html>+ "#+ ),+ "https://www.a.com/this/is/a/pattern",+ ),+ backlinks: vec![],+ host_centrality: 0.0,+ page_centrality: 0.0,+ fetch_time_ms: 500,+ pre_computed_score: 0.0,+ primary_image: None,+ node_id: Some(+ *centrality_store+ .approx_harmonic+ .node2id+ .get(&Node::from("www.a.com").into_host())+ .unwrap(),+ ),+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.b.com/this/is/b/pattern",+ ),+ backlinks: vec![],+ host_centrality: 0.0001,+ page_centrality: 0.0,+ primary_image: None,+ pre_computed_score: 0.0,+ fetch_time_ms: 500,+ node_id: Some(+ *centrality_store+ .approx_harmonic+ .node2id+ .get(&Node::from("www.b.com").into_host())+ .unwrap(),+ ),+ })+ .expect("failed to insert webpage");+ index+ .insert(Webpage {+ html: Html::parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Website B</title>+ </head>+ <body>+ {CONTENT}+ </body>+ </html>+ "#+ ),+ "https://www.c.com/this/is/c/pattern",+ ),+ backlinks: vec![],+ host_centrality: 0.0002,+ page_centrality: 0.0,+ primary_image: None,+ pre_computed_score: 0.0,+ fetch_time_ms: 500,+ node_id: Some(+ *centrality_store+ .approx_harmonic+ .node2id+ .get(&Node::from("www.c.com").into_host())+ .unwrap(),+ ),+ })+ .expect("failed to insert webpage");++ index.commit().expect("failed to commit index");+ let mut searcher = LocalSearcher::from(index);++ searcher.set_centrality_store(centrality_store);++ let res = searcher+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: Some(+ r#"+ liked = [+ www.a.com,+ www.b.com,+ ]+ disliked = [+ www.c.com,+ ]+ "#+ .to_string(),+ ),+ skip_pages: None,+ site_rankings: None,+ })+ .unwrap()+ .into_websites()+ .unwrap()+ .webpages+ .documents;++ assert_eq!(res.len(), 3);+ assert_eq!(res[0].url, "https://www.b.com/this/is/b/pattern");+ assert_eq!(res[1].url, "https://www.a.com/this/is/a/pattern");+ assert_eq!(res[2].url, "https://www.c.com/this/is/c/pattern");+ }}diff --git a/src/ranking/signal.rs b/src/ranking/signal.rsindex b38dd886..fe38f808 100644--- a/src/ranking/signal.rs+++ b/src/ranking/signal.rs@@ -139,7 +139,7 @@ impl Signal {Signal::UpdateTimestamp => 80.0,Signal::NumTrackers => 20.0,Signal::Region => 60.0,- Signal::PersonalCentrality => 1024.0,+ Signal::PersonalCentrality => 2048.0,}}diff --git a/src/webgraph/centrality/approximate_harmonic.rs b/src/webgraph/centrality/approximate_harmonic.rsindex e9ba5b75..4eb302fb 100644--- a/src/webgraph/centrality/approximate_harmonic.rs+++ b/src/webgraph/centrality/approximate_harmonic.rs@@ -153,23 +153,26 @@ impl Scorer {return *cached;}- let res = ((self- .liked_nodes- .iter()- .filter_map(|liked_node| liked_node.best_dist(&node).map(|dist| (dist, liked_node)))- .map(|(dist, liked_node)| liked_node.weight as f64 / dist as f64)- .sum::<f64>()- - self- .disliked_nodes+ let res = (1.0+ + (self+ .liked_nodes.iter()- .filter_map(|disliked_node| {- disliked_node- .best_dist(&node)- .map(|dist| (dist, disliked_node))+ .filter_map(|liked_node| {+ liked_node.best_dist(&node).map(|dist| (dist, liked_node))})- .map(|(dist, disliked_node)| disliked_node.weight as f64 / dist as f64)- .sum::<f64>())- / self.num_liked_nodes as f64)+ .map(|(dist, liked_node)| liked_node.weight as f64 / dist as f64)+ .sum::<f64>()+ - self+ .disliked_nodes+ .iter()+ .filter_map(|disliked_node| {+ disliked_node+ .best_dist(&node)+ .map(|dist| (dist, disliked_node))+ })+ .map(|(dist, disliked_node)| disliked_node.weight as f64 / dist as f64)+ .sum::<f64>())+ / self.num_liked_nodes as f64).max(0.0);self.cache.lock().unwrap().insert(node, res);@@ -308,7 +311,7 @@ impl ApproximatedHarmonicCentrality {let fixed_scores: HashMap<_, _> = liked_nodes.iter()- .map(|node| (*node, 1.0))+ .map(|node| (*node, 2.0)).chain(disliked_nodes.iter().map(|node| (*node, 0.0))).collect();@@ -422,7 +425,7 @@ mod tests {let scorer = centrality.scorer(&liked_nodes, &[]);for node in &liked_nodes {- assert_eq!(scorer.score(*centrality.node2id.get(node).unwrap()), 1.0);+ assert_eq!(scorer.score(*centrality.node2id.get(node).unwrap()), 2.0);}}