Skip to content

web search done right

turnoff betweenness centrality for proxy node selection (wasn't fast enough)

Mikkel Denker938d8d402c58
diff --git a/core/src/entrypoint/centrality.rs b/core/src/entrypoint/centrality.rs
index 76d5975d..c6aafa36 100644
--- a/core/src/entrypoint/centrality.rs
+++ b/core/src/entrypoint/centrality.rs
@@ -25,7 +25,6 @@ impl Centrality {
let graph = WebgraphBuilder::new(webgraph_path)
.read_only(true)
.with_host_graph()
- .with_full_graph()
.open();
CentralityStore::build(&graph, output_path);
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index d6079f7f..81ccbaff 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -64,12 +64,11 @@ impl CentralityStore {
}
}
- pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
- let mut store = CentralityStore::open(output_path.as_ref());
-
- store.approx_harmonic = ApproximatedHarmonicCentrality::new(graph);
- let harmonic_centrality = HarmonicCentrality::calculate(graph);
-
+ fn store_full<P: AsRef<Path>>(
+ output_path: P,
+ store: &mut CentralityStore,
+ harmonic_centrality: &HarmonicCentrality,
+ ) {
let csv_file = File::options()
.write(true)
.create(true)
@@ -78,7 +77,7 @@ impl CentralityStore {
.unwrap();
let mut wtr = csv::Writer::from_writer(csv_file);
- let mut full: Vec<_> = harmonic_centrality.full.into_iter().collect();
+ let mut full: Vec<_> = harmonic_centrality.full.clone().into_iter().collect();
full.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
for (node, centrality) in full {
store.harmonic.full.insert(node.name.clone(), centrality);
@@ -87,7 +86,13 @@ impl CentralityStore {
}
wtr.flush().unwrap();
+ }
+ fn store_host<P: AsRef<Path>>(
+ output_path: P,
+ store: &mut CentralityStore,
+ harmonic_centrality: &HarmonicCentrality,
+ ) {
let csv_file = File::options()
.write(true)
.create(true)
@@ -95,7 +100,7 @@ impl CentralityStore {
.open(output_path.as_ref().join("harmonic_host.csv"))
.unwrap();
- let mut host: Vec<_> = harmonic_centrality.host.into_iter().collect();
+ let mut host: Vec<_> = harmonic_centrality.host.clone().into_iter().collect();
host.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
let mut wtr = csv::Writer::from_writer(csv_file);
for (node, centrality) in host {
@@ -104,6 +109,22 @@ impl CentralityStore {
.unwrap();
}
wtr.flush().unwrap();
+ }
+
+ pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
+ let mut store = CentralityStore::open(output_path.as_ref());
+
+ let harmonic_centrality = HarmonicCentrality::calculate(graph);
+
+ if graph.full.is_some() {
+ Self::store_full(&output_path, &mut store, &harmonic_centrality)
+ }
+
+ if graph.host.is_some() {
+ Self::store_host(&output_path, &mut store, &harmonic_centrality)
+ }
+
+ store.approx_harmonic = ApproximatedHarmonicCentrality::new(graph);
store.flush();
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 0983ad26..82d55af1 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -171,7 +171,7 @@ impl LocalSearcher {
if !top_host_nodes.is_empty() {
let approx = centrality_store
.approx_harmonic
- .scorer_without_fixed_from_ids(&top_host_nodes, &[]);
+ .scorer_from_ids(&top_host_nodes, &[]);
ranker.set_query_centrality(approx);
}
}
diff --git a/core/src/webgraph/centrality/approximate_harmonic.rs b/core/src/webgraph/centrality/approximate_harmonic.rs
index 8f61dde9..6893c290 100644
--- a/core/src/webgraph/centrality/approximate_harmonic.rs
+++ b/core/src/webgraph/centrality/approximate_harmonic.rs
@@ -35,10 +35,10 @@ use serde::{Deserialize, Serialize};
use crate::intmap::IntMap;
use crate::webgraph::Node;
-use crate::webgraph::{centrality::betweenness::Betweenness, NodeID, Webgraph};
+use crate::webgraph::{NodeID, Webgraph};
use crate::Result;
-const NUM_PROXY_NODES: usize = 10_000;
+const NUM_PROXY_NODES: usize = 100_000;
const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;
const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
pub const SHIFT: f64 = 1.0;
@@ -94,21 +94,10 @@ pub struct Scorer {
num_liked_nodes: usize,
}
-fn create_user_nodes(
- nodes: &[NodeID],
- proxy_nodes: &[Arc<ProxyNode>],
- betweenness: &HashMap<NodeID, f64>,
-) -> Vec<UserNode> {
+fn create_user_nodes(nodes: &[NodeID], proxy_nodes: &[Arc<ProxyNode>]) -> Vec<UserNode> {
let mut res = Vec::new();
- let mut nodes = nodes.to_vec();
- nodes.sort_by(|a, b| {
- betweenness
- .get(b)
- .unwrap_or(&0.0)
- .partial_cmp(betweenness.get(a).unwrap_or(&0.0))
- .unwrap_or(std::cmp::Ordering::Equal)
- });
+ let nodes = nodes.to_vec();
for id in nodes {
let user_node = UserNode::new(id, proxy_nodes);
@@ -133,15 +122,14 @@ impl Scorer {
liked_nodes: &[NodeID],
disliked_nodes: &[NodeID],
fixed_scores: HashMap<NodeID, f64>,
- betweenness: &HashMap<NodeID, f64>,
) -> Self {
let num_liked_nodes = liked_nodes.len();
Self {
fixed_scores,
num_liked_nodes,
- liked_nodes: create_user_nodes(liked_nodes, proxy_nodes, betweenness),
- disliked_nodes: create_user_nodes(disliked_nodes, proxy_nodes, betweenness),
+ liked_nodes: create_user_nodes(liked_nodes, proxy_nodes),
+ disliked_nodes: create_user_nodes(disliked_nodes, proxy_nodes),
cache: Mutex::new(HashMap::new()),
}
}
@@ -244,7 +232,6 @@ impl UserNode {
pub struct ApproximatedHarmonicCentrality {
pub node2id: HashMap<Node, NodeID>,
pub proxy_nodes: Vec<Arc<ProxyNode>>,
- betweenness: HashMap<NodeID, f64>,
}
impl ApproximatedHarmonicCentrality {
@@ -253,16 +240,19 @@ impl ApproximatedHarmonicCentrality {
}
fn new_with_num_proxy(graph: &Webgraph, num_proxy_nodes: usize) -> Self {
- let betweenness = Betweenness::calculate_with_progress(graph);
-
- let mut nodes = betweenness.centrality.iter().collect_vec();
- nodes.sort_by(|a, b| b.1.partial_cmp(a.1).unwrap_or(std::cmp::Ordering::Equal));
-
let mut node2id: HashMap<Node, NodeID> = HashMap::new();
+ // we should probably choose the proxy nodes based on their
+ // betweenness centrality, but I don't know how we can approximate betweenness
+ // on a graph that cannot be in memory
+ let mut nodes = Vec::new();
if let Some(store) = graph.host.as_ref() {
for id in store.nodes() {
if let Some(node) = store.id2node(&id) {
+ if nodes.len() < num_proxy_nodes {
+ nodes.push(node.clone());
+ }
+
node2id.insert(node, id);
}
}
@@ -271,7 +261,6 @@ impl ApproximatedHarmonicCentrality {
let proxy_nodes: Vec<_> = nodes
.into_par_iter()
.take(num_proxy_nodes)
- .map(|(node, _)| node)
.filter(|node| node2id.contains_key(node))
.map(|node| {
let dist_to_node = graph.raw_host_distances(node.clone()).into_iter().collect();
@@ -281,7 +270,7 @@ impl ApproximatedHarmonicCentrality {
.collect();
Arc::new(ProxyNode {
- id: *node2id.get(node).unwrap(),
+ id: *node2id.get(&node).unwrap(),
dist_to_node,
dist_from_node,
})
@@ -290,11 +279,6 @@ impl ApproximatedHarmonicCentrality {
Self {
proxy_nodes,
- betweenness: betweenness
- .centrality
- .into_iter()
- .filter_map(|(node, centrality)| node2id.get(&node).map(|id| (*id, centrality)))
- .collect(),
node2id,
}
}
@@ -312,48 +296,15 @@ impl ApproximatedHarmonicCentrality {
.filter_map(|node| self.node2id.get(&node).copied())
.collect_vec();
- let fixed_scores: HashMap<_, _> = liked_nodes
- .iter()
- .map(|node| (*node, 2.0))
- .chain(disliked_nodes.iter().map(|node| (*node, 0.0)))
- .collect();
-
- Scorer::new(
- &self.proxy_nodes,
- &liked_nodes,
- &disliked_nodes,
- fixed_scores,
- &self.betweenness,
- )
+ self.scorer_from_ids(&liked_nodes, &disliked_nodes)
}
- pub fn scorer_without_fixed(&self, liked_nodes: &[Node], disliked_nodes: &[Node]) -> Scorer {
- let liked_nodes = liked_nodes
- .iter()
- .map(|node| node.clone().into_host())
- .filter_map(|node| self.node2id.get(&node).copied())
- .collect_vec();
-
- let disliked_nodes = disliked_nodes
- .iter()
- .map(|node| node.clone().into_host())
- .filter_map(|node| self.node2id.get(&node).copied())
- .collect_vec();
-
- self.scorer_without_fixed_from_ids(&liked_nodes, &disliked_nodes)
- }
-
- pub fn scorer_without_fixed_from_ids(
- &self,
- liked_nodes: &[NodeID],
- disliked_nodes: &[NodeID],
- ) -> Scorer {
+ pub fn scorer_from_ids(&self, liked_nodes: &[NodeID], disliked_nodes: &[NodeID]) -> Scorer {
Scorer::new(
&self.proxy_nodes,
liked_nodes,
disliked_nodes,
HashMap::new(),
- &self.betweenness,
)
}
@@ -418,54 +369,10 @@ mod tests {
graph
}
- #[test]
- fn proxy_nodes_selection() {
- // Liked nodes: E, C
- let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);
- let id2node: HashMap<NodeID, Node> = centrality
- .node2id
- .clone()
- .into_iter()
- .map(|(node, id)| (id, node))
- .collect();
-
- assert_eq!(centrality.proxy_nodes.len(), 3);
- let mut proxy_nodes = centrality
- .proxy_nodes
- .iter()
- .map(|node| id2node.get(&node.id).unwrap().clone())
- .collect_vec();
- proxy_nodes.sort();
-
- assert_eq!(
- proxy_nodes,
- vec![
- Node::from("D".to_string()),
- Node::from("E".to_string()),
- Node::from("H".to_string())
- ]
- );
- }
-
- #[test]
- fn liked_nodes_centrality() {
- let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);
-
- let liked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];
-
- let scorer = centrality.scorer(&liked_nodes, &[]);
-
- for node in &liked_nodes {
- assert_eq!(scorer.score(*centrality.node2id.get(node).unwrap()), 2.0);
- }
- }
-
#[test]
fn ordering() {
let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);
+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);
let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];
@@ -516,7 +423,7 @@ mod tests {
#[test]
fn disliked_nodes_centrality() {
let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);
+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);
let disliked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];
@@ -530,7 +437,7 @@ mod tests {
#[test]
fn ordering_with_dislikes() {
let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);
+ let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 5);
let liked_nodes = vec![Node::from("B".to_string()), Node::from("E".to_string())];
let disliked_nodes = vec![Node::from("F".to_string())];
@@ -579,24 +486,4 @@ mod tests {
)
);
}
-
- #[test]
- fn liked_nodes_centrality_without_fix() {
- let graph = test_graph();
- let centrality = ApproximatedHarmonicCentrality::new_with_num_proxy(&graph, 3);
-
- let liked_nodes = vec![Node::from("D".to_string()), Node::from("E".to_string())];
-
- let scorer = centrality.scorer_without_fixed(&liked_nodes, &[]);
-
- assert_eq!(
- scorer.score(*centrality.node2id.get(&liked_nodes[0]).unwrap()),
- 1.625
- );
-
- assert_eq!(
- scorer.score(*centrality.node2id.get(&liked_nodes[1]).unwrap()),
- 1.75
- );
- }
}
diff --git a/core/src/webgraph/centrality/topic.rs b/core/src/webgraph/centrality/topic.rs
index c81fbfac..44639ead 100644
--- a/core/src/webgraph/centrality/topic.rs
+++ b/core/src/webgraph/centrality/topic.rs
@@ -177,7 +177,7 @@ impl TopicCentrality {
assert!(!top_sites.is_empty());
- let scorer = approx.scorer_without_fixed(&top_sites, &[]);
+ let scorer = approx.scorer(&top_sites, &[]);
for node in &nodes {
node_scores[*node as usize][i] = scorer.score(*node) - SHIFT;