web search done right
Hopefully reduce memory fragmentation when building inbound similarity vectors
Mikkel Denkerc8c79f56720a
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 62fc190a..66e3eeff 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -110,17 +110,16 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {}){let mut html = Html::parse_without_text(&record.response.body, &record.request.url);+ let node_id = worker.centrality_store.node2id.get(+ &html+ .url()+ .host_without_specific_subdomains_and_query()+ .to_string()+ .into(),+ );- let host_centrality = worker- .centrality_store- .harmonic- .host- .get(- &html- .url()- .host_without_specific_subdomains_and_query()- .to_string(),- )+ let host_centrality = node_id+ .and_then(|node_id| worker.centrality_store.harmonic.host.get(node_id)).unwrap_or_default();if let Some(host_centrality_threshold) = job.host_centrality_threshold {@@ -154,11 +153,13 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {}).unwrap_or_else(Vec::new);- let page_centrality = worker+ let node_id = worker.centrality_store- .harmonic- .full- .get(&html.url().without_protocol().to_lowercase())+ .node2id+ .get(&html.url().without_protocol().to_lowercase().into());++ let page_centrality = node_id+ .and_then(|node_id| worker.centrality_store.harmonic.full.get(node_id)).unwrap_or_default();let fetch_time_ms = record.metadata.fetch_time_ms as u64;diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rsindex ff3ab1d9..45556538 100644--- a/core/src/ranking/bitvec_similarity.rs+++ b/core/src/ranking/bitvec_similarity.rs@@ -30,14 +30,14 @@ pub struct BitVec {}impl BitVec {- pub fn new(data: Vec<bool>) -> Self {+ pub fn new(data: &[bool]) -> Self {let mut blocks = Vec::new();let mut len = 0;- for (offset, chunk) in data.into_iter().chunks(64).into_iter().enumerate() {+ for (offset, chunk) in data.iter().chunks(64).into_iter().enumerate() {let mut data = 0u64;for (i, b) in chunk.into_iter().enumerate() {- if b {+ if *b {data |= 1u64 << (63 - i);len += 1;}@@ -117,8 +117,8 @@ mod tests {assert!(expected > 0.894);- let a = BitVec::new(a);- let b = BitVec::new(b);+ let a = BitVec::new(&a);+ let b = BitVec::new(&b);assert!((expected - a.sim(&b)).abs() < 0.00001);}@@ -129,8 +129,8 @@ mod tests {let b: Vec<_> = repeat(true).take(1000).collect();- let a = BitVec::new(a);- let b = BitVec::new(b);+ let a = BitVec::new(&a);+ let b = BitVec::new(&b);assert_eq!(a.sim(&b), 0.0);}@@ -140,8 +140,8 @@ mod tests {let a: Vec<_> = Vec::new();let b: Vec<_> = Vec::new();- let a = BitVec::new(a);- let b = BitVec::new(b);+ let a = BitVec::new(&a);+ let b = BitVec::new(&b);assert_eq!(a.sim(&b), 0.0);}@@ -163,8 +163,8 @@ mod tests {assert!(expected < 0.01);- let a = BitVec::new(a);- let b = BitVec::new(b);+ let a = BitVec::new(&a);+ let b = BitVec::new(&b);assert!((expected - a.sim(&b)).abs() < 0.00001);}diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rsindex 93364a7b..c3e58427 100644--- a/core/src/ranking/centrality_store.rs+++ b/core/src/ranking/centrality_store.rs@@ -29,8 +29,8 @@ use crate::{use super::inbound_similarity::InboundSimilarity;pub struct HarmonicCentralityStore {- pub host: Box<dyn Kv<String, f64>>,- pub full: Box<dyn Kv<String, f64>>,+ pub host: Box<dyn Kv<NodeID, f64>>,+ pub full: Box<dyn Kv<NodeID, f64>>,}impl HarmonicCentralityStore {@@ -149,7 +149,8 @@ impl CentralityStore {host.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));let mut wtr = csv::Writer::from_writer(csv_file);for (node, centrality) in host {- store.harmonic.host.insert(node.name.clone(), centrality);+ let node_id = store.node2id.get(&node).unwrap();+ store.harmonic.host.insert(*node_id, centrality);wtr.write_record(&[node.name, centrality.to_string()]).unwrap();}@@ -160,6 +161,7 @@ impl CentralityStore {pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {let mut store = CentralityStore::open(output_path.as_ref());+ store.node2id = graph.node_ids().collect();let harmonic_centrality = HarmonicCentrality::calculate(graph);Self::store_host(&output_path, &mut store, harmonic_centrality);@@ -167,8 +169,6 @@ impl CentralityStore {store.online_harmonic = OnlineHarmonicCentrality::new(graph, &store.harmonic);store.inbound_similarity = InboundSimilarity::build(graph, &store.harmonic);- store.node2id = graph.node_ids().collect();-store.flush();storediff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex 4500758d..b362a352 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -31,7 +31,7 @@ use crate::{use super::{bitvec_similarity, centrality_store::HarmonicCentralityStore};const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_INBOUND: f64 = 0.038;-const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES: f64 = 0.03;+const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES: f64 = 0.038;const SCORE_SCALE: f64 = 5.0;pub struct Scorer {@@ -80,33 +80,27 @@ impl InboundSimilarity {let nodes: Vec<_> = graph.nodes().collect();if let Some(max_node) = nodes.iter().max().copied() {- for node_id in nodes- .into_iter()- .filter(|node_id| match graph.id2node(node_id) {- Some(node) => {- let score = harmonic.host.get(&node.into_host().name).unwrap_or(0.0);- score >= harmonic_centrality_threshold_nodes- }- None => false,- })- {- let mut buf = vec![false; max_node.0 as usize];+ let mut buf = vec![false; max_node.0 as usize];++ for node_id in nodes.into_iter().filter(|node_id| {+ let score = harmonic.host.get(node_id).unwrap_or(0.0);+ score >= harmonic_centrality_threshold_nodes+ }) {+ buf.clear();+ buf.resize(max_node.0 as usize, false);for edge in graph.raw_ingoing_edges(&node_id).into_iter()- .filter(|edge| match graph.id2node(&edge.from) {- Some(node) => {- let score = harmonic.host.get(&node.into_host().name).unwrap_or(0.0);- score >= harmonic_centrality_threshold_inbound- }- None => false,+ .filter(|edge| {+ let score = harmonic.host.get(&edge.from).unwrap_or(0.0);+ score >= harmonic_centrality_threshold_inbound}){buf[edge.from.0 as usize] = true;}- vectors.insert(node_id.0, bitvec_similarity::BitVec::new(buf));+ vectors.insert(node_id.0, bitvec_similarity::BitVec::new(&buf));}}@@ -188,7 +182,9 @@ mod tests {let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());for (node, centrality) in harmonic.host {- harmonic_centrality_store.host.insert(node.name, centrality);+ harmonic_centrality_store+ .host+ .insert(graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -216,7 +212,9 @@ mod tests {let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());for (node, centrality) in harmonic.host {- harmonic_centrality_store.host.insert(node.name, centrality);+ harmonic_centrality_store+ .host+ .insert(graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 8a8bbecb..49c231c9 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -1250,7 +1250,9 @@ mod tests {let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());for (node, centrality) in harmonic.host {- harmonic_centrality_store.host.insert(node.name, centrality);+ harmonic_centrality_store+ .host+ .insert(webgraph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 933c87f5..6103397d 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -353,8 +353,8 @@ impl OnlineHarmonicCentrality {// as an estimatelet mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();- for (node, _) in graph.node_ids() {- let score = centrality.host.get(&node.name).unwrap_or(0.0);+ for (node, node_id) in graph.node_ids() {+ let score = centrality.host.get(&node_id).unwrap_or(0.0);let candidate = ProxyNodeCandidate { node, score };if nodes.len() >= num_proxy_nodes {@@ -469,7 +469,9 @@ mod tests {let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());for (node, centrality) in harmonic.host {- harmonic_centrality_store.host.insert(node.name, centrality);+ harmonic_centrality_store+ .host+ .insert(graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -505,7 +507,9 @@ mod tests {let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());for (node, centrality) in harmonic.host {- harmonic_centrality_store.host.insert(node.name, centrality);+ harmonic_centrality_store+ .host+ .insert(graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();@@ -531,7 +535,9 @@ mod tests {let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());for (node, centrality) in harmonic.host {- harmonic_centrality_store.host.insert(node.name, centrality);+ harmonic_centrality_store+ .host+ .insert(graph.node2id(&node).unwrap(), centrality);}harmonic_centrality_store.host.flush();