Skip to content

web search done right

Hopefully reduce memory fragmentation when building inbound similarity vectors

Mikkel Denkerc8c79f56720a
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 62fc190a..66e3eeff 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -110,17 +110,16 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
})
{
let mut html = Html::parse_without_text(&record.response.body, &record.request.url);
+ let node_id = worker.centrality_store.node2id.get(
+ &html
+ .url()
+ .host_without_specific_subdomains_and_query()
+ .to_string()
+ .into(),
+ );
- let host_centrality = worker
- .centrality_store
- .harmonic
- .host
- .get(
- &html
- .url()
- .host_without_specific_subdomains_and_query()
- .to_string(),
- )
+ let host_centrality = node_id
+ .and_then(|node_id| worker.centrality_store.harmonic.host.get(node_id))
.unwrap_or_default();
if let Some(host_centrality_threshold) = job.host_centrality_threshold {
@@ -154,11 +153,13 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
})
.unwrap_or_else(Vec::new);
- let page_centrality = worker
+ let node_id = worker
.centrality_store
- .harmonic
- .full
- .get(&html.url().without_protocol().to_lowercase())
+ .node2id
+ .get(&html.url().without_protocol().to_lowercase().into());
+
+ let page_centrality = node_id
+ .and_then(|node_id| worker.centrality_store.harmonic.full.get(node_id))
.unwrap_or_default();
let fetch_time_ms = record.metadata.fetch_time_ms as u64;
diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rs
index ff3ab1d9..45556538 100644
--- a/core/src/ranking/bitvec_similarity.rs
+++ b/core/src/ranking/bitvec_similarity.rs
@@ -30,14 +30,14 @@ pub struct BitVec {
}
impl BitVec {
- pub fn new(data: Vec<bool>) -> Self {
+ pub fn new(data: &[bool]) -> Self {
let mut blocks = Vec::new();
let mut len = 0;
- for (offset, chunk) in data.into_iter().chunks(64).into_iter().enumerate() {
+ for (offset, chunk) in data.iter().chunks(64).into_iter().enumerate() {
let mut data = 0u64;
for (i, b) in chunk.into_iter().enumerate() {
- if b {
+ if *b {
data |= 1u64 << (63 - i);
len += 1;
}
@@ -117,8 +117,8 @@ mod tests {
assert!(expected > 0.894);
- let a = BitVec::new(a);
- let b = BitVec::new(b);
+ let a = BitVec::new(&a);
+ let b = BitVec::new(&b);
assert!((expected - a.sim(&b)).abs() < 0.00001);
}
@@ -129,8 +129,8 @@ mod tests {
let b: Vec<_> = repeat(true).take(1000).collect();
- let a = BitVec::new(a);
- let b = BitVec::new(b);
+ let a = BitVec::new(&a);
+ let b = BitVec::new(&b);
assert_eq!(a.sim(&b), 0.0);
}
@@ -140,8 +140,8 @@ mod tests {
let a: Vec<_> = Vec::new();
let b: Vec<_> = Vec::new();
- let a = BitVec::new(a);
- let b = BitVec::new(b);
+ let a = BitVec::new(&a);
+ let b = BitVec::new(&b);
assert_eq!(a.sim(&b), 0.0);
}
@@ -163,8 +163,8 @@ mod tests {
assert!(expected < 0.01);
- let a = BitVec::new(a);
- let b = BitVec::new(b);
+ let a = BitVec::new(&a);
+ let b = BitVec::new(&b);
assert!((expected - a.sim(&b)).abs() < 0.00001);
}
diff --git a/core/src/ranking/centrality_store.rs b/core/src/ranking/centrality_store.rs
index 93364a7b..c3e58427 100644
--- a/core/src/ranking/centrality_store.rs
+++ b/core/src/ranking/centrality_store.rs
@@ -29,8 +29,8 @@ use crate::{
use super::inbound_similarity::InboundSimilarity;
pub struct HarmonicCentralityStore {
- pub host: Box<dyn Kv<String, f64>>,
- pub full: Box<dyn Kv<String, f64>>,
+ pub host: Box<dyn Kv<NodeID, f64>>,
+ pub full: Box<dyn Kv<NodeID, f64>>,
}
impl HarmonicCentralityStore {
@@ -149,7 +149,8 @@ impl CentralityStore {
host.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
let mut wtr = csv::Writer::from_writer(csv_file);
for (node, centrality) in host {
- store.harmonic.host.insert(node.name.clone(), centrality);
+ let node_id = store.node2id.get(&node).unwrap();
+ store.harmonic.host.insert(*node_id, centrality);
wtr.write_record(&[node.name, centrality.to_string()])
.unwrap();
}
@@ -160,6 +161,7 @@ impl CentralityStore {
pub fn build<P: AsRef<Path>>(graph: &Webgraph, output_path: P) -> Self {
let mut store = CentralityStore::open(output_path.as_ref());
+ store.node2id = graph.node_ids().collect();
let harmonic_centrality = HarmonicCentrality::calculate(graph);
Self::store_host(&output_path, &mut store, harmonic_centrality);
@@ -167,8 +169,6 @@ impl CentralityStore {
store.online_harmonic = OnlineHarmonicCentrality::new(graph, &store.harmonic);
store.inbound_similarity = InboundSimilarity::build(graph, &store.harmonic);
- store.node2id = graph.node_ids().collect();
-
store.flush();
store
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 4500758d..b362a352 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -31,7 +31,7 @@ use crate::{
use super::{bitvec_similarity, centrality_store::HarmonicCentralityStore};
const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_INBOUND: f64 = 0.038;
-const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES: f64 = 0.03;
+const DEFAULT_HARMONIC_CENTRALITY_THRESHOLD_FOR_NODES: f64 = 0.038;
const SCORE_SCALE: f64 = 5.0;
pub struct Scorer {
@@ -80,33 +80,27 @@ impl InboundSimilarity {
let nodes: Vec<_> = graph.nodes().collect();
if let Some(max_node) = nodes.iter().max().copied() {
- for node_id in nodes
- .into_iter()
- .filter(|node_id| match graph.id2node(node_id) {
- Some(node) => {
- let score = harmonic.host.get(&node.into_host().name).unwrap_or(0.0);
- score >= harmonic_centrality_threshold_nodes
- }
- None => false,
- })
- {
- let mut buf = vec![false; max_node.0 as usize];
+ let mut buf = vec![false; max_node.0 as usize];
+
+ for node_id in nodes.into_iter().filter(|node_id| {
+ let score = harmonic.host.get(node_id).unwrap_or(0.0);
+ score >= harmonic_centrality_threshold_nodes
+ }) {
+ buf.clear();
+ buf.resize(max_node.0 as usize, false);
for edge in graph
.raw_ingoing_edges(&node_id)
.into_iter()
- .filter(|edge| match graph.id2node(&edge.from) {
- Some(node) => {
- let score = harmonic.host.get(&node.into_host().name).unwrap_or(0.0);
- score >= harmonic_centrality_threshold_inbound
- }
- None => false,
+ .filter(|edge| {
+ let score = harmonic.host.get(&edge.from).unwrap_or(0.0);
+ score >= harmonic_centrality_threshold_inbound
})
{
buf[edge.from.0 as usize] = true;
}
- vectors.insert(node_id.0, bitvec_similarity::BitVec::new(buf));
+ vectors.insert(node_id.0, bitvec_similarity::BitVec::new(&buf));
}
}
@@ -188,7 +182,9 @@ mod tests {
let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
for (node, centrality) in harmonic.host {
- harmonic_centrality_store.host.insert(node.name, centrality);
+ harmonic_centrality_store
+ .host
+ .insert(graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -216,7 +212,9 @@ mod tests {
let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
for (node, centrality) in harmonic.host {
- harmonic_centrality_store.host.insert(node.name, centrality);
+ harmonic_centrality_store
+ .host
+ .insert(graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 8a8bbecb..49c231c9 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -1250,7 +1250,9 @@ mod tests {
let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
for (node, centrality) in harmonic.host {
- harmonic_centrality_store.host.insert(node.name, centrality);
+ harmonic_centrality_store
+ .host
+ .insert(webgraph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 933c87f5..6103397d 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -353,8 +353,8 @@ impl OnlineHarmonicCentrality {
// as an estimate
let mut nodes: BinaryHeap<ProxyNodeCandidate> = BinaryHeap::new();
- for (node, _) in graph.node_ids() {
- let score = centrality.host.get(&node.name).unwrap_or(0.0);
+ for (node, node_id) in graph.node_ids() {
+ let score = centrality.host.get(&node_id).unwrap_or(0.0);
let candidate = ProxyNodeCandidate { node, score };
if nodes.len() >= num_proxy_nodes {
@@ -469,7 +469,9 @@ mod tests {
let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
for (node, centrality) in harmonic.host {
- harmonic_centrality_store.host.insert(node.name, centrality);
+ harmonic_centrality_store
+ .host
+ .insert(graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -505,7 +507,9 @@ mod tests {
let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
for (node, centrality) in harmonic.host {
- harmonic_centrality_store.host.insert(node.name, centrality);
+ harmonic_centrality_store
+ .host
+ .insert(graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();
@@ -531,7 +535,9 @@ mod tests {
let harmonic_centrality_store = HarmonicCentralityStore::open(crate::gen_temp_path());
for (node, centrality) in harmonic.host {
- harmonic_centrality_store.host.insert(node.name, centrality);
+ harmonic_centrality_store
+ .host
+ .insert(graph.node2id(&node).unwrap(), centrality);
}
harmonic_centrality_store.host.flush();