Skip to content

web search done right

BTreeMap instead of DashMap in harmonic centrality calculation. Should reduce memory consumption but cannot be parallel

Mikkel Denkerb3608f249c55
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index bbadc3cf..69a6f7d5 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -19,7 +19,7 @@ use crate::{
entrypoint::download_all_warc_files,
mapreduce::Worker,
webgraph::{self, Node, WebgraphWriter},
- webpage::{url_ext::UrlExt, Html},
+ webpage::Html,
Result,
};
use itertools::Itertools;
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index c41a5e30..0adf3ec5 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -15,15 +15,13 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{
- collections::BTreeMap,
+ collections::{BTreeMap, BTreeSet},
sync::{atomic::AtomicBool, Mutex},
};
use std::sync::atomic::Ordering;
use bitvec::vec::BitVec;
-use dashmap::{DashMap, DashSet};
-use rayon::prelude::*;
use tracing::info;
use crate::{
@@ -87,8 +85,8 @@ impl JankyBloomFilter {
fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let mut num_nodes = 0;
- let mut counters: DashMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = DashMap::new();
- let centralities: DashMap<NodeID, KahanSum> = DashMap::new();
+ let mut counters: BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = BTreeMap::new();
+ let mut centralities: BTreeMap<NodeID, KahanSum> = BTreeMap::new();
for node in graph.nodes() {
let mut counter = HyperLogLog::default();
@@ -114,14 +112,14 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let has_changes = AtomicBool::new(true);
let mut t = 0;
- let mut exact_changed_nodes: DashSet<NodeID> = DashSet::default();
+ let mut exact_changed_nodes: BTreeSet<NodeID> = BTreeSet::default();
loop {
if !has_changes.load(Ordering::Relaxed) {
break;
}
- let new_counters: DashMap<_, _> = counters.clone();
+ let mut new_counters: BTreeMap<_, _> = counters.clone();
has_changes.store(false, Ordering::Relaxed);
let new_changed_nodes = Mutex::new(JankyBloomFilter::new(num_nodes as u64, 0.05));
@@ -129,11 +127,11 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
if !exact_changed_nodes.is_empty()
&& exact_changed_nodes.len() as u64 <= exact_counting_threshold
{
- let new_exact_changed_nodes = DashSet::default();
+ let mut new_exact_changed_nodes = BTreeSet::default();
- exact_changed_nodes.par_iter().for_each(|changed_node| {
- for edge in graph.raw_outgoing_edges(&changed_node) {
- if let (Some(mut counter_to), Some(counter_from)) =
+ exact_changed_nodes.iter().for_each(|changed_node| {
+ for edge in graph.raw_outgoing_edges(changed_node) {
+ if let (Some(counter_to), Some(counter_from)) =
(new_counters.get_mut(&edge.to), counters.get(&edge.from))
{
if counter_to
@@ -142,7 +140,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
.zip(counter_from.registers().iter())
.any(|(to, from)| *from > *to)
{
- counter_to.merge(&counter_from);
+ counter_to.merge(counter_from);
new_changed_nodes.lock().unwrap().insert(edge.to.bit_64());
new_exact_changed_nodes.insert(edge.to);
@@ -155,10 +153,10 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
exact_changed_nodes = new_exact_changed_nodes;
} else {
- exact_changed_nodes = DashSet::default();
- graph.par_edges().for_each(|edge| {
+ exact_changed_nodes = BTreeSet::default();
+ graph.edges().for_each(|edge| {
if changed_nodes.contains(&edge.from.bit_64()) {
- if let (Some(mut counter_to), Some(counter_from)) =
+ if let (Some(counter_to), Some(counter_from)) =
(new_counters.get_mut(&edge.to), counters.get(&edge.from))
{
if counter_to
@@ -167,7 +165,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
.zip(counter_from.registers().iter())
.any(|(to, from)| *from > *to)
{
- counter_to.merge(&counter_from);
+ counter_to.merge(counter_from);
new_changed_nodes.lock().unwrap().insert(edge.to.bit_64());
if exact_counting {
@@ -181,8 +179,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
})
}
- centralities.par_iter_mut().for_each(|mut r| {
- let (node, score) = r.pair_mut();
+ centralities.iter_mut().for_each(|(node, score)| {
*score += new_counters
.get(node)
.map(|counter| counter.size())