Skip to content

web search done right

optionally store pages with 0 centrality from approx harmonic

Mikkel Denker4fbffe63b4f0
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index ec0af33c..803d8575 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -195,4 +195,8 @@ impl ApproxHarmonic {
pub fn max_distance() -> u8 {
7
}
+
+ pub fn save_centralities_with_zero() -> bool {
+ false
+ }
}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 996c9d92..274bedd2 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -590,6 +590,9 @@ pub struct ApproxHarmonicCoordinatorConfig {
#[serde(default = "defaults::ApproxHarmonic::max_distance")]
pub max_distance: u8,
+
+ #[serde(default = "defaults::ApproxHarmonic::save_centralities_with_zero")]
+ pub save_centralities_with_zero: bool,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index b0a359ed..4bcd11ae 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -352,7 +352,9 @@ impl CrawlPlanner {
add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);
}
- // make sure frontpage is added for all hosts
+ // make sure frontpage is added for all hosts with a budget.
+ // this is necessary to ensure that we crawl at least one page
+ // for each host.
let mut futures = FuturesOrdered::new();
let num_missing_budgets = budgets.len();
let missing_budgets = budgets.clone();
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
index b2c8e0de..a5d41d38 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs
@@ -140,12 +140,17 @@ pub fn build(
dht: &[(ShardId, SocketAddr)],
workers: Vec<RemoteApproxCentralityWorker>,
sample_rate: f64,
+ save_centralities_with_zero: bool,
) -> Coordinator<ApproxCentralityJob> {
let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);
- Coordinator::new(setup, workers.clone())
- .with_mapper(ApproxCentralityMapper::InitCentrality)
- .with_mapper(ApproxCentralityMapper::ApproximateCentrality)
+ let mut coord = Coordinator::new(setup, workers.clone());
+
+ if save_centralities_with_zero {
+ coord = coord.with_mapper(ApproxCentralityMapper::InitCentrality)
+ }
+
+ coord.with_mapper(ApproxCentralityMapper::ApproximateCentrality)
}
struct ClusterInfo {
@@ -228,13 +233,27 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {
})
.collect();
- let coordinator = build(&cluster.dht, cluster.workers.clone(), config.sample_rate);
+ let coordinator = build(
+ &cluster.dht,
+ cluster.workers.clone(),
+ config.sample_rate,
+ config.save_centralities_with_zero,
+ );
let res = coordinator.run(jobs, ApproxCentralityFinish)?;
let output_path = Path::new(&config.output_path);
let store = store_harmonic(
- res.centrality.iter().map(|(n, c)| (n, f64::from(c))),
+ res.centrality
+ .iter()
+ .map(|(n, c)| (n, f64::from(c)))
+ .filter(|(_, c)| {
+ if config.save_centralities_with_zero {
+ true
+ } else {
+ *c > 0.0
+ }
+ }),
output_path,
);