web search done right
optionally store pages with 0 centrality from approx harmonic
Mikkel Denker4fbffe63b4f0
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex ec0af33c..803d8575 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -195,4 +195,8 @@ impl ApproxHarmonic {pub fn max_distance() -> u8 {7}++ pub fn save_centralities_with_zero() -> bool {+ false+ }}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 996c9d92..274bedd2 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -590,6 +590,9 @@ pub struct ApproxHarmonicCoordinatorConfig {#[serde(default = "defaults::ApproxHarmonic::max_distance")]pub max_distance: u8,++ #[serde(default = "defaults::ApproxHarmonic::save_centralities_with_zero")]+ pub save_centralities_with_zero: bool,}#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex b0a359ed..4bcd11ae 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -352,7 +352,9 @@ impl CrawlPlanner {add_nodes_to_grouper(nodes.into_iter().flatten(), &mut grouper, &mut budgets);}- // make sure frontpage is added for all hosts+ // make sure frontpage is added for all hosts with a budget.+ // this is necessary to ensure that we crawl at least one page+ // for each host.let mut futures = FuturesOrdered::new();let num_missing_budgets = budgets.len();let missing_budgets = budgets.clone();diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rsindex b2c8e0de..a5d41d38 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/coordinator.rs@@ -140,12 +140,17 @@ pub fn build(dht: &[(ShardId, SocketAddr)],workers: Vec<RemoteApproxCentralityWorker>,sample_rate: f64,+ save_centralities_with_zero: bool,) -> Coordinator<ApproxCentralityJob> {let setup = ApproxCentralitySetup::new_for_dht_members(dht, workers.clone(), sample_rate);- Coordinator::new(setup, workers.clone())- .with_mapper(ApproxCentralityMapper::InitCentrality)- .with_mapper(ApproxCentralityMapper::ApproximateCentrality)+ let mut coord = Coordinator::new(setup, workers.clone());++ if save_centralities_with_zero {+ coord = coord.with_mapper(ApproxCentralityMapper::InitCentrality)+ }++ coord.with_mapper(ApproxCentralityMapper::ApproximateCentrality)}struct ClusterInfo {@@ -228,13 +233,27 @@ pub fn run(config: ApproxHarmonicCoordinatorConfig) -> Result<()> {}).collect();- let coordinator = build(&cluster.dht, cluster.workers.clone(), config.sample_rate);+ let coordinator = build(+ &cluster.dht,+ cluster.workers.clone(),+ config.sample_rate,+ config.save_centralities_with_zero,+ );let res = coordinator.run(jobs, ApproxCentralityFinish)?;let output_path = Path::new(&config.output_path);let store = store_harmonic(- res.centrality.iter().map(|(n, c)| (n, f64::from(c))),+ res.centrality+ .iter()+ .map(|(n, c)| (n, f64::from(c)))+ .filter(|(_, c)| {+ if config.save_centralities_with_zero {+ true+ } else {+ *c > 0.0+ }+ }),output_path,);