web search done right
coordinate changed nodes between workers in distrivuted harmonic to make sure a node that has been update in worker A is also considered for updates on worker B
Mikkel Denkerb302a8d5c70a
diff --git a/Cargo.lock b/Cargo.lockindex 52a72701..92e88ffe 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -545,6 +545,7 @@ version = "0.1.0"dependencies = ["bincode","bitvec",+ "serde","xxhash-rust",]diff --git a/crates/bloom/Cargo.toml b/crates/bloom/Cargo.tomlindex 951c9743..3be433db 100644--- a/crates/bloom/Cargo.toml+++ b/crates/bloom/Cargo.toml@@ -1,12 +1,13 @@[package]-name = "bloom"-version = "0.1.0"edition = "2021"license = "AGPL-3.0"+name = "bloom"+version = "0.1.0"# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html[dependencies]bincode.workspace = truebitvec.workspace = true+serde.workspace = truexxhash-rust = "0.8.10"diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rsindex 47d3b163..78ebd0a8 100644--- a/crates/bloom/src/lib.rs+++ b/crates/bloom/src/lib.rs@@ -47,8 +47,9 @@ fn num_hashes(num_bits: u64, estimated_items: u64) -> u64 {(((num_bits as f64) / estimated_items as f64 * 2.0_f64.ln()).ceil() as u64).max(1)}-#[derive(Clone)]+#[derive(Clone, bincode::Encode, bincode::Decode, Debug, serde::Serialize, serde::Deserialize)]pub struct U64BloomFilter {+ #[bincode(with_serde)]bit_vec: BitVec,}@@ -98,7 +99,9 @@ impl U64BloomFilter {.unwrap_or_default()}- pub fn merge(&mut self, other: Self) {+ pub fn union(&mut self, other: Self) {+ debug_assert_eq!(self.bit_vec.len(), other.bit_vec.len());+self.bit_vec |= other.bit_vec;}}diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 92101078..b7257c67 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -14,6 +14,7 @@ cors = []default = ["cors"]dev = ["cors"]prod = ["cors"]+return_body = [][[bin]]name = "stract"diff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rsindex 4067ffa1..e6eb7306 100644--- a/crates/core/examples/distributed_harmonic.rs+++ b/crates/core/examples/distributed_harmonic.rs@@ -1,26 +1,27 @@-use std::time::Duration;+use std::{net::SocketAddr, time::Duration};use clap::Parser;-use stract::distributed::member::ShardId;+use stract::{distributed::member::ShardId, webgraph::Node, webpage::url_ext::UrlExt};use tracing_subscriber::util::SubscriberInitExt;#[derive(Parser)]struct Args {+ warc_path: String,graph_path: String,output_path: String,}-fn start_dht_thread() {- std::thread::spawn(|| {+fn start_dht_thread(id: u64, host: SocketAddr, gossip: SocketAddr) {+ std::thread::spawn(move || {let config = stract::entrypoint::ampc::dht::Config {- node_id: 1,- host: "0.0.0.0:3000".parse().unwrap(),- shard: ShardId::new(1),+ node_id: id,+ host,+ shard: ShardId::new(id),seed_node: None,gossip: Some(stract::config::GossipConfig {cluster_id: "test".to_string(),- seed_nodes: None,- addr: "0.0.0.0:3001".parse().unwrap(),+ seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),+ addr: gossip,}),};@@ -33,46 +34,135 @@ fn start_dht_thread() {});}-fn start_worker_thread(graph_path: String) {- std::thread::spawn(|| {+fn start_worker_thread(graph_path: String, shard: ShardId, host: SocketAddr, gossip: SocketAddr) {+ std::thread::spawn(move || {let config = stract::config::HarmonicWorkerConfig {gossip: stract::config::GossipConfig {cluster_id: "test".to_string(),seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),- addr: "0.0.0.0:3003".parse().unwrap(),+ addr: gossip,},- shard: ShardId::new(1),+ shard,graph_path,- host: "0.0.0.0:3002".parse().unwrap(),+ host,};stract::entrypoint::ampc::harmonic_centrality::worker::run(config).unwrap();});}+fn build_graphs_if_not_exist(warc_path: &str, graph_path: &str) -> anyhow::Result<()> {+ let path = std::path::Path::new(graph_path);++ if path.exists() {+ return Ok(());+ }++ tracing::info!("Building graphs from warc file: {}", warc_path);++ std::fs::create_dir_all(path)?;+ let warc = stract::warc::WarcFile::open(warc_path)?;++ let num_records = warc.records().count();++ let a_path = path.join("graph_a");+ let b_path = path.join("graph_b");++ let mut a =+ stract::webgraph::WebgraphWriter::new(a_path, Default::default(), Default::default(), None);++ let mut b =+ stract::webgraph::WebgraphWriter::new(b_path, Default::default(), Default::default(), None);++ for (i, record) in warc.records().flatten().enumerate() {+ let webpage = match stract::webpage::Html::parse_without_text(+ &record.response.body,+ &record.request.url,+ ) {+ Ok(webpage) => webpage,+ Err(err) => {+ tracing::error!("error parsing webpage: {}", err);+ continue;+ }+ };++ for link in webpage+ .anchor_links()+ .into_iter()+ .filter(|link| matches!(link.destination.scheme(), "http" | "https"))+ {+ let source = Node::from(link.source.clone()).into_host();+ let destination = Node::from(link.destination.clone()).into_host();++ let dest_domain = link.destination.root_domain();+ let source_domain = link.source.root_domain();++ if dest_domain.is_some() && source_domain.is_some() && dest_domain != source_domain {+ if i <= num_records / 2 {+ a.insert(source, destination, link.text);+ } else {+ b.insert(source, destination, link.text);+ }+ }+ }+ }++ a.commit();+ b.commit();++ a.finalize();+ b.finalize();++ Ok(())+}+fn main() -> anyhow::Result<()> {tracing_subscriber::fmt()- .with_max_level(tracing::Level::INFO)+ .with_env_filter(tracing_subscriber::EnvFilter::builder().from_env_lossy()).without_time().with_target(false).finish().init();+ let args = Args::parse();++ build_graphs_if_not_exist(&args.warc_path, &args.graph_path)?;+tracing::info!("Starting distributed harmonic centrality worker");- let args = Args::parse();- start_dht_thread();- std::thread::sleep(Duration::from_secs(1));- start_worker_thread(args.graph_path);+ start_dht_thread(+ 1,+ "0.0.0.0:3000".parse().unwrap(),+ "0.0.0.0:3001".parse().unwrap(),+ );+ start_dht_thread(+ 2,+ "0.0.0.0:2998".parse().unwrap(),+ "0.0.0.0:2999".parse().unwrap(),+ );std::thread::sleep(Duration::from_secs(1));+ let graph_path = std::path::Path::new(&args.graph_path);+ start_worker_thread(+ graph_path.join("graph_a").to_str().unwrap().to_string(),+ ShardId::new(1),+ "0.0.0.0:3002".parse().unwrap(),+ "0.0.0.0:3003".parse().unwrap(),+ );+ start_worker_thread(+ graph_path.join("graph_b").to_str().unwrap().to_string(),+ ShardId::new(2),+ "0.0.0.0:3004".parse().unwrap(),+ "0.0.0.0:3005".parse().unwrap(),+ );+ std::thread::sleep(Duration::from_secs(3));let config = stract::config::HarmonicCoordinatorConfig {gossip: stract::config::GossipConfig {cluster_id: "test".to_string(),seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),- addr: "0.0.0.0:3005".parse().unwrap(),+ addr: "0.0.0.0:3007".parse().unwrap(),},- host: "0.0.0.0:3004".parse().unwrap(),+ host: "0.0.0.0:3006".parse().unwrap(),output_path: args.output_path,};diff --git a/crates/core/src/ampc/dht/client.rs b/crates/core/src/ampc/dht/client.rsindex 39d2da8f..a448a9e1 100644--- a/crates/core/src/ampc/dht/client.rs+++ b/crates/core/src/ampc/dht/client.rs@@ -101,6 +101,10 @@ impl Node {Self { api }}+ pub async fn num_keys(&self, table: Table) -> Result<u64> {+ self.api.num_keys(table).await+ }+pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {self.api.get(table, key).await}@@ -213,6 +217,10 @@ impl Shard {self.node().batch_get(table, keys).await}+ pub async fn num_keys(&self, table: Table) -> Result<u64> {+ self.node().num_keys(table).await+ }+pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {self.node().set(table, key, value).await}@@ -342,6 +350,16 @@ impl Client {Ok(())}+ pub async fn num_keys(&self, table: Table) -> Result<u64> {+ let mut total = 0;++ for shard in self.shards.values() {+ total += shard.num_keys(table.clone()).await?;+ }++ Ok(total)+ }+pub async fn upsert<F: Into<UpsertEnum>>(&self,table: Table,diff --git a/crates/core/src/ampc/dht/network/api.rs b/crates/core/src/ampc/dht/network/api.rsindex 169ac692..a6807ae3 100644--- a/crates/core/src/ampc/dht/network/api.rs+++ b/crates/core/src/ampc/dht/network/api.rs@@ -78,6 +78,11 @@ pub struct BatchGet {pub keys: Arc<Vec<Key>>,}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct NumKeys {+ pub table: Table,+}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct DropTable {pub table: Table,@@ -202,6 +207,20 @@ impl sonic::service::Message<Server> for BatchGet {}}+impl sonic::service::Message<Server> for NumKeys {+ type Response = u64;++ async fn handle(self, server: &Server) -> Self::Response {+ server+ .state_machine_store+ .state_machine+ .read()+ .await+ .db+ .num_keys(&self.table) as u64+ }+}+impl sonic::service::Message<Server> for DropTable {type Response = Result<(),@@ -442,6 +461,39 @@ impl RemoteClient {Err(anyhow!("failed to batch set values"))}+ pub async fn num_keys(&self, table: Table) -> Result<u64> {+ for backoff in Self::retry_strat() {+ match self+ .self_remote+ .send_with_timeout(+ NumKeys {+ table: table.clone(),+ },+ Duration::from_secs(5),+ )+ .await+ {+ Ok(res) => return Ok(res),+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolGet => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to get number of keys"))+ }+pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {for backoff in Self::retry_strat() {match selfdiff --git a/crates/core/src/ampc/dht/network/mod.rs b/crates/core/src/ampc/dht/network/mod.rsindex 95a89f55..48c66f8f 100644--- a/crates/core/src/ampc/dht/network/mod.rs+++ b/crates/core/src/ampc/dht/network/mod.rs@@ -18,8 +18,8 @@ pub mod api;pub mod raft;use api::{- AllTables, BatchGet, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Get, RangeGet,- Set, Upsert,+ AllTables, BatchGet, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Get, NumKeys,+ RangeGet, Set, Upsert,};use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};@@ -89,6 +89,7 @@ sonic_service!(BatchGet,Set,BatchSet,+ NumKeys,Upsert,BatchUpsert,DropTable,diff --git a/crates/core/src/ampc/dht/store.rs b/crates/core/src/ampc/dht/store.rsindex b6c0973e..62a4bec7 100644--- a/crates/core/src/ampc/dht/store.rs+++ b/crates/core/src/ampc/dht/store.rs@@ -192,6 +192,10 @@ impl Db {}}+ pub fn num_keys(&self, table: &Table) -> usize {+ self.data.get(table).map(|m| m.len()).unwrap_or(0)+ }+pub fn upsert(&mut self,table: Table,diff --git a/crates/core/src/ampc/dht_conn.rs b/crates/core/src/ampc/dht_conn.rsindex c29f9df9..b6b4d9e1 100644--- a/crates/core/src/ampc/dht_conn.rs+++ b/crates/core/src/ampc/dht_conn.rs@@ -202,6 +202,10 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {block_on(self.client().batch_set(self.table().dht(), pairs)).unwrap();}+ fn num_keys(&self) -> u64 {+ block_on(self.client().num_keys(self.table().dht())).unwrap()+ }+fn upsert<F: Into<UpsertEnum>>(&self,upsert: F,diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rsindex 1a4d20b1..07e9ac7c 100644--- a/crates/core/src/api/search.rs+++ b/crates/core/src/api/search.rs@@ -75,6 +75,7 @@ pub struct ApiSearchQuery {#[serde(default = "defaults::SearchQuery::return_structured_data")]pub return_structured_data: bool,+ #[cfg(feature = "return_body")]pub return_body: Option<ReturnBody>,}@@ -100,7 +101,10 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {return_ranking_signals: api.return_ranking_signals,safe_search: api.safe_search.unwrap_or(default.safe_search),count_results_exact: api.count_results_exact,+ #[cfg(feature = "return_body")]return_body: api.return_body,+ #[cfg(not(feature = "return_body"))]+ return_body: None,return_structured_data: api.return_structured_data,})}diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rsindex 06ca6dfd..b1c38445 100644--- a/crates/core/src/distributed/sonic/mod.rs+++ b/crates/core/src/distributed/sonic/mod.rs@@ -81,6 +81,8 @@ wherematch tokio::time::timeout(timeout, TcpStream::connect(server)).await {Ok(stream) => {let stream = stream?;+ stream.set_nodelay(true)?;+Ok(Connection {stream,marker: PhantomData,diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rsindex b39d2618..b06648cf 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs@@ -32,11 +32,12 @@ use super::{CentralityJob, CentralityMapper, CentralityTables, Meta, RemoteCentrpub struct CentralitySetup {dht: DhtConn<CentralityTables>,+ workers: Vec<RemoteCentralityWorker>,}impl CentralitySetup {pub async fn new(cluster: &Cluster) -> Self {- let members: Vec<_> = cluster+ let dht_members: Vec<_> = cluster.members().await.into_iter()@@ -49,19 +50,36 @@ impl CentralitySetup {}).collect();- Self::new_for_dht_members(&members)+ let workers: Vec<_> = cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::HarmonicWorker { host, shard } = member.service {+ Some(RemoteCentralityWorker::new(shard, host))+ } else {+ None+ }+ })+ .collect();++ Self::new_for_dht_members(&dht_members, workers)}- pub fn new_for_dht_members(members: &[(ShardId, SocketAddr)]) -> Self {+ pub fn new_for_dht_members(+ dht_members: &[(ShardId, SocketAddr)],+ workers: Vec<RemoteCentralityWorker>,+ ) -> Self {let initial = CentralityTables {- counters: DefaultDhtTable::new(members, "counters"),- meta: DefaultDhtTable::new(members, "meta"),- centrality: DefaultDhtTable::new(members, "centrality"),+ counters: DefaultDhtTable::new(dht_members, "counters"),+ meta: DefaultDhtTable::new(dht_members, "meta"),+ centrality: DefaultDhtTable::new(dht_members, "centrality"),+ changed_nodes: DefaultDhtTable::new(dht_members, "changed_nodes"),};let dht = DhtConn::new(initial);- Self { dht }+ Self { dht, workers }}}@@ -73,19 +91,20 @@ impl Setup for CentralitySetup {}fn setup_round(&self, dht: &Self::DhtTables) {- dht.meta.set(- (),- Meta {- round_had_changes: false,- },- );+ let mut meta = dht.meta.get(()).unwrap();+ meta.round_had_changes = false;++ dht.meta.set((), meta);}fn setup_first_round(&self, dht: &Self::DhtTables) {+ let upper_bound_num_nodes = self.workers.iter().map(|w| w.num_nodes()).sum();+dht.meta.set((),Meta {round_had_changes: true, // force first round to run+ upper_bound_num_nodes,},);}@@ -105,11 +124,14 @@ pub fn build(dht: &[(ShardId, SocketAddr)],workers: Vec<RemoteCentralityWorker>,) -> Coordinator<CentralityJob> {- let setup = CentralitySetup::new_for_dht_members(dht);+ let setup = CentralitySetup::new_for_dht_members(dht, workers.clone());Coordinator::new(setup, workers).with_mapper(CentralityMapper::SetupCounters)+ .with_mapper(CentralityMapper::SetupBloom).with_mapper(CentralityMapper::Cardinalities)+ .with_mapper(CentralityMapper::SaveBloom)+ .with_mapper(CentralityMapper::UpdateBloom).with_mapper(CentralityMapper::Centralities)}@@ -182,7 +204,7 @@ pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {let coordinator = build(&cluster.dht, cluster.workers.clone());let res = coordinator.run(jobs, CentralityFinish)?;- let num_nodes = cluster.workers.iter().map(|w| w.num_nodes()).sum::<u64>();+ let num_nodes = res.counters.num_keys();let output_path = Path::new(&config.output_path);let store = store_harmonic(diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rsindex 1c37452b..bd0c0852 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs@@ -31,15 +31,18 @@ use crate::{webgraph,};-use super::{CentralityJob, CentralityTables, CentralityWorker, Meta};+use super::{CentralityJob, CentralityTables, CentralityWorker};const OPS_BATCH_PER_SHARD: u64 = 4096;#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub enum CentralityMapper {SetupCounters,+ SetupBloom,Cardinalities,Centralities,+ SaveBloom,+ UpdateBloom,}impl CentralityMapper {@@ -122,6 +125,7 @@ impl CentralityMapper {}fn update_dht(+ worker: &CentralityWorker,batch: &[webgraph::Edge<()>],new_changed_nodes: &Mutex<U64BloomFilter>,dht: &DhtConn<CentralityTables>,@@ -134,17 +138,19 @@ impl CentralityMapper {Self::update_changed_nodes(&changes, new_changed_nodes);// if any nodes changed, indicate in dht that we aren't finished yet- if changes.iter().any(|(_, upsert_res)| match upsert_res {- UpsertAction::Merged => true,- UpsertAction::NoChange => false,- UpsertAction::Inserted => true,- }) {- dht.next().meta.set(- (),- Meta {- round_had_changes: true,- },- )+ if !worker.has_updated_meta_for_round()+ && changes.iter().any(|(_, upsert_res)| match upsert_res {+ UpsertAction::Merged => true,+ UpsertAction::NoChange => false,+ UpsertAction::Inserted => true,+ })+ {+ // it's okay that this is not atomic as it will just cause an extra meta update+ worker.set_has_updated_meta_for_round(true);++ let mut meta = dht.next().meta.get(()).unwrap();+ meta.round_had_changes = true;+ dht.next().meta.set((), meta)}}@@ -203,37 +209,49 @@ impl CentralityMapper {}fn map_setup_counters(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ if worker.round() != 0 {+ return;+ }+// shards are the same for both prev and nextlet num_shards = dht.prev().num_shards();let batch_size = (num_shards * OPS_BATCH_PER_SHARD) as usize;let pool = rayon::ThreadPoolBuilder::new().build().unwrap();- if worker.round() == 0 {- pool.scope(|s| {- let mut batch = Vec::with_capacity(batch_size);- let mut changed_nodes = worker.changed_nodes().lock().unwrap();-- for node in worker.graph().nodes() {- changed_nodes.insert(node.as_u64());- batch.push(node);- if batch.len() >= batch_size {- let update_batch = batch.clone();- s.spawn(move |_| Self::setup_counters(&update_batch, dht));-- batch.clear();- }- }+ pool.scope(|s| {+ let mut batch = Vec::with_capacity(batch_size);+ let mut changed_nodes = worker.changed_nodes().lock().unwrap();- if !batch.is_empty() {+ for node in worker.graph().nodes() {+ changed_nodes.insert(node.as_u64());+ batch.push(node);+ if batch.len() >= batch_size {let update_batch = batch.clone();s.spawn(move |_| Self::setup_counters(&update_batch, dht));++ batch.clear();}- });+ }++ if !batch.is_empty() {+ let update_batch = batch.clone();+ s.spawn(move |_| Self::setup_counters(&update_batch, dht));+ }+ });+ }++ fn map_setup_bloom(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ if worker.round() != 0 {+ return;}++ let upper_bound_num_nodes = dht.prev().meta.get(()).unwrap().upper_bound_num_nodes;+ worker.setup_changed_nodes(upper_bound_num_nodes);}fn map_cardinalities(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ worker.set_has_updated_meta_for_round(false);// shards are the same for both prev and nextlet num_shards = dht.prev().num_shards();let batch_size = (num_shards * OPS_BATCH_PER_SHARD) as usize;@@ -258,7 +276,9 @@ impl CentralityMapper {let new_changed_nodes = Arc::clone(&new_changed_nodes);let update_batch = batch.clone();- s.spawn(move |_| Self::update_dht(&update_batch, &new_changed_nodes, dht));+ s.spawn(move |_| {+ Self::update_dht(worker, &update_batch, &new_changed_nodes, dht)+ });batch.clear();}@@ -268,7 +288,7 @@ impl CentralityMapper {let new_changed_nodes = Arc::clone(&new_changed_nodes);let update_batch = batch.clone();- s.spawn(move |_| Self::update_dht(&update_batch, &new_changed_nodes, dht));+ s.spawn(move |_| Self::update_dht(worker, &update_batch, &new_changed_nodes, dht));}});*worker.changed_nodes().lock().unwrap() = new_changed_nodes.lock().unwrap().clone();@@ -309,6 +329,26 @@ impl CentralityMapper {}});}++ fn map_save_bloom(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ let changed_nodes = worker.changed_nodes().lock().unwrap();+ dht.next()+ .changed_nodes+ .set(worker.shard(), changed_nodes.clone());+ }++ fn map_update_bloom(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ let changed_nodes: Vec<_> = dht.next().changed_nodes.iter().map(|(_, v)| v).collect();++ let mut new_changed_nodes =+ U64BloomFilter::empty_from(&worker.changed_nodes().lock().unwrap());++ for bloom in changed_nodes {+ new_changed_nodes.union(bloom.clone());+ }++ *worker.changed_nodes().lock().unwrap() = new_changed_nodes;+ }}impl Mapper for CentralityMapper {@@ -316,8 +356,11 @@ impl Mapper for CentralityMapper {fn map(&self, _: Self::Job, worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {match self {+ CentralityMapper::SetupBloom => Self::map_setup_bloom(worker, dht),CentralityMapper::SetupCounters => Self::map_setup_counters(worker, dht),CentralityMapper::Cardinalities => Self::map_cardinalities(worker, dht),+ CentralityMapper::SaveBloom => Self::map_save_bloom(worker, dht),+ CentralityMapper::UpdateBloom => Self::map_update_bloom(worker, dht),CentralityMapper::Centralities => Self::map_centralities(worker, dht),}}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rsindex 1595a8b9..7c6c8ecb 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs@@ -24,6 +24,7 @@ pub mod coordinator;mod mapper;pub mod worker;+use bloom::U64BloomFilter;pub use coordinator::{CentralityFinish, CentralitySetup};pub use mapper::CentralityMapper;pub use worker::{CentralityWorker, RemoteCentralityWorker};@@ -31,6 +32,7 @@ pub use worker::{CentralityWorker, RemoteCentralityWorker};#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct Meta {round_had_changes: bool,+ upper_bound_num_nodes: u64,}#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]@@ -38,6 +40,7 @@ pub struct CentralityTables {counters: DefaultDhtTable<webgraph::NodeID, HyperLogLog<64>>,meta: DefaultDhtTable<(), Meta>,centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,+ changed_nodes: DefaultDhtTable<ShardId, U64BloomFilter>,}impl CentralityTables {@@ -46,7 +49,10 @@ impl CentralityTables {}}-impl_dht_tables!(CentralityTables, [counters, meta, centrality]);+impl_dht_tables!(+ CentralityTables,+ [counters, meta, centrality, changed_nodes]+);#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct CentralityJob {@@ -67,17 +73,59 @@ impl Job for CentralityJob {mod tests {use tracing_test::traced_test;- use crate::{free_socket_addr, webgraph::centrality::harmonic::HarmonicCentrality};+ use crate::{+ executor::Executor,+ free_socket_addr,+ webgraph::{centrality::harmonic::HarmonicCentrality, Compression, WebgraphWriter},+ };use super::*;#[test]#[traced_test]fn test_simple_graph() {- let graph = crate::webgraph::tests::test_graph();- let expected = HarmonicCentrality::calculate(&graph);- let num_nodes = graph.nodes().count();- let worker = CentralityWorker::new(1.into(), graph);+ let mut combined = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ let mut a = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );+ let mut b = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );++ let edges = crate::webgraph::tests::test_edges();++ for (i, (from, to, label)) in edges.into_iter().enumerate() {+ combined.insert(from.clone(), to.clone(), label.clone());++ if i % 2 == 0 {+ a.insert(from, to, label);+ } else {+ b.insert(from, to, label);+ }+ }++ combined.commit();+ a.commit();+ b.commit();++ let combined = combined.finalize();+ let a = a.finalize();+ let b = b.finalize();++ let expected = HarmonicCentrality::calculate(&combined);+ let num_nodes = combined.nodes().count();+ let worker = CentralityWorker::new(1.into(), a);let worker_addr = free_socket_addr();@@ -85,15 +133,30 @@ mod tests {worker.run(worker_addr).unwrap();});- std::thread::sleep(std::time::Duration::from_secs(1)); // Wait for worker to start+ std::thread::sleep(std::time::Duration::from_secs(2)); // Wait for worker to start+ let a = RemoteCentralityWorker::new(1.into(), worker_addr);++ let worker = CentralityWorker::new(2.into(), b);+ let worker_addr = free_socket_addr();+ std::thread::spawn(move || {+ worker.run(worker_addr).unwrap();+ });++ std::thread::sleep(std::time::Duration::from_secs(2)); // Wait for worker to start- let remote_worker = RemoteCentralityWorker::new(1.into(), worker_addr);+ let b = RemoteCentralityWorker::new(2.into(), worker_addr);- assert_eq!(remote_worker.num_nodes(), num_nodes as u64);+ // assert_eq!(a.num_nodes() + b.num_nodes(), num_nodes as u64);let (dht_shard, dht_addr) = crate::entrypoint::ampc::dht::tests::setup();- let res = coordinator::build(&[(dht_shard, dht_addr)], vec![remote_worker])- .run(vec![CentralityJob { shard: 1.into() }], CentralityFinish)+ let res = coordinator::build(&[(dht_shard, dht_addr)], vec![a, b])+ .run(+ vec![+ CentralityJob { shard: 1.into() },+ CentralityJob { shard: 2.into() },+ ],+ CentralityFinish,+ ).unwrap();let mut actual = resdiff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rsindex 34fb932a..786114f4 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs@@ -25,7 +25,10 @@ use crate::{};use std::{net::SocketAddr,- sync::{atomic::AtomicU64, Arc, Mutex},+ sync::{+ atomic::{AtomicBool, AtomicU64},+ Arc, Mutex,+ },};use super::CentralityJob;@@ -35,6 +38,7 @@ pub struct CentralityWorker {graph: Webgraph,changed_nodes: Arc<Mutex<U64BloomFilter>>,round: AtomicU64,+ has_updated_meta_for_round: AtomicBool,}impl CentralityWorker {@@ -51,9 +55,20 @@ impl CentralityWorker {graph,changed_nodes: Arc::new(Mutex::new(changed_nodes)),round: AtomicU64::new(0),+ has_updated_meta_for_round: AtomicBool::new(false),}}+ pub fn setup_changed_nodes(&self, upper_bound_num_nodes: u64) {+ let mut new_changed_nodes = U64BloomFilter::new(upper_bound_num_nodes, 0.05);++ for node in self.graph.nodes() {+ new_changed_nodes.insert(node.as_u64());+ }++ *self.changed_nodes.lock().unwrap() = new_changed_nodes;+ }+pub fn shard(&self) -> ShardId {self.shard}@@ -70,6 +85,16 @@ impl CentralityWorker {self.round.load(std::sync::atomic::Ordering::Relaxed)}+ pub fn has_updated_meta_for_round(&self) -> bool {+ self.has_updated_meta_for_round+ .load(std::sync::atomic::Ordering::Relaxed)+ }++ pub fn set_has_updated_meta_for_round(&self, value: bool) {+ self.has_updated_meta_for_round+ .store(value, std::sync::atomic::Ordering::Relaxed);+ }+pub fn inc_round(&self) -> u64 {self.round.fetch_add(1, std::sync::atomic::Ordering::Relaxed)diff --git a/crates/core/src/executor.rs b/crates/core/src/executor.rsindex 8556548c..fb0de0b3 100644--- a/crates/core/src/executor.rs+++ b/crates/core/src/executor.rs@@ -19,7 +19,9 @@ use crate::{Error, Result};use crossbeam_channel::unbounded;use rayon::{ThreadPool, ThreadPoolBuilder};+#[derive(Default)]pub enum Executor {+ #[default]#[allow(unused)]SingleThread,ThreadPool(ThreadPool),diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex f0d73d91..550d4759 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -52,7 +52,7 @@ pub mod crawler;pub mod distributed;pub mod entity_index;mod enum_map;-mod executor;+pub mod executor;mod external_sort;mod fastfield_reader;pub mod feed;@@ -237,7 +237,6 @@ fn free_socket_addr() -> std::net::SocketAddr {use std::net::{Ipv4Addr, TcpListener};let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();-listener.local_addr().unwrap()}diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rsindex a3a9826a..4e27ec8f 100644--- a/crates/core/src/search_prettifier/mod.rs+++ b/crates/core/src/search_prettifier/mod.rs@@ -25,8 +25,10 @@ use chrono::{NaiveDateTime, Utc};use url::Url;use utoipa::ToSchema;+#[cfg(feature = "return_body")]+use crate::api::search::ReturnBody;+use crate::{- api::search::ReturnBody,highlighted::HighlightedFragment,inverted_index::RetrievedWebpage,ranking::{SignalEnumDiscriminants, SignalScore},@@ -190,6 +192,7 @@ pub struct DisplayedWebpage {pub domain: String,pub pretty_url: String,pub snippet: Snippet,+ #[cfg(feature = "return_body")]pub body: Option<String>,pub rich_snippet: Option<RichSnippet>,pub ranking_signals: Option<HashMap<SignalEnumDiscriminants, SignalScore>>,@@ -220,11 +223,6 @@ impl DisplayedWebpage {let domain = url.root_domain().unwrap_or_default().to_string();let pretty_url = prettify_url(&url);- let body = query.return_body.map(|r| match r {- ReturnBody::All => webpage.body,- ReturnBody::Truncated(n) => webpage.body.chars().take(n).collect::<String>(),- });-let structured_data = if query.return_structured_data {Some(webpage@@ -237,6 +235,12 @@ impl DisplayedWebpage {None};+ #[cfg(feature = "return_body")]+ let body = query.return_body.map(|r| match r {+ ReturnBody::All => webpage.body,+ ReturnBody::Truncated(n) => webpage.body.chars().take(n).collect::<String>(),+ });+Self {title: webpage.title,site: url.normalized_host().unwrap_or_default().to_string(),@@ -244,6 +248,7 @@ impl DisplayedWebpage {pretty_url,domain,snippet,+ #[cfg(feature = "return_body")]body,ranking_signals: None,score: None,diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex 76622f01..3a5d546b 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -25,8 +25,9 @@ use optics::{HostRankings, Optic};use utoipa::ToSchema;+use crate::api::search::ReturnBody;+use crate::{- api::search::ReturnBody,bangs::BangHit,collector::approx_count::Count,config::defaults,diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rsindex 54548eba..d5505de6 100644--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs+++ b/crates/core/src/webgraph/centrality/derived_harmonic.rs@@ -52,7 +52,7 @@ impl BloomMap {let mut bf = self.map.pop().unwrap().into_inner().unwrap();for m in self.map {- bf.merge(m.into_inner().unwrap());+ bf.union(m.into_inner().unwrap());}bfdiff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 62a46071..1d91822a 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -345,7 +345,7 @@ impl Webgraph {pub mod tests {use super::*;- fn test_edges() -> Vec<(Node, Node, String)> {+ pub fn test_edges() -> Vec<(Node, Node, String)> {vec![(Node::from("A"), Node::from("B"), String::new()),(Node::from("B"), Node::from("C"), String::new()),diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex f227ef46..ffedc5aa 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -196,7 +196,6 @@ export type ApiSearchQuery = {optic?: string;page?: number;query: string;- returnBody?: ReturnBody;returnRankingSignals?: boolean;returnStructuredData?: boolean;safeSearch?: boolean;@@ -273,7 +272,6 @@ export type DisplayedSidebar =};};export type DisplayedWebpage = {- body?: string;domain: string;likelyHasAds: boolean;likelyHasPaywall: boolean;