Skip to content

web search done right

coordinate changed nodes between workers in distrivuted harmonic to make sure a node that has been update in worker A is also considered for updates on worker B

Mikkel Denkerb302a8d5c70a
diff --git a/Cargo.lock b/Cargo.lock
index 52a72701..92e88ffe 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -545,6 +545,7 @@ version = "0.1.0"
dependencies = [
"bincode",
"bitvec",
+ "serde",
"xxhash-rust",
]
diff --git a/crates/bloom/Cargo.toml b/crates/bloom/Cargo.toml
index 951c9743..3be433db 100644
--- a/crates/bloom/Cargo.toml
+++ b/crates/bloom/Cargo.toml
@@ -1,12 +1,13 @@
[package]
-name = "bloom"
-version = "0.1.0"
edition = "2021"
license = "AGPL-3.0"
+name = "bloom"
+version = "0.1.0"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bincode.workspace = true
bitvec.workspace = true
+serde.workspace = true
xxhash-rust = "0.8.10"
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rs
index 47d3b163..78ebd0a8 100644
--- a/crates/bloom/src/lib.rs
+++ b/crates/bloom/src/lib.rs
@@ -47,8 +47,9 @@ fn num_hashes(num_bits: u64, estimated_items: u64) -> u64 {
(((num_bits as f64) / estimated_items as f64 * 2.0_f64.ln()).ceil() as u64).max(1)
}
-#[derive(Clone)]
+#[derive(Clone, bincode::Encode, bincode::Decode, Debug, serde::Serialize, serde::Deserialize)]
pub struct U64BloomFilter {
+ #[bincode(with_serde)]
bit_vec: BitVec,
}
@@ -98,7 +99,9 @@ impl U64BloomFilter {
.unwrap_or_default()
}
- pub fn merge(&mut self, other: Self) {
+ pub fn union(&mut self, other: Self) {
+ debug_assert_eq!(self.bit_vec.len(), other.bit_vec.len());
+
self.bit_vec |= other.bit_vec;
}
}
diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
index 92101078..b7257c67 100644
--- a/crates/core/Cargo.toml
+++ b/crates/core/Cargo.toml
@@ -14,6 +14,7 @@ cors = []
default = ["cors"]
dev = ["cors"]
prod = ["cors"]
+return_body = []
[[bin]]
name = "stract"
diff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rs
index 4067ffa1..e6eb7306 100644
--- a/crates/core/examples/distributed_harmonic.rs
+++ b/crates/core/examples/distributed_harmonic.rs
@@ -1,26 +1,27 @@
-use std::time::Duration;
+use std::{net::SocketAddr, time::Duration};
use clap::Parser;
-use stract::distributed::member::ShardId;
+use stract::{distributed::member::ShardId, webgraph::Node, webpage::url_ext::UrlExt};
use tracing_subscriber::util::SubscriberInitExt;
#[derive(Parser)]
struct Args {
+ warc_path: String,
graph_path: String,
output_path: String,
}
-fn start_dht_thread() {
- std::thread::spawn(|| {
+fn start_dht_thread(id: u64, host: SocketAddr, gossip: SocketAddr) {
+ std::thread::spawn(move || {
let config = stract::entrypoint::ampc::dht::Config {
- node_id: 1,
- host: "0.0.0.0:3000".parse().unwrap(),
- shard: ShardId::new(1),
+ node_id: id,
+ host,
+ shard: ShardId::new(id),
seed_node: None,
gossip: Some(stract::config::GossipConfig {
cluster_id: "test".to_string(),
- seed_nodes: None,
- addr: "0.0.0.0:3001".parse().unwrap(),
+ seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),
+ addr: gossip,
}),
};
@@ -33,46 +34,135 @@ fn start_dht_thread() {
});
}
-fn start_worker_thread(graph_path: String) {
- std::thread::spawn(|| {
+fn start_worker_thread(graph_path: String, shard: ShardId, host: SocketAddr, gossip: SocketAddr) {
+ std::thread::spawn(move || {
let config = stract::config::HarmonicWorkerConfig {
gossip: stract::config::GossipConfig {
cluster_id: "test".to_string(),
seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),
- addr: "0.0.0.0:3003".parse().unwrap(),
+ addr: gossip,
},
- shard: ShardId::new(1),
+ shard,
graph_path,
- host: "0.0.0.0:3002".parse().unwrap(),
+ host,
};
stract::entrypoint::ampc::harmonic_centrality::worker::run(config).unwrap();
});
}
+fn build_graphs_if_not_exist(warc_path: &str, graph_path: &str) -> anyhow::Result<()> {
+ let path = std::path::Path::new(graph_path);
+
+ if path.exists() {
+ return Ok(());
+ }
+
+ tracing::info!("Building graphs from warc file: {}", warc_path);
+
+ std::fs::create_dir_all(path)?;
+ let warc = stract::warc::WarcFile::open(warc_path)?;
+
+ let num_records = warc.records().count();
+
+ let a_path = path.join("graph_a");
+ let b_path = path.join("graph_b");
+
+ let mut a =
+ stract::webgraph::WebgraphWriter::new(a_path, Default::default(), Default::default(), None);
+
+ let mut b =
+ stract::webgraph::WebgraphWriter::new(b_path, Default::default(), Default::default(), None);
+
+ for (i, record) in warc.records().flatten().enumerate() {
+ let webpage = match stract::webpage::Html::parse_without_text(
+ &record.response.body,
+ &record.request.url,
+ ) {
+ Ok(webpage) => webpage,
+ Err(err) => {
+ tracing::error!("error parsing webpage: {}", err);
+ continue;
+ }
+ };
+
+ for link in webpage
+ .anchor_links()
+ .into_iter()
+ .filter(|link| matches!(link.destination.scheme(), "http" | "https"))
+ {
+ let source = Node::from(link.source.clone()).into_host();
+ let destination = Node::from(link.destination.clone()).into_host();
+
+ let dest_domain = link.destination.root_domain();
+ let source_domain = link.source.root_domain();
+
+ if dest_domain.is_some() && source_domain.is_some() && dest_domain != source_domain {
+ if i <= num_records / 2 {
+ a.insert(source, destination, link.text);
+ } else {
+ b.insert(source, destination, link.text);
+ }
+ }
+ }
+ }
+
+ a.commit();
+ b.commit();
+
+ a.finalize();
+ b.finalize();
+
+ Ok(())
+}
+
fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt()
- .with_max_level(tracing::Level::INFO)
+ .with_env_filter(tracing_subscriber::EnvFilter::builder().from_env_lossy())
.without_time()
.with_target(false)
.finish()
.init();
+ let args = Args::parse();
+
+ build_graphs_if_not_exist(&args.warc_path, &args.graph_path)?;
+
tracing::info!("Starting distributed harmonic centrality worker");
- let args = Args::parse();
- start_dht_thread();
- std::thread::sleep(Duration::from_secs(1));
- start_worker_thread(args.graph_path);
+ start_dht_thread(
+ 1,
+ "0.0.0.0:3000".parse().unwrap(),
+ "0.0.0.0:3001".parse().unwrap(),
+ );
+ start_dht_thread(
+ 2,
+ "0.0.0.0:2998".parse().unwrap(),
+ "0.0.0.0:2999".parse().unwrap(),
+ );
std::thread::sleep(Duration::from_secs(1));
+ let graph_path = std::path::Path::new(&args.graph_path);
+ start_worker_thread(
+ graph_path.join("graph_a").to_str().unwrap().to_string(),
+ ShardId::new(1),
+ "0.0.0.0:3002".parse().unwrap(),
+ "0.0.0.0:3003".parse().unwrap(),
+ );
+ start_worker_thread(
+ graph_path.join("graph_b").to_str().unwrap().to_string(),
+ ShardId::new(2),
+ "0.0.0.0:3004".parse().unwrap(),
+ "0.0.0.0:3005".parse().unwrap(),
+ );
+ std::thread::sleep(Duration::from_secs(3));
let config = stract::config::HarmonicCoordinatorConfig {
gossip: stract::config::GossipConfig {
cluster_id: "test".to_string(),
seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),
- addr: "0.0.0.0:3005".parse().unwrap(),
+ addr: "0.0.0.0:3007".parse().unwrap(),
},
- host: "0.0.0.0:3004".parse().unwrap(),
+ host: "0.0.0.0:3006".parse().unwrap(),
output_path: args.output_path,
};
diff --git a/crates/core/src/ampc/dht/client.rs b/crates/core/src/ampc/dht/client.rs
index 39d2da8f..a448a9e1 100644
--- a/crates/core/src/ampc/dht/client.rs
+++ b/crates/core/src/ampc/dht/client.rs
@@ -101,6 +101,10 @@ impl Node {
Self { api }
}
+ pub async fn num_keys(&self, table: Table) -> Result<u64> {
+ self.api.num_keys(table).await
+ }
+
pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {
self.api.get(table, key).await
}
@@ -213,6 +217,10 @@ impl Shard {
self.node().batch_get(table, keys).await
}
+ pub async fn num_keys(&self, table: Table) -> Result<u64> {
+ self.node().num_keys(table).await
+ }
+
pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {
self.node().set(table, key, value).await
}
@@ -342,6 +350,16 @@ impl Client {
Ok(())
}
+ pub async fn num_keys(&self, table: Table) -> Result<u64> {
+ let mut total = 0;
+
+ for shard in self.shards.values() {
+ total += shard.num_keys(table.clone()).await?;
+ }
+
+ Ok(total)
+ }
+
pub async fn upsert<F: Into<UpsertEnum>>(
&self,
table: Table,
diff --git a/crates/core/src/ampc/dht/network/api.rs b/crates/core/src/ampc/dht/network/api.rs
index 169ac692..a6807ae3 100644
--- a/crates/core/src/ampc/dht/network/api.rs
+++ b/crates/core/src/ampc/dht/network/api.rs
@@ -78,6 +78,11 @@ pub struct BatchGet {
pub keys: Arc<Vec<Key>>,
}
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
+pub struct NumKeys {
+ pub table: Table,
+}
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct DropTable {
pub table: Table,
@@ -202,6 +207,20 @@ impl sonic::service::Message<Server> for BatchGet {
}
}
+impl sonic::service::Message<Server> for NumKeys {
+ type Response = u64;
+
+ async fn handle(self, server: &Server) -> Self::Response {
+ server
+ .state_machine_store
+ .state_machine
+ .read()
+ .await
+ .db
+ .num_keys(&self.table) as u64
+ }
+}
+
impl sonic::service::Message<Server> for DropTable {
type Response = Result<
(),
@@ -442,6 +461,39 @@ impl RemoteClient {
Err(anyhow!("failed to batch set values"))
}
+ pub async fn num_keys(&self, table: Table) -> Result<u64> {
+ for backoff in Self::retry_strat() {
+ match self
+ .self_remote
+ .send_with_timeout(
+ NumKeys {
+ table: table.clone(),
+ },
+ Duration::from_secs(5),
+ )
+ .await
+ {
+ Ok(res) => return Ok(res),
+ Err(e) => match e {
+ sonic::Error::IO(_)
+ | sonic::Error::ConnectionTimeout
+ | sonic::Error::RequestTimeout
+ | sonic::Error::PoolGet => {
+ tokio::time::sleep(backoff).await;
+ }
+ sonic::Error::BadRequest
+ | sonic::Error::BodyTooLarge {
+ body_size: _,
+ max_size: _,
+ }
+ | sonic::Error::Application(_) => return Err(e.into()),
+ },
+ }
+ }
+
+ Err(anyhow!("failed to get number of keys"))
+ }
+
pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {
for backoff in Self::retry_strat() {
match self
diff --git a/crates/core/src/ampc/dht/network/mod.rs b/crates/core/src/ampc/dht/network/mod.rs
index 95a89f55..48c66f8f 100644
--- a/crates/core/src/ampc/dht/network/mod.rs
+++ b/crates/core/src/ampc/dht/network/mod.rs
@@ -18,8 +18,8 @@ pub mod api;
pub mod raft;
use api::{
- AllTables, BatchGet, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Get, RangeGet,
- Set, Upsert,
+ AllTables, BatchGet, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Get, NumKeys,
+ RangeGet, Set, Upsert,
};
use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};
@@ -89,6 +89,7 @@ sonic_service!(
BatchGet,
Set,
BatchSet,
+ NumKeys,
Upsert,
BatchUpsert,
DropTable,
diff --git a/crates/core/src/ampc/dht/store.rs b/crates/core/src/ampc/dht/store.rs
index b6c0973e..62a4bec7 100644
--- a/crates/core/src/ampc/dht/store.rs
+++ b/crates/core/src/ampc/dht/store.rs
@@ -192,6 +192,10 @@ impl Db {
}
}
+ pub fn num_keys(&self, table: &Table) -> usize {
+ self.data.get(table).map(|m| m.len()).unwrap_or(0)
+ }
+
pub fn upsert(
&mut self,
table: Table,
diff --git a/crates/core/src/ampc/dht_conn.rs b/crates/core/src/ampc/dht_conn.rs
index c29f9df9..b6b4d9e1 100644
--- a/crates/core/src/ampc/dht_conn.rs
+++ b/crates/core/src/ampc/dht_conn.rs
@@ -202,6 +202,10 @@ pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {
block_on(self.client().batch_set(self.table().dht(), pairs)).unwrap();
}
+ fn num_keys(&self) -> u64 {
+ block_on(self.client().num_keys(self.table().dht())).unwrap()
+ }
+
fn upsert<F: Into<UpsertEnum>>(
&self,
upsert: F,
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rs
index 1a4d20b1..07e9ac7c 100644
--- a/crates/core/src/api/search.rs
+++ b/crates/core/src/api/search.rs
@@ -75,6 +75,7 @@ pub struct ApiSearchQuery {
#[serde(default = "defaults::SearchQuery::return_structured_data")]
pub return_structured_data: bool,
+ #[cfg(feature = "return_body")]
pub return_body: Option<ReturnBody>,
}
@@ -100,7 +101,10 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {
return_ranking_signals: api.return_ranking_signals,
safe_search: api.safe_search.unwrap_or(default.safe_search),
count_results_exact: api.count_results_exact,
+ #[cfg(feature = "return_body")]
return_body: api.return_body,
+ #[cfg(not(feature = "return_body"))]
+ return_body: None,
return_structured_data: api.return_structured_data,
})
}
diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rs
index 06ca6dfd..b1c38445 100644
--- a/crates/core/src/distributed/sonic/mod.rs
+++ b/crates/core/src/distributed/sonic/mod.rs
@@ -81,6 +81,8 @@ where
match tokio::time::timeout(timeout, TcpStream::connect(server)).await {
Ok(stream) => {
let stream = stream?;
+ stream.set_nodelay(true)?;
+
Ok(Connection {
stream,
marker: PhantomData,
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
index b39d2618..b06648cf 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs
@@ -32,11 +32,12 @@ use super::{CentralityJob, CentralityMapper, CentralityTables, Meta, RemoteCentr
pub struct CentralitySetup {
dht: DhtConn<CentralityTables>,
+ workers: Vec<RemoteCentralityWorker>,
}
impl CentralitySetup {
pub async fn new(cluster: &Cluster) -> Self {
- let members: Vec<_> = cluster
+ let dht_members: Vec<_> = cluster
.members()
.await
.into_iter()
@@ -49,19 +50,36 @@ impl CentralitySetup {
})
.collect();
- Self::new_for_dht_members(&members)
+ let workers: Vec<_> = cluster
+ .members()
+ .await
+ .into_iter()
+ .filter_map(|member| {
+ if let Service::HarmonicWorker { host, shard } = member.service {
+ Some(RemoteCentralityWorker::new(shard, host))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ Self::new_for_dht_members(&dht_members, workers)
}
- pub fn new_for_dht_members(members: &[(ShardId, SocketAddr)]) -> Self {
+ pub fn new_for_dht_members(
+ dht_members: &[(ShardId, SocketAddr)],
+ workers: Vec<RemoteCentralityWorker>,
+ ) -> Self {
let initial = CentralityTables {
- counters: DefaultDhtTable::new(members, "counters"),
- meta: DefaultDhtTable::new(members, "meta"),
- centrality: DefaultDhtTable::new(members, "centrality"),
+ counters: DefaultDhtTable::new(dht_members, "counters"),
+ meta: DefaultDhtTable::new(dht_members, "meta"),
+ centrality: DefaultDhtTable::new(dht_members, "centrality"),
+ changed_nodes: DefaultDhtTable::new(dht_members, "changed_nodes"),
};
let dht = DhtConn::new(initial);
- Self { dht }
+ Self { dht, workers }
}
}
@@ -73,19 +91,20 @@ impl Setup for CentralitySetup {
}
fn setup_round(&self, dht: &Self::DhtTables) {
- dht.meta.set(
- (),
- Meta {
- round_had_changes: false,
- },
- );
+ let mut meta = dht.meta.get(()).unwrap();
+ meta.round_had_changes = false;
+
+ dht.meta.set((), meta);
}
fn setup_first_round(&self, dht: &Self::DhtTables) {
+ let upper_bound_num_nodes = self.workers.iter().map(|w| w.num_nodes()).sum();
+
dht.meta.set(
(),
Meta {
round_had_changes: true, // force first round to run
+ upper_bound_num_nodes,
},
);
}
@@ -105,11 +124,14 @@ pub fn build(
dht: &[(ShardId, SocketAddr)],
workers: Vec<RemoteCentralityWorker>,
) -> Coordinator<CentralityJob> {
- let setup = CentralitySetup::new_for_dht_members(dht);
+ let setup = CentralitySetup::new_for_dht_members(dht, workers.clone());
Coordinator::new(setup, workers)
.with_mapper(CentralityMapper::SetupCounters)
+ .with_mapper(CentralityMapper::SetupBloom)
.with_mapper(CentralityMapper::Cardinalities)
+ .with_mapper(CentralityMapper::SaveBloom)
+ .with_mapper(CentralityMapper::UpdateBloom)
.with_mapper(CentralityMapper::Centralities)
}
@@ -182,7 +204,7 @@ pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {
let coordinator = build(&cluster.dht, cluster.workers.clone());
let res = coordinator.run(jobs, CentralityFinish)?;
- let num_nodes = cluster.workers.iter().map(|w| w.num_nodes()).sum::<u64>();
+ let num_nodes = res.counters.num_keys();
let output_path = Path::new(&config.output_path);
let store = store_harmonic(
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
index 1c37452b..bd0c0852 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
@@ -31,15 +31,18 @@ use crate::{
webgraph,
};
-use super::{CentralityJob, CentralityTables, CentralityWorker, Meta};
+use super::{CentralityJob, CentralityTables, CentralityWorker};
const OPS_BATCH_PER_SHARD: u64 = 4096;
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub enum CentralityMapper {
SetupCounters,
+ SetupBloom,
Cardinalities,
Centralities,
+ SaveBloom,
+ UpdateBloom,
}
impl CentralityMapper {
@@ -122,6 +125,7 @@ impl CentralityMapper {
}
fn update_dht(
+ worker: &CentralityWorker,
batch: &[webgraph::Edge<()>],
new_changed_nodes: &Mutex<U64BloomFilter>,
dht: &DhtConn<CentralityTables>,
@@ -134,17 +138,19 @@ impl CentralityMapper {
Self::update_changed_nodes(&changes, new_changed_nodes);
// if any nodes changed, indicate in dht that we aren't finished yet
- if changes.iter().any(|(_, upsert_res)| match upsert_res {
- UpsertAction::Merged => true,
- UpsertAction::NoChange => false,
- UpsertAction::Inserted => true,
- }) {
- dht.next().meta.set(
- (),
- Meta {
- round_had_changes: true,
- },
- )
+ if !worker.has_updated_meta_for_round()
+ && changes.iter().any(|(_, upsert_res)| match upsert_res {
+ UpsertAction::Merged => true,
+ UpsertAction::NoChange => false,
+ UpsertAction::Inserted => true,
+ })
+ {
+ // it's okay that this is not atomic as it will just cause an extra meta update
+ worker.set_has_updated_meta_for_round(true);
+
+ let mut meta = dht.next().meta.get(()).unwrap();
+ meta.round_had_changes = true;
+ dht.next().meta.set((), meta)
}
}
@@ -203,37 +209,49 @@ impl CentralityMapper {
}
fn map_setup_counters(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {
+ if worker.round() != 0 {
+ return;
+ }
+
// shards are the same for both prev and next
let num_shards = dht.prev().num_shards();
let batch_size = (num_shards * OPS_BATCH_PER_SHARD) as usize;
let pool = rayon::ThreadPoolBuilder::new().build().unwrap();
- if worker.round() == 0 {
- pool.scope(|s| {
- let mut batch = Vec::with_capacity(batch_size);
- let mut changed_nodes = worker.changed_nodes().lock().unwrap();
-
- for node in worker.graph().nodes() {
- changed_nodes.insert(node.as_u64());
- batch.push(node);
- if batch.len() >= batch_size {
- let update_batch = batch.clone();
- s.spawn(move |_| Self::setup_counters(&update_batch, dht));
-
- batch.clear();
- }
- }
+ pool.scope(|s| {
+ let mut batch = Vec::with_capacity(batch_size);
+ let mut changed_nodes = worker.changed_nodes().lock().unwrap();
- if !batch.is_empty() {
+ for node in worker.graph().nodes() {
+ changed_nodes.insert(node.as_u64());
+ batch.push(node);
+ if batch.len() >= batch_size {
let update_batch = batch.clone();
s.spawn(move |_| Self::setup_counters(&update_batch, dht));
+
+ batch.clear();
}
- });
+ }
+
+ if !batch.is_empty() {
+ let update_batch = batch.clone();
+ s.spawn(move |_| Self::setup_counters(&update_batch, dht));
+ }
+ });
+ }
+
+ fn map_setup_bloom(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {
+ if worker.round() != 0 {
+ return;
}
+
+ let upper_bound_num_nodes = dht.prev().meta.get(()).unwrap().upper_bound_num_nodes;
+ worker.setup_changed_nodes(upper_bound_num_nodes);
}
fn map_cardinalities(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {
+ worker.set_has_updated_meta_for_round(false);
// shards are the same for both prev and next
let num_shards = dht.prev().num_shards();
let batch_size = (num_shards * OPS_BATCH_PER_SHARD) as usize;
@@ -258,7 +276,9 @@ impl CentralityMapper {
let new_changed_nodes = Arc::clone(&new_changed_nodes);
let update_batch = batch.clone();
- s.spawn(move |_| Self::update_dht(&update_batch, &new_changed_nodes, dht));
+ s.spawn(move |_| {
+ Self::update_dht(worker, &update_batch, &new_changed_nodes, dht)
+ });
batch.clear();
}
@@ -268,7 +288,7 @@ impl CentralityMapper {
let new_changed_nodes = Arc::clone(&new_changed_nodes);
let update_batch = batch.clone();
- s.spawn(move |_| Self::update_dht(&update_batch, &new_changed_nodes, dht));
+ s.spawn(move |_| Self::update_dht(worker, &update_batch, &new_changed_nodes, dht));
}
});
*worker.changed_nodes().lock().unwrap() = new_changed_nodes.lock().unwrap().clone();
@@ -309,6 +329,26 @@ impl CentralityMapper {
}
});
}
+
+ fn map_save_bloom(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {
+ let changed_nodes = worker.changed_nodes().lock().unwrap();
+ dht.next()
+ .changed_nodes
+ .set(worker.shard(), changed_nodes.clone());
+ }
+
+ fn map_update_bloom(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {
+ let changed_nodes: Vec<_> = dht.next().changed_nodes.iter().map(|(_, v)| v).collect();
+
+ let mut new_changed_nodes =
+ U64BloomFilter::empty_from(&worker.changed_nodes().lock().unwrap());
+
+ for bloom in changed_nodes {
+ new_changed_nodes.union(bloom.clone());
+ }
+
+ *worker.changed_nodes().lock().unwrap() = new_changed_nodes;
+ }
}
impl Mapper for CentralityMapper {
@@ -316,8 +356,11 @@ impl Mapper for CentralityMapper {
fn map(&self, _: Self::Job, worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {
match self {
+ CentralityMapper::SetupBloom => Self::map_setup_bloom(worker, dht),
CentralityMapper::SetupCounters => Self::map_setup_counters(worker, dht),
CentralityMapper::Cardinalities => Self::map_cardinalities(worker, dht),
+ CentralityMapper::SaveBloom => Self::map_save_bloom(worker, dht),
+ CentralityMapper::UpdateBloom => Self::map_update_bloom(worker, dht),
CentralityMapper::Centralities => Self::map_centralities(worker, dht),
}
}
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
index 1595a8b9..7c6c8ecb 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs
@@ -24,6 +24,7 @@ pub mod coordinator;
mod mapper;
pub mod worker;
+use bloom::U64BloomFilter;
pub use coordinator::{CentralityFinish, CentralitySetup};
pub use mapper::CentralityMapper;
pub use worker::{CentralityWorker, RemoteCentralityWorker};
@@ -31,6 +32,7 @@ pub use worker::{CentralityWorker, RemoteCentralityWorker};
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct Meta {
round_had_changes: bool,
+ upper_bound_num_nodes: u64,
}
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
@@ -38,6 +40,7 @@ pub struct CentralityTables {
counters: DefaultDhtTable<webgraph::NodeID, HyperLogLog<64>>,
meta: DefaultDhtTable<(), Meta>,
centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,
+ changed_nodes: DefaultDhtTable<ShardId, U64BloomFilter>,
}
impl CentralityTables {
@@ -46,7 +49,10 @@ impl CentralityTables {
}
}
-impl_dht_tables!(CentralityTables, [counters, meta, centrality]);
+impl_dht_tables!(
+ CentralityTables,
+ [counters, meta, centrality, changed_nodes]
+);
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
pub struct CentralityJob {
@@ -67,17 +73,59 @@ impl Job for CentralityJob {
mod tests {
use tracing_test::traced_test;
- use crate::{free_socket_addr, webgraph::centrality::harmonic::HarmonicCentrality};
+ use crate::{
+ executor::Executor,
+ free_socket_addr,
+ webgraph::{centrality::harmonic::HarmonicCentrality, Compression, WebgraphWriter},
+ };
use super::*;
#[test]
#[traced_test]
fn test_simple_graph() {
- let graph = crate::webgraph::tests::test_graph();
- let expected = HarmonicCentrality::calculate(&graph);
- let num_nodes = graph.nodes().count();
- let worker = CentralityWorker::new(1.into(), graph);
+ let mut combined = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ let mut a = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+ let mut b = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ None,
+ );
+
+ let edges = crate::webgraph::tests::test_edges();
+
+ for (i, (from, to, label)) in edges.into_iter().enumerate() {
+ combined.insert(from.clone(), to.clone(), label.clone());
+
+ if i % 2 == 0 {
+ a.insert(from, to, label);
+ } else {
+ b.insert(from, to, label);
+ }
+ }
+
+ combined.commit();
+ a.commit();
+ b.commit();
+
+ let combined = combined.finalize();
+ let a = a.finalize();
+ let b = b.finalize();
+
+ let expected = HarmonicCentrality::calculate(&combined);
+ let num_nodes = combined.nodes().count();
+ let worker = CentralityWorker::new(1.into(), a);
let worker_addr = free_socket_addr();
@@ -85,15 +133,30 @@ mod tests {
worker.run(worker_addr).unwrap();
});
- std::thread::sleep(std::time::Duration::from_secs(1)); // Wait for worker to start
+ std::thread::sleep(std::time::Duration::from_secs(2)); // Wait for worker to start
+ let a = RemoteCentralityWorker::new(1.into(), worker_addr);
+
+ let worker = CentralityWorker::new(2.into(), b);
+ let worker_addr = free_socket_addr();
+ std::thread::spawn(move || {
+ worker.run(worker_addr).unwrap();
+ });
+
+ std::thread::sleep(std::time::Duration::from_secs(2)); // Wait for worker to start
- let remote_worker = RemoteCentralityWorker::new(1.into(), worker_addr);
+ let b = RemoteCentralityWorker::new(2.into(), worker_addr);
- assert_eq!(remote_worker.num_nodes(), num_nodes as u64);
+ // assert_eq!(a.num_nodes() + b.num_nodes(), num_nodes as u64);
let (dht_shard, dht_addr) = crate::entrypoint::ampc::dht::tests::setup();
- let res = coordinator::build(&[(dht_shard, dht_addr)], vec![remote_worker])
- .run(vec![CentralityJob { shard: 1.into() }], CentralityFinish)
+ let res = coordinator::build(&[(dht_shard, dht_addr)], vec![a, b])
+ .run(
+ vec![
+ CentralityJob { shard: 1.into() },
+ CentralityJob { shard: 2.into() },
+ ],
+ CentralityFinish,
+ )
.unwrap();
let mut actual = res
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
index 34fb932a..786114f4 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs
@@ -25,7 +25,10 @@ use crate::{
};
use std::{
net::SocketAddr,
- sync::{atomic::AtomicU64, Arc, Mutex},
+ sync::{
+ atomic::{AtomicBool, AtomicU64},
+ Arc, Mutex,
+ },
};
use super::CentralityJob;
@@ -35,6 +38,7 @@ pub struct CentralityWorker {
graph: Webgraph,
changed_nodes: Arc<Mutex<U64BloomFilter>>,
round: AtomicU64,
+ has_updated_meta_for_round: AtomicBool,
}
impl CentralityWorker {
@@ -51,9 +55,20 @@ impl CentralityWorker {
graph,
changed_nodes: Arc::new(Mutex::new(changed_nodes)),
round: AtomicU64::new(0),
+ has_updated_meta_for_round: AtomicBool::new(false),
}
}
+ pub fn setup_changed_nodes(&self, upper_bound_num_nodes: u64) {
+ let mut new_changed_nodes = U64BloomFilter::new(upper_bound_num_nodes, 0.05);
+
+ for node in self.graph.nodes() {
+ new_changed_nodes.insert(node.as_u64());
+ }
+
+ *self.changed_nodes.lock().unwrap() = new_changed_nodes;
+ }
+
pub fn shard(&self) -> ShardId {
self.shard
}
@@ -70,6 +85,16 @@ impl CentralityWorker {
self.round.load(std::sync::atomic::Ordering::Relaxed)
}
+ pub fn has_updated_meta_for_round(&self) -> bool {
+ self.has_updated_meta_for_round
+ .load(std::sync::atomic::Ordering::Relaxed)
+ }
+
+ pub fn set_has_updated_meta_for_round(&self, value: bool) {
+ self.has_updated_meta_for_round
+ .store(value, std::sync::atomic::Ordering::Relaxed);
+ }
+
pub fn inc_round(&self) -> u64 {
self.round
.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
diff --git a/crates/core/src/executor.rs b/crates/core/src/executor.rs
index 8556548c..fb0de0b3 100644
--- a/crates/core/src/executor.rs
+++ b/crates/core/src/executor.rs
@@ -19,7 +19,9 @@ use crate::{Error, Result};
use crossbeam_channel::unbounded;
use rayon::{ThreadPool, ThreadPoolBuilder};
+#[derive(Default)]
pub enum Executor {
+ #[default]
#[allow(unused)]
SingleThread,
ThreadPool(ThreadPool),
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index f0d73d91..550d4759 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -52,7 +52,7 @@ pub mod crawler;
pub mod distributed;
pub mod entity_index;
mod enum_map;
-mod executor;
+pub mod executor;
mod external_sort;
mod fastfield_reader;
pub mod feed;
@@ -237,7 +237,6 @@ fn free_socket_addr() -> std::net::SocketAddr {
use std::net::{Ipv4Addr, TcpListener};
let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();
-
listener.local_addr().unwrap()
}
diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rs
index a3a9826a..4e27ec8f 100644
--- a/crates/core/src/search_prettifier/mod.rs
+++ b/crates/core/src/search_prettifier/mod.rs
@@ -25,8 +25,10 @@ use chrono::{NaiveDateTime, Utc};
use url::Url;
use utoipa::ToSchema;
+#[cfg(feature = "return_body")]
+use crate::api::search::ReturnBody;
+
use crate::{
- api::search::ReturnBody,
highlighted::HighlightedFragment,
inverted_index::RetrievedWebpage,
ranking::{SignalEnumDiscriminants, SignalScore},
@@ -190,6 +192,7 @@ pub struct DisplayedWebpage {
pub domain: String,
pub pretty_url: String,
pub snippet: Snippet,
+ #[cfg(feature = "return_body")]
pub body: Option<String>,
pub rich_snippet: Option<RichSnippet>,
pub ranking_signals: Option<HashMap<SignalEnumDiscriminants, SignalScore>>,
@@ -220,11 +223,6 @@ impl DisplayedWebpage {
let domain = url.root_domain().unwrap_or_default().to_string();
let pretty_url = prettify_url(&url);
- let body = query.return_body.map(|r| match r {
- ReturnBody::All => webpage.body,
- ReturnBody::Truncated(n) => webpage.body.chars().take(n).collect::<String>(),
- });
-
let structured_data = if query.return_structured_data {
Some(
webpage
@@ -237,6 +235,12 @@ impl DisplayedWebpage {
None
};
+ #[cfg(feature = "return_body")]
+ let body = query.return_body.map(|r| match r {
+ ReturnBody::All => webpage.body,
+ ReturnBody::Truncated(n) => webpage.body.chars().take(n).collect::<String>(),
+ });
+
Self {
title: webpage.title,
site: url.normalized_host().unwrap_or_default().to_string(),
@@ -244,6 +248,7 @@ impl DisplayedWebpage {
pretty_url,
domain,
snippet,
+ #[cfg(feature = "return_body")]
body,
ranking_signals: None,
score: None,
diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rs
index 76622f01..3a5d546b 100644
--- a/crates/core/src/searcher/mod.rs
+++ b/crates/core/src/searcher/mod.rs
@@ -25,8 +25,9 @@ use optics::{HostRankings, Optic};
use utoipa::ToSchema;
+use crate::api::search::ReturnBody;
+
use crate::{
- api::search::ReturnBody,
bangs::BangHit,
collector::approx_count::Count,
config::defaults,
diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rs
index 54548eba..d5505de6 100644
--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs
+++ b/crates/core/src/webgraph/centrality/derived_harmonic.rs
@@ -52,7 +52,7 @@ impl BloomMap {
let mut bf = self.map.pop().unwrap().into_inner().unwrap();
for m in self.map {
- bf.merge(m.into_inner().unwrap());
+ bf.union(m.into_inner().unwrap());
}
bf
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 62a46071..1d91822a 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -345,7 +345,7 @@ impl Webgraph {
pub mod tests {
use super::*;
- fn test_edges() -> Vec<(Node, Node, String)> {
+ pub fn test_edges() -> Vec<(Node, Node, String)> {
vec![
(Node::from("A"), Node::from("B"), String::new()),
(Node::from("B"), Node::from("C"), String::new()),
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index f227ef46..ffedc5aa 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -196,7 +196,6 @@ export type ApiSearchQuery = {
optic?: string;
page?: number;
query: string;
- returnBody?: ReturnBody;
returnRankingSignals?: boolean;
returnStructuredData?: boolean;
safeSearch?: boolean;
@@ -273,7 +272,6 @@ export type DisplayedSidebar =
};
};
export type DisplayedWebpage = {
- body?: string;
domain: string;
likelyHasAds: boolean;
likelyHasPaywall: boolean;