Skip to content

web search done right

explicitly wait for graph nodes to come online

Mikkel Denkerf6f39b77cad6
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index b01c04a3..fcceb3a6 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -184,7 +184,7 @@ pub async fn router(
.iter()
.any(|m| m.service.is_searcher())
{
- log::warn!("Waiting for search nodes to join the cluster");
+ log::info!("Waiting for search nodes to join the cluster");
cluster.await_member(|m| m.service.is_searcher()).await;
log::info!("Search nodes joined the cluster");
}
diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rs
index 4a10bbec..de407448 100644
--- a/crates/core/src/crawler/planner.rs
+++ b/crates/core/src/crawler/planner.rs
@@ -166,7 +166,9 @@ impl CrawlPlanner {
Self::check_config(&config)?;
let page_graph = RemoteWebgraph::new(cluster.clone()).await;
+ page_graph.await_ready().await;
let host_graph = RemoteWebgraph::new(cluster.clone()).await;
+ host_graph.await_ready().await;
let domain_boosts = config
.domain_boosts
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 979b9218..186b69e8 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -152,10 +152,14 @@ impl Webgraph {
),
IndexerGraphConfig::Remote { gossip } => {
let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);
- Self::Remote(RemoteWebgraph::new(cluster).await)
+ let remote = RemoteWebgraph::new(cluster).await;
+ remote.await_ready().await;
+ Self::Remote(remote)
}
IndexerGraphConfig::Existing { cluster } => {
- Self::Remote(RemoteWebgraph::new(cluster.clone()).await)
+ let remote = RemoteWebgraph::new(cluster.clone()).await;
+ remote.await_ready().await;
+ Self::Remote(remote)
}
}
}
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index 411529d0..a4f566fe 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -114,37 +114,38 @@ where
#[derive(Clone)]
pub struct RemoteWebgraph<G: WebgraphGranularity> {
client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager<G>>>>,
+ cluster: Arc<Cluster>,
}
impl<G: WebgraphGranularity> RemoteWebgraph<G> {
pub async fn new(cluster: Arc<Cluster>) -> Self {
- #[cfg(feature = "dev")]
- {
- let granularity = G::granularity();
- tracing::info!("waiting for {granularity} webgraph to come online...");
- cluster
- .await_member(|member| {
- if let Service::Webgraph {
- host: _,
- shard: _,
- granularity: remote_granularity,
- } = member.service
- {
- granularity == remote_granularity
- } else {
- false
- }
- })
- .await;
- }
-
Self {
client: Arc::new(Mutex::new(
- sonic::replication::ReusableShardedClient::new(cluster).await,
+ sonic::replication::ReusableShardedClient::new(cluster.clone()).await,
)),
+ cluster,
}
}
+ pub async fn await_ready(&self) {
+ let granularity = G::granularity();
+ tracing::info!("waiting for {granularity} webgraph to come online...");
+ self.cluster
+ .await_member(|member| {
+ if let Service::Webgraph {
+ host: _,
+ shard: _,
+ granularity: remote_granularity,
+ } = member.service
+ {
+ granularity == remote_granularity
+ } else {
+ false
+ }
+ })
+ .await;
+ }
+
async fn conn(&self) -> Arc<sonic::replication::ShardedClient<WebGraphService, ShardId>> {
self.client.lock().await.conn().await
}