web search done right
explicitly wait for graph nodes to come online
Mikkel Denkerf6f39b77cad6
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex b01c04a3..fcceb3a6 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -184,7 +184,7 @@ pub async fn router(.iter().any(|m| m.service.is_searcher()){- log::warn!("Waiting for search nodes to join the cluster");+ log::info!("Waiting for search nodes to join the cluster");cluster.await_member(|m| m.service.is_searcher()).await;log::info!("Search nodes joined the cluster");}diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 4a10bbec..de407448 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -166,7 +166,9 @@ impl CrawlPlanner {Self::check_config(&config)?;let page_graph = RemoteWebgraph::new(cluster.clone()).await;+ page_graph.await_ready().await;let host_graph = RemoteWebgraph::new(cluster.clone()).await;+ host_graph.await_ready().await;let domain_boosts = config.domain_boostsdiff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 979b9218..186b69e8 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -152,10 +152,14 @@ impl Webgraph {),IndexerGraphConfig::Remote { gossip } => {let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);- Self::Remote(RemoteWebgraph::new(cluster).await)+ let remote = RemoteWebgraph::new(cluster).await;+ remote.await_ready().await;+ Self::Remote(remote)}IndexerGraphConfig::Existing { cluster } => {- Self::Remote(RemoteWebgraph::new(cluster.clone()).await)+ let remote = RemoteWebgraph::new(cluster.clone()).await;+ remote.await_ready().await;+ Self::Remote(remote)}}}diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex 411529d0..a4f566fe 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -114,37 +114,38 @@ where#[derive(Clone)]pub struct RemoteWebgraph<G: WebgraphGranularity> {client: Arc<Mutex<sonic::replication::ReusableShardedClient<WebgraphClientManager<G>>>>,+ cluster: Arc<Cluster>,}impl<G: WebgraphGranularity> RemoteWebgraph<G> {pub async fn new(cluster: Arc<Cluster>) -> Self {- #[cfg(feature = "dev")]- {- let granularity = G::granularity();- tracing::info!("waiting for {granularity} webgraph to come online...");- cluster- .await_member(|member| {- if let Service::Webgraph {- host: _,- shard: _,- granularity: remote_granularity,- } = member.service- {- granularity == remote_granularity- } else {- false- }- })- .await;- }-Self {client: Arc::new(Mutex::new(- sonic::replication::ReusableShardedClient::new(cluster).await,+ sonic::replication::ReusableShardedClient::new(cluster.clone()).await,)),+ cluster,}}+ pub async fn await_ready(&self) {+ let granularity = G::granularity();+ tracing::info!("waiting for {granularity} webgraph to come online...");+ self.cluster+ .await_member(|member| {+ if let Service::Webgraph {+ host: _,+ shard: _,+ granularity: remote_granularity,+ } = member.service+ {+ granularity == remote_granularity+ } else {+ false+ }+ })+ .await;+ }+async fn conn(&self) -> Arc<sonic::replication::ShardedClient<WebGraphService, ShardId>> {self.client.lock().await.conn().await}