Skip to content

web search done right

only search single replica with retries

Mikkel Denkerbf68865f894f
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 26fb949d..13744360 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -26,6 +26,7 @@ use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration};
use futures::stream::FuturesUnordered;
use futures::StreamExt;
+use rand::seq::SliceRandom;
use serde::{Deserialize, Serialize};
use thiserror::Error;
@@ -33,6 +34,8 @@ use crate::distributed::sonic;
use super::{InitialWebsiteResult, SearchQuery};
+const NUM_REPLICA_RETRIES: usize = 3;
+
struct RemoteSearcher {
addr: SocketAddr,
}
@@ -154,21 +157,21 @@ impl Shard {
}
}
+ fn replica(&self) -> &RemoteSearcher {
+ self.replicas.choose(&mut rand::thread_rng()).unwrap()
+ }
+
async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {
- match self
- .replicas
- .iter()
- .map(|remote| remote.search(query))
- .collect::<FuturesUnordered<_>>()
- .next()
- .await
- {
- Some(result) => Ok(InitialSearchResultShard {
- local_result: result?,
- shard: self.id.clone(),
- }),
- None => Err(Error::SearchFailed.into()),
+ for _ in 0..NUM_REPLICA_RETRIES {
+ if let Ok(result) = self.replica().search(query).await {
+ return Ok(InitialSearchResultShard {
+ local_result: result,
+ shard: self.id.clone(),
+ });
+ }
}
+
+ Err(Error::SearchFailed.into())
}
async fn retrieve_websites(
@@ -176,45 +179,31 @@ impl Shard {
pointers: &[inverted_index::WebsitePointer],
original_query: &str,
) -> Result<Vec<RetrievedWebpage>> {
- match self
- .replicas
- .iter()
- .map(|remote| remote.retrieve_websites(pointers, original_query))
- .collect::<FuturesUnordered<_>>()
- .next()
- .await
- {
- Some(Ok(websites)) => Ok(websites),
- _ => Err(Error::SearchFailed.into()),
+ for _ in 0..NUM_REPLICA_RETRIES {
+ if let Ok(res) = self
+ .replica()
+ .retrieve_websites(pointers, original_query)
+ .await
+ {
+ return Ok(res);
+ }
}
+
+ Err(Error::SearchFailed.into())
}
async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- match self
- .replicas
- .iter()
- .map(|remote| remote.get_webpage(url))
- .collect::<FuturesUnordered<_>>()
- .next()
- .await
- {
- Some(t) => t,
- _ => Err(Error::WebpageNotFound.into()),
+ for _ in 0..NUM_REPLICA_RETRIES {
+ if let Ok(res) = self.replica().get_webpage(url).await {
+ return Ok(res);
+ }
}
+
+ Err(Error::SearchFailed.into())
}
async fn get_homepage_descriptions(&self, urls: &[String]) -> HashMap<Url, String> {
- match self
- .replicas
- .iter()
- .map(|remote| remote.get_homepage_descriptions(urls))
- .collect::<FuturesUnordered<_>>()
- .next()
- .await
- {
- Some(t) => t,
- _ => HashMap::new(),
- }
+ self.replica().get_homepage_descriptions(urls).await
}
}