Skip to content

web search done right

Search live index in api searcher

Mikkel Denker0a70fef7c251
diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rs
index d32a9239..28df66c4 100644
--- a/core/src/api/summarize.rs
+++ b/core/src/api/summarize.rs
@@ -47,7 +47,12 @@ async fn summarize(
params: SummarizeParams,
state: Arc<State>,
) -> Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>> {
- let webpage = state.searcher.get_webpage(&params.url).await?;
+ let webpage = state
+ .searcher
+ .get_webpage(&params.url)
+ .await?
+ .ok_or_else(|| anyhow::anyhow!("Webpage not found: {}", params.url))?;
+
let (tx, mut rx) = unbounded_channel();
let summarizer = Arc::clone(&state.summarizer);
diff --git a/core/src/distributed/sonic/mod.rs b/core/src/distributed/sonic/mod.rs
index 1519e56d..05a71ad3 100644
--- a/core/src/distributed/sonic/mod.rs
+++ b/core/src/distributed/sonic/mod.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+pub mod replication;
pub mod service;
use std::{marker::PhantomData, time::Duration};
diff --git a/core/src/distributed/sonic/replication.rs b/core/src/distributed/sonic/replication.rs
new file mode 100644
index 00000000..c1f36bbf
--- /dev/null
+++ b/core/src/distributed/sonic/replication.rs
@@ -0,0 +1,243 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use futures::future::join_all;
+use rand::seq::IteratorRandom;
+
+use super::Result;
+use crate::distributed::{retry_strategy::ExponentialBackoff, sonic};
+use std::{net::SocketAddr, time::Duration};
+
+#[derive(Debug, Clone)]
+pub struct RemoteClient<S: sonic::service::Service> {
+ addr: SocketAddr,
+ _phantom: std::marker::PhantomData<S>,
+}
+
+impl<S> RemoteClient<S>
+where
+ S: sonic::service::Service,
+{
+ pub fn new(addr: SocketAddr) -> Self {
+ Self {
+ addr,
+ _phantom: std::marker::PhantomData,
+ }
+ }
+}
+
+impl<S> RemoteClient<S>
+where
+ S: sonic::service::Service,
+{
+ async fn conn(&self) -> Result<sonic::service::ResilientConnection<S>> {
+ let retry = ExponentialBackoff::from_millis(30)
+ .with_limit(Duration::from_millis(200))
+ .take(5);
+
+ sonic::service::ResilientConnection::create_with_timeout(
+ self.addr,
+ Duration::from_secs(30),
+ retry,
+ )
+ .await
+ }
+
+ async fn send<R: sonic::service::Wrapper<S>>(&self, req: &R) -> Result<R::Response> {
+ let conn = self.conn().await?;
+ conn.send_with_timeout(req, Duration::from_secs(60)).await
+ }
+}
+
+pub trait ReplicaSelector<S: sonic::service::Service> {
+ fn select<'a>(&self, replicas: &'a [RemoteClient<S>]) -> Vec<&'a RemoteClient<S>>;
+}
+
+pub struct RandomReplicaSelector;
+
+impl<S> ReplicaSelector<S> for RandomReplicaSelector
+where
+ S: sonic::service::Service,
+{
+ fn select<'a>(&self, replicas: &'a [RemoteClient<S>]) -> Vec<&'a RemoteClient<S>> {
+ let mut rng = rand::thread_rng();
+ replicas.iter().choose_multiple(&mut rng, 1)
+ }
+}
+
+pub struct AllReplicaSelector;
+
+impl<S> ReplicaSelector<S> for AllReplicaSelector
+where
+ S: sonic::service::Service,
+{
+ fn select<'a>(&self, replicas: &'a [RemoteClient<S>]) -> Vec<&'a RemoteClient<S>> {
+ replicas.iter().collect()
+ }
+}
+
+pub struct ReplicatedClient<S: sonic::service::Service> {
+ clients: Vec<RemoteClient<S>>,
+}
+
+impl<S> ReplicatedClient<S>
+where
+ S: sonic::service::Service,
+{
+ pub fn new(clients: Vec<RemoteClient<S>>) -> Self {
+ Self { clients }
+ }
+
+ pub async fn send<Req, Rep>(&self, req: &Req, selector: &Rep) -> Result<Vec<Req::Response>>
+ where
+ Req: sonic::service::Wrapper<S>,
+ Rep: ReplicaSelector<S>,
+ {
+ let mut futures = Vec::new();
+ for client in selector.select(&self.clients) {
+ futures.push(client.send(req));
+ }
+
+ let mut results = Vec::new();
+ for r in join_all(futures).await {
+ match r {
+ Ok(r) => results.push(r),
+ Err(e) => {
+ tracing::error!("Failed to send request: {:?}", e);
+ }
+ }
+ }
+
+ Ok(results)
+ }
+}
+
+pub trait ShardIdentifier: PartialEq + Eq + Clone {}
+
+pub trait ShardSelector<S: sonic::service::Service, Id: ShardIdentifier> {
+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>>;
+}
+
+pub struct AllShardsSelector;
+
+impl<S, Id> ShardSelector<S, Id> for AllShardsSelector
+where
+ S: sonic::service::Service,
+ Id: ShardIdentifier,
+{
+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>> {
+ shards.iter().collect()
+ }
+}
+
+pub struct RandomShardSelector;
+
+impl<S, Id> ShardSelector<S, Id> for RandomShardSelector
+where
+ S: sonic::service::Service,
+ Id: ShardIdentifier,
+{
+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>> {
+ let mut rng = rand::thread_rng();
+ shards.iter().choose_multiple(&mut rng, 1)
+ }
+}
+
+pub struct SpecificShardSelector<Id: ShardIdentifier>(pub Id);
+
+impl<S, Id> ShardSelector<S, Id> for SpecificShardSelector<Id>
+where
+ S: sonic::service::Service,
+ Id: ShardIdentifier,
+{
+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>> {
+ shards.iter().find(|s| s.id == self.0).into_iter().collect()
+ }
+}
+
+pub struct Shard<S: sonic::service::Service, Id: ShardIdentifier> {
+ replicas: ReplicatedClient<S>,
+ id: Id,
+}
+
+impl<S, Id> Shard<S, Id>
+where
+ S: sonic::service::Service,
+ Id: ShardIdentifier,
+{
+ pub fn new(id: Id, replicas: ReplicatedClient<S>) -> Self {
+ Self { replicas, id }
+ }
+}
+
+pub struct ShardedClient<S: sonic::service::Service, Id: ShardIdentifier> {
+ shards: Vec<Shard<S, Id>>,
+}
+
+impl<S, Id> ShardedClient<S, Id>
+where
+ S: sonic::service::Service,
+ Id: ShardIdentifier,
+{
+ pub fn new(shards: Vec<Shard<S, Id>>) -> Self {
+ Self { shards }
+ }
+
+ async fn send_single<Req, RSel>(
+ &self,
+ req: &Req,
+ shard: &Shard<S, Id>,
+ replica_selector: &RSel,
+ ) -> Result<(Id, Vec<Req::Response>)>
+ where
+ Req: sonic::service::Wrapper<S>,
+ RSel: ReplicaSelector<S>,
+ {
+ Ok((
+ shard.id.clone(),
+ shard.replicas.send(req, replica_selector).await?,
+ ))
+ }
+
+ pub async fn send<Req, SSel, RSel>(
+ &self,
+ req: &Req,
+ shard_selector: &SSel,
+ replica_selector: &RSel,
+ ) -> Result<Vec<(Id, Vec<Req::Response>)>>
+ where
+ Req: sonic::service::Wrapper<S>,
+ SSel: ShardSelector<S, Id>,
+ RSel: ReplicaSelector<S>,
+ {
+ let mut futures = Vec::new();
+ for shard in shard_selector.select(&self.shards) {
+ futures.push(self.send_single(req, shard, replica_selector));
+ }
+
+ let mut results = Vec::new();
+ for r in join_all(futures).await {
+ match r {
+ Ok(r) => results.push(r),
+ Err(e) => {
+ tracing::error!("Failed to send request: {:?}", e);
+ }
+ }
+ }
+
+ Ok(results)
+ }
+}
diff --git a/core/src/distributed/sonic/service.rs b/core/src/distributed/sonic/service.rs
index 15fa1d4e..66c5862b 100644
--- a/core/src/distributed/sonic/service.rs
+++ b/core/src/distributed/sonic/service.rs
@@ -1,3 +1,19 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
use std::{net::SocketAddr, sync::Arc, time::Duration};
use tokio::net::ToSocketAddrs;
diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rs
index 8a60cbbd..6100da9a 100644
--- a/core/src/feed/scheduler.rs
+++ b/core/src/feed/scheduler.rs
@@ -49,6 +49,12 @@ impl From<&Url> for Domain {
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]
pub struct SplitId(uuid::Uuid);
+impl SplitId {
+ pub fn id(&self) -> uuid::Uuid {
+ self.0
+ }
+}
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct Split {
pub id: SplitId,
diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rs
index 948d7b50..5c13cafa 100644
--- a/core/src/searcher/api.rs
+++ b/core/src/searcher/api.rs
@@ -30,6 +30,7 @@ use crate::image_store::Image;
use crate::inverted_index::RetrievedWebpage;
#[cfg(not(feature = "libtorch"))]
use crate::ranking::models::cross_encoder::DummyCrossEncoder;
+use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite};
use crate::ranking::ALL_SIGNALS;
use crate::search_prettifier::{
create_stackoverflow_sidebar, DisplayedAnswer, DisplayedEntity, DisplayedSidebar,
@@ -46,14 +47,35 @@ use crate::{ceil_char_boundary, floor_char_boundary, query, Result};
#[cfg(feature = "libtorch")]
use crate::{qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel};
-use super::{
- distributed, DistributedSearcher, InitialSearchResultShard, ScoredWebsitePointer, SearchQuery,
- SearchResult, WebsitesResult,
-};
+use super::live::LiveSearcher;
+use super::{distributed, live, DistributedSearcher, SearchQuery, SearchResult, WebsitesResult};
+
+#[derive(Clone)]
+enum ScoredWebsitePointer {
+ Normal(distributed::ScoredWebsitePointer),
+ Live(live::ScoredWebsitePointer),
+}
+
+impl AsRankingWebsite for ScoredWebsitePointer {
+ fn as_ranking(&self) -> &RankingWebsite {
+ match self {
+ ScoredWebsitePointer::Normal(p) => &p.website,
+ ScoredWebsitePointer::Live(p) => &p.website,
+ }
+ }
+
+ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {
+ match self {
+ ScoredWebsitePointer::Normal(p) => &mut p.website,
+ ScoredWebsitePointer::Live(p) => &mut p.website,
+ }
+ }
+}
#[cfg(feature = "libtorch")]
pub struct ApiSearcher {
distributed_searcher: DistributedSearcher,
+ live_searcher: LiveSearcher,
cross_encoder: Option<Arc<CrossEncoderModel>>,
lambda_model: Option<Arc<LambdaMART>>,
qa_model: Option<Arc<QaModel>>,
@@ -84,7 +106,8 @@ impl ApiSearcher {
config: ApiConfig,
) -> Self {
Self {
- distributed_searcher: DistributedSearcher::new(cluster),
+ distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),
+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),
cross_encoder: cross_encoder.map(Arc::new),
lambda_model: lambda_model.map(Arc::new),
qa_model: qa_model.map(Arc::new),
@@ -114,7 +137,8 @@ impl ApiSearcher {
fn combine_results(
&self,
- initial_results: Vec<InitialSearchResultShard>,
+ initial_results: Vec<distributed::InitialSearchResultShard>,
+ live_results: Vec<live::InitialSearchResultSplit>,
pipeline: RankingPipeline<ScoredWebsitePointer>,
) -> (Vec<ScoredWebsitePointer>, bool) {
let mut collector =
@@ -124,11 +148,27 @@ impl ApiSearcher {
for result in initial_results {
for website in result.local_result.websites {
num_sites += 1;
- let pointer = ScoredWebsitePointer {
+ let pointer = distributed::ScoredWebsitePointer {
website,
- shard: result.shard.clone(),
+ shard: result.shard,
};
+ let pointer = ScoredWebsitePointer::Normal(pointer);
+
+ collector.insert(pointer);
+ }
+ }
+
+ for result in live_results {
+ for website in result.local_result.websites {
+ num_sites += 1;
+ let pointer = live::ScoredWebsitePointer {
+ website,
+ split_id: result.split_id.clone(),
+ };
+
+ let pointer = ScoredWebsitePointer::Live(pointer);
+
collector.insert(pointer);
}
}
@@ -174,13 +214,14 @@ impl ApiSearcher {
if let Some((shard, website)) = results.pop() {
if website.score > self.thresholds.stackoverflow {
- let scored_websites = vec![ScoredWebsitePointer { website, shard }];
+ let scored_websites =
+ vec![(0, distributed::ScoredWebsitePointer { website, shard })];
let mut retrieved = self
.distributed_searcher
.retrieve_webpages(&scored_websites, &query.query)
.await;
- if let Some(res) = retrieved.pop() {
+ if let Some((_, res)) = retrieved.pop() {
return Ok(Some(create_stackoverflow_sidebar(
res.schema_org,
Url::parse(&res.url).unwrap(),
@@ -194,7 +235,7 @@ impl ApiSearcher {
async fn sidebar(
&self,
- initial_results: &[InitialSearchResultShard],
+ initial_results: &[distributed::InitialSearchResultShard],
query: &SearchQuery,
) -> Result<Option<DisplayedSidebar>> {
let entity = initial_results
@@ -303,11 +344,11 @@ impl ApiSearcher {
return Ok(None);
}
- let (top_websites, _) = self.combine_results(initial_results, pipeline);
+ let (top_websites, _) = self.combine_results(initial_results, vec![], pipeline);
let scores: Vec<_> = top_websites
.iter()
- .map(|pointer| pointer.website.score)
+ .map(|pointer| pointer.as_ranking().score)
.collect();
let median = if scores.len() % 2 == 0 {
@@ -322,9 +363,24 @@ impl ApiSearcher {
let mut result: Vec<DisplayedWebpage> = self
.distributed_searcher
- .retrieve_webpages(&top_websites, &query.query)
+ .retrieve_webpages(
+ &top_websites
+ .clone()
+ .into_iter()
+ .filter_map(|pointer| {
+ if let ScoredWebsitePointer::Normal(p) = pointer {
+ Some(p)
+ } else {
+ None
+ }
+ })
+ .enumerate()
+ .collect_vec(),
+ &query.query,
+ )
.await
.into_iter()
+ .map(|(_, webpage)| webpage)
.map(DisplayedWebpage::from)
.collect();
@@ -344,7 +400,7 @@ impl ApiSearcher {
let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());
for signal in ALL_SIGNALS {
- if let Some(signal_value) = pointer.website.signals.get(signal) {
+ if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {
signals.insert(signal, *signal_value);
}
}
@@ -378,17 +434,16 @@ impl ApiSearcher {
self.collector_config.clone(),
)?;
- let initial_results = self
- .distributed_searcher
- .search_initial(&search_query)
- .await;
+ let (initial_results, live_results, widget, discussions) = tokio::join!(
+ self.distributed_searcher.search_initial(&search_query),
+ self.live_searcher.search_initial(&search_query),
+ self.widget(query),
+ self.discussions_widget(query)
+ );
+ let discussions = discussions?;
let sidebar = self.sidebar(&initial_results, query).await?;
- let discussions = self.discussions_widget(query).await?;
-
- let widget = self.widget(query);
-
let spell_corrected_query = if widget.is_none() {
initial_results
.first()
@@ -403,14 +458,51 @@ impl ApiSearcher {
.map(|result| result.local_result.num_websites)
.sum();
- let (top_websites, has_more_results) = self.combine_results(initial_results, pipeline);
+ let (top_websites, has_more_results) =
+ self.combine_results(initial_results, live_results, pipeline);
// retrieve webpages
- let mut retrieved_webpages: Vec<_> = self
+ let normal: Vec<_> = top_websites
+ .iter()
+ .enumerate()
+ .filter_map(|(i, pointer)| {
+ if let ScoredWebsitePointer::Normal(p) = pointer {
+ Some((i, p.clone()))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let live: Vec<_> = top_websites
+ .iter()
+ .enumerate()
+ .filter_map(|(i, pointer)| {
+ if let ScoredWebsitePointer::Live(p) = pointer {
+ Some((i, p.clone()))
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let retrieved_normal: Vec<_> = self
.distributed_searcher
- .retrieve_webpages(&top_websites, &query.query)
- .await
+ .retrieve_webpages(&normal, &query.query)
+ .await;
+
+ let retrieved_live: Vec<_> = self
+ .live_searcher
+ .retrieve_webpages(&live, &query.query)
+ .await;
+
+ let mut retrieved_webpages: Vec<_> =
+ retrieved_normal.into_iter().chain(retrieved_live).collect();
+ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+ let mut retrieved_webpages: Vec<_> = retrieved_webpages
.into_iter()
+ .map(|(_, webpage)| webpage)
.map(DisplayedWebpage::from)
.collect();
@@ -447,7 +539,7 @@ impl ApiSearcher {
Ok(SearchResult::Websites(self.search_websites(query).await?))
}
- fn widget(&self, query: &SearchQuery) -> Option<Widget> {
+ async fn widget(&self, query: &SearchQuery) -> Option<Widget> {
if query.page > 0 {
return None;
}
@@ -508,7 +600,7 @@ impl ApiSearcher {
None
}
- pub async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {
+ pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
self.distributed_searcher.get_webpage(url).await
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 6199f07a..eb107a16 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -15,33 +15,31 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff},
+ distributed::{
+ cluster::Cluster,
+ member::Service,
+ sonic::replication::{
+ AllShardsSelector, RandomReplicaSelector, RandomShardSelector, RemoteClient,
+ ReplicatedClient, Shard, ShardIdentifier, ShardedClient, SpecificShardSelector,
+ },
+ },
entrypoint::search_server::{self, SearchService},
image_store::Image,
- inverted_index::{self, RetrievedWebpage},
- ranking::pipeline::{AsRankingWebsite, RankingWebsite},
+ inverted_index::{RetrievedWebpage, WebsitePointer},
+ ranking::pipeline::RankingWebsite,
Result,
};
-use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc};
-use futures::stream::FuturesUnordered;
-use futures::StreamExt;
-use rand::seq::SliceRandom;
+use futures::future::join_all;
+use itertools::Itertools;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use url::Url;
-use crate::distributed::sonic;
-
use super::{InitialWebsiteResult, SearchQuery};
-const NUM_REPLICA_RETRIES: usize = 3;
-
-struct RemoteSearcher {
- addr: SocketAddr,
-}
-
#[derive(Error, Debug)]
pub enum Error {
#[error("Failed to get search result")]
@@ -54,188 +52,16 @@ pub enum Error {
WebpageNotFound,
}
-impl RemoteSearcher {
- async fn search(&self, query: &SearchQuery) -> Result<InitialWebsiteResult> {
- let conn = self.conn().await;
-
- if let Ok(Some(body)) = conn
- .send_with_timeout(
- &search_server::Search {
- query: query.clone(),
- },
- Duration::from_secs(60),
- )
- .await
- {
- return Ok(body);
- }
-
- Err(Error::SearchFailed.into())
- }
-
- async fn retrieve_websites(
- &self,
- pointers: &[inverted_index::WebsitePointer],
- original_query: &str,
- ) -> Result<Vec<RetrievedWebpage>> {
- let conn = self.conn().await;
-
- if let Ok(Some(body)) = conn
- .send_with_timeout(
- &search_server::RetrieveWebsites {
- websites: pointers.to_vec(),
- query: original_query.to_string(),
- },
- Duration::from_secs(60),
- )
- .await
- {
- return Ok(body);
- }
- Err(Error::SearchFailed.into())
- }
-
- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- let conn = self.conn().await;
-
- if let Ok(body) = conn
- .send_with_timeout(
- &search_server::GetWebpage {
- url: url.to_string(),
- },
- Duration::from_secs(60),
- )
- .await
- {
- return Ok(body);
- }
-
- Err(Error::WebpageNotFound.into())
- }
-
- async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
- let conn = self.conn().await;
-
- if let Ok(body) = conn
- .send_with_timeout(
- &search_server::GetHomepageDescriptions {
- urls: urls.to_vec(),
- },
- Duration::from_secs(60),
- )
- .await
- {
- return body;
- }
-
- HashMap::new()
- }
-
- async fn conn(&self) -> sonic::service::ResilientConnection<SearchService> {
- let retry = ExponentialBackoff::from_millis(30)
- .with_limit(Duration::from_millis(200))
- .take(5);
-
- sonic::service::ResilientConnection::create_with_timeout(
- self.addr,
- Duration::from_secs(30),
- retry,
- )
- .await
- .unwrap()
- }
-
- async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
- let conn = self.conn().await;
-
- conn.send_with_timeout(
- &search_server::GetEntityImage {
- image_id: image_id.to_string(),
- },
- Duration::from_secs(60),
- )
- .await
- .map_err(|e| e.into())
- }
+#[derive(Clone, Debug)]
+pub struct ScoredWebsitePointer {
+ pub website: RankingWebsite,
+ pub shard: ShardId,
}
-#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]
+#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]
pub struct ShardId(u64);
-pub struct Shard {
- id: ShardId,
- replicas: Vec<RemoteSearcher>,
-}
-
-impl Shard {
- pub fn new(id: u64, replicas: Vec<String>) -> Self {
- let mut parsed_replicas = Vec::new();
-
- for replica in replicas {
- parsed_replicas.push(RemoteSearcher {
- addr: replica.parse().unwrap(),
- });
- }
-
- Self {
- id: ShardId(id),
- replicas: parsed_replicas,
- }
- }
-
- fn replica(&self) -> &RemoteSearcher {
- self.replicas.choose(&mut rand::thread_rng()).unwrap()
- }
-
- async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {
- for _ in 0..NUM_REPLICA_RETRIES {
- if let Ok(result) = self.replica().search(query).await {
- return Ok(InitialSearchResultShard {
- local_result: result,
- shard: self.id.clone(),
- });
- }
- }
-
- Err(Error::SearchFailed.into())
- }
-
- async fn retrieve_websites(
- &self,
- pointers: &[inverted_index::WebsitePointer],
- original_query: &str,
- ) -> Result<Vec<RetrievedWebpage>> {
- for _ in 0..NUM_REPLICA_RETRIES {
- if let Ok(res) = self
- .replica()
- .retrieve_websites(pointers, original_query)
- .await
- {
- return Ok(res);
- }
- }
-
- Err(Error::SearchFailed.into())
- }
-
- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- for _ in 0..NUM_REPLICA_RETRIES {
- if let Ok(res) = self.replica().get_webpage(url).await {
- return Ok(res);
- }
- }
-
- Err(Error::SearchFailed.into())
- }
-
- async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
- self.replica().get_homepage_descriptions(urls).await
- }
-
- async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
- self.replica().get_entity_image(image_id).await
- }
-}
+impl ShardIdentifier for ShardId {}
#[derive(Debug)]
pub struct InitialSearchResultShard {
@@ -243,22 +69,6 @@ pub struct InitialSearchResultShard {
pub shard: ShardId,
}
-#[derive(Clone, Debug)]
-pub struct ScoredWebsitePointer {
- pub website: RankingWebsite,
- pub shard: ShardId,
-}
-
-impl AsRankingWebsite for ScoredWebsitePointer {
- fn as_ranking(&self) -> &RankingWebsite {
- &self.website
- }
-
- fn as_mut_ranking(&mut self) -> &mut RankingWebsite {
- &mut self.website
- }
-}
-
pub struct DistributedSearcher {
cluster: Arc<Cluster>,
}
@@ -268,7 +78,7 @@ impl DistributedSearcher {
Self { cluster }
}
- async fn shards(&self) -> Vec<Shard> {
+ async fn client(&self) -> ShardedClient<SearchService, ShardId> {
let mut shards = HashMap::new();
for member in self.cluster.members().await {
if let Service::Searcher { host, shard } = member.service {
@@ -276,97 +86,165 @@ impl DistributedSearcher {
}
}
- shards
- .into_iter()
- .map(|(shard, replicas)| Shard {
- id: shard,
- replicas: replicas
- .into_iter()
- .map(|addr| RemoteSearcher { addr })
- .collect(),
- })
- .collect()
+ let mut shard_clients = Vec::new();
+
+ for (id, replicas) in shards {
+ let replicated =
+ ReplicatedClient::new(replicas.into_iter().map(RemoteClient::new).collect());
+ let shard = Shard::new(id, replicated);
+ shard_clients.push(shard);
+ }
+
+ ShardedClient::new(shard_clients)
}
- pub async fn retrieve_webpages(
+ pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {
+ let client = self.client().await;
+ let mut results = Vec::new();
+
+ if let Ok(res) = client
+ .send(
+ &search_server::Search {
+ query: query.clone(),
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ {
+ for (shard_id, mut res) in res {
+ if let Some(Some(res)) = res.pop() {
+ results.push(InitialSearchResultShard {
+ local_result: res,
+ shard: shard_id,
+ });
+ }
+ }
+ }
+
+ results
+ }
+
+ async fn retrieve_webpages_from_shard(
&self,
- top_websites: &[ScoredWebsitePointer],
+ shard: ShardId,
+ client: &ShardedClient<SearchService, ShardId>,
query: &str,
- ) -> Vec<RetrievedWebpage> {
- let mut retrieved_webpages = Vec::new();
+ pointers: Vec<(usize, WebsitePointer)>,
+ ) -> Vec<(usize, RetrievedWebpage)> {
+ let (idxs, pointers): (Vec<usize>, Vec<WebsitePointer>) = pointers.into_iter().unzip();
- for _ in 0..top_websites.len() {
- retrieved_webpages.push(None);
+ match client
+ .send(
+ &search_server::RetrieveWebsites {
+ websites: pointers,
+ query: query.to_string(),
+ },
+ &SpecificShardSelector(shard),
+ &RandomReplicaSelector,
+ )
+ .await
+ {
+ Ok(v) => v
+ .into_iter()
+ .flat_map(|(_, v)| v)
+ .flatten()
+ .flatten()
+ .zip_eq(idxs)
+ .map(|(v, i)| (i, v))
+ .collect(),
+ _ => vec![],
}
+ }
- for shard in self.shards().await.iter() {
- let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites
- .iter()
- .enumerate()
- .filter(|(_, pointer)| pointer.shard == shard.id)
- .map(|(idx, pointer)| (idx, pointer.website.pointer.clone()))
- .unzip();
-
- if let Ok(websites) = shard.retrieve_websites(&pointers, query).await {
- for (index, website) in indexes.into_iter().zip(websites) {
- retrieved_webpages[index] = Some(website);
- }
- }
+ pub async fn retrieve_webpages(
+ &self,
+ top_websites: &[(usize, ScoredWebsitePointer)],
+ query: &str,
+ ) -> Vec<(usize, RetrievedWebpage)> {
+ let mut pointers: HashMap<_, Vec<_>> = HashMap::new();
+
+ for (i, pointer) in top_websites {
+ pointers
+ .entry(pointer.shard)
+ .or_default()
+ .push((*i, pointer.website.pointer.clone()));
+ }
+
+ let client = self.client().await;
+ let mut futures = Vec::new();
+ for (shard, pointers) in pointers {
+ futures.push(self.retrieve_webpages_from_shard(shard, &client, query, pointers));
}
- let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();
+ let mut retrieved_webpages = Vec::new();
+ for pages in join_all(futures).await {
+ retrieved_webpages.extend(pages);
+ }
debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
+ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));
+
retrieved_webpages
}
- pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {
- self.shards()
- .await
- .iter()
- .map(|shard| shard.search(query))
- .collect::<FuturesUnordered<_>>()
- .collect::<Vec<_>>()
- .await
- .into_iter()
- .filter_map(|result| result.ok())
- .collect::<Vec<_>>()
- }
+ pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
+ let client = self.client().await;
- pub async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {
- self.shards()
- .await
- .iter()
- .map(|shard| shard.get_webpage(url))
- .collect::<FuturesUnordered<_>>()
- .collect::<Vec<_>>()
+ let res = client
+ .send(
+ &search_server::GetWebpage {
+ url: url.to_string(),
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
.await
- .into_iter()
- .filter_map(|result| result.ok())
- .flatten()
- .collect::<Vec<_>>()
- .pop()
- .ok_or(Error::WebpageNotFound.into())
+ .map_err(|_| Error::SearchFailed)?;
+
+ if let Some(Some(res)) = res.into_iter().flat_map(|(_, v)| v).next() {
+ Ok(Some(res))
+ } else {
+ Err(Error::WebpageNotFound.into())
+ }
}
pub async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
- self.shards()
- .await
- .iter()
- .map(|shard| shard.get_homepage_descriptions(urls))
- .collect::<FuturesUnordered<_>>()
- .collect::<Vec<_>>()
- .await
- .into_iter()
- .flatten()
- .collect()
+ let client = self.client().await;
+
+ let res = client
+ .send(
+ &search_server::GetHomepageDescriptions {
+ urls: urls.to_vec(),
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await;
+
+ match res {
+ Ok(v) => v.into_iter().flat_map(|(_, v)| v).flatten().collect(),
+ _ => HashMap::new(),
+ }
}
pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {
- match self.shards().await.first() {
- Some(s) => s.get_entity_image(image_id).await,
- None => Ok(None),
- }
+ let client = self.client().await;
+
+ let (_, mut res) = client
+ .send(
+ &search_server::GetEntityImage {
+ image_id: image_id.to_string(),
+ },
+ &RandomShardSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ .map_err(|_| Error::SearchFailed)?
+ .pop()
+ .unwrap();
+
+ Ok(res.pop().flatten())
}
}
diff --git a/core/src/searcher/live.rs b/core/src/searcher/live.rs
new file mode 100644
index 00000000..6593d10b
--- /dev/null
+++ b/core/src/searcher/live.rs
@@ -0,0 +1,172 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ distributed::{
+ cluster::Cluster,
+ member::Service,
+ sonic::replication::{
+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,
+ ShardIdentifier, ShardedClient, SpecificShardSelector,
+ },
+ },
+ entrypoint::search_server::{self, SearchService},
+ feed::scheduler::SplitId,
+ inverted_index::{RetrievedWebpage, WebsitePointer},
+ ranking::pipeline::RankingWebsite,
+};
+
+use std::{collections::HashMap, sync::Arc};
+
+use futures::future::join_all;
+use itertools::Itertools;
+
+use super::{InitialWebsiteResult, SearchQuery};
+
+#[derive(Clone, Debug)]
+pub struct ScoredWebsitePointer {
+ pub website: RankingWebsite,
+ pub split_id: SplitId,
+}
+
+impl ShardIdentifier for SplitId {}
+
+#[derive(Debug)]
+pub struct InitialSearchResultSplit {
+ pub local_result: InitialWebsiteResult,
+ pub split_id: SplitId,
+}
+
+pub struct LiveSearcher {
+ cluster: Arc<Cluster>,
+}
+
+impl LiveSearcher {
+ pub fn new(cluster: Arc<Cluster>) -> Self {
+ Self { cluster }
+ }
+
+ async fn client(&self) -> ShardedClient<SearchService, SplitId> {
+ let mut shards = HashMap::new();
+ for member in self.cluster.members().await {
+ if let Service::LiveIndex { host, split_id } = member.service {
+ shards.entry(split_id).or_insert_with(Vec::new).push(host);
+ }
+ }
+
+ let mut shard_clients = Vec::new();
+
+ for (id, replicas) in shards {
+ let replicated =
+ ReplicatedClient::new(replicas.into_iter().map(RemoteClient::new).collect());
+ let shard = Shard::new(id, replicated);
+ shard_clients.push(shard);
+ }
+
+ ShardedClient::new(shard_clients)
+ }
+
+ pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultSplit> {
+ let client = self.client().await;
+ let mut results = Vec::new();
+
+ if let Ok(res) = client
+ .send(
+ &search_server::Search {
+ query: query.clone(),
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ {
+ for (shard_id, mut res) in res {
+ if let Some(Some(res)) = res.pop() {
+ results.push(InitialSearchResultSplit {
+ local_result: res,
+ split_id: shard_id,
+ });
+ }
+ }
+ }
+
+ results
+ }
+
+ async fn retrieve_webpages_from_shard(
+ &self,
+ split: SplitId,
+ client: &ShardedClient<SearchService, SplitId>,
+ query: &str,
+ pointers: Vec<(usize, WebsitePointer)>,
+ ) -> Vec<(usize, RetrievedWebpage)> {
+ let (idxs, pointers): (Vec<usize>, Vec<WebsitePointer>) = pointers.into_iter().unzip();
+
+ match client
+ .send(
+ &search_server::RetrieveWebsites {
+ websites: pointers,
+ query: query.to_string(),
+ },
+ &SpecificShardSelector(split),
+ &RandomReplicaSelector,
+ )
+ .await
+ {
+ Ok(v) => v
+ .into_iter()
+ .flat_map(|(_, v)| v)
+ .flatten()
+ .flatten()
+ .zip_eq(idxs)
+ .map(|(v, i)| (i, v))
+ .collect(),
+ _ => vec![],
+ }
+ }
+
+ pub async fn retrieve_webpages(
+ &self,
+ top_websites: &[(usize, ScoredWebsitePointer)],
+ query: &str,
+ ) -> Vec<(usize, RetrievedWebpage)> {
+ let mut pointers: HashMap<_, Vec<_>> = HashMap::new();
+
+ for (i, pointer) in top_websites {
+ pointers
+ .entry(pointer.split_id.clone())
+ .or_default()
+ .push((*i, pointer.website.pointer.clone()));
+ }
+
+ let client = self.client().await;
+ let mut futures = Vec::new();
+ for (split_id, pointers) in pointers {
+ futures.push(self.retrieve_webpages_from_shard(split_id, &client, query, pointers));
+ }
+
+ let mut retrieved_webpages = Vec::new();
+ for pages in join_all(futures).await {
+ retrieved_webpages.extend(pages);
+ }
+
+ debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
+
+ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));
+
+ retrieved_webpages
+ }
+}
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 592d2552..2ed4d250 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -16,6 +16,7 @@
pub mod api;
pub mod distributed;
+mod live;
pub mod local;
pub use distributed::*;