web search done right
Search live index in api searcher
Mikkel Denker0a70fef7c251
diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rsindex d32a9239..28df66c4 100644--- a/core/src/api/summarize.rs+++ b/core/src/api/summarize.rs@@ -47,7 +47,12 @@ async fn summarize(params: SummarizeParams,state: Arc<State>,) -> Result<Sse<impl Stream<Item = std::result::Result<Event, Infallible>>>> {- let webpage = state.searcher.get_webpage(¶ms.url).await?;+ let webpage = state+ .searcher+ .get_webpage(¶ms.url)+ .await?+ .ok_or_else(|| anyhow::anyhow!("Webpage not found: {}", params.url))?;+let (tx, mut rx) = unbounded_channel();let summarizer = Arc::clone(&state.summarizer);diff --git a/core/src/distributed/sonic/mod.rs b/core/src/distributed/sonic/mod.rsindex 1519e56d..05a71ad3 100644--- a/core/src/distributed/sonic/mod.rs+++ b/core/src/distributed/sonic/mod.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod replication;pub mod service;use std::{marker::PhantomData, time::Duration};diff --git a/core/src/distributed/sonic/replication.rs b/core/src/distributed/sonic/replication.rsnew file mode 100644index 00000000..c1f36bbf--- /dev/null+++ b/core/src/distributed/sonic/replication.rs@@ -0,0 +1,243 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use futures::future::join_all;+use rand::seq::IteratorRandom;++use super::Result;+use crate::distributed::{retry_strategy::ExponentialBackoff, sonic};+use std::{net::SocketAddr, time::Duration};++#[derive(Debug, Clone)]+pub struct RemoteClient<S: sonic::service::Service> {+ addr: SocketAddr,+ _phantom: std::marker::PhantomData<S>,+}++impl<S> RemoteClient<S>+where+ S: sonic::service::Service,+{+ pub fn new(addr: SocketAddr) -> Self {+ Self {+ addr,+ _phantom: std::marker::PhantomData,+ }+ }+}++impl<S> RemoteClient<S>+where+ S: sonic::service::Service,+{+ async fn conn(&self) -> Result<sonic::service::ResilientConnection<S>> {+ let retry = ExponentialBackoff::from_millis(30)+ .with_limit(Duration::from_millis(200))+ .take(5);++ sonic::service::ResilientConnection::create_with_timeout(+ self.addr,+ Duration::from_secs(30),+ retry,+ )+ .await+ }++ async fn send<R: sonic::service::Wrapper<S>>(&self, req: &R) -> Result<R::Response> {+ let conn = self.conn().await?;+ conn.send_with_timeout(req, Duration::from_secs(60)).await+ }+}++pub trait ReplicaSelector<S: sonic::service::Service> {+ fn select<'a>(&self, replicas: &'a [RemoteClient<S>]) -> Vec<&'a RemoteClient<S>>;+}++pub struct RandomReplicaSelector;++impl<S> ReplicaSelector<S> for RandomReplicaSelector+where+ S: sonic::service::Service,+{+ fn select<'a>(&self, replicas: &'a [RemoteClient<S>]) -> Vec<&'a RemoteClient<S>> {+ let mut rng = rand::thread_rng();+ replicas.iter().choose_multiple(&mut rng, 1)+ }+}++pub struct AllReplicaSelector;++impl<S> ReplicaSelector<S> for AllReplicaSelector+where+ S: sonic::service::Service,+{+ fn select<'a>(&self, replicas: &'a [RemoteClient<S>]) -> Vec<&'a RemoteClient<S>> {+ replicas.iter().collect()+ }+}++pub struct ReplicatedClient<S: sonic::service::Service> {+ clients: Vec<RemoteClient<S>>,+}++impl<S> ReplicatedClient<S>+where+ S: sonic::service::Service,+{+ pub fn new(clients: Vec<RemoteClient<S>>) -> Self {+ Self { clients }+ }++ pub async fn send<Req, Rep>(&self, req: &Req, selector: &Rep) -> Result<Vec<Req::Response>>+ where+ Req: sonic::service::Wrapper<S>,+ Rep: ReplicaSelector<S>,+ {+ let mut futures = Vec::new();+ for client in selector.select(&self.clients) {+ futures.push(client.send(req));+ }++ let mut results = Vec::new();+ for r in join_all(futures).await {+ match r {+ Ok(r) => results.push(r),+ Err(e) => {+ tracing::error!("Failed to send request: {:?}", e);+ }+ }+ }++ Ok(results)+ }+}++pub trait ShardIdentifier: PartialEq + Eq + Clone {}++pub trait ShardSelector<S: sonic::service::Service, Id: ShardIdentifier> {+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>>;+}++pub struct AllShardsSelector;++impl<S, Id> ShardSelector<S, Id> for AllShardsSelector+where+ S: sonic::service::Service,+ Id: ShardIdentifier,+{+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>> {+ shards.iter().collect()+ }+}++pub struct RandomShardSelector;++impl<S, Id> ShardSelector<S, Id> for RandomShardSelector+where+ S: sonic::service::Service,+ Id: ShardIdentifier,+{+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>> {+ let mut rng = rand::thread_rng();+ shards.iter().choose_multiple(&mut rng, 1)+ }+}++pub struct SpecificShardSelector<Id: ShardIdentifier>(pub Id);++impl<S, Id> ShardSelector<S, Id> for SpecificShardSelector<Id>+where+ S: sonic::service::Service,+ Id: ShardIdentifier,+{+ fn select<'a>(&self, shards: &'a [Shard<S, Id>]) -> Vec<&'a Shard<S, Id>> {+ shards.iter().find(|s| s.id == self.0).into_iter().collect()+ }+}++pub struct Shard<S: sonic::service::Service, Id: ShardIdentifier> {+ replicas: ReplicatedClient<S>,+ id: Id,+}++impl<S, Id> Shard<S, Id>+where+ S: sonic::service::Service,+ Id: ShardIdentifier,+{+ pub fn new(id: Id, replicas: ReplicatedClient<S>) -> Self {+ Self { replicas, id }+ }+}++pub struct ShardedClient<S: sonic::service::Service, Id: ShardIdentifier> {+ shards: Vec<Shard<S, Id>>,+}++impl<S, Id> ShardedClient<S, Id>+where+ S: sonic::service::Service,+ Id: ShardIdentifier,+{+ pub fn new(shards: Vec<Shard<S, Id>>) -> Self {+ Self { shards }+ }++ async fn send_single<Req, RSel>(+ &self,+ req: &Req,+ shard: &Shard<S, Id>,+ replica_selector: &RSel,+ ) -> Result<(Id, Vec<Req::Response>)>+ where+ Req: sonic::service::Wrapper<S>,+ RSel: ReplicaSelector<S>,+ {+ Ok((+ shard.id.clone(),+ shard.replicas.send(req, replica_selector).await?,+ ))+ }++ pub async fn send<Req, SSel, RSel>(+ &self,+ req: &Req,+ shard_selector: &SSel,+ replica_selector: &RSel,+ ) -> Result<Vec<(Id, Vec<Req::Response>)>>+ where+ Req: sonic::service::Wrapper<S>,+ SSel: ShardSelector<S, Id>,+ RSel: ReplicaSelector<S>,+ {+ let mut futures = Vec::new();+ for shard in shard_selector.select(&self.shards) {+ futures.push(self.send_single(req, shard, replica_selector));+ }++ let mut results = Vec::new();+ for r in join_all(futures).await {+ match r {+ Ok(r) => results.push(r),+ Err(e) => {+ tracing::error!("Failed to send request: {:?}", e);+ }+ }+ }++ Ok(results)+ }+}diff --git a/core/src/distributed/sonic/service.rs b/core/src/distributed/sonic/service.rsindex 15fa1d4e..66c5862b 100644--- a/core/src/distributed/sonic/service.rs+++ b/core/src/distributed/sonic/service.rs@@ -1,3 +1,19 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use std::{net::SocketAddr, sync::Arc, time::Duration};use tokio::net::ToSocketAddrs;diff --git a/core/src/feed/scheduler.rs b/core/src/feed/scheduler.rsindex 8a60cbbd..6100da9a 100644--- a/core/src/feed/scheduler.rs+++ b/core/src/feed/scheduler.rs@@ -49,6 +49,12 @@ impl From<&Url> for Domain {#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub struct SplitId(uuid::Uuid);+impl SplitId {+ pub fn id(&self) -> uuid::Uuid {+ self.0+ }+}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]pub struct Split {pub id: SplitId,diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex 948d7b50..5c13cafa 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -30,6 +30,7 @@ use crate::image_store::Image;use crate::inverted_index::RetrievedWebpage;#[cfg(not(feature = "libtorch"))]use crate::ranking::models::cross_encoder::DummyCrossEncoder;+use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite};use crate::ranking::ALL_SIGNALS;use crate::search_prettifier::{create_stackoverflow_sidebar, DisplayedAnswer, DisplayedEntity, DisplayedSidebar,@@ -46,14 +47,35 @@ use crate::{ceil_char_boundary, floor_char_boundary, query, Result};#[cfg(feature = "libtorch")]use crate::{qa_model::QaModel, ranking::models::cross_encoder::CrossEncoderModel};-use super::{- distributed, DistributedSearcher, InitialSearchResultShard, ScoredWebsitePointer, SearchQuery,- SearchResult, WebsitesResult,-};+use super::live::LiveSearcher;+use super::{distributed, live, DistributedSearcher, SearchQuery, SearchResult, WebsitesResult};++#[derive(Clone)]+enum ScoredWebsitePointer {+ Normal(distributed::ScoredWebsitePointer),+ Live(live::ScoredWebsitePointer),+}++impl AsRankingWebsite for ScoredWebsitePointer {+ fn as_ranking(&self) -> &RankingWebsite {+ match self {+ ScoredWebsitePointer::Normal(p) => &p.website,+ ScoredWebsitePointer::Live(p) => &p.website,+ }+ }++ fn as_mut_ranking(&mut self) -> &mut RankingWebsite {+ match self {+ ScoredWebsitePointer::Normal(p) => &mut p.website,+ ScoredWebsitePointer::Live(p) => &mut p.website,+ }+ }+}#[cfg(feature = "libtorch")]pub struct ApiSearcher {distributed_searcher: DistributedSearcher,+ live_searcher: LiveSearcher,cross_encoder: Option<Arc<CrossEncoderModel>>,lambda_model: Option<Arc<LambdaMART>>,qa_model: Option<Arc<QaModel>>,@@ -84,7 +106,8 @@ impl ApiSearcher {config: ApiConfig,) -> Self {Self {- distributed_searcher: DistributedSearcher::new(cluster),+ distributed_searcher: DistributedSearcher::new(Arc::clone(&cluster)),+ live_searcher: LiveSearcher::new(Arc::clone(&cluster)),cross_encoder: cross_encoder.map(Arc::new),lambda_model: lambda_model.map(Arc::new),qa_model: qa_model.map(Arc::new),@@ -114,7 +137,8 @@ impl ApiSearcher {fn combine_results(&self,- initial_results: Vec<InitialSearchResultShard>,+ initial_results: Vec<distributed::InitialSearchResultShard>,+ live_results: Vec<live::InitialSearchResultSplit>,pipeline: RankingPipeline<ScoredWebsitePointer>,) -> (Vec<ScoredWebsitePointer>, bool) {let mut collector =@@ -124,11 +148,27 @@ impl ApiSearcher {for result in initial_results {for website in result.local_result.websites {num_sites += 1;- let pointer = ScoredWebsitePointer {+ let pointer = distributed::ScoredWebsitePointer {website,- shard: result.shard.clone(),+ shard: result.shard,};+ let pointer = ScoredWebsitePointer::Normal(pointer);++ collector.insert(pointer);+ }+ }++ for result in live_results {+ for website in result.local_result.websites {+ num_sites += 1;+ let pointer = live::ScoredWebsitePointer {+ website,+ split_id: result.split_id.clone(),+ };++ let pointer = ScoredWebsitePointer::Live(pointer);+collector.insert(pointer);}}@@ -174,13 +214,14 @@ impl ApiSearcher {if let Some((shard, website)) = results.pop() {if website.score > self.thresholds.stackoverflow {- let scored_websites = vec![ScoredWebsitePointer { website, shard }];+ let scored_websites =+ vec![(0, distributed::ScoredWebsitePointer { website, shard })];let mut retrieved = self.distributed_searcher.retrieve_webpages(&scored_websites, &query.query).await;- if let Some(res) = retrieved.pop() {+ if let Some((_, res)) = retrieved.pop() {return Ok(Some(create_stackoverflow_sidebar(res.schema_org,Url::parse(&res.url).unwrap(),@@ -194,7 +235,7 @@ impl ApiSearcher {async fn sidebar(&self,- initial_results: &[InitialSearchResultShard],+ initial_results: &[distributed::InitialSearchResultShard],query: &SearchQuery,) -> Result<Option<DisplayedSidebar>> {let entity = initial_results@@ -303,11 +344,11 @@ impl ApiSearcher {return Ok(None);}- let (top_websites, _) = self.combine_results(initial_results, pipeline);+ let (top_websites, _) = self.combine_results(initial_results, vec![], pipeline);let scores: Vec<_> = top_websites.iter()- .map(|pointer| pointer.website.score)+ .map(|pointer| pointer.as_ranking().score).collect();let median = if scores.len() % 2 == 0 {@@ -322,9 +363,24 @@ impl ApiSearcher {let mut result: Vec<DisplayedWebpage> = self.distributed_searcher- .retrieve_webpages(&top_websites, &query.query)+ .retrieve_webpages(+ &top_websites+ .clone()+ .into_iter()+ .filter_map(|pointer| {+ if let ScoredWebsitePointer::Normal(p) = pointer {+ Some(p)+ } else {+ None+ }+ })+ .enumerate()+ .collect_vec(),+ &query.query,+ ).await.into_iter()+ .map(|(_, webpage)| webpage).map(DisplayedWebpage::from).collect();@@ -344,7 +400,7 @@ impl ApiSearcher {let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());for signal in ALL_SIGNALS {- if let Some(signal_value) = pointer.website.signals.get(signal) {+ if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {signals.insert(signal, *signal_value);}}@@ -378,17 +434,16 @@ impl ApiSearcher {self.collector_config.clone(),)?;- let initial_results = self- .distributed_searcher- .search_initial(&search_query)- .await;+ let (initial_results, live_results, widget, discussions) = tokio::join!(+ self.distributed_searcher.search_initial(&search_query),+ self.live_searcher.search_initial(&search_query),+ self.widget(query),+ self.discussions_widget(query)+ );+ let discussions = discussions?;let sidebar = self.sidebar(&initial_results, query).await?;- let discussions = self.discussions_widget(query).await?;-- let widget = self.widget(query);-let spell_corrected_query = if widget.is_none() {initial_results.first()@@ -403,14 +458,51 @@ impl ApiSearcher {.map(|result| result.local_result.num_websites).sum();- let (top_websites, has_more_results) = self.combine_results(initial_results, pipeline);+ let (top_websites, has_more_results) =+ self.combine_results(initial_results, live_results, pipeline);// retrieve webpages- let mut retrieved_webpages: Vec<_> = self+ let normal: Vec<_> = top_websites+ .iter()+ .enumerate()+ .filter_map(|(i, pointer)| {+ if let ScoredWebsitePointer::Normal(p) = pointer {+ Some((i, p.clone()))+ } else {+ None+ }+ })+ .collect();++ let live: Vec<_> = top_websites+ .iter()+ .enumerate()+ .filter_map(|(i, pointer)| {+ if let ScoredWebsitePointer::Live(p) = pointer {+ Some((i, p.clone()))+ } else {+ None+ }+ })+ .collect();++ let retrieved_normal: Vec<_> = self.distributed_searcher- .retrieve_webpages(&top_websites, &query.query)- .await+ .retrieve_webpages(&normal, &query.query)+ .await;++ let retrieved_live: Vec<_> = self+ .live_searcher+ .retrieve_webpages(&live, &query.query)+ .await;++ let mut retrieved_webpages: Vec<_> =+ retrieved_normal.into_iter().chain(retrieved_live).collect();+ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));++ let mut retrieved_webpages: Vec<_> = retrieved_webpages.into_iter()+ .map(|(_, webpage)| webpage).map(DisplayedWebpage::from).collect();@@ -447,7 +539,7 @@ impl ApiSearcher {Ok(SearchResult::Websites(self.search_websites(query).await?))}- fn widget(&self, query: &SearchQuery) -> Option<Widget> {+ async fn widget(&self, query: &SearchQuery) -> Option<Widget> {if query.page > 0 {return None;}@@ -508,7 +600,7 @@ impl ApiSearcher {None}- pub async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {+ pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {self.distributed_searcher.get_webpage(url).await}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 6199f07a..eb107a16 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -15,33 +15,31 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff},+ distributed::{+ cluster::Cluster,+ member::Service,+ sonic::replication::{+ AllShardsSelector, RandomReplicaSelector, RandomShardSelector, RemoteClient,+ ReplicatedClient, Shard, ShardIdentifier, ShardedClient, SpecificShardSelector,+ },+ },entrypoint::search_server::{self, SearchService},image_store::Image,- inverted_index::{self, RetrievedWebpage},- ranking::pipeline::{AsRankingWebsite, RankingWebsite},+ inverted_index::{RetrievedWebpage, WebsitePointer},+ ranking::pipeline::RankingWebsite,Result,};-use std::{collections::HashMap, net::SocketAddr, sync::Arc, time::Duration};+use std::{collections::HashMap, sync::Arc};-use futures::stream::FuturesUnordered;-use futures::StreamExt;-use rand::seq::SliceRandom;+use futures::future::join_all;+use itertools::Itertools;use serde::{Deserialize, Serialize};use thiserror::Error;use url::Url;-use crate::distributed::sonic;-use super::{InitialWebsiteResult, SearchQuery};-const NUM_REPLICA_RETRIES: usize = 3;--struct RemoteSearcher {- addr: SocketAddr,-}-#[derive(Error, Debug)]pub enum Error {#[error("Failed to get search result")]@@ -54,188 +52,16 @@ pub enum Error {WebpageNotFound,}-impl RemoteSearcher {- async fn search(&self, query: &SearchQuery) -> Result<InitialWebsiteResult> {- let conn = self.conn().await;-- if let Ok(Some(body)) = conn- .send_with_timeout(- &search_server::Search {- query: query.clone(),- },- Duration::from_secs(60),- )- .await- {- return Ok(body);- }-- Err(Error::SearchFailed.into())- }-- async fn retrieve_websites(- &self,- pointers: &[inverted_index::WebsitePointer],- original_query: &str,- ) -> Result<Vec<RetrievedWebpage>> {- let conn = self.conn().await;-- if let Ok(Some(body)) = conn- .send_with_timeout(- &search_server::RetrieveWebsites {- websites: pointers.to_vec(),- query: original_query.to_string(),- },- Duration::from_secs(60),- )- .await- {- return Ok(body);- }- Err(Error::SearchFailed.into())- }-- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- let conn = self.conn().await;-- if let Ok(body) = conn- .send_with_timeout(- &search_server::GetWebpage {- url: url.to_string(),- },- Duration::from_secs(60),- )- .await- {- return Ok(body);- }-- Err(Error::WebpageNotFound.into())- }-- async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {- let conn = self.conn().await;-- if let Ok(body) = conn- .send_with_timeout(- &search_server::GetHomepageDescriptions {- urls: urls.to_vec(),- },- Duration::from_secs(60),- )- .await- {- return body;- }-- HashMap::new()- }-- async fn conn(&self) -> sonic::service::ResilientConnection<SearchService> {- let retry = ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5);-- sonic::service::ResilientConnection::create_with_timeout(- self.addr,- Duration::from_secs(30),- retry,- )- .await- .unwrap()- }-- async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {- let conn = self.conn().await;-- conn.send_with_timeout(- &search_server::GetEntityImage {- image_id: image_id.to_string(),- },- Duration::from_secs(60),- )- .await- .map_err(|e| e.into())- }+#[derive(Clone, Debug)]+pub struct ScoredWebsitePointer {+ pub website: RankingWebsite,+ pub shard: ShardId,}-#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]+#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]pub struct ShardId(u64);-pub struct Shard {- id: ShardId,- replicas: Vec<RemoteSearcher>,-}--impl Shard {- pub fn new(id: u64, replicas: Vec<String>) -> Self {- let mut parsed_replicas = Vec::new();-- for replica in replicas {- parsed_replicas.push(RemoteSearcher {- addr: replica.parse().unwrap(),- });- }-- Self {- id: ShardId(id),- replicas: parsed_replicas,- }- }-- fn replica(&self) -> &RemoteSearcher {- self.replicas.choose(&mut rand::thread_rng()).unwrap()- }-- async fn search(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {- for _ in 0..NUM_REPLICA_RETRIES {- if let Ok(result) = self.replica().search(query).await {- return Ok(InitialSearchResultShard {- local_result: result,- shard: self.id.clone(),- });- }- }-- Err(Error::SearchFailed.into())- }-- async fn retrieve_websites(- &self,- pointers: &[inverted_index::WebsitePointer],- original_query: &str,- ) -> Result<Vec<RetrievedWebpage>> {- for _ in 0..NUM_REPLICA_RETRIES {- if let Ok(res) = self- .replica()- .retrieve_websites(pointers, original_query)- .await- {- return Ok(res);- }- }-- Err(Error::SearchFailed.into())- }-- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- for _ in 0..NUM_REPLICA_RETRIES {- if let Ok(res) = self.replica().get_webpage(url).await {- return Ok(res);- }- }-- Err(Error::SearchFailed.into())- }-- async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {- self.replica().get_homepage_descriptions(urls).await- }-- async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {- self.replica().get_entity_image(image_id).await- }-}+impl ShardIdentifier for ShardId {}#[derive(Debug)]pub struct InitialSearchResultShard {@@ -243,22 +69,6 @@ pub struct InitialSearchResultShard {pub shard: ShardId,}-#[derive(Clone, Debug)]-pub struct ScoredWebsitePointer {- pub website: RankingWebsite,- pub shard: ShardId,-}--impl AsRankingWebsite for ScoredWebsitePointer {- fn as_ranking(&self) -> &RankingWebsite {- &self.website- }-- fn as_mut_ranking(&mut self) -> &mut RankingWebsite {- &mut self.website- }-}-pub struct DistributedSearcher {cluster: Arc<Cluster>,}@@ -268,7 +78,7 @@ impl DistributedSearcher {Self { cluster }}- async fn shards(&self) -> Vec<Shard> {+ async fn client(&self) -> ShardedClient<SearchService, ShardId> {let mut shards = HashMap::new();for member in self.cluster.members().await {if let Service::Searcher { host, shard } = member.service {@@ -276,97 +86,165 @@ impl DistributedSearcher {}}- shards- .into_iter()- .map(|(shard, replicas)| Shard {- id: shard,- replicas: replicas- .into_iter()- .map(|addr| RemoteSearcher { addr })- .collect(),- })- .collect()+ let mut shard_clients = Vec::new();++ for (id, replicas) in shards {+ let replicated =+ ReplicatedClient::new(replicas.into_iter().map(RemoteClient::new).collect());+ let shard = Shard::new(id, replicated);+ shard_clients.push(shard);+ }++ ShardedClient::new(shard_clients)}- pub async fn retrieve_webpages(+ pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {+ let client = self.client().await;+ let mut results = Vec::new();++ if let Ok(res) = client+ .send(+ &search_server::Search {+ query: query.clone(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ {+ for (shard_id, mut res) in res {+ if let Some(Some(res)) = res.pop() {+ results.push(InitialSearchResultShard {+ local_result: res,+ shard: shard_id,+ });+ }+ }+ }++ results+ }++ async fn retrieve_webpages_from_shard(&self,- top_websites: &[ScoredWebsitePointer],+ shard: ShardId,+ client: &ShardedClient<SearchService, ShardId>,query: &str,- ) -> Vec<RetrievedWebpage> {- let mut retrieved_webpages = Vec::new();+ pointers: Vec<(usize, WebsitePointer)>,+ ) -> Vec<(usize, RetrievedWebpage)> {+ let (idxs, pointers): (Vec<usize>, Vec<WebsitePointer>) = pointers.into_iter().unzip();- for _ in 0..top_websites.len() {- retrieved_webpages.push(None);+ match client+ .send(+ &search_server::RetrieveWebsites {+ websites: pointers,+ query: query.to_string(),+ },+ &SpecificShardSelector(shard),+ &RandomReplicaSelector,+ )+ .await+ {+ Ok(v) => v+ .into_iter()+ .flat_map(|(_, v)| v)+ .flatten()+ .flatten()+ .zip_eq(idxs)+ .map(|(v, i)| (i, v))+ .collect(),+ _ => vec![],}+ }- for shard in self.shards().await.iter() {- let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites- .iter()- .enumerate()- .filter(|(_, pointer)| pointer.shard == shard.id)- .map(|(idx, pointer)| (idx, pointer.website.pointer.clone()))- .unzip();-- if let Ok(websites) = shard.retrieve_websites(&pointers, query).await {- for (index, website) in indexes.into_iter().zip(websites) {- retrieved_webpages[index] = Some(website);- }- }+ pub async fn retrieve_webpages(+ &self,+ top_websites: &[(usize, ScoredWebsitePointer)],+ query: &str,+ ) -> Vec<(usize, RetrievedWebpage)> {+ let mut pointers: HashMap<_, Vec<_>> = HashMap::new();++ for (i, pointer) in top_websites {+ pointers+ .entry(pointer.shard)+ .or_default()+ .push((*i, pointer.website.pointer.clone()));+ }++ let client = self.client().await;+ let mut futures = Vec::new();+ for (shard, pointers) in pointers {+ futures.push(self.retrieve_webpages_from_shard(shard, &client, query, pointers));}- let retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().flatten().collect();+ let mut retrieved_webpages = Vec::new();+ for pages in join_all(futures).await {+ retrieved_webpages.extend(pages);+ }debug_assert_eq!(retrieved_webpages.len(), top_websites.len());+ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));+retrieved_webpages}- pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {- self.shards()- .await- .iter()- .map(|shard| shard.search(query))- .collect::<FuturesUnordered<_>>()- .collect::<Vec<_>>()- .await- .into_iter()- .filter_map(|result| result.ok())- .collect::<Vec<_>>()- }+ pub async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {+ let client = self.client().await;- pub async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {- self.shards()- .await- .iter()- .map(|shard| shard.get_webpage(url))- .collect::<FuturesUnordered<_>>()- .collect::<Vec<_>>()+ let res = client+ .send(+ &search_server::GetWebpage {+ url: url.to_string(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ ).await- .into_iter()- .filter_map(|result| result.ok())- .flatten()- .collect::<Vec<_>>()- .pop()- .ok_or(Error::WebpageNotFound.into())+ .map_err(|_| Error::SearchFailed)?;++ if let Some(Some(res)) = res.into_iter().flat_map(|(_, v)| v).next() {+ Ok(Some(res))+ } else {+ Err(Error::WebpageNotFound.into())+ }}pub async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {- self.shards()- .await- .iter()- .map(|shard| shard.get_homepage_descriptions(urls))- .collect::<FuturesUnordered<_>>()- .collect::<Vec<_>>()- .await- .into_iter()- .flatten()- .collect()+ let client = self.client().await;++ let res = client+ .send(+ &search_server::GetHomepageDescriptions {+ urls: urls.to_vec(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await;++ match res {+ Ok(v) => v.into_iter().flat_map(|(_, v)| v).flatten().collect(),+ _ => HashMap::new(),+ }}pub async fn get_entity_image(&self, image_id: &str) -> Result<Option<Image>> {- match self.shards().await.first() {- Some(s) => s.get_entity_image(image_id).await,- None => Ok(None),- }+ let client = self.client().await;++ let (_, mut res) = client+ .send(+ &search_server::GetEntityImage {+ image_id: image_id.to_string(),+ },+ &RandomShardSelector,+ &RandomReplicaSelector,+ )+ .await+ .map_err(|_| Error::SearchFailed)?+ .pop()+ .unwrap();++ Ok(res.pop().flatten())}}diff --git a/core/src/searcher/live.rs b/core/src/searcher/live.rsnew file mode 100644index 00000000..6593d10b--- /dev/null+++ b/core/src/searcher/live.rs@@ -0,0 +1,172 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ distributed::{+ cluster::Cluster,+ member::Service,+ sonic::replication::{+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,+ ShardIdentifier, ShardedClient, SpecificShardSelector,+ },+ },+ entrypoint::search_server::{self, SearchService},+ feed::scheduler::SplitId,+ inverted_index::{RetrievedWebpage, WebsitePointer},+ ranking::pipeline::RankingWebsite,+};++use std::{collections::HashMap, sync::Arc};++use futures::future::join_all;+use itertools::Itertools;++use super::{InitialWebsiteResult, SearchQuery};++#[derive(Clone, Debug)]+pub struct ScoredWebsitePointer {+ pub website: RankingWebsite,+ pub split_id: SplitId,+}++impl ShardIdentifier for SplitId {}++#[derive(Debug)]+pub struct InitialSearchResultSplit {+ pub local_result: InitialWebsiteResult,+ pub split_id: SplitId,+}++pub struct LiveSearcher {+ cluster: Arc<Cluster>,+}++impl LiveSearcher {+ pub fn new(cluster: Arc<Cluster>) -> Self {+ Self { cluster }+ }++ async fn client(&self) -> ShardedClient<SearchService, SplitId> {+ let mut shards = HashMap::new();+ for member in self.cluster.members().await {+ if let Service::LiveIndex { host, split_id } = member.service {+ shards.entry(split_id).or_insert_with(Vec::new).push(host);+ }+ }++ let mut shard_clients = Vec::new();++ for (id, replicas) in shards {+ let replicated =+ ReplicatedClient::new(replicas.into_iter().map(RemoteClient::new).collect());+ let shard = Shard::new(id, replicated);+ shard_clients.push(shard);+ }++ ShardedClient::new(shard_clients)+ }++ pub async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultSplit> {+ let client = self.client().await;+ let mut results = Vec::new();++ if let Ok(res) = client+ .send(+ &search_server::Search {+ query: query.clone(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ {+ for (shard_id, mut res) in res {+ if let Some(Some(res)) = res.pop() {+ results.push(InitialSearchResultSplit {+ local_result: res,+ split_id: shard_id,+ });+ }+ }+ }++ results+ }++ async fn retrieve_webpages_from_shard(+ &self,+ split: SplitId,+ client: &ShardedClient<SearchService, SplitId>,+ query: &str,+ pointers: Vec<(usize, WebsitePointer)>,+ ) -> Vec<(usize, RetrievedWebpage)> {+ let (idxs, pointers): (Vec<usize>, Vec<WebsitePointer>) = pointers.into_iter().unzip();++ match client+ .send(+ &search_server::RetrieveWebsites {+ websites: pointers,+ query: query.to_string(),+ },+ &SpecificShardSelector(split),+ &RandomReplicaSelector,+ )+ .await+ {+ Ok(v) => v+ .into_iter()+ .flat_map(|(_, v)| v)+ .flatten()+ .flatten()+ .zip_eq(idxs)+ .map(|(v, i)| (i, v))+ .collect(),+ _ => vec![],+ }+ }++ pub async fn retrieve_webpages(+ &self,+ top_websites: &[(usize, ScoredWebsitePointer)],+ query: &str,+ ) -> Vec<(usize, RetrievedWebpage)> {+ let mut pointers: HashMap<_, Vec<_>> = HashMap::new();++ for (i, pointer) in top_websites {+ pointers+ .entry(pointer.split_id.clone())+ .or_default()+ .push((*i, pointer.website.pointer.clone()));+ }++ let client = self.client().await;+ let mut futures = Vec::new();+ for (split_id, pointers) in pointers {+ futures.push(self.retrieve_webpages_from_shard(split_id, &client, query, pointers));+ }++ let mut retrieved_webpages = Vec::new();+ for pages in join_all(futures).await {+ retrieved_webpages.extend(pages);+ }++ debug_assert_eq!(retrieved_webpages.len(), top_websites.len());++ retrieved_webpages.sort_by(|(a, _), (b, _)| a.cmp(b));++ retrieved_webpages+ }+}diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 592d2552..2ed4d250 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -16,6 +16,7 @@pub mod api;pub mod distributed;+mod live;pub mod local;pub use distributed::*;