web search done right
[search_server[ implement GetSiteUrls as generic query
Mikkel Denker14e6e110581a
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 2d2b85e3..cf99f3be 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -76,7 +76,9 @@ pub async fn main() {let webgraph = Webgraph::open("data/webgraph", 0u64.into()).unwrap();let searcher: ApiSearcher<LocalSearchClient, LiveSearcher, Webgraph> =- ApiSearcher::new(searcher, bangs, config).with_webgraph(webgraph);+ ApiSearcher::new(searcher, None, bangs, config)+ .await+ .with_webgraph(webgraph);for query in queries {let mut desc = "search '".to_string();diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex 03dbfc50..aca12d2f 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -209,7 +209,9 @@ pub async fn router(}let mut searcher =- ApiSearcher::new(dist_searcher, bangs, config.clone()).with_live(live_searcher);+ ApiSearcher::new(dist_searcher, Some(cluster.clone()), bangs, config.clone())+ .await+ .with_live(live_searcher);if let Some(cross_encoder) = cross_encoder {searcher = searcher.with_cross_encoder(cross_encoder);diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rsindex 586b275e..be018477 100644--- a/crates/core/src/distributed/sonic/replication.rs+++ b/crates/core/src/distributed/sonic/replication.rs@@ -511,6 +511,19 @@ wherelast_client_update: std::time::Instant,}+impl<M> Clone for ReusableShardedClient<M>+where+ M: ReusableClientManager,+{+ fn clone(&self) -> Self {+ Self {+ cluster: self.cluster.clone(),+ client: self.client.clone(),+ last_client_update: self.last_client_update,+ }+ }+}+impl<M> ReusableShardedClient<M>whereM: ReusableClientManager,diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsindex 2e9003cd..936f52c8 100644--- a/crates/core/src/entrypoint/live_index/search_server.rs+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -44,7 +44,7 @@ use tracing::info;use crate::entrypoint::{indexer::{self, IndexableWebpage},- search_server::{GetSiteUrls, RetrieveWebsites, Search, SiteUrls},+ search_server::{RetrieveWebsites, Search},};const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);@@ -55,7 +55,6 @@ sonic_service!([RetrieveWebsites,Search,- GetSiteUrls,IndexWebpages,GetIndexPath,RemoteDownload@@ -351,18 +350,6 @@ impl sonic::service::Message<LiveIndexService> for Search {}}-impl sonic::service::Message<LiveIndexService> for GetSiteUrls {- type Response = SiteUrls;- async fn handle(self, server: &LiveIndexService) -> Self::Response {- let urls = server- .local_searcher- .get_site_urls(&self.site, self.offset as usize, self.limit as usize)- .await;-- SiteUrls { urls }- }-}-#[derive(Clone, Debug)]struct RemoteIndex {host: SocketAddr,diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 135cf9a2..21582d72 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -17,7 +17,6 @@use std::sync::Arc;use tracing::info;-use url::Url;use crate::{config,@@ -26,7 +25,9 @@ use crate::{member::{Member, Service},sonic::{self, service::sonic_service},},- generic_query::{self, GetHomepageQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},+ generic_query::{+ self, GetHomepageQuery, GetSiteUrlsQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery,+ },index::Index,inverted_index,models::dual_encoder::DualEncoder,@@ -116,7 +117,6 @@ macro_rules! impl_search {sonic_service!(SearchService, [RetrieveWebsites,Search,- GetSiteUrls,$($q,[<$q Retrieve>],@@ -132,6 +132,7 @@ impl_search!([SizeQuery,GetWebpageQuery,GetHomepageQuery,+ GetSiteUrlsQuery,]);pub struct SearchService {@@ -221,28 +222,3 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {}}}--#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct GetSiteUrls {- pub site: String,- pub offset: u64,- pub limit: u64,-}--#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct SiteUrls {- #[bincode(with_serde)]- pub urls: Vec<Url>,-}--impl sonic::service::Message<SearchService> for GetSiteUrls {- type Response = SiteUrls;- async fn handle(self, server: &SearchService) -> Self::Response {- let urls = server- .local_searcher- .get_site_urls(&self.site, self.offset as usize, self.limit as usize)- .await;-- SiteUrls { urls }- }-}diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rsindex a0b407ef..230339c5 100644--- a/crates/core/src/generic_query/collector/mod.rs+++ b/crates/core/src/generic_query/collector/mod.rs@@ -23,6 +23,9 @@ pub use size::SizeCollector;mod first_doc;pub use first_doc::FirstDocCollector;+mod top_docs;+pub use top_docs::TopDocsCollector;+use tantivy::{collector::{Fruit, SegmentCollector},query::Weight,diff --git a/crates/core/src/generic_query/collector/top_docs.rs b/crates/core/src/generic_query/collector/top_docs.rsnew file mode 100644index 00000000..084aec58--- /dev/null+++ b/crates/core/src/generic_query/collector/top_docs.rs@@ -0,0 +1,209 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{distributed::member::ShardId, inverted_index::DocAddress};+use tantivy::{+ collector::{SegmentCollector, TopNComputer},+ DocId, SegmentOrdinal,+};++use super::Collector;++pub struct TopDocsCollector {+ shard_id: Option<ShardId>,+ limit: Option<usize>,+ offset: Option<usize>,+ perform_offset: bool,+}++impl Default for TopDocsCollector {+ fn default() -> Self {+ Self::new()+ }+}++impl TopDocsCollector {+ pub fn new() -> Self {+ Self {+ shard_id: None,+ limit: None,+ offset: None,+ perform_offset: true,+ }+ }+}++impl TopDocsCollector {+ pub fn with_shard_id(self, shard_id: ShardId) -> Self {+ Self {+ shard_id: Some(shard_id),+ ..self+ }+ }++ pub fn with_offset(self, offset: usize) -> Self {+ Self {+ offset: Some(offset),+ ..self+ }+ }++ pub fn with_limit(self, limit: usize) -> Self {+ Self {+ limit: Some(limit),+ ..self+ }+ }++ pub fn enable_offset(self) -> Self {+ Self {+ perform_offset: true,+ ..self+ }+ }++ pub fn disable_offset(self) -> Self {+ Self {+ perform_offset: false,+ ..self+ }+ }+}++impl TopDocsCollector {+ fn computer(&self) -> Computer {+ match (self.offset, self.limit) {+ (Some(offset), Some(limit)) => Computer::TopN(TopNComputer::new(limit + offset)),+ (Some(_), None) => Computer::All(AllComputer::new()),+ (None, Some(limit)) => Computer::TopN(TopNComputer::new(limit)),+ (None, None) => Computer::All(AllComputer::new()),+ }+ }+}++impl Collector for TopDocsCollector {+ type Fruit = Vec<(tantivy::Score, DocAddress)>;++ type Child = TopDocsSegmentCollector;++ fn for_segment(+ &self,+ segment_ord: SegmentOrdinal,+ _: &tantivy::SegmentReader,+ ) -> crate::Result<Self::Child> {+ Ok(TopDocsSegmentCollector {+ shard_id: self.shard_id.unwrap(),+ computer: self.computer(),+ segment_ord,+ })+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> crate::Result<Self::Fruit> {+ let mut computer = self.computer();++ let fruits: Vec<_> = segment_fruits.into_iter().flatten().collect();++ for (score, doc) in fruits {+ computer.push(score, doc);+ }++ let result = computer.harvest();++ if self.perform_offset {+ Ok(result+ .into_iter()+ .skip(self.offset.unwrap_or(0))+ .take(self.limit.unwrap_or(usize::MAX))+ .collect())+ } else {+ Ok(result+ .into_iter()+ .take(self.limit.unwrap_or(usize::MAX))+ .collect())+ }+ }+}++enum Computer {+ TopN(TopNComputer<tantivy::Score, DocAddress>),+ All(AllComputer),+}++impl Computer {+ fn push(&mut self, score: tantivy::Score, doc: DocAddress) {+ match self {+ Computer::TopN(computer) => computer.push(score, doc),+ Computer::All(computer) => computer.push(score, doc),+ }+ }++ fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {+ match self {+ Computer::TopN(computer) => computer+ .into_sorted_vec()+ .into_iter()+ .map(|comparable_doc| (comparable_doc.feature, comparable_doc.doc))+ .collect(),+ Computer::All(computer) => computer.harvest(),+ }+ }+}++struct AllComputer {+ docs: Vec<(tantivy::Score, DocAddress)>,+}++impl AllComputer {+ fn new() -> Self {+ Self { docs: Vec::new() }+ }++ fn push(&mut self, score: tantivy::Score, doc: DocAddress) {+ self.docs.push((score, doc));+ }++ fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {+ let mut docs = self.docs;+ docs.sort_by(|(score1, _), (score2, _)| score2.total_cmp(score1));+ docs+ }+}++pub struct TopDocsSegmentCollector {+ shard_id: ShardId,+ computer: Computer,+ segment_ord: SegmentOrdinal,+}++impl SegmentCollector for TopDocsSegmentCollector {+ type Fruit = Vec<(tantivy::Score, DocAddress)>;++ fn collect(&mut self, doc: DocId, score: tantivy::Score) {+ if doc == tantivy::TERMINATED {+ return;+ }++ self.computer+ .push(score, DocAddress::new(self.segment_ord, doc, self.shard_id));+ }++ fn harvest(self) -> Self::Fruit {+ self.computer.harvest()+ }+}diff --git a/crates/core/src/generic_query/get_site_urls.rs b/crates/core/src/generic_query/get_site_urls.rsnew file mode 100644index 00000000..e14510d0--- /dev/null+++ b/crates/core/src/generic_query/get_site_urls.rs@@ -0,0 +1,110 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::TantivyDocument;+use url::Url;++use crate::{+ inverted_index::RetrievedWebpage,+ schema::text_field::{self, TextField},+ search_ctx,+ webpage::url_ext::UrlExt,+};++use super::{collector::TopDocsCollector, GenericQuery};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct GetSiteUrlsQuery {+ pub site: String,+ pub limit: u64,+ pub offset: Option<u64>,+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct GetSiteUrlsResponse {+ #[bincode(with_serde)]+ pub urls: Vec<Url>,+}++impl GenericQuery for GetSiteUrlsQuery {+ type Collector = TopDocsCollector;+ type TantivyQuery = Box<dyn tantivy::query::Query>;+ type IntermediateOutput = GetSiteUrlsResponse;+ type Output = GetSiteUrlsResponse;++ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery {+ let field = ctx+ .tv_searcher+ .schema()+ .get_field(text_field::SiteNoTokenizer.name())+ .unwrap();+ let term = tantivy::Term::from_field_text(field, &self.site);+ Box::new(tantivy::query::TermQuery::new(+ term,+ tantivy::schema::IndexRecordOption::Basic,+ ))+ }++ fn collector(&self, _: &search_ctx::Ctx) -> Self::Collector {+ Self::Collector::new()+ .with_limit(self.limit as usize)+ .with_offset(self.offset.unwrap_or(0) as usize)+ .disable_offset()+ }++ fn remote_collector(&self) -> Self::Collector {+ Self::Collector::new()+ .with_limit(self.limit as usize)+ .with_offset(self.offset.unwrap_or(0) as usize)+ .enable_offset()+ }++ fn filter_fruit_shards(+ &self,+ shard_id: crate::ampc::dht::ShardId,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> <Self::Collector as super::Collector>::Fruit {+ fruit+ .into_iter()+ .filter(|(_, addr)| addr.shard_id == shard_id)+ .collect()+ }++ fn retrieve(+ &self,+ ctx: &search_ctx::Ctx,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> crate::Result<Self::IntermediateOutput> {+ let mut urls = Vec::new();++ for (_, addr) in fruit {+ let doc: TantivyDocument = ctx.tv_searcher.doc(addr.into())?;+ let doc = RetrievedWebpage::from(doc);+ urls.push(Url::robust_parse(&doc.url)?);+ }++ Ok(GetSiteUrlsResponse { urls })+ }++ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {+ let mut urls: Vec<_> = results.into_iter().flat_map(|r| r.urls).collect();++ urls.sort_by_key(|url| url.to_string());+ urls.dedup();++ GetSiteUrlsResponse { urls }+ }+}diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rsindex 2ecc5365..a5958b42 100644--- a/crates/core/src/generic_query/mod.rs+++ b/crates/core/src/generic_query/mod.rs@@ -28,6 +28,9 @@ pub use get_webpage::GetWebpageQuery;pub mod get_homepage;pub use get_homepage::GetHomepageQuery;+pub mod get_site_urls;+pub use get_site_urls::GetSiteUrlsQuery;+pub mod collector;pub use collector::Collector;diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 840f01d4..4d6a1ad0 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -83,6 +83,9 @@ pub struct WebpagePointer {Clone,Copy,PartialEq,+ Eq,+ PartialOrd,+ Ord,)]pub struct DocAddress {pub segment: u32,diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex 396e2794..13db884d 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -82,10 +82,6 @@ impl Client {self.live_index.lock().await.conn().await}- async fn search_conn(&self) -> Arc<ShardedClient<search_server::SearchService, ShardId>> {- self.search.lock().await.conn().await- }-pub async fn index(&self, pages: Vec<IndexableWebpage>) -> Result<()> {let mut conn = self.live_conn().await;@@ -112,23 +108,10 @@ impl Client {Ok(())}- async fn get_site_urls_from_backbone(&self, site: &str) -> Vec<Url> {- let mut res = Vec::new();- let conn = self.search_conn().await;- let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();-- while let Some(url) = stream.next().await {- res.push(url);- }-- res- }-- async fn get_site_urls_from_live(&self, site: &str) -> Vec<Url> {+ pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {let mut res = Vec::new();- let conn = self.live_conn().await;-- let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();+ let mut stream =+ SiteUrlStream::new(site.to_string(), self.search.lock().await.clone()).stream();while let Some(url) = stream.next().await {res.push(url);@@ -136,15 +119,6 @@ impl Client {res}-- pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {- let mut res = Vec::new();-- res.extend(self.get_site_urls_from_backbone(site).await);- res.extend(self.get_site_urls_from_live(site).await);-- res- }}impl From<LiveCrawlerConfig> for CrawlerConfig {diff --git a/crates/core/src/live_index/crawler/site_url_stream.rs b/crates/core/src/live_index/crawler/site_url_stream.rsindex 835aa78d..f22c5454 100644--- a/crates/core/src/live_index/crawler/site_url_stream.rs+++ b/crates/core/src/live_index/crawler/site_url_stream.rs@@ -16,83 +16,45 @@const SITE_URL_BATCH_SIZE: usize = 100;-use std::sync::Arc;-use url::Url;use crate::{- ampc::dht::ShardId,distributed::{- sonic::{- replication::{AllShardsSelector, RandomReplicaSelector, ShardedClient},- service::Service,- },- streaming_response::StreamingResponse,+ sonic::replication::ReusableShardedClient, streaming_response::StreamingResponse,},- entrypoint::{live_index, search_server},+ entrypoint::search_server,+ generic_query::GetSiteUrlsQuery,+ searcher::{DistributedSearcher, SearchClient},Result,};-pub struct SiteUrlStream<S: Service> {+pub struct SiteUrlStream {site: String,offset: usize,- conn: Arc<ShardedClient<S, ShardId>>,+ searcher: DistributedSearcher,}-impl<S: Service> SiteUrlStream<S> {- pub fn new(site: String, conn: Arc<ShardedClient<S, ShardId>>) -> Self {+impl SiteUrlStream {+ pub fn new(site: String, conn: ReusableShardedClient<search_server::SearchService>) -> Self {Self {site,offset: 0,- conn,+ searcher: DistributedSearcher::from_client(conn),}}}-impl StreamingResponse for SiteUrlStream<search_server::SearchService> {- type Item = Url;-- async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {- let req = search_server::GetSiteUrls {- site: self.site.clone(),- offset: self.offset as u64,- limit: SITE_URL_BATCH_SIZE as u64,- };-- self.offset += SITE_URL_BATCH_SIZE;-- let res = self- .conn- .send(req, &AllShardsSelector, &RandomReplicaSelector)- .await?;-- Ok(res- .into_iter()- .flatten()- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))- .collect())- }-}--impl StreamingResponse for SiteUrlStream<live_index::LiveIndexService> {+impl StreamingResponse for SiteUrlStream {type Item = Url;async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {- let req = search_server::GetSiteUrls {- site: self.site.clone(),- offset: self.offset as u64,- limit: SITE_URL_BATCH_SIZE as u64,- };-- let res = self- .conn- .send(req, &AllShardsSelector, &RandomReplicaSelector)- .await?;-- Ok(res- .into_iter()- .flatten()- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))- .collect())+ self.searcher+ .search_generic(GetSiteUrlsQuery {+ site: self.site.clone(),+ offset: Some(self.offset as u64),+ limit: SITE_URL_BATCH_SIZE as u64,+ })+ .await+ .map(|res| res.urls)}}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 32691255..9ddf7b94 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -737,9 +737,11 @@ mod tests {index.commit().expect("failed to commit index");let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),+ None,Bangs::empty(),Config::default(),)+ .await.with_webgraph(graph);let res = searcherdiff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 593391f8..454bb0f8 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -364,9 +364,11 @@ mod tests {let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),+ None,Bangs::empty(),crate::searcher::api::Config::default(),)+ .await.with_webgraph(graph);let res = searcherdiff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 2537bdd9..96a8b0d6 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -190,9 +190,11 @@ mod tests {index.commit().expect("failed to commit index");let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),+ None,Bangs::empty(),crate::searcher::api::Config::default(),)+ .await.with_webgraph(graph);let result = searcherdiff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 7bff888e..26fc856d 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -29,6 +29,7 @@ use ahash::AHashMap as HashMap;use crate::bangs::{Bang, BangHit};use crate::collector::{self, approx_count};use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};+use crate::distributed::cluster::Cluster;use crate::enum_map::EnumMap;use crate::image_store::Image;use crate::models::dual_encoder::DualEncoder;@@ -244,7 +245,7 @@ wherepub struct ApiSearcher<S, L, G> {distributed_searcher: Arc<S>,- sidebar_manager: SidebarManager<S>,+ sidebar_manager: Option<SidebarManager>,live_searcher: Option<L>,cross_encoder: Option<Arc<CrossEncoderModel>>,lambda_model: Option<Arc<LambdaMART>>,@@ -262,14 +263,21 @@ whereL: live::SearchClient,G: Graph,{- pub fn new<C>(dist_searcher: S, bangs: Bangs, config: C) -> Self+ pub async fn new<C>(+ dist_searcher: S,+ cluster: Option<Arc<Cluster>>,+ bangs: Bangs,+ config: C,+ ) -> SelfwhereC: Into<Config>,{let config: Config = config.into();let dist_searcher = Arc::new(dist_searcher);- let sidebar_manager =- SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());+ let sidebar_manager = match cluster {+ Some(cluster) => Some(SidebarManager::new(cluster, config.thresholds.clone()).await),+ None => None,+ };let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());@@ -358,7 +366,10 @@ where}pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {- self.sidebar_manager.sidebar(query).await+ match &self.sidebar_manager {+ Some(sidebar_manager) => sidebar_manager.sidebar(query).await,+ None => None,+ }}pub fn spell_check(&self, query: &str) -> Option<HighlightedSpellCorrection> {@@ -778,9 +789,14 @@ wheremax_height: Option<u64>,max_width: Option<u64>,) -> Result<Option<Image>> {- self.distributed_searcher- .get_entity_image(image_id, max_height, max_width)- .await+ match &self.sidebar_manager {+ Some(sidebar_manager) => {+ sidebar_manager+ .get_entity_image(image_id, max_height, max_width)+ .await+ }+ None => Ok(None),+ }}pub async fn warmup(&self, queries: impl Iterator<Item = String>) {diff --git a/crates/core/src/searcher/api/sidebar.rs b/crates/core/src/searcher/api/sidebar.rsindex 50aa65aa..75e629eb 100644--- a/crates/core/src/searcher/api/sidebar.rs+++ b/crates/core/src/searcher/api/sidebar.rs@@ -14,10 +14,25 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{config::ApiThresholds, ranking::pipeline::RecallRankingWebpage, Result};+use crate::{+ config::ApiThresholds,+ distributed::{+ cluster::Cluster,+ sonic::replication::{+ AllShardsSelector, RandomReplicaSelector, ReusableShardedClient, ShardedClient,+ },+ },+ entity_index::EntityMatch,+ entrypoint::entity_search_server,+ image_store::Image,+ ranking::pipeline::RecallRankingWebpage,+ searcher::{DistributedSearcher, SearchClient},+ Result,+};use std::{cmp::Ordering, sync::Arc};use optics::Optic;+use tokio::sync::Mutex;use url::Url;use crate::{@@ -25,22 +40,72 @@ use crate::{searcher::{distributed, SearchQuery},};-pub struct SidebarManager<S> {- distributed_searcher: Arc<S>,+pub struct SidebarManager {+ distributed_searcher: DistributedSearcher,+ entity_searcher: Mutex<ReusableShardedClient<entity_search_server::SearchService>>,thresholds: ApiThresholds,}-impl<S> SidebarManager<S>-where- S: distributed::SearchClient,-{- pub fn new(distributed_searcher: Arc<S>, thresholds: ApiThresholds) -> SidebarManager<S> {+impl SidebarManager {+ pub async fn new(cluster: Arc<Cluster>, thresholds: ApiThresholds) -> Self {Self {- distributed_searcher,+ distributed_searcher: DistributedSearcher::new(cluster.clone()).await,+ entity_searcher: Mutex::new(ReusableShardedClient::new(cluster.clone()).await),thresholds,}}+ async fn entity_conn(&self) -> Arc<ShardedClient<entity_search_server::SearchService, ()>> {+ self.entity_searcher.lock().await.conn().await+ }++ pub async fn get_entity_image(+ &self,+ image_id: &str,+ max_height: Option<u64>,+ max_width: Option<u64>,+ ) -> Result<Option<Image>> {+ let client = self.entity_conn().await;++ Ok(client+ .send(+ entity_search_server::GetEntityImage {+ image_id: image_id.to_string(),+ max_height,+ max_width,+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ .map_err(|_| anyhow::anyhow!("Failed to get entity image"))?+ .into_iter()+ .flatten()+ .next()+ .and_then(|(_, mut v)| v.pop())+ .and_then(|(_, v)| v))+ }++ async fn search_entity(&self, query: &str) -> Option<EntityMatch> {+ let client = self.entity_conn().await;++ client+ .send(+ entity_search_server::Search {+ query: query.to_string(),+ },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await+ .ok()?+ .into_iter()+ .flatten()+ .next()+ .and_then(|(_, mut v)| v.pop())+ .and_then(|(_, v)| v)+ }+pub async fn stackoverflow(&self, query: &str) -> Result<Option<DisplayedSidebar>> {let query = SearchQuery {query: query.to_string(),@@ -93,10 +158,8 @@ where}pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {- let (entity, stackoverflow) = futures::join!(- self.distributed_searcher.search_entity(query),- self.stackoverflow(query)- );+ let (entity, stackoverflow) =+ futures::join!(self.search_entity(query), self.stackoverflow(query));if let Some(entity) = entity {if entity.score as f64 > self.thresholds.entity_sidebar {diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 1a44e3c2..380623bb 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -27,14 +27,12 @@ use crate::{},},},- entity_index::EntityMatch,entrypoint::{entity_search_server,live_index::LiveIndexService,search_server::{self, SearchService, RetrieveReq},},generic_query::{self, Collector},- image_store::Image,index::Index,inverted_index::{RetrievedWebpage, WebpagePointer},ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},@@ -78,15 +76,6 @@ pub trait SearchClient {query: &str,) -> impl Future<Output = Vec<(usize, PrecisionRankingWebpage)>> + Send;- fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;-- fn get_entity_image(- &self,- image_id: &str,- max_height: Option<u64>,- max_width: Option<u64>,- ) -> impl Future<Output = Result<Option<Image>>> + Send;-fn search_initial_generic<Q>(&self,query: Q,@@ -285,14 +274,16 @@ impl ReusableClientManager for LiveIndexService {pub struct DistributedSearcher {client: Mutex<ReusableShardedClient<SearchService>>,- entiy_client: Mutex<ReusableShardedClient<entity_search_server::SearchService>>,}impl DistributedSearcher {pub async fn new(cluster: Arc<Cluster>) -> Self {+ Self::from_client(ReusableShardedClient::new(cluster.clone()).await)+ }++ pub fn from_client(client: ReusableShardedClient<SearchService>) -> Self {Self {- client: Mutex::new(ReusableShardedClient::new(cluster.clone()).await),- entiy_client: Mutex::new(ReusableShardedClient::new(cluster).await),+ client: Mutex::new(client),}}@@ -300,10 +291,6 @@ impl DistributedSearcher {self.client.lock().await.conn().await}- async fn entity_conn(&self) -> Arc<ShardedClient<entity_search_server::SearchService, ()>> {- self.entiy_client.lock().await.conn().await- }-async fn retrieve_webpages_from_shard(&self,shard: ShardId,@@ -404,53 +391,6 @@ impl SearchClient for DistributedSearcher {retrieved_webpages}- async fn get_entity_image(- &self,- image_id: &str,- max_height: Option<u64>,- max_width: Option<u64>,- ) -> Result<Option<Image>> {- let client = self.entity_conn().await;-- Ok(client- .send(- entity_search_server::GetEntityImage {- image_id: image_id.to_string(),- max_height,- max_width,- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await- .map_err(|_| Error::SearchFailed)?- .into_iter()- .flatten()- .next()- .and_then(|(_, mut v)| v.pop())- .and_then(|(_, v)| v))- }-- async fn search_entity(&self, query: &str) -> Option<EntityMatch> {- let client = self.entity_conn().await;-- client- .send(- entity_search_server::Search {- query: query.to_string(),- },- &AllShardsSelector,- &RandomReplicaSelector,- )- .await- .ok()?- .into_iter()- .flatten()- .next()- .and_then(|(_, mut v)| v.pop())- .and_then(|(_, v)| v)- }-async fn search_initial_generic<Q>(&self,query: Q,@@ -673,19 +613,6 @@ impl SearchClient for LocalSearchClient {res}- async fn get_entity_image(- &self,- _image_id: &str,- _max_height: Option<u64>,- _max_width: Option<u64>,- ) -> Result<Option<Image>> {- Ok(None)- }-- async fn search_entity(&self, _query: &str) -> Option<EntityMatch> {- None- }-async fn search_initial_generic<Q>(&self,query: Q,