Skip to content

web search done right

[search_server[ implement GetSiteUrls as generic query

Mikkel Denker14e6e110581a
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index 2d2b85e3..cf99f3be 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -76,7 +76,9 @@ pub async fn main() {
let webgraph = Webgraph::open("data/webgraph", 0u64.into()).unwrap();
let searcher: ApiSearcher<LocalSearchClient, LiveSearcher, Webgraph> =
- ApiSearcher::new(searcher, bangs, config).with_webgraph(webgraph);
+ ApiSearcher::new(searcher, None, bangs, config)
+ .await
+ .with_webgraph(webgraph);
for query in queries {
let mut desc = "search '".to_string();
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index 03dbfc50..aca12d2f 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -209,7 +209,9 @@ pub async fn router(
}
let mut searcher =
- ApiSearcher::new(dist_searcher, bangs, config.clone()).with_live(live_searcher);
+ ApiSearcher::new(dist_searcher, Some(cluster.clone()), bangs, config.clone())
+ .await
+ .with_live(live_searcher);
if let Some(cross_encoder) = cross_encoder {
searcher = searcher.with_cross_encoder(cross_encoder);
diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rs
index 586b275e..be018477 100644
--- a/crates/core/src/distributed/sonic/replication.rs
+++ b/crates/core/src/distributed/sonic/replication.rs
@@ -511,6 +511,19 @@ where
last_client_update: std::time::Instant,
}
+impl<M> Clone for ReusableShardedClient<M>
+where
+ M: ReusableClientManager,
+{
+ fn clone(&self) -> Self {
+ Self {
+ cluster: self.cluster.clone(),
+ client: self.client.clone(),
+ last_client_update: self.last_client_update,
+ }
+ }
+}
+
impl<M> ReusableShardedClient<M>
where
M: ReusableClientManager,
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
index 2e9003cd..936f52c8 100644
--- a/crates/core/src/entrypoint/live_index/search_server.rs
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -44,7 +44,7 @@ use tracing::info;
use crate::entrypoint::{
indexer::{self, IndexableWebpage},
- search_server::{GetSiteUrls, RetrieveWebsites, Search, SiteUrls},
+ search_server::{RetrieveWebsites, Search},
};
const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);
@@ -55,7 +55,6 @@ sonic_service!(
[
RetrieveWebsites,
Search,
- GetSiteUrls,
IndexWebpages,
GetIndexPath,
RemoteDownload
@@ -351,18 +350,6 @@ impl sonic::service::Message<LiveIndexService> for Search {
}
}
-impl sonic::service::Message<LiveIndexService> for GetSiteUrls {
- type Response = SiteUrls;
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- let urls = server
- .local_searcher
- .get_site_urls(&self.site, self.offset as usize, self.limit as usize)
- .await;
-
- SiteUrls { urls }
- }
-}
-
#[derive(Clone, Debug)]
struct RemoteIndex {
host: SocketAddr,
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 135cf9a2..21582d72 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -17,7 +17,6 @@
use std::sync::Arc;
use tracing::info;
-use url::Url;
use crate::{
config,
@@ -26,7 +25,9 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
- generic_query::{self, GetHomepageQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery},
+ generic_query::{
+ self, GetHomepageQuery, GetSiteUrlsQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery,
+ },
index::Index,
inverted_index,
models::dual_encoder::DualEncoder,
@@ -116,7 +117,6 @@ macro_rules! impl_search {
sonic_service!(SearchService, [
RetrieveWebsites,
Search,
- GetSiteUrls,
$(
$q,
[<$q Retrieve>],
@@ -132,6 +132,7 @@ impl_search!([
SizeQuery,
GetWebpageQuery,
GetHomepageQuery,
+ GetSiteUrlsQuery,
]);
pub struct SearchService {
@@ -221,28 +222,3 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {
}
}
}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct GetSiteUrls {
- pub site: String,
- pub offset: u64,
- pub limit: u64,
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct SiteUrls {
- #[bincode(with_serde)]
- pub urls: Vec<Url>,
-}
-
-impl sonic::service::Message<SearchService> for GetSiteUrls {
- type Response = SiteUrls;
- async fn handle(self, server: &SearchService) -> Self::Response {
- let urls = server
- .local_searcher
- .get_site_urls(&self.site, self.offset as usize, self.limit as usize)
- .await;
-
- SiteUrls { urls }
- }
-}
diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rs
index a0b407ef..230339c5 100644
--- a/crates/core/src/generic_query/collector/mod.rs
+++ b/crates/core/src/generic_query/collector/mod.rs
@@ -23,6 +23,9 @@ pub use size::SizeCollector;
mod first_doc;
pub use first_doc::FirstDocCollector;
+mod top_docs;
+pub use top_docs::TopDocsCollector;
+
use tantivy::{
collector::{Fruit, SegmentCollector},
query::Weight,
diff --git a/crates/core/src/generic_query/collector/top_docs.rs b/crates/core/src/generic_query/collector/top_docs.rs
new file mode 100644
index 00000000..084aec58
--- /dev/null
+++ b/crates/core/src/generic_query/collector/top_docs.rs
@@ -0,0 +1,209 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{distributed::member::ShardId, inverted_index::DocAddress};
+use tantivy::{
+ collector::{SegmentCollector, TopNComputer},
+ DocId, SegmentOrdinal,
+};
+
+use super::Collector;
+
+pub struct TopDocsCollector {
+ shard_id: Option<ShardId>,
+ limit: Option<usize>,
+ offset: Option<usize>,
+ perform_offset: bool,
+}
+
+impl Default for TopDocsCollector {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl TopDocsCollector {
+ pub fn new() -> Self {
+ Self {
+ shard_id: None,
+ limit: None,
+ offset: None,
+ perform_offset: true,
+ }
+ }
+}
+
+impl TopDocsCollector {
+ pub fn with_shard_id(self, shard_id: ShardId) -> Self {
+ Self {
+ shard_id: Some(shard_id),
+ ..self
+ }
+ }
+
+ pub fn with_offset(self, offset: usize) -> Self {
+ Self {
+ offset: Some(offset),
+ ..self
+ }
+ }
+
+ pub fn with_limit(self, limit: usize) -> Self {
+ Self {
+ limit: Some(limit),
+ ..self
+ }
+ }
+
+ pub fn enable_offset(self) -> Self {
+ Self {
+ perform_offset: true,
+ ..self
+ }
+ }
+
+ pub fn disable_offset(self) -> Self {
+ Self {
+ perform_offset: false,
+ ..self
+ }
+ }
+}
+
+impl TopDocsCollector {
+ fn computer(&self) -> Computer {
+ match (self.offset, self.limit) {
+ (Some(offset), Some(limit)) => Computer::TopN(TopNComputer::new(limit + offset)),
+ (Some(_), None) => Computer::All(AllComputer::new()),
+ (None, Some(limit)) => Computer::TopN(TopNComputer::new(limit)),
+ (None, None) => Computer::All(AllComputer::new()),
+ }
+ }
+}
+
+impl Collector for TopDocsCollector {
+ type Fruit = Vec<(tantivy::Score, DocAddress)>;
+
+ type Child = TopDocsSegmentCollector;
+
+ fn for_segment(
+ &self,
+ segment_ord: SegmentOrdinal,
+ _: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ Ok(TopDocsSegmentCollector {
+ shard_id: self.shard_id.unwrap(),
+ computer: self.computer(),
+ segment_ord,
+ })
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ let mut computer = self.computer();
+
+ let fruits: Vec<_> = segment_fruits.into_iter().flatten().collect();
+
+ for (score, doc) in fruits {
+ computer.push(score, doc);
+ }
+
+ let result = computer.harvest();
+
+ if self.perform_offset {
+ Ok(result
+ .into_iter()
+ .skip(self.offset.unwrap_or(0))
+ .take(self.limit.unwrap_or(usize::MAX))
+ .collect())
+ } else {
+ Ok(result
+ .into_iter()
+ .take(self.limit.unwrap_or(usize::MAX))
+ .collect())
+ }
+ }
+}
+
+enum Computer {
+ TopN(TopNComputer<tantivy::Score, DocAddress>),
+ All(AllComputer),
+}
+
+impl Computer {
+ fn push(&mut self, score: tantivy::Score, doc: DocAddress) {
+ match self {
+ Computer::TopN(computer) => computer.push(score, doc),
+ Computer::All(computer) => computer.push(score, doc),
+ }
+ }
+
+ fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {
+ match self {
+ Computer::TopN(computer) => computer
+ .into_sorted_vec()
+ .into_iter()
+ .map(|comparable_doc| (comparable_doc.feature, comparable_doc.doc))
+ .collect(),
+ Computer::All(computer) => computer.harvest(),
+ }
+ }
+}
+
+struct AllComputer {
+ docs: Vec<(tantivy::Score, DocAddress)>,
+}
+
+impl AllComputer {
+ fn new() -> Self {
+ Self { docs: Vec::new() }
+ }
+
+ fn push(&mut self, score: tantivy::Score, doc: DocAddress) {
+ self.docs.push((score, doc));
+ }
+
+ fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {
+ let mut docs = self.docs;
+ docs.sort_by(|(score1, _), (score2, _)| score2.total_cmp(score1));
+ docs
+ }
+}
+
+pub struct TopDocsSegmentCollector {
+ shard_id: ShardId,
+ computer: Computer,
+ segment_ord: SegmentOrdinal,
+}
+
+impl SegmentCollector for TopDocsSegmentCollector {
+ type Fruit = Vec<(tantivy::Score, DocAddress)>;
+
+ fn collect(&mut self, doc: DocId, score: tantivy::Score) {
+ if doc == tantivy::TERMINATED {
+ return;
+ }
+
+ self.computer
+ .push(score, DocAddress::new(self.segment_ord, doc, self.shard_id));
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.computer.harvest()
+ }
+}
diff --git a/crates/core/src/generic_query/get_site_urls.rs b/crates/core/src/generic_query/get_site_urls.rs
new file mode 100644
index 00000000..e14510d0
--- /dev/null
+++ b/crates/core/src/generic_query/get_site_urls.rs
@@ -0,0 +1,110 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::TantivyDocument;
+use url::Url;
+
+use crate::{
+ inverted_index::RetrievedWebpage,
+ schema::text_field::{self, TextField},
+ search_ctx,
+ webpage::url_ext::UrlExt,
+};
+
+use super::{collector::TopDocsCollector, GenericQuery};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct GetSiteUrlsQuery {
+ pub site: String,
+ pub limit: u64,
+ pub offset: Option<u64>,
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct GetSiteUrlsResponse {
+ #[bincode(with_serde)]
+ pub urls: Vec<Url>,
+}
+
+impl GenericQuery for GetSiteUrlsQuery {
+ type Collector = TopDocsCollector;
+ type TantivyQuery = Box<dyn tantivy::query::Query>;
+ type IntermediateOutput = GetSiteUrlsResponse;
+ type Output = GetSiteUrlsResponse;
+
+ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery {
+ let field = ctx
+ .tv_searcher
+ .schema()
+ .get_field(text_field::SiteNoTokenizer.name())
+ .unwrap();
+ let term = tantivy::Term::from_field_text(field, &self.site);
+ Box::new(tantivy::query::TermQuery::new(
+ term,
+ tantivy::schema::IndexRecordOption::Basic,
+ ))
+ }
+
+ fn collector(&self, _: &search_ctx::Ctx) -> Self::Collector {
+ Self::Collector::new()
+ .with_limit(self.limit as usize)
+ .with_offset(self.offset.unwrap_or(0) as usize)
+ .disable_offset()
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ Self::Collector::new()
+ .with_limit(self.limit as usize)
+ .with_offset(self.offset.unwrap_or(0) as usize)
+ .enable_offset()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(_, addr)| addr.shard_id == shard_id)
+ .collect()
+ }
+
+ fn retrieve(
+ &self,
+ ctx: &search_ctx::Ctx,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ let mut urls = Vec::new();
+
+ for (_, addr) in fruit {
+ let doc: TantivyDocument = ctx.tv_searcher.doc(addr.into())?;
+ let doc = RetrievedWebpage::from(doc);
+ urls.push(Url::robust_parse(&doc.url)?);
+ }
+
+ Ok(GetSiteUrlsResponse { urls })
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut urls: Vec<_> = results.into_iter().flat_map(|r| r.urls).collect();
+
+ urls.sort_by_key(|url| url.to_string());
+ urls.dedup();
+
+ GetSiteUrlsResponse { urls }
+ }
+}
diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rs
index 2ecc5365..a5958b42 100644
--- a/crates/core/src/generic_query/mod.rs
+++ b/crates/core/src/generic_query/mod.rs
@@ -28,6 +28,9 @@ pub use get_webpage::GetWebpageQuery;
pub mod get_homepage;
pub use get_homepage::GetHomepageQuery;
+pub mod get_site_urls;
+pub use get_site_urls::GetSiteUrlsQuery;
+
pub mod collector;
pub use collector::Collector;
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 840f01d4..4d6a1ad0 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -83,6 +83,9 @@ pub struct WebpagePointer {
Clone,
Copy,
PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
)]
pub struct DocAddress {
pub segment: u32,
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index 396e2794..13db884d 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -82,10 +82,6 @@ impl Client {
self.live_index.lock().await.conn().await
}
- async fn search_conn(&self) -> Arc<ShardedClient<search_server::SearchService, ShardId>> {
- self.search.lock().await.conn().await
- }
-
pub async fn index(&self, pages: Vec<IndexableWebpage>) -> Result<()> {
let mut conn = self.live_conn().await;
@@ -112,23 +108,10 @@ impl Client {
Ok(())
}
- async fn get_site_urls_from_backbone(&self, site: &str) -> Vec<Url> {
- let mut res = Vec::new();
- let conn = self.search_conn().await;
- let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();
-
- while let Some(url) = stream.next().await {
- res.push(url);
- }
-
- res
- }
-
- async fn get_site_urls_from_live(&self, site: &str) -> Vec<Url> {
+ pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {
let mut res = Vec::new();
- let conn = self.live_conn().await;
-
- let mut stream = SiteUrlStream::new(site.to_string(), conn).stream();
+ let mut stream =
+ SiteUrlStream::new(site.to_string(), self.search.lock().await.clone()).stream();
while let Some(url) = stream.next().await {
res.push(url);
@@ -136,15 +119,6 @@ impl Client {
res
}
-
- pub async fn get_site_urls(&self, site: &str) -> Vec<Url> {
- let mut res = Vec::new();
-
- res.extend(self.get_site_urls_from_backbone(site).await);
- res.extend(self.get_site_urls_from_live(site).await);
-
- res
- }
}
impl From<LiveCrawlerConfig> for CrawlerConfig {
diff --git a/crates/core/src/live_index/crawler/site_url_stream.rs b/crates/core/src/live_index/crawler/site_url_stream.rs
index 835aa78d..f22c5454 100644
--- a/crates/core/src/live_index/crawler/site_url_stream.rs
+++ b/crates/core/src/live_index/crawler/site_url_stream.rs
@@ -16,83 +16,45 @@
const SITE_URL_BATCH_SIZE: usize = 100;
-use std::sync::Arc;
-
use url::Url;
use crate::{
- ampc::dht::ShardId,
distributed::{
- sonic::{
- replication::{AllShardsSelector, RandomReplicaSelector, ShardedClient},
- service::Service,
- },
- streaming_response::StreamingResponse,
+ sonic::replication::ReusableShardedClient, streaming_response::StreamingResponse,
},
- entrypoint::{live_index, search_server},
+ entrypoint::search_server,
+ generic_query::GetSiteUrlsQuery,
+ searcher::{DistributedSearcher, SearchClient},
Result,
};
-pub struct SiteUrlStream<S: Service> {
+pub struct SiteUrlStream {
site: String,
offset: usize,
- conn: Arc<ShardedClient<S, ShardId>>,
+ searcher: DistributedSearcher,
}
-impl<S: Service> SiteUrlStream<S> {
- pub fn new(site: String, conn: Arc<ShardedClient<S, ShardId>>) -> Self {
+impl SiteUrlStream {
+ pub fn new(site: String, conn: ReusableShardedClient<search_server::SearchService>) -> Self {
Self {
site,
offset: 0,
- conn,
+ searcher: DistributedSearcher::from_client(conn),
}
}
}
-impl StreamingResponse for SiteUrlStream<search_server::SearchService> {
- type Item = Url;
-
- async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {
- let req = search_server::GetSiteUrls {
- site: self.site.clone(),
- offset: self.offset as u64,
- limit: SITE_URL_BATCH_SIZE as u64,
- };
-
- self.offset += SITE_URL_BATCH_SIZE;
-
- let res = self
- .conn
- .send(req, &AllShardsSelector, &RandomReplicaSelector)
- .await?;
-
- Ok(res
- .into_iter()
- .flatten()
- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
- .collect())
- }
-}
-
-impl StreamingResponse for SiteUrlStream<live_index::LiveIndexService> {
+impl StreamingResponse for SiteUrlStream {
type Item = Url;
async fn next_batch(&mut self) -> Result<Vec<Self::Item>> {
- let req = search_server::GetSiteUrls {
- site: self.site.clone(),
- offset: self.offset as u64,
- limit: SITE_URL_BATCH_SIZE as u64,
- };
-
- let res = self
- .conn
- .send(req, &AllShardsSelector, &RandomReplicaSelector)
- .await?;
-
- Ok(res
- .into_iter()
- .flatten()
- .flat_map(|(_, v)| v.into_iter().flat_map(|(_, v)| v.urls))
- .collect())
+ self.searcher
+ .search_generic(GetSiteUrlsQuery {
+ site: self.site.clone(),
+ offset: Some(self.offset as u64),
+ limit: SITE_URL_BATCH_SIZE as u64,
+ })
+ .await
+ .map(|res| res.urls)
}
}
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 32691255..9ddf7b94 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -737,9 +737,11 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(
LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),
+ None,
Bangs::empty(),
Config::default(),
)
+ .await
.with_webgraph(graph);
let res = searcher
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rs
index 593391f8..454bb0f8 100644
--- a/crates/core/src/ranking/inbound_similarity.rs
+++ b/crates/core/src/ranking/inbound_similarity.rs
@@ -364,9 +364,11 @@ mod tests {
let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(
LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),
+ None,
Bangs::empty(),
crate::searcher::api::Config::default(),
)
+ .await
.with_webgraph(graph);
let res = searcher
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index 2537bdd9..96a8b0d6 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -190,9 +190,11 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(
LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),
+ None,
Bangs::empty(),
crate::searcher::api::Config::default(),
)
+ .await
.with_webgraph(graph);
let result = searcher
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 7bff888e..26fc856d 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -29,6 +29,7 @@ use ahash::AHashMap as HashMap;
use crate::bangs::{Bang, BangHit};
use crate::collector::{self, approx_count};
use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};
+use crate::distributed::cluster::Cluster;
use crate::enum_map::EnumMap;
use crate::image_store::Image;
use crate::models::dual_encoder::DualEncoder;
@@ -244,7 +245,7 @@ where
pub struct ApiSearcher<S, L, G> {
distributed_searcher: Arc<S>,
- sidebar_manager: SidebarManager<S>,
+ sidebar_manager: Option<SidebarManager>,
live_searcher: Option<L>,
cross_encoder: Option<Arc<CrossEncoderModel>>,
lambda_model: Option<Arc<LambdaMART>>,
@@ -262,14 +263,21 @@ where
L: live::SearchClient,
G: Graph,
{
- pub fn new<C>(dist_searcher: S, bangs: Bangs, config: C) -> Self
+ pub async fn new<C>(
+ dist_searcher: S,
+ cluster: Option<Arc<Cluster>>,
+ bangs: Bangs,
+ config: C,
+ ) -> Self
where
C: Into<Config>,
{
let config: Config = config.into();
let dist_searcher = Arc::new(dist_searcher);
- let sidebar_manager =
- SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());
+ let sidebar_manager = match cluster {
+ Some(cluster) => Some(SidebarManager::new(cluster, config.thresholds.clone()).await),
+ None => None,
+ };
let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());
@@ -358,7 +366,10 @@ where
}
pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {
- self.sidebar_manager.sidebar(query).await
+ match &self.sidebar_manager {
+ Some(sidebar_manager) => sidebar_manager.sidebar(query).await,
+ None => None,
+ }
}
pub fn spell_check(&self, query: &str) -> Option<HighlightedSpellCorrection> {
@@ -778,9 +789,14 @@ where
max_height: Option<u64>,
max_width: Option<u64>,
) -> Result<Option<Image>> {
- self.distributed_searcher
- .get_entity_image(image_id, max_height, max_width)
- .await
+ match &self.sidebar_manager {
+ Some(sidebar_manager) => {
+ sidebar_manager
+ .get_entity_image(image_id, max_height, max_width)
+ .await
+ }
+ None => Ok(None),
+ }
}
pub async fn warmup(&self, queries: impl Iterator<Item = String>) {
diff --git a/crates/core/src/searcher/api/sidebar.rs b/crates/core/src/searcher/api/sidebar.rs
index 50aa65aa..75e629eb 100644
--- a/crates/core/src/searcher/api/sidebar.rs
+++ b/crates/core/src/searcher/api/sidebar.rs
@@ -14,10 +14,25 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{config::ApiThresholds, ranking::pipeline::RecallRankingWebpage, Result};
+use crate::{
+ config::ApiThresholds,
+ distributed::{
+ cluster::Cluster,
+ sonic::replication::{
+ AllShardsSelector, RandomReplicaSelector, ReusableShardedClient, ShardedClient,
+ },
+ },
+ entity_index::EntityMatch,
+ entrypoint::entity_search_server,
+ image_store::Image,
+ ranking::pipeline::RecallRankingWebpage,
+ searcher::{DistributedSearcher, SearchClient},
+ Result,
+};
use std::{cmp::Ordering, sync::Arc};
use optics::Optic;
+use tokio::sync::Mutex;
use url::Url;
use crate::{
@@ -25,22 +40,72 @@ use crate::{
searcher::{distributed, SearchQuery},
};
-pub struct SidebarManager<S> {
- distributed_searcher: Arc<S>,
+pub struct SidebarManager {
+ distributed_searcher: DistributedSearcher,
+ entity_searcher: Mutex<ReusableShardedClient<entity_search_server::SearchService>>,
thresholds: ApiThresholds,
}
-impl<S> SidebarManager<S>
-where
- S: distributed::SearchClient,
-{
- pub fn new(distributed_searcher: Arc<S>, thresholds: ApiThresholds) -> SidebarManager<S> {
+impl SidebarManager {
+ pub async fn new(cluster: Arc<Cluster>, thresholds: ApiThresholds) -> Self {
Self {
- distributed_searcher,
+ distributed_searcher: DistributedSearcher::new(cluster.clone()).await,
+ entity_searcher: Mutex::new(ReusableShardedClient::new(cluster.clone()).await),
thresholds,
}
}
+ async fn entity_conn(&self) -> Arc<ShardedClient<entity_search_server::SearchService, ()>> {
+ self.entity_searcher.lock().await.conn().await
+ }
+
+ pub async fn get_entity_image(
+ &self,
+ image_id: &str,
+ max_height: Option<u64>,
+ max_width: Option<u64>,
+ ) -> Result<Option<Image>> {
+ let client = self.entity_conn().await;
+
+ Ok(client
+ .send(
+ entity_search_server::GetEntityImage {
+ image_id: image_id.to_string(),
+ max_height,
+ max_width,
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ .map_err(|_| anyhow::anyhow!("Failed to get entity image"))?
+ .into_iter()
+ .flatten()
+ .next()
+ .and_then(|(_, mut v)| v.pop())
+ .and_then(|(_, v)| v))
+ }
+
+ async fn search_entity(&self, query: &str) -> Option<EntityMatch> {
+ let client = self.entity_conn().await;
+
+ client
+ .send(
+ entity_search_server::Search {
+ query: query.to_string(),
+ },
+ &AllShardsSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ .ok()?
+ .into_iter()
+ .flatten()
+ .next()
+ .and_then(|(_, mut v)| v.pop())
+ .and_then(|(_, v)| v)
+ }
+
pub async fn stackoverflow(&self, query: &str) -> Result<Option<DisplayedSidebar>> {
let query = SearchQuery {
query: query.to_string(),
@@ -93,10 +158,8 @@ where
}
pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {
- let (entity, stackoverflow) = futures::join!(
- self.distributed_searcher.search_entity(query),
- self.stackoverflow(query)
- );
+ let (entity, stackoverflow) =
+ futures::join!(self.search_entity(query), self.stackoverflow(query));
if let Some(entity) = entity {
if entity.score as f64 > self.thresholds.entity_sidebar {
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index 1a44e3c2..380623bb 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -27,14 +27,12 @@ use crate::{
},
},
},
- entity_index::EntityMatch,
entrypoint::{
entity_search_server,
live_index::LiveIndexService,
search_server::{self, SearchService, RetrieveReq},
},
generic_query::{self, Collector},
- image_store::Image,
index::Index,
inverted_index::{RetrievedWebpage, WebpagePointer},
ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},
@@ -78,15 +76,6 @@ pub trait SearchClient {
query: &str,
) -> impl Future<Output = Vec<(usize, PrecisionRankingWebpage)>> + Send;
- fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;
-
- fn get_entity_image(
- &self,
- image_id: &str,
- max_height: Option<u64>,
- max_width: Option<u64>,
- ) -> impl Future<Output = Result<Option<Image>>> + Send;
-
fn search_initial_generic<Q>(
&self,
query: Q,
@@ -285,14 +274,16 @@ impl ReusableClientManager for LiveIndexService {
pub struct DistributedSearcher {
client: Mutex<ReusableShardedClient<SearchService>>,
- entiy_client: Mutex<ReusableShardedClient<entity_search_server::SearchService>>,
}
impl DistributedSearcher {
pub async fn new(cluster: Arc<Cluster>) -> Self {
+ Self::from_client(ReusableShardedClient::new(cluster.clone()).await)
+ }
+
+ pub fn from_client(client: ReusableShardedClient<SearchService>) -> Self {
Self {
- client: Mutex::new(ReusableShardedClient::new(cluster.clone()).await),
- entiy_client: Mutex::new(ReusableShardedClient::new(cluster).await),
+ client: Mutex::new(client),
}
}
@@ -300,10 +291,6 @@ impl DistributedSearcher {
self.client.lock().await.conn().await
}
- async fn entity_conn(&self) -> Arc<ShardedClient<entity_search_server::SearchService, ()>> {
- self.entiy_client.lock().await.conn().await
- }
-
async fn retrieve_webpages_from_shard(
&self,
shard: ShardId,
@@ -404,53 +391,6 @@ impl SearchClient for DistributedSearcher {
retrieved_webpages
}
- async fn get_entity_image(
- &self,
- image_id: &str,
- max_height: Option<u64>,
- max_width: Option<u64>,
- ) -> Result<Option<Image>> {
- let client = self.entity_conn().await;
-
- Ok(client
- .send(
- entity_search_server::GetEntityImage {
- image_id: image_id.to_string(),
- max_height,
- max_width,
- },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await
- .map_err(|_| Error::SearchFailed)?
- .into_iter()
- .flatten()
- .next()
- .and_then(|(_, mut v)| v.pop())
- .and_then(|(_, v)| v))
- }
-
- async fn search_entity(&self, query: &str) -> Option<EntityMatch> {
- let client = self.entity_conn().await;
-
- client
- .send(
- entity_search_server::Search {
- query: query.to_string(),
- },
- &AllShardsSelector,
- &RandomReplicaSelector,
- )
- .await
- .ok()?
- .into_iter()
- .flatten()
- .next()
- .and_then(|(_, mut v)| v.pop())
- .and_then(|(_, v)| v)
- }
-
async fn search_initial_generic<Q>(
&self,
query: Q,
@@ -673,19 +613,6 @@ impl SearchClient for LocalSearchClient {
res
}
- async fn get_entity_image(
- &self,
- _image_id: &str,
- _max_height: Option<u64>,
- _max_width: Option<u64>,
- ) -> Result<Option<Image>> {
- Ok(None)
- }
-
- async fn search_entity(&self, _query: &str) -> Option<EntityMatch> {
- None
- }
-
async fn search_initial_generic<Q>(
&self,
query: Q,