Skip to content

web search done right

[search_server] implement size query as GenericQuery

Mikkel Denker11e30b139118
diff --git a/crates/core/src/entrypoint/admin/mod.rs b/crates/core/src/entrypoint/admin/mod.rs
index 281f99f6..674666fd 100644
--- a/crates/core/src/entrypoint/admin/mod.rs
+++ b/crates/core/src/entrypoint/admin/mod.rs
@@ -19,7 +19,7 @@ use std::{
path::{Path, PathBuf},
};
-use crate::{distributed::sonic, entrypoint::api, Result};
+use crate::{distributed::sonic, entrypoint::api, generic_query, Result};
const CONFIG_FOLDER: &str = "~/.config/stract";
const CONFIG_NAME: &str = "admin.toml";
@@ -120,7 +120,7 @@ pub async fn index_size() -> Result<()> {
let mut conn: sonic::service::Connection<api::ManagementService> =
sonic::service::Connection::create(config.host).await?;
- let size: api::SizeResponse = conn.send_without_timeout(api::Size).await?;
+ let size: generic_query::size::SizeResponse = conn.send_without_timeout(api::Size).await?;
println!("Number of pages in index: {}", size.pages);
diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rs
index 7fb39f16..0e0198de 100644
--- a/crates/core/src/entrypoint/api.rs
+++ b/crates/core/src/entrypoint/api.rs
@@ -29,14 +29,12 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
- generic_query::TopKeyPhrasesQuery,
+ generic_query::{self, TopKeyPhrasesQuery},
inverted_index::KeyPhrase,
metrics::Label,
searcher::{DistributedSearcher, SearchClient},
};
-use super::search_server::SearchService;
-
fn counters(registry: &mut crate::metrics::PrometheusRegistry) -> Result<Counters> {
let search_counter_success = crate::metrics::Counter::default();
let search_counter_fail = crate::metrics::Counter::default();
@@ -145,49 +143,14 @@ impl sonic::service::Message<ManagementService> for ClusterStatus {
#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]
pub struct Size;
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct SizeResponse {
- pub pages: u64,
-}
-
-impl std::ops::Add for SizeResponse {
- type Output = Self;
- fn add(mut self, other: Self) -> Self {
- self += other;
- self
- }
-}
-
-impl std::ops::AddAssign for SizeResponse {
- fn add_assign(&mut self, other: Self) {
- self.pages += other.pages;
- }
-}
-
impl sonic::service::Message<ManagementService> for Size {
- type Response = SizeResponse;
+ type Response = generic_query::size::SizeResponse;
async fn handle(self, server: &ManagementService) -> Self::Response {
- let mut res = SizeResponse { pages: 0 };
-
- let mut checked_shards = std::collections::HashSet::new();
-
- for member in server.cluster.members().await {
- if let Service::Searcher { host, shard } = member.service {
- if checked_shards.contains(&shard) {
- continue;
- }
-
- let mut client: sonic::service::Connection<SearchService> =
- sonic::service::Connection::create(host).await.unwrap();
- let size = client.send_without_timeout(Size).await.unwrap();
-
- res += size;
-
- checked_shards.insert(shard);
- }
- }
-
- res
+ server
+ .searcher
+ .search_generic(generic_query::SizeQuery)
+ .await
+ .unwrap()
}
}
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 87dee138..61b6b916 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -26,7 +26,7 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
- generic_query::{self, TopKeyPhrasesQuery},
+ generic_query::{self, SizeQuery, TopKeyPhrasesQuery},
index::Index,
inverted_index::{self, RetrievedWebpage},
models::dual_encoder::DualEncoder,
@@ -35,8 +35,6 @@ use crate::{
Result,
};
-use super::api::{Size, SizeResponse};
-
pub trait RetrieveReq:
bincode::Encode + bincode::Decode + Clone + sonic::service::Wrapper<SearchService>
{
@@ -118,7 +116,6 @@ macro_rules! impl_search {
sonic_service!(SearchService, [
RetrieveWebsites,
Search,
- Size,
GetWebpage,
GetHomepageDescriptions,
GetSiteUrls,
@@ -132,7 +129,7 @@ macro_rules! impl_search {
}
}
-impl_search!([TopKeyPhrasesQuery,]);
+impl_search!([TopKeyPhrasesQuery, SizeQuery,]);
pub struct SearchService {
local_searcher: LocalSearcher<Arc<Index>>,
@@ -255,15 +252,6 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {
}
}
-impl sonic::service::Message<SearchService> for Size {
- type Response = SizeResponse;
- async fn handle(self, server: &SearchService) -> Self::Response {
- SizeResponse {
- pages: server.local_searcher.num_documents().await,
- }
- }
-}
-
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct GetSiteUrls {
pub site: String,
diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rs
index cdb4cf9e..35425c6b 100644
--- a/crates/core/src/generic_query/collector/mod.rs
+++ b/crates/core/src/generic_query/collector/mod.rs
@@ -17,6 +17,9 @@
mod top_key_phrases;
pub use top_key_phrases::TopKeyPhrasesCollector;
+mod size;
+pub use size::SizeCollector;
+
use tantivy::{
collector::{Fruit, SegmentCollector},
query::Weight,
diff --git a/crates/core/src/generic_query/collector/size.rs b/crates/core/src/generic_query/collector/size.rs
new file mode 100644
index 00000000..91468278
--- /dev/null
+++ b/crates/core/src/generic_query/collector/size.rs
@@ -0,0 +1,117 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rustc_hash::FxHashMap;
+
+use crate::{ampc::dht::ShardId, generic_query::size::SizeResponse};
+
+use super::Collector;
+
+pub struct SizeCollector {
+ shard_id: Option<ShardId>,
+}
+
+impl Default for SizeCollector {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SizeCollector {
+ pub fn new() -> Self {
+ Self { shard_id: None }
+ }
+
+ pub fn with_shard_id(mut self, shard_id: ShardId) -> Self {
+ self.shard_id = Some(shard_id);
+ self
+ }
+}
+
+impl Collector for SizeCollector {
+ type Fruit = FxHashMap<ShardId, SizeResponse>;
+ type Child = SizeSegmentCollector;
+
+ fn for_segment(
+ &self,
+ _: tantivy::SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ Ok(SizeSegmentCollector::new(self.shard_id.unwrap(), segment))
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ let mut map = FxHashMap::default();
+
+ for fruit in segment_fruits {
+ for (shard_id, size_response) in fruit {
+ map.entry(shard_id).or_insert_with(|| size_response);
+ }
+ }
+
+ Ok(map)
+ }
+
+ fn collect_segment(
+ &self,
+ _: &dyn tantivy::query::Weight,
+ segment_ord: u32,
+ reader: &tantivy::SegmentReader,
+ ) -> crate::Result<<Self::Child as tantivy::collector::SegmentCollector>::Fruit> {
+ let child = self.for_segment(segment_ord, reader)?;
+ Ok(child.fruit())
+ }
+}
+
+pub struct SizeSegmentCollector {
+ shard_id: ShardId,
+ num_pages: u64,
+}
+
+impl SizeSegmentCollector {
+ pub fn new(shard_id: ShardId, segment: &tantivy::SegmentReader) -> Self {
+ Self {
+ shard_id,
+ num_pages: segment.num_docs() as u64,
+ }
+ }
+
+ pub fn fruit(self) -> <Self as tantivy::collector::SegmentCollector>::Fruit {
+ let mut map = FxHashMap::default();
+ map.insert(
+ self.shard_id,
+ SizeResponse {
+ pages: self.num_pages,
+ },
+ );
+ map
+ }
+}
+
+impl tantivy::collector::SegmentCollector for SizeSegmentCollector {
+ type Fruit = FxHashMap<ShardId, SizeResponse>;
+
+ fn collect(&mut self, _: tantivy::DocId, _: tantivy::Score) {
+ unimplemented!()
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ unimplemented!()
+ }
+}
diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rs
index 5ca2ed04..6b988657 100644
--- a/crates/core/src/generic_query/mod.rs
+++ b/crates/core/src/generic_query/mod.rs
@@ -19,6 +19,9 @@ use crate::{ampc::dht::ShardId, search_ctx, Result};
pub mod top_key_phrases;
pub use top_key_phrases::TopKeyPhrasesQuery;
+pub mod size;
+pub use size::SizeQuery;
+
pub mod collector;
pub use collector::Collector;
diff --git a/crates/core/src/generic_query/size.rs b/crates/core/src/generic_query/size.rs
new file mode 100644
index 00000000..872e19ae
--- /dev/null
+++ b/crates/core/src/generic_query/size.rs
@@ -0,0 +1,105 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rustc_hash::FxHashMap;
+
+use crate::generic_query::{Collector, GenericQuery};
+
+use super::collector::SizeCollector;
+
+#[derive(
+ Debug,
+ Default,
+ Clone,
+ Copy,
+ bincode::Encode,
+ bincode::Decode,
+ serde::Serialize,
+ serde::Deserialize,
+)]
+pub struct SizeResponse {
+ pub pages: u64,
+}
+
+impl std::ops::Add for SizeResponse {
+ type Output = Self;
+
+ fn add(self, rhs: Self) -> Self::Output {
+ Self {
+ pages: self.pages + rhs.pages,
+ }
+ }
+}
+
+impl std::ops::AddAssign for SizeResponse {
+ fn add_assign(&mut self, rhs: Self) {
+ self.pages += rhs.pages;
+ }
+}
+
+#[derive(Debug, Default, Clone, Copy, bincode::Encode, bincode::Decode)]
+pub struct SizeQuery;
+
+impl GenericQuery for SizeQuery {
+ type Collector = SizeCollector;
+ type TantivyQuery = tantivy::query::EmptyQuery;
+ type IntermediateOutput = SizeResponse;
+ type Output = SizeResponse;
+
+ fn tantivy_query(&self, _: &crate::search_ctx::Ctx) -> Self::TantivyQuery {
+ tantivy::query::EmptyQuery
+ }
+
+ fn collector(&self, ctx: &crate::search_ctx::Ctx) -> Self::Collector {
+ SizeCollector::new().with_shard_id(ctx.shard_id)
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ SizeCollector::new()
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as Collector>::Fruit,
+ ) -> <Self::Collector as Collector>::Fruit {
+ let mut map = FxHashMap::default();
+
+ for (fruit_shard_id, size_response) in fruit {
+ if shard_id == fruit_shard_id {
+ map.insert(shard_id, size_response);
+ }
+ }
+
+ map
+ }
+
+ fn retrieve(
+ &self,
+ ctx: &crate::search_ctx::Ctx,
+ fruit: <Self::Collector as Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ Ok(*fruit.get(&ctx.shard_id).unwrap_or(&SizeResponse::default()))
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut res = SizeResponse { pages: 0 };
+ for r in results {
+ res += r;
+ }
+ res
+ }
+}