web search done right
[search_server] implement size query as GenericQuery
Mikkel Denker11e30b139118
diff --git a/crates/core/src/entrypoint/admin/mod.rs b/crates/core/src/entrypoint/admin/mod.rsindex 281f99f6..674666fd 100644--- a/crates/core/src/entrypoint/admin/mod.rs+++ b/crates/core/src/entrypoint/admin/mod.rs@@ -19,7 +19,7 @@ use std::{path::{Path, PathBuf},};-use crate::{distributed::sonic, entrypoint::api, Result};+use crate::{distributed::sonic, entrypoint::api, generic_query, Result};const CONFIG_FOLDER: &str = "~/.config/stract";const CONFIG_NAME: &str = "admin.toml";@@ -120,7 +120,7 @@ pub async fn index_size() -> Result<()> {let mut conn: sonic::service::Connection<api::ManagementService> =sonic::service::Connection::create(config.host).await?;- let size: api::SizeResponse = conn.send_without_timeout(api::Size).await?;+ let size: generic_query::size::SizeResponse = conn.send_without_timeout(api::Size).await?;println!("Number of pages in index: {}", size.pages);diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rsindex 7fb39f16..0e0198de 100644--- a/crates/core/src/entrypoint/api.rs+++ b/crates/core/src/entrypoint/api.rs@@ -29,14 +29,12 @@ use crate::{member::{Member, Service},sonic::{self, service::sonic_service},},- generic_query::TopKeyPhrasesQuery,+ generic_query::{self, TopKeyPhrasesQuery},inverted_index::KeyPhrase,metrics::Label,searcher::{DistributedSearcher, SearchClient},};-use super::search_server::SearchService;-fn counters(registry: &mut crate::metrics::PrometheusRegistry) -> Result<Counters> {let search_counter_success = crate::metrics::Counter::default();let search_counter_fail = crate::metrics::Counter::default();@@ -145,49 +143,14 @@ impl sonic::service::Message<ManagementService> for ClusterStatus {#[derive(Debug, Clone, Copy, bincode::Encode, bincode::Decode)]pub struct Size;-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct SizeResponse {- pub pages: u64,-}--impl std::ops::Add for SizeResponse {- type Output = Self;- fn add(mut self, other: Self) -> Self {- self += other;- self- }-}--impl std::ops::AddAssign for SizeResponse {- fn add_assign(&mut self, other: Self) {- self.pages += other.pages;- }-}-impl sonic::service::Message<ManagementService> for Size {- type Response = SizeResponse;+ type Response = generic_query::size::SizeResponse;async fn handle(self, server: &ManagementService) -> Self::Response {- let mut res = SizeResponse { pages: 0 };-- let mut checked_shards = std::collections::HashSet::new();-- for member in server.cluster.members().await {- if let Service::Searcher { host, shard } = member.service {- if checked_shards.contains(&shard) {- continue;- }-- let mut client: sonic::service::Connection<SearchService> =- sonic::service::Connection::create(host).await.unwrap();- let size = client.send_without_timeout(Size).await.unwrap();-- res += size;-- checked_shards.insert(shard);- }- }-- res+ server+ .searcher+ .search_generic(generic_query::SizeQuery)+ .await+ .unwrap()}}diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 87dee138..61b6b916 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -26,7 +26,7 @@ use crate::{member::{Member, Service},sonic::{self, service::sonic_service},},- generic_query::{self, TopKeyPhrasesQuery},+ generic_query::{self, SizeQuery, TopKeyPhrasesQuery},index::Index,inverted_index::{self, RetrievedWebpage},models::dual_encoder::DualEncoder,@@ -35,8 +35,6 @@ use crate::{Result,};-use super::api::{Size, SizeResponse};-pub trait RetrieveReq:bincode::Encode + bincode::Decode + Clone + sonic::service::Wrapper<SearchService>{@@ -118,7 +116,6 @@ macro_rules! impl_search {sonic_service!(SearchService, [RetrieveWebsites,Search,- Size,GetWebpage,GetHomepageDescriptions,GetSiteUrls,@@ -132,7 +129,7 @@ macro_rules! impl_search {}}-impl_search!([TopKeyPhrasesQuery,]);+impl_search!([TopKeyPhrasesQuery, SizeQuery,]);pub struct SearchService {local_searcher: LocalSearcher<Arc<Index>>,@@ -255,15 +252,6 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {}}-impl sonic::service::Message<SearchService> for Size {- type Response = SizeResponse;- async fn handle(self, server: &SearchService) -> Self::Response {- SizeResponse {- pages: server.local_searcher.num_documents().await,- }- }-}-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct GetSiteUrls {pub site: String,diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rsindex cdb4cf9e..35425c6b 100644--- a/crates/core/src/generic_query/collector/mod.rs+++ b/crates/core/src/generic_query/collector/mod.rs@@ -17,6 +17,9 @@mod top_key_phrases;pub use top_key_phrases::TopKeyPhrasesCollector;+mod size;+pub use size::SizeCollector;+use tantivy::{collector::{Fruit, SegmentCollector},query::Weight,diff --git a/crates/core/src/generic_query/collector/size.rs b/crates/core/src/generic_query/collector/size.rsnew file mode 100644index 00000000..91468278--- /dev/null+++ b/crates/core/src/generic_query/collector/size.rs@@ -0,0 +1,117 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use rustc_hash::FxHashMap;++use crate::{ampc::dht::ShardId, generic_query::size::SizeResponse};++use super::Collector;++pub struct SizeCollector {+ shard_id: Option<ShardId>,+}++impl Default for SizeCollector {+ fn default() -> Self {+ Self::new()+ }+}++impl SizeCollector {+ pub fn new() -> Self {+ Self { shard_id: None }+ }++ pub fn with_shard_id(mut self, shard_id: ShardId) -> Self {+ self.shard_id = Some(shard_id);+ self+ }+}++impl Collector for SizeCollector {+ type Fruit = FxHashMap<ShardId, SizeResponse>;+ type Child = SizeSegmentCollector;++ fn for_segment(+ &self,+ _: tantivy::SegmentOrdinal,+ segment: &tantivy::SegmentReader,+ ) -> crate::Result<Self::Child> {+ Ok(SizeSegmentCollector::new(self.shard_id.unwrap(), segment))+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> crate::Result<Self::Fruit> {+ let mut map = FxHashMap::default();++ for fruit in segment_fruits {+ for (shard_id, size_response) in fruit {+ map.entry(shard_id).or_insert_with(|| size_response);+ }+ }++ Ok(map)+ }++ fn collect_segment(+ &self,+ _: &dyn tantivy::query::Weight,+ segment_ord: u32,+ reader: &tantivy::SegmentReader,+ ) -> crate::Result<<Self::Child as tantivy::collector::SegmentCollector>::Fruit> {+ let child = self.for_segment(segment_ord, reader)?;+ Ok(child.fruit())+ }+}++pub struct SizeSegmentCollector {+ shard_id: ShardId,+ num_pages: u64,+}++impl SizeSegmentCollector {+ pub fn new(shard_id: ShardId, segment: &tantivy::SegmentReader) -> Self {+ Self {+ shard_id,+ num_pages: segment.num_docs() as u64,+ }+ }++ pub fn fruit(self) -> <Self as tantivy::collector::SegmentCollector>::Fruit {+ let mut map = FxHashMap::default();+ map.insert(+ self.shard_id,+ SizeResponse {+ pages: self.num_pages,+ },+ );+ map+ }+}++impl tantivy::collector::SegmentCollector for SizeSegmentCollector {+ type Fruit = FxHashMap<ShardId, SizeResponse>;++ fn collect(&mut self, _: tantivy::DocId, _: tantivy::Score) {+ unimplemented!()+ }++ fn harvest(self) -> Self::Fruit {+ unimplemented!()+ }+}diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rsindex 5ca2ed04..6b988657 100644--- a/crates/core/src/generic_query/mod.rs+++ b/crates/core/src/generic_query/mod.rs@@ -19,6 +19,9 @@ use crate::{ampc::dht::ShardId, search_ctx, Result};pub mod top_key_phrases;pub use top_key_phrases::TopKeyPhrasesQuery;+pub mod size;+pub use size::SizeQuery;+pub mod collector;pub use collector::Collector;diff --git a/crates/core/src/generic_query/size.rs b/crates/core/src/generic_query/size.rsnew file mode 100644index 00000000..872e19ae--- /dev/null+++ b/crates/core/src/generic_query/size.rs@@ -0,0 +1,105 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use rustc_hash::FxHashMap;++use crate::generic_query::{Collector, GenericQuery};++use super::collector::SizeCollector;++#[derive(+ Debug,+ Default,+ Clone,+ Copy,+ bincode::Encode,+ bincode::Decode,+ serde::Serialize,+ serde::Deserialize,+)]+pub struct SizeResponse {+ pub pages: u64,+}++impl std::ops::Add for SizeResponse {+ type Output = Self;++ fn add(self, rhs: Self) -> Self::Output {+ Self {+ pages: self.pages + rhs.pages,+ }+ }+}++impl std::ops::AddAssign for SizeResponse {+ fn add_assign(&mut self, rhs: Self) {+ self.pages += rhs.pages;+ }+}++#[derive(Debug, Default, Clone, Copy, bincode::Encode, bincode::Decode)]+pub struct SizeQuery;++impl GenericQuery for SizeQuery {+ type Collector = SizeCollector;+ type TantivyQuery = tantivy::query::EmptyQuery;+ type IntermediateOutput = SizeResponse;+ type Output = SizeResponse;++ fn tantivy_query(&self, _: &crate::search_ctx::Ctx) -> Self::TantivyQuery {+ tantivy::query::EmptyQuery+ }++ fn collector(&self, ctx: &crate::search_ctx::Ctx) -> Self::Collector {+ SizeCollector::new().with_shard_id(ctx.shard_id)+ }++ fn remote_collector(&self) -> Self::Collector {+ SizeCollector::new()+ }++ fn filter_fruit_shards(+ &self,+ shard_id: crate::ampc::dht::ShardId,+ fruit: <Self::Collector as Collector>::Fruit,+ ) -> <Self::Collector as Collector>::Fruit {+ let mut map = FxHashMap::default();++ for (fruit_shard_id, size_response) in fruit {+ if shard_id == fruit_shard_id {+ map.insert(shard_id, size_response);+ }+ }++ map+ }++ fn retrieve(+ &self,+ ctx: &crate::search_ctx::Ctx,+ fruit: <Self::Collector as Collector>::Fruit,+ ) -> crate::Result<Self::IntermediateOutput> {+ Ok(*fruit.get(&ctx.shard_id).unwrap_or(&SizeResponse::default()))+ }++ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {+ let mut res = SizeResponse { pages: 0 };+ for r in results {+ res += r;+ }+ res+ }+}