web search done right
[search_server] add a 'GenericQuery' trait so the search index can handle queries that are not typical search queries
Mikkel Denker1a3a9cbbb5cd
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsindex 83129dfc..2e9003cd 100644--- a/crates/core/src/entrypoint/live_index/search_server.rs+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -204,6 +204,7 @@ impl LiveIndexService {let index = Arc::new(LiveIndex::new(index_path,+ config.shard_id,indexer::worker::Config {host_centrality_store_path: config.host_centrality_store_path.clone(),page_centrality_store_path: config.page_centrality_store_path.clone(),diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rsindex e0effdc3..2ff1d4a0 100644--- a/crates/core/src/entrypoint/live_index/tests.rs+++ b/crates/core/src/entrypoint/live_index/tests.rs@@ -414,7 +414,7 @@ async fn test_meta_segments() -> Result<()> {dual_encoder: None,};- let index = LiveIndex::new(&config.index_path, indexer_config.clone()).await?;+ let index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config.clone()).await?;assert!(index.meta().await.segments().is_empty());index@@ -436,7 +436,7 @@ async fn test_meta_segments() -> Result<()> {assert_eq!(index.meta().await.segments().len(), 1);- let copy_index = LiveIndex::new(&config.index_path, indexer_config).await?;+ let copy_index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?;assert_eq!(copy_index.meta().await.segments().len(), 1);@@ -455,7 +455,8 @@ async fn test_segment_compaction() -> Result<()> {dual_encoder: None,};- let index = Arc::new(LiveIndex::new(&config.index_path, indexer_config).await?);+ let index =+ Arc::new(LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?);index.insert(&[IndexableWebpage {diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 646b277a..811353fb 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -26,6 +26,7 @@ use crate::{member::{Member, Service},sonic::{self, service::sonic_service},},+ generic_query,index::Index,inverted_index::{self, KeyPhrase, RetrievedWebpage},models::dual_encoder::DualEncoder,@@ -36,18 +37,98 @@ use crate::{use super::api::{Size, SizeResponse};-sonic_service!(- SearchService,- [- RetrieveWebsites,- Search,- GetWebpage,- GetHomepageDescriptions,- TopKeyPhrases,- Size,- GetSiteUrls,- ]-);+pub trait RetrieveReq: bincode::Encode + bincode::Decode + Clone {+ type Query: generic_query::GenericQuery + bincode::Encode + bincode::Decode;+ fn new(+ query: Self::Query,+ fruit: <<Self::Query as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit,+ ) -> Self;+}++pub trait Query+where+ Self: generic_query::GenericQuery+ + bincode::Encode+ + bincode::Decode+ + sonic::service::Wrapper<SearchService>,+{+ type RetrieveReq: RetrieveReq<Query = Self>;+}++#[derive(bincode::Encode, bincode::Decode, Clone)]+pub struct EncodedError {+ pub msg: String,+}++impl std::fmt::Display for EncodedError {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ write!(f, "{}", self.msg)+ }+}++macro_rules! impl_search {+ ([$($q:ident),*$(,)?]) => {+ $(+ impl Message<SearchService> for $q {+ type Response = Result<<<$q as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit, EncodedError>;++ async fn handle(self, server: &SearchService) -> Self::Response {+ server.local_searcher.search_initial_generic(self).await.map_err(|e| EncodedError { msg: e.to_string() })+ }+ }++ paste::item! {+ #[derive(bincode::Encode, bincode::Decode, Clone)]+ pub struct [<$q Retrieve>] {+ pub query: $q,+ #[bincode(with_serde)]+ pub fruit: <<$q as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit,+ }++ impl Message<SearchService> for [<$q Retrieve>] {+ type Response = Result<<$q as generic_query::GenericQuery>::IntermediateOutput, EncodedError>;+ async fn handle(self, server: &SearchService) -> Self::Response {+ server+ .local_searcher+ .retrieve_generic(self.query, self.fruit)+ .await+ .map_err(|e| EncodedError { msg: e.to_string() })+ }+ }++ impl Query for $q {+ type RetrieveReq = [<$q Retrieve>];+ }++ impl RetrieveReq for [<$q Retrieve>] {+ type Query = $q;+ fn new(query: Self::Query, fruit: <<Self::Query as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit) -> Self {+ Self { query, fruit }+ }+ }+ }+ )*++ paste::item! {+ sonic_service!(SearchService, [+ RetrieveWebsites,+ Search,+ GetWebpage,+ GetHomepageDescriptions,+ TopKeyPhrases,+ Size,+ GetSiteUrls,+ $(+ $q,+ [<$q Retrieve>],+ )*+ ]);+ }++ }+}++impl_search!([]);pub struct SearchService {local_searcher: LocalSearcher<Arc<Index>>,@@ -62,6 +143,7 @@ impl SearchService {search_index.inverted_index.set_snippet_config(config.snippet);+ search_index.inverted_index.set_shard_id(config.shard);let mut local_searcher = LocalSearcher::builder(Arc::new(search_index));diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 54344935..2d238fc2 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -182,7 +182,7 @@ impl Webgraph {let mut handlers = Vec::new();let graph_path = &config.graph_base_path;- const MAX_FINALIZE_CONCURRENT: usize = 8;+ const MAX_FINALIZE_CONCURRENT: usize = 2;let (s, r) = crossbeam_channel::bounded(MAX_FINALIZE_CONCURRENT);for _ in 0..MAX_FINALIZE_CONCURRENT {diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rsnew file mode 100644index 00000000..50c8f460--- /dev/null+++ b/crates/core/src/generic_query/collector/mod.rs@@ -0,0 +1,135 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::{+ collector::{Fruit, SegmentCollector},+ query::Weight,+ SegmentOrdinal, SegmentReader,+};++pub trait Collector: Sync + Send {+ type Fruit: Fruit;+ type Child: SegmentCollector;++ fn for_segment(+ &self,+ segment_local_id: SegmentOrdinal,+ segment: &SegmentReader,+ ) -> crate::Result<Self::Child>;++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,+ ) -> crate::Result<Self::Fruit>;++ fn collect_segment(+ &self,+ weight: &dyn Weight,+ segment_ord: u32,+ reader: &SegmentReader,+ ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {+ let mut segment_collector = self.for_segment(segment_ord, reader)?;++ weight.for_each_no_score(reader, &mut |docs| {+ segment_collector.collect_block(docs);+ })?;++ Ok(segment_collector.harvest())+ }+}++impl<T> Collector for T+where+ T: tantivy::collector::Collector,+{+ type Fruit = <T as tantivy::collector::Collector>::Fruit;+ type Child = <T as tantivy::collector::Collector>::Child;++ fn for_segment(+ &self,+ segment_local_id: SegmentOrdinal,+ segment: &SegmentReader,+ ) -> crate::Result<Self::Child> {+ T::for_segment(self, segment_local_id, segment).map_err(|e| anyhow::anyhow!(e))+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,+ ) -> crate::Result<Self::Fruit> {+ T::merge_fruits(self, segment_fruits).map_err(|e| anyhow::anyhow!(e))+ }++ fn collect_segment(+ &self,+ weight: &dyn Weight,+ segment_ord: u32,+ reader: &SegmentReader,+ ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {+ T::collect_segment(self, weight, segment_ord, reader).map_err(|e| anyhow::anyhow!(e))+ }+}++pub struct TantivyCollector<'a, T>(&'a T);++impl<'a, T> From<&'a T> for TantivyCollector<'a, T>+where+ T: Collector,+{+ fn from(collector: &'a T) -> Self {+ TantivyCollector(collector)+ }+}++impl<'a, T> tantivy::collector::Collector for TantivyCollector<'a, T>+where+ T: Collector,+{+ type Fruit = T::Fruit;++ type Child = T::Child;++ fn for_segment(+ &self,+ segment_local_id: SegmentOrdinal,+ segment: &SegmentReader,+ ) -> tantivy::Result<Self::Child> {+ T::for_segment(self.0, segment_local_id, segment)+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))+ }++ fn requires_scoring(&self) -> bool {+ false+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> tantivy::Result<Self::Fruit> {+ T::merge_fruits(self.0, segment_fruits)+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))+ }++ fn collect_segment(+ &self,+ weight: &dyn tantivy::query::Weight,+ segment_ord: u32,+ reader: &SegmentReader,+ ) -> tantivy::Result<<Self::Child as tantivy::collector::SegmentCollector>::Fruit> {+ T::collect_segment(self.0, weight, segment_ord, reader)+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))+ }+}diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rsnew file mode 100644index 00000000..a19dbedb--- /dev/null+++ b/crates/core/src/generic_query/mod.rs@@ -0,0 +1,45 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{ampc::dht::ShardId, search_ctx, Result};++pub mod collector;+pub use collector::Collector;++pub trait GenericQuery: Send + Sync + bincode::Encode + bincode::Decode + Clone {+ type Collector: Collector;+ type TantivyQuery: tantivy::query::Query;+ type IntermediateOutput: Send + Sync;+ type Output: Send + Sync;++ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery;+ fn collector(&self, ctx: &search_ctx::Ctx) -> Self::Collector;+ fn remote_collector(&self) -> Self::Collector;++ fn filter_fruit_shards(+ &self,+ shard_id: ShardId,+ fruit: <Self::Collector as Collector>::Fruit,+ ) -> <Self::Collector as Collector>::Fruit;++ fn retrieve(+ &self,+ ctx: &search_ctx::Ctx,+ fruit: <Self::Collector as Collector>::Fruit,+ ) -> Result<Self::IntermediateOutput>;++ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output;+}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 7347bd13..f0553d64 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -42,6 +42,7 @@ use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;use tantivy::{IndexReader, IndexWriter};+use crate::ampc::dht::ShardId;use crate::collector::{approx_count, Hashes};use crate::config::SnippetConfig;use crate::numericalfield_reader::NumericalFieldReader;@@ -140,6 +141,7 @@ pub struct InvertedIndex {schema: Arc<Schema>,snippet_config: SnippetConfig,columnfield_reader: NumericalFieldReader,+ shard_id: Option<ShardId>,}impl InvertedIndex {@@ -180,9 +182,14 @@ impl InvertedIndex {tantivy_index,snippet_config: SnippetConfig::default(),columnfield_reader,+ shard_id: None,})}+ pub fn set_shard_id(&mut self, shard_id: ShardId) {+ self.shard_id = Some(shard_id);+ }+pub fn re_open(&mut self) -> Result<()> {*self = Self::open(self.path.clone())?;Ok(())diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 6293727d..efcc020e 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -24,9 +24,11 @@ use tantivy::schema::Value;use tantivy::TantivyDocument;use url::Url;+use crate::ampc::dht::ShardId;use crate::collector::approx_count::ApproxCount;use crate::collector::{approx_count, MainCollector};+use crate::generic_query::{self, Collector as _, GenericQuery};use crate::highlighted::HighlightedFragment;use crate::numericalfield_reader::NumericalFieldReader;use crate::query::Query;@@ -97,6 +99,7 @@ impl InvertedIndex {pub fn local_search_ctx(&self) -> Ctx {let tv_searcher = self.tv_searcher();Ctx {+ shard_id: self.shard_id.unwrap_or(ShardId::new(u64::MAX)),columnfield_reader: self.columnfield_reader.clone(),tv_searcher,}@@ -347,4 +350,39 @@ impl InvertedIndex {Err(_) => vec![],}}++ pub fn search_initial_generic<Q: GenericQuery>(+ &self,+ query: &Q,+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit> {+ let ctx = self.local_search_ctx();++ let res = ctx.tv_searcher.search(+ &query.tantivy_query(&ctx),+ &generic_query::collector::TantivyCollector::from(&query.collector(&ctx)),+ )?;++ Ok(res)+ }++ pub fn retrieve_generic<Q: GenericQuery>(+ &self,+ query: &Q,+ fruit: <Q::Collector as generic_query::Collector>::Fruit,+ ) -> Result<Q::IntermediateOutput> {+ let ctx = self.local_search_ctx();+ query.retrieve(&ctx, fruit)+ }++ pub fn search_generic<Q>(&self, query: &Q) -> Result<Q::Output>+ where+ Q: GenericQuery,+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:+ From<<Q::Collector as generic_query::Collector>::Fruit>,+ {+ let fruit = self.search_initial_generic(query)?;+ let fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;+ let res = self.retrieve_generic(query, fruit)?;+ Ok(Q::merge_results(vec![res]))+ }}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex ca790d9a..a89c485a 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -97,6 +97,8 @@ pub mod webgraph;pub mod webpage;mod widgets;+pub mod generic_query;+pub use block_on::block_on;#[derive(Error, Debug)]diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex 70a53e5b..b9a04059 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -33,6 +33,7 @@ use tantivy::{use std::collections::{HashMap, HashSet};use crate::{+ ampc::dht::ShardId,config::SnippetConfig,entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},inverted_index::InvertedIndex,@@ -118,10 +119,12 @@ pub struct InnerIndex {impl InnerIndex {pub async fn new<P: AsRef<Path>>(path: P,+ shard_id: ShardId,indexer_worker_config: indexer::worker::Config,) -> Result<Self> {let mut index = crate::index::Index::open(path.as_ref())?;index.prepare_writer()?;+ index.inverted_index.set_shard_id(shard_id);let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;let wal_count = write_ahead_log.iter()?.count();@@ -354,11 +357,12 @@ pub struct LiveIndex {impl LiveIndex {pub async fn new<P: AsRef<Path>>(path: P,+ shard_id: ShardId,indexer_worker_config: indexer::worker::Config,) -> Result<Self> {Ok(Self {inner: Arc::new(RwLock::new(- InnerIndex::new(path, indexer_worker_config).await?,+ InnerIndex::new(path, shard_id, indexer_worker_config).await?,)),})}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex f7795af3..32691255 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -677,6 +677,7 @@ mod tests {"https://www.a.com/this/is/a/pattern",).unwrap(),+ host_centrality: 0.0001,fetch_time_ms: 500,node_id: Some(Node::from("www.a.com").into_host().id()),..Default::default()diff --git a/crates/core/src/search_ctx.rs b/crates/core/src/search_ctx.rsindex db45efe1..4188fb64 100644--- a/crates/core/src/search_ctx.rs+++ b/crates/core/src/search_ctx.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,10 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::numericalfield_reader::NumericalFieldReader;+use crate::{ampc::dht::ShardId, numericalfield_reader::NumericalFieldReader};#[derive(Clone)]pub struct Ctx {pub tv_searcher: tantivy::Searcher,pub columnfield_reader: NumericalFieldReader,+ pub shard_id: ShardId,}diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 0c2987a0..561505d9 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -18,18 +18,22 @@ use crate::{distributed::{cluster::Cluster,member::{LiveIndexState, Service, ShardId},- sonic::replication::{- AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,- ReusableClientManager, ReusableShardedClient, Shard, ShardIdentifier, ShardedClient,- SpecificShardSelector,+ sonic::{+ self,+ replication::{+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,+ ReusableClientManager, ReusableShardedClient, Shard, ShardIdentifier,+ ShardedClient, SpecificShardSelector,+ },},},entity_index::EntityMatch,entrypoint::{entity_search_server,live_index::LiveIndexService,- search_server::{self, SearchService},+ search_server::{self, SearchService, RetrieveReq},},+ generic_query::{self, Collector},image_store::Image,index::Index,inverted_index::{KeyPhrase, RetrievedWebpage, WebpagePointer},@@ -40,7 +44,7 @@ use crate::{use std::{collections::HashMap, sync::Arc, time::Duration};use fnv::FnvHashMap;-use futures::future::join_all;+use futures::{future::join_all, stream::FuturesUnordered, StreamExt};use itertools::Itertools;use std::future::Future;use thiserror::Error;@@ -95,6 +99,118 @@ pub trait SearchClient {) -> impl Future<Output = Result<Option<Image>>> + Send;fn top_key_phrases(&self, top_n: usize) -> impl Future<Output = Vec<KeyPhrase>> + Send;++ fn search_initial_generic<Q>(+ &self,+ query: Q,+ ) -> impl Future<Output = Result<<Q::Collector as generic_query::Collector>::Fruit>> + Send+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,+ Result<+ <Q::Collector as generic_query::Collector>::Fruit,+ search_server::EncodedError,+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:+ From<<Q::Collector as generic_query::Collector>::Fruit>;++ fn retrieve_generic<Q>(+ &self,+ query: Q,+ fruit: <Q::Collector as generic_query::Collector>::Fruit,+ ) -> impl Future<Output = Result<Vec<Q::IntermediateOutput>>> + Send+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as search_server::Query>::RetrieveReq: Send + Sync,+ <Q::Collector as generic_query::Collector>::Fruit: Clone,+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<SearchService>>::Response: Send + Sync,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,+ Result<Q::IntermediateOutput, search_server::EncodedError>:+ From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ >;++ fn search_generic<Q>(&self, query: Q) -> impl Future<Output = Result<Q::Output>> + Send+ where+ Self: Send + Sync,+ Q: search_server::Query + Send + Sync + 'static,+ <Q as search_server::Query>::RetrieveReq: Send + Sync,+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<SearchService>>::Response: Send + Sync,+ Result<+ <Q::Collector as generic_query::Collector>::Fruit,+ search_server::EncodedError,+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:+ From<<Q::Collector as generic_query::Collector>::Fruit>,+ <Q::Collector as generic_query::Collector>::Fruit: Clone,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,+ Result<Q::IntermediateOutput, search_server::EncodedError>:+ From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ >+ {+ async {+ let fruit = self.search_initial_generic(query.clone()).await?;+ let res = self.retrieve_generic(query, fruit).await?;+ let output = Q::merge_results(res);+ Ok(output)+ }+ }++ fn batch_search_initial_generic<Q>(+ &self,+ queries: Vec<Q>,+ ) -> impl Future<Output = Result<Vec<<Q::Collector as generic_query::Collector>::Fruit>>> + Send+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:+ From<<Q as sonic::service::Message<SearchService>>::Response>;++ fn batch_retrieve_generic<Q>(+ &self,+ queries: Vec<(Q, <Q::Collector as generic_query::Collector>::Fruit)>,+ ) -> impl Future<Output = Result<Vec<Vec<Q::IntermediateOutput>>>> + Send+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ >,+ <Q::Collector as generic_query::Collector>::Fruit: Clone;++ fn batch_search_generic<Q>(&self, queries: Vec<Q>) -> impl Future<Output = Result<Vec<Q::Output>>> + Send+ where+ Q: search_server::Query + Send + Sync + 'static,+ Self: Send + Sync,+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:+ From<<Q as sonic::service::Message<SearchService>>::Response>,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ >,+ <Q::Collector as generic_query::Collector>::Fruit: Clone+ {+ async {+ let res = self.batch_search_initial_generic(queries.clone()).await?;+ let res = self+ .batch_retrieve_generic(queries.into_iter().zip(res).collect())+ .await?;+ Ok(res.into_iter().map(|v| Q::merge_results(v)).collect())+ }+ }}#[derive(Clone, Debug)]@@ -115,7 +231,6 @@ impl ReusableClientManager for SearchService {const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;type Service = SearchService;-type ShardId = ShardId;async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {@@ -453,6 +568,193 @@ impl SearchClient for DistributedSearcher {}}}++ async fn search_initial_generic<Q>(+ &self,+ query: Q,+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit>+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,+ Result<+ <Q::Collector as generic_query::Collector>::Fruit,+ search_server::EncodedError,+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:+ From<<Q::Collector as generic_query::Collector>::Fruit> {+ let collector = query.remote_collector();++ let res = self.conn().await+ .send(query, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ let fruits: Vec<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit> = res+ .into_iter()+ .flatten()+ .flat_map(|(_, reps)| reps)+ .filter_map(|(_, rep)| {+ Result::<+ <Q::Collector as generic_query::Collector>::Fruit,+ search_server::EncodedError,+ >::from(rep)+ .ok()+ })+ .map(|fruit| {+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit::from(fruit)+ })+ .collect();++ collector+ .merge_fruits(fruits)+ .map_err(|_| anyhow::anyhow!("failed to merge fruits"))+ }++ async fn retrieve_generic<Q>(+ &self,+ query: Q,+ fruit: <Q::Collector as generic_query::Collector>::Fruit,+ ) -> Result<Vec<Q::IntermediateOutput>>+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as search_server::Query>::RetrieveReq: Send + Sync,+ <Q::Collector as generic_query::Collector>::Fruit: Clone,+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<SearchService>>::Response: Send + Sync,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,+ Result<Q::IntermediateOutput, search_server::EncodedError>:+ From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ >+ {+ let conn = self.conn().await;+ let mut results = FuturesUnordered::new();+ for shard in conn.shards() {+ let fruit = query.filter_fruit_shards(*shard.id(), fruit.clone());+ let req = Q::RetrieveReq::new(query.clone(), fruit);+ results.push(shard.replicas().send(req, &RandomReplicaSelector));+ }+ let mut res = Vec::new();++ while let Some(shard_res) = results.next().await {+ if let Ok(shard_res) = shard_res {+ res.push(shard_res);+ }+ }++ Ok(res+ .into_iter()+ .flatten()+ .filter_map(|(_, res)| {+ Result::<Q::IntermediateOutput, search_server::EncodedError>::from(res).ok()+ })+ .collect())+ }++ async fn batch_search_initial_generic<Q>(+ &self,+ queries: Vec<Q>,+ ) -> Result<Vec<<Q::Collector as generic_query::Collector>::Fruit>>+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:+ From<<Q as sonic::service::Message<SearchService>>::Response>,+ {+ let res = self+ .conn()+ .await+ .batch_send(&queries, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ let mut fruits = Vec::with_capacity(queries.len());++ for _ in 0..queries.len() {+ fruits.push(Vec::new());+ }++ for (_, replica_results) in res.into_iter() {+ debug_assert_eq!(replica_results.len(), 1);++ for (_, shard_results) in replica_results.into_iter() {+ for (i, shard_result) in shard_results.into_iter().enumerate() {+ if let Ok(shard_result) =+ Result::<_, search_server::EncodedError>::from(shard_result)+ {+ fruits[i].push(shard_result);+ }+ }+ }+ }++ queries+ .iter()+ .zip_eq(fruits.into_iter())+ .map(|(query, shard_fruits)| query.remote_collector().merge_fruits(shard_fruits))+ .collect::<Result<Vec<_>, _>>()+ }++ async fn batch_retrieve_generic<Q>(+ &self,+ queries: Vec<(Q, <Q::Collector as generic_query::Collector>::Fruit)>,+ ) -> Result<Vec<Vec<Q::IntermediateOutput>>>+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ >,+ <Q::Collector as generic_query::Collector>::Fruit: Clone,+ {+ let conn = self.conn().await;+ let mut results = FuturesUnordered::new();++ for shard in conn.shards() {+ let retrieve_requests: Vec<_> = queries+ .iter()+ .map(|(query, fruit)| {+ let fruit = query.filter_fruit_shards(*shard.id(), fruit.clone());+ Q::RetrieveReq::new(query.clone(), fruit)+ })+ .collect();++ results.push(async move {+ let retrieve_requests = retrieve_requests; // move lifetime+ shard+ .replicas()+ .batch_send(&retrieve_requests, &RandomReplicaSelector)+ .await+ });+ }++ let mut res = Vec::new();++ for _ in 0..queries.len() {+ res.push(Vec::new());+ }++ while let Some(shard_res) = results.next().await {+ for (_, shard_res) in shard_res? {+ assert_eq!(shard_res.len(), queries.len());++ for (i, query_res) in shard_res.into_iter().enumerate() {+ res[i].push(+ <Result<Q::IntermediateOutput, search_server::EncodedError>>::from(+ query_res,+ )+ .map_err(|e| anyhow::anyhow!("{e}"))?,+ );+ }+ }+ }++ Ok(res)+ }++}/// This should only be used for testing and benchmarks.@@ -533,4 +835,80 @@ impl SearchClient for LocalSearchClient {async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {self.0.top_key_phrases(top_n).await}++ async fn search_initial_generic<Q>(+ &self,+ query: Q,+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit>+ where+ Q: search_server::Query + 'static,+ Result<+ <Q::Collector as generic_query::Collector>::Fruit,+ search_server::EncodedError,+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:+ From<<Q::Collector as generic_query::Collector>::Fruit> {+ self.0.search_initial_generic(query).await+ }++ async fn retrieve_generic<Q>(+ &self,+ query: Q,+ fruit: <Q::Collector as generic_query::Collector>::Fruit,+ ) -> Result<Vec<Q::IntermediateOutput>>+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as search_server::Query>::RetrieveReq: Send + Sync,+ <Q::Collector as generic_query::Collector>::Fruit: Clone,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,+ Result<Q::IntermediateOutput, search_server::EncodedError>:+ From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ > {+ Ok(vec![self.0.retrieve_generic(query, fruit).await?])+ }++ async fn batch_search_initial_generic<Q>(+ &self,+ queries: Vec<Q>,+ ) -> Result<Vec<<Q::Collector as generic_query::Collector>::Fruit>>+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:+ From<<Q as sonic::service::Message<SearchService>>::Response> {+ let mut res = Vec::new();++ for query in queries {+ res.push(self.0.search_initial_generic(query).await?);+ }++ Ok(res)+ }++ async fn batch_retrieve_generic<Q>(+ &self,+ queries: Vec<(Q, <Q::Collector as generic_query::Collector>::Fruit)>,+ ) -> Result<Vec<Vec<Q::IntermediateOutput>>>+ where+ Q: search_server::Query + Send + Sync + 'static,+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<+ SearchService,+ >>::Response,+ >,+ <Q::Collector as generic_query::Collector>::Fruit: Clone,+ {+ let mut res = Vec::new();++ for (query, fruit) in queries {+ res.push(vec![self.0.retrieve_generic(query, fruit).await?]);+ }++ Ok(res)+ }}diff --git a/crates/core/src/searcher/local/guard.rs b/crates/core/src/searcher/local/guard.rsnew file mode 100644index 00000000..78a03d53--- /dev/null+++ b/crates/core/src/searcher/local/guard.rs@@ -0,0 +1,61 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;+use tokio::sync::OwnedRwLockReadGuard;++use crate::index::Index;+use crate::inverted_index::InvertedIndex;+use crate::live_index;++pub trait SearchGuard: Send + Sync {+ fn search_index(&self) -> &Index;+ fn inverted_index(&self) -> &InvertedIndex {+ &self.search_index().inverted_index+ }+}++pub struct NormalIndexSearchGuard {+ search_index: Arc<Index>,+}++impl NormalIndexSearchGuard {+ pub fn new(search_index: Arc<Index>) -> Self {+ Self { search_index }+ }+}++impl SearchGuard for NormalIndexSearchGuard {+ fn search_index(&self) -> &Index {+ self.search_index.as_ref()+ }+}++pub struct LiveIndexSearchGuard {+ lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>,+}++impl LiveIndexSearchGuard {+ pub fn new(lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>) -> Self {+ Self { lock_guard }+ }+}++impl SearchGuard for LiveIndexSearchGuard {+ fn search_index(&self) -> &Index {+ self.lock_guard.index()+ }+}diff --git a/crates/core/src/searcher/local/inner.rs b/crates/core/src/searcher/local/inner.rsnew file mode 100644index 00000000..87c49aad--- /dev/null+++ b/crates/core/src/searcher/local/inner.rs@@ -0,0 +1,199 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ generic_query::{self, GenericQuery},+ inverted_index,+ ranking::{LocalRanker, SignalComputer},+ searcher::InitialWebsiteResult,+ Result,+};+use std::sync::Arc;++use crate::{+ config::CollectorConfig, models::dual_encoder::DualEncoder, query::Query,+ ranking::models::linear::LinearRegression, search_ctx::Ctx, searcher::SearchQuery,+};++use super::{InvertedIndexResult, SearchGuard, SearchableIndex};++pub struct InnerLocalSearcher<I: SearchableIndex> {+ index: I,+ linear_regression: Option<Arc<LinearRegression>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,+}++impl<I> InnerLocalSearcher<I>+where+ I: SearchableIndex,+{+ pub fn new(index: I) -> Self {+ Self {+ index,+ linear_regression: None,+ dual_encoder: None,+ collector_config: CollectorConfig::default(),+ }+ }++ pub async fn guard(&self) -> I::SearchGuard {+ self.index.guard().await+ }++ pub fn set_linear_model(&mut self, model: LinearRegression) {+ self.linear_regression = Some(Arc::new(model));+ }++ pub fn set_dual_encoder(&mut self, dual_encoder: DualEncoder) {+ self.dual_encoder = Some(Arc::new(dual_encoder));+ }++ pub fn set_collector_config(&mut self, config: CollectorConfig) {+ self.collector_config = config;+ }++ fn parse_query<G: SearchGuard>(+ &self,+ ctx: &Ctx,+ guard: &G,+ query: &SearchQuery,+ ) -> Result<Query> {+ Query::parse(ctx, query, guard.inverted_index())+ }++ fn ranker<G: SearchGuard>(+ &self,+ query: &Query,+ guard: &G,+ de_rank_similar: bool,+ computer: SignalComputer,+ ) -> Result<LocalRanker> {+ let mut ranker = LocalRanker::new(+ computer,+ guard.inverted_index().columnfield_reader(),+ self.collector_config.clone(),+ );++ ranker.de_rank_similar(de_rank_similar);++ Ok(ranker+ .with_max_docs(+ self.collector_config.max_docs_considered,+ guard.inverted_index().num_segments(),+ )+ .with_num_results(query.num_results())+ .with_offset(query.offset()))+ }++ fn search_inverted_index<G: SearchGuard>(+ &self,+ ctx: &Ctx,+ guard: &G,+ query: &SearchQuery,+ de_rank_similar: bool,+ ) -> Result<InvertedIndexResult> {+ let parsed_query = self.parse_query(ctx, guard, query)?;++ let mut computer = SignalComputer::new(Some(&parsed_query));++ computer.set_region_count(+ guard+ .search_index()+ .region_count+ .lock()+ .unwrap_or_else(|e| e.into_inner())+ .clone(),+ );++ if let Some(model) = self.linear_regression.as_ref() {+ computer.set_linear_model(model.clone());+ }++ let ranker = self.ranker(&parsed_query, guard, de_rank_similar, computer)?;++ let res = guard.inverted_index().search_initial(+ &parsed_query,+ ctx,+ ranker.collector(ctx.clone()),+ )?;++ let columnfield_reader = guard.inverted_index().columnfield_reader();++ let ranking_websites = guard.inverted_index().retrieve_ranking_websites(+ ctx,+ res.top_websites,+ ranker.computer(),+ &columnfield_reader,+ )?;++ Ok(InvertedIndexResult {+ webpages: ranking_websites,+ num_hits: res.num_websites,+ })+ }++ pub fn search_initial(+ &self,+ query: &SearchQuery,+ guard: &I::SearchGuard,+ de_rank_similar: bool,+ ) -> Result<InitialWebsiteResult> {+ let query = query.clone();++ let ctx = guard.inverted_index().local_search_ctx();+ let inverted_index_result =+ self.search_inverted_index(&ctx, guard, &query, de_rank_similar)?;++ Ok(InitialWebsiteResult {+ websites: inverted_index_result.webpages,+ num_websites: inverted_index_result.num_hits,+ })+ }++ pub fn retrieve_websites(+ &self,+ websites: &[inverted_index::WebpagePointer],+ query: &str,+ guard: &I::SearchGuard,+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {+ let ctx = guard.inverted_index().local_search_ctx();+ let query = SearchQuery {+ query: query.to_string(),+ ..Default::default()+ };+ let query = Query::parse(&ctx, &query, guard.inverted_index())?;++ guard.inverted_index().retrieve_websites(websites, &query)+ }++ pub fn search_initial_generic<Q: GenericQuery>(+ &self,+ query: &Q,+ guard: &I::SearchGuard,+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit> {+ guard.inverted_index().search_initial_generic(query)+ }++ pub fn retrieve_generic<Q: GenericQuery>(+ &self,+ query: &Q,+ fruit: <Q::Collector as generic_query::Collector>::Fruit,+ guard: &I::SearchGuard,+ ) -> Result<Q::IntermediateOutput> {+ guard.inverted_index().retrieve_generic(query, fruit)+ }+}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local/mod.rssimilarity index 63%rename from crates/core/src/searcher/local.rsrename to crates/core/src/searcher/local/mod.rsindex e03a462c..c95036b8 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local/mod.rs@@ -14,26 +14,30 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+mod guard;+use guard::{LiveIndexSearchGuard, NormalIndexSearchGuard, SearchGuard};++mod inner;+use inner::InnerLocalSearcher;+use std::collections::HashMap;use std::future::Future;use std::sync::Arc;-use tokio::sync::OwnedRwLockReadGuard;use itertools::Itertools;use url::Url;use crate::collector::approx_count;use crate::config::CollectorConfig;+use crate::generic_query::{self, GenericQuery};use crate::index::Index;-use crate::inverted_index::{InvertedIndex, KeyPhrase, RetrievedWebpage};+use crate::inverted_index::{KeyPhrase, RetrievedWebpage};use crate::models::dual_encoder::DualEncoder;-use crate::query::Query;use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{LocalRecallRankingWebpage, PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage,};-use crate::ranking::{LocalRanker, SignalComputer, SignalEnum, SignalScore};-use crate::search_ctx::Ctx;+use crate::ranking::{SignalEnum, SignalScore};use crate::search_prettifier::DisplayedWebpage;use crate::{inverted_index, live_index, Result};@@ -46,30 +50,11 @@ pub trait SearchableIndex: Send + Sync + 'static {fn guard(&self) -> impl Future<Output = Self::SearchGuard>;}-pub trait SearchGuard: Send + Sync {- fn search_index(&self) -> &Index;- fn inverted_index(&self) -> &InvertedIndex {- &self.search_index().inverted_index- }-}-impl SearchableIndex for Arc<Index> {type SearchGuard = NormalIndexSearchGuard;async fn guard(&self) -> Self::SearchGuard {- NormalIndexSearchGuard {- search_index: self.clone(),- }- }-}--pub struct NormalIndexSearchGuard {- search_index: Arc<Index>,-}--impl SearchGuard for NormalIndexSearchGuard {- fn search_index(&self) -> &Index {- self.search_index.as_ref()+ NormalIndexSearchGuard::new(self.clone())}}@@ -77,19 +62,7 @@ impl SearchableIndex for Arc<live_index::LiveIndex> {type SearchGuard = LiveIndexSearchGuard;async fn guard(&self) -> Self::SearchGuard {- LiveIndexSearchGuard {- lock_guard: self.read().await,- }- }-}--pub struct LiveIndexSearchGuard {- lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>,-}--impl SearchGuard for LiveIndexSearchGuard {- fn search_index(&self) -> &Index {- self.lock_guard.index()+ LiveIndexSearchGuard::new(self.read().await)}}@@ -270,162 +243,34 @@ wherepub async fn num_documents(&self) -> u64 {self.inner.guard().await.inverted_index().num_documents()}-}--struct InnerLocalSearcher<I: SearchableIndex> {- index: I,- linear_regression: Option<Arc<LinearRegression>>,- dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,-}--struct InvertedIndexResult {- webpages: Vec<LocalRecallRankingWebpage>,- num_hits: approx_count::Count,-}--impl<I> InnerLocalSearcher<I>-where- I: SearchableIndex,-{- pub fn new(index: I) -> Self {- Self {- index,- linear_regression: None,- dual_encoder: None,- collector_config: CollectorConfig::default(),- }- }-- async fn guard(&self) -> I::SearchGuard {- self.index.guard().await- }-- pub fn set_linear_model(&mut self, model: LinearRegression) {- self.linear_regression = Some(Arc::new(model));- }-- pub fn set_dual_encoder(&mut self, dual_encoder: DualEncoder) {- self.dual_encoder = Some(Arc::new(dual_encoder));- }-- pub fn set_collector_config(&mut self, config: CollectorConfig) {- self.collector_config = config;- }- fn parse_query<G: SearchGuard>(+ pub async fn search_initial_generic<Q: GenericQuery + 'static>(&self,- ctx: &Ctx,- guard: &G,- query: &SearchQuery,- ) -> Result<Query> {- Query::parse(ctx, query, guard.inverted_index())- }-- fn ranker<G: SearchGuard>(- &self,- query: &Query,- guard: &G,- de_rank_similar: bool,- computer: SignalComputer,- ) -> Result<LocalRanker> {- let mut ranker = LocalRanker::new(- computer,- guard.inverted_index().columnfield_reader(),- self.collector_config.clone(),- );-- ranker.de_rank_similar(de_rank_similar);-- Ok(ranker- .with_max_docs(- self.collector_config.max_docs_considered,- guard.inverted_index().num_segments(),- )- .with_num_results(query.num_results())- .with_offset(query.offset()))- }-- fn search_inverted_index<G: SearchGuard>(- &self,- ctx: &Ctx,- guard: &G,- query: &SearchQuery,- de_rank_similar: bool,- ) -> Result<InvertedIndexResult> {- let parsed_query = self.parse_query(ctx, guard, query)?;-- let mut computer = SignalComputer::new(Some(&parsed_query));-- computer.set_region_count(- guard- .search_index()- .region_count- .lock()- .unwrap_or_else(|e| e.into_inner())- .clone(),- );-- if let Some(model) = self.linear_regression.as_ref() {- computer.set_linear_model(model.clone());- }-- let ranker = self.ranker(&parsed_query, guard, de_rank_similar, computer)?;-- let res = guard.inverted_index().search_initial(- &parsed_query,- ctx,- ranker.collector(ctx.clone()),- )?;-- let columnfield_reader = guard.inverted_index().columnfield_reader();-- let ranking_websites = guard.inverted_index().retrieve_ranking_websites(- ctx,- res.top_websites,- ranker.computer(),- &columnfield_reader,- )?;-- Ok(InvertedIndexResult {- webpages: ranking_websites,- num_hits: res.num_websites,- })+ query: Q,+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit> {+ let inner = self.inner.clone();+ let guard = inner.guard().await;+ tokio::task::spawn_blocking(move || inner.search_initial_generic(&query, &guard))+ .await+ .unwrap()}- fn search_initial(+ pub async fn retrieve_generic<Q: GenericQuery + 'static>(&self,- query: &SearchQuery,- guard: &I::SearchGuard,- de_rank_similar: bool,- ) -> Result<InitialWebsiteResult> {- let query = query.clone();-- let ctx = guard.inverted_index().local_search_ctx();- let inverted_index_result =- self.search_inverted_index(&ctx, guard, &query, de_rank_similar)?;-- Ok(InitialWebsiteResult {- websites: inverted_index_result.webpages,- num_websites: inverted_index_result.num_hits,- })+ query: Q,+ fruit: <Q::Collector as generic_query::Collector>::Fruit,+ ) -> Result<Q::IntermediateOutput> {+ let inner = self.inner.clone();+ let guard = inner.guard().await;+ tokio::task::spawn_blocking(move || inner.retrieve_generic(&query, fruit, &guard))+ .await+ .unwrap()}+}- fn retrieve_websites(- &self,- websites: &[inverted_index::WebpagePointer],- query: &str,- guard: &I::SearchGuard,- ) -> Result<Vec<inverted_index::RetrievedWebpage>> {- let ctx = guard.inverted_index().local_search_ctx();- let query = SearchQuery {- query: query.to_string(),- ..Default::default()- };- let query = Query::parse(&ctx, &query, guard.inverted_index())?;-- guard.inverted_index().retrieve_websites(websites, &query)- }+struct InvertedIndexResult {+ webpages: Vec<LocalRecallRankingWebpage>,+ num_hits: approx_count::Count,}#[cfg(test)]diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex f2c32c07..2e2e4caf 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -279,20 +279,34 @@ impl EdgeStore {pub fn iter_pages_small(&self) -> impl Iterator<Item = SmallEdge> + '_ {let searcher = self.reader.searcher();let segment_readers: Vec<_> = searcher.segment_readers().to_vec();-- segment_readers.into_iter().flat_map(|segment| {- SmallSegmentEdgesIter::new(&segment, FromId, ToId, 0..segment.max_doc())+ let warm_column_fields = self.warmed_column_fields.clone();++ segment_readers.into_iter().flat_map(move |segment| {+ SmallSegmentEdgesIter::new(+ &segment,+ &warm_column_fields,+ FromId,+ ToId,+ 0..segment.max_doc(),+ )})}pub fn iter_hosts_small(&self) -> impl Iterator<Item = SmallEdge> + '_ {let searcher = self.reader.searcher();let segment_readers: Vec<_> = searcher.segment_readers().to_vec();+ let warm_column_fields = self.warmed_column_fields.clone();segment_readers.into_iter()- .flat_map(|segment| {- SmallSegmentEdgesIter::new(&segment, FromHostId, ToHostId, 0..segment.max_doc())+ .flat_map(move |segment| {+ SmallSegmentEdgesIter::new(+ &segment,+ &warm_column_fields,+ FromHostId,+ ToHostId,+ 0..segment.max_doc(),+ )}).unique_by(|e| (e.from, e.to))}@@ -300,13 +314,20 @@ impl EdgeStore {pub fn iter_page_node_ids(&self, offset: u32, limit: u32) -> impl Iterator<Item = NodeID> + '_ {let searcher = self.reader.searcher();let segment_readers: Vec<_> = searcher.segment_readers().to_vec();+ let warm_column_fields = self.warmed_column_fields.clone();let range = offset..limit;segment_readers.into_iter().flat_map(move |segment| {- SmallSegmentEdgesIter::new(&segment, FromId, ToId, range.clone())+ SmallSegmentEdgesIter::new(+ &segment,+ &warm_column_fields,+ FromId,+ ToId,+ range.clone(),+ )}).flat_map(|e| [e.from, e.to]).unique()@@ -315,13 +336,19 @@ impl EdgeStore {pub fn iter_host_node_ids(&self, offset: u32, limit: u32) -> impl Iterator<Item = NodeID> + '_ {let searcher = self.reader.searcher();let segment_readers: Vec<_> = searcher.segment_readers().to_vec();-+ let warm_column_fields = self.warmed_column_fields.clone();let range = offset..limit;segment_readers.into_iter().flat_map(move |segment| {- SmallSegmentEdgesIter::new(&segment, FromHostId, ToHostId, range.clone())+ SmallSegmentEdgesIter::new(+ &segment,+ &warm_column_fields,+ FromHostId,+ ToHostId,+ range.clone(),+ )}).flat_map(|e| [e.from, e.to]).unique()@@ -339,6 +366,7 @@ pub struct SmallSegmentEdgesIter {impl SmallSegmentEdgesIter {fn new<F1, F2>(segment: &SegmentReader,+ warm_column_fields: &WarmedColumnFields,from_id: F1,to_id: F2,mut doc_range: Range<DocId>,@@ -347,16 +375,16 @@ impl SmallSegmentEdgesIter {F1: Field,F2: Field,{- let columns = segment.column_fields();+ let columns = warm_column_fields.segment(&segment.segment_id());if doc_range.end > segment.max_doc() {doc_range.end = segment.max_doc();}Self {- from_id: columns.u64(from_id.name()).unwrap(),- to_id: columns.u64(to_id.name()).unwrap(),- rel_flags: columns.u64(schema::RelFlags.name()).unwrap(),+ from_id: columns.u64(from_id).unwrap(),+ to_id: columns.u64(to_id).unwrap(),+ rel_flags: columns.u64(schema::RelFlags).unwrap(),current_doc: doc_range.start,doc_range,}