Skip to content

web search done right

[search_server] add a 'GenericQuery' trait so the search index can handle queries that are not typical search queries

Mikkel Denker1a3a9cbbb5cd
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
index 83129dfc..2e9003cd 100644
--- a/crates/core/src/entrypoint/live_index/search_server.rs
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -204,6 +204,7 @@ impl LiveIndexService {
let index = Arc::new(
LiveIndex::new(
index_path,
+ config.shard_id,
indexer::worker::Config {
host_centrality_store_path: config.host_centrality_store_path.clone(),
page_centrality_store_path: config.page_centrality_store_path.clone(),
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
index e0effdc3..2ff1d4a0 100644
--- a/crates/core/src/entrypoint/live_index/tests.rs
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -414,7 +414,7 @@ async fn test_meta_segments() -> Result<()> {
dual_encoder: None,
};
- let index = LiveIndex::new(&config.index_path, indexer_config.clone()).await?;
+ let index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config.clone()).await?;
assert!(index.meta().await.segments().is_empty());
index
@@ -436,7 +436,7 @@ async fn test_meta_segments() -> Result<()> {
assert_eq!(index.meta().await.segments().len(), 1);
- let copy_index = LiveIndex::new(&config.index_path, indexer_config).await?;
+ let copy_index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?;
assert_eq!(copy_index.meta().await.segments().len(), 1);
@@ -455,7 +455,8 @@ async fn test_segment_compaction() -> Result<()> {
dual_encoder: None,
};
- let index = Arc::new(LiveIndex::new(&config.index_path, indexer_config).await?);
+ let index =
+ Arc::new(LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?);
index
.insert(&[IndexableWebpage {
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 646b277a..811353fb 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -26,6 +26,7 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
+ generic_query,
index::Index,
inverted_index::{self, KeyPhrase, RetrievedWebpage},
models::dual_encoder::DualEncoder,
@@ -36,18 +37,98 @@ use crate::{
use super::api::{Size, SizeResponse};
-sonic_service!(
- SearchService,
- [
- RetrieveWebsites,
- Search,
- GetWebpage,
- GetHomepageDescriptions,
- TopKeyPhrases,
- Size,
- GetSiteUrls,
- ]
-);
+pub trait RetrieveReq: bincode::Encode + bincode::Decode + Clone {
+ type Query: generic_query::GenericQuery + bincode::Encode + bincode::Decode;
+ fn new(
+ query: Self::Query,
+ fruit: <<Self::Query as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit,
+ ) -> Self;
+}
+
+pub trait Query
+where
+ Self: generic_query::GenericQuery
+ + bincode::Encode
+ + bincode::Decode
+ + sonic::service::Wrapper<SearchService>,
+{
+ type RetrieveReq: RetrieveReq<Query = Self>;
+}
+
+#[derive(bincode::Encode, bincode::Decode, Clone)]
+pub struct EncodedError {
+ pub msg: String,
+}
+
+impl std::fmt::Display for EncodedError {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ write!(f, "{}", self.msg)
+ }
+}
+
+macro_rules! impl_search {
+ ([$($q:ident),*$(,)?]) => {
+ $(
+ impl Message<SearchService> for $q {
+ type Response = Result<<<$q as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit, EncodedError>;
+
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server.local_searcher.search_initial_generic(self).await.map_err(|e| EncodedError { msg: e.to_string() })
+ }
+ }
+
+ paste::item! {
+ #[derive(bincode::Encode, bincode::Decode, Clone)]
+ pub struct [<$q Retrieve>] {
+ pub query: $q,
+ #[bincode(with_serde)]
+ pub fruit: <<$q as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit,
+ }
+
+ impl Message<SearchService> for [<$q Retrieve>] {
+ type Response = Result<<$q as generic_query::GenericQuery>::IntermediateOutput, EncodedError>;
+ async fn handle(self, server: &SearchService) -> Self::Response {
+ server
+ .local_searcher
+ .retrieve_generic(self.query, self.fruit)
+ .await
+ .map_err(|e| EncodedError { msg: e.to_string() })
+ }
+ }
+
+ impl Query for $q {
+ type RetrieveReq = [<$q Retrieve>];
+ }
+
+ impl RetrieveReq for [<$q Retrieve>] {
+ type Query = $q;
+ fn new(query: Self::Query, fruit: <<Self::Query as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit) -> Self {
+ Self { query, fruit }
+ }
+ }
+ }
+ )*
+
+ paste::item! {
+ sonic_service!(SearchService, [
+ RetrieveWebsites,
+ Search,
+ GetWebpage,
+ GetHomepageDescriptions,
+ TopKeyPhrases,
+ Size,
+ GetSiteUrls,
+ $(
+ $q,
+ [<$q Retrieve>],
+ )*
+ ]);
+ }
+
+ }
+}
+
+impl_search!([]);
pub struct SearchService {
local_searcher: LocalSearcher<Arc<Index>>,
@@ -62,6 +143,7 @@ impl SearchService {
search_index
.inverted_index
.set_snippet_config(config.snippet);
+ search_index.inverted_index.set_shard_id(config.shard);
let mut local_searcher = LocalSearcher::builder(Arc::new(search_index));
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 54344935..2d238fc2 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -182,7 +182,7 @@ impl Webgraph {
let mut handlers = Vec::new();
let graph_path = &config.graph_base_path;
- const MAX_FINALIZE_CONCURRENT: usize = 8;
+ const MAX_FINALIZE_CONCURRENT: usize = 2;
let (s, r) = crossbeam_channel::bounded(MAX_FINALIZE_CONCURRENT);
for _ in 0..MAX_FINALIZE_CONCURRENT {
diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rs
new file mode 100644
index 00000000..50c8f460
--- /dev/null
+++ b/crates/core/src/generic_query/collector/mod.rs
@@ -0,0 +1,135 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{
+ collector::{Fruit, SegmentCollector},
+ query::Weight,
+ SegmentOrdinal, SegmentReader,
+};
+
+pub trait Collector: Sync + Send {
+ type Fruit: Fruit;
+ type Child: SegmentCollector;
+
+ fn for_segment(
+ &self,
+ segment_local_id: SegmentOrdinal,
+ segment: &SegmentReader,
+ ) -> crate::Result<Self::Child>;
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit>;
+
+ fn collect_segment(
+ &self,
+ weight: &dyn Weight,
+ segment_ord: u32,
+ reader: &SegmentReader,
+ ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
+ let mut segment_collector = self.for_segment(segment_ord, reader)?;
+
+ weight.for_each_no_score(reader, &mut |docs| {
+ segment_collector.collect_block(docs);
+ })?;
+
+ Ok(segment_collector.harvest())
+ }
+}
+
+impl<T> Collector for T
+where
+ T: tantivy::collector::Collector,
+{
+ type Fruit = <T as tantivy::collector::Collector>::Fruit;
+ type Child = <T as tantivy::collector::Collector>::Child;
+
+ fn for_segment(
+ &self,
+ segment_local_id: SegmentOrdinal,
+ segment: &SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ T::for_segment(self, segment_local_id, segment).map_err(|e| anyhow::anyhow!(e))
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ T::merge_fruits(self, segment_fruits).map_err(|e| anyhow::anyhow!(e))
+ }
+
+ fn collect_segment(
+ &self,
+ weight: &dyn Weight,
+ segment_ord: u32,
+ reader: &SegmentReader,
+ ) -> crate::Result<<Self::Child as SegmentCollector>::Fruit> {
+ T::collect_segment(self, weight, segment_ord, reader).map_err(|e| anyhow::anyhow!(e))
+ }
+}
+
+pub struct TantivyCollector<'a, T>(&'a T);
+
+impl<'a, T> From<&'a T> for TantivyCollector<'a, T>
+where
+ T: Collector,
+{
+ fn from(collector: &'a T) -> Self {
+ TantivyCollector(collector)
+ }
+}
+
+impl<'a, T> tantivy::collector::Collector for TantivyCollector<'a, T>
+where
+ T: Collector,
+{
+ type Fruit = T::Fruit;
+
+ type Child = T::Child;
+
+ fn for_segment(
+ &self,
+ segment_local_id: SegmentOrdinal,
+ segment: &SegmentReader,
+ ) -> tantivy::Result<Self::Child> {
+ T::for_segment(self.0, segment_local_id, segment)
+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))
+ }
+
+ fn requires_scoring(&self) -> bool {
+ false
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> tantivy::Result<Self::Fruit> {
+ T::merge_fruits(self.0, segment_fruits)
+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))
+ }
+
+ fn collect_segment(
+ &self,
+ weight: &dyn tantivy::query::Weight,
+ segment_ord: u32,
+ reader: &SegmentReader,
+ ) -> tantivy::Result<<Self::Child as tantivy::collector::SegmentCollector>::Fruit> {
+ T::collect_segment(self.0, weight, segment_ord, reader)
+ .map_err(|e| tantivy::TantivyError::InternalError(e.to_string()))
+ }
+}
diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rs
new file mode 100644
index 00000000..a19dbedb
--- /dev/null
+++ b/crates/core/src/generic_query/mod.rs
@@ -0,0 +1,45 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{ampc::dht::ShardId, search_ctx, Result};
+
+pub mod collector;
+pub use collector::Collector;
+
+pub trait GenericQuery: Send + Sync + bincode::Encode + bincode::Decode + Clone {
+ type Collector: Collector;
+ type TantivyQuery: tantivy::query::Query;
+ type IntermediateOutput: Send + Sync;
+ type Output: Send + Sync;
+
+ fn tantivy_query(&self, ctx: &search_ctx::Ctx) -> Self::TantivyQuery;
+ fn collector(&self, ctx: &search_ctx::Ctx) -> Self::Collector;
+ fn remote_collector(&self) -> Self::Collector;
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: ShardId,
+ fruit: <Self::Collector as Collector>::Fruit,
+ ) -> <Self::Collector as Collector>::Fruit;
+
+ fn retrieve(
+ &self,
+ ctx: &search_ctx::Ctx,
+ fruit: <Self::Collector as Collector>::Fruit,
+ ) -> Result<Self::IntermediateOutput>;
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output;
+}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 7347bd13..f0553d64 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -42,6 +42,7 @@ use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
use tantivy::{IndexReader, IndexWriter};
+use crate::ampc::dht::ShardId;
use crate::collector::{approx_count, Hashes};
use crate::config::SnippetConfig;
use crate::numericalfield_reader::NumericalFieldReader;
@@ -140,6 +141,7 @@ pub struct InvertedIndex {
schema: Arc<Schema>,
snippet_config: SnippetConfig,
columnfield_reader: NumericalFieldReader,
+ shard_id: Option<ShardId>,
}
impl InvertedIndex {
@@ -180,9 +182,14 @@ impl InvertedIndex {
tantivy_index,
snippet_config: SnippetConfig::default(),
columnfield_reader,
+ shard_id: None,
})
}
+ pub fn set_shard_id(&mut self, shard_id: ShardId) {
+ self.shard_id = Some(shard_id);
+ }
+
pub fn re_open(&mut self) -> Result<()> {
*self = Self::open(self.path.clone())?;
Ok(())
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 6293727d..efcc020e 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -24,9 +24,11 @@ use tantivy::schema::Value;
use tantivy::TantivyDocument;
use url::Url;
+use crate::ampc::dht::ShardId;
use crate::collector::approx_count::ApproxCount;
use crate::collector::{approx_count, MainCollector};
+use crate::generic_query::{self, Collector as _, GenericQuery};
use crate::highlighted::HighlightedFragment;
use crate::numericalfield_reader::NumericalFieldReader;
use crate::query::Query;
@@ -97,6 +99,7 @@ impl InvertedIndex {
pub fn local_search_ctx(&self) -> Ctx {
let tv_searcher = self.tv_searcher();
Ctx {
+ shard_id: self.shard_id.unwrap_or(ShardId::new(u64::MAX)),
columnfield_reader: self.columnfield_reader.clone(),
tv_searcher,
}
@@ -347,4 +350,39 @@ impl InvertedIndex {
Err(_) => vec![],
}
}
+
+ pub fn search_initial_generic<Q: GenericQuery>(
+ &self,
+ query: &Q,
+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit> {
+ let ctx = self.local_search_ctx();
+
+ let res = ctx.tv_searcher.search(
+ &query.tantivy_query(&ctx),
+ &generic_query::collector::TantivyCollector::from(&query.collector(&ctx)),
+ )?;
+
+ Ok(res)
+ }
+
+ pub fn retrieve_generic<Q: GenericQuery>(
+ &self,
+ query: &Q,
+ fruit: <Q::Collector as generic_query::Collector>::Fruit,
+ ) -> Result<Q::IntermediateOutput> {
+ let ctx = self.local_search_ctx();
+ query.retrieve(&ctx, fruit)
+ }
+
+ pub fn search_generic<Q>(&self, query: &Q) -> Result<Q::Output>
+ where
+ Q: GenericQuery,
+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as generic_query::Collector>::Fruit>,
+ {
+ let fruit = self.search_initial_generic(query)?;
+ let fruit = query.remote_collector().merge_fruits(vec![fruit.into()])?;
+ let res = self.retrieve_generic(query, fruit)?;
+ Ok(Q::merge_results(vec![res]))
+ }
}
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index ca790d9a..a89c485a 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -97,6 +97,8 @@ pub mod webgraph;
pub mod webpage;
mod widgets;
+pub mod generic_query;
+
pub use block_on::block_on;
#[derive(Error, Debug)]
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index 70a53e5b..b9a04059 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -33,6 +33,7 @@ use tantivy::{
use std::collections::{HashMap, HashSet};
use crate::{
+ ampc::dht::ShardId,
config::SnippetConfig,
entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},
inverted_index::InvertedIndex,
@@ -118,10 +119,12 @@ pub struct InnerIndex {
impl InnerIndex {
pub async fn new<P: AsRef<Path>>(
path: P,
+ shard_id: ShardId,
indexer_worker_config: indexer::worker::Config,
) -> Result<Self> {
let mut index = crate::index::Index::open(path.as_ref())?;
index.prepare_writer()?;
+ index.inverted_index.set_shard_id(shard_id);
let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;
let wal_count = write_ahead_log.iter()?.count();
@@ -354,11 +357,12 @@ pub struct LiveIndex {
impl LiveIndex {
pub async fn new<P: AsRef<Path>>(
path: P,
+ shard_id: ShardId,
indexer_worker_config: indexer::worker::Config,
) -> Result<Self> {
Ok(Self {
inner: Arc::new(RwLock::new(
- InnerIndex::new(path, indexer_worker_config).await?,
+ InnerIndex::new(path, shard_id, indexer_worker_config).await?,
)),
})
}
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index f7795af3..32691255 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -677,6 +677,7 @@ mod tests {
"https://www.a.com/this/is/a/pattern",
)
.unwrap(),
+ host_centrality: 0.0001,
fetch_time_ms: 500,
node_id: Some(Node::from("www.a.com").into_host().id()),
..Default::default()
diff --git a/crates/core/src/search_ctx.rs b/crates/core/src/search_ctx.rs
index db45efe1..4188fb64 100644
--- a/crates/core/src/search_ctx.rs
+++ b/crates/core/src/search_ctx.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -14,10 +14,11 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::numericalfield_reader::NumericalFieldReader;
+use crate::{ampc::dht::ShardId, numericalfield_reader::NumericalFieldReader};
#[derive(Clone)]
pub struct Ctx {
pub tv_searcher: tantivy::Searcher,
pub columnfield_reader: NumericalFieldReader,
+ pub shard_id: ShardId,
}
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index 0c2987a0..561505d9 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -18,18 +18,22 @@ use crate::{
distributed::{
cluster::Cluster,
member::{LiveIndexState, Service, ShardId},
- sonic::replication::{
- AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,
- ReusableClientManager, ReusableShardedClient, Shard, ShardIdentifier, ShardedClient,
- SpecificShardSelector,
+ sonic::{
+ self,
+ replication::{
+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,
+ ReusableClientManager, ReusableShardedClient, Shard, ShardIdentifier,
+ ShardedClient, SpecificShardSelector,
+ },
},
},
entity_index::EntityMatch,
entrypoint::{
entity_search_server,
live_index::LiveIndexService,
- search_server::{self, SearchService},
+ search_server::{self, SearchService, RetrieveReq},
},
+ generic_query::{self, Collector},
image_store::Image,
index::Index,
inverted_index::{KeyPhrase, RetrievedWebpage, WebpagePointer},
@@ -40,7 +44,7 @@ use crate::{
use std::{collections::HashMap, sync::Arc, time::Duration};
use fnv::FnvHashMap;
-use futures::future::join_all;
+use futures::{future::join_all, stream::FuturesUnordered, StreamExt};
use itertools::Itertools;
use std::future::Future;
use thiserror::Error;
@@ -95,6 +99,118 @@ pub trait SearchClient {
) -> impl Future<Output = Result<Option<Image>>> + Send;
fn top_key_phrases(&self, top_n: usize) -> impl Future<Output = Vec<KeyPhrase>> + Send;
+
+ fn search_initial_generic<Q>(
+ &self,
+ query: Q,
+ ) -> impl Future<Output = Result<<Q::Collector as generic_query::Collector>::Fruit>> + Send
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ Result<
+ <Q::Collector as generic_query::Collector>::Fruit,
+ search_server::EncodedError,
+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,
+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as generic_query::Collector>::Fruit>;
+
+ fn retrieve_generic<Q>(
+ &self,
+ query: Q,
+ fruit: <Q::Collector as generic_query::Collector>::Fruit,
+ ) -> impl Future<Output = Result<Vec<Q::IntermediateOutput>>> + Send
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as search_server::Query>::RetrieveReq: Send + Sync,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone,
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,
+ Result<Q::IntermediateOutput, search_server::EncodedError>:
+ From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ >;
+
+ fn search_generic<Q>(&self, query: Q) -> impl Future<Output = Result<Q::Output>> + Send
+ where
+ Self: Send + Sync,
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as search_server::Query>::RetrieveReq: Send + Sync,
+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ Result<
+ <Q::Collector as generic_query::Collector>::Fruit,
+ search_server::EncodedError,
+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,
+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as generic_query::Collector>::Fruit>,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,
+ Result<Q::IntermediateOutput, search_server::EncodedError>:
+ From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ >
+ {
+ async {
+ let fruit = self.search_initial_generic(query.clone()).await?;
+ let res = self.retrieve_generic(query, fruit).await?;
+ let output = Q::merge_results(res);
+ Ok(output)
+ }
+ }
+
+ fn batch_search_initial_generic<Q>(
+ &self,
+ queries: Vec<Q>,
+ ) -> impl Future<Output = Result<Vec<<Q::Collector as generic_query::Collector>::Fruit>>> + Send
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:
+ From<<Q as sonic::service::Message<SearchService>>::Response>;
+
+ fn batch_retrieve_generic<Q>(
+ &self,
+ queries: Vec<(Q, <Q::Collector as generic_query::Collector>::Fruit)>,
+ ) -> impl Future<Output = Result<Vec<Vec<Q::IntermediateOutput>>>> + Send
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,
+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ >,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone;
+
+ fn batch_search_generic<Q>(&self, queries: Vec<Q>) -> impl Future<Output = Result<Vec<Q::Output>>> + Send
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ Self: Send + Sync,
+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:
+ From<<Q as sonic::service::Message<SearchService>>::Response>,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,
+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ >,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone
+ {
+ async {
+ let res = self.batch_search_initial_generic(queries.clone()).await?;
+ let res = self
+ .batch_retrieve_generic(queries.into_iter().zip(res).collect())
+ .await?;
+ Ok(res.into_iter().map(|v| Q::merge_results(v)).collect())
+ }
+ }
}
#[derive(Clone, Debug)]
@@ -115,7 +231,6 @@ impl ReusableClientManager for SearchService {
const CLIENT_REFRESH_INTERVAL: std::time::Duration = CLIENT_REFRESH_INTERVAL;
type Service = SearchService;
-
type ShardId = ShardId;
async fn new_client(cluster: &Cluster) -> ShardedClient<Self::Service, Self::ShardId> {
@@ -453,6 +568,193 @@ impl SearchClient for DistributedSearcher {
}
}
}
+
+ async fn search_initial_generic<Q>(
+ &self,
+ query: Q,
+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit>
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ Result<
+ <Q::Collector as generic_query::Collector>::Fruit,
+ search_server::EncodedError,
+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,
+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as generic_query::Collector>::Fruit> {
+ let collector = query.remote_collector();
+
+ let res = self.conn().await
+ .send(query, &AllShardsSelector, &RandomReplicaSelector)
+ .await?;
+
+ let fruits: Vec<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit> = res
+ .into_iter()
+ .flatten()
+ .flat_map(|(_, reps)| reps)
+ .filter_map(|(_, rep)| {
+ Result::<
+ <Q::Collector as generic_query::Collector>::Fruit,
+ search_server::EncodedError,
+ >::from(rep)
+ .ok()
+ })
+ .map(|fruit| {
+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit::from(fruit)
+ })
+ .collect();
+
+ collector
+ .merge_fruits(fruits)
+ .map_err(|_| anyhow::anyhow!("failed to merge fruits"))
+ }
+
+ async fn retrieve_generic<Q>(
+ &self,
+ query: Q,
+ fruit: <Q::Collector as generic_query::Collector>::Fruit,
+ ) -> Result<Vec<Q::IntermediateOutput>>
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as search_server::Query>::RetrieveReq: Send + Sync,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone,
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,
+ Result<Q::IntermediateOutput, search_server::EncodedError>:
+ From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ >
+ {
+ let conn = self.conn().await;
+ let mut results = FuturesUnordered::new();
+ for shard in conn.shards() {
+ let fruit = query.filter_fruit_shards(*shard.id(), fruit.clone());
+ let req = Q::RetrieveReq::new(query.clone(), fruit);
+ results.push(shard.replicas().send(req, &RandomReplicaSelector));
+ }
+ let mut res = Vec::new();
+
+ while let Some(shard_res) = results.next().await {
+ if let Ok(shard_res) = shard_res {
+ res.push(shard_res);
+ }
+ }
+
+ Ok(res
+ .into_iter()
+ .flatten()
+ .filter_map(|(_, res)| {
+ Result::<Q::IntermediateOutput, search_server::EncodedError>::from(res).ok()
+ })
+ .collect())
+ }
+
+ async fn batch_search_initial_generic<Q>(
+ &self,
+ queries: Vec<Q>,
+ ) -> Result<Vec<<Q::Collector as generic_query::Collector>::Fruit>>
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:
+ From<<Q as sonic::service::Message<SearchService>>::Response>,
+ {
+ let res = self
+ .conn()
+ .await
+ .batch_send(&queries, &AllShardsSelector, &RandomReplicaSelector)
+ .await?;
+
+ let mut fruits = Vec::with_capacity(queries.len());
+
+ for _ in 0..queries.len() {
+ fruits.push(Vec::new());
+ }
+
+ for (_, replica_results) in res.into_iter() {
+ debug_assert_eq!(replica_results.len(), 1);
+
+ for (_, shard_results) in replica_results.into_iter() {
+ for (i, shard_result) in shard_results.into_iter().enumerate() {
+ if let Ok(shard_result) =
+ Result::<_, search_server::EncodedError>::from(shard_result)
+ {
+ fruits[i].push(shard_result);
+ }
+ }
+ }
+ }
+
+ queries
+ .iter()
+ .zip_eq(fruits.into_iter())
+ .map(|(query, shard_fruits)| query.remote_collector().merge_fruits(shard_fruits))
+ .collect::<Result<Vec<_>, _>>()
+ }
+
+ async fn batch_retrieve_generic<Q>(
+ &self,
+ queries: Vec<(Q, <Q::Collector as generic_query::Collector>::Fruit)>,
+ ) -> Result<Vec<Vec<Q::IntermediateOutput>>>
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,
+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ >,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone,
+ {
+ let conn = self.conn().await;
+ let mut results = FuturesUnordered::new();
+
+ for shard in conn.shards() {
+ let retrieve_requests: Vec<_> = queries
+ .iter()
+ .map(|(query, fruit)| {
+ let fruit = query.filter_fruit_shards(*shard.id(), fruit.clone());
+ Q::RetrieveReq::new(query.clone(), fruit)
+ })
+ .collect();
+
+ results.push(async move {
+ let retrieve_requests = retrieve_requests; // move lifetime
+ shard
+ .replicas()
+ .batch_send(&retrieve_requests, &RandomReplicaSelector)
+ .await
+ });
+ }
+
+ let mut res = Vec::new();
+
+ for _ in 0..queries.len() {
+ res.push(Vec::new());
+ }
+
+ while let Some(shard_res) = results.next().await {
+ for (_, shard_res) in shard_res? {
+ assert_eq!(shard_res.len(), queries.len());
+
+ for (i, query_res) in shard_res.into_iter().enumerate() {
+ res[i].push(
+ <Result<Q::IntermediateOutput, search_server::EncodedError>>::from(
+ query_res,
+ )
+ .map_err(|e| anyhow::anyhow!("{e}"))?,
+ );
+ }
+ }
+ }
+
+ Ok(res)
+ }
+
+
}
/// This should only be used for testing and benchmarks.
@@ -533,4 +835,80 @@ impl SearchClient for LocalSearchClient {
async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
self.0.top_key_phrases(top_n).await
}
+
+ async fn search_initial_generic<Q>(
+ &self,
+ query: Q,
+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit>
+ where
+ Q: search_server::Query + 'static,
+ Result<
+ <Q::Collector as generic_query::Collector>::Fruit,
+ search_server::EncodedError,
+ >: From<<Q as sonic::service::Message<SearchService>>::Response>,
+ <<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit:
+ From<<Q::Collector as generic_query::Collector>::Fruit> {
+ self.0.search_initial_generic(query).await
+ }
+
+ async fn retrieve_generic<Q>(
+ &self,
+ query: Q,
+ fruit: <Q::Collector as generic_query::Collector>::Fruit,
+ ) -> Result<Vec<Q::IntermediateOutput>>
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as search_server::Query>::RetrieveReq: Send + Sync,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService>,
+ Result<Q::IntermediateOutput, search_server::EncodedError>:
+ From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ > {
+ Ok(vec![self.0.retrieve_generic(query, fruit).await?])
+ }
+
+ async fn batch_search_initial_generic<Q>(
+ &self,
+ queries: Vec<Q>,
+ ) -> Result<Vec<<Q::Collector as generic_query::Collector>::Fruit>>
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as sonic::service::Message<SearchService>>::Response: Send + Sync,
+ Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:
+ From<<Q as sonic::service::Message<SearchService>>::Response> {
+ let mut res = Vec::new();
+
+ for query in queries {
+ res.push(self.0.search_initial_generic(query).await?);
+ }
+
+ Ok(res)
+ }
+
+ async fn batch_retrieve_generic<Q>(
+ &self,
+ queries: Vec<(Q, <Q::Collector as generic_query::Collector>::Fruit)>,
+ ) -> Result<Vec<Vec<Q::IntermediateOutput>>>
+ where
+ Q: search_server::Query + Send + Sync + 'static,
+ <Q as search_server::Query>::RetrieveReq: sonic::service::Wrapper<SearchService> + Send + Sync,
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<search_server::SearchService>>::Response: Send + Sync,
+ Result<Q::IntermediateOutput, search_server::EncodedError>: From<
+ <<Q as search_server::Query>::RetrieveReq as sonic::service::Message<
+ SearchService,
+ >>::Response,
+ >,
+ <Q::Collector as generic_query::Collector>::Fruit: Clone,
+ {
+ let mut res = Vec::new();
+
+ for (query, fruit) in queries {
+ res.push(vec![self.0.retrieve_generic(query, fruit).await?]);
+ }
+
+ Ok(res)
+ }
}
diff --git a/crates/core/src/searcher/local/guard.rs b/crates/core/src/searcher/local/guard.rs
new file mode 100644
index 00000000..78a03d53
--- /dev/null
+++ b/crates/core/src/searcher/local/guard.rs
@@ -0,0 +1,61 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+use tokio::sync::OwnedRwLockReadGuard;
+
+use crate::index::Index;
+use crate::inverted_index::InvertedIndex;
+use crate::live_index;
+
+pub trait SearchGuard: Send + Sync {
+ fn search_index(&self) -> &Index;
+ fn inverted_index(&self) -> &InvertedIndex {
+ &self.search_index().inverted_index
+ }
+}
+
+pub struct NormalIndexSearchGuard {
+ search_index: Arc<Index>,
+}
+
+impl NormalIndexSearchGuard {
+ pub fn new(search_index: Arc<Index>) -> Self {
+ Self { search_index }
+ }
+}
+
+impl SearchGuard for NormalIndexSearchGuard {
+ fn search_index(&self) -> &Index {
+ self.search_index.as_ref()
+ }
+}
+
+pub struct LiveIndexSearchGuard {
+ lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>,
+}
+
+impl LiveIndexSearchGuard {
+ pub fn new(lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>) -> Self {
+ Self { lock_guard }
+ }
+}
+
+impl SearchGuard for LiveIndexSearchGuard {
+ fn search_index(&self) -> &Index {
+ self.lock_guard.index()
+ }
+}
diff --git a/crates/core/src/searcher/local/inner.rs b/crates/core/src/searcher/local/inner.rs
new file mode 100644
index 00000000..87c49aad
--- /dev/null
+++ b/crates/core/src/searcher/local/inner.rs
@@ -0,0 +1,199 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ generic_query::{self, GenericQuery},
+ inverted_index,
+ ranking::{LocalRanker, SignalComputer},
+ searcher::InitialWebsiteResult,
+ Result,
+};
+use std::sync::Arc;
+
+use crate::{
+ config::CollectorConfig, models::dual_encoder::DualEncoder, query::Query,
+ ranking::models::linear::LinearRegression, search_ctx::Ctx, searcher::SearchQuery,
+};
+
+use super::{InvertedIndexResult, SearchGuard, SearchableIndex};
+
+pub struct InnerLocalSearcher<I: SearchableIndex> {
+ index: I,
+ linear_regression: Option<Arc<LinearRegression>>,
+ dual_encoder: Option<Arc<DualEncoder>>,
+ collector_config: CollectorConfig,
+}
+
+impl<I> InnerLocalSearcher<I>
+where
+ I: SearchableIndex,
+{
+ pub fn new(index: I) -> Self {
+ Self {
+ index,
+ linear_regression: None,
+ dual_encoder: None,
+ collector_config: CollectorConfig::default(),
+ }
+ }
+
+ pub async fn guard(&self) -> I::SearchGuard {
+ self.index.guard().await
+ }
+
+ pub fn set_linear_model(&mut self, model: LinearRegression) {
+ self.linear_regression = Some(Arc::new(model));
+ }
+
+ pub fn set_dual_encoder(&mut self, dual_encoder: DualEncoder) {
+ self.dual_encoder = Some(Arc::new(dual_encoder));
+ }
+
+ pub fn set_collector_config(&mut self, config: CollectorConfig) {
+ self.collector_config = config;
+ }
+
+ fn parse_query<G: SearchGuard>(
+ &self,
+ ctx: &Ctx,
+ guard: &G,
+ query: &SearchQuery,
+ ) -> Result<Query> {
+ Query::parse(ctx, query, guard.inverted_index())
+ }
+
+ fn ranker<G: SearchGuard>(
+ &self,
+ query: &Query,
+ guard: &G,
+ de_rank_similar: bool,
+ computer: SignalComputer,
+ ) -> Result<LocalRanker> {
+ let mut ranker = LocalRanker::new(
+ computer,
+ guard.inverted_index().columnfield_reader(),
+ self.collector_config.clone(),
+ );
+
+ ranker.de_rank_similar(de_rank_similar);
+
+ Ok(ranker
+ .with_max_docs(
+ self.collector_config.max_docs_considered,
+ guard.inverted_index().num_segments(),
+ )
+ .with_num_results(query.num_results())
+ .with_offset(query.offset()))
+ }
+
+ fn search_inverted_index<G: SearchGuard>(
+ &self,
+ ctx: &Ctx,
+ guard: &G,
+ query: &SearchQuery,
+ de_rank_similar: bool,
+ ) -> Result<InvertedIndexResult> {
+ let parsed_query = self.parse_query(ctx, guard, query)?;
+
+ let mut computer = SignalComputer::new(Some(&parsed_query));
+
+ computer.set_region_count(
+ guard
+ .search_index()
+ .region_count
+ .lock()
+ .unwrap_or_else(|e| e.into_inner())
+ .clone(),
+ );
+
+ if let Some(model) = self.linear_regression.as_ref() {
+ computer.set_linear_model(model.clone());
+ }
+
+ let ranker = self.ranker(&parsed_query, guard, de_rank_similar, computer)?;
+
+ let res = guard.inverted_index().search_initial(
+ &parsed_query,
+ ctx,
+ ranker.collector(ctx.clone()),
+ )?;
+
+ let columnfield_reader = guard.inverted_index().columnfield_reader();
+
+ let ranking_websites = guard.inverted_index().retrieve_ranking_websites(
+ ctx,
+ res.top_websites,
+ ranker.computer(),
+ &columnfield_reader,
+ )?;
+
+ Ok(InvertedIndexResult {
+ webpages: ranking_websites,
+ num_hits: res.num_websites,
+ })
+ }
+
+ pub fn search_initial(
+ &self,
+ query: &SearchQuery,
+ guard: &I::SearchGuard,
+ de_rank_similar: bool,
+ ) -> Result<InitialWebsiteResult> {
+ let query = query.clone();
+
+ let ctx = guard.inverted_index().local_search_ctx();
+ let inverted_index_result =
+ self.search_inverted_index(&ctx, guard, &query, de_rank_similar)?;
+
+ Ok(InitialWebsiteResult {
+ websites: inverted_index_result.webpages,
+ num_websites: inverted_index_result.num_hits,
+ })
+ }
+
+ pub fn retrieve_websites(
+ &self,
+ websites: &[inverted_index::WebpagePointer],
+ query: &str,
+ guard: &I::SearchGuard,
+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {
+ let ctx = guard.inverted_index().local_search_ctx();
+ let query = SearchQuery {
+ query: query.to_string(),
+ ..Default::default()
+ };
+ let query = Query::parse(&ctx, &query, guard.inverted_index())?;
+
+ guard.inverted_index().retrieve_websites(websites, &query)
+ }
+
+ pub fn search_initial_generic<Q: GenericQuery>(
+ &self,
+ query: &Q,
+ guard: &I::SearchGuard,
+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit> {
+ guard.inverted_index().search_initial_generic(query)
+ }
+
+ pub fn retrieve_generic<Q: GenericQuery>(
+ &self,
+ query: &Q,
+ fruit: <Q::Collector as generic_query::Collector>::Fruit,
+ guard: &I::SearchGuard,
+ ) -> Result<Q::IntermediateOutput> {
+ guard.inverted_index().retrieve_generic(query, fruit)
+ }
+}
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local/mod.rs
similarity index 63%
rename from crates/core/src/searcher/local.rs
rename to crates/core/src/searcher/local/mod.rs
index e03a462c..c95036b8 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local/mod.rs
@@ -14,26 +14,30 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod guard;
+use guard::{LiveIndexSearchGuard, NormalIndexSearchGuard, SearchGuard};
+
+mod inner;
+use inner::InnerLocalSearcher;
+
use std::collections::HashMap;
use std::future::Future;
use std::sync::Arc;
-use tokio::sync::OwnedRwLockReadGuard;
use itertools::Itertools;
use url::Url;
use crate::collector::approx_count;
use crate::config::CollectorConfig;
+use crate::generic_query::{self, GenericQuery};
use crate::index::Index;
-use crate::inverted_index::{InvertedIndex, KeyPhrase, RetrievedWebpage};
+use crate::inverted_index::{KeyPhrase, RetrievedWebpage};
use crate::models::dual_encoder::DualEncoder;
-use crate::query::Query;
use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{
LocalRecallRankingWebpage, PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage,
};
-use crate::ranking::{LocalRanker, SignalComputer, SignalEnum, SignalScore};
-use crate::search_ctx::Ctx;
+use crate::ranking::{SignalEnum, SignalScore};
use crate::search_prettifier::DisplayedWebpage;
use crate::{inverted_index, live_index, Result};
@@ -46,30 +50,11 @@ pub trait SearchableIndex: Send + Sync + 'static {
fn guard(&self) -> impl Future<Output = Self::SearchGuard>;
}
-pub trait SearchGuard: Send + Sync {
- fn search_index(&self) -> &Index;
- fn inverted_index(&self) -> &InvertedIndex {
- &self.search_index().inverted_index
- }
-}
-
impl SearchableIndex for Arc<Index> {
type SearchGuard = NormalIndexSearchGuard;
async fn guard(&self) -> Self::SearchGuard {
- NormalIndexSearchGuard {
- search_index: self.clone(),
- }
- }
-}
-
-pub struct NormalIndexSearchGuard {
- search_index: Arc<Index>,
-}
-
-impl SearchGuard for NormalIndexSearchGuard {
- fn search_index(&self) -> &Index {
- self.search_index.as_ref()
+ NormalIndexSearchGuard::new(self.clone())
}
}
@@ -77,19 +62,7 @@ impl SearchableIndex for Arc<live_index::LiveIndex> {
type SearchGuard = LiveIndexSearchGuard;
async fn guard(&self) -> Self::SearchGuard {
- LiveIndexSearchGuard {
- lock_guard: self.read().await,
- }
- }
-}
-
-pub struct LiveIndexSearchGuard {
- lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>,
-}
-
-impl SearchGuard for LiveIndexSearchGuard {
- fn search_index(&self) -> &Index {
- self.lock_guard.index()
+ LiveIndexSearchGuard::new(self.read().await)
}
}
@@ -270,162 +243,34 @@ where
pub async fn num_documents(&self) -> u64 {
self.inner.guard().await.inverted_index().num_documents()
}
-}
-
-struct InnerLocalSearcher<I: SearchableIndex> {
- index: I,
- linear_regression: Option<Arc<LinearRegression>>,
- dual_encoder: Option<Arc<DualEncoder>>,
- collector_config: CollectorConfig,
-}
-
-struct InvertedIndexResult {
- webpages: Vec<LocalRecallRankingWebpage>,
- num_hits: approx_count::Count,
-}
-
-impl<I> InnerLocalSearcher<I>
-where
- I: SearchableIndex,
-{
- pub fn new(index: I) -> Self {
- Self {
- index,
- linear_regression: None,
- dual_encoder: None,
- collector_config: CollectorConfig::default(),
- }
- }
-
- async fn guard(&self) -> I::SearchGuard {
- self.index.guard().await
- }
-
- pub fn set_linear_model(&mut self, model: LinearRegression) {
- self.linear_regression = Some(Arc::new(model));
- }
-
- pub fn set_dual_encoder(&mut self, dual_encoder: DualEncoder) {
- self.dual_encoder = Some(Arc::new(dual_encoder));
- }
-
- pub fn set_collector_config(&mut self, config: CollectorConfig) {
- self.collector_config = config;
- }
- fn parse_query<G: SearchGuard>(
+ pub async fn search_initial_generic<Q: GenericQuery + 'static>(
&self,
- ctx: &Ctx,
- guard: &G,
- query: &SearchQuery,
- ) -> Result<Query> {
- Query::parse(ctx, query, guard.inverted_index())
- }
-
- fn ranker<G: SearchGuard>(
- &self,
- query: &Query,
- guard: &G,
- de_rank_similar: bool,
- computer: SignalComputer,
- ) -> Result<LocalRanker> {
- let mut ranker = LocalRanker::new(
- computer,
- guard.inverted_index().columnfield_reader(),
- self.collector_config.clone(),
- );
-
- ranker.de_rank_similar(de_rank_similar);
-
- Ok(ranker
- .with_max_docs(
- self.collector_config.max_docs_considered,
- guard.inverted_index().num_segments(),
- )
- .with_num_results(query.num_results())
- .with_offset(query.offset()))
- }
-
- fn search_inverted_index<G: SearchGuard>(
- &self,
- ctx: &Ctx,
- guard: &G,
- query: &SearchQuery,
- de_rank_similar: bool,
- ) -> Result<InvertedIndexResult> {
- let parsed_query = self.parse_query(ctx, guard, query)?;
-
- let mut computer = SignalComputer::new(Some(&parsed_query));
-
- computer.set_region_count(
- guard
- .search_index()
- .region_count
- .lock()
- .unwrap_or_else(|e| e.into_inner())
- .clone(),
- );
-
- if let Some(model) = self.linear_regression.as_ref() {
- computer.set_linear_model(model.clone());
- }
-
- let ranker = self.ranker(&parsed_query, guard, de_rank_similar, computer)?;
-
- let res = guard.inverted_index().search_initial(
- &parsed_query,
- ctx,
- ranker.collector(ctx.clone()),
- )?;
-
- let columnfield_reader = guard.inverted_index().columnfield_reader();
-
- let ranking_websites = guard.inverted_index().retrieve_ranking_websites(
- ctx,
- res.top_websites,
- ranker.computer(),
- &columnfield_reader,
- )?;
-
- Ok(InvertedIndexResult {
- webpages: ranking_websites,
- num_hits: res.num_websites,
- })
+ query: Q,
+ ) -> Result<<Q::Collector as generic_query::Collector>::Fruit> {
+ let inner = self.inner.clone();
+ let guard = inner.guard().await;
+ tokio::task::spawn_blocking(move || inner.search_initial_generic(&query, &guard))
+ .await
+ .unwrap()
}
- fn search_initial(
+ pub async fn retrieve_generic<Q: GenericQuery + 'static>(
&self,
- query: &SearchQuery,
- guard: &I::SearchGuard,
- de_rank_similar: bool,
- ) -> Result<InitialWebsiteResult> {
- let query = query.clone();
-
- let ctx = guard.inverted_index().local_search_ctx();
- let inverted_index_result =
- self.search_inverted_index(&ctx, guard, &query, de_rank_similar)?;
-
- Ok(InitialWebsiteResult {
- websites: inverted_index_result.webpages,
- num_websites: inverted_index_result.num_hits,
- })
+ query: Q,
+ fruit: <Q::Collector as generic_query::Collector>::Fruit,
+ ) -> Result<Q::IntermediateOutput> {
+ let inner = self.inner.clone();
+ let guard = inner.guard().await;
+ tokio::task::spawn_blocking(move || inner.retrieve_generic(&query, fruit, &guard))
+ .await
+ .unwrap()
}
+}
- fn retrieve_websites(
- &self,
- websites: &[inverted_index::WebpagePointer],
- query: &str,
- guard: &I::SearchGuard,
- ) -> Result<Vec<inverted_index::RetrievedWebpage>> {
- let ctx = guard.inverted_index().local_search_ctx();
- let query = SearchQuery {
- query: query.to_string(),
- ..Default::default()
- };
- let query = Query::parse(&ctx, &query, guard.inverted_index())?;
-
- guard.inverted_index().retrieve_websites(websites, &query)
- }
+struct InvertedIndexResult {
+ webpages: Vec<LocalRecallRankingWebpage>,
+ num_hits: approx_count::Count,
}
#[cfg(test)]
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index f2c32c07..2e2e4caf 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -279,20 +279,34 @@ impl EdgeStore {
pub fn iter_pages_small(&self) -> impl Iterator<Item = SmallEdge> + '_ {
let searcher = self.reader.searcher();
let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
-
- segment_readers.into_iter().flat_map(|segment| {
- SmallSegmentEdgesIter::new(&segment, FromId, ToId, 0..segment.max_doc())
+ let warm_column_fields = self.warmed_column_fields.clone();
+
+ segment_readers.into_iter().flat_map(move |segment| {
+ SmallSegmentEdgesIter::new(
+ &segment,
+ &warm_column_fields,
+ FromId,
+ ToId,
+ 0..segment.max_doc(),
+ )
})
}
pub fn iter_hosts_small(&self) -> impl Iterator<Item = SmallEdge> + '_ {
let searcher = self.reader.searcher();
let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
+ let warm_column_fields = self.warmed_column_fields.clone();
segment_readers
.into_iter()
- .flat_map(|segment| {
- SmallSegmentEdgesIter::new(&segment, FromHostId, ToHostId, 0..segment.max_doc())
+ .flat_map(move |segment| {
+ SmallSegmentEdgesIter::new(
+ &segment,
+ &warm_column_fields,
+ FromHostId,
+ ToHostId,
+ 0..segment.max_doc(),
+ )
})
.unique_by(|e| (e.from, e.to))
}
@@ -300,13 +314,20 @@ impl EdgeStore {
pub fn iter_page_node_ids(&self, offset: u32, limit: u32) -> impl Iterator<Item = NodeID> + '_ {
let searcher = self.reader.searcher();
let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
+ let warm_column_fields = self.warmed_column_fields.clone();
let range = offset..limit;
segment_readers
.into_iter()
.flat_map(move |segment| {
- SmallSegmentEdgesIter::new(&segment, FromId, ToId, range.clone())
+ SmallSegmentEdgesIter::new(
+ &segment,
+ &warm_column_fields,
+ FromId,
+ ToId,
+ range.clone(),
+ )
})
.flat_map(|e| [e.from, e.to])
.unique()
@@ -315,13 +336,19 @@ impl EdgeStore {
pub fn iter_host_node_ids(&self, offset: u32, limit: u32) -> impl Iterator<Item = NodeID> + '_ {
let searcher = self.reader.searcher();
let segment_readers: Vec<_> = searcher.segment_readers().to_vec();
-
+ let warm_column_fields = self.warmed_column_fields.clone();
let range = offset..limit;
segment_readers
.into_iter()
.flat_map(move |segment| {
- SmallSegmentEdgesIter::new(&segment, FromHostId, ToHostId, range.clone())
+ SmallSegmentEdgesIter::new(
+ &segment,
+ &warm_column_fields,
+ FromHostId,
+ ToHostId,
+ range.clone(),
+ )
})
.flat_map(|e| [e.from, e.to])
.unique()
@@ -339,6 +366,7 @@ pub struct SmallSegmentEdgesIter {
impl SmallSegmentEdgesIter {
fn new<F1, F2>(
segment: &SegmentReader,
+ warm_column_fields: &WarmedColumnFields,
from_id: F1,
to_id: F2,
mut doc_range: Range<DocId>,
@@ -347,16 +375,16 @@ impl SmallSegmentEdgesIter {
F1: Field,
F2: Field,
{
- let columns = segment.column_fields();
+ let columns = warm_column_fields.segment(&segment.segment_id());
if doc_range.end > segment.max_doc() {
doc_range.end = segment.max_doc();
}
Self {
- from_id: columns.u64(from_id.name()).unwrap(),
- to_id: columns.u64(to_id.name()).unwrap(),
- rel_flags: columns.u64(schema::RelFlags.name()).unwrap(),
+ from_id: columns.u64(from_id).unwrap(),
+ to_id: columns.u64(to_id).unwrap(),
+ rel_flags: columns.u64(schema::RelFlags).unwrap(),
current_doc: doc_range.start,
doc_range,
}