Skip to content

web search done right

[search_server] implement top key phrases as generic query

Mikkel Denkeraba861ea3798
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index 11c6da3a..2d2b85e3 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -8,6 +8,7 @@ use stract::{
defaults, ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, SnippetConfig,
WidgetsConfig,
},
+ generic_query::TopKeyPhrasesQuery,
index::Index,
searcher::{
api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,
@@ -59,9 +60,10 @@ pub async fn main() {
.set_collector_config(collector_conf)
.build();
- let mut queries: Vec<_> = searcher
- .top_key_phrases(1_000_000)
+ let mut queries: Vec<String> = searcher
+ .search_generic(TopKeyPhrasesQuery { top_n: 1_000_000 })
.await
+ .unwrap()
.into_iter()
.map(|phrase| phrase.text().to_string())
.collect();
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index 56919e55..03dbfc50 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -27,6 +27,7 @@ use crate::{
bangs::Bangs,
config::ApiConfig,
distributed::cluster::Cluster,
+ generic_query::TopKeyPhrasesQuery,
improvement::{store_improvements_loop, ImprovementEvent},
leaky_queue::LeakyQueue,
models::dual_encoder::DualEncoder,
@@ -195,8 +196,9 @@ pub async fn router(
log::info!("Building autosuggest");
let autosuggest = Autosuggest::from_key_phrases(
dist_searcher
- .top_key_phrases(config.top_phrases_for_autosuggest)
- .await,
+ .search_generic(TopKeyPhrasesQuery::new(config.top_phrases_for_autosuggest))
+ .await
+ .unwrap_or_default(),
)?;
let state = {
diff --git a/crates/core/src/entrypoint/api.rs b/crates/core/src/entrypoint/api.rs
index 3efe16a3..7fb39f16 100644
--- a/crates/core/src/entrypoint/api.rs
+++ b/crates/core/src/entrypoint/api.rs
@@ -29,6 +29,7 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
+ generic_query::TopKeyPhrasesQuery,
inverted_index::KeyPhrase,
metrics::Label,
searcher::{DistributedSearcher, SearchClient},
@@ -117,7 +118,11 @@ pub struct TopKeyphrases {
impl sonic::service::Message<ManagementService> for TopKeyphrases {
type Response = Vec<KeyPhrase>;
async fn handle(self, server: &ManagementService) -> Self::Response {
- server.searcher.top_key_phrases(self.top).await
+ server
+ .searcher
+ .search_generic(TopKeyPhrasesQuery::new(self.top))
+ .await
+ .unwrap_or_default()
}
}
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 8ff85b63..87dee138 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -26,9 +26,9 @@ use crate::{
member::{Member, Service},
sonic::{self, service::sonic_service},
},
- generic_query,
+ generic_query::{self, TopKeyPhrasesQuery},
index::Index,
- inverted_index::{self, KeyPhrase, RetrievedWebpage},
+ inverted_index::{self, RetrievedWebpage},
models::dual_encoder::DualEncoder,
ranking::models::linear::LinearRegression,
searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},
@@ -74,7 +74,7 @@ impl std::fmt::Display for EncodedError {
macro_rules! impl_search {
([$($q:ident),*$(,)?]) => {
$(
- impl Message<SearchService> for $q {
+ impl sonic::service::Message<SearchService> for $q {
type Response = Result<<<$q as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit, EncodedError>;
async fn handle(self, server: &SearchService) -> Self::Response {
@@ -90,7 +90,7 @@ macro_rules! impl_search {
pub fruit: <<$q as generic_query::GenericQuery>::Collector as generic_query::Collector>::Fruit,
}
- impl Message<SearchService> for [<$q Retrieve>] {
+ impl sonic::service::Message<SearchService> for [<$q Retrieve>] {
type Response = Result<<$q as generic_query::GenericQuery>::IntermediateOutput, EncodedError>;
async fn handle(self, server: &SearchService) -> Self::Response {
server
@@ -118,10 +118,9 @@ macro_rules! impl_search {
sonic_service!(SearchService, [
RetrieveWebsites,
Search,
+ Size,
GetWebpage,
GetHomepageDescriptions,
- TopKeyPhrases,
- Size,
GetSiteUrls,
$(
$q,
@@ -133,7 +132,7 @@ macro_rules! impl_search {
}
}
-impl_search!([]);
+impl_search!([TopKeyPhrasesQuery,]);
pub struct SearchService {
local_searcher: LocalSearcher<Arc<Index>>,
@@ -243,17 +242,6 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
}
}
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct TopKeyPhrases {
- pub top_n: usize,
-}
-impl sonic::service::Message<SearchService> for TopKeyPhrases {
- type Response = Vec<KeyPhrase>;
- async fn handle(self, server: &SearchService) -> Self::Response {
- server.local_searcher.top_key_phrases(self.top_n).await
- }
-}
-
pub async fn run(config: config::SearchServerConfig) -> Result<()> {
let addr = config.host;
let server = SearchService::new(config).await?.bind(addr).await.unwrap();
diff --git a/crates/core/src/generic_query/collector/mod.rs b/crates/core/src/generic_query/collector/mod.rs
index 50c8f460..cdb4cf9e 100644
--- a/crates/core/src/generic_query/collector/mod.rs
+++ b/crates/core/src/generic_query/collector/mod.rs
@@ -14,6 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+mod top_key_phrases;
+pub use top_key_phrases::TopKeyPhrasesCollector;
+
use tantivy::{
collector::{Fruit, SegmentCollector},
query::Weight,
diff --git a/crates/core/src/generic_query/collector/top_key_phrases.rs b/crates/core/src/generic_query/collector/top_key_phrases.rs
new file mode 100644
index 00000000..da76e24a
--- /dev/null
+++ b/crates/core/src/generic_query/collector/top_key_phrases.rs
@@ -0,0 +1,187 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{ampc::dht::ShardId, Result, SortableFloat};
+use std::{
+ cmp::Reverse,
+ collections::{BinaryHeap, HashMap},
+ sync::Arc,
+};
+
+use lending_iter::LendingIterator;
+use tantivy::collector::SegmentCollector;
+
+use crate::{
+ inverted_index::KeyPhrase,
+ schema::text_field::{self, TextField},
+};
+use std::str;
+
+use super::Collector;
+
+const NON_ALPHABETIC_CHAR_THRESHOLD: f64 = 0.25;
+
+pub struct TopKeyPhrasesCollector {
+ pub top_n: usize,
+ pub shard_id: Option<ShardId>,
+}
+
+impl TopKeyPhrasesCollector {
+ pub fn new(top_n: usize) -> Self {
+ Self {
+ top_n,
+ shard_id: None,
+ }
+ }
+
+ pub fn with_shard_id(mut self, shard_id: ShardId) -> Self {
+ self.shard_id = Some(shard_id);
+ self
+ }
+}
+
+impl Collector for TopKeyPhrasesCollector {
+ type Fruit = Vec<(ShardId, KeyPhrase)>;
+ type Child = TopKeyPhrasesSegmentCollector;
+
+ fn for_segment(
+ &self,
+ _: tantivy::SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ TopKeyPhrasesSegmentCollector::new(self.top_n, self.shard_id.unwrap(), segment)
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ let mut phrases = HashMap::new();
+
+ for (shard_id, fruit) in segment_fruits.into_iter().flatten() {
+ *phrases
+ .entry((shard_id, fruit.text().to_string()))
+ .or_default() += fruit.score();
+ }
+
+ let mut res: Vec<_> = phrases
+ .into_iter()
+ .map(|((shard_id, phrase), score)| (shard_id, KeyPhrase::new(phrase, score)))
+ .collect();
+
+ res.sort_by(|(_, a), (_, b)| b.score().total_cmp(&a.score()));
+ res.truncate(self.top_n);
+
+ Ok(res)
+ }
+
+ fn collect_segment(
+ &self,
+ _: &dyn tantivy::query::Weight,
+ segment_ord: u32,
+ reader: &tantivy::SegmentReader,
+ ) -> crate::Result<<Self::Child as tantivy::collector::SegmentCollector>::Fruit> {
+ let child = self.for_segment(segment_ord, reader)?;
+ Ok(child.top_phrases())
+ }
+}
+
+pub struct TopKeyPhrasesSegmentCollector {
+ top_n: usize,
+ shard_id: ShardId,
+ inverted_index: Arc<tantivy::InvertedIndexReader>,
+}
+
+impl TopKeyPhrasesSegmentCollector {
+ pub fn new(top_n: usize, shard_id: ShardId, reader: &tantivy::SegmentReader) -> Result<Self> {
+ let field = reader
+ .schema()
+ .get_field(text_field::KeyPhrases.name())
+ .unwrap();
+
+ let inverted_index = reader.inverted_index(field)?;
+
+ Ok(Self {
+ top_n,
+ shard_id,
+ inverted_index,
+ })
+ }
+
+ pub fn top_phrases(&self) -> Vec<(ShardId, KeyPhrase)> {
+ let mut keywords: BinaryHeap<(Reverse<SortableFloat>, String)> =
+ BinaryHeap::with_capacity(self.top_n);
+
+ let mut stream = self.inverted_index.terms().stream().unwrap();
+ while let Some((term, info)) = stream.next() {
+ let term_str = str::from_utf8(term).unwrap().to_string();
+ let num_chars = term_str.chars().count();
+
+ if term_str.chars().filter(|c| !c.is_alphabetic()).count() as f64 / num_chars as f64
+ > NON_ALPHABETIC_CHAR_THRESHOLD
+ {
+ continue;
+ }
+
+ let left_paren = term_str.chars().filter(|c| c == &'(').count();
+ let right_paren = term_str.chars().filter(|c| c == &')').count();
+
+ if left_paren != right_paren {
+ continue;
+ }
+
+ let words = term_str.split_whitespace().collect::<Vec<_>>();
+
+ if words.is_empty() {
+ continue;
+ }
+
+ let score = info.doc_freq as f64;
+
+ if score.is_normal() {
+ let term_str = words.join(" ");
+
+ if keywords.len() >= self.top_n {
+ if let Some(mut min) = keywords.peek_mut() {
+ if score > min.0 .0.into() {
+ *min = (Reverse(score.into()), term_str);
+ }
+ }
+ } else {
+ keywords.push((Reverse(score.into()), term_str));
+ }
+ }
+ }
+
+ keywords
+ .into_iter()
+ .map(|(Reverse(score), phrase)| KeyPhrase::new(phrase, score.into()))
+ .map(|phrase| (self.shard_id, phrase))
+ .collect()
+ }
+}
+
+impl SegmentCollector for TopKeyPhrasesSegmentCollector {
+ type Fruit = Vec<(ShardId, KeyPhrase)>;
+
+ fn collect(&mut self, _: tantivy::DocId, _: tantivy::Score) {
+ unimplemented!()
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ unimplemented!()
+ }
+}
diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rs
index a19dbedb..5ca2ed04 100644
--- a/crates/core/src/generic_query/mod.rs
+++ b/crates/core/src/generic_query/mod.rs
@@ -16,6 +16,9 @@
use crate::{ampc::dht::ShardId, search_ctx, Result};
+pub mod top_key_phrases;
+pub use top_key_phrases::TopKeyPhrasesQuery;
+
pub mod collector;
pub use collector::Collector;
diff --git a/crates/core/src/generic_query/top_key_phrases.rs b/crates/core/src/generic_query/top_key_phrases.rs
new file mode 100644
index 00000000..efe01317
--- /dev/null
+++ b/crates/core/src/generic_query/top_key_phrases.rs
@@ -0,0 +1,86 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::HashMap;
+
+use itertools::Itertools;
+
+use crate::{ampc::dht::ShardId, inverted_index::KeyPhrase};
+
+use super::{collector::TopKeyPhrasesCollector, GenericQuery};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct TopKeyPhrasesQuery {
+ pub top_n: usize,
+}
+
+impl TopKeyPhrasesQuery {
+ pub fn new(top_n: usize) -> Self {
+ Self { top_n }
+ }
+}
+
+impl GenericQuery for TopKeyPhrasesQuery {
+ type Collector = TopKeyPhrasesCollector;
+ type TantivyQuery = tantivy::query::EmptyQuery;
+ type IntermediateOutput = Vec<(ShardId, KeyPhrase)>;
+ type Output = Vec<KeyPhrase>;
+
+ fn tantivy_query(&self, _: &crate::search_ctx::Ctx) -> Self::TantivyQuery {
+ tantivy::query::EmptyQuery
+ }
+
+ fn collector(&self, ctx: &crate::search_ctx::Ctx) -> Self::Collector {
+ TopKeyPhrasesCollector::new(self.top_n).with_shard_id(ctx.shard_id)
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ TopKeyPhrasesCollector::new(self.top_n)
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ shard_id: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ .into_iter()
+ .filter(|(id, _)| id == &shard_id)
+ .collect()
+ }
+
+ fn retrieve(
+ &self,
+ _: &crate::search_ctx::Ctx,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ Ok(fruit)
+ }
+
+ fn merge_results(results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ let mut phrases = HashMap::new();
+
+ for (_, phrase) in results.into_iter().flatten() {
+ *phrases.entry(phrase.text().to_string()).or_default() += phrase.score();
+ }
+
+ phrases
+ .into_iter()
+ .map(|(phrase, score)| KeyPhrase::new(phrase, score))
+ .sorted_by(|a, b| b.score().total_cmp(&a.score()))
+ .collect()
+ }
+}
diff --git a/crates/core/src/inverted_index/key_phrase.rs b/crates/core/src/inverted_index/key_phrase.rs
index 853bf4f9..86b41afb 100644
--- a/crates/core/src/inverted_index/key_phrase.rs
+++ b/crates/core/src/inverted_index/key_phrase.rs
@@ -14,19 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use lending_iter::LendingIterator;
-use std::cmp::Reverse;
-use std::collections::BinaryHeap;
-use std::str;
-
-use crate::{
- schema::text_field::{self, TextField},
- SortableFloat,
-};
-
-const NON_ALPHABETIC_CHAR_THRESHOLD: f64 = 0.25;
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize)]
pub struct KeyPhrase {
phrase: String,
score: f64,
@@ -37,66 +25,6 @@ impl KeyPhrase {
Self { phrase, score }
}
- pub fn compute_top(reader: &tantivy::IndexReader, top_n: usize) -> Vec<Self> {
- let searcher = reader.searcher();
- let field = searcher
- .schema()
- .get_field(text_field::KeyPhrases.name())
- .unwrap();
-
- let mut keywords: BinaryHeap<(Reverse<SortableFloat>, String)> =
- BinaryHeap::with_capacity(top_n);
-
- for seg_reader in searcher.segment_readers() {
- let inv_index = seg_reader.inverted_index(field).unwrap();
- let mut stream = inv_index.terms().stream().unwrap();
- while let Some((term, info)) = stream.next() {
- let term_str = str::from_utf8(term).unwrap().to_string();
- let num_chars = term_str.chars().count();
-
- if term_str.chars().filter(|c| !c.is_alphabetic()).count() as f64 / num_chars as f64
- > NON_ALPHABETIC_CHAR_THRESHOLD
- {
- continue;
- }
-
- let left_paren = term_str.chars().filter(|c| c == &'(').count();
- let right_paren = term_str.chars().filter(|c| c == &')').count();
-
- if left_paren != right_paren {
- continue;
- }
-
- let words = term_str.split_whitespace().collect::<Vec<_>>();
-
- if words.is_empty() {
- continue;
- }
-
- let score = info.doc_freq as f64;
-
- if score.is_normal() {
- let term_str = words.join(" ");
-
- if keywords.len() >= top_n {
- if let Some(mut min) = keywords.peek_mut() {
- if score > min.0 .0.into() {
- *min = (Reverse(score.into()), term_str);
- }
- }
- } else {
- keywords.push((Reverse(score.into()), term_str));
- }
- }
- }
- }
-
- keywords
- .into_iter()
- .map(|(Reverse(score), phrase)| KeyPhrase::new(phrase, score.into()))
- .collect()
- }
-
pub fn score(&self) -> f64 {
self.score
}
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index efcc020e..447ff0d6 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -14,7 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
-use super::key_phrase::KeyPhrase;
use super::{DocAddress, InitialSearchResult, InvertedIndex, RetrievedWebpage, WebpagePointer};
use itertools::Itertools;
use tantivy::collector::Count;
@@ -321,10 +320,6 @@ impl InvertedIndex {
.map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).unwrap())
}
- pub(crate) fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
- KeyPhrase::compute_top(&self.reader, top_n)
- }
-
pub(crate) fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
let ctx = self.local_search_ctx();
let tv_searcher = ctx.tv_searcher;
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index 7801ed7a..e1f8a566 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -36,12 +36,12 @@ use crate::{
generic_query::{self, Collector},
image_store::Image,
index::Index,
- inverted_index::{KeyPhrase, RetrievedWebpage, WebpagePointer},
+ inverted_index::{RetrievedWebpage, WebpagePointer},
ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},
Result,
};
-use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::{collections::HashMap, sync::Arc};
use fnv::FnvHashMap;
use futures::{future::join_all, stream::FuturesUnordered, StreamExt};
@@ -98,8 +98,6 @@ pub trait SearchClient {
max_width: Option<u64>,
) -> impl Future<Output = Result<Option<Image>>> + Send;
- fn top_key_phrases(&self, top_n: usize) -> impl Future<Output = Vec<KeyPhrase>> + Send;
-
fn search_initial_generic<Q>(
&self,
query: Q,
@@ -182,7 +180,6 @@ pub trait SearchClient {
where
Q: search_server::Query,
Self: Send + Sync,
-
Result<<<Q::Collector as generic_query::Collector>::Child as tantivy::collector::SegmentCollector>::Fruit, search_server::EncodedError>:
From<<Q as sonic::service::Message<SearchService>>::Response>,
Result<Q::IntermediateOutput, search_server::EncodedError>: From<
@@ -519,45 +516,6 @@ impl SearchClient for DistributedSearcher {
.and_then(|(_, v)| v)
}
- async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
- let client = self.conn().await;
-
- let res = client
- .send_with_timeout(
- search_server::TopKeyPhrases { top_n },
- &AllShardsSelector,
- &RandomReplicaSelector,
- Duration::from_secs(60 * 60),
- )
- .await;
-
- match res {
- Ok(res) => {
- let mut phrases = HashMap::new();
-
- for (_, v) in res.into_iter().flatten() {
- for (_, v) in v {
- for phrase in v {
- *phrases.entry(phrase.text().to_string()).or_default() +=
- phrase.score();
- }
- }
- }
-
- phrases
- .into_iter()
- .map(|(phrase, score)| KeyPhrase::new(phrase, score))
- .sorted_by(|a, b| b.score().partial_cmp(&a.score()).unwrap())
- .take(top_n)
- .collect()
- }
- Err(e) => {
- tracing::error!("failed to get key phrases: {:?}", e);
- Vec::new()
- }
- }
- }
-
async fn search_initial_generic<Q>(
&self,
query: Q,
@@ -814,10 +772,6 @@ impl SearchClient for LocalSearchClient {
None
}
- async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
- self.0.top_key_phrases(top_n).await
- }
-
async fn search_initial_generic<Q>(
&self,
query: Q,
diff --git a/crates/core/src/searcher/local/inner.rs b/crates/core/src/searcher/local/inner.rs
index 87c49aad..a66c3e71 100644
--- a/crates/core/src/searcher/local/inner.rs
+++ b/crates/core/src/searcher/local/inner.rs
@@ -196,4 +196,15 @@ where
) -> Result<Q::IntermediateOutput> {
guard.inverted_index().retrieve_generic(query, fruit)
}
+
+ pub fn search_generic<Q: GenericQuery>(
+ &self,
+ query: Q,
+ guard: &I::SearchGuard,
+ ) -> Result<Q::Output> {
+ let fruit = self.search_initial_generic(&query, guard)?;
+ Ok(Q::merge_results(vec![
+ self.retrieve_generic(&query, fruit, guard)?
+ ]))
+ }
}
diff --git a/crates/core/src/searcher/local/mod.rs b/crates/core/src/searcher/local/mod.rs
index c95036b8..39185b3f 100644
--- a/crates/core/src/searcher/local/mod.rs
+++ b/crates/core/src/searcher/local/mod.rs
@@ -31,7 +31,7 @@ use crate::collector::approx_count;
use crate::config::CollectorConfig;
use crate::generic_query::{self, GenericQuery};
use crate::index::Index;
-use crate::inverted_index::{KeyPhrase, RetrievedWebpage};
+use crate::inverted_index::RetrievedWebpage;
use crate::models::dual_encoder::DualEncoder;
use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{
@@ -224,14 +224,6 @@ where
self.inner.guard().await.inverted_index().get_homepage(url)
}
- pub async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
- self.inner
- .guard()
- .await
- .inverted_index()
- .top_key_phrases(top_n)
- }
-
pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
self.inner
.guard()
@@ -266,6 +258,14 @@ where
.await
.unwrap()
}
+
+ pub async fn search_generic<Q: GenericQuery + 'static>(&self, query: Q) -> Result<Q::Output> {
+ let inner = self.inner.clone();
+ let guard = inner.guard().await;
+ tokio::task::spawn_blocking(move || inner.search_generic(query, &guard))
+ .await
+ .unwrap()
+ }
}
struct InvertedIndexResult {