Skip to content

web search done right

improve compaction performance in live index by performing initial segment merge on a read lock and only switch to the new segments on a write lock. this ensures that search requests can still be performed while the heavy part of merge is executing

Mikkel Denker3a127af0a780
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index c338cbe6..be538ed1 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -51,17 +51,20 @@ pub async fn main() {
let mut queries: Vec<_> = searcher
.top_key_phrases(1_000_000)
+ .await
.into_iter()
.map(|phrase| phrase.text().to_string())
.collect();
queries.shuffle(&mut rand::thread_rng());
searcher.set_collector_config(collector_conf);
- searcher.set_snippet_config(SnippetConfig {
- num_words_for_lang_detection: Some(250),
- max_considered_words: Some(10_000),
- ..Default::default()
- });
+ searcher
+ .set_snippet_config(SnippetConfig {
+ num_words_for_lang_detection: Some(250),
+ max_considered_words: Some(10_000),
+ ..Default::default()
+ })
+ .await;
let bangs = Bangs::from_path(config.bangs_path.as_ref().unwrap());
let searcher = stract::searcher::LocalSearchClient::from(searcher);
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
index 37e94139..be27bf74 100644
--- a/crates/core/src/entrypoint/live_index/search_server.rs
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -134,8 +134,8 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)
if let Some(other) = others.pop() {
let mut conn: sonic::service::Connection<LiveIndexService> =
sonic::service::Connection::create(other).await?;
- index.delete_all_pages();
- let local_path = index.path();
+ index.delete_all_pages().await;
+ let local_path = index.path().await;
let remote_path = conn.send(GetIndexPath).await?;
remote_cp::Request::download(
@@ -146,7 +146,7 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)
},
)
.await;
- index.re_open().unwrap();
+ index.re_open().await.unwrap();
}
let mut wal = temp_wal
@@ -157,9 +157,9 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)
for pages in wal.iter().unwrap().chunks(512).into_iter() {
let pages: Vec<_> = pages.into_iter().collect();
- index.insert(&pages);
+ index.insert(&pages).await;
}
- index.commit();
+ index.commit().await;
wal.clear().unwrap();
@@ -241,7 +241,9 @@ impl LiveIndexService {
let temp_wal = self.temp_wal.clone();
let cluster = self.cluster_handle.clone();
- tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });
+ std::thread::spawn(move || {
+ crate::block_on(setup(index, cluster, temp_wal)).unwrap();
+ });
}
async fn index_webpages_in_replicas(
@@ -330,6 +332,7 @@ impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {
server
.local_searcher
.retrieve_websites(&self.websites, &self.query)
+ .await
.ok()
}
}
@@ -337,18 +340,21 @@ impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {
impl sonic::service::Message<LiveIndexService> for Search {
type Response = Option<InitialWebsiteResult>;
async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server.local_searcher.search_initial(&self.query, true).ok()
+ server
+ .local_searcher
+ .search_initial(&self.query, true)
+ .await
+ .ok()
}
}
impl sonic::service::Message<LiveIndexService> for GetSiteUrls {
type Response = SiteUrls;
async fn handle(self, server: &LiveIndexService) -> Self::Response {
- let urls = server.local_searcher.get_site_urls(
- &self.site,
- self.offset as usize,
- self.limit as usize,
- );
+ let urls = server
+ .local_searcher
+ .get_site_urls(&self.site, self.offset as usize, self.limit as usize)
+ .await;
SiteUrls { urls }
}
@@ -408,7 +414,7 @@ impl sonic::service::Message<LiveIndexService> for IndexWebpages {
if let Some(wal) = server.temp_wal.lock().await.as_mut() {
wal.batch_write(self.pages.iter()).unwrap();
} else {
- server.index.insert(&self.pages);
+ server.index.insert(&self.pages).await;
if let Some(consistency_fraction) = self.consistency_fraction {
server
@@ -428,7 +434,7 @@ impl sonic::service::Message<LiveIndexService> for GetIndexPath {
type Response = PathBuf;
async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server.index.path()
+ server.index.path().await
}
}
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
index c605dd74..4ecadf13 100644
--- a/crates/core/src/entrypoint/live_index/tests.rs
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -171,7 +171,7 @@ impl RemoteIndex {
}
async fn commit_underlying(&self) {
- self.underlying_index.commit();
+ self.underlying_index.commit().await;
}
async fn kill(self) -> Result<()> {
@@ -415,28 +415,30 @@ async fn test_meta_segments() -> Result<()> {
};
let index = LiveIndex::new(&config.index_path, indexer_config.clone()).await?;
- assert!(index.meta().segments().is_empty());
+ assert!(index.meta().await.segments().is_empty());
- index.insert(&[IndexableWebpage {
- url: "https://a.com/".to_string(),
- body: "
+ index
+ .insert(&[IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
<title>test page</title>
Example webpage
"
- .to_string(),
- fetch_time_ms: 100,
- }]);
- index.commit();
+ .to_string(),
+ fetch_time_ms: 100,
+ }])
+ .await;
+ index.commit().await;
- assert_eq!(index.meta().segments().len(), 1);
+ assert_eq!(index.meta().await.segments().len(), 1);
- index.re_open()?;
+ index.re_open().await?;
- assert_eq!(index.meta().segments().len(), 1);
+ assert_eq!(index.meta().await.segments().len(), 1);
let copy_index = LiveIndex::new(&config.index_path, indexer_config).await?;
- assert_eq!(copy_index.meta().segments().len(), 1);
+ assert_eq!(copy_index.meta().await.segments().len(), 1);
Ok(())
}
@@ -455,42 +457,48 @@ async fn test_segment_compaction() -> Result<()> {
let index = Arc::new(LiveIndex::new(&config.index_path, indexer_config).await?);
- index.insert(&[IndexableWebpage {
- url: "https://a.com/".to_string(),
- body: "
+ index
+ .insert(&[IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
<title>test page</title>
Example webpage
"
- .to_string(),
- fetch_time_ms: 100,
- }]);
+ .to_string(),
+ fetch_time_ms: 100,
+ }])
+ .await;
- index.commit();
+ index.commit().await;
- index.insert(&[IndexableWebpage {
- url: "https://b.com/".to_string(),
- body: "
+ index
+ .insert(&[IndexableWebpage {
+ url: "https://b.com/".to_string(),
+ body: "
<title>test page</title>
Example webpage
"
- .to_string(),
- fetch_time_ms: 100,
- }]);
+ .to_string(),
+ fetch_time_ms: 100,
+ }])
+ .await;
- index.commit();
+ index.commit().await;
- assert_eq!(index.meta().segments().len(), 2);
+ assert_eq!(index.meta().await.segments().len(), 2);
- index.compact_segments_by_date();
+ index.compact_segments_by_date().await?;
- assert_eq!(index.meta().segments().len(), 1);
+ assert_eq!(index.meta().await.segments().len(), 1);
let searcher = LocalSearcher::from(index.clone());
- let res = searcher.search(&SearchQuery {
- query: "test".to_string(),
- ..Default::default()
- })?;
+ let res = searcher
+ .search(&SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ })
+ .await?;
assert_eq!(res.webpages.len(), 2);
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 9316f633..13096fd1 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -71,7 +71,7 @@ impl SearchService {
}
local_searcher.set_collector_config(config.collector);
- local_searcher.set_snippet_config(config.snippet);
+ local_searcher.set_snippet_config(config.snippet).await;
let cluster_handle = Cluster::join(
Member::new(Service::Searcher {
@@ -101,6 +101,7 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {
server
.local_searcher
.retrieve_websites(&self.websites, &self.query)
+ .await
.ok()
}
}
@@ -112,7 +113,11 @@ pub struct Search {
impl sonic::service::Message<SearchService> for Search {
type Response = Option<InitialWebsiteResult>;
async fn handle(self, server: &SearchService) -> Self::Response {
- server.local_searcher.search_initial(&self.query, true).ok()
+ server
+ .local_searcher
+ .search_initial(&self.query, true)
+ .await
+ .ok()
}
}
@@ -123,7 +128,7 @@ pub struct GetWebpage {
impl sonic::service::Message<SearchService> for GetWebpage {
type Response = Option<RetrievedWebpage>;
async fn handle(self, server: &SearchService) -> Self::Response {
- server.local_searcher.get_webpage(&self.url)
+ server.local_searcher.get_webpage(&self.url).await
}
}
@@ -138,7 +143,7 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
let mut result = HashMap::with_capacity(self.urls.len());
for url in &self.urls {
- if let Some(homepage) = server.local_searcher.get_homepage(url) {
+ if let Some(homepage) = server.local_searcher.get_homepage(url).await {
if let Some(desc) = homepage.description() {
result.insert(url.clone(), desc.clone());
}
@@ -156,7 +161,7 @@ pub struct TopKeyPhrases {
impl sonic::service::Message<SearchService> for TopKeyPhrases {
type Response = Vec<KeyPhrase>;
async fn handle(self, server: &SearchService) -> Self::Response {
- server.local_searcher.top_key_phrases(self.top_n)
+ server.local_searcher.top_key_phrases(self.top_n).await
}
}
@@ -181,6 +186,7 @@ impl sonic::service::Message<SearchService> for Size {
.local_searcher
.index()
.guard()
+ .await
.inverted_index()
.num_documents(),
}
@@ -203,11 +209,10 @@ pub struct SiteUrls {
impl sonic::service::Message<SearchService> for GetSiteUrls {
type Response = SiteUrls;
async fn handle(self, server: &SearchService) -> Self::Response {
- let urls = server.local_searcher.get_site_urls(
- &self.site,
- self.offset as usize,
- self.limit as usize,
- );
+ let urls = server
+ .local_searcher
+ .get_site_urls(&self.site, self.offset as usize, self.limit as usize)
+ .await;
SiteUrls { urls }
}
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index b93cd2ab..03b8f906 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -232,7 +232,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test".to_string(),
return_ranking_signals: true,
..Default::default()
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index 9a4a62c2..c077df92 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -15,6 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>
use tantivy::index::SegmentId;
+use tantivy::indexer::{MergeOperation, SegmentEntry};
use tantivy::merge_policy::NoMergePolicy;
use tantivy::{IndexWriter, SegmentMeta};
@@ -153,20 +154,35 @@ impl InvertedIndex {
Ok(())
}
- #[allow(clippy::missing_panics_doc)] // should not panic as writer is prepared
- pub fn merge_segments_by_id(&mut self, segments: &[SegmentId]) -> Result<Option<SegmentId>> {
- self.prepare_writer()?;
-
+ pub async fn start_merge_segments_by_id(
+ &self,
+ segments: &[SegmentId],
+ ) -> Result<(Option<SegmentEntry>, MergeOperation)> {
if segments.is_empty() {
- return Ok(None);
+ anyhow::bail!("no segments to merge");
}
+ let (entry, op) = self
+ .writer
+ .as_ref()
+ .expect("writer has not been prepared")
+ .start_merge(segments)
+ .await?;
+
+ Ok((entry, op))
+ }
+
+ pub fn end_merge_segments_by_id(
+ &mut self,
+ merge_operation: MergeOperation,
+ segment_entry: Option<SegmentEntry>,
+ ) -> Result<Option<SegmentId>> {
+ self.prepare_writer()?;
let res = self
.writer
.as_mut()
.expect("writer has not been prepared")
- .merge(segments)
- .wait()?;
+ .end_merge(merge_operation, segment_entry)?;
Ok(res.map(|seg| seg.id()))
}
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index 5058afd5..cd3d1e1f 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -17,19 +17,25 @@
use std::{
io::{BufReader, BufWriter},
path::{Path, PathBuf},
- sync::{Arc, RwLock, RwLockReadGuard},
+ sync::Arc,
};
+use tokio::sync::{RwLock, RwLockReadGuard};
+
use chrono::{DateTime, NaiveDate, Utc};
use itertools::Itertools;
use simple_wal::Wal;
-use tantivy::index::SegmentId;
+use tantivy::{
+ index::SegmentId,
+ indexer::{MergeOperation, SegmentEntry},
+};
use std::collections::{HashMap, HashSet};
use crate::{
config::SnippetConfig,
entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},
+ inverted_index::InvertedIndex,
live_index::{BATCH_SIZE, TTL},
searcher::SearchableIndex,
Result,
@@ -51,6 +57,19 @@ impl Segment {
}
}
+pub struct CompactOperation {
+ segments: Vec<Segment>,
+ entry: Option<SegmentEntry>,
+ merge_op: MergeOperation,
+}
+
+impl CompactOperation {
+ pub fn end(self, index: &mut InvertedIndex) -> Result<Option<SegmentId>> {
+ let res = index.end_merge_segments_by_id(self.merge_op, self.entry)?;
+ Ok(res)
+ }
+}
+
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
pub struct Meta {
segments: Vec<Segment>,
@@ -145,20 +164,40 @@ impl InnerIndex {
self.re_open();
}
- pub fn compact_segments_by_date(&mut self) {
+ pub async fn start_compact_segments_by_date(&self) -> Result<Vec<CompactOperation>> {
let segments_to_compact = self.prepare_segments_for_compaction();
+ let mut operations = Vec::new();
+
for (_, segments) in segments_to_compact {
if segments.len() <= 1 {
continue;
}
let segment_ids: Vec<SegmentId> = segments.iter().map(|s| s.id).collect();
- let newest_creation_date = segments.iter().map(|s| s.created).max().unwrap();
- let merge_result = self.index.inverted_index.merge_segments_by_id(&segment_ids);
+ let (entry, merge_op) = self
+ .index
+ .inverted_index
+ .start_merge_segments_by_id(&segment_ids)
+ .await?;
+
+ operations.push(CompactOperation {
+ segments,
+ entry,
+ merge_op,
+ });
+ }
+
+ Ok(operations)
+ }
- if let Ok(Some(new_segment_id)) = merge_result {
+ fn end_compact_segments_by_date(&mut self, operations: Vec<CompactOperation>) -> Result<()> {
+ for op in operations {
+ let newest_creation_date = op.segments.iter().map(|s| s.created).max().unwrap();
+ let segment_ids: Vec<SegmentId> = op.segments.iter().map(|s| s.id).collect();
+
+ if let Ok(Some(new_segment_id)) = op.end(&mut self.index.inverted_index) {
self.update_meta_after_compaction(
segment_ids,
new_segment_id,
@@ -169,6 +208,8 @@ impl InnerIndex {
self.save_meta();
self.re_open();
+
+ Ok(())
}
fn prepare_segments_for_compaction(&self) -> HashMap<NaiveDate, Vec<Segment>> {
@@ -323,89 +364,70 @@ impl LiveIndex {
})
}
- pub fn commit(&self) {
+ pub async fn commit(&self) {
tracing::debug!("committing index");
- futures::executor::block_on(
- self.inner
- .write()
- .unwrap_or_else(|e| e.into_inner())
- .commit(),
- );
- tracing::debug!("index committed");
+ self.inner.write().await.commit().await;
}
- pub fn prune_segments(&self) {
+ pub async fn prune_segments(&self) {
tracing::debug!("pruning segments");
- self.inner
- .write()
- .unwrap_or_else(|e| e.into_inner())
- .prune_segments()
+ self.inner.write().await.prune_segments()
}
- pub fn has_inserts(&self) -> bool {
- self.inner
- .read()
- .unwrap_or_else(|e| e.into_inner())
- .has_inserts()
+ pub async fn has_inserts(&self) -> bool {
+ self.inner.read().await.has_inserts()
}
- pub fn compact_segments_by_date(&self) {
+ pub async fn compact_segments_by_date(&self) -> Result<()> {
tracing::debug!("compacting segments by date");
+ let operations = self
+ .inner
+ .read()
+ .await
+ .start_compact_segments_by_date()
+ .await?;
+
self.inner
.write()
- .unwrap_or_else(|e| e.into_inner())
- .compact_segments_by_date()
+ .await
+ .end_compact_segments_by_date(operations)?;
+
+ Ok(())
}
- pub fn insert(&self, pages: &[IndexableWebpage]) {
+ pub async fn insert(&self, pages: &[IndexableWebpage]) {
tracing::debug!("inserting {} pages into index", pages.len());
- self.inner
- .write()
- .unwrap_or_else(|e| e.into_inner())
- .insert(pages)
+ self.inner.write().await.insert(pages)
}
- pub fn read(&self) -> RwLockReadGuard<'_, InnerIndex> {
- self.inner.read().unwrap_or_else(|e| e.into_inner())
+ pub async fn read(&self) -> RwLockReadGuard<'_, InnerIndex> {
+ self.inner.read().await
}
- pub fn set_snippet_config(&self, config: SnippetConfig) {
+ pub async fn set_snippet_config(&self, config: SnippetConfig) {
self.inner
.write()
- .unwrap_or_else(|e| e.into_inner())
+ .await
.index
.set_snippet_config(config)
+ .await
}
- pub fn path(&self) -> PathBuf {
- self.inner
- .read()
- .unwrap_or_else(|e| e.into_inner())
- .path
- .to_path_buf()
+ pub async fn path(&self) -> PathBuf {
+ self.inner.read().await.path.to_path_buf()
}
- pub fn delete_all_pages(&self) {
- self.inner
- .write()
- .unwrap_or_else(|e| e.into_inner())
- .delete_all_pages();
+ pub async fn delete_all_pages(&self) {
+ self.inner.write().await.delete_all_pages();
}
- pub fn re_open(&self) -> Result<()> {
- self.inner
- .write()
- .unwrap_or_else(|e| e.into_inner())
- .re_open();
+ pub async fn re_open(&self) -> Result<()> {
+ self.inner.write().await.re_open();
Ok(())
}
- pub fn meta(&self) -> Meta {
- self.inner
- .read()
- .unwrap_or_else(|e| e.into_inner())
- .meta
- .clone()
+ pub async fn meta(&self) -> Meta {
+ self.inner.read().await.meta.clone()
}
}
diff --git a/crates/core/src/live_index/index_manager.rs b/crates/core/src/live_index/index_manager.rs
index 1f0e902a..966662af 100644
--- a/crates/core/src/live_index/index_manager.rs
+++ b/crates/core/src/live_index/index_manager.rs
@@ -38,17 +38,22 @@ impl IndexManager {
loop {
if last_prune + PRUNE_INTERVAL < Utc::now() {
- self.index.prune_segments();
+ crate::block_on(self.index.prune_segments());
last_prune = Utc::now();
}
- if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && self.index.has_inserts() {
- self.index.commit();
+ if last_commit + AUTO_COMMIT_INTERVAL < Utc::now()
+ && crate::block_on(self.index.has_inserts())
+ {
+ crate::block_on(self.index.commit());
last_commit = Utc::now();
}
if last_compact + COMPACT_INTERVAL < Utc::now() {
- self.index.compact_segments_by_date();
+ if let Err(e) = crate::block_on(self.index.compact_segments_by_date()) {
+ tracing::error!("Failed to compact segments: {}", e);
+ }
+
last_compact = Utc::now();
}
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index 6382f52f..fbcb479a 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -384,7 +384,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.second.com/");
}
@@ -454,7 +454,7 @@ mod tests {
query: "test site:first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
@@ -462,7 +462,7 @@ mod tests {
query: "test site:www.first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
@@ -470,7 +470,7 @@ mod tests {
query: "test -site:first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
assert!(result
@@ -528,7 +528,7 @@ mod tests {
query: "test linksto:first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.second.com/example/abc");
@@ -536,7 +536,7 @@ mod tests {
query: "test linkto:www.first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.second.com/example/abc");
@@ -544,7 +544,7 @@ mod tests {
query: "test -linkto:first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
@@ -552,7 +552,7 @@ mod tests {
query: "test linkto:second.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
@@ -560,7 +560,7 @@ mod tests {
query: "test linkto:www.second.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
@@ -568,7 +568,7 @@ mod tests {
query: "test linkto:second.com/example".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
@@ -576,7 +576,7 @@ mod tests {
query: "test linksto:second.com/example/abc".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
@@ -630,7 +630,7 @@ mod tests {
query: "test linkto:second.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
@@ -682,7 +682,7 @@ mod tests {
query: "intitle:website".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
@@ -734,7 +734,7 @@ mod tests {
query: "test inurl:forum".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/forum");
}
@@ -829,14 +829,14 @@ mod tests {
query: "test site:first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 0);
let query = SearchQuery {
query: "test site:the-first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.the-first.com/");
@@ -844,7 +844,7 @@ mod tests {
query: "test site:www.the-first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.the-first.com/");
}
@@ -902,7 +902,7 @@ mod tests {
query: "\"Test website\"".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.first.com/");
@@ -910,7 +910,7 @@ mod tests {
query: "\"Test website\" site:www.second.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 0);
}
@@ -970,7 +970,7 @@ mod tests {
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
let query = SearchQuery {
@@ -978,7 +978,7 @@ mod tests {
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
}
@@ -1053,7 +1053,7 @@ mod tests {
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
let query = SearchQuery {
@@ -1062,7 +1062,7 @@ mod tests {
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
assert_eq!(result.webpages[0].url, "https://www.sfw.com/");
@@ -1142,28 +1142,28 @@ mod tests {
query: "test site:.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
let query = SearchQuery {
query: "test site:.com/example".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
query: "test site:first.com/example".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
query: "test site:first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
@@ -1171,7 +1171,7 @@ mod tests {
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
}
@@ -1249,14 +1249,14 @@ mod tests {
query: "test exacturl:https://www.first.com/example".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
let query = SearchQuery {
query: "test exacturl:https://www.first.com".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 0);
}
@@ -1334,7 +1334,7 @@ mod tests {
query: "\"test test\" website".to_string(),
..Default::default()
};
- let result = searcher.search(&query).expect("Search failed");
+ let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 2);
}
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 5a4df880..ab5dbecc 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -323,7 +323,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
signal_coefficients: enum_map! {
crate::ranking::SignalEnum::from(crate::ranking::signals::core::Bm25Title) => 1_000_000.0
@@ -340,7 +340,7 @@ mod tests {
assert_eq!(res[1].url, "https://www.a.com/");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(
@@ -364,7 +364,7 @@ mod tests {
assert_eq!(res[0].url, "https://www.a.com/");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(
@@ -444,7 +444,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let _ = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(include_str!(
@@ -458,7 +458,7 @@ mod tests {
.webpages;
let _ = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(include_str!(
@@ -472,7 +472,7 @@ mod tests {
.webpages;
let _ = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(include_str!(
@@ -568,7 +568,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(
@@ -844,7 +844,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(
@@ -868,7 +868,7 @@ mod tests {
assert_eq!(res[0].url, "https://www.b.com/");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(
@@ -892,7 +892,7 @@ mod tests {
assert_eq!(res[0].url, "https://www.b.com/");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(
@@ -916,7 +916,7 @@ mod tests {
assert_eq!(res[0].url, "https://www.a.com/");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "website".to_string(),
optic: Some(
Optic::parse(
@@ -973,7 +973,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "site:stackoverflow.com".to_string(),
optic: Some(
Optic::parse(
@@ -1063,7 +1063,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1125,7 +1125,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
..Default::default()
})
@@ -1135,7 +1135,7 @@ mod tests {
assert_eq!(res[0].url, "https://example.com/");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"is\") }, Action(Discard) }").unwrap(),
@@ -1147,7 +1147,7 @@ mod tests {
assert_eq!(res.len(), 0);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"|is\") }, Action(Discard) }").unwrap(),
@@ -1159,7 +1159,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"|This\") }, Action(Discard) }").unwrap(),
@@ -1171,7 +1171,7 @@ mod tests {
assert_eq!(res.len(), 0);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"|This an\") }, Action(Discard) }")
@@ -1184,7 +1184,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"|This * an\") }, Action(Discard) }")
@@ -1197,7 +1197,7 @@ mod tests {
assert_eq!(res.len(), 0);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Site(\"example.com\") }, Action(Discard) }")
@@ -1210,7 +1210,7 @@ mod tests {
assert_eq!(res.len(), 0);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Site(\"|example.com\") }, Action(Discard) }")
@@ -1223,7 +1223,7 @@ mod tests {
assert_eq!(res.len(), 0);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Site(\"|example.com|\") }, Action(Discard) }")
@@ -1236,7 +1236,7 @@ mod tests {
assert_eq!(res.len(), 0);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"website.com|\") }, Action(Discard) }")
@@ -1282,7 +1282,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1297,7 +1297,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1350,7 +1350,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(Optic::parse("").unwrap()),
..Default::default()
@@ -1360,7 +1360,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"\") }, Action(Discard) }").unwrap(),
@@ -1430,7 +1430,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"*\") }, Action(Discard) }").unwrap(),
@@ -1442,7 +1442,7 @@ mod tests {
assert_eq!(res.len(), 0);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"* is\") }, Action(Discard) }").unwrap(),
@@ -1454,7 +1454,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"* This is\") }, Action(Discard) }")
@@ -1467,7 +1467,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"example *\") }, Action(Discard) }")
@@ -1480,7 +1480,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1526,7 +1526,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
..Default::default()
})
@@ -1535,7 +1535,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("DiscardNonMatching; Rule { Matches { Content(\"||\") }, Action(Boost(0)) }")
@@ -1548,7 +1548,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1622,7 +1622,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
..Default::default()
})
@@ -1631,7 +1631,7 @@ mod tests {
assert_eq!(res.len(), 2);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1701,7 +1701,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
..Default::default()
})
@@ -1710,7 +1710,7 @@ mod tests {
assert_eq!(res.len(), 2);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1726,7 +1726,7 @@ mod tests {
assert_eq!(res[0].url, "https://example.com/test");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Site(\"|example.com|\") }, Action(Discard) }")
@@ -1815,7 +1815,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("Rule { Matches { Title(\"*'s collection\") }, Action(Discard) }")
@@ -1903,7 +1903,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("DiscardNonMatching; Rule { Matches { Title(\"*'s collection\") }, Action(Discard) }; Rule { Matches { Site(\"*.com\") } }")
@@ -1945,7 +1945,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1960,7 +1960,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse(
@@ -1975,7 +1975,7 @@ mod tests {
assert_eq!(res.len(), 1);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
optic: Some(
Optic::parse("DiscardNonMatching; Rule { Matches { Domain(\"|another.example.com|\") } } ")
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index de2e0bfe..e5133904 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -198,7 +198,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
..Default::default()
})
@@ -264,7 +264,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
..Default::default()
})
@@ -332,7 +332,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "title".to_string(),
return_ranking_signals: true,
signal_coefficients: crate::enum_map! {
@@ -405,7 +405,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test".to_string(),
..Default::default()
})
@@ -471,7 +471,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test".to_string(),
..Default::default()
})
@@ -557,7 +557,7 @@ mod tests {
let searcher = LocalSearcher::new(index);
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
signal_coefficients: crate::enum_map! {
@@ -573,7 +573,7 @@ mod tests {
assert_eq!(&res.webpages[0].url, "https://www.title.com/");
let res = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "example".to_string(),
signal_coefficients: crate::enum_map! {
@@ -643,7 +643,7 @@ mod tests {
let searcher = LocalSearcher::new(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test".to_string(),
signal_coefficients: crate::enum_map! {
crate::ranking::SignalEnum::from(crate::ranking::signals::FetchTimeMs) => 100_000.0,
@@ -736,7 +736,7 @@ mod tests {
let searcher = LocalSearcher::new(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test".to_string(),
signal_coefficients: crate::enum_map! {
@@ -859,7 +859,7 @@ mod tests {
.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "best chocolate cake".to_string(),
signal_coefficients: crate::enum_map! {
@@ -947,7 +947,7 @@ mod tests {
.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "best chocolate cake".to_string(),
signal_coefficients: crate::enum_map! {
@@ -994,7 +994,7 @@ mod tests {
let searcher = LocalSearcher::new(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test website".to_string(),
return_ranking_signals: true,
..Default::default()
@@ -1017,7 +1017,7 @@ mod tests {
);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test example".to_string(),
return_ranking_signals: true,
..Default::default()
@@ -1074,7 +1074,7 @@ mod tests {
let searcher = LocalSearcher::new(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test website".to_string(),
return_ranking_signals: true,
..Default::default()
@@ -1097,7 +1097,7 @@ mod tests {
);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test b".to_string(),
return_ranking_signals: true,
..Default::default()
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index dd927676..b3c4839b 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -458,7 +458,7 @@ impl From<LocalSearcher<Index>> for LocalSearchClient {
impl SearchClient for LocalSearchClient {
async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {
- let res = self.0.search_initial(query, true).unwrap();
+ let res = self.0.search_initial(query, true).await.unwrap();
vec![InitialSearchResultShard {
local_result: res,
@@ -479,6 +479,7 @@ impl SearchClient for LocalSearchClient {
let res = self
.0
.retrieve_websites(&pointers, query)
+ .await
.unwrap()
.into_iter()
.zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))
@@ -489,7 +490,7 @@ impl SearchClient for LocalSearchClient {
}
async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {
- Ok(self.0.get_webpage(url))
+ Ok(self.0.get_webpage(url).await)
}
async fn get_homepage_descriptions(
@@ -499,7 +500,7 @@ impl SearchClient for LocalSearchClient {
let mut res = std::collections::HashMap::new();
for url in urls {
- if let Some(homepage) = self.0.get_homepage(url) {
+ if let Some(homepage) = self.0.get_homepage(url).await {
if let Some(desc) = homepage.description() {
res.insert(url.clone(), desc.clone());
}
@@ -523,6 +524,6 @@ impl SearchClient for LocalSearchClient {
}
async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
- self.0.top_key_phrases(top_n)
+ self.0.top_key_phrases(top_n).await
}
}
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index 58efa36c..31c4cf3a 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -15,7 +15,9 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::collections::HashMap;
-use std::sync::{Arc, RwLockReadGuard};
+use std::future::Future;
+use std::sync::Arc;
+use tokio::sync::RwLockReadGuard;
use itertools::Itertools;
use url::Url;
@@ -43,8 +45,8 @@ pub trait SearchableIndex {
where
Self: 'a;
- fn guard(&self) -> Self::SearchGuard<'_>;
- fn set_snippet_config(&mut self, config: SnippetConfig);
+ fn guard(&self) -> impl Future<Output = Self::SearchGuard<'_>>;
+ fn set_snippet_config(&mut self, config: SnippetConfig) -> impl Future<Output = ()>;
}
pub trait SearchGuard<'a> {
@@ -57,11 +59,11 @@ pub trait SearchGuard<'a> {
impl SearchableIndex for Index {
type SearchGuard<'a> = NormalIndexSearchGuard<'a>;
- fn guard(&self) -> Self::SearchGuard<'_> {
+ async fn guard(&self) -> Self::SearchGuard<'_> {
NormalIndexSearchGuard { search_index: self }
}
- fn set_snippet_config(&mut self, config: SnippetConfig) {
+ async fn set_snippet_config(&mut self, config: SnippetConfig) {
self.inverted_index.set_snippet_config(config);
}
}
@@ -79,14 +81,14 @@ impl<'a> SearchGuard<'a> for NormalIndexSearchGuard<'a> {
impl SearchableIndex for Arc<live_index::LiveIndex> {
type SearchGuard<'a> = LiveIndexSearchGuard<'a>;
- fn guard(&self) -> Self::SearchGuard<'_> {
+ async fn guard(&self) -> Self::SearchGuard<'_> {
LiveIndexSearchGuard {
- lock_guard: self.read(),
+ lock_guard: self.read().await,
}
}
- fn set_snippet_config(&mut self, config: SnippetConfig) {
- live_index::LiveIndex::set_snippet_config(self, config)
+ async fn set_snippet_config(&mut self, config: SnippetConfig) {
+ live_index::LiveIndex::set_snippet_config(self, config).await
}
}
@@ -146,8 +148,8 @@ where
self.collector_config = config;
}
- pub fn set_snippet_config(&mut self, config: SnippetConfig) {
- self.index.set_snippet_config(config);
+ pub async fn set_snippet_config(&mut self, config: SnippetConfig) {
+ self.index.set_snippet_config(config).await;
}
fn parse_query<'a, G: SearchGuard<'a>>(
@@ -234,12 +236,12 @@ where
&self.index
}
- pub fn search_initial(
+ pub async fn search_initial(
&self,
query: &SearchQuery,
de_rank_similar: bool,
) -> Result<InitialWebsiteResult> {
- let guard = self.index.guard();
+ let guard = self.index.guard().await;
let ctx = guard.inverted_index().local_search_ctx();
let inverted_index_result =
self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;
@@ -250,12 +252,12 @@ where
})
}
- pub fn retrieve_websites(
+ pub async fn retrieve_websites(
&self,
websites: &[inverted_index::WebpagePointer],
query: &str,
) -> Result<Vec<inverted_index::RetrievedWebpage>> {
- let guard = self.index.guard();
+ let guard = self.index.guard().await;
let ctx = guard.inverted_index().local_search_ctx();
let query = SearchQuery {
query: query.to_string(),
@@ -266,14 +268,13 @@ where
guard.inverted_index().retrieve_websites(websites, &query)
}
- /// This function is mainly used for tests and benchmarks
- pub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {
+ pub async fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {
use std::time::Instant;
let start = Instant::now();
let search_query = query.clone();
- let search_result = self.search_initial(&search_query, true)?;
+ let search_result = self.search_initial(&search_query, true).await?;
let pointers: Vec<_> = search_result
.websites
@@ -282,7 +283,8 @@ where
.collect();
let websites: Vec<_> = self
- .retrieve_websites(&pointers, &query.query)?
+ .retrieve_websites(&pointers, &query.query)
+ .await?
.into_iter()
.zip_eq(search_result.websites)
.map(|(webpage, ranking)| {
@@ -296,7 +298,9 @@ where
.map(|website| website.ranking().pointer().clone())
.collect();
- let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;
+ let retrieved_sites = self
+ .retrieve_websites(&pointers, &search_query.query)
+ .await?;
let coefficients = query.signal_coefficients();
@@ -332,21 +336,31 @@ where
})
}
- pub fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
- self.index.guard().inverted_index().get_webpage(url)
+ /// This function is mainly used for tests and benchmarks
+ pub fn search_sync(&self, query: &SearchQuery) -> Result<WebsitesResult> {
+ crate::block_on(self.search(query))
+ }
+
+ pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
+ self.index.guard().await.inverted_index().get_webpage(url)
}
- pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
- self.index.guard().inverted_index().get_homepage(url)
+ pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
+ self.index.guard().await.inverted_index().get_homepage(url)
}
- pub fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
- self.index.guard().inverted_index().top_key_phrases(top_n)
+ pub async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
+ self.index
+ .guard()
+ .await
+ .inverted_index()
+ .top_key_phrases(top_n)
}
- pub fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
+ pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
self.index
.guard()
+ .await
.inverted_index()
.get_site_urls(site, offset, limit)
}
@@ -398,7 +412,7 @@ mod tests {
for p in 0..NUM_PAGES {
let urls: Vec<_> = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "test".to_string(),
page: p,
..Default::default()
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index 6bc3dfc2..e5214443 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -416,7 +416,7 @@ Survey in 2016, 2017, and 2018."#;
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "rust language".to_string(),
..Default::default()
})
@@ -455,7 +455,7 @@ Survey in 2016, 2017, and 2018."#;
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "describe".to_string(),
..Default::default()
})
@@ -494,7 +494,7 @@ Survey in 2016, 2017, and 2018."#;
let searcher = LocalSearcher::from(index);
let result = searcher
- .search(&SearchQuery {
+ .search_sync(&SearchQuery {
query: "paradigms".to_string(),
..Default::default()
})
diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rs
index 09fe63b0..b27889e3 100644
--- a/crates/tantivy/src/indexer/index_writer.rs
+++ b/crates/tantivy/src/indexer/index_writer.rs
@@ -7,7 +7,7 @@ use smallvec::smallvec;
use super::operation::{AddOperation, UserOperation};
use super::segment_updater::SegmentUpdater;
-use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};
+use super::{AddBatch, AddBatchReceiver, AddBatchSender, MergeOperation, PreparedCommit};
use crate::directory::GarbageCollectionResult;
use crate::error::TantivyError;
use crate::index::{Index, Segment, SegmentId, SegmentMeta};
@@ -331,7 +331,33 @@ impl<D: Document> IndexWriter<D> {
pub fn merge(&mut self, segment_ids: &[SegmentId]) -> FutureResult<Option<SegmentMeta>> {
let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
let segment_updater = self.segment_updater.clone();
- segment_updater.start_merge(merge_operation)
+ segment_updater.merge(merge_operation)
+ }
+
+ /// Start a merge operation.
+ ///
+ /// Returns a future that resolves to a tuple of the segment entry and the merge operation.
+ /// When the future is resolved, the merged segment has been created and committed.
+ ///
+ /// `end_merge` must be called to complete the merge operation.
+ pub fn start_merge(
+ &self,
+ segment_ids: &[SegmentId],
+ ) -> FutureResult<(Option<SegmentEntry>, MergeOperation)> {
+ let merge_operation = self.segment_updater.make_merge_operation(segment_ids);
+ self.segment_updater.start_merge(merge_operation)
+ }
+
+ /// End a merge operation.
+ ///
+ /// This method must be called to perform the necessary cleanup after a merge operation.
+ pub fn end_merge(
+ &mut self,
+ merge_operation: MergeOperation,
+ segment_entry: Option<SegmentEntry>,
+ ) -> crate::Result<Option<SegmentMeta>> {
+ self.segment_updater
+ .end_merge(merge_operation, segment_entry)
}
/// Closes the current document channel send.
diff --git a/crates/tantivy/src/indexer/segment_updater.rs b/crates/tantivy/src/indexer/segment_updater.rs
index 88793188..5a5edf97 100644
--- a/crates/tantivy/src/indexer/segment_updater.rs
+++ b/crates/tantivy/src/indexer/segment_updater.rs
@@ -445,7 +445,7 @@ impl SegmentUpdater {
pub fn start_merge(
&self,
merge_operation: MergeOperation,
- ) -> FutureResult<Option<SegmentMeta>> {
+ ) -> FutureResult<(Option<SegmentEntry>, MergeOperation)> {
assert!(
!merge_operation.segment_ids().is_empty(),
"Segment_ids cannot be empty."
@@ -478,8 +478,8 @@ impl SegmentUpdater {
// candidate for another merge.
match merge(&segment_updater.index, segment_entries) {
Ok(after_merge_segment_entry) => {
- let res = segment_updater.end_merge(merge_operation, after_merge_segment_entry);
- let _send_result = merging_future_send.send(res);
+ let _send_result =
+ merging_future_send.send(Ok((after_merge_segment_entry, merge_operation)));
}
Err(merge_error) => {
warn!(
@@ -530,14 +530,21 @@ impl SegmentUpdater {
merge_candidates.extend(committed_merge_candidates);
for merge_operation in merge_candidates {
- // If a merge cannot be started this is not a fatal error.
- // We do log a warning in `start_merge`.
- drop(self.start_merge(merge_operation));
+ let segment_updater = self.clone();
+ self.merge_thread_pool.spawn(move || {
+ // If a merge cannot be started this is not a fatal error.
+ // We do log a warning in `start_merge`.
+ if let Ok((entry, merge_operation)) =
+ segment_updater.start_merge(merge_operation).wait()
+ {
+ let _ = segment_updater.end_merge(merge_operation, entry);
+ }
+ });
}
}
/// Queues a `end_merge` in the segment updater and blocks until it is successfully processed.
- fn end_merge(
+ pub(crate) fn end_merge(
&self,
merge_operation: MergeOperation,
after_merge_segment_entry: Option<SegmentEntry>,
@@ -572,6 +579,23 @@ impl SegmentUpdater {
Ok(after_merge_segment_meta)
}
+ /// Merge a list of segments.
+ ///
+ /// The future will resolve when the merge is complete and the resulting segment is committed.
+ pub fn merge(&self, merge_operation: MergeOperation) -> FutureResult<Option<SegmentMeta>> {
+ let segment_updater = self.clone();
+ let (scheduled_result, sender) = FutureResult::create("Merge operation failed.");
+ self.merge_thread_pool.spawn(move || {
+ if let Ok((entry, merge_operation)) =
+ segment_updater.start_merge(merge_operation).wait()
+ {
+ let segment_meta = segment_updater.end_merge(merge_operation, entry);
+ let _ = sender.send(segment_meta);
+ }
+ });
+ scheduled_result
+ }
+
/// Wait for current merging threads.
///
/// Upon termination of the current merging threads,
diff --git a/fuzz/fuzz_targets/query_parser.rs b/fuzz/fuzz_targets/query_parser.rs
index facb424d..ea03cf23 100644
--- a/fuzz/fuzz_targets/query_parser.rs
+++ b/fuzz/fuzz_targets/query_parser.rs
@@ -10,7 +10,7 @@ use stract::{
fuzz_target!(|query: &str| {
let index = Index::open("/tmp/stract/fuzz-index").unwrap();
- let guard = index.guard();
+ let guard = stract::block_on(index.guard());
let ctx = guard.inverted_index().local_search_ctx();
let _ = Query::parse(