web search done right
improve compaction performance in live index by performing initial segment merge on a read lock and only switch to the new segments on a write lock. this ensures that search requests can still be performed while the heavy part of merge is executing
Mikkel Denker3a127af0a780
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex c338cbe6..be538ed1 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -51,17 +51,20 @@ pub async fn main() {let mut queries: Vec<_> = searcher.top_key_phrases(1_000_000)+ .await.into_iter().map(|phrase| phrase.text().to_string()).collect();queries.shuffle(&mut rand::thread_rng());searcher.set_collector_config(collector_conf);- searcher.set_snippet_config(SnippetConfig {- num_words_for_lang_detection: Some(250),- max_considered_words: Some(10_000),- ..Default::default()- });+ searcher+ .set_snippet_config(SnippetConfig {+ num_words_for_lang_detection: Some(250),+ max_considered_words: Some(10_000),+ ..Default::default()+ })+ .await;let bangs = Bangs::from_path(config.bangs_path.as_ref().unwrap());let searcher = stract::searcher::LocalSearchClient::from(searcher);diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsindex 37e94139..be27bf74 100644--- a/crates/core/src/entrypoint/live_index/search_server.rs+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -134,8 +134,8 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)if let Some(other) = others.pop() {let mut conn: sonic::service::Connection<LiveIndexService> =sonic::service::Connection::create(other).await?;- index.delete_all_pages();- let local_path = index.path();+ index.delete_all_pages().await;+ let local_path = index.path().await;let remote_path = conn.send(GetIndexPath).await?;remote_cp::Request::download(@@ -146,7 +146,7 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)},).await;- index.re_open().unwrap();+ index.re_open().await.unwrap();}let mut wal = temp_wal@@ -157,9 +157,9 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)for pages in wal.iter().unwrap().chunks(512).into_iter() {let pages: Vec<_> = pages.into_iter().collect();- index.insert(&pages);+ index.insert(&pages).await;}- index.commit();+ index.commit().await;wal.clear().unwrap();@@ -241,7 +241,9 @@ impl LiveIndexService {let temp_wal = self.temp_wal.clone();let cluster = self.cluster_handle.clone();- tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });+ std::thread::spawn(move || {+ crate::block_on(setup(index, cluster, temp_wal)).unwrap();+ });}async fn index_webpages_in_replicas(@@ -330,6 +332,7 @@ impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {server.local_searcher.retrieve_websites(&self.websites, &self.query)+ .await.ok()}}@@ -337,18 +340,21 @@ impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {impl sonic::service::Message<LiveIndexService> for Search {type Response = Option<InitialWebsiteResult>;async fn handle(self, server: &LiveIndexService) -> Self::Response {- server.local_searcher.search_initial(&self.query, true).ok()+ server+ .local_searcher+ .search_initial(&self.query, true)+ .await+ .ok()}}impl sonic::service::Message<LiveIndexService> for GetSiteUrls {type Response = SiteUrls;async fn handle(self, server: &LiveIndexService) -> Self::Response {- let urls = server.local_searcher.get_site_urls(- &self.site,- self.offset as usize,- self.limit as usize,- );+ let urls = server+ .local_searcher+ .get_site_urls(&self.site, self.offset as usize, self.limit as usize)+ .await;SiteUrls { urls }}@@ -408,7 +414,7 @@ impl sonic::service::Message<LiveIndexService> for IndexWebpages {if let Some(wal) = server.temp_wal.lock().await.as_mut() {wal.batch_write(self.pages.iter()).unwrap();} else {- server.index.insert(&self.pages);+ server.index.insert(&self.pages).await;if let Some(consistency_fraction) = self.consistency_fraction {server@@ -428,7 +434,7 @@ impl sonic::service::Message<LiveIndexService> for GetIndexPath {type Response = PathBuf;async fn handle(self, server: &LiveIndexService) -> Self::Response {- server.index.path()+ server.index.path().await}}diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rsindex c605dd74..4ecadf13 100644--- a/crates/core/src/entrypoint/live_index/tests.rs+++ b/crates/core/src/entrypoint/live_index/tests.rs@@ -171,7 +171,7 @@ impl RemoteIndex {}async fn commit_underlying(&self) {- self.underlying_index.commit();+ self.underlying_index.commit().await;}async fn kill(self) -> Result<()> {@@ -415,28 +415,30 @@ async fn test_meta_segments() -> Result<()> {};let index = LiveIndex::new(&config.index_path, indexer_config.clone()).await?;- assert!(index.meta().segments().is_empty());+ assert!(index.meta().await.segments().is_empty());- index.insert(&[IndexableWebpage {- url: "https://a.com/".to_string(),- body: "+ index+ .insert(&[IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "<title>test page</title>Example webpage"- .to_string(),- fetch_time_ms: 100,- }]);- index.commit();+ .to_string(),+ fetch_time_ms: 100,+ }])+ .await;+ index.commit().await;- assert_eq!(index.meta().segments().len(), 1);+ assert_eq!(index.meta().await.segments().len(), 1);- index.re_open()?;+ index.re_open().await?;- assert_eq!(index.meta().segments().len(), 1);+ assert_eq!(index.meta().await.segments().len(), 1);let copy_index = LiveIndex::new(&config.index_path, indexer_config).await?;- assert_eq!(copy_index.meta().segments().len(), 1);+ assert_eq!(copy_index.meta().await.segments().len(), 1);Ok(())}@@ -455,42 +457,48 @@ async fn test_segment_compaction() -> Result<()> {let index = Arc::new(LiveIndex::new(&config.index_path, indexer_config).await?);- index.insert(&[IndexableWebpage {- url: "https://a.com/".to_string(),- body: "+ index+ .insert(&[IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "<title>test page</title>Example webpage"- .to_string(),- fetch_time_ms: 100,- }]);+ .to_string(),+ fetch_time_ms: 100,+ }])+ .await;- index.commit();+ index.commit().await;- index.insert(&[IndexableWebpage {- url: "https://b.com/".to_string(),- body: "+ index+ .insert(&[IndexableWebpage {+ url: "https://b.com/".to_string(),+ body: "<title>test page</title>Example webpage"- .to_string(),- fetch_time_ms: 100,- }]);+ .to_string(),+ fetch_time_ms: 100,+ }])+ .await;- index.commit();+ index.commit().await;- assert_eq!(index.meta().segments().len(), 2);+ assert_eq!(index.meta().await.segments().len(), 2);- index.compact_segments_by_date();+ index.compact_segments_by_date().await?;- assert_eq!(index.meta().segments().len(), 1);+ assert_eq!(index.meta().await.segments().len(), 1);let searcher = LocalSearcher::from(index.clone());- let res = searcher.search(&SearchQuery {- query: "test".to_string(),- ..Default::default()- })?;+ let res = searcher+ .search(&SearchQuery {+ query: "test".to_string(),+ ..Default::default()+ })+ .await?;assert_eq!(res.webpages.len(), 2);diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 9316f633..13096fd1 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -71,7 +71,7 @@ impl SearchService {}local_searcher.set_collector_config(config.collector);- local_searcher.set_snippet_config(config.snippet);+ local_searcher.set_snippet_config(config.snippet).await;let cluster_handle = Cluster::join(Member::new(Service::Searcher {@@ -101,6 +101,7 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {server.local_searcher.retrieve_websites(&self.websites, &self.query)+ .await.ok()}}@@ -112,7 +113,11 @@ pub struct Search {impl sonic::service::Message<SearchService> for Search {type Response = Option<InitialWebsiteResult>;async fn handle(self, server: &SearchService) -> Self::Response {- server.local_searcher.search_initial(&self.query, true).ok()+ server+ .local_searcher+ .search_initial(&self.query, true)+ .await+ .ok()}}@@ -123,7 +128,7 @@ pub struct GetWebpage {impl sonic::service::Message<SearchService> for GetWebpage {type Response = Option<RetrievedWebpage>;async fn handle(self, server: &SearchService) -> Self::Response {- server.local_searcher.get_webpage(&self.url)+ server.local_searcher.get_webpage(&self.url).await}}@@ -138,7 +143,7 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {let mut result = HashMap::with_capacity(self.urls.len());for url in &self.urls {- if let Some(homepage) = server.local_searcher.get_homepage(url) {+ if let Some(homepage) = server.local_searcher.get_homepage(url).await {if let Some(desc) = homepage.description() {result.insert(url.clone(), desc.clone());}@@ -156,7 +161,7 @@ pub struct TopKeyPhrases {impl sonic::service::Message<SearchService> for TopKeyPhrases {type Response = Vec<KeyPhrase>;async fn handle(self, server: &SearchService) -> Self::Response {- server.local_searcher.top_key_phrases(self.top_n)+ server.local_searcher.top_key_phrases(self.top_n).await}}@@ -181,6 +186,7 @@ impl sonic::service::Message<SearchService> for Size {.local_searcher.index().guard()+ .await.inverted_index().num_documents(),}@@ -203,11 +209,10 @@ pub struct SiteUrls {impl sonic::service::Message<SearchService> for GetSiteUrls {type Response = SiteUrls;async fn handle(self, server: &SearchService) -> Self::Response {- let urls = server.local_searcher.get_site_urls(- &self.site,- self.offset as usize,- self.limit as usize,- );+ let urls = server+ .local_searcher+ .get_site_urls(&self.site, self.offset as usize, self.limit as usize)+ .await;SiteUrls { urls }}diff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex b93cd2ab..03b8f906 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -232,7 +232,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test".to_string(),return_ranking_signals: true,..Default::default()diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rsindex 9a4a62c2..c077df92 100644--- a/crates/core/src/inverted_index/indexing.rs+++ b/crates/core/src/inverted_index/indexing.rs@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>use tantivy::index::SegmentId;+use tantivy::indexer::{MergeOperation, SegmentEntry};use tantivy::merge_policy::NoMergePolicy;use tantivy::{IndexWriter, SegmentMeta};@@ -153,20 +154,35 @@ impl InvertedIndex {Ok(())}- #[allow(clippy::missing_panics_doc)] // should not panic as writer is prepared- pub fn merge_segments_by_id(&mut self, segments: &[SegmentId]) -> Result<Option<SegmentId>> {- self.prepare_writer()?;-+ pub async fn start_merge_segments_by_id(+ &self,+ segments: &[SegmentId],+ ) -> Result<(Option<SegmentEntry>, MergeOperation)> {if segments.is_empty() {- return Ok(None);+ anyhow::bail!("no segments to merge");}+ let (entry, op) = self+ .writer+ .as_ref()+ .expect("writer has not been prepared")+ .start_merge(segments)+ .await?;++ Ok((entry, op))+ }++ pub fn end_merge_segments_by_id(+ &mut self,+ merge_operation: MergeOperation,+ segment_entry: Option<SegmentEntry>,+ ) -> Result<Option<SegmentId>> {+ self.prepare_writer()?;let res = self.writer.as_mut().expect("writer has not been prepared")- .merge(segments)- .wait()?;+ .end_merge(merge_operation, segment_entry)?;Ok(res.map(|seg| seg.id()))}diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex 5058afd5..cd3d1e1f 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -17,19 +17,25 @@use std::{io::{BufReader, BufWriter},path::{Path, PathBuf},- sync::{Arc, RwLock, RwLockReadGuard},+ sync::Arc,};+use tokio::sync::{RwLock, RwLockReadGuard};+use chrono::{DateTime, NaiveDate, Utc};use itertools::Itertools;use simple_wal::Wal;-use tantivy::index::SegmentId;+use tantivy::{+ index::SegmentId,+ indexer::{MergeOperation, SegmentEntry},+};use std::collections::{HashMap, HashSet};use crate::{config::SnippetConfig,entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},+ inverted_index::InvertedIndex,live_index::{BATCH_SIZE, TTL},searcher::SearchableIndex,Result,@@ -51,6 +57,19 @@ impl Segment {}}+pub struct CompactOperation {+ segments: Vec<Segment>,+ entry: Option<SegmentEntry>,+ merge_op: MergeOperation,+}++impl CompactOperation {+ pub fn end(self, index: &mut InvertedIndex) -> Result<Option<SegmentId>> {+ let res = index.end_merge_segments_by_id(self.merge_op, self.entry)?;+ Ok(res)+ }+}+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]pub struct Meta {segments: Vec<Segment>,@@ -145,20 +164,40 @@ impl InnerIndex {self.re_open();}- pub fn compact_segments_by_date(&mut self) {+ pub async fn start_compact_segments_by_date(&self) -> Result<Vec<CompactOperation>> {let segments_to_compact = self.prepare_segments_for_compaction();+ let mut operations = Vec::new();+for (_, segments) in segments_to_compact {if segments.len() <= 1 {continue;}let segment_ids: Vec<SegmentId> = segments.iter().map(|s| s.id).collect();- let newest_creation_date = segments.iter().map(|s| s.created).max().unwrap();- let merge_result = self.index.inverted_index.merge_segments_by_id(&segment_ids);+ let (entry, merge_op) = self+ .index+ .inverted_index+ .start_merge_segments_by_id(&segment_ids)+ .await?;++ operations.push(CompactOperation {+ segments,+ entry,+ merge_op,+ });+ }++ Ok(operations)+ }- if let Ok(Some(new_segment_id)) = merge_result {+ fn end_compact_segments_by_date(&mut self, operations: Vec<CompactOperation>) -> Result<()> {+ for op in operations {+ let newest_creation_date = op.segments.iter().map(|s| s.created).max().unwrap();+ let segment_ids: Vec<SegmentId> = op.segments.iter().map(|s| s.id).collect();++ if let Ok(Some(new_segment_id)) = op.end(&mut self.index.inverted_index) {self.update_meta_after_compaction(segment_ids,new_segment_id,@@ -169,6 +208,8 @@ impl InnerIndex {self.save_meta();self.re_open();++ Ok(())}fn prepare_segments_for_compaction(&self) -> HashMap<NaiveDate, Vec<Segment>> {@@ -323,89 +364,70 @@ impl LiveIndex {})}- pub fn commit(&self) {+ pub async fn commit(&self) {tracing::debug!("committing index");- futures::executor::block_on(- self.inner- .write()- .unwrap_or_else(|e| e.into_inner())- .commit(),- );- tracing::debug!("index committed");+ self.inner.write().await.commit().await;}- pub fn prune_segments(&self) {+ pub async fn prune_segments(&self) {tracing::debug!("pruning segments");- self.inner- .write()- .unwrap_or_else(|e| e.into_inner())- .prune_segments()+ self.inner.write().await.prune_segments()}- pub fn has_inserts(&self) -> bool {- self.inner- .read()- .unwrap_or_else(|e| e.into_inner())- .has_inserts()+ pub async fn has_inserts(&self) -> bool {+ self.inner.read().await.has_inserts()}- pub fn compact_segments_by_date(&self) {+ pub async fn compact_segments_by_date(&self) -> Result<()> {tracing::debug!("compacting segments by date");+ let operations = self+ .inner+ .read()+ .await+ .start_compact_segments_by_date()+ .await?;+self.inner.write()- .unwrap_or_else(|e| e.into_inner())- .compact_segments_by_date()+ .await+ .end_compact_segments_by_date(operations)?;++ Ok(())}- pub fn insert(&self, pages: &[IndexableWebpage]) {+ pub async fn insert(&self, pages: &[IndexableWebpage]) {tracing::debug!("inserting {} pages into index", pages.len());- self.inner- .write()- .unwrap_or_else(|e| e.into_inner())- .insert(pages)+ self.inner.write().await.insert(pages)}- pub fn read(&self) -> RwLockReadGuard<'_, InnerIndex> {- self.inner.read().unwrap_or_else(|e| e.into_inner())+ pub async fn read(&self) -> RwLockReadGuard<'_, InnerIndex> {+ self.inner.read().await}- pub fn set_snippet_config(&self, config: SnippetConfig) {+ pub async fn set_snippet_config(&self, config: SnippetConfig) {self.inner.write()- .unwrap_or_else(|e| e.into_inner())+ .await.index.set_snippet_config(config)+ .await}- pub fn path(&self) -> PathBuf {- self.inner- .read()- .unwrap_or_else(|e| e.into_inner())- .path- .to_path_buf()+ pub async fn path(&self) -> PathBuf {+ self.inner.read().await.path.to_path_buf()}- pub fn delete_all_pages(&self) {- self.inner- .write()- .unwrap_or_else(|e| e.into_inner())- .delete_all_pages();+ pub async fn delete_all_pages(&self) {+ self.inner.write().await.delete_all_pages();}- pub fn re_open(&self) -> Result<()> {- self.inner- .write()- .unwrap_or_else(|e| e.into_inner())- .re_open();+ pub async fn re_open(&self) -> Result<()> {+ self.inner.write().await.re_open();Ok(())}- pub fn meta(&self) -> Meta {- self.inner- .read()- .unwrap_or_else(|e| e.into_inner())- .meta- .clone()+ pub async fn meta(&self) -> Meta {+ self.inner.read().await.meta.clone()}}diff --git a/crates/core/src/live_index/index_manager.rs b/crates/core/src/live_index/index_manager.rsindex 1f0e902a..966662af 100644--- a/crates/core/src/live_index/index_manager.rs+++ b/crates/core/src/live_index/index_manager.rs@@ -38,17 +38,22 @@ impl IndexManager {loop {if last_prune + PRUNE_INTERVAL < Utc::now() {- self.index.prune_segments();+ crate::block_on(self.index.prune_segments());last_prune = Utc::now();}- if last_commit + AUTO_COMMIT_INTERVAL < Utc::now() && self.index.has_inserts() {- self.index.commit();+ if last_commit + AUTO_COMMIT_INTERVAL < Utc::now()+ && crate::block_on(self.index.has_inserts())+ {+ crate::block_on(self.index.commit());last_commit = Utc::now();}if last_compact + COMPACT_INTERVAL < Utc::now() {- self.index.compact_segments_by_date();+ if let Err(e) = crate::block_on(self.index.compact_segments_by_date()) {+ tracing::error!("Failed to compact segments: {}", e);+ }+last_compact = Utc::now();}diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 6382f52f..fbcb479a 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -384,7 +384,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.second.com/");}@@ -454,7 +454,7 @@ mod tests {query: "test site:first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -462,7 +462,7 @@ mod tests {query: "test site:www.first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -470,7 +470,7 @@ mod tests {query: "test -site:first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 2);assert!(result@@ -528,7 +528,7 @@ mod tests {query: "test linksto:first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.second.com/example/abc");@@ -536,7 +536,7 @@ mod tests {query: "test linkto:www.first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.second.com/example/abc");@@ -544,7 +544,7 @@ mod tests {query: "test -linkto:first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -552,7 +552,7 @@ mod tests {query: "test linkto:second.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -560,7 +560,7 @@ mod tests {query: "test linkto:www.second.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -568,7 +568,7 @@ mod tests {query: "test linkto:second.com/example".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -576,7 +576,7 @@ mod tests {query: "test linksto:second.com/example/abc".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");}@@ -630,7 +630,7 @@ mod tests {query: "test linkto:second.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");}@@ -682,7 +682,7 @@ mod tests {query: "intitle:website".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");}@@ -734,7 +734,7 @@ mod tests {query: "test inurl:forum".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/forum");}@@ -829,14 +829,14 @@ mod tests {query: "test site:first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 0);let query = SearchQuery {query: "test site:the-first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.the-first.com/");@@ -844,7 +844,7 @@ mod tests {query: "test site:www.the-first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.the-first.com/");}@@ -902,7 +902,7 @@ mod tests {query: "\"Test website\"".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -910,7 +910,7 @@ mod tests {query: "\"Test website\" site:www.second.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 0);}@@ -970,7 +970,7 @@ mod tests {..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 2);let query = SearchQuery {@@ -978,7 +978,7 @@ mod tests {..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 2);}@@ -1053,7 +1053,7 @@ mod tests {..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 2);let query = SearchQuery {@@ -1062,7 +1062,7 @@ mod tests {..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.sfw.com/");@@ -1142,28 +1142,28 @@ mod tests {query: "test site:.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 2);let query = SearchQuery {query: "test site:.com/example".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);let query = SearchQuery {query: "test site:first.com/example".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);let query = SearchQuery {query: "test site:first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);let query = SearchQuery {@@ -1171,7 +1171,7 @@ mod tests {..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);}@@ -1249,14 +1249,14 @@ mod tests {query: "test exacturl:https://www.first.com/example".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);let query = SearchQuery {query: "test exacturl:https://www.first.com".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 0);}@@ -1334,7 +1334,7 @@ mod tests {query: "\"test test\" website".to_string(),..Default::default()};- let result = searcher.search(&query).expect("Search failed");+ let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 2);}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 5a4df880..ab5dbecc 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -323,7 +323,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),signal_coefficients: enum_map! {crate::ranking::SignalEnum::from(crate::ranking::signals::core::Bm25Title) => 1_000_000.0@@ -340,7 +340,7 @@ mod tests {assert_eq!(res[1].url, "https://www.a.com/");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(@@ -364,7 +364,7 @@ mod tests {assert_eq!(res[0].url, "https://www.a.com/");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(@@ -444,7 +444,7 @@ mod tests {let searcher = LocalSearcher::from(index);let _ = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(include_str!(@@ -458,7 +458,7 @@ mod tests {.webpages;let _ = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(include_str!(@@ -472,7 +472,7 @@ mod tests {.webpages;let _ = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(include_str!(@@ -568,7 +568,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(@@ -844,7 +844,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(@@ -868,7 +868,7 @@ mod tests {assert_eq!(res[0].url, "https://www.b.com/");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(@@ -892,7 +892,7 @@ mod tests {assert_eq!(res[0].url, "https://www.b.com/");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(@@ -916,7 +916,7 @@ mod tests {assert_eq!(res[0].url, "https://www.a.com/");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "website".to_string(),optic: Some(Optic::parse(@@ -973,7 +973,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "site:stackoverflow.com".to_string(),optic: Some(Optic::parse(@@ -1063,7 +1063,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1125,7 +1125,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),..Default::default()})@@ -1135,7 +1135,7 @@ mod tests {assert_eq!(res[0].url, "https://example.com/");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"is\") }, Action(Discard) }").unwrap(),@@ -1147,7 +1147,7 @@ mod tests {assert_eq!(res.len(), 0);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"|is\") }, Action(Discard) }").unwrap(),@@ -1159,7 +1159,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"|This\") }, Action(Discard) }").unwrap(),@@ -1171,7 +1171,7 @@ mod tests {assert_eq!(res.len(), 0);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"|This an\") }, Action(Discard) }")@@ -1184,7 +1184,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"|This * an\") }, Action(Discard) }")@@ -1197,7 +1197,7 @@ mod tests {assert_eq!(res.len(), 0);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Site(\"example.com\") }, Action(Discard) }")@@ -1210,7 +1210,7 @@ mod tests {assert_eq!(res.len(), 0);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Site(\"|example.com\") }, Action(Discard) }")@@ -1223,7 +1223,7 @@ mod tests {assert_eq!(res.len(), 0);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Site(\"|example.com|\") }, Action(Discard) }")@@ -1236,7 +1236,7 @@ mod tests {assert_eq!(res.len(), 0);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"website.com|\") }, Action(Discard) }")@@ -1282,7 +1282,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1297,7 +1297,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1350,7 +1350,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("").unwrap()),..Default::default()@@ -1360,7 +1360,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"\") }, Action(Discard) }").unwrap(),@@ -1430,7 +1430,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"*\") }, Action(Discard) }").unwrap(),@@ -1442,7 +1442,7 @@ mod tests {assert_eq!(res.len(), 0);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"* is\") }, Action(Discard) }").unwrap(),@@ -1454,7 +1454,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"* This is\") }, Action(Discard) }")@@ -1467,7 +1467,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"example *\") }, Action(Discard) }")@@ -1480,7 +1480,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1526,7 +1526,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),..Default::default()})@@ -1535,7 +1535,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("DiscardNonMatching; Rule { Matches { Content(\"||\") }, Action(Boost(0)) }")@@ -1548,7 +1548,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1622,7 +1622,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),..Default::default()})@@ -1631,7 +1631,7 @@ mod tests {assert_eq!(res.len(), 2);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1701,7 +1701,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),..Default::default()})@@ -1710,7 +1710,7 @@ mod tests {assert_eq!(res.len(), 2);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1726,7 +1726,7 @@ mod tests {assert_eq!(res[0].url, "https://example.com/test");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Site(\"|example.com|\") }, Action(Discard) }")@@ -1815,7 +1815,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("Rule { Matches { Title(\"*'s collection\") }, Action(Discard) }")@@ -1903,7 +1903,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("DiscardNonMatching; Rule { Matches { Title(\"*'s collection\") }, Action(Discard) }; Rule { Matches { Site(\"*.com\") } }")@@ -1945,7 +1945,7 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1960,7 +1960,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse(@@ -1975,7 +1975,7 @@ mod tests {assert_eq!(res.len(), 1);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),optic: Some(Optic::parse("DiscardNonMatching; Rule { Matches { Domain(\"|another.example.com|\") } } ")diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex de2e0bfe..e5133904 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -198,7 +198,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),..Default::default()})@@ -264,7 +264,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),..Default::default()})@@ -332,7 +332,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "title".to_string(),return_ranking_signals: true,signal_coefficients: crate::enum_map! {@@ -405,7 +405,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test".to_string(),..Default::default()})@@ -471,7 +471,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test".to_string(),..Default::default()})@@ -557,7 +557,7 @@ mod tests {let searcher = LocalSearcher::new(index);let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),signal_coefficients: crate::enum_map! {@@ -573,7 +573,7 @@ mod tests {assert_eq!(&res.webpages[0].url, "https://www.title.com/");let res = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "example".to_string(),signal_coefficients: crate::enum_map! {@@ -643,7 +643,7 @@ mod tests {let searcher = LocalSearcher::new(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test".to_string(),signal_coefficients: crate::enum_map! {crate::ranking::SignalEnum::from(crate::ranking::signals::FetchTimeMs) => 100_000.0,@@ -736,7 +736,7 @@ mod tests {let searcher = LocalSearcher::new(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test".to_string(),signal_coefficients: crate::enum_map! {@@ -859,7 +859,7 @@ mod tests {.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "best chocolate cake".to_string(),signal_coefficients: crate::enum_map! {@@ -947,7 +947,7 @@ mod tests {.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "best chocolate cake".to_string(),signal_coefficients: crate::enum_map! {@@ -994,7 +994,7 @@ mod tests {let searcher = LocalSearcher::new(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test website".to_string(),return_ranking_signals: true,..Default::default()@@ -1017,7 +1017,7 @@ mod tests {);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test example".to_string(),return_ranking_signals: true,..Default::default()@@ -1074,7 +1074,7 @@ mod tests {let searcher = LocalSearcher::new(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test website".to_string(),return_ranking_signals: true,..Default::default()@@ -1097,7 +1097,7 @@ mod tests {);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test b".to_string(),return_ranking_signals: true,..Default::default()diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex dd927676..b3c4839b 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -458,7 +458,7 @@ impl From<LocalSearcher<Index>> for LocalSearchClient {impl SearchClient for LocalSearchClient {async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {- let res = self.0.search_initial(query, true).unwrap();+ let res = self.0.search_initial(query, true).await.unwrap();vec![InitialSearchResultShard {local_result: res,@@ -479,6 +479,7 @@ impl SearchClient for LocalSearchClient {let res = self.0.retrieve_websites(&pointers, query)+ .await.unwrap().into_iter().zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))@@ -489,7 +490,7 @@ impl SearchClient for LocalSearchClient {}async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- Ok(self.0.get_webpage(url))+ Ok(self.0.get_webpage(url).await)}async fn get_homepage_descriptions(@@ -499,7 +500,7 @@ impl SearchClient for LocalSearchClient {let mut res = std::collections::HashMap::new();for url in urls {- if let Some(homepage) = self.0.get_homepage(url) {+ if let Some(homepage) = self.0.get_homepage(url).await {if let Some(desc) = homepage.description() {res.insert(url.clone(), desc.clone());}@@ -523,6 +524,6 @@ impl SearchClient for LocalSearchClient {}async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {- self.0.top_key_phrases(top_n)+ self.0.top_key_phrases(top_n).await}}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex 58efa36c..31c4cf3a 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -15,7 +15,9 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::collections::HashMap;-use std::sync::{Arc, RwLockReadGuard};+use std::future::Future;+use std::sync::Arc;+use tokio::sync::RwLockReadGuard;use itertools::Itertools;use url::Url;@@ -43,8 +45,8 @@ pub trait SearchableIndex {whereSelf: 'a;- fn guard(&self) -> Self::SearchGuard<'_>;- fn set_snippet_config(&mut self, config: SnippetConfig);+ fn guard(&self) -> impl Future<Output = Self::SearchGuard<'_>>;+ fn set_snippet_config(&mut self, config: SnippetConfig) -> impl Future<Output = ()>;}pub trait SearchGuard<'a> {@@ -57,11 +59,11 @@ pub trait SearchGuard<'a> {impl SearchableIndex for Index {type SearchGuard<'a> = NormalIndexSearchGuard<'a>;- fn guard(&self) -> Self::SearchGuard<'_> {+ async fn guard(&self) -> Self::SearchGuard<'_> {NormalIndexSearchGuard { search_index: self }}- fn set_snippet_config(&mut self, config: SnippetConfig) {+ async fn set_snippet_config(&mut self, config: SnippetConfig) {self.inverted_index.set_snippet_config(config);}}@@ -79,14 +81,14 @@ impl<'a> SearchGuard<'a> for NormalIndexSearchGuard<'a> {impl SearchableIndex for Arc<live_index::LiveIndex> {type SearchGuard<'a> = LiveIndexSearchGuard<'a>;- fn guard(&self) -> Self::SearchGuard<'_> {+ async fn guard(&self) -> Self::SearchGuard<'_> {LiveIndexSearchGuard {- lock_guard: self.read(),+ lock_guard: self.read().await,}}- fn set_snippet_config(&mut self, config: SnippetConfig) {- live_index::LiveIndex::set_snippet_config(self, config)+ async fn set_snippet_config(&mut self, config: SnippetConfig) {+ live_index::LiveIndex::set_snippet_config(self, config).await}}@@ -146,8 +148,8 @@ whereself.collector_config = config;}- pub fn set_snippet_config(&mut self, config: SnippetConfig) {- self.index.set_snippet_config(config);+ pub async fn set_snippet_config(&mut self, config: SnippetConfig) {+ self.index.set_snippet_config(config).await;}fn parse_query<'a, G: SearchGuard<'a>>(@@ -234,12 +236,12 @@ where&self.index}- pub fn search_initial(+ pub async fn search_initial(&self,query: &SearchQuery,de_rank_similar: bool,) -> Result<InitialWebsiteResult> {- let guard = self.index.guard();+ let guard = self.index.guard().await;let ctx = guard.inverted_index().local_search_ctx();let inverted_index_result =self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;@@ -250,12 +252,12 @@ where})}- pub fn retrieve_websites(+ pub async fn retrieve_websites(&self,websites: &[inverted_index::WebpagePointer],query: &str,) -> Result<Vec<inverted_index::RetrievedWebpage>> {- let guard = self.index.guard();+ let guard = self.index.guard().await;let ctx = guard.inverted_index().local_search_ctx();let query = SearchQuery {query: query.to_string(),@@ -266,14 +268,13 @@ whereguard.inverted_index().retrieve_websites(websites, &query)}- /// This function is mainly used for tests and benchmarks- pub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {+ pub async fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {use std::time::Instant;let start = Instant::now();let search_query = query.clone();- let search_result = self.search_initial(&search_query, true)?;+ let search_result = self.search_initial(&search_query, true).await?;let pointers: Vec<_> = search_result.websites@@ -282,7 +283,8 @@ where.collect();let websites: Vec<_> = self- .retrieve_websites(&pointers, &query.query)?+ .retrieve_websites(&pointers, &query.query)+ .await?.into_iter().zip_eq(search_result.websites).map(|(webpage, ranking)| {@@ -296,7 +298,9 @@ where.map(|website| website.ranking().pointer().clone()).collect();- let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;+ let retrieved_sites = self+ .retrieve_websites(&pointers, &search_query.query)+ .await?;let coefficients = query.signal_coefficients();@@ -332,21 +336,31 @@ where})}- pub fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {- self.index.guard().inverted_index().get_webpage(url)+ /// This function is mainly used for tests and benchmarks+ pub fn search_sync(&self, query: &SearchQuery) -> Result<WebsitesResult> {+ crate::block_on(self.search(query))+ }++ pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {+ self.index.guard().await.inverted_index().get_webpage(url)}- pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {- self.index.guard().inverted_index().get_homepage(url)+ pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {+ self.index.guard().await.inverted_index().get_homepage(url)}- pub fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {- self.index.guard().inverted_index().top_key_phrases(top_n)+ pub async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {+ self.index+ .guard()+ .await+ .inverted_index()+ .top_key_phrases(top_n)}- pub fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {+ pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {self.index.guard()+ .await.inverted_index().get_site_urls(site, offset, limit)}@@ -398,7 +412,7 @@ mod tests {for p in 0..NUM_PAGES {let urls: Vec<_> = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "test".to_string(),page: p,..Default::default()diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex 6bc3dfc2..e5214443 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -416,7 +416,7 @@ Survey in 2016, 2017, and 2018."#;let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "rust language".to_string(),..Default::default()})@@ -455,7 +455,7 @@ Survey in 2016, 2017, and 2018."#;let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "describe".to_string(),..Default::default()})@@ -494,7 +494,7 @@ Survey in 2016, 2017, and 2018."#;let searcher = LocalSearcher::from(index);let result = searcher- .search(&SearchQuery {+ .search_sync(&SearchQuery {query: "paradigms".to_string(),..Default::default()})diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rsindex 09fe63b0..b27889e3 100644--- a/crates/tantivy/src/indexer/index_writer.rs+++ b/crates/tantivy/src/indexer/index_writer.rs@@ -7,7 +7,7 @@ use smallvec::smallvec;use super::operation::{AddOperation, UserOperation};use super::segment_updater::SegmentUpdater;-use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};+use super::{AddBatch, AddBatchReceiver, AddBatchSender, MergeOperation, PreparedCommit};use crate::directory::GarbageCollectionResult;use crate::error::TantivyError;use crate::index::{Index, Segment, SegmentId, SegmentMeta};@@ -331,7 +331,33 @@ impl<D: Document> IndexWriter<D> {pub fn merge(&mut self, segment_ids: &[SegmentId]) -> FutureResult<Option<SegmentMeta>> {let merge_operation = self.segment_updater.make_merge_operation(segment_ids);let segment_updater = self.segment_updater.clone();- segment_updater.start_merge(merge_operation)+ segment_updater.merge(merge_operation)+ }++ /// Start a merge operation.+ ///+ /// Returns a future that resolves to a tuple of the segment entry and the merge operation.+ /// When the future is resolved, the merged segment has been created and committed.+ ///+ /// `end_merge` must be called to complete the merge operation.+ pub fn start_merge(+ &self,+ segment_ids: &[SegmentId],+ ) -> FutureResult<(Option<SegmentEntry>, MergeOperation)> {+ let merge_operation = self.segment_updater.make_merge_operation(segment_ids);+ self.segment_updater.start_merge(merge_operation)+ }++ /// End a merge operation.+ ///+ /// This method must be called to perform the necessary cleanup after a merge operation.+ pub fn end_merge(+ &mut self,+ merge_operation: MergeOperation,+ segment_entry: Option<SegmentEntry>,+ ) -> crate::Result<Option<SegmentMeta>> {+ self.segment_updater+ .end_merge(merge_operation, segment_entry)}/// Closes the current document channel send.diff --git a/crates/tantivy/src/indexer/segment_updater.rs b/crates/tantivy/src/indexer/segment_updater.rsindex 88793188..5a5edf97 100644--- a/crates/tantivy/src/indexer/segment_updater.rs+++ b/crates/tantivy/src/indexer/segment_updater.rs@@ -445,7 +445,7 @@ impl SegmentUpdater {pub fn start_merge(&self,merge_operation: MergeOperation,- ) -> FutureResult<Option<SegmentMeta>> {+ ) -> FutureResult<(Option<SegmentEntry>, MergeOperation)> {assert!(!merge_operation.segment_ids().is_empty(),"Segment_ids cannot be empty."@@ -478,8 +478,8 @@ impl SegmentUpdater {// candidate for another merge.match merge(&segment_updater.index, segment_entries) {Ok(after_merge_segment_entry) => {- let res = segment_updater.end_merge(merge_operation, after_merge_segment_entry);- let _send_result = merging_future_send.send(res);+ let _send_result =+ merging_future_send.send(Ok((after_merge_segment_entry, merge_operation)));}Err(merge_error) => {warn!(@@ -530,14 +530,21 @@ impl SegmentUpdater {merge_candidates.extend(committed_merge_candidates);for merge_operation in merge_candidates {- // If a merge cannot be started this is not a fatal error.- // We do log a warning in `start_merge`.- drop(self.start_merge(merge_operation));+ let segment_updater = self.clone();+ self.merge_thread_pool.spawn(move || {+ // If a merge cannot be started this is not a fatal error.+ // We do log a warning in `start_merge`.+ if let Ok((entry, merge_operation)) =+ segment_updater.start_merge(merge_operation).wait()+ {+ let _ = segment_updater.end_merge(merge_operation, entry);+ }+ });}}/// Queues a `end_merge` in the segment updater and blocks until it is successfully processed.- fn end_merge(+ pub(crate) fn end_merge(&self,merge_operation: MergeOperation,after_merge_segment_entry: Option<SegmentEntry>,@@ -572,6 +579,23 @@ impl SegmentUpdater {Ok(after_merge_segment_meta)}+ /// Merge a list of segments.+ ///+ /// The future will resolve when the merge is complete and the resulting segment is committed.+ pub fn merge(&self, merge_operation: MergeOperation) -> FutureResult<Option<SegmentMeta>> {+ let segment_updater = self.clone();+ let (scheduled_result, sender) = FutureResult::create("Merge operation failed.");+ self.merge_thread_pool.spawn(move || {+ if let Ok((entry, merge_operation)) =+ segment_updater.start_merge(merge_operation).wait()+ {+ let segment_meta = segment_updater.end_merge(merge_operation, entry);+ let _ = sender.send(segment_meta);+ }+ });+ scheduled_result+ }+/// Wait for current merging threads.////// Upon termination of the current merging threads,diff --git a/fuzz/fuzz_targets/query_parser.rs b/fuzz/fuzz_targets/query_parser.rsindex facb424d..ea03cf23 100644--- a/fuzz/fuzz_targets/query_parser.rs+++ b/fuzz/fuzz_targets/query_parser.rs@@ -10,7 +10,7 @@ use stract::{fuzz_target!(|query: &str| {let index = Index::open("/tmp/stract/fuzz-index").unwrap();- let guard = index.guard();+ let guard = stract::block_on(index.guard());let ctx = guard.inverted_index().local_search_ctx();let _ = Query::parse(