web search done right
[search_server] use spawn_blocking for search requests to improve concurrency
Mikkel Denkerb93afff31989
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 442e2b03..11c6da3a 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -1,3 +1,5 @@+use std::sync::Arc;+use optics::HostRankings;use rand::seq::SliceRandom;use stract::{@@ -15,7 +17,7 @@ use stract::{#[tokio::main]pub async fn main() {- let index = Index::open("data/index").unwrap();+ let mut index = Index::open("data/index").unwrap();let collector_conf = CollectorConfig {..Default::default()@@ -47,7 +49,15 @@ pub async fn main() {top_phrases_for_autosuggest: defaults::Api::top_phrases_for_autosuggest(),};- let mut searcher = LocalSearcher::new(index);+ index.inverted_index.set_snippet_config(SnippetConfig {+ num_words_for_lang_detection: Some(250),+ max_considered_words: Some(10_000),+ ..Default::default()+ });++ let searcher = LocalSearcher::builder(Arc::new(index))+ .set_collector_config(collector_conf)+ .build();let mut queries: Vec<_> = searcher.top_key_phrases(1_000_000)@@ -57,14 +67,6 @@ pub async fn main() {.collect();queries.shuffle(&mut rand::thread_rng());- searcher.set_collector_config(collector_conf);- searcher- .set_snippet_config(SnippetConfig {- num_words_for_lang_detection: Some(250),- max_considered_words: Some(10_000),- ..Default::default()- })- .await;let bangs = Bangs::from_path(config.bangs_path.as_ref().unwrap());let searcher = stract::searcher::LocalSearchClient::from(searcher);diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsindex be27bf74..83129dfc 100644--- a/crates/core/src/entrypoint/live_index/search_server.rs+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -216,7 +216,9 @@ impl LiveIndexService {).await?,);- let local_searcher = LocalSearcher::new(index.clone());+ let local_searcher = LocalSearcher::builder(index.clone())+ .set_collector_config(config.collector)+ .build();let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rsindex 4ecadf13..e0effdc3 100644--- a/crates/core/src/entrypoint/live_index/tests.rs+++ b/crates/core/src/entrypoint/live_index/tests.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -491,7 +491,7 @@ async fn test_segment_compaction() -> Result<()> {assert_eq!(index.meta().await.segments().len(), 1);- let searcher = LocalSearcher::from(index.clone());+ let searcher = LocalSearcher::builder(index.clone()).build();let res = searcher.search(&SearchQuery {diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 13096fd1..646b277a 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::collections::HashMap;+use std::{collections::HashMap, sync::Arc};use tracing::info;use url::Url;@@ -30,7 +30,7 @@ use crate::{inverted_index::{self, KeyPhrase, RetrievedWebpage},models::dual_encoder::DualEncoder,ranking::models::linear::LinearRegression,- searcher::{InitialWebsiteResult, LocalSearcher, SearchGuard, SearchQuery, SearchableIndex},+ searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},Result,};@@ -50,7 +50,7 @@ sonic_service!();pub struct SearchService {- local_searcher: LocalSearcher<Index>,+ local_searcher: LocalSearcher<Arc<Index>>,// dropping the handle leaves the cluster#[allow(unused)]cluster_handle: Cluster,@@ -58,20 +58,22 @@ pub struct SearchService {impl SearchService {async fn new(config: config::SearchServerConfig) -> Result<Self> {- let search_index = Index::open(config.index_path)?;+ let mut search_index = Index::open(config.index_path)?;+ search_index+ .inverted_index+ .set_snippet_config(config.snippet);- let mut local_searcher = LocalSearcher::new(search_index);+ let mut local_searcher = LocalSearcher::builder(Arc::new(search_index));if let Some(model_path) = config.linear_model_path {- local_searcher.set_linear_model(LinearRegression::open(model_path)?);+ local_searcher = local_searcher.set_linear_model(LinearRegression::open(model_path)?);}if let Some(model_path) = config.dual_encoder_model_path {- local_searcher.set_dual_encoder(DualEncoder::open(model_path)?);+ local_searcher = local_searcher.set_dual_encoder(DualEncoder::open(model_path)?);}- local_searcher.set_collector_config(config.collector);- local_searcher.set_snippet_config(config.snippet).await;+ local_searcher = local_searcher.set_collector_config(config.collector);let cluster_handle = Cluster::join(Member::new(Service::Searcher {@@ -84,7 +86,7 @@ impl SearchService {.await?;Ok(SearchService {- local_searcher,+ local_searcher: local_searcher.build(),cluster_handle,})}@@ -182,13 +184,7 @@ impl sonic::service::Message<SearchService> for Size {type Response = SizeResponse;async fn handle(self, server: &SearchService) -> Self::Response {SizeResponse {- pages: server- .local_searcher- .index()- .guard()- .await- .inverted_index()- .num_documents(),+ pages: server.local_searcher.num_documents().await,}}}diff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex a902883a..85bf32ca 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -145,6 +145,8 @@ impl Index {#[cfg(test)]mod tests {+ use std::sync::Arc;+use crate::{ranking,searcher::{LocalSearcher, SearchQuery},@@ -224,7 +226,7 @@ mod tests {index.commit().unwrap();- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {query: "test".to_string(),diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex cd3d1e1f..70a53e5b 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -20,7 +20,7 @@ use std::{sync::Arc,};-use tokio::sync::{RwLock, RwLockReadGuard};+use tokio::sync::{OwnedRwLockReadGuard, RwLock};use chrono::{DateTime, NaiveDate, Utc};use itertools::Itertools;@@ -37,7 +37,6 @@ use crate::{entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},inverted_index::InvertedIndex,live_index::{BATCH_SIZE, TTL},- searcher::SearchableIndex,Result,};@@ -400,8 +399,8 @@ impl LiveIndex {self.inner.write().await.insert(pages)}- pub async fn read(&self) -> RwLockReadGuard<'_, InnerIndex> {- self.inner.read().await+ pub async fn read(&self) -> OwnedRwLockReadGuard<InnerIndex> {+ RwLock::read_owned(self.inner.clone()).await}pub async fn set_snippet_config(&self, config: SnippetConfig) {@@ -409,8 +408,8 @@ impl LiveIndex {.write().await.index+ .inverted_index.set_snippet_config(config)- .await}pub async fn path(&self) -> PathBuf {diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex f413692e..c2b1e0f0 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -205,6 +205,8 @@ impl tantivy::query::Query for Query {#[cfg(test)]mod tests {+ use std::sync::Arc;+use crate::{index::Index, rand_words, searcher::LocalSearcher, webpage::Webpage};use proptest::prelude::*;@@ -381,7 +383,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);@@ -447,7 +449,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test site:first.com".to_string(),@@ -521,7 +523,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test linksto:first.com".to_string(),@@ -623,7 +625,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test linkto:second.com".to_string(),@@ -675,7 +677,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "intitle:website".to_string(),@@ -727,7 +729,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test inurl:forum".to_string(),@@ -822,7 +824,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test site:first.com".to_string(),@@ -895,7 +897,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "\"Test website\"".to_string(),@@ -962,7 +964,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "testwebsite".to_string(),@@ -1044,7 +1046,7 @@ mod tests {index.insert(&webpage).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test".to_string(),@@ -1135,7 +1137,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test site:.com".to_string(),@@ -1242,7 +1244,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "test exacturl:https://www.first.com/example".to_string(),@@ -1327,7 +1329,7 @@ mod tests {).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let query = SearchQuery {query: "\"test test\" website".to_string(),diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 6f0ce316..f7795af3 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -248,6 +248,8 @@ impl AsTantivyQuery for Matching {#[cfg(test)]mod tests {+ use std::sync::Arc;+use optics::{HostRankings, Optic};use crate::{@@ -320,7 +322,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -441,7 +443,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let _ = searcher.search_sync(&SearchQuery {@@ -565,7 +567,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -733,7 +735,7 @@ mod tests {index.commit().expect("failed to commit index");let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(- LocalSearchClient::from(LocalSearcher::from(index)),+ LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),Bangs::empty(),Config::default(),)@@ -849,7 +851,7 @@ mod tests {.expect("failed to insert webpage");index.commit().unwrap();- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -978,7 +980,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1068,7 +1070,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1131,7 +1133,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {query: "example".to_string(),@@ -1287,7 +1289,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1355,7 +1357,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1435,7 +1437,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1531,7 +1533,7 @@ mod tests {index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1627,7 +1629,7 @@ mod tests {index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1706,7 +1708,7 @@ mod tests {index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1820,7 +1822,7 @@ mod tests {index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1908,7 +1910,7 @@ mod tests {index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -1950,7 +1952,7 @@ mod tests {index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 2db05dd0..593391f8 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -139,6 +139,8 @@ impl Scorer {#[cfg(test)]mod tests {+ use std::sync::Arc;+use optics::HostRankings;use crate::{@@ -361,7 +363,7 @@ mod tests {index.commit().unwrap();let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(- LocalSearchClient::from(LocalSearcher::new(index)),+ LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),Bangs::empty(),crate::searcher::api::Config::default(),)diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 3f12e758..e1e764ee 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -127,7 +127,7 @@ impl LocalRanker {#[cfg(test)]mod tests {- use std::path::Path;+ use std::{path::Path, sync::Arc};use crate::{config::{IndexerConfig, IndexerDualEncoderConfig, WarcSource},@@ -196,7 +196,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {query: "example".to_string(),@@ -262,7 +262,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {query: "example".to_string(),@@ -330,7 +330,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {query: "title".to_string(),@@ -403,7 +403,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {query: "test".to_string(),@@ -469,7 +469,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {query: "test".to_string(),@@ -554,7 +554,7 @@ mod tests {index.commit().unwrap();- let searcher = LocalSearcher::new(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let res = searcher.search_sync(&SearchQuery {@@ -640,7 +640,7 @@ mod tests {}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {@@ -733,7 +733,7 @@ mod tests {}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {@@ -854,9 +854,10 @@ mod tests {index.commit().expect("failed to commit index");- let mut searcher = LocalSearcher::new(index);- searcher+ let mut searcher = LocalSearcher::builder(Arc::new(index));+ searcher = searcher.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));+ let searcher = searcher.build();let result = searcher.search_sync(&SearchQuery {@@ -942,9 +943,10 @@ mod tests {index.commit().expect("failed to commit index");- let mut searcher = LocalSearcher::new(index);- searcher+ let mut searcher = LocalSearcher::builder(Arc::new(index));+ searcher = searcher.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));+ let searcher = searcher.build();let result = searcher.search_sync(&SearchQuery {@@ -991,7 +993,7 @@ mod tests {}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {@@ -1071,7 +1073,7 @@ mod tests {index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::new(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 65cf33a3..2537bdd9 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -16,6 +16,8 @@#[cfg(test)]mod tests {+ use std::sync::Arc;+use optics::HostRankings;use crate::{@@ -187,7 +189,7 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(- LocalSearchClient::from(LocalSearcher::new(index)),+ LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),Bangs::empty(),crate::searcher::api::Config::default(),)diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex eb9cba77..0c2987a0 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -456,9 +456,9 @@ impl SearchClient for DistributedSearcher {}/// This should only be used for testing and benchmarks.-pub struct LocalSearchClient(LocalSearcher<Index>);-impl From<LocalSearcher<Index>> for LocalSearchClient {- fn from(searcher: LocalSearcher<Index>) -> Self {+pub struct LocalSearchClient(LocalSearcher<Arc<Index>>);+impl From<LocalSearcher<Arc<Index>>> for LocalSearchClient {+ fn from(searcher: LocalSearcher<Arc<Index>>) -> Self {Self(searcher)}}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex 31c4cf3a..e03a462c 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -17,13 +17,13 @@use std::collections::HashMap;use std::future::Future;use std::sync::Arc;-use tokio::sync::RwLockReadGuard;+use tokio::sync::OwnedRwLockReadGuard;use itertools::Itertools;use url::Url;use crate::collector::approx_count;-use crate::config::{CollectorConfig, SnippetConfig};+use crate::config::CollectorConfig;use crate::index::Index;use crate::inverted_index::{InvertedIndex, KeyPhrase, RetrievedWebpage};use crate::models::dual_encoder::DualEncoder;@@ -40,82 +40,243 @@ use crate::{inverted_index, live_index, Result};use super::WebsitesResult;use super::{InitialWebsiteResult, SearchQuery};-pub trait SearchableIndex {- type SearchGuard<'a>: SearchGuard<'a>- where- Self: 'a;+pub trait SearchableIndex: Send + Sync + 'static {+ type SearchGuard: SearchGuard;- fn guard(&self) -> impl Future<Output = Self::SearchGuard<'_>>;- fn set_snippet_config(&mut self, config: SnippetConfig) -> impl Future<Output = ()>;+ fn guard(&self) -> impl Future<Output = Self::SearchGuard>;}-pub trait SearchGuard<'a> {- fn search_index(&self) -> &'_ Index;- fn inverted_index(&self) -> &'_ InvertedIndex {+pub trait SearchGuard: Send + Sync {+ fn search_index(&self) -> &Index;+ fn inverted_index(&self) -> &InvertedIndex {&self.search_index().inverted_index}}-impl SearchableIndex for Index {- type SearchGuard<'a> = NormalIndexSearchGuard<'a>;+impl SearchableIndex for Arc<Index> {+ type SearchGuard = NormalIndexSearchGuard;- async fn guard(&self) -> Self::SearchGuard<'_> {- NormalIndexSearchGuard { search_index: self }- }-- async fn set_snippet_config(&mut self, config: SnippetConfig) {- self.inverted_index.set_snippet_config(config);+ async fn guard(&self) -> Self::SearchGuard {+ NormalIndexSearchGuard {+ search_index: self.clone(),+ }}}-pub struct NormalIndexSearchGuard<'a> {- search_index: &'a Index,+pub struct NormalIndexSearchGuard {+ search_index: Arc<Index>,}-impl<'a> SearchGuard<'a> for NormalIndexSearchGuard<'a> {- fn search_index(&self) -> &'_ Index {- self.search_index+impl SearchGuard for NormalIndexSearchGuard {+ fn search_index(&self) -> &Index {+ self.search_index.as_ref()}}impl SearchableIndex for Arc<live_index::LiveIndex> {- type SearchGuard<'a> = LiveIndexSearchGuard<'a>;+ type SearchGuard = LiveIndexSearchGuard;- async fn guard(&self) -> Self::SearchGuard<'_> {+ async fn guard(&self) -> Self::SearchGuard {LiveIndexSearchGuard {lock_guard: self.read().await,}}+}- async fn set_snippet_config(&mut self, config: SnippetConfig) {- live_index::LiveIndex::set_snippet_config(self, config).await+pub struct LiveIndexSearchGuard {+ lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>,+}++impl SearchGuard for LiveIndexSearchGuard {+ fn search_index(&self) -> &Index {+ self.lock_guard.index()}}-pub struct LiveIndexSearchGuard<'a> {- lock_guard: RwLockReadGuard<'a, live_index::index::InnerIndex>,+pub struct LocalSearcherBuilder<I: SearchableIndex> {+ inner: InnerLocalSearcher<I>,}-impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {- fn search_index(&self) -> &'_ Index {- self.lock_guard.index()+impl<I> LocalSearcherBuilder<I>+where+ I: SearchableIndex,+{+ pub fn new(index: I) -> Self {+ Self {+ inner: InnerLocalSearcher::new(index),+ }+ }++ pub fn set_linear_model(mut self, model: LinearRegression) -> Self {+ self.inner.set_linear_model(model);+ self+ }++ pub fn set_dual_encoder(mut self, dual_encoder: DualEncoder) -> Self {+ self.inner.set_dual_encoder(dual_encoder);+ self+ }++ pub fn set_collector_config(mut self, config: CollectorConfig) -> Self {+ self.inner.set_collector_config(config);+ self+ }++ pub fn build(self) -> LocalSearcher<I> {+ LocalSearcher {+ inner: Arc::new(self.inner),+ }}}pub struct LocalSearcher<I: SearchableIndex> {- index: I,- linear_regression: Option<Arc<LinearRegression>>,- dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,+ inner: Arc<InnerLocalSearcher<I>>,}-impl<I> From<I> for LocalSearcher<I>+impl<I> LocalSearcher<I>whereI: SearchableIndex,{- fn from(index: I) -> Self {- Self::new(index)+ pub fn builder(index: I) -> LocalSearcherBuilder<I> {+ LocalSearcherBuilder::new(index)+ }++ pub async fn search_initial(+ &self,+ query: &SearchQuery,+ de_rank_similar: bool,+ ) -> Result<InitialWebsiteResult> {+ let inner = self.inner.clone();+ let query = query.clone();+ let guard = inner.guard().await;++ tokio::task::spawn_blocking(move || inner.search_initial(&query, &guard, de_rank_similar))+ .await+ .unwrap()+ }++ pub async fn retrieve_websites(+ &self,+ websites: &[inverted_index::WebpagePointer],+ query: &str,+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {+ let inner = self.inner.clone();+ let guard = inner.guard().await;+ let query = query.to_string();+ let websites = websites.to_vec();++ tokio::task::spawn_blocking(move || inner.retrieve_websites(&websites, &query, &guard))+ .await+ .unwrap()+ }++ pub async fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {+ use std::time::Instant;++ let start = Instant::now();+ let search_query = query.clone();++ let search_result = self.search_initial(&search_query, true).await?;++ let pointers: Vec<_> = search_result+ .websites+ .iter()+ .map(|website| website.pointer().clone())+ .collect();++ let websites: Vec<_> = self+ .retrieve_websites(&pointers, &query.query)+ .await?+ .into_iter()+ .zip_eq(search_result.websites)+ .map(|(webpage, ranking)| {+ let ranking = RecallRankingWebpage::new(ranking, Default::default());+ PrecisionRankingWebpage::new(webpage, ranking)+ })+ .collect();++ let pointers: Vec<_> = websites+ .iter()+ .map(|website| website.ranking().pointer().clone())+ .collect();++ let retrieved_sites = self+ .retrieve_websites(&pointers, &search_query.query)+ .await?;++ let coefficients = query.signal_coefficients();++ let mut webpages: Vec<_> = retrieved_sites+ .into_iter()+ .map(|webpage| DisplayedWebpage::new(webpage, query))+ .collect();++ for (webpage, ranking) in webpages.iter_mut().zip(websites) {+ let mut ranking_signals = HashMap::new();++ for signal in SignalEnum::all() {+ if let Some(calc) = ranking.ranking().signals().get(signal) {+ ranking_signals.insert(+ signal.into(),+ SignalScore {+ value: calc.score,+ coefficient: coefficients.get(&signal),+ },+ );+ }+ }++ webpage.ranking_signals = Some(ranking_signals);+ }++ Ok(WebsitesResult {+ num_hits: search_result.num_websites,+ webpages,+ search_duration_ms: start.elapsed().as_millis(),+ has_more_results: (search_result.num_websites.as_u64() as usize)+ > query.offset() + query.num_results(),+ })+ }++ /// This function is mainly used for tests and benchmarks+ pub fn search_sync(&self, query: &SearchQuery) -> Result<WebsitesResult> {+ crate::block_on(self.search(query))+ }++ pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {+ self.inner.guard().await.inverted_index().get_webpage(url)+ }++ pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {+ self.inner.guard().await.inverted_index().get_homepage(url)+ }++ pub async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {+ self.inner+ .guard()+ .await+ .inverted_index()+ .top_key_phrases(top_n)+ }++ pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {+ self.inner+ .guard()+ .await+ .inverted_index()+ .get_site_urls(site, offset, limit)}++ pub async fn num_documents(&self) -> u64 {+ self.inner.guard().await.inverted_index().num_documents()+ }+}++struct InnerLocalSearcher<I: SearchableIndex> {+ index: I,+ linear_regression: Option<Arc<LinearRegression>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,}struct InvertedIndexResult {@@ -123,12 +284,12 @@ struct InvertedIndexResult {num_hits: approx_count::Count,}-impl<I> LocalSearcher<I>+impl<I> InnerLocalSearcher<I>whereI: SearchableIndex,{pub fn new(index: I) -> Self {- LocalSearcher {+ Self {index,linear_regression: None,dual_encoder: None,@@ -136,6 +297,10 @@ where}}+ async fn guard(&self) -> I::SearchGuard {+ self.index.guard().await+ }+pub fn set_linear_model(&mut self, model: LinearRegression) {self.linear_regression = Some(Arc::new(model));}@@ -148,12 +313,8 @@ whereself.collector_config = config;}- pub async fn set_snippet_config(&mut self, config: SnippetConfig) {- self.index.set_snippet_config(config).await;- }-- fn parse_query<'a, G: SearchGuard<'a>>(- &'a self,+ fn parse_query<G: SearchGuard>(+ &self,ctx: &Ctx,guard: &G,query: &SearchQuery,@@ -161,8 +322,8 @@ whereQuery::parse(ctx, query, guard.inverted_index())}- fn ranker<'a, G: SearchGuard<'a>>(- &'a self,+ fn ranker<G: SearchGuard>(+ &self,query: &Query,guard: &G,de_rank_similar: bool,@@ -185,8 +346,8 @@ where.with_offset(query.offset()))}- fn search_inverted_index<'a, G: SearchGuard<'a>>(- &'a self,+ fn search_inverted_index<G: SearchGuard>(+ &self,ctx: &Ctx,guard: &G,query: &SearchQuery,@@ -232,19 +393,17 @@ where})}- pub fn index(&self) -> &I {- &self.index- }-- pub async fn search_initial(+ fn search_initial(&self,query: &SearchQuery,+ guard: &I::SearchGuard,de_rank_similar: bool,) -> Result<InitialWebsiteResult> {- let guard = self.index.guard().await;+ let query = query.clone();+let ctx = guard.inverted_index().local_search_ctx();let inverted_index_result =- self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;+ self.search_inverted_index(&ctx, guard, &query, de_rank_similar)?;Ok(InitialWebsiteResult {websites: inverted_index_result.webpages,@@ -252,12 +411,12 @@ where})}- pub async fn retrieve_websites(+ fn retrieve_websites(&self,websites: &[inverted_index::WebpagePointer],query: &str,+ guard: &I::SearchGuard,) -> Result<Vec<inverted_index::RetrievedWebpage>> {- let guard = self.index.guard().await;let ctx = guard.inverted_index().local_search_ctx();let query = SearchQuery {query: query.to_string(),@@ -267,103 +426,6 @@ whereguard.inverted_index().retrieve_websites(websites, &query)}-- pub async fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {- use std::time::Instant;-- let start = Instant::now();- let search_query = query.clone();-- let search_result = self.search_initial(&search_query, true).await?;-- let pointers: Vec<_> = search_result- .websites- .iter()- .map(|website| website.pointer().clone())- .collect();-- let websites: Vec<_> = self- .retrieve_websites(&pointers, &query.query)- .await?- .into_iter()- .zip_eq(search_result.websites)- .map(|(webpage, ranking)| {- let ranking = RecallRankingWebpage::new(ranking, Default::default());- PrecisionRankingWebpage::new(webpage, ranking)- })- .collect();-- let pointers: Vec<_> = websites- .iter()- .map(|website| website.ranking().pointer().clone())- .collect();-- let retrieved_sites = self- .retrieve_websites(&pointers, &search_query.query)- .await?;-- let coefficients = query.signal_coefficients();-- let mut webpages: Vec<_> = retrieved_sites- .into_iter()- .map(|webpage| DisplayedWebpage::new(webpage, query))- .collect();-- for (webpage, ranking) in webpages.iter_mut().zip(websites) {- let mut ranking_signals = HashMap::new();-- for signal in SignalEnum::all() {- if let Some(calc) = ranking.ranking().signals().get(signal) {- ranking_signals.insert(- signal.into(),- SignalScore {- value: calc.score,- coefficient: coefficients.get(&signal),- },- );- }- }-- webpage.ranking_signals = Some(ranking_signals);- }-- Ok(WebsitesResult {- num_hits: search_result.num_websites,- webpages,- search_duration_ms: start.elapsed().as_millis(),- has_more_results: (search_result.num_websites.as_u64() as usize)- > query.offset() + query.num_results(),- })- }-- /// This function is mainly used for tests and benchmarks- pub fn search_sync(&self, query: &SearchQuery) -> Result<WebsitesResult> {- crate::block_on(self.search(query))- }-- pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {- self.index.guard().await.inverted_index().get_webpage(url)- }-- pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {- self.index.guard().await.inverted_index().get_homepage(url)- }-- pub async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {- self.index- .guard()- .await- .inverted_index()- .top_key_phrases(top_n)- }-- pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {- self.index- .guard()- .await- .inverted_index()- .get_site_urls(site, offset, limit)- }}#[cfg(test)]@@ -408,7 +470,7 @@ mod tests {index.commit().unwrap();- let searcher = LocalSearcher::new(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();for p in 0..NUM_PAGES {let urls: Vec<_> = searcherdiff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex e5214443..2e86f34f 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -349,6 +349,8 @@ pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfi#[cfg(test)]mod tests {+ use std::sync::Arc;+use super::*;use crate::{index::Index,@@ -413,7 +415,7 @@ Survey in 2016, 2017, and 2018."#;.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {@@ -452,7 +454,7 @@ Survey in 2016, 2017, and 2018."#;.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {@@ -491,7 +493,7 @@ Survey in 2016, 2017, and 2018."#;.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);+ let searcher = LocalSearcher::builder(Arc::new(index)).build();let result = searcher.search_sync(&SearchQuery {diff --git a/fuzz/fuzz_targets/query_parser.rs b/fuzz/fuzz_targets/query_parser.rsindex ea03cf23..a1d83a15 100644--- a/fuzz/fuzz_targets/query_parser.rs+++ b/fuzz/fuzz_targets/query_parser.rs@@ -1,17 +1,12 @@#![no_main]use libfuzzer_sys::fuzz_target;-use stract::{- index::Index,- query::Query,- searcher::{SearchGuard, SearchQuery, SearchableIndex},-};+use stract::{index::Index, query::Query, searcher::SearchQuery};fuzz_target!(|query: &str| {let index = Index::open("/tmp/stract/fuzz-index").unwrap();- let guard = stract::block_on(index.guard());- let ctx = guard.inverted_index().local_search_ctx();+ let ctx = index.inverted_index.local_search_ctx();let _ = Query::parse(&ctx,