Skip to content

web search done right

[search_server] use spawn_blocking for search requests to improve concurrency

Mikkel Denkerb93afff31989
diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rs
index 442e2b03..11c6da3a 100644
--- a/crates/core/examples/search_preindexed.rs
+++ b/crates/core/examples/search_preindexed.rs
@@ -1,3 +1,5 @@
+use std::sync::Arc;
+
use optics::HostRankings;
use rand::seq::SliceRandom;
use stract::{
@@ -15,7 +17,7 @@ use stract::{
#[tokio::main]
pub async fn main() {
- let index = Index::open("data/index").unwrap();
+ let mut index = Index::open("data/index").unwrap();
let collector_conf = CollectorConfig {
..Default::default()
@@ -47,7 +49,15 @@ pub async fn main() {
top_phrases_for_autosuggest: defaults::Api::top_phrases_for_autosuggest(),
};
- let mut searcher = LocalSearcher::new(index);
+ index.inverted_index.set_snippet_config(SnippetConfig {
+ num_words_for_lang_detection: Some(250),
+ max_considered_words: Some(10_000),
+ ..Default::default()
+ });
+
+ let searcher = LocalSearcher::builder(Arc::new(index))
+ .set_collector_config(collector_conf)
+ .build();
let mut queries: Vec<_> = searcher
.top_key_phrases(1_000_000)
@@ -57,14 +67,6 @@ pub async fn main() {
.collect();
queries.shuffle(&mut rand::thread_rng());
- searcher.set_collector_config(collector_conf);
- searcher
- .set_snippet_config(SnippetConfig {
- num_words_for_lang_detection: Some(250),
- max_considered_words: Some(10_000),
- ..Default::default()
- })
- .await;
let bangs = Bangs::from_path(config.bangs_path.as_ref().unwrap());
let searcher = stract::searcher::LocalSearchClient::from(searcher);
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
index be27bf74..83129dfc 100644
--- a/crates/core/src/entrypoint/live_index/search_server.rs
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -216,7 +216,9 @@ impl LiveIndexService {
)
.await?,
);
- let local_searcher = LocalSearcher::new(index.clone());
+ let local_searcher = LocalSearcher::builder(index.clone())
+ .set_collector_config(config.collector)
+ .build();
let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
index 4ecadf13..e0effdc3 100644
--- a/crates/core/src/entrypoint/live_index/tests.rs
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -491,7 +491,7 @@ async fn test_segment_compaction() -> Result<()> {
assert_eq!(index.meta().await.segments().len(), 1);
- let searcher = LocalSearcher::from(index.clone());
+ let searcher = LocalSearcher::builder(index.clone()).build();
let res = searcher
.search(&SearchQuery {
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 13096fd1..646b277a 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::collections::HashMap;
+use std::{collections::HashMap, sync::Arc};
use tracing::info;
use url::Url;
@@ -30,7 +30,7 @@ use crate::{
inverted_index::{self, KeyPhrase, RetrievedWebpage},
models::dual_encoder::DualEncoder,
ranking::models::linear::LinearRegression,
- searcher::{InitialWebsiteResult, LocalSearcher, SearchGuard, SearchQuery, SearchableIndex},
+ searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},
Result,
};
@@ -50,7 +50,7 @@ sonic_service!(
);
pub struct SearchService {
- local_searcher: LocalSearcher<Index>,
+ local_searcher: LocalSearcher<Arc<Index>>,
// dropping the handle leaves the cluster
#[allow(unused)]
cluster_handle: Cluster,
@@ -58,20 +58,22 @@ pub struct SearchService {
impl SearchService {
async fn new(config: config::SearchServerConfig) -> Result<Self> {
- let search_index = Index::open(config.index_path)?;
+ let mut search_index = Index::open(config.index_path)?;
+ search_index
+ .inverted_index
+ .set_snippet_config(config.snippet);
- let mut local_searcher = LocalSearcher::new(search_index);
+ let mut local_searcher = LocalSearcher::builder(Arc::new(search_index));
if let Some(model_path) = config.linear_model_path {
- local_searcher.set_linear_model(LinearRegression::open(model_path)?);
+ local_searcher = local_searcher.set_linear_model(LinearRegression::open(model_path)?);
}
if let Some(model_path) = config.dual_encoder_model_path {
- local_searcher.set_dual_encoder(DualEncoder::open(model_path)?);
+ local_searcher = local_searcher.set_dual_encoder(DualEncoder::open(model_path)?);
}
- local_searcher.set_collector_config(config.collector);
- local_searcher.set_snippet_config(config.snippet).await;
+ local_searcher = local_searcher.set_collector_config(config.collector);
let cluster_handle = Cluster::join(
Member::new(Service::Searcher {
@@ -84,7 +86,7 @@ impl SearchService {
.await?;
Ok(SearchService {
- local_searcher,
+ local_searcher: local_searcher.build(),
cluster_handle,
})
}
@@ -182,13 +184,7 @@ impl sonic::service::Message<SearchService> for Size {
type Response = SizeResponse;
async fn handle(self, server: &SearchService) -> Self::Response {
SizeResponse {
- pages: server
- .local_searcher
- .index()
- .guard()
- .await
- .inverted_index()
- .num_documents(),
+ pages: server.local_searcher.num_documents().await,
}
}
}
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index a902883a..85bf32ca 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -145,6 +145,8 @@ impl Index {
#[cfg(test)]
mod tests {
+ use std::sync::Arc;
+
use crate::{
ranking,
searcher::{LocalSearcher, SearchQuery},
@@ -224,7 +226,7 @@ mod tests {
index.commit().unwrap();
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
query: "test".to_string(),
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index cd3d1e1f..70a53e5b 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -20,7 +20,7 @@ use std::{
sync::Arc,
};
-use tokio::sync::{RwLock, RwLockReadGuard};
+use tokio::sync::{OwnedRwLockReadGuard, RwLock};
use chrono::{DateTime, NaiveDate, Utc};
use itertools::Itertools;
@@ -37,7 +37,6 @@ use crate::{
entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},
inverted_index::InvertedIndex,
live_index::{BATCH_SIZE, TTL},
- searcher::SearchableIndex,
Result,
};
@@ -400,8 +399,8 @@ impl LiveIndex {
self.inner.write().await.insert(pages)
}
- pub async fn read(&self) -> RwLockReadGuard<'_, InnerIndex> {
- self.inner.read().await
+ pub async fn read(&self) -> OwnedRwLockReadGuard<InnerIndex> {
+ RwLock::read_owned(self.inner.clone()).await
}
pub async fn set_snippet_config(&self, config: SnippetConfig) {
@@ -409,8 +408,8 @@ impl LiveIndex {
.write()
.await
.index
+ .inverted_index
.set_snippet_config(config)
- .await
}
pub async fn path(&self) -> PathBuf {
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index f413692e..c2b1e0f0 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -205,6 +205,8 @@ impl tantivy::query::Query for Query {
#[cfg(test)]
mod tests {
+ use std::sync::Arc;
+
use crate::{index::Index, rand_words, searcher::LocalSearcher, webpage::Webpage};
use proptest::prelude::*;
@@ -381,7 +383,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher.search_sync(&query).expect("Search failed");
assert_eq!(result.webpages.len(), 1);
@@ -447,7 +449,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test site:first.com".to_string(),
@@ -521,7 +523,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test linksto:first.com".to_string(),
@@ -623,7 +625,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test linkto:second.com".to_string(),
@@ -675,7 +677,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "intitle:website".to_string(),
@@ -727,7 +729,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test inurl:forum".to_string(),
@@ -822,7 +824,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test site:first.com".to_string(),
@@ -895,7 +897,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "\"Test website\"".to_string(),
@@ -962,7 +964,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "testwebsite".to_string(),
@@ -1044,7 +1046,7 @@ mod tests {
index.insert(&webpage).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test".to_string(),
@@ -1135,7 +1137,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test site:.com".to_string(),
@@ -1242,7 +1244,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "test exacturl:https://www.first.com/example".to_string(),
@@ -1327,7 +1329,7 @@ mod tests {
)
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let query = SearchQuery {
query: "\"test test\" website".to_string(),
diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rs
index 6f0ce316..f7795af3 100644
--- a/crates/core/src/query/optic.rs
+++ b/crates/core/src/query/optic.rs
@@ -248,6 +248,8 @@ impl AsTantivyQuery for Matching {
#[cfg(test)]
mod tests {
+ use std::sync::Arc;
+
use optics::{HostRankings, Optic};
use crate::{
@@ -320,7 +322,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -441,7 +443,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let _ = searcher
.search_sync(&SearchQuery {
@@ -565,7 +567,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -733,7 +735,7 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(
- LocalSearchClient::from(LocalSearcher::from(index)),
+ LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),
Bangs::empty(),
Config::default(),
)
@@ -849,7 +851,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().unwrap();
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -978,7 +980,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1068,7 +1070,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1131,7 +1133,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
query: "example".to_string(),
@@ -1287,7 +1289,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1355,7 +1357,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1435,7 +1437,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1531,7 +1533,7 @@ mod tests {
index.insert(&page).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1627,7 +1629,7 @@ mod tests {
index.insert(&page).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1706,7 +1708,7 @@ mod tests {
index.insert(&page).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1820,7 +1822,7 @@ mod tests {
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1908,7 +1910,7 @@ mod tests {
index.insert(&page).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -1950,7 +1952,7 @@ mod tests {
index.insert(&page).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rs
index 2db05dd0..593391f8 100644
--- a/crates/core/src/ranking/inbound_similarity.rs
+++ b/crates/core/src/ranking/inbound_similarity.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -139,6 +139,8 @@ impl Scorer {
#[cfg(test)]
mod tests {
+ use std::sync::Arc;
+
use optics::HostRankings;
use crate::{
@@ -361,7 +363,7 @@ mod tests {
index.commit().unwrap();
let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(
- LocalSearchClient::from(LocalSearcher::new(index)),
+ LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),
Bangs::empty(),
crate::searcher::api::Config::default(),
)
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index 3f12e758..e1e764ee 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -127,7 +127,7 @@ impl LocalRanker {
#[cfg(test)]
mod tests {
- use std::path::Path;
+ use std::{path::Path, sync::Arc};
use crate::{
config::{IndexerConfig, IndexerDualEncoderConfig, WarcSource},
@@ -196,7 +196,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
query: "example".to_string(),
@@ -262,7 +262,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
query: "example".to_string(),
@@ -330,7 +330,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
query: "title".to_string(),
@@ -403,7 +403,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
query: "test".to_string(),
@@ -469,7 +469,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
query: "test".to_string(),
@@ -554,7 +554,7 @@ mod tests {
index.commit().unwrap();
- let searcher = LocalSearcher::new(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let res = searcher
.search_sync(&SearchQuery {
@@ -640,7 +640,7 @@ mod tests {
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::new(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
@@ -733,7 +733,7 @@ mod tests {
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::new(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
@@ -854,9 +854,10 @@ mod tests {
index.commit().expect("failed to commit index");
- let mut searcher = LocalSearcher::new(index);
- searcher
+ let mut searcher = LocalSearcher::builder(Arc::new(index));
+ searcher = searcher
.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));
+ let searcher = searcher.build();
let result = searcher
.search_sync(&SearchQuery {
@@ -942,9 +943,10 @@ mod tests {
index.commit().expect("failed to commit index");
- let mut searcher = LocalSearcher::new(index);
- searcher
+ let mut searcher = LocalSearcher::builder(Arc::new(index));
+ searcher = searcher
.set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));
+ let searcher = searcher.build();
let result = searcher
.search_sync(&SearchQuery {
@@ -991,7 +993,7 @@ mod tests {
})
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::new(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
@@ -1071,7 +1073,7 @@ mod tests {
index.insert(&page).expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::new(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rs
index 65cf33a3..2537bdd9 100644
--- a/crates/core/src/ranking/optics.rs
+++ b/crates/core/src/ranking/optics.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -16,6 +16,8 @@
#[cfg(test)]
mod tests {
+ use std::sync::Arc;
+
use optics::HostRankings;
use crate::{
@@ -187,7 +189,7 @@ mod tests {
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(
- LocalSearchClient::from(LocalSearcher::new(index)),
+ LocalSearchClient::from(LocalSearcher::builder(Arc::new(index)).build()),
Bangs::empty(),
crate::searcher::api::Config::default(),
)
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index eb9cba77..0c2987a0 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -456,9 +456,9 @@ impl SearchClient for DistributedSearcher {
}
/// This should only be used for testing and benchmarks.
-pub struct LocalSearchClient(LocalSearcher<Index>);
-impl From<LocalSearcher<Index>> for LocalSearchClient {
- fn from(searcher: LocalSearcher<Index>) -> Self {
+pub struct LocalSearchClient(LocalSearcher<Arc<Index>>);
+impl From<LocalSearcher<Arc<Index>>> for LocalSearchClient {
+ fn from(searcher: LocalSearcher<Arc<Index>>) -> Self {
Self(searcher)
}
}
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index 31c4cf3a..e03a462c 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -1,5 +1,5 @@
// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
+// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
@@ -17,13 +17,13 @@
use std::collections::HashMap;
use std::future::Future;
use std::sync::Arc;
-use tokio::sync::RwLockReadGuard;
+use tokio::sync::OwnedRwLockReadGuard;
use itertools::Itertools;
use url::Url;
use crate::collector::approx_count;
-use crate::config::{CollectorConfig, SnippetConfig};
+use crate::config::CollectorConfig;
use crate::index::Index;
use crate::inverted_index::{InvertedIndex, KeyPhrase, RetrievedWebpage};
use crate::models::dual_encoder::DualEncoder;
@@ -40,82 +40,243 @@ use crate::{inverted_index, live_index, Result};
use super::WebsitesResult;
use super::{InitialWebsiteResult, SearchQuery};
-pub trait SearchableIndex {
- type SearchGuard<'a>: SearchGuard<'a>
- where
- Self: 'a;
+pub trait SearchableIndex: Send + Sync + 'static {
+ type SearchGuard: SearchGuard;
- fn guard(&self) -> impl Future<Output = Self::SearchGuard<'_>>;
- fn set_snippet_config(&mut self, config: SnippetConfig) -> impl Future<Output = ()>;
+ fn guard(&self) -> impl Future<Output = Self::SearchGuard>;
}
-pub trait SearchGuard<'a> {
- fn search_index(&self) -> &'_ Index;
- fn inverted_index(&self) -> &'_ InvertedIndex {
+pub trait SearchGuard: Send + Sync {
+ fn search_index(&self) -> &Index;
+ fn inverted_index(&self) -> &InvertedIndex {
&self.search_index().inverted_index
}
}
-impl SearchableIndex for Index {
- type SearchGuard<'a> = NormalIndexSearchGuard<'a>;
+impl SearchableIndex for Arc<Index> {
+ type SearchGuard = NormalIndexSearchGuard;
- async fn guard(&self) -> Self::SearchGuard<'_> {
- NormalIndexSearchGuard { search_index: self }
- }
-
- async fn set_snippet_config(&mut self, config: SnippetConfig) {
- self.inverted_index.set_snippet_config(config);
+ async fn guard(&self) -> Self::SearchGuard {
+ NormalIndexSearchGuard {
+ search_index: self.clone(),
+ }
}
}
-pub struct NormalIndexSearchGuard<'a> {
- search_index: &'a Index,
+pub struct NormalIndexSearchGuard {
+ search_index: Arc<Index>,
}
-impl<'a> SearchGuard<'a> for NormalIndexSearchGuard<'a> {
- fn search_index(&self) -> &'_ Index {
- self.search_index
+impl SearchGuard for NormalIndexSearchGuard {
+ fn search_index(&self) -> &Index {
+ self.search_index.as_ref()
}
}
impl SearchableIndex for Arc<live_index::LiveIndex> {
- type SearchGuard<'a> = LiveIndexSearchGuard<'a>;
+ type SearchGuard = LiveIndexSearchGuard;
- async fn guard(&self) -> Self::SearchGuard<'_> {
+ async fn guard(&self) -> Self::SearchGuard {
LiveIndexSearchGuard {
lock_guard: self.read().await,
}
}
+}
- async fn set_snippet_config(&mut self, config: SnippetConfig) {
- live_index::LiveIndex::set_snippet_config(self, config).await
+pub struct LiveIndexSearchGuard {
+ lock_guard: OwnedRwLockReadGuard<live_index::index::InnerIndex>,
+}
+
+impl SearchGuard for LiveIndexSearchGuard {
+ fn search_index(&self) -> &Index {
+ self.lock_guard.index()
}
}
-pub struct LiveIndexSearchGuard<'a> {
- lock_guard: RwLockReadGuard<'a, live_index::index::InnerIndex>,
+pub struct LocalSearcherBuilder<I: SearchableIndex> {
+ inner: InnerLocalSearcher<I>,
}
-impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {
- fn search_index(&self) -> &'_ Index {
- self.lock_guard.index()
+impl<I> LocalSearcherBuilder<I>
+where
+ I: SearchableIndex,
+{
+ pub fn new(index: I) -> Self {
+ Self {
+ inner: InnerLocalSearcher::new(index),
+ }
+ }
+
+ pub fn set_linear_model(mut self, model: LinearRegression) -> Self {
+ self.inner.set_linear_model(model);
+ self
+ }
+
+ pub fn set_dual_encoder(mut self, dual_encoder: DualEncoder) -> Self {
+ self.inner.set_dual_encoder(dual_encoder);
+ self
+ }
+
+ pub fn set_collector_config(mut self, config: CollectorConfig) -> Self {
+ self.inner.set_collector_config(config);
+ self
+ }
+
+ pub fn build(self) -> LocalSearcher<I> {
+ LocalSearcher {
+ inner: Arc::new(self.inner),
+ }
}
}
pub struct LocalSearcher<I: SearchableIndex> {
- index: I,
- linear_regression: Option<Arc<LinearRegression>>,
- dual_encoder: Option<Arc<DualEncoder>>,
- collector_config: CollectorConfig,
+ inner: Arc<InnerLocalSearcher<I>>,
}
-impl<I> From<I> for LocalSearcher<I>
+impl<I> LocalSearcher<I>
where
I: SearchableIndex,
{
- fn from(index: I) -> Self {
- Self::new(index)
+ pub fn builder(index: I) -> LocalSearcherBuilder<I> {
+ LocalSearcherBuilder::new(index)
+ }
+
+ pub async fn search_initial(
+ &self,
+ query: &SearchQuery,
+ de_rank_similar: bool,
+ ) -> Result<InitialWebsiteResult> {
+ let inner = self.inner.clone();
+ let query = query.clone();
+ let guard = inner.guard().await;
+
+ tokio::task::spawn_blocking(move || inner.search_initial(&query, &guard, de_rank_similar))
+ .await
+ .unwrap()
+ }
+
+ pub async fn retrieve_websites(
+ &self,
+ websites: &[inverted_index::WebpagePointer],
+ query: &str,
+ ) -> Result<Vec<inverted_index::RetrievedWebpage>> {
+ let inner = self.inner.clone();
+ let guard = inner.guard().await;
+ let query = query.to_string();
+ let websites = websites.to_vec();
+
+ tokio::task::spawn_blocking(move || inner.retrieve_websites(&websites, &query, &guard))
+ .await
+ .unwrap()
+ }
+
+ pub async fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {
+ use std::time::Instant;
+
+ let start = Instant::now();
+ let search_query = query.clone();
+
+ let search_result = self.search_initial(&search_query, true).await?;
+
+ let pointers: Vec<_> = search_result
+ .websites
+ .iter()
+ .map(|website| website.pointer().clone())
+ .collect();
+
+ let websites: Vec<_> = self
+ .retrieve_websites(&pointers, &query.query)
+ .await?
+ .into_iter()
+ .zip_eq(search_result.websites)
+ .map(|(webpage, ranking)| {
+ let ranking = RecallRankingWebpage::new(ranking, Default::default());
+ PrecisionRankingWebpage::new(webpage, ranking)
+ })
+ .collect();
+
+ let pointers: Vec<_> = websites
+ .iter()
+ .map(|website| website.ranking().pointer().clone())
+ .collect();
+
+ let retrieved_sites = self
+ .retrieve_websites(&pointers, &search_query.query)
+ .await?;
+
+ let coefficients = query.signal_coefficients();
+
+ let mut webpages: Vec<_> = retrieved_sites
+ .into_iter()
+ .map(|webpage| DisplayedWebpage::new(webpage, query))
+ .collect();
+
+ for (webpage, ranking) in webpages.iter_mut().zip(websites) {
+ let mut ranking_signals = HashMap::new();
+
+ for signal in SignalEnum::all() {
+ if let Some(calc) = ranking.ranking().signals().get(signal) {
+ ranking_signals.insert(
+ signal.into(),
+ SignalScore {
+ value: calc.score,
+ coefficient: coefficients.get(&signal),
+ },
+ );
+ }
+ }
+
+ webpage.ranking_signals = Some(ranking_signals);
+ }
+
+ Ok(WebsitesResult {
+ num_hits: search_result.num_websites,
+ webpages,
+ search_duration_ms: start.elapsed().as_millis(),
+ has_more_results: (search_result.num_websites.as_u64() as usize)
+ > query.offset() + query.num_results(),
+ })
+ }
+
+ /// This function is mainly used for tests and benchmarks
+ pub fn search_sync(&self, query: &SearchQuery) -> Result<WebsitesResult> {
+ crate::block_on(self.search(query))
+ }
+
+ pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
+ self.inner.guard().await.inverted_index().get_webpage(url)
+ }
+
+ pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
+ self.inner.guard().await.inverted_index().get_homepage(url)
+ }
+
+ pub async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
+ self.inner
+ .guard()
+ .await
+ .inverted_index()
+ .top_key_phrases(top_n)
+ }
+
+ pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
+ self.inner
+ .guard()
+ .await
+ .inverted_index()
+ .get_site_urls(site, offset, limit)
}
+
+ pub async fn num_documents(&self) -> u64 {
+ self.inner.guard().await.inverted_index().num_documents()
+ }
+}
+
+struct InnerLocalSearcher<I: SearchableIndex> {
+ index: I,
+ linear_regression: Option<Arc<LinearRegression>>,
+ dual_encoder: Option<Arc<DualEncoder>>,
+ collector_config: CollectorConfig,
}
struct InvertedIndexResult {
@@ -123,12 +284,12 @@ struct InvertedIndexResult {
num_hits: approx_count::Count,
}
-impl<I> LocalSearcher<I>
+impl<I> InnerLocalSearcher<I>
where
I: SearchableIndex,
{
pub fn new(index: I) -> Self {
- LocalSearcher {
+ Self {
index,
linear_regression: None,
dual_encoder: None,
@@ -136,6 +297,10 @@ where
}
}
+ async fn guard(&self) -> I::SearchGuard {
+ self.index.guard().await
+ }
+
pub fn set_linear_model(&mut self, model: LinearRegression) {
self.linear_regression = Some(Arc::new(model));
}
@@ -148,12 +313,8 @@ where
self.collector_config = config;
}
- pub async fn set_snippet_config(&mut self, config: SnippetConfig) {
- self.index.set_snippet_config(config).await;
- }
-
- fn parse_query<'a, G: SearchGuard<'a>>(
- &'a self,
+ fn parse_query<G: SearchGuard>(
+ &self,
ctx: &Ctx,
guard: &G,
query: &SearchQuery,
@@ -161,8 +322,8 @@ where
Query::parse(ctx, query, guard.inverted_index())
}
- fn ranker<'a, G: SearchGuard<'a>>(
- &'a self,
+ fn ranker<G: SearchGuard>(
+ &self,
query: &Query,
guard: &G,
de_rank_similar: bool,
@@ -185,8 +346,8 @@ where
.with_offset(query.offset()))
}
- fn search_inverted_index<'a, G: SearchGuard<'a>>(
- &'a self,
+ fn search_inverted_index<G: SearchGuard>(
+ &self,
ctx: &Ctx,
guard: &G,
query: &SearchQuery,
@@ -232,19 +393,17 @@ where
})
}
- pub fn index(&self) -> &I {
- &self.index
- }
-
- pub async fn search_initial(
+ fn search_initial(
&self,
query: &SearchQuery,
+ guard: &I::SearchGuard,
de_rank_similar: bool,
) -> Result<InitialWebsiteResult> {
- let guard = self.index.guard().await;
+ let query = query.clone();
+
let ctx = guard.inverted_index().local_search_ctx();
let inverted_index_result =
- self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;
+ self.search_inverted_index(&ctx, guard, &query, de_rank_similar)?;
Ok(InitialWebsiteResult {
websites: inverted_index_result.webpages,
@@ -252,12 +411,12 @@ where
})
}
- pub async fn retrieve_websites(
+ fn retrieve_websites(
&self,
websites: &[inverted_index::WebpagePointer],
query: &str,
+ guard: &I::SearchGuard,
) -> Result<Vec<inverted_index::RetrievedWebpage>> {
- let guard = self.index.guard().await;
let ctx = guard.inverted_index().local_search_ctx();
let query = SearchQuery {
query: query.to_string(),
@@ -267,103 +426,6 @@ where
guard.inverted_index().retrieve_websites(websites, &query)
}
-
- pub async fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {
- use std::time::Instant;
-
- let start = Instant::now();
- let search_query = query.clone();
-
- let search_result = self.search_initial(&search_query, true).await?;
-
- let pointers: Vec<_> = search_result
- .websites
- .iter()
- .map(|website| website.pointer().clone())
- .collect();
-
- let websites: Vec<_> = self
- .retrieve_websites(&pointers, &query.query)
- .await?
- .into_iter()
- .zip_eq(search_result.websites)
- .map(|(webpage, ranking)| {
- let ranking = RecallRankingWebpage::new(ranking, Default::default());
- PrecisionRankingWebpage::new(webpage, ranking)
- })
- .collect();
-
- let pointers: Vec<_> = websites
- .iter()
- .map(|website| website.ranking().pointer().clone())
- .collect();
-
- let retrieved_sites = self
- .retrieve_websites(&pointers, &search_query.query)
- .await?;
-
- let coefficients = query.signal_coefficients();
-
- let mut webpages: Vec<_> = retrieved_sites
- .into_iter()
- .map(|webpage| DisplayedWebpage::new(webpage, query))
- .collect();
-
- for (webpage, ranking) in webpages.iter_mut().zip(websites) {
- let mut ranking_signals = HashMap::new();
-
- for signal in SignalEnum::all() {
- if let Some(calc) = ranking.ranking().signals().get(signal) {
- ranking_signals.insert(
- signal.into(),
- SignalScore {
- value: calc.score,
- coefficient: coefficients.get(&signal),
- },
- );
- }
- }
-
- webpage.ranking_signals = Some(ranking_signals);
- }
-
- Ok(WebsitesResult {
- num_hits: search_result.num_websites,
- webpages,
- search_duration_ms: start.elapsed().as_millis(),
- has_more_results: (search_result.num_websites.as_u64() as usize)
- > query.offset() + query.num_results(),
- })
- }
-
- /// This function is mainly used for tests and benchmarks
- pub fn search_sync(&self, query: &SearchQuery) -> Result<WebsitesResult> {
- crate::block_on(self.search(query))
- }
-
- pub async fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
- self.index.guard().await.inverted_index().get_webpage(url)
- }
-
- pub async fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
- self.index.guard().await.inverted_index().get_homepage(url)
- }
-
- pub async fn top_key_phrases(&self, top_n: usize) -> Vec<KeyPhrase> {
- self.index
- .guard()
- .await
- .inverted_index()
- .top_key_phrases(top_n)
- }
-
- pub async fn get_site_urls(&self, site: &str, offset: usize, limit: usize) -> Vec<Url> {
- self.index
- .guard()
- .await
- .inverted_index()
- .get_site_urls(site, offset, limit)
- }
}
#[cfg(test)]
@@ -408,7 +470,7 @@ mod tests {
index.commit().unwrap();
- let searcher = LocalSearcher::new(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
for p in 0..NUM_PAGES {
let urls: Vec<_> = searcher
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index e5214443..2e86f34f 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -349,6 +349,8 @@ pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfi
#[cfg(test)]
mod tests {
+ use std::sync::Arc;
+
use super::*;
use crate::{
index::Index,
@@ -413,7 +415,7 @@ Survey in 2016, 2017, and 2018."#;
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
@@ -452,7 +454,7 @@ Survey in 2016, 2017, and 2018."#;
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
@@ -491,7 +493,7 @@ Survey in 2016, 2017, and 2018."#;
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
+ let searcher = LocalSearcher::builder(Arc::new(index)).build();
let result = searcher
.search_sync(&SearchQuery {
diff --git a/fuzz/fuzz_targets/query_parser.rs b/fuzz/fuzz_targets/query_parser.rs
index ea03cf23..a1d83a15 100644
--- a/fuzz/fuzz_targets/query_parser.rs
+++ b/fuzz/fuzz_targets/query_parser.rs
@@ -1,17 +1,12 @@
#![no_main]
use libfuzzer_sys::fuzz_target;
-use stract::{
- index::Index,
- query::Query,
- searcher::{SearchGuard, SearchQuery, SearchableIndex},
-};
+use stract::{index::Index, query::Query, searcher::SearchQuery};
fuzz_target!(|query: &str| {
let index = Index::open("/tmp/stract/fuzz-index").unwrap();
- let guard = stract::block_on(index.guard());
- let ctx = guard.inverted_index().local_search_ctx();
+ let ctx = index.inverted_index.local_search_ctx();
let _ = Query::parse(
&ctx,