Skip to content

web search done right

Make spell dictionary optional

Mikkel Denker28c8da906a72
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index e51937a0..ba4a376c 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -105,3 +105,11 @@ impl WebgraphServer {
1_000
}
}
+
+pub struct SearchServer;
+
+impl SearchServer {
+ pub fn build_spell_dictionary() -> bool {
+ false
+ }
+}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 7228acd7..787a3a6d 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -255,6 +255,9 @@ pub struct SearchServerConfig {
#[serde(default)]
pub snippet: SnippetConfig,
+
+ #[serde(default = "defaults::SearchServer::build_spell_dictionary")]
+ pub build_spell_dictionary: bool,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 66bef9d3..ff68fddb 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -83,6 +83,10 @@ impl SearchService {
local_searcher.set_lambda_model(LambdaMART::open(model_path)?);
}
+ if config.build_spell_dictionary {
+ local_searcher.build_spell_dict();
+ }
+
local_searcher.set_collector_config(config.collector);
local_searcher.set_snippet_config(config.snippet);
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 369dcce2..a0c64ced 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -43,7 +43,7 @@ use super::{InitialWebsiteResult, SearchQuery};
pub struct LocalSearcher {
index: Index,
- spell: Spell,
+ spell: Option<Spell>,
entity_index: Option<EntityIndex>,
centrality_store: Option<SearchCentralityStore>,
linear_regression: Option<Arc<LinearRegression>>,
@@ -65,14 +65,12 @@ struct InvertedIndexResult {
impl LocalSearcher {
pub fn new(index: Index) -> Self {
- let spell = Spell::for_index(&index);
-
let mut index = index;
index.optimize_for_search().unwrap();
LocalSearcher {
index,
- spell,
+ spell: None,
entity_index: None,
centrality_store: None,
linear_regression: None,
@@ -81,6 +79,10 @@ impl LocalSearcher {
}
}
+ pub fn build_spell_dict(&mut self) {
+ self.spell = Some(Spell::for_index(&self.index));
+ }
+
pub fn set_entity_index(&mut self, entity_index: EntityIndex) {
self.entity_index = Some(entity_index);
}
@@ -281,7 +283,7 @@ impl LocalSearcher {
) -> Result<InitialWebsiteResult> {
let ctx = self.index.inverted_index.local_search_ctx();
let inverted_index_result = self.search_inverted_index(&ctx, query, de_rank_similar)?;
- let correction = self.spell.correction(query);
+ let correction = self.spell.as_ref().and_then(|s| s.correction(query));
let sidebar = self
.entity_sidebar(query)
.map(|entity| DisplayedEntity::from(entity, self));
@@ -504,12 +506,15 @@ mod tests {
index.commit().unwrap();
- let searcher = LocalSearcher::new(index);
+ let mut searcher = LocalSearcher::new(index);
+ searcher.build_spell_dict();
assert_eq!(
String::from(
searcher
.spell
+ .as_ref()
+ .unwrap()
.correction(&SearchQuery {
query: "th best".to_string(),
..Default::default()
@@ -519,7 +524,7 @@ mod tests {
"the best".to_string()
);
assert_eq!(
- searcher.spell.correction(&SearchQuery {
+ searcher.spell.as_ref().unwrap().correction(&SearchQuery {
query: "the best".to_string(),
..Default::default()
}),