Skip to content

web search done right

Ability to limit number of words considered in snippet generation. Some search results has a very large amount of text, which results in the vast majority of time being spent in snippet generation compared to search.

Mikkel Denker68ba1330452f
diff --git a/configs/search_server.toml b/configs/search_server.toml
index 021d3642..35c9b30b 100644
--- a/configs/search_server.toml
+++ b/configs/search_server.toml
@@ -7,4 +7,8 @@ index_path = "data/index"
entity_index_path = "data/entity"
host_centrality_store_path = "data/centrality"
# linear_model_path = "data/linear_model.json"
-# lambda_model_path = "data/lambdamart.txt"
\ No newline at end of file
+# lambda_model_path = "data/lambdamart.txt"
+
+[snippet]
+max_considered_words = 10_000
+num_words_for_lang_detection = 1_000
\ No newline at end of file
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rs
index 86b66d57..c22b9e3a 100644
--- a/core/benches/search-preindexed.rs
+++ b/core/benches/search-preindexed.rs
@@ -16,21 +16,24 @@ macro_rules! bench {
desc.push('\'');
$c.bench_function(desc.as_str(), |b| {
b.iter(|| {
- $searcher
- .search(&SearchQuery {
- query: $query.to_string(),
- site_rankings: Some(SiteRankings {
- liked: vec![
- "docs.rs".to_string(),
- "news.ycombinator.com".to_string(),
- "pubmed.ncbi.nlm.nih.gov".to_string(),
- ],
- disliked: vec!["www.pinterest.com".to_string()],
- blocked: vec![],
- }),
- ..Default::default()
- })
- .unwrap()
+ dbg!(
+ $searcher
+ .search(&SearchQuery {
+ query: $query.to_string(),
+ site_rankings: Some(SiteRankings {
+ liked: vec![
+ "docs.rs".to_string(),
+ "news.ycombinator.com".to_string(),
+ "pubmed.ncbi.nlm.nih.gov".to_string(),
+ ],
+ disliked: vec!["www.pinterest.com".to_string()],
+ blocked: vec![],
+ }),
+ ..Default::default()
+ })
+ .unwrap()
+ .num_hits
+ )
})
});
};
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 42f5a51d..66096a4b 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -225,6 +225,9 @@ pub struct SnippetConfig {
#[serde(default = "defaults::Snippet::min_passage_width")]
pub min_passage_width: usize,
+
+ pub max_considered_words: Option<usize>,
+ pub num_words_for_lang_detection: Option<usize>,
}
impl Default for SnippetConfig {
@@ -233,6 +236,8 @@ impl Default for SnippetConfig {
desired_num_chars: defaults::Snippet::desired_num_chars(),
delta_num_chars: defaults::Snippet::delta_num_chars(),
min_passage_width: defaults::Snippet::min_passage_width(),
+ max_considered_words: None,
+ num_words_for_lang_detection: None,
}
}
}
diff --git a/core/src/query/parser.rs b/core/src/query/parser.rs
index c8206c8c..e841d46d 100644
--- a/core/src/query/parser.rs
+++ b/core/src/query/parser.rs
@@ -21,6 +21,7 @@ use tantivy::{
use crate::{
bangs::BANG_PREFIX,
+ floor_char_boundary,
schema::{Field, TextField, ALL_FIELDS},
};
@@ -339,6 +340,8 @@ pub fn parse(query: &str) -> Vec<Box<Term>> {
continue;
}
+ cur_term_begin = floor_char_boundary(&query, cur_term_begin);
+
if query[cur_term_begin..].starts_with('"') {
if let Some(offset) = query[cur_term_begin + 1..].find('"') {
let offset = offset + cur_term_begin + 1;
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index fc81d589..8a1dcf3c 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -20,7 +20,7 @@ use crate::search_prettifier::html_escape;
use crate::spell::sentence_ranges;
use crate::tokenizer::{BigramTokenizer, Stemmed, Tokenizer, TrigramTokenizer};
use crate::webpage::region::Region;
-use std::collections::{HashMap, HashSet};
+use hashbrown::{HashMap, HashSet};
use std::ops::Range;
use itertools::Itertools;
@@ -239,16 +239,36 @@ fn snippet_string(
pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfig) -> String {
let lang = match region.lang() {
Some(lang) => lang,
- None => whatlang::detect_lang(text).unwrap_or(Lang::Eng),
+ None => match config.num_words_for_lang_detection {
+ Some(num_words) => whatlang::detect_lang(
+ text.split_whitespace()
+ .take(num_words)
+ .collect::<String>()
+ .as_str(),
+ ),
+ None => whatlang::detect_lang(text),
+ }
+ .unwrap_or(Lang::Eng),
};
if text.is_empty() {
return text.to_string();
}
- let snippet = snippet_string(text, query.simple_terms(), lang, config);
+ match config.max_considered_words {
+ Some(num_words) => {
+ let text = text.split_whitespace().take(num_words).join(" ");
+
+ let snippet = snippet_string(&text, query.simple_terms(), lang, config);
+
+ snippet.to_html()
+ }
+ None => {
+ let snippet = snippet_string(text, query.simple_terms(), lang, config);
- snippet.to_html()
+ snippet.to_html()
+ }
+ }
}
#[cfg(test)]