web search done right
Ability to limit number of words considered in snippet generation. Some search results has a very large amount of text, which results in the vast majority of time being spent in snippet generation compared to search.
Mikkel Denker68ba1330452f
diff --git a/configs/search_server.toml b/configs/search_server.tomlindex 021d3642..35c9b30b 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -7,4 +7,8 @@ index_path = "data/index"entity_index_path = "data/entity"host_centrality_store_path = "data/centrality"# linear_model_path = "data/linear_model.json"-# lambda_model_path = "data/lambdamart.txt"\ No newline at end of file+# lambda_model_path = "data/lambdamart.txt"++[snippet]+max_considered_words = 10_000+num_words_for_lang_detection = 1_000\ No newline at end of filediff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rsindex 86b66d57..c22b9e3a 100644--- a/core/benches/search-preindexed.rs+++ b/core/benches/search-preindexed.rs@@ -16,21 +16,24 @@ macro_rules! bench {desc.push('\'');$c.bench_function(desc.as_str(), |b| {b.iter(|| {- $searcher- .search(&SearchQuery {- query: $query.to_string(),- site_rankings: Some(SiteRankings {- liked: vec![- "docs.rs".to_string(),- "news.ycombinator.com".to_string(),- "pubmed.ncbi.nlm.nih.gov".to_string(),- ],- disliked: vec!["www.pinterest.com".to_string()],- blocked: vec![],- }),- ..Default::default()- })- .unwrap()+ dbg!(+ $searcher+ .search(&SearchQuery {+ query: $query.to_string(),+ site_rankings: Some(SiteRankings {+ liked: vec![+ "docs.rs".to_string(),+ "news.ycombinator.com".to_string(),+ "pubmed.ncbi.nlm.nih.gov".to_string(),+ ],+ disliked: vec!["www.pinterest.com".to_string()],+ blocked: vec![],+ }),+ ..Default::default()+ })+ .unwrap()+ .num_hits+ )})});};diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 42f5a51d..66096a4b 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -225,6 +225,9 @@ pub struct SnippetConfig {#[serde(default = "defaults::Snippet::min_passage_width")]pub min_passage_width: usize,++ pub max_considered_words: Option<usize>,+ pub num_words_for_lang_detection: Option<usize>,}impl Default for SnippetConfig {@@ -233,6 +236,8 @@ impl Default for SnippetConfig {desired_num_chars: defaults::Snippet::desired_num_chars(),delta_num_chars: defaults::Snippet::delta_num_chars(),min_passage_width: defaults::Snippet::min_passage_width(),+ max_considered_words: None,+ num_words_for_lang_detection: None,}}}diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex c8206c8c..e841d46d 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -21,6 +21,7 @@ use tantivy::{use crate::{bangs::BANG_PREFIX,+ floor_char_boundary,schema::{Field, TextField, ALL_FIELDS},};@@ -339,6 +340,8 @@ pub fn parse(query: &str) -> Vec<Box<Term>> {continue;}+ cur_term_begin = floor_char_boundary(&query, cur_term_begin);+if query[cur_term_begin..].starts_with('"') {if let Some(offset) = query[cur_term_begin + 1..].find('"') {let offset = offset + cur_term_begin + 1;diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex fc81d589..8a1dcf3c 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -20,7 +20,7 @@ use crate::search_prettifier::html_escape;use crate::spell::sentence_ranges;use crate::tokenizer::{BigramTokenizer, Stemmed, Tokenizer, TrigramTokenizer};use crate::webpage::region::Region;-use std::collections::{HashMap, HashSet};+use hashbrown::{HashMap, HashSet};use std::ops::Range;use itertools::Itertools;@@ -239,16 +239,36 @@ fn snippet_string(pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfig) -> String {let lang = match region.lang() {Some(lang) => lang,- None => whatlang::detect_lang(text).unwrap_or(Lang::Eng),+ None => match config.num_words_for_lang_detection {+ Some(num_words) => whatlang::detect_lang(+ text.split_whitespace()+ .take(num_words)+ .collect::<String>()+ .as_str(),+ ),+ None => whatlang::detect_lang(text),+ }+ .unwrap_or(Lang::Eng),};if text.is_empty() {return text.to_string();}- let snippet = snippet_string(text, query.simple_terms(), lang, config);+ match config.max_considered_words {+ Some(num_words) => {+ let text = text.split_whitespace().take(num_words).join(" ");++ let snippet = snippet_string(&text, query.simple_terms(), lang, config);++ snippet.to_html()+ }+ None => {+ let snippet = snippet_string(text, query.simple_terms(), lang, config);- snippet.to_html()+ snippet.to_html()+ }+ }}#[cfg(test)]