web search done right
move some of the hardcoded snippet choices into the configuration file
Mikkel Denkeraa8981390645
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex ee338dff..58a305a4 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -64,6 +64,22 @@ impl Snippet {pub fn min_passage_width() -> usize {20}++ pub fn empty_query_snippet_words() -> usize {+ 50+ }++ pub fn min_description_words() -> usize {+ 10+ }++ pub fn min_body_length() -> usize {+ 256+ }++ pub fn min_body_length_homepage() -> usize {+ 1024+ }}pub struct Crawler;diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 4f91bc6e..11f4bf7a 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -235,6 +235,11 @@ pub struct SnippetConfig {pub max_considered_words: Option<usize>,pub num_words_for_lang_detection: Option<usize>,++ pub empty_query_snippet_words: usize,+ pub min_description_words: usize,+ pub min_body_length: usize,+ pub min_body_length_homepage: usize,}impl Default for SnippetConfig {@@ -245,6 +250,10 @@ impl Default for SnippetConfig {min_passage_width: defaults::Snippet::min_passage_width(),max_considered_words: None,num_words_for_lang_detection: None,+ empty_query_snippet_words: defaults::Snippet::empty_query_snippet_words(),+ min_description_words: defaults::Snippet::min_description_words(),+ min_body_length: defaults::Snippet::min_body_length(),+ min_body_length_homepage: defaults::Snippet::min_body_length_homepage(),}}}diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rsindex f4a16534..38ed2a38 100644--- a/crates/core/src/inverted_index.rs+++ b/crates/core/src/inverted_index.rs@@ -447,22 +447,35 @@ impl InvertedIndex {}) {if query.simple_terms().is_empty() {let snippet = if let Some(description) = page.description.as_deref() {- let snip = description.split_whitespace().take(50).join(" ");-- if snip.split_whitespace().count() < 10 {- page.body.split_whitespace().take(50).join(" ")+ let snip = description+ .split_whitespace()+ .take(self.snippet_config.empty_query_snippet_words)+ .join(" ");++ if snip.split_whitespace().count() < self.snippet_config.min_description_words {+ page.body+ .split_whitespace()+ .take(self.snippet_config.empty_query_snippet_words)+ .join(" ")} else {snip}} else {- page.body.split_whitespace().take(50).join(" ")+ page.body+ .split_whitespace()+ .take(self.snippet_config.empty_query_snippet_words)+ .join(" ")};page.snippet = TextSnippet {fragments: vec![TextSnippetFragment::new_unhighlighted(snippet)],};} else {- let min_body_len = if url.is_homepage() { 1024 } else { 256 };+ let min_body_len = if url.is_homepage() {+ self.snippet_config.min_body_length_homepage+ } else {+ self.snippet_config.min_body_length+ };if page.body.split_whitespace().count() < min_body_len&& page@@ -471,7 +484,7 @@ impl InvertedIndex {.unwrap_or_default().split_whitespace().count()- >= 8+ >= self.snippet_config.min_description_words{page.snippet = snippet::generate(query,diff --git a/scripts/requirements.txt b/scripts/requirements.txtindex 3766728c..f6ad1f5c 100644--- a/scripts/requirements.txt+++ b/scripts/requirements.txt@@ -1,6 +1,4 @@transformers-tensorboard>1.15.0-torch>=2.0.0datasetsgensimsafetensors@@ -8,4 +6,3 @@ pandasllama-cpp-python[server]llama-cpp-pythonhuggingface-hub>=0.20.3-gguf