Skip to content

web search done right

move some of the hardcoded snippet choices into the configuration file

Mikkel Denkeraa8981390645
diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rs
index ee338dff..58a305a4 100644
--- a/crates/core/src/config/defaults.rs
+++ b/crates/core/src/config/defaults.rs
@@ -64,6 +64,22 @@ impl Snippet {
pub fn min_passage_width() -> usize {
20
}
+
+ pub fn empty_query_snippet_words() -> usize {
+ 50
+ }
+
+ pub fn min_description_words() -> usize {
+ 10
+ }
+
+ pub fn min_body_length() -> usize {
+ 256
+ }
+
+ pub fn min_body_length_homepage() -> usize {
+ 1024
+ }
}
pub struct Crawler;
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 4f91bc6e..11f4bf7a 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -235,6 +235,11 @@ pub struct SnippetConfig {
pub max_considered_words: Option<usize>,
pub num_words_for_lang_detection: Option<usize>,
+
+ pub empty_query_snippet_words: usize,
+ pub min_description_words: usize,
+ pub min_body_length: usize,
+ pub min_body_length_homepage: usize,
}
impl Default for SnippetConfig {
@@ -245,6 +250,10 @@ impl Default for SnippetConfig {
min_passage_width: defaults::Snippet::min_passage_width(),
max_considered_words: None,
num_words_for_lang_detection: None,
+ empty_query_snippet_words: defaults::Snippet::empty_query_snippet_words(),
+ min_description_words: defaults::Snippet::min_description_words(),
+ min_body_length: defaults::Snippet::min_body_length(),
+ min_body_length_homepage: defaults::Snippet::min_body_length_homepage(),
}
}
}
diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rs
index f4a16534..38ed2a38 100644
--- a/crates/core/src/inverted_index.rs
+++ b/crates/core/src/inverted_index.rs
@@ -447,22 +447,35 @@ impl InvertedIndex {
}) {
if query.simple_terms().is_empty() {
let snippet = if let Some(description) = page.description.as_deref() {
- let snip = description.split_whitespace().take(50).join(" ");
-
- if snip.split_whitespace().count() < 10 {
- page.body.split_whitespace().take(50).join(" ")
+ let snip = description
+ .split_whitespace()
+ .take(self.snippet_config.empty_query_snippet_words)
+ .join(" ");
+
+ if snip.split_whitespace().count() < self.snippet_config.min_description_words {
+ page.body
+ .split_whitespace()
+ .take(self.snippet_config.empty_query_snippet_words)
+ .join(" ")
} else {
snip
}
} else {
- page.body.split_whitespace().take(50).join(" ")
+ page.body
+ .split_whitespace()
+ .take(self.snippet_config.empty_query_snippet_words)
+ .join(" ")
};
page.snippet = TextSnippet {
fragments: vec![TextSnippetFragment::new_unhighlighted(snippet)],
};
} else {
- let min_body_len = if url.is_homepage() { 1024 } else { 256 };
+ let min_body_len = if url.is_homepage() {
+ self.snippet_config.min_body_length_homepage
+ } else {
+ self.snippet_config.min_body_length
+ };
if page.body.split_whitespace().count() < min_body_len
&& page
@@ -471,7 +484,7 @@ impl InvertedIndex {
.unwrap_or_default()
.split_whitespace()
.count()
- >= 8
+ >= self.snippet_config.min_description_words
{
page.snippet = snippet::generate(
query,
diff --git a/scripts/requirements.txt b/scripts/requirements.txt
index 3766728c..f6ad1f5c 100644
--- a/scripts/requirements.txt
+++ b/scripts/requirements.txt
@@ -1,6 +1,4 @@
transformers
-tensorboard>1.15.0
-torch>=2.0.0
datasets
gensim
safetensors
@@ -8,4 +6,3 @@ pandas
llama-cpp-python[server]
llama-cpp-python
huggingface-hub>=0.20.3
-gguf