Skip to content

web search done right

use description meta tag for snippets when clean_body is small. right now the cutoff is at 256 chars. if the body is less than this amount, generate the snippet based on description meta tag. the cutoff for homepages is 1024 as we want to prioritise the use of the description tag in snippets for homepages. apparently many developers think they can control the snippet with this tag, so we will probably get better snippets with these changes.

Mikkel Denker159537a8fa99
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index ded79e9f..09fca40c 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -435,17 +435,24 @@ impl InvertedIndex {
.filter_map(|res| res.ok())
.collect();
- for page in &mut webpages {
- if !page.body.is_empty() {
- page.snippet =
- snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());
- } else {
+ for (url, page) in webpages.iter_mut().filter_map(|page| {
+ let url = Url::parse(&page.url).ok()?;
+ Some((url, page))
+ }) {
+ let min_body_len = if url.is_homepage() { 1024 } else { 256 };
+
+ if page.body.len() < min_body_len
+ && !page.description.as_deref().unwrap_or_default().is_empty()
+ {
page.snippet = snippet::generate(
query,
page.description.as_deref().unwrap_or_default(),
&page.region,
self.snippet_config.clone(),
);
+ } else {
+ page.snippet =
+ snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());
}
}
diff --git a/core/src/webpage/html/mod.rs b/core/src/webpage/html/mod.rs
index ff2aeb4d..95ec0db3 100644
--- a/core/src/webpage/html/mod.rs
+++ b/core/src/webpage/html/mod.rs
@@ -443,7 +443,7 @@ impl Html {
}
pub fn is_homepage(&self) -> bool {
- self.url().path() == "/" && self.url().query().is_none()
+ self.url().is_homepage()
}
}
diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rs
index 42f0eec0..48bd5034 100644
--- a/core/src/webpage/url_ext.rs
+++ b/core/src/webpage/url_ext.rs
@@ -33,6 +33,7 @@ pub trait UrlExt {
fn root_domain(&self) -> Option<&str>;
fn normalized_host(&self) -> Option<&str>;
fn subdomain(&self) -> Option<&str>;
+ fn is_homepage(&self) -> bool;
fn tld(&self) -> Option<&str>;
}
@@ -67,6 +68,10 @@ impl UrlExt for url::Url {
Some(subdomain)
}
+ fn is_homepage(&self) -> bool {
+ self.path() == "/" && self.query().is_none()
+ }
+
fn tld(&self) -> Option<&str> {
let host = self.host_str()?;
let suffix = std::str::from_utf8(ICANN_LIST.suffix(host.as_bytes())?.as_bytes()).ok()?;