web search done right
use description meta tag for snippets when clean_body is small. right now the cutoff is at 256 chars. if the body is less than this amount, generate the snippet based on description meta tag. the cutoff for homepages is 1024 as we want to prioritise the use of the description tag in snippets for homepages. apparently many developers think they can control the snippet with this tag, so we will probably get better snippets with these changes.
Mikkel Denker159537a8fa99
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex ded79e9f..09fca40c 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -435,17 +435,24 @@ impl InvertedIndex {.filter_map(|res| res.ok()).collect();- for page in &mut webpages {- if !page.body.is_empty() {- page.snippet =- snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());- } else {+ for (url, page) in webpages.iter_mut().filter_map(|page| {+ let url = Url::parse(&page.url).ok()?;+ Some((url, page))+ }) {+ let min_body_len = if url.is_homepage() { 1024 } else { 256 };++ if page.body.len() < min_body_len+ && !page.description.as_deref().unwrap_or_default().is_empty()+ {page.snippet = snippet::generate(query,page.description.as_deref().unwrap_or_default(),&page.region,self.snippet_config.clone(),);+ } else {+ page.snippet =+ snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());}}diff --git a/core/src/webpage/html/mod.rs b/core/src/webpage/html/mod.rsindex ff2aeb4d..95ec0db3 100644--- a/core/src/webpage/html/mod.rs+++ b/core/src/webpage/html/mod.rs@@ -443,7 +443,7 @@ impl Html {}pub fn is_homepage(&self) -> bool {- self.url().path() == "/" && self.url().query().is_none()+ self.url().is_homepage()}}diff --git a/core/src/webpage/url_ext.rs b/core/src/webpage/url_ext.rsindex 42f0eec0..48bd5034 100644--- a/core/src/webpage/url_ext.rs+++ b/core/src/webpage/url_ext.rs@@ -33,6 +33,7 @@ pub trait UrlExt {fn root_domain(&self) -> Option<&str>;fn normalized_host(&self) -> Option<&str>;fn subdomain(&self) -> Option<&str>;+ fn is_homepage(&self) -> bool;fn tld(&self) -> Option<&str>;}@@ -67,6 +68,10 @@ impl UrlExt for url::Url {Some(subdomain)}+ fn is_homepage(&self) -> bool {+ self.path() == "/" && self.query().is_none()+ }+fn tld(&self) -> Option<&str> {let host = self.host_str()?;let suffix = std::str::from_utf8(ICANN_LIST.suffix(host.as_bytes())?.as_bytes()).ok()?;