web search done right
pre-calculate and cache webpage texts
Mikkel Denkeraf61109248a2
diff --git a/src/ranking/initial.rs b/src/ranking/initial.rsindex 5aa7fdda..4535304b 100644--- a/src/ranking/initial.rs+++ b/src/ranking/initial.rs@@ -133,7 +133,7 @@ fn region_score(webpage_region: Region,) -> f64 {let boost = selected_region- .map(|region| if region == webpage_region { 15.0 } else { 0.0 })+ .map(|region| if region == webpage_region { 50.0 } else { 0.0 }).unwrap_or(0.0);boost + region_count.score(&webpage_region)diff --git a/src/schema.rs b/src/schema.rsindex f76d8acd..4ce019f5 100644--- a/src/schema.rs+++ b/src/schema.rs@@ -173,7 +173,7 @@ impl Field {Field::CleanBody => Some(4.0),Field::Title => Some(10.0),Field::Url => Some(1.0),- Field::Domain => Some(4.0),+ Field::Domain => Some(2.0),Field::AllBody => Some(0.01),Field::StemmedAllBody => Some(0.001),Field::BacklinkText => Some(4.0),diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex 03cd0d4f..0db54a36 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -195,14 +195,21 @@ struct Script {pub struct Html {url: Url,root: NodeRef, // this is reference counted (cheap to clone)+ all_text: Option<String>,+ clean_text: Option<String>,}impl Html {pub fn parse(html: &str, url: &str) -> Self {let root = kuchiki::parse_html().one(html);+ let all_text = Html::calculate_all_text(root.clone());+ let clean_text = Html::calculate_clean_text(root.clone());+Self {root,+ all_text,+ clean_text,url: url.to_string().into(),}}@@ -298,8 +305,8 @@ impl Html {None}- pub fn clean_text(&self) -> Option<String> {- let text = JustText::default().extract(self.root.clone());+ fn calculate_clean_text(node: NodeRef) -> Option<String> {+ let text = JustText::default().extract(node);if text.is_empty() {None@@ -308,7 +315,11 @@ impl Html {}}- pub fn all_text(&self) -> Option<String> {+ pub fn clean_text(&self) -> Option<String> {+ self.clean_text.clone()+ }++ fn calculate_all_text(node: NodeRef) -> Option<String> {let text = JustText {max_link_density: 2.0,length_low: 0,@@ -317,7 +328,7 @@ impl Html {stopwords_high: -1.0,max_heading_distance: 10000,}- .extract(self.root.clone());+ .extract(node);if text.is_empty() {None@@ -326,9 +337,18 @@ impl Html {}}+ fn all_text(&self) -> Option<String> {+ self.all_text.clone()+ }+pub fn title(&self) -> Option<String> {if let Ok(title) = self.root.select_first("title") {- Some(title.text_contents().trim().to_string())+ let title = title.text_contents().trim().to_string();+ if !title.is_empty() {+ Some(title)+ } else {+ None+ }} else {None}@@ -666,6 +686,26 @@ mod tests {assert_eq!(webpage.domain(), "example.com");}+ #[test]+ fn empty_title() {+ let raw = format!(+ r#"+ <html>+ <head>+ <title></title>+ </head>+ <body>+ <p>{CONTENT}</p>+ </body>+ </html>+ "#+ );++ let webpage = Html::parse(&raw, "https://www.example.com/whatever");++ assert_eq!(webpage.title(), None);+ }+#[test]fn text_raw_body() {let raw = format!(