web search done right
<title> tag inside an <svg> tag would mistakenly be chosen as the title of the page
Mikkel Denker44cc5a8d0f5b
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex 7a0a49b1..dfde819c 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -156,16 +156,35 @@ impl Html {}pub fn title(&self) -> Option<String> {- if let Some(title) = self.root.select_first("title") {- let title = title.text_contents().trim().to_string();- if title.is_empty() {- None- } else {- Some(title)+ let mut preprocessor =+ super::just_text::Preprocessor::new(["script", "style", "embed", "iframe", "svg"]);++ for edge in self.root.traverse() {+ preprocessor.update(&edge);++ if preprocessor.is_inside_removed() {+ continue;+ }++ if let kuchiki::iter::NodeEdge::Start(node) = edge {+ if let Some(title) = node.as_element().and_then(|e| {+ if &e.name.local == "title" {+ let title = node.text_contents().trim().to_string();+ if !title.is_empty() {+ Some(title)+ } else {+ None+ }+ } else {+ None+ }+ }) {+ return Some(title);+ }}- } else {- None}++ None}pub fn url(&self) -> &Url {@@ -567,6 +586,29 @@ mod tests {assert_eq!(webpage.title(), None);}+ #[test]+ fn title_in_svg() {+ let raw = format!(+ r#"+ <html>+ <head>+ <svg>+ <title>SVG title</title>+ </svg>+ <title>Best website</title>+ </head>+ <body>+ <p>{CONTENT}</p>+ </body>+ </html>+ "#+ );++ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();++ assert_eq!(webpage.title(), Some("Best website".to_string()));+ }+#[test]fn text_raw_body() {let raw = format!(diff --git a/crates/core/src/webpage/just_text.rs b/crates/core/src/webpage/just_text.rsindex b5446bf3..ff0b2eb0 100644--- a/crates/core/src/webpage/just_text.rs+++ b/crates/core/src/webpage/just_text.rs@@ -239,8 +239,9 @@ impl JustText {pub fn paragraphs(root: NodeRef) -> Vec<Paragraph> {let mut res = Vec::new();- let mut preprocessor =- Preprocessor::new(["script", "style", "embed", "head", "noscript", "iframe"]);+ let mut preprocessor = Preprocessor::new([+ "script", "style", "embed", "head", "noscript", "iframe", "svg",+ ]);let mut br = false;let mut link = false;diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex 2e897ef0..c22b4834 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -19,7 +19,7 @@ param_grid = {"max_depth": [-1],# "num_leaves": [7, 15, 31],"num_leaves": [63],- "lambda_l2": [2.0, 2.5],+ "lambda_l2": [2.5],"linear_tree": [False],}diff --git a/ltr/stract.py b/ltr/stract.pyindex 2617ffd8..c9698e14 100644--- a/ltr/stract.py+++ b/ltr/stract.py@@ -1,7 +1,7 @@import requests-# API = "https://stract.com/beta/api/search"-API = "http://localhost:3000/beta/api/search"+API = "https://stract.com/beta/api/search"+# API = "http://localhost:3000/beta/api/search"NUM_RESULTS_PER_QUERY = 20