Skip to content

web search done right

<title> tag inside an <svg> tag would mistakenly be chosen as the title of the page

Mikkel Denker44cc5a8d0f5b
diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rs
index 7a0a49b1..dfde819c 100644
--- a/crates/core/src/webpage/html/mod.rs
+++ b/crates/core/src/webpage/html/mod.rs
@@ -156,16 +156,35 @@ impl Html {
}
pub fn title(&self) -> Option<String> {
- if let Some(title) = self.root.select_first("title") {
- let title = title.text_contents().trim().to_string();
- if title.is_empty() {
- None
- } else {
- Some(title)
+ let mut preprocessor =
+ super::just_text::Preprocessor::new(["script", "style", "embed", "iframe", "svg"]);
+
+ for edge in self.root.traverse() {
+ preprocessor.update(&edge);
+
+ if preprocessor.is_inside_removed() {
+ continue;
+ }
+
+ if let kuchiki::iter::NodeEdge::Start(node) = edge {
+ if let Some(title) = node.as_element().and_then(|e| {
+ if &e.name.local == "title" {
+ let title = node.text_contents().trim().to_string();
+ if !title.is_empty() {
+ Some(title)
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ }) {
+ return Some(title);
+ }
}
- } else {
- None
}
+
+ None
}
pub fn url(&self) -> &Url {
@@ -567,6 +586,29 @@ mod tests {
assert_eq!(webpage.title(), None);
}
+ #[test]
+ fn title_in_svg() {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <svg>
+ <title>SVG title</title>
+ </svg>
+ <title>Best website</title>
+ </head>
+ <body>
+ <p>{CONTENT}</p>
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
+
+ assert_eq!(webpage.title(), Some("Best website".to_string()));
+ }
+
#[test]
fn text_raw_body() {
let raw = format!(
diff --git a/crates/core/src/webpage/just_text.rs b/crates/core/src/webpage/just_text.rs
index b5446bf3..ff0b2eb0 100644
--- a/crates/core/src/webpage/just_text.rs
+++ b/crates/core/src/webpage/just_text.rs
@@ -239,8 +239,9 @@ impl JustText {
pub fn paragraphs(root: NodeRef) -> Vec<Paragraph> {
let mut res = Vec::new();
- let mut preprocessor =
- Preprocessor::new(["script", "style", "embed", "head", "noscript", "iframe"]);
+ let mut preprocessor = Preprocessor::new([
+ "script", "style", "embed", "head", "noscript", "iframe", "svg",
+ ]);
let mut br = false;
let mut link = false;
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index 2e897ef0..c22b4834 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -19,7 +19,7 @@ param_grid = {
"max_depth": [-1],
# "num_leaves": [7, 15, 31],
"num_leaves": [63],
- "lambda_l2": [2.0, 2.5],
+ "lambda_l2": [2.5],
"linear_tree": [False],
}
diff --git a/ltr/stract.py b/ltr/stract.py
index 2617ffd8..c9698e14 100644
--- a/ltr/stract.py
+++ b/ltr/stract.py
@@ -1,7 +1,7 @@
import requests
-# API = "https://stract.com/beta/api/search"
-API = "http://localhost:3000/beta/api/search"
+API = "https://stract.com/beta/api/search"
+# API = "http://localhost:3000/beta/api/search"
NUM_RESULTS_PER_QUERY = 20