Skip to content

web search done right

style+link tags are ignored when extracting text

Mikkel Denker3b4d1aa9295e
diff --git a/src/indexer.rs b/src/indexer.rs
index 1a23a454..3fb63af4 100644
--- a/src/indexer.rs
+++ b/src/indexer.rs
@@ -5,6 +5,7 @@ use std::io::{self, BufRead};
use tokio::io::AsyncReadExt;
use crate::warc::WarcFile;
+use crate::webpage::Webpage;
use crate::{Config, Error, Result};
pub struct Indexer {
@@ -38,8 +39,11 @@ impl Indexer {
println!("Downloaded {} bytes", raw_object.len());
let warc = WarcFile::new(&raw_object[..]);
for record in warc {
- println!("TEST: {:?}", record);
- panic!();
+ if let Ok(record) = record {
+ let webpage = Webpage::parse(&record.response.body);
+ println!("TEST: {:?}", webpage.text());
+ panic!();
+ }
}
}
diff --git a/src/warc.rs b/src/warc.rs
index c7befe88..1cd376e3 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -159,8 +159,14 @@ pub(crate) struct Response {
impl Response {
fn from_raw(record: RawWarcRecord) -> Result<Self> {
+ let content = decode(record.content);
+
+ let (_header, content) = content
+ .split_once("\r\n\r\n")
+ .ok_or(Error::WarcParse("Invalid http body"))?;
+
Ok(Self {
- body: decode(record.content),
+ body: content.to_string(),
})
}
}
@@ -281,7 +287,9 @@ mod tests {
\r\n\
warc/1.0\r\n\
warc-tYPE: response\r\n\
- cONTENT-lENGTH: 16\r\n\
+ cONTENT-lENGTH: 29\r\n\
+ \r\n\
+ http-body\r\n\
\r\n\
body of response\r\n\
\r\n\
diff --git a/src/webpage.rs b/src/webpage.rs
index 104a3298..3416ee77 100644
--- a/src/webpage.rs
+++ b/src/webpage.rs
@@ -1,5 +1,5 @@
use itertools::Itertools;
-use scraper::{Html, Selector};
+use scraper::{Html, Node, Selector};
use std::collections::BTreeMap;
#[derive(Debug)]
@@ -35,14 +35,28 @@ impl Webpage {
fn grab_texts(&self, selector: &Selector) -> Vec<String> {
self.dom
.select(selector)
- .flat_map(|el| el.text())
+ .filter(|el| selector.matches(el))
+ .map(|el| {
+ if let Some(node) = (*el).first_child() {
+ if let Node::Text(text) = node.value() {
+ Some(text)
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ })
+ .filter(|text| text.is_some())
+ .map(|text| text.unwrap())
.map(|t| String::from(t.trim()))
.filter(|t| !t.is_empty())
.collect::<Vec<String>>()
}
pub(crate) fn text(&self) -> String {
- let selector = Selector::parse("body").expect("Failed to parse selector");
+ let selector =
+ Selector::parse("body :not(script):not(style)").expect("Failed to parse selector");
Itertools::intersperse(self.grab_texts(&selector).into_iter(), "\n".to_string())
.collect::<String>()
.trim()
@@ -113,4 +127,62 @@ mod tests {
assert_eq!(webpage.metadata(), vec![expected_meta]);
}
+
+ #[test]
+ fn script_tags_text_ignored() {
+ let raw = r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ <script>this should not be extracted</script>
+ </head>
+ <body>
+ <script>this should not be extracted</script>
+ <p>This text should be the first text extracted</p>
+ <div>
+ <script>this should not be extracted</script>
+ <p>This text should be the second text extracted</p>
+ </div>
+ <script>this should not be extracted</script>
+ </body>
+ </html>
+ "#;
+
+ let webpage = Webpage::parse(raw);
+
+ assert_eq!(
+ webpage.text(),
+ "This text should be the first text extracted\nThis text should be the second text extracted"
+ );
+ }
+
+ #[test]
+ fn style_tags_text_ignored() {
+ let raw = r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ <style>this should not be extracted</style>
+ </head>
+ <body>
+ <style>this should not be extracted</style>
+ <p>This text should be the first text extracted</p>
+ <div>
+ <style>this should not be extracted</style>
+ <p>This text should be the second text extracted</p>
+ </div>
+ <style>this should not be extracted</style>
+ </body>
+ </html>
+ "#;
+
+ let webpage = Webpage::parse(raw);
+
+ assert_eq!(
+ webpage.text(),
+ "This text should be the first text extracted\nThis text should be the second text extracted"
+ );
+ }
}