web search done right
style+link tags are ignored when extracting text
Mikkel Denker3b4d1aa9295e
diff --git a/src/indexer.rs b/src/indexer.rsindex 1a23a454..3fb63af4 100644--- a/src/indexer.rs+++ b/src/indexer.rs@@ -5,6 +5,7 @@ use std::io::{self, BufRead};use tokio::io::AsyncReadExt;use crate::warc::WarcFile;+use crate::webpage::Webpage;use crate::{Config, Error, Result};pub struct Indexer {@@ -38,8 +39,11 @@ impl Indexer {println!("Downloaded {} bytes", raw_object.len());let warc = WarcFile::new(&raw_object[..]);for record in warc {- println!("TEST: {:?}", record);- panic!();+ if let Ok(record) = record {+ let webpage = Webpage::parse(&record.response.body);+ println!("TEST: {:?}", webpage.text());+ panic!();+ }}}diff --git a/src/warc.rs b/src/warc.rsindex c7befe88..1cd376e3 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -159,8 +159,14 @@ pub(crate) struct Response {impl Response {fn from_raw(record: RawWarcRecord) -> Result<Self> {+ let content = decode(record.content);++ let (_header, content) = content+ .split_once("\r\n\r\n")+ .ok_or(Error::WarcParse("Invalid http body"))?;+Ok(Self {- body: decode(record.content),+ body: content.to_string(),})}}@@ -281,7 +287,9 @@ mod tests {\r\n\warc/1.0\r\n\warc-tYPE: response\r\n\- cONTENT-lENGTH: 16\r\n\+ cONTENT-lENGTH: 29\r\n\+ \r\n\+ http-body\r\n\\r\n\body of response\r\n\\r\n\diff --git a/src/webpage.rs b/src/webpage.rsindex 104a3298..3416ee77 100644--- a/src/webpage.rs+++ b/src/webpage.rs@@ -1,5 +1,5 @@use itertools::Itertools;-use scraper::{Html, Selector};+use scraper::{Html, Node, Selector};use std::collections::BTreeMap;#[derive(Debug)]@@ -35,14 +35,28 @@ impl Webpage {fn grab_texts(&self, selector: &Selector) -> Vec<String> {self.dom.select(selector)- .flat_map(|el| el.text())+ .filter(|el| selector.matches(el))+ .map(|el| {+ if let Some(node) = (*el).first_child() {+ if let Node::Text(text) = node.value() {+ Some(text)+ } else {+ None+ }+ } else {+ None+ }+ })+ .filter(|text| text.is_some())+ .map(|text| text.unwrap()).map(|t| String::from(t.trim())).filter(|t| !t.is_empty()).collect::<Vec<String>>()}pub(crate) fn text(&self) -> String {- let selector = Selector::parse("body").expect("Failed to parse selector");+ let selector =+ Selector::parse("body :not(script):not(style)").expect("Failed to parse selector");Itertools::intersperse(self.grab_texts(&selector).into_iter(), "\n".to_string()).collect::<String>().trim()@@ -113,4 +127,62 @@ mod tests {assert_eq!(webpage.metadata(), vec![expected_meta]);}++ #[test]+ fn script_tags_text_ignored() {+ let raw = r#"+ <html>+ <head>+ <title>Best website</title>+ <meta name="meta1" content="value">+ <script>this should not be extracted</script>+ </head>+ <body>+ <script>this should not be extracted</script>+ <p>This text should be the first text extracted</p>+ <div>+ <script>this should not be extracted</script>+ <p>This text should be the second text extracted</p>+ </div>+ <script>this should not be extracted</script>+ </body>+ </html>+ "#;++ let webpage = Webpage::parse(raw);++ assert_eq!(+ webpage.text(),+ "This text should be the first text extracted\nThis text should be the second text extracted"+ );+ }++ #[test]+ fn style_tags_text_ignored() {+ let raw = r#"+ <html>+ <head>+ <title>Best website</title>+ <meta name="meta1" content="value">+ <style>this should not be extracted</style>+ </head>+ <body>+ <style>this should not be extracted</style>+ <p>This text should be the first text extracted</p>+ <div>+ <style>this should not be extracted</style>+ <p>This text should be the second text extracted</p>+ </div>+ <style>this should not be extracted</style>+ </body>+ </html>+ "#;++ let webpage = Webpage::parse(raw);++ assert_eq!(+ webpage.text(),+ "This text should be the first text extracted\nThis text should be the second text extracted"+ );+ }}