web search done right
parse webpage
Mikkel Denkerd41e69831974
diff --git a/Cargo.toml b/Cargo.tomlindex c5331f98..6fadab0d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,8 +11,10 @@ name = "cuely"[dependencies]encoding_rs = "0.8.31"flate2 = "1.0.23"+itertools = "0.10.3"rusoto_core = "0.48.0"rusoto_s3 = "0.48.0"+scraper = "0.13.0"serde = "1.0.137"thiserror = "1.0.31"tokio = { version = "1.18.2", features = ["full"] }diff --git a/src/lib.rs b/src/lib.rsindex 2eac4ae7..aec43650 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -5,6 +5,7 @@ use thiserror::Error;mod indexer;mod warc;+mod webpage;pub use indexer::Indexer;diff --git a/src/warc.rs b/src/warc.rsindex 55bad3e8..c7befe88 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -129,15 +129,15 @@ struct RawWarcRecord {#[derive(Debug)]pub(crate) struct WarcRecord {- request: Request,- response: Response,- metadata: Metadata,+ pub(crate) request: Request,+ pub(crate) response: Response,+ pub(crate) metadata: Metadata,}#[derive(Debug)]-struct Request {+pub(crate) struct Request {// WARC-Target-URI- url: String,+ pub(crate) url: String,}impl Request {@@ -153,8 +153,8 @@ impl Request {}#[derive(Debug)]-struct Response {- body: String,+pub(crate) struct Response {+ pub(crate) body: String,}impl Response {@@ -166,9 +166,9 @@ impl Response {}#[derive(Debug)]-struct Metadata {+pub(crate) struct Metadata {// fetchTimeMs- fetch_time_ms: usize,+ pub(crate) fetch_time_ms: usize,}impl Metadata {@@ -208,14 +208,12 @@ impl<R: BufRead> Iterator for WarcFile<R> {let mut metadata = None;for item in items {- if item.is_none() {- return None;- }- let item = item.unwrap();+ let item = item?;if item.is_err() {return Some(Err(item.err().unwrap()));}+let item = item.unwrap();if let Some(warc_type) = item.header.get("WARC-TYPE") {diff --git a/src/webpage.rs b/src/webpage.rsnew file mode 100644index 00000000..104a3298--- /dev/null+++ b/src/webpage.rs@@ -0,0 +1,116 @@+use itertools::Itertools;+use scraper::{Html, Selector};+use std::collections::BTreeMap;++#[derive(Debug)]+pub(crate) struct Webpage {+ dom: Html,+}++impl Webpage {+ pub(crate) fn parse(html: &str) -> Self {+ Self {+ dom: Html::parse_document(html),+ }+ }++ pub(crate) fn links(&self) -> Vec<Link> {+ let selector = Selector::parse("a").expect("Failed to parse selector");+ self.dom+ .select(&selector)+ .map(|el| {+ let destination = el.value().attr("href");+ let text = el.text().collect::<String>();++ (destination, text)+ })+ .filter(|(dest, _)| dest.is_some())+ .map(|(destination, text)| {+ let destination = destination.unwrap().to_string();+ Link { destination, text }+ })+ .collect()+ }++ fn grab_texts(&self, selector: &Selector) -> Vec<String> {+ self.dom+ .select(selector)+ .flat_map(|el| el.text())+ .map(|t| String::from(t.trim()))+ .filter(|t| !t.is_empty())+ .collect::<Vec<String>>()+ }++ pub(crate) fn text(&self) -> String {+ let selector = Selector::parse("body").expect("Failed to parse selector");+ Itertools::intersperse(self.grab_texts(&selector).into_iter(), "\n".to_string())+ .collect::<String>()+ .trim()+ .to_string()+ }++ pub(crate) fn title(&self) -> Option<String> {+ let selector = Selector::parse("title").expect("Failed to parse selector");+ self.grab_texts(&selector).get(0).cloned()+ }++ pub(crate) fn metadata(&self) -> Vec<Meta> {+ let selector = Selector::parse("meta").expect("Failed to parse selector");+ self.dom+ .select(&selector)+ .map(|el| {+ el.value()+ .attrs()+ .into_iter()+ .map(|(k, v)| (k.to_string(), v.to_string()))+ .collect::<BTreeMap<String, String>>()+ })+ .collect()+ }+}++#[derive(Debug, PartialEq)]+pub(crate) struct Link {+ destination: String,+ text: String,+}++pub(crate) type Meta = BTreeMap<String, String>;++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn simple() {+ let raw = r#"+ <html>+ <head>+ <title>Best website</title>+ <meta name="meta1" content="value">+ </head>+ <body>+ <a href="example.com">Best example website ever</a>+ </body>+ </html>+ "#;++ let webpage = Webpage::parse(raw);++ assert_eq!(&webpage.text(), "Best example website ever");+ assert_eq!(webpage.title(), Some("Best website".to_string()));+ assert_eq!(+ webpage.links(),+ vec![Link {+ destination: "example.com".to_string(),+ text: "Best example website ever".to_string()+ }]+ );++ let mut expected_meta = BTreeMap::new();+ expected_meta.insert("name".to_string(), "meta1".to_string());+ expected_meta.insert("content".to_string(), "value".to_string());++ assert_eq!(webpage.metadata(), vec![expected_meta]);+ }+}