Skip to content

web search done right

parse webpage

Mikkel Denkerd41e69831974
diff --git a/Cargo.toml b/Cargo.toml
index c5331f98..6fadab0d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,8 +11,10 @@ name = "cuely"
[dependencies]
encoding_rs = "0.8.31"
flate2 = "1.0.23"
+itertools = "0.10.3"
rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
+scraper = "0.13.0"
serde = "1.0.137"
thiserror = "1.0.31"
tokio = { version = "1.18.2", features = ["full"] }
diff --git a/src/lib.rs b/src/lib.rs
index 2eac4ae7..aec43650 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -5,6 +5,7 @@ use thiserror::Error;
mod indexer;
mod warc;
+mod webpage;
pub use indexer::Indexer;
diff --git a/src/warc.rs b/src/warc.rs
index 55bad3e8..c7befe88 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -129,15 +129,15 @@ struct RawWarcRecord {
#[derive(Debug)]
pub(crate) struct WarcRecord {
- request: Request,
- response: Response,
- metadata: Metadata,
+ pub(crate) request: Request,
+ pub(crate) response: Response,
+ pub(crate) metadata: Metadata,
}
#[derive(Debug)]
-struct Request {
+pub(crate) struct Request {
// WARC-Target-URI
- url: String,
+ pub(crate) url: String,
}
impl Request {
@@ -153,8 +153,8 @@ impl Request {
}
#[derive(Debug)]
-struct Response {
- body: String,
+pub(crate) struct Response {
+ pub(crate) body: String,
}
impl Response {
@@ -166,9 +166,9 @@ impl Response {
}
#[derive(Debug)]
-struct Metadata {
+pub(crate) struct Metadata {
// fetchTimeMs
- fetch_time_ms: usize,
+ pub(crate) fetch_time_ms: usize,
}
impl Metadata {
@@ -208,14 +208,12 @@ impl<R: BufRead> Iterator for WarcFile<R> {
let mut metadata = None;
for item in items {
- if item.is_none() {
- return None;
- }
- let item = item.unwrap();
+ let item = item?;
if item.is_err() {
return Some(Err(item.err().unwrap()));
}
+
let item = item.unwrap();
if let Some(warc_type) = item.header.get("WARC-TYPE") {
diff --git a/src/webpage.rs b/src/webpage.rs
new file mode 100644
index 00000000..104a3298
--- /dev/null
+++ b/src/webpage.rs
@@ -0,0 +1,116 @@
+use itertools::Itertools;
+use scraper::{Html, Selector};
+use std::collections::BTreeMap;
+
+#[derive(Debug)]
+pub(crate) struct Webpage {
+ dom: Html,
+}
+
+impl Webpage {
+ pub(crate) fn parse(html: &str) -> Self {
+ Self {
+ dom: Html::parse_document(html),
+ }
+ }
+
+ pub(crate) fn links(&self) -> Vec<Link> {
+ let selector = Selector::parse("a").expect("Failed to parse selector");
+ self.dom
+ .select(&selector)
+ .map(|el| {
+ let destination = el.value().attr("href");
+ let text = el.text().collect::<String>();
+
+ (destination, text)
+ })
+ .filter(|(dest, _)| dest.is_some())
+ .map(|(destination, text)| {
+ let destination = destination.unwrap().to_string();
+ Link { destination, text }
+ })
+ .collect()
+ }
+
+ fn grab_texts(&self, selector: &Selector) -> Vec<String> {
+ self.dom
+ .select(selector)
+ .flat_map(|el| el.text())
+ .map(|t| String::from(t.trim()))
+ .filter(|t| !t.is_empty())
+ .collect::<Vec<String>>()
+ }
+
+ pub(crate) fn text(&self) -> String {
+ let selector = Selector::parse("body").expect("Failed to parse selector");
+ Itertools::intersperse(self.grab_texts(&selector).into_iter(), "\n".to_string())
+ .collect::<String>()
+ .trim()
+ .to_string()
+ }
+
+ pub(crate) fn title(&self) -> Option<String> {
+ let selector = Selector::parse("title").expect("Failed to parse selector");
+ self.grab_texts(&selector).get(0).cloned()
+ }
+
+ pub(crate) fn metadata(&self) -> Vec<Meta> {
+ let selector = Selector::parse("meta").expect("Failed to parse selector");
+ self.dom
+ .select(&selector)
+ .map(|el| {
+ el.value()
+ .attrs()
+ .into_iter()
+ .map(|(k, v)| (k.to_string(), v.to_string()))
+ .collect::<BTreeMap<String, String>>()
+ })
+ .collect()
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub(crate) struct Link {
+ destination: String,
+ text: String,
+}
+
+pub(crate) type Meta = BTreeMap<String, String>;
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple() {
+ let raw = r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ </head>
+ <body>
+ <a href="example.com">Best example website ever</a>
+ </body>
+ </html>
+ "#;
+
+ let webpage = Webpage::parse(raw);
+
+ assert_eq!(&webpage.text(), "Best example website ever");
+ assert_eq!(webpage.title(), Some("Best website".to_string()));
+ assert_eq!(
+ webpage.links(),
+ vec![Link {
+ destination: "example.com".to_string(),
+ text: "Best example website ever".to_string()
+ }]
+ );
+
+ let mut expected_meta = BTreeMap::new();
+ expected_meta.insert("name".to_string(), "meta1".to_string());
+ expected_meta.insert("content".to_string(), "value".to_string());
+
+ assert_eq!(webpage.metadata(), vec![expected_meta]);
+ }
+}