Skip to content

web search done right

Remove custom html lexer Kuchiki (the new parser) builds on top of html5ever and should be waaaay more robust.

Mikkel Denkercfeb68ac1c78
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index e03f0f49..68353c7d 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -277,7 +277,7 @@ mod tests {
a.appendChild(r);
})(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
</script>
- <script src="https://thirdparty.com/js" />
+ <script src="https://thirdparty.com/js"></script>
<link href='//securepubads.g.doubleclick.net' rel='preconnect'>
<title>Test site</title>
</head>
diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rs
index 9e479ad1..c9c2b80a 100644
--- a/src/webpage/just_text.rs
+++ b/src/webpage/just_text.rs
@@ -16,10 +16,10 @@
use std::collections::{HashMap, HashSet};
-use logos::{Lexer, Logos};
+use kuchiki::{iter::NodeEdge, NodeRef};
use whatlang::Lang;
-use super::{lexer::Token, Preprocessor};
+use super::Preprocessor;
// implementation of the JustText algorithm described in this thesis: https://is.muni.cz/th/45523/fi_d/phdthesis.pdf
// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py
@@ -194,7 +194,7 @@ struct ClassifiedParagraph {
}
impl JustText {
- fn paragraphs<'a>(&self, mut tokens: Lexer<'a, Token<'a>>, raw: &str) -> Vec<Paragraph> {
+ fn paragraphs(&self, root: NodeRef) -> Vec<Paragraph> {
let mut res = Vec::new();
let mut preprocessor = Preprocessor::new(["script", "style", "embed", "form", "head"]);
@@ -205,116 +205,123 @@ impl JustText {
let mut heading_count = 0;
- while let Some(tok) = tokens.next() {
- preprocessor.update(&tok);
-
+ for edge in root.traverse() {
+ preprocessor.update(&edge);
if preprocessor.is_inside_removed() {
continue;
}
- match tok {
- Token::StartTag(tag) => {
- if matches!(tag.name(), "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
- heading_count += 1;
- }
-
- if matches!(
- tag.name(),
- "body"
- | "blockquote"
- | "caption"
- | "center"
- | "col"
- | "colgroup"
- | "dd"
- | "div"
- | "dl"
- | "dt"
- | "fieldset"
- | "form"
- | "legend"
- | "optgroup"
- | "option"
- | "p"
- | "pre"
- | "table"
- | "td"
- | "textarea"
- | "tfoot"
- | "th"
- | "thead"
- | "tr"
- | "ul"
- | "li"
- | "h1"
- | "h2"
- | "h3"
- | "h4"
- | "h5"
- | "h6"
- ) || (tag.name() == "br" && br)
- {
- if tag.name() == "br" {
- // the <br><br> is a paragraph separator and should
- // not be included in the number of tags within the
- // paragraph
- paragraph.tags_count -= 1;
- }
-
- paragraph.is_heading = heading_count > 0;
+ match edge {
+ NodeEdge::Start(node) => {
+ if let Some(element) = node.as_element() {
+ let name: &str = &element.name.local;
- if paragraph.contains_text() {
- res.push(paragraph);
+ if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
+ heading_count += 1;
}
- paragraph = Paragraph::new();
- } else {
- br = tag.name() == "br";
- if br {
- paragraph.append_text(" ");
- } else if tag.name() == "a" {
- link = true;
- }
- paragraph.tags_count += 1;
- }
- }
- Token::EndTag(tag) => {
- if matches!(tag.name(), "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
- heading_count -= 1;
- }
+ if matches!(
+ name,
+ "body"
+ | "blockquote"
+ | "caption"
+ | "center"
+ | "col"
+ | "colgroup"
+ | "dd"
+ | "div"
+ | "dl"
+ | "dt"
+ | "fieldset"
+ | "form"
+ | "legend"
+ | "optgroup"
+ | "option"
+ | "p"
+ | "pre"
+ | "table"
+ | "td"
+ | "textarea"
+ | "tfoot"
+ | "th"
+ | "thead"
+ | "tr"
+ | "ul"
+ | "li"
+ | "h1"
+ | "h2"
+ | "h3"
+ | "h4"
+ | "h5"
+ | "h6"
+ ) || (name == "br" && br)
+ {
+ if name == "br" {
+ // the <br><br> is a paragraph separator and should
+ // not be included in the number of tags within the
+ // paragraph
+ paragraph.tags_count -= 1;
+ }
- paragraph.append_text(" ");
+ paragraph.is_heading = heading_count > 0;
- match tag.name() {
- "body" | "blockquote" | "caption" | "center" | "col" | "colgroup"
- | "dd" | "div" | "dl" | "dt" | "fieldset" | "form" | "legend"
- | "optgroup" | "option" | "p" | "pre" | "table" | "td" | "textarea"
- | "tfoot" | "th" | "thead" | "tr" | "ul" | "li" | "h1" | "h2" | "h3"
- | "h4" | "h5" | "h6" => {
if paragraph.contains_text() {
res.push(paragraph);
}
paragraph = Paragraph::new();
+ } else {
+ br = name == "br";
+ if br {
+ paragraph.append_text(" ");
+ } else if name == "a" {
+ link = true;
+ }
+ paragraph.tags_count += 1;
}
- _ => {}
}
- if tag.name() == "a" {
- link = false;
+ if let Some(text) = node.as_text() {
+ let raw_text = text.borrow();
+ let text = raw_text.as_str();
+ let chars_added = paragraph.append_text(text);
+
+ if link {
+ paragraph.chars_count_in_links += chars_added;
+ }
+
+ br = false;
}
}
- Token::Error => {
- let span = tokens.span();
- let chars_added = paragraph.append_text(&raw[span]);
+ NodeEdge::End(node) => {
+ if let Some(element) = node.as_element() {
+ let name: &str = &element.name.local;
+ if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {
+ heading_count -= 1;
+ }
- if link {
- paragraph.chars_count_in_links += chars_added;
- }
+ paragraph.append_text(" ");
+
+ match name {
+ "body" | "blockquote" | "caption" | "center" | "col" | "colgroup"
+ | "dd" | "div" | "dl" | "dt" | "fieldset" | "form" | "legend"
+ | "optgroup" | "option" | "p" | "pre" | "table" | "td" | "textarea"
+ | "tfoot" | "th" | "thead" | "tr" | "ul" | "li" | "h1" | "h2"
+ | "h3" | "h4" | "h5" | "h6" => {
+ if paragraph.contains_text() {
+ res.push(paragraph);
+ }
- br = false;
+ paragraph = Paragraph::new();
+ }
+ _ => {}
+ }
+
+ if name == "a" {
+ link = false;
+ }
+ }
}
- Token::SelfTerminatingTag(_) | Token::BeginComment | Token::EndComment => {}
}
}
@@ -578,10 +585,8 @@ impl JustText {
.join(" ")
}
- pub fn extract(&self, html: &str) -> String {
- let lex = Token::lexer(html);
-
- let paragraphs = self.paragraphs(lex, html);
+ pub fn extract(&self, root: NodeRef) -> String {
+ let paragraphs = self.paragraphs(root);
let mut classified = self
.initial_classification(paragraphs)
.into_iter()
diff --git a/src/webpage/lexer.rs b/src/webpage/lexer.rs
deleted file mode 100644
index fc414ec3..00000000
--- a/src/webpage/lexer.rs
+++ /dev/null
@@ -1,384 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use logos::{Lexer, Logos};
-use std::{borrow::Cow, collections::HashMap};
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Tag<'a> {
- name: Cow<'a, str>,
- raw: &'a str,
-}
-
-fn addr_of(s: &str) -> usize {
- s.as_ptr() as usize
-}
-
-fn split_whitespace_indices(s: &str) -> impl Iterator<Item = (usize, &str)> {
- s.split_whitespace()
- .map(move |sub| (addr_of(sub) - addr_of(s), sub))
-}
-
-impl<'a> Tag<'a> {
- pub fn attributes(&self) -> HashMap<&'a str, &'a str> {
- let mut attributes = HashMap::new();
-
- let mut long_attribute: Option<usize> = None;
- let mut is_first_token = true;
- for (start_idx, tok) in split_whitespace_indices(self.raw) {
- let num_quotes = tok.chars().filter(|c| matches!(*c, '"' | '\'')).count();
- if (tok.contains('>') && num_quotes == 0 && is_first_token)
- || tok.contains('<')
- || tok == "/>"
- || tok.starts_with(self.name.as_ref())
- {
- is_first_token = false;
- continue;
- }
- is_first_token = false;
-
- if num_quotes == 0 {
- if long_attribute.is_none() {
- // attribute without value (e.g. autofocus)
- let to_insert = trim_html_stuff(tok, true);
- if !to_insert.is_empty() {
- attributes.insert(to_insert, "");
- }
- }
- } else if num_quotes == 1 {
- if let Some(attribute_start_idx) = long_attribute {
- // closing attribute
- let tok = &self.raw[attribute_start_idx..start_idx + tok.len()];
- if tok.contains('=') {
- if let Some((key, value)) = tok.split_once('=') {
- attributes.insert(key.trim(), trim_html_stuff(value, false));
- }
- }
- long_attribute = None;
- } else {
- // opening attribute
- long_attribute = Some(start_idx);
- }
- } else if num_quotes == 2 {
- // self-containing attribute
- if tok.contains('=') {
- if let Some((key, value)) = tok.split_once('=') {
- attributes.insert(key.trim(), trim_html_stuff(value, false));
- }
- }
- }
- }
-
- attributes
- }
-
- pub fn name(&self) -> &str {
- self.name.as_ref()
- }
-
- pub fn raw(&self) -> &str {
- self.raw
- }
-}
-
-fn trim_html_stuff(s: &str, break_on_dash: bool) -> &str {
- let mut start_val = 0;
- let mut end_val = s.len();
-
- for (idx, c) in s.char_indices() {
- start_val = idx;
- if !matches!(c, ' ' | '"' | '\'') {
- break;
- }
- }
-
- for (idx, c) in s[start_val..].char_indices() {
- end_val = start_val + idx;
- if break_on_dash && c == '/' {
- break;
- }
-
- if matches!(c, '>' | '"' | '\'') {
- break;
- }
- }
-
- &s[start_val..end_val]
-}
-
-fn start_tag<'a>(lex: &mut Lexer<'a, Token<'a>>) -> Option<Tag<'a>> {
- let slice = lex.slice();
- let name = name(slice);
-
- let name = if name.chars().any(|c| c.is_uppercase()) {
- Cow::Owned(name.to_ascii_lowercase())
- } else {
- Cow::Borrowed(name)
- };
-
- Some(Tag { name, raw: slice })
-}
-
-fn end_tag<'a>(lex: &mut Lexer<'a, Token<'a>>) -> Option<Tag<'a>> {
- let slice = lex.slice();
- let name = name(slice);
-
- let name = if name.chars().any(|c| c.is_uppercase()) {
- Cow::Owned(name.to_ascii_lowercase())
- } else {
- Cow::Borrowed(name)
- };
-
- Some(Tag { name, raw: slice })
-}
-
-fn name(s: &str) -> &str {
- let mut start = 0;
-
- for (idx, c) in s.char_indices() {
- start = idx;
- if !matches!(c, '<' | '/' | '\\' | ' ') {
- break;
- }
- }
-
- let mut end = start;
- for (idx, c) in s.char_indices().skip(start) {
- end = idx;
-
- if matches!(c, ' ' | '>' | '/') {
- break;
- }
- }
-
- s[start..end].trim()
-}
-
-#[derive(Clone, Logos, Debug, PartialEq, Eq)]
-pub enum Token<'a> {
- #[regex(r"<[^><]+[^/]?>", start_tag, priority = 0)]
- StartTag(Tag<'a>),
-
- #[regex(r"<?\\?/[a-zA-Z]+>", end_tag, priority = 1)]
- EndTag(Tag<'a>),
-
- // #[regex(r"<[^[/>]]+/>", start_tag)]
- #[regex(r"<[^[/>]]+/>", start_tag)]
- SelfTerminatingTag(Tag<'a>),
-
- #[token("<!--")]
- BeginComment,
-
- #[token("-->")]
- EndComment,
-
- #[error]
- Error,
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use maplit::hashmap;
-
- #[test]
- fn simple_link_tag() {
- let mut lex = Token::lexer("<a href=\"test\">");
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
-
- assert_eq!(tag.name(), "a");
- assert_eq!(tag.attributes(), hashmap!["href" => "test"]);
- assert_eq!(lex.next(), None);
- }
-
- #[test]
- fn tag_without_attributes() {
- let mut lex = Token::lexer("<a>");
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "a");
- assert_eq!(tag.attributes(), hashmap![]);
- assert_eq!(lex.next(), None);
-
- let mut lex = Token::lexer("< a>");
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "a");
- assert_eq!(tag.attributes(), hashmap![]);
- assert_eq!(lex.next(), None);
-
- let mut lex = Token::lexer("< a >");
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "a");
- assert_eq!(tag.attributes(), hashmap![]);
- assert_eq!(lex.next(), None);
-
- let mut lex = Token::lexer("<a >");
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "a");
- assert_eq!(tag.attributes(), hashmap![]);
- assert_eq!(lex.next(), None);
- }
-
- #[test]
- fn key_without_value() {
- let mut lex = Token::lexer("<input autofocus>");
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "input");
- assert_eq!(tag.attributes(), hashmap!["autofocus" => ""]);
- assert_eq!(lex.next(), None);
-
- let mut lex = Token::lexer("<input autofocus/>");
-
- let tag = match lex.next().unwrap() {
- Token::SelfTerminatingTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "input");
- assert_eq!(tag.attributes(), hashmap!["autofocus" => ""]);
- assert_eq!(lex.next(), None);
- }
-
- #[test]
- fn self_terminating_tag() {
- let mut lex = Token::lexer("<a href=\"test\" />");
-
- let tag = match lex.next().unwrap() {
- Token::SelfTerminatingTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "a");
- assert_eq!(tag.attributes(), hashmap!["href" => "test"]);
- assert_eq!(lex.next(), None);
- }
-
- #[test]
- fn end_tag() {
- let raw = r#"<a href="test">this is some text</a>"#;
- let mut lex = Token::lexer(raw);
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => {
- panic!("{:?}", whut)
- }
- };
- assert_eq!(tag.name(), "a");
- assert_eq!(tag.attributes(), hashmap!["href" => "test"]);
-
- let mut tok = lex.next();
-
- while let Some(Token::Error) = tok {
- tok = lex.next();
- }
-
- let tag = match tok.unwrap() {
- Token::EndTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
- assert_eq!(tag.name(), "a");
- assert_eq!(lex.next(), None);
- }
-
- #[test]
- fn test_split_whitespace_indices() {
- let mut iter = split_whitespace_indices(" Hello world");
-
- assert_eq!(Some((1, "Hello")), iter.next());
- assert_eq!(Some((7, "world")), iter.next());
- }
-
- #[test]
- fn attribute_containing_spaces() {
- let mut lex = Token::lexer("<meta description=\"this is a long attribute\">");
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- _ => panic!(),
- };
-
- assert_eq!(
- tag.attributes(),
- hashmap!["description" => "this is a long attribute"]
- );
- assert_eq!(lex.next(), None);
- }
-
- #[test]
- fn script_with_lt() {
- let raw = "<script>if d < a</script>";
- let mut lex = Token::lexer(raw);
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
-
- assert_eq!(tag.name(), "script");
-
- let mut text = String::new();
-
- let mut tok = lex.next();
- while let Some(Token::Error) = tok {
- text.push_str(&raw[lex.span()]);
- tok = lex.next();
- }
-
- assert_eq!(text.as_str(), "if d < a");
-
- assert!(matches!(tok.unwrap(), Token::EndTag(_)));
- assert_eq!(lex.next(), None);
- }
-
- #[test]
- fn link_attributes() {
- let raw = "<link href='//securepubads.g.doubleclick.net' rel='preconnect'>";
-
- let mut lex = Token::lexer(raw);
-
- let tag = match lex.next().unwrap() {
- Token::StartTag(tag) => tag,
- whut => panic!("{:?}", whut),
- };
-
- assert_eq!(
- tag.attributes().get("href"),
- Some(&"//securepubads.g.doubleclick.net")
- );
- }
-}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index ee2400e4..03cd0d4f 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -16,20 +16,19 @@
use crate::{schema_org::SchemaOrg, Error, Result};
use chrono::{DateTime, FixedOffset};
use itertools::Itertools;
-use logos::{Lexer, Logos};
+use kuchiki::{iter::NodeEdge, traits::*, NodeRef};
use regex::Regex;
-use std::collections::HashMap;
+use std::{borrow::Borrow, collections::HashMap};
use uuid::Uuid;
mod just_text;
-mod lexer;
pub mod region;
mod url;
use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};
pub use self::url::Url;
-use self::{just_text::JustText, lexer::Token, region::Region};
+use self::{just_text::JustText, region::Region};
static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
Regex::new(r#"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b"#).unwrap()
@@ -46,7 +45,6 @@ pub struct FaviconLink {
pub struct Preprocessor<const N: usize> {
removed_tags: [&'static str; N],
num_open_tags: [i64; N],
- open_comments: i64,
}
impl<const N: usize> Preprocessor<N> {
@@ -54,55 +52,57 @@ impl<const N: usize> Preprocessor<N> {
Self {
removed_tags,
num_open_tags: [0; N],
- open_comments: 0,
}
}
- pub fn update(&mut self, tok: &Token) {
- match tok {
- Token::StartTag(tag) if !tag.raw().contains("/>") => {
- if let Some((_, n)) = self
- .removed_tags
- .iter()
- .zip(self.num_open_tags.iter_mut())
- .find(|(name, _)| **name == tag.name())
- {
- *n += 1;
+ pub fn update(&mut self, edge: &NodeEdge<NodeRef>) {
+ match edge {
+ NodeEdge::Start(node) => {
+ if let Some(element) = node.as_element() {
+ let element_name: &str = element.name.local.borrow();
+ if let Some((_, n)) = self
+ .removed_tags
+ .iter()
+ .zip(self.num_open_tags.iter_mut())
+ .find(|(name, _)| **name == element_name)
+ {
+ *n += 1;
+ }
}
}
- Token::EndTag(tag) => {
- if let Some((_, n)) = self
- .removed_tags
- .iter()
- .zip(self.num_open_tags.iter_mut())
- .find(|(name, _)| **name == tag.name())
- {
- *n -= 1;
+ NodeEdge::End(node) => {
+ if let Some(element) = node.as_element() {
+ let element_name: &str = element.name.local.borrow();
+ if let Some((_, n)) = self
+ .removed_tags
+ .iter()
+ .zip(self.num_open_tags.iter_mut())
+ .find(|(name, _)| **name == element_name)
+ {
+ *n -= 1;
+ }
}
}
- Token::BeginComment => self.open_comments += 1,
- Token::EndComment => self.open_comments -= 1,
- _ => {}
}
}
pub fn is_inside_removed(&self) -> bool {
- self.num_open_tags.iter().any(|n| *n > 0) || self.open_comments > 0
+ self.num_open_tags.iter().any(|n| *n > 0)
}
}
-pub struct Webpage<'a> {
- pub html: Html<'a>,
+pub struct Webpage {
+ pub html: Html,
pub backlinks: Vec<Link>,
pub centrality: f64,
pub fetch_time_ms: u64,
pub primary_image_uuid: Option<Uuid>,
}
-impl<'a> Webpage<'a> {
+impl Webpage {
#[cfg(test)]
pub fn new(
- html: &'a str,
+ html: &str,
url: &str,
backlinks: Vec<Link>,
centrality: f64,
@@ -186,71 +186,73 @@ impl<'a> Webpage<'a> {
}
}
+struct Script {
+ attributes: HashMap<String, String>,
+ content: String,
+}
+
#[derive(Debug)]
-pub struct Html<'a> {
- raw: &'a str,
- tokens: Lexer<'a, Token<'a>>,
+pub struct Html {
url: Url,
+ root: NodeRef, // this is reference counted (cheap to clone)
}
-impl<'a> Html<'a> {
- pub fn parse(html: &'a str, url: &str) -> Self {
- let tokens = Token::lexer(html);
+impl Html {
+ pub fn parse(html: &str, url: &str) -> Self {
+ let root = kuchiki::parse_html().one(html);
Self {
- raw: html,
- tokens,
+ root,
url: url.to_string().into(),
}
}
pub fn links(&self) -> Vec<Link> {
- let mut tokens = self.tokens.clone();
let mut links = Vec::new();
let mut open_links = Vec::new();
let mut preprocessor = Preprocessor::new(["script", "style", "head"]);
- while let Some(tok) = tokens.next() {
- preprocessor.update(&tok);
+ for edge in self.root.traverse() {
+ preprocessor.update(&edge);
if preprocessor.is_inside_removed() {
continue;
}
- match tok {
- Token::StartTag(tag) if tag.name() == "a" => {
- open_links.push((String::new(), tag.attributes()));
- }
- Token::EndTag(tag) if tag.name() == "a" => {
- if let Some((text, attributes)) = open_links.pop() {
- if let Some(dest) = attributes.get("href") {
- links.push(Link {
- source: self.url.clone(),
- destination: dest.to_string().into(),
- text,
- })
+ match edge {
+ NodeEdge::Start(node) => {
+ if let Some(element) = node.as_element() {
+ if &element.name.local == "a" {
+ open_links.push((String::new(), element.attributes.clone()));
}
}
}
- Token::Error => {
- for (text, _) in &mut open_links {
- let span = tokens.span();
- text.push_str(&self.raw[span]);
+ NodeEdge::End(node) => {
+ if let Some(element) = node.as_element() {
+ if &element.name.local == "a" {
+ if let Some((text, attributes)) = open_links.pop() {
+ if let Some(dest) = attributes.borrow().get("href") {
+ links.push(Link {
+ source: self.url.clone(),
+ destination: dest.to_string().into(),
+ text: text.trim().to_string(),
+ })
+ }
+ }
+ }
}
- }
- Token::SelfTerminatingTag(tag) if tag.name() == "a" => {
- if let Some(dest) = tag.attributes().get("href") {
- links.push(Link {
- source: self.url.clone(),
- destination: dest.to_string().into(),
- text: String::new(),
- })
+
+ if let Some(text) = node.as_text() {
+ let raw_text = text.borrow();
+ let text = raw_text.trim();
+
+ if !text.is_empty() {
+ for (link_text, _) in &mut open_links {
+ link_text.push('\n');
+ link_text.push_str(text);
+ }
+ }
}
}
- Token::StartTag(_)
- | Token::EndTag(_)
- | Token::SelfTerminatingTag(_)
- | Token::BeginComment
- | Token::EndComment => {}
}
}
@@ -258,59 +260,38 @@ impl<'a> Html<'a> {
}
pub fn favicon(&self) -> Option<FaviconLink> {
- let mut preprocessor = Preprocessor::new(["script", "style", "body"]);
-
- let tokens = self.tokens.clone();
- for tok in tokens {
- preprocessor.update(&tok);
- if preprocessor.is_inside_removed() {
+ for node in self.root.select("link").unwrap() {
+ if !matches!(node.attributes.borrow().get("rel"), Some("icon")) {
continue;
}
- match tok {
- Token::StartTag(tag) | Token::SelfTerminatingTag(tag) if tag.name() == "link" => {
- let rel = tag.attributes().get("rel").cloned();
- if rel.is_none() {
- continue;
- }
- if rel.unwrap() != "icon" {
- continue;
+ if let Some(link) = node.attributes.borrow().get("href") {
+ let (width, height) = match node.attributes.borrow().get("sizes") {
+ Some(size) => {
+ if let Some((width, height)) = size.split_once('x') {
+ (width.parse().ok(), height.parse().ok())
+ } else {
+ (None, None)
+ }
}
+ _ => (None, None),
+ };
- if let Some(link) = tag.attributes().get("href") {
- let (width, height) = match tag.attributes().get("sizes") {
- Some(size) => {
- if let Some((width, height)) = size.split_once('x') {
- (width.parse().ok(), height.parse().ok())
- } else {
- (None, None)
- }
- }
- _ => (None, None),
- };
+ let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());
+ let mut link: Url = link.to_string().into();
- let image_type = tag.attributes().get("type").map(|t| t.to_string());
- let mut link: Url = link.to_string().into();
+ if !link.is_full_path() {
+ link.prefix_with(&self.url);
+ }
- if !link.is_full_path() {
- link.prefix_with(&self.url);
- }
+ let favicon = FaviconLink {
+ link,
+ image_type,
+ width,
+ height,
+ };
- let favicon = FaviconLink {
- link,
- image_type,
- width,
- height,
- };
- return Some(favicon);
- }
- }
- Token::StartTag(_)
- | Token::EndTag(_)
- | Token::SelfTerminatingTag(_)
- | Token::BeginComment
- | Token::Error
- | Token::EndComment => {}
+ return Some(favicon);
}
}
@@ -318,7 +299,7 @@ impl<'a> Html<'a> {
}
pub fn clean_text(&self) -> Option<String> {
- let text = JustText::default().extract(self.raw);
+ let text = JustText::default().extract(self.root.clone());
if text.is_empty() {
None
@@ -336,7 +317,7 @@ impl<'a> Html<'a> {
stopwords_high: -1.0,
max_heading_distance: 10000,
}
- .extract(self.raw);
+ .extract(self.root.clone());
if text.is_empty() {
None
@@ -346,47 +327,11 @@ impl<'a> Html<'a> {
}
pub fn title(&self) -> Option<String> {
- let mut tokens = self.tokens.clone();
- let mut title = None;
- let mut open_tags = 0;
- let mut preprocessor = Preprocessor::new(["script", "style"]);
-
- while let Some(tok) = tokens.next() {
- preprocessor.update(&tok);
- if preprocessor.is_inside_removed() {
- continue;
- }
-
- match tok {
- Token::StartTag(tag) if tag.name() == "title" => {
- open_tags += 1;
- }
- Token::EndTag(tag) if tag.name() == "title" => {
- open_tags -= 1;
-
- if open_tags == 0 {
- break;
- }
- }
- Token::Error => {
- if open_tags > 0 {
- let span = tokens.span();
- if let Some(cur_title) = title {
- title = Some(cur_title + &self.raw[span]);
- } else {
- title = Some(self.raw[span].to_string());
- }
- }
- }
- Token::SelfTerminatingTag(_)
- | Token::StartTag(_)
- | Token::EndTag(_)
- | Token::BeginComment
- | Token::EndComment => {}
- }
+ if let Ok(title) = self.root.select_first("title") {
+ Some(title.text_contents().trim().to_string())
+ } else {
+ None
}
-
- title
}
pub fn url(&self) -> &Url {
@@ -406,39 +351,24 @@ impl<'a> Html<'a> {
}
pub fn metadata(&self) -> Vec<Meta> {
- let tokens = self.tokens.clone();
let mut metas = Vec::new();
- let mut preprocessor = Preprocessor::new(["script", "style"]);
- for tok in tokens {
- preprocessor.update(&tok);
- if preprocessor.is_inside_removed() {
- continue;
- }
-
- match tok {
- Token::StartTag(tag) if tag.name() == "meta" => {
- metas.push(
- tag.attributes()
- .into_iter()
- .map(|(key, value)| (key.to_string(), value.to_string()))
- .collect(),
- );
- }
- Token::SelfTerminatingTag(tag) if tag.name() == "meta" => {
- metas.push(
- tag.attributes()
- .into_iter()
- .map(|(key, value)| (key.to_string(), value.to_string()))
- .collect(),
- );
- }
- Token::StartTag(_)
- | Token::EndTag(_)
- | Token::SelfTerminatingTag(_)
- | Token::Error
- | Token::BeginComment
- | Token::EndComment => {}
+ for node in self.root.select("meta").unwrap() {
+ if let Some(element) = node.as_node().as_element() {
+ metas.push(
+ element
+ .attributes
+ .borrow()
+ .map
+ .iter()
+ .map(|(name, attr)| {
+ (
+ name.local.borrow().to_string(),
+ attr.borrow().value.to_string(),
+ )
+ })
+ .collect(),
+ );
}
}
@@ -511,50 +441,75 @@ impl<'a> Html<'a> {
Ok(doc)
}
+ fn scripts(&self) -> Vec<Script> {
+ let mut scripts = Vec::new();
+
+ for node in self.root.select("script").unwrap() {
+ let content = node.text_contents().trim().to_string();
+ let attributes = node
+ .attributes
+ .borrow()
+ .map
+ .iter()
+ .map(|(name, attr)| {
+ (
+ name.local.borrow().to_string(),
+ attr.borrow().value.to_string(),
+ )
+ })
+ .collect();
+
+ scripts.push(Script {
+ attributes,
+ content,
+ })
+ }
+
+ scripts
+ }
+
pub fn schema_org(&self) -> Vec<SchemaOrg> {
- let mut tokens = self.tokens.clone();
let mut schemas = Vec::new();
- let mut preprocessor = Preprocessor::new([]);
- let mut open_schemas = 0;
- let mut json = String::new();
- while let Some(tok) = tokens.next() {
- preprocessor.update(&tok);
- if preprocessor.is_inside_removed() {
- continue;
+ for schema in self.scripts().into_iter().filter(|script| {
+ matches!(
+ script.attributes.get("type").map(|s| s.as_str()),
+ Some("application/ld+json")
+ )
+ }) {
+ if let Ok(schema) = serde_json::from_str(&schema.content) {
+ schemas.push(schema);
}
+ }
- match tok {
- Token::StartTag(tag) if tag.name() == "script" => {
- if let Some(&"application/ld+json") = tag.attributes().get("type") {
- open_schemas += 1;
- }
- }
- Token::EndTag(tag) if tag.name() == "script" => {
- if open_schemas > 0 {
- open_schemas -= 1;
+ schemas
+ }
- if open_schemas == 0 {
- if let Ok(schema) = serde_json::from_str(&json) {
- schemas.push(schema);
- }
- json = String::new();
- }
- }
- }
- Token::Error => {
- let span = tokens.span();
- json.push_str(&self.raw[span]);
- }
- Token::StartTag(_)
- | Token::EndTag(_)
- | Token::SelfTerminatingTag(_)
- | Token::BeginComment
- | Token::EndComment => {}
+ pub fn trackers(&self) -> Vec<Url> {
+ let mut links: Vec<Url> = Vec::new();
+
+ for script in self.scripts() {
+ if let Some(link) = script.attributes.get("src") {
+ links.push(link.to_string().into());
+ }
+
+ for res in URL_REGEX.find_iter(&script.content) {
+ links.push(res.as_str().to_string().into());
}
}
- schemas
+ for node in self.root.select("link").unwrap() {
+ if let Some(link) = node.attributes.borrow().get("href") {
+ links.push(link.to_string().into())
+ }
+ }
+
+ links
+ .into_iter()
+ .filter(|link| !link.host().is_empty())
+ .filter(|link| link.host() != self.host())
+ .unique_by(|link| link.host().to_string())
+ .collect()
}
fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {
@@ -646,77 +601,6 @@ impl<'a> Html<'a> {
})
.and_then(|metadata| metadata.get("content").cloned())
}
-
- pub fn trackers(&self) -> Vec<Url> {
- let mut tokens = self.tokens.clone();
-
- let mut links: Vec<Url> = Vec::new();
- let mut open_scripts = 0;
- let mut script_text = String::new();
- let mut preprocessor = Preprocessor::new([]);
-
- while let Some(tok) = tokens.next() {
- preprocessor.update(&tok);
- if preprocessor.is_inside_removed() {
- continue;
- }
-
- match tok {
- Token::StartTag(tag) if tag.name() == "script" => {
- if !tag.raw().contains("/>") {
- open_scripts += 1;
- }
- if let Some(link) = tag.attributes().get("src") {
- links.push(link.to_string().into());
- }
- }
- Token::EndTag(tag) if tag.name() == "script" => {
- if open_scripts > 0 {
- open_scripts -= 1;
-
- if open_scripts == 0 {
- for res in URL_REGEX.find_iter(script_text.as_str()) {
- links.push(res.as_str().to_string().into());
- }
- script_text.clear();
- }
- }
- }
- _ if open_scripts > 0 => {
- let span = tokens.span();
- script_text.push_str(&self.raw[span]);
- }
- Token::Error => {
- if open_scripts > 0 {
- let span = tokens.span();
- script_text.push_str(&self.raw[span]);
- }
- }
- Token::SelfTerminatingTag(tag) if tag.name() == "script" => {
- if let Some(link) = tag.attributes().get("src") {
- links.push(link.to_string().into());
- }
- }
- Token::StartTag(tag) | Token::SelfTerminatingTag(tag) if tag.name() == "link" => {
- if let Some(link) = tag.attributes().get("href") {
- links.push(link.to_string().into());
- }
- }
- Token::StartTag(_)
- | Token::EndTag(_)
- | Token::SelfTerminatingTag(_)
- | Token::BeginComment
- | Token::EndComment => {}
- }
- }
-
- links
- .into_iter()
- .filter(|link| !link.host().is_empty())
- .filter(|link| link.host() != self.host())
- .unique_by(|link| link.host().to_string())
- .collect()
- }
}
#[derive(Debug, PartialEq, Eq)]
@@ -1273,12 +1157,11 @@ mod tests {
a.appendChild(r);
})(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
</script>
- <script src="https://thirdparty.com/js" />
- <script src="https://example.com/js" />
+ <script src="https://thirdparty.com/js"></script>
+ <script src="https://example.com/js"></script>
<link href='//securepubads.g.doubleclick.net' rel='preconnect'>
- <script src="https://thirdparty.com/js" />
- <script src="/js/file" />
- <meta property="article:modified_time" content="2022-06-22T19:37:34+00:00" />
+ <script src="https://thirdparty.com/js"></script>
+ <script src="/js/file"></script>
</head>
<body>
</body>
@@ -1318,7 +1201,7 @@ mod tests {
a.appendChild(r);
})(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
</script>
- <script src="https://thirdparty.com/js" />
+ <script src="https://thirdparty.com/js"></script>
<link href='//securepubads.g.doubleclick.net' rel='preconnect'>
<title>Test site</title>
</head>