web search done right
Remove custom html lexer Kuchiki (the new parser) builds on top of html5ever and should be waaaay more robust.
Mikkel Denkercfeb68ac1c78
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex e03f0f49..68353c7d 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -277,7 +277,7 @@ mod tests {a.appendChild(r);})(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');</script>- <script src="https://thirdparty.com/js" />+ <script src="https://thirdparty.com/js"></script><link href='//securepubads.g.doubleclick.net' rel='preconnect'><title>Test site</title></head>diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rsindex 9e479ad1..c9c2b80a 100644--- a/src/webpage/just_text.rs+++ b/src/webpage/just_text.rs@@ -16,10 +16,10 @@use std::collections::{HashMap, HashSet};-use logos::{Lexer, Logos};+use kuchiki::{iter::NodeEdge, NodeRef};use whatlang::Lang;-use super::{lexer::Token, Preprocessor};+use super::Preprocessor;// implementation of the JustText algorithm described in this thesis: https://is.muni.cz/th/45523/fi_d/phdthesis.pdf// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py@@ -194,7 +194,7 @@ struct ClassifiedParagraph {}impl JustText {- fn paragraphs<'a>(&self, mut tokens: Lexer<'a, Token<'a>>, raw: &str) -> Vec<Paragraph> {+ fn paragraphs(&self, root: NodeRef) -> Vec<Paragraph> {let mut res = Vec::new();let mut preprocessor = Preprocessor::new(["script", "style", "embed", "form", "head"]);@@ -205,116 +205,123 @@ impl JustText {let mut heading_count = 0;- while let Some(tok) = tokens.next() {- preprocessor.update(&tok);-+ for edge in root.traverse() {+ preprocessor.update(&edge);if preprocessor.is_inside_removed() {continue;}- match tok {- Token::StartTag(tag) => {- if matches!(tag.name(), "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {- heading_count += 1;- }-- if matches!(- tag.name(),- "body"- | "blockquote"- | "caption"- | "center"- | "col"- | "colgroup"- | "dd"- | "div"- | "dl"- | "dt"- | "fieldset"- | "form"- | "legend"- | "optgroup"- | "option"- | "p"- | "pre"- | "table"- | "td"- | "textarea"- | "tfoot"- | "th"- | "thead"- | "tr"- | "ul"- | "li"- | "h1"- | "h2"- | "h3"- | "h4"- | "h5"- | "h6"- ) || (tag.name() == "br" && br)- {- if tag.name() == "br" {- // the <br><br> is a paragraph separator and should- // not be included in the number of tags within the- // paragraph- paragraph.tags_count -= 1;- }-- paragraph.is_heading = heading_count > 0;+ match edge {+ NodeEdge::Start(node) => {+ if let Some(element) = node.as_element() {+ let name: &str = &element.name.local;- if paragraph.contains_text() {- res.push(paragraph);+ if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {+ heading_count += 1;}- paragraph = Paragraph::new();- } else {- br = tag.name() == "br";- if br {- paragraph.append_text(" ");- } else if tag.name() == "a" {- link = true;- }- paragraph.tags_count += 1;- }- }- Token::EndTag(tag) => {- if matches!(tag.name(), "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {- heading_count -= 1;- }+ if matches!(+ name,+ "body"+ | "blockquote"+ | "caption"+ | "center"+ | "col"+ | "colgroup"+ | "dd"+ | "div"+ | "dl"+ | "dt"+ | "fieldset"+ | "form"+ | "legend"+ | "optgroup"+ | "option"+ | "p"+ | "pre"+ | "table"+ | "td"+ | "textarea"+ | "tfoot"+ | "th"+ | "thead"+ | "tr"+ | "ul"+ | "li"+ | "h1"+ | "h2"+ | "h3"+ | "h4"+ | "h5"+ | "h6"+ ) || (name == "br" && br)+ {+ if name == "br" {+ // the <br><br> is a paragraph separator and should+ // not be included in the number of tags within the+ // paragraph+ paragraph.tags_count -= 1;+ }- paragraph.append_text(" ");+ paragraph.is_heading = heading_count > 0;- match tag.name() {- "body" | "blockquote" | "caption" | "center" | "col" | "colgroup"- | "dd" | "div" | "dl" | "dt" | "fieldset" | "form" | "legend"- | "optgroup" | "option" | "p" | "pre" | "table" | "td" | "textarea"- | "tfoot" | "th" | "thead" | "tr" | "ul" | "li" | "h1" | "h2" | "h3"- | "h4" | "h5" | "h6" => {if paragraph.contains_text() {res.push(paragraph);}paragraph = Paragraph::new();+ } else {+ br = name == "br";+ if br {+ paragraph.append_text(" ");+ } else if name == "a" {+ link = true;+ }+ paragraph.tags_count += 1;}- _ => {}}- if tag.name() == "a" {- link = false;+ if let Some(text) = node.as_text() {+ let raw_text = text.borrow();+ let text = raw_text.as_str();+ let chars_added = paragraph.append_text(text);++ if link {+ paragraph.chars_count_in_links += chars_added;+ }++ br = false;}}- Token::Error => {- let span = tokens.span();- let chars_added = paragraph.append_text(&raw[span]);+ NodeEdge::End(node) => {+ if let Some(element) = node.as_element() {+ let name: &str = &element.name.local;+ if matches!(name, "h1" | "h2" | "h3" | "h4" | "h5" | "h6") {+ heading_count -= 1;+ }- if link {- paragraph.chars_count_in_links += chars_added;- }+ paragraph.append_text(" ");++ match name {+ "body" | "blockquote" | "caption" | "center" | "col" | "colgroup"+ | "dd" | "div" | "dl" | "dt" | "fieldset" | "form" | "legend"+ | "optgroup" | "option" | "p" | "pre" | "table" | "td" | "textarea"+ | "tfoot" | "th" | "thead" | "tr" | "ul" | "li" | "h1" | "h2"+ | "h3" | "h4" | "h5" | "h6" => {+ if paragraph.contains_text() {+ res.push(paragraph);+ }- br = false;+ paragraph = Paragraph::new();+ }+ _ => {}+ }++ if name == "a" {+ link = false;+ }+ }}- Token::SelfTerminatingTag(_) | Token::BeginComment | Token::EndComment => {}}}@@ -578,10 +585,8 @@ impl JustText {.join(" ")}- pub fn extract(&self, html: &str) -> String {- let lex = Token::lexer(html);-- let paragraphs = self.paragraphs(lex, html);+ pub fn extract(&self, root: NodeRef) -> String {+ let paragraphs = self.paragraphs(root);let mut classified = self.initial_classification(paragraphs).into_iter()diff --git a/src/webpage/lexer.rs b/src/webpage/lexer.rsdeleted file mode 100644index fc414ec3..00000000--- a/src/webpage/lexer.rs+++ /dev/null@@ -1,384 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use logos::{Lexer, Logos};-use std::{borrow::Cow, collections::HashMap};--#[derive(Debug, Clone, PartialEq, Eq)]-pub struct Tag<'a> {- name: Cow<'a, str>,- raw: &'a str,-}--fn addr_of(s: &str) -> usize {- s.as_ptr() as usize-}--fn split_whitespace_indices(s: &str) -> impl Iterator<Item = (usize, &str)> {- s.split_whitespace()- .map(move |sub| (addr_of(sub) - addr_of(s), sub))-}--impl<'a> Tag<'a> {- pub fn attributes(&self) -> HashMap<&'a str, &'a str> {- let mut attributes = HashMap::new();-- let mut long_attribute: Option<usize> = None;- let mut is_first_token = true;- for (start_idx, tok) in split_whitespace_indices(self.raw) {- let num_quotes = tok.chars().filter(|c| matches!(*c, '"' | '\'')).count();- if (tok.contains('>') && num_quotes == 0 && is_first_token)- || tok.contains('<')- || tok == "/>"- || tok.starts_with(self.name.as_ref())- {- is_first_token = false;- continue;- }- is_first_token = false;-- if num_quotes == 0 {- if long_attribute.is_none() {- // attribute without value (e.g. autofocus)- let to_insert = trim_html_stuff(tok, true);- if !to_insert.is_empty() {- attributes.insert(to_insert, "");- }- }- } else if num_quotes == 1 {- if let Some(attribute_start_idx) = long_attribute {- // closing attribute- let tok = &self.raw[attribute_start_idx..start_idx + tok.len()];- if tok.contains('=') {- if let Some((key, value)) = tok.split_once('=') {- attributes.insert(key.trim(), trim_html_stuff(value, false));- }- }- long_attribute = None;- } else {- // opening attribute- long_attribute = Some(start_idx);- }- } else if num_quotes == 2 {- // self-containing attribute- if tok.contains('=') {- if let Some((key, value)) = tok.split_once('=') {- attributes.insert(key.trim(), trim_html_stuff(value, false));- }- }- }- }-- attributes- }-- pub fn name(&self) -> &str {- self.name.as_ref()- }-- pub fn raw(&self) -> &str {- self.raw- }-}--fn trim_html_stuff(s: &str, break_on_dash: bool) -> &str {- let mut start_val = 0;- let mut end_val = s.len();-- for (idx, c) in s.char_indices() {- start_val = idx;- if !matches!(c, ' ' | '"' | '\'') {- break;- }- }-- for (idx, c) in s[start_val..].char_indices() {- end_val = start_val + idx;- if break_on_dash && c == '/' {- break;- }-- if matches!(c, '>' | '"' | '\'') {- break;- }- }-- &s[start_val..end_val]-}--fn start_tag<'a>(lex: &mut Lexer<'a, Token<'a>>) -> Option<Tag<'a>> {- let slice = lex.slice();- let name = name(slice);-- let name = if name.chars().any(|c| c.is_uppercase()) {- Cow::Owned(name.to_ascii_lowercase())- } else {- Cow::Borrowed(name)- };-- Some(Tag { name, raw: slice })-}--fn end_tag<'a>(lex: &mut Lexer<'a, Token<'a>>) -> Option<Tag<'a>> {- let slice = lex.slice();- let name = name(slice);-- let name = if name.chars().any(|c| c.is_uppercase()) {- Cow::Owned(name.to_ascii_lowercase())- } else {- Cow::Borrowed(name)- };-- Some(Tag { name, raw: slice })-}--fn name(s: &str) -> &str {- let mut start = 0;-- for (idx, c) in s.char_indices() {- start = idx;- if !matches!(c, '<' | '/' | '\\' | ' ') {- break;- }- }-- let mut end = start;- for (idx, c) in s.char_indices().skip(start) {- end = idx;-- if matches!(c, ' ' | '>' | '/') {- break;- }- }-- s[start..end].trim()-}--#[derive(Clone, Logos, Debug, PartialEq, Eq)]-pub enum Token<'a> {- #[regex(r"<[^><]+[^/]?>", start_tag, priority = 0)]- StartTag(Tag<'a>),-- #[regex(r"<?\\?/[a-zA-Z]+>", end_tag, priority = 1)]- EndTag(Tag<'a>),-- // #[regex(r"<[^[/>]]+/>", start_tag)]- #[regex(r"<[^[/>]]+/>", start_tag)]- SelfTerminatingTag(Tag<'a>),-- #[token("<!--")]- BeginComment,-- #[token("-->")]- EndComment,-- #[error]- Error,-}--#[cfg(test)]-mod tests {- use super::*;- use maplit::hashmap;-- #[test]- fn simple_link_tag() {- let mut lex = Token::lexer("<a href=\"test\">");-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };-- assert_eq!(tag.name(), "a");- assert_eq!(tag.attributes(), hashmap!["href" => "test"]);- assert_eq!(lex.next(), None);- }-- #[test]- fn tag_without_attributes() {- let mut lex = Token::lexer("<a>");-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "a");- assert_eq!(tag.attributes(), hashmap![]);- assert_eq!(lex.next(), None);-- let mut lex = Token::lexer("< a>");-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "a");- assert_eq!(tag.attributes(), hashmap![]);- assert_eq!(lex.next(), None);-- let mut lex = Token::lexer("< a >");-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "a");- assert_eq!(tag.attributes(), hashmap![]);- assert_eq!(lex.next(), None);-- let mut lex = Token::lexer("<a >");-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "a");- assert_eq!(tag.attributes(), hashmap![]);- assert_eq!(lex.next(), None);- }-- #[test]- fn key_without_value() {- let mut lex = Token::lexer("<input autofocus>");-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "input");- assert_eq!(tag.attributes(), hashmap!["autofocus" => ""]);- assert_eq!(lex.next(), None);-- let mut lex = Token::lexer("<input autofocus/>");-- let tag = match lex.next().unwrap() {- Token::SelfTerminatingTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "input");- assert_eq!(tag.attributes(), hashmap!["autofocus" => ""]);- assert_eq!(lex.next(), None);- }-- #[test]- fn self_terminating_tag() {- let mut lex = Token::lexer("<a href=\"test\" />");-- let tag = match lex.next().unwrap() {- Token::SelfTerminatingTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "a");- assert_eq!(tag.attributes(), hashmap!["href" => "test"]);- assert_eq!(lex.next(), None);- }-- #[test]- fn end_tag() {- let raw = r#"<a href="test">this is some text</a>"#;- let mut lex = Token::lexer(raw);-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => {- panic!("{:?}", whut)- }- };- assert_eq!(tag.name(), "a");- assert_eq!(tag.attributes(), hashmap!["href" => "test"]);-- let mut tok = lex.next();-- while let Some(Token::Error) = tok {- tok = lex.next();- }-- let tag = match tok.unwrap() {- Token::EndTag(tag) => tag,- whut => panic!("{:?}", whut),- };- assert_eq!(tag.name(), "a");- assert_eq!(lex.next(), None);- }-- #[test]- fn test_split_whitespace_indices() {- let mut iter = split_whitespace_indices(" Hello world");-- assert_eq!(Some((1, "Hello")), iter.next());- assert_eq!(Some((7, "world")), iter.next());- }-- #[test]- fn attribute_containing_spaces() {- let mut lex = Token::lexer("<meta description=\"this is a long attribute\">");-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- _ => panic!(),- };-- assert_eq!(- tag.attributes(),- hashmap!["description" => "this is a long attribute"]- );- assert_eq!(lex.next(), None);- }-- #[test]- fn script_with_lt() {- let raw = "<script>if d < a</script>";- let mut lex = Token::lexer(raw);-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };-- assert_eq!(tag.name(), "script");-- let mut text = String::new();-- let mut tok = lex.next();- while let Some(Token::Error) = tok {- text.push_str(&raw[lex.span()]);- tok = lex.next();- }-- assert_eq!(text.as_str(), "if d < a");-- assert!(matches!(tok.unwrap(), Token::EndTag(_)));- assert_eq!(lex.next(), None);- }-- #[test]- fn link_attributes() {- let raw = "<link href='//securepubads.g.doubleclick.net' rel='preconnect'>";-- let mut lex = Token::lexer(raw);-- let tag = match lex.next().unwrap() {- Token::StartTag(tag) => tag,- whut => panic!("{:?}", whut),- };-- assert_eq!(- tag.attributes().get("href"),- Some(&"//securepubads.g.doubleclick.net")- );- }-}diff --git a/src/webpage/mod.rs b/src/webpage/mod.rsindex ee2400e4..03cd0d4f 100644--- a/src/webpage/mod.rs+++ b/src/webpage/mod.rs@@ -16,20 +16,19 @@use crate::{schema_org::SchemaOrg, Error, Result};use chrono::{DateTime, FixedOffset};use itertools::Itertools;-use logos::{Lexer, Logos};+use kuchiki::{iter::NodeEdge, traits::*, NodeRef};use regex::Regex;-use std::collections::HashMap;+use std::{borrow::Borrow, collections::HashMap};use uuid::Uuid;mod just_text;-mod lexer;pub mod region;mod url;use crate::schema::{Field, ALL_FIELDS, CENTRALITY_SCALING};pub use self::url::Url;-use self::{just_text::JustText, lexer::Token, region::Region};+use self::{just_text::JustText, region::Region};static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {Regex::new(r#"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b"#).unwrap()@@ -46,7 +45,6 @@ pub struct FaviconLink {pub struct Preprocessor<const N: usize> {removed_tags: [&'static str; N],num_open_tags: [i64; N],- open_comments: i64,}impl<const N: usize> Preprocessor<N> {@@ -54,55 +52,57 @@ impl<const N: usize> Preprocessor<N> {Self {removed_tags,num_open_tags: [0; N],- open_comments: 0,}}- pub fn update(&mut self, tok: &Token) {- match tok {- Token::StartTag(tag) if !tag.raw().contains("/>") => {- if let Some((_, n)) = self- .removed_tags- .iter()- .zip(self.num_open_tags.iter_mut())- .find(|(name, _)| **name == tag.name())- {- *n += 1;+ pub fn update(&mut self, edge: &NodeEdge<NodeRef>) {+ match edge {+ NodeEdge::Start(node) => {+ if let Some(element) = node.as_element() {+ let element_name: &str = element.name.local.borrow();+ if let Some((_, n)) = self+ .removed_tags+ .iter()+ .zip(self.num_open_tags.iter_mut())+ .find(|(name, _)| **name == element_name)+ {+ *n += 1;+ }}}- Token::EndTag(tag) => {- if let Some((_, n)) = self- .removed_tags- .iter()- .zip(self.num_open_tags.iter_mut())- .find(|(name, _)| **name == tag.name())- {- *n -= 1;+ NodeEdge::End(node) => {+ if let Some(element) = node.as_element() {+ let element_name: &str = element.name.local.borrow();+ if let Some((_, n)) = self+ .removed_tags+ .iter()+ .zip(self.num_open_tags.iter_mut())+ .find(|(name, _)| **name == element_name)+ {+ *n -= 1;+ }}}- Token::BeginComment => self.open_comments += 1,- Token::EndComment => self.open_comments -= 1,- _ => {}}}pub fn is_inside_removed(&self) -> bool {- self.num_open_tags.iter().any(|n| *n > 0) || self.open_comments > 0+ self.num_open_tags.iter().any(|n| *n > 0)}}-pub struct Webpage<'a> {- pub html: Html<'a>,+pub struct Webpage {+ pub html: Html,pub backlinks: Vec<Link>,pub centrality: f64,pub fetch_time_ms: u64,pub primary_image_uuid: Option<Uuid>,}-impl<'a> Webpage<'a> {+impl Webpage {#[cfg(test)]pub fn new(- html: &'a str,+ html: &str,url: &str,backlinks: Vec<Link>,centrality: f64,@@ -186,71 +186,73 @@ impl<'a> Webpage<'a> {}}+struct Script {+ attributes: HashMap<String, String>,+ content: String,+}+#[derive(Debug)]-pub struct Html<'a> {- raw: &'a str,- tokens: Lexer<'a, Token<'a>>,+pub struct Html {url: Url,+ root: NodeRef, // this is reference counted (cheap to clone)}-impl<'a> Html<'a> {- pub fn parse(html: &'a str, url: &str) -> Self {- let tokens = Token::lexer(html);+impl Html {+ pub fn parse(html: &str, url: &str) -> Self {+ let root = kuchiki::parse_html().one(html);Self {- raw: html,- tokens,+ root,url: url.to_string().into(),}}pub fn links(&self) -> Vec<Link> {- let mut tokens = self.tokens.clone();let mut links = Vec::new();let mut open_links = Vec::new();let mut preprocessor = Preprocessor::new(["script", "style", "head"]);- while let Some(tok) = tokens.next() {- preprocessor.update(&tok);+ for edge in self.root.traverse() {+ preprocessor.update(&edge);if preprocessor.is_inside_removed() {continue;}- match tok {- Token::StartTag(tag) if tag.name() == "a" => {- open_links.push((String::new(), tag.attributes()));- }- Token::EndTag(tag) if tag.name() == "a" => {- if let Some((text, attributes)) = open_links.pop() {- if let Some(dest) = attributes.get("href") {- links.push(Link {- source: self.url.clone(),- destination: dest.to_string().into(),- text,- })+ match edge {+ NodeEdge::Start(node) => {+ if let Some(element) = node.as_element() {+ if &element.name.local == "a" {+ open_links.push((String::new(), element.attributes.clone()));}}}- Token::Error => {- for (text, _) in &mut open_links {- let span = tokens.span();- text.push_str(&self.raw[span]);+ NodeEdge::End(node) => {+ if let Some(element) = node.as_element() {+ if &element.name.local == "a" {+ if let Some((text, attributes)) = open_links.pop() {+ if let Some(dest) = attributes.borrow().get("href") {+ links.push(Link {+ source: self.url.clone(),+ destination: dest.to_string().into(),+ text: text.trim().to_string(),+ })+ }+ }+ }}- }- Token::SelfTerminatingTag(tag) if tag.name() == "a" => {- if let Some(dest) = tag.attributes().get("href") {- links.push(Link {- source: self.url.clone(),- destination: dest.to_string().into(),- text: String::new(),- })++ if let Some(text) = node.as_text() {+ let raw_text = text.borrow();+ let text = raw_text.trim();++ if !text.is_empty() {+ for (link_text, _) in &mut open_links {+ link_text.push('\n');+ link_text.push_str(text);+ }+ }}}- Token::StartTag(_)- | Token::EndTag(_)- | Token::SelfTerminatingTag(_)- | Token::BeginComment- | Token::EndComment => {}}}@@ -258,59 +260,38 @@ impl<'a> Html<'a> {}pub fn favicon(&self) -> Option<FaviconLink> {- let mut preprocessor = Preprocessor::new(["script", "style", "body"]);-- let tokens = self.tokens.clone();- for tok in tokens {- preprocessor.update(&tok);- if preprocessor.is_inside_removed() {+ for node in self.root.select("link").unwrap() {+ if !matches!(node.attributes.borrow().get("rel"), Some("icon")) {continue;}- match tok {- Token::StartTag(tag) | Token::SelfTerminatingTag(tag) if tag.name() == "link" => {- let rel = tag.attributes().get("rel").cloned();- if rel.is_none() {- continue;- }- if rel.unwrap() != "icon" {- continue;+ if let Some(link) = node.attributes.borrow().get("href") {+ let (width, height) = match node.attributes.borrow().get("sizes") {+ Some(size) => {+ if let Some((width, height)) = size.split_once('x') {+ (width.parse().ok(), height.parse().ok())+ } else {+ (None, None)+ }}+ _ => (None, None),+ };- if let Some(link) = tag.attributes().get("href") {- let (width, height) = match tag.attributes().get("sizes") {- Some(size) => {- if let Some((width, height)) = size.split_once('x') {- (width.parse().ok(), height.parse().ok())- } else {- (None, None)- }- }- _ => (None, None),- };+ let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());+ let mut link: Url = link.to_string().into();- let image_type = tag.attributes().get("type").map(|t| t.to_string());- let mut link: Url = link.to_string().into();+ if !link.is_full_path() {+ link.prefix_with(&self.url);+ }- if !link.is_full_path() {- link.prefix_with(&self.url);- }+ let favicon = FaviconLink {+ link,+ image_type,+ width,+ height,+ };- let favicon = FaviconLink {- link,- image_type,- width,- height,- };- return Some(favicon);- }- }- Token::StartTag(_)- | Token::EndTag(_)- | Token::SelfTerminatingTag(_)- | Token::BeginComment- | Token::Error- | Token::EndComment => {}+ return Some(favicon);}}@@ -318,7 +299,7 @@ impl<'a> Html<'a> {}pub fn clean_text(&self) -> Option<String> {- let text = JustText::default().extract(self.raw);+ let text = JustText::default().extract(self.root.clone());if text.is_empty() {None@@ -336,7 +317,7 @@ impl<'a> Html<'a> {stopwords_high: -1.0,max_heading_distance: 10000,}- .extract(self.raw);+ .extract(self.root.clone());if text.is_empty() {None@@ -346,47 +327,11 @@ impl<'a> Html<'a> {}pub fn title(&self) -> Option<String> {- let mut tokens = self.tokens.clone();- let mut title = None;- let mut open_tags = 0;- let mut preprocessor = Preprocessor::new(["script", "style"]);-- while let Some(tok) = tokens.next() {- preprocessor.update(&tok);- if preprocessor.is_inside_removed() {- continue;- }-- match tok {- Token::StartTag(tag) if tag.name() == "title" => {- open_tags += 1;- }- Token::EndTag(tag) if tag.name() == "title" => {- open_tags -= 1;-- if open_tags == 0 {- break;- }- }- Token::Error => {- if open_tags > 0 {- let span = tokens.span();- if let Some(cur_title) = title {- title = Some(cur_title + &self.raw[span]);- } else {- title = Some(self.raw[span].to_string());- }- }- }- Token::SelfTerminatingTag(_)- | Token::StartTag(_)- | Token::EndTag(_)- | Token::BeginComment- | Token::EndComment => {}- }+ if let Ok(title) = self.root.select_first("title") {+ Some(title.text_contents().trim().to_string())+ } else {+ None}-- title}pub fn url(&self) -> &Url {@@ -406,39 +351,24 @@ impl<'a> Html<'a> {}pub fn metadata(&self) -> Vec<Meta> {- let tokens = self.tokens.clone();let mut metas = Vec::new();- let mut preprocessor = Preprocessor::new(["script", "style"]);- for tok in tokens {- preprocessor.update(&tok);- if preprocessor.is_inside_removed() {- continue;- }-- match tok {- Token::StartTag(tag) if tag.name() == "meta" => {- metas.push(- tag.attributes()- .into_iter()- .map(|(key, value)| (key.to_string(), value.to_string()))- .collect(),- );- }- Token::SelfTerminatingTag(tag) if tag.name() == "meta" => {- metas.push(- tag.attributes()- .into_iter()- .map(|(key, value)| (key.to_string(), value.to_string()))- .collect(),- );- }- Token::StartTag(_)- | Token::EndTag(_)- | Token::SelfTerminatingTag(_)- | Token::Error- | Token::BeginComment- | Token::EndComment => {}+ for node in self.root.select("meta").unwrap() {+ if let Some(element) = node.as_node().as_element() {+ metas.push(+ element+ .attributes+ .borrow()+ .map+ .iter()+ .map(|(name, attr)| {+ (+ name.local.borrow().to_string(),+ attr.borrow().value.to_string(),+ )+ })+ .collect(),+ );}}@@ -511,50 +441,75 @@ impl<'a> Html<'a> {Ok(doc)}+ fn scripts(&self) -> Vec<Script> {+ let mut scripts = Vec::new();++ for node in self.root.select("script").unwrap() {+ let content = node.text_contents().trim().to_string();+ let attributes = node+ .attributes+ .borrow()+ .map+ .iter()+ .map(|(name, attr)| {+ (+ name.local.borrow().to_string(),+ attr.borrow().value.to_string(),+ )+ })+ .collect();++ scripts.push(Script {+ attributes,+ content,+ })+ }++ scripts+ }+pub fn schema_org(&self) -> Vec<SchemaOrg> {- let mut tokens = self.tokens.clone();let mut schemas = Vec::new();- let mut preprocessor = Preprocessor::new([]);- let mut open_schemas = 0;- let mut json = String::new();- while let Some(tok) = tokens.next() {- preprocessor.update(&tok);- if preprocessor.is_inside_removed() {- continue;+ for schema in self.scripts().into_iter().filter(|script| {+ matches!(+ script.attributes.get("type").map(|s| s.as_str()),+ Some("application/ld+json")+ )+ }) {+ if let Ok(schema) = serde_json::from_str(&schema.content) {+ schemas.push(schema);}+ }- match tok {- Token::StartTag(tag) if tag.name() == "script" => {- if let Some(&"application/ld+json") = tag.attributes().get("type") {- open_schemas += 1;- }- }- Token::EndTag(tag) if tag.name() == "script" => {- if open_schemas > 0 {- open_schemas -= 1;+ schemas+ }- if open_schemas == 0 {- if let Ok(schema) = serde_json::from_str(&json) {- schemas.push(schema);- }- json = String::new();- }- }- }- Token::Error => {- let span = tokens.span();- json.push_str(&self.raw[span]);- }- Token::StartTag(_)- | Token::EndTag(_)- | Token::SelfTerminatingTag(_)- | Token::BeginComment- | Token::EndComment => {}+ pub fn trackers(&self) -> Vec<Url> {+ let mut links: Vec<Url> = Vec::new();++ for script in self.scripts() {+ if let Some(link) = script.attributes.get("src") {+ links.push(link.to_string().into());+ }++ for res in URL_REGEX.find_iter(&script.content) {+ links.push(res.as_str().to_string().into());}}- schemas+ for node in self.root.select("link").unwrap() {+ if let Some(link) = node.attributes.borrow().get("href") {+ links.push(link.to_string().into())+ }+ }++ links+ .into_iter()+ .filter(|link| !link.host().is_empty())+ .filter(|link| link.host() != self.host())+ .unique_by(|link| link.host().to_string())+ .collect()}fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {@@ -646,77 +601,6 @@ impl<'a> Html<'a> {}).and_then(|metadata| metadata.get("content").cloned())}-- pub fn trackers(&self) -> Vec<Url> {- let mut tokens = self.tokens.clone();-- let mut links: Vec<Url> = Vec::new();- let mut open_scripts = 0;- let mut script_text = String::new();- let mut preprocessor = Preprocessor::new([]);-- while let Some(tok) = tokens.next() {- preprocessor.update(&tok);- if preprocessor.is_inside_removed() {- continue;- }-- match tok {- Token::StartTag(tag) if tag.name() == "script" => {- if !tag.raw().contains("/>") {- open_scripts += 1;- }- if let Some(link) = tag.attributes().get("src") {- links.push(link.to_string().into());- }- }- Token::EndTag(tag) if tag.name() == "script" => {- if open_scripts > 0 {- open_scripts -= 1;-- if open_scripts == 0 {- for res in URL_REGEX.find_iter(script_text.as_str()) {- links.push(res.as_str().to_string().into());- }- script_text.clear();- }- }- }- _ if open_scripts > 0 => {- let span = tokens.span();- script_text.push_str(&self.raw[span]);- }- Token::Error => {- if open_scripts > 0 {- let span = tokens.span();- script_text.push_str(&self.raw[span]);- }- }- Token::SelfTerminatingTag(tag) if tag.name() == "script" => {- if let Some(link) = tag.attributes().get("src") {- links.push(link.to_string().into());- }- }- Token::StartTag(tag) | Token::SelfTerminatingTag(tag) if tag.name() == "link" => {- if let Some(link) = tag.attributes().get("href") {- links.push(link.to_string().into());- }- }- Token::StartTag(_)- | Token::EndTag(_)- | Token::SelfTerminatingTag(_)- | Token::BeginComment- | Token::EndComment => {}- }- }-- links- .into_iter()- .filter(|link| !link.host().is_empty())- .filter(|link| link.host() != self.host())- .unique_by(|link| link.host().to_string())- .collect()- }}#[derive(Debug, PartialEq, Eq)]@@ -1273,12 +1157,11 @@ mod tests {a.appendChild(r);})(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');</script>- <script src="https://thirdparty.com/js" />- <script src="https://example.com/js" />+ <script src="https://thirdparty.com/js"></script>+ <script src="https://example.com/js"></script><link href='//securepubads.g.doubleclick.net' rel='preconnect'>- <script src="https://thirdparty.com/js" />- <script src="/js/file" />- <meta property="article:modified_time" content="2022-06-22T19:37:34+00:00" />+ <script src="https://thirdparty.com/js"></script>+ <script src="/js/file"></script></head><body></body>@@ -1318,7 +1201,7 @@ mod tests {a.appendChild(r);})(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');</script>- <script src="https://thirdparty.com/js" />+ <script src="https://thirdparty.com/js"></script><link href='//securepubads.g.doubleclick.net' rel='preconnect'><title>Test site</title></head>