Skip to content

web search done right

Handle HTML comments in lexer

Mikkel Denkerb526a782a63f
diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rs
index 56d4a358..c01e92b4 100644
--- a/src/webpage/just_text.rs
+++ b/src/webpage/just_text.rs
@@ -20,7 +20,7 @@ use lazy_static::lazy_static;
use logos::{Lexer, Logos};
use whatlang::Lang;
-use super::lexer::Token;
+use super::{lexer::Token, Preprocessor};
// implementation of the JustText algorithm described in this thesis: https://is.muni.cz/th/45523/fi_d/phdthesis.pdf
// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py
@@ -194,51 +194,6 @@ struct ClassifiedParagraph {
classification: Classification,
}
-// preprocessor removes scripts, comments, style, embed, forms and head.
-struct Preprocessor<const N: usize> {
- removed_tags: [&'static str; N],
- num_open_tags: [usize; N],
-}
-
-impl<const N: usize> Preprocessor<N> {
- fn new(removed_tags: [&'static str; N]) -> Self {
- Self {
- removed_tags,
- num_open_tags: [0; N],
- }
- }
-
- fn update(&mut self, tok: &Token) {
- match tok {
- Token::StartTag(tag) => {
- if let Some((_, n)) = self
- .removed_tags
- .iter()
- .zip(self.num_open_tags.iter_mut())
- .find(|(name, _)| **name == tag.name())
- {
- *n += 1;
- }
- }
- Token::EndTag(tag) => {
- if let Some((_, n)) = self
- .removed_tags
- .iter()
- .zip(self.num_open_tags.iter_mut())
- .find(|(name, _)| **name == tag.name())
- {
- *n -= 1;
- }
- }
- Token::SelfTerminatingTag(_) | Token::Error => {}
- }
- }
-
- fn is_inside_removed(&self) -> bool {
- self.num_open_tags.iter().any(|n| *n > 0)
- }
-}
-
impl JustText {
fn paragraphs<'a>(&self, mut tokens: Lexer<'a, Token<'a>>, raw: &str) -> Vec<Paragraph> {
let mut res = Vec::new();
@@ -360,7 +315,7 @@ impl JustText {
br = false;
}
- Token::SelfTerminatingTag(_) => {}
+ Token::SelfTerminatingTag(_) | Token::BeginComment | Token::EndComment => {}
}
}
diff --git a/src/webpage/lexer.rs b/src/webpage/lexer.rs
index b84cc6b0..99dea3d2 100644
--- a/src/webpage/lexer.rs
+++ b/src/webpage/lexer.rs
@@ -137,6 +137,12 @@ pub enum Token<'a> {
#[regex(r"<[^[/>]]+/>", start_tag)]
SelfTerminatingTag(Tag<'a>),
+ #[token("<!--")]
+ BeginComment,
+
+ #[token("-->")]
+ EndComment,
+
#[error]
Error,
}
diff --git a/src/webpage/mod.rs b/src/webpage/mod.rs
index 18aaa358..eb17229b 100644
--- a/src/webpage/mod.rs
+++ b/src/webpage/mod.rs
@@ -81,6 +81,54 @@ pub fn domain(url: &str) -> &'_ str {
}
}
+pub struct Preprocessor<const N: usize> {
+ removed_tags: [&'static str; N],
+ num_open_tags: [usize; N],
+ open_comments: usize,
+}
+
+impl<const N: usize> Preprocessor<N> {
+ pub fn new(removed_tags: [&'static str; N]) -> Self {
+ Self {
+ removed_tags,
+ num_open_tags: [0; N],
+ open_comments: 0,
+ }
+ }
+
+ pub fn update(&mut self, tok: &Token) {
+ match tok {
+ Token::StartTag(tag) => {
+ if let Some((_, n)) = self
+ .removed_tags
+ .iter()
+ .zip(self.num_open_tags.iter_mut())
+ .find(|(name, _)| **name == tag.name())
+ {
+ *n += 1;
+ }
+ }
+ Token::EndTag(tag) => {
+ if let Some((_, n)) = self
+ .removed_tags
+ .iter()
+ .zip(self.num_open_tags.iter_mut())
+ .find(|(name, _)| **name == tag.name())
+ {
+ *n -= 1;
+ }
+ }
+ Token::SelfTerminatingTag(_) | Token::Error => {}
+ Token::BeginComment => self.open_comments += 1,
+ Token::EndComment => self.open_comments -= 1,
+ }
+ }
+
+ pub fn is_inside_removed(&self) -> bool {
+ self.num_open_tags.iter().any(|n| *n > 0) || self.open_comments > 0
+ }
+}
+
pub struct Webpage<'a> {
pub html: Html<'a>,
pub backlinks: Vec<Link>,
@@ -147,8 +195,14 @@ impl<'a> Html<'a> {
let mut tokens = self.tokens.clone();
let mut links = Vec::new();
let mut open_links = Vec::new();
+ let mut preprocessor = Preprocessor::new(["script", "style", "head"]);
while let Some(tok) = tokens.next() {
+ preprocessor.update(&tok);
+ if preprocessor.is_inside_removed() {
+ continue;
+ }
+
match tok {
Token::StartTag(tag) if tag.name() == "a" => {
open_links.push((String::new(), tag.attributes()));
@@ -179,7 +233,11 @@ impl<'a> Html<'a> {
})
}
}
- Token::StartTag(_) | Token::EndTag(_) | Token::SelfTerminatingTag(_) => {}
+ Token::StartTag(_)
+ | Token::EndTag(_)
+ | Token::SelfTerminatingTag(_)
+ | Token::BeginComment
+ | Token::EndComment => {}
}
}
@@ -200,8 +258,14 @@ impl<'a> Html<'a> {
let mut tokens = self.tokens.clone();
let mut title = None;
let mut open_tags = 0;
+ let mut preprocessor = Preprocessor::new(["script", "style"]);
while let Some(tok) = tokens.next() {
+ preprocessor.update(&tok);
+ if preprocessor.is_inside_removed() {
+ continue;
+ }
+
match tok {
Token::StartTag(tag) if tag.name() == "title" => {
open_tags += 1;
@@ -223,7 +287,11 @@ impl<'a> Html<'a> {
}
}
}
- Token::SelfTerminatingTag(_) | Token::StartTag(_) | Token::EndTag(_) => {}
+ Token::SelfTerminatingTag(_)
+ | Token::StartTag(_)
+ | Token::EndTag(_)
+ | Token::BeginComment
+ | Token::EndComment => {}
}
}
@@ -249,8 +317,14 @@ impl<'a> Html<'a> {
pub fn metadata(&self) -> Vec<Meta> {
let tokens = self.tokens.clone();
let mut metas = Vec::new();
+ let mut preprocessor = Preprocessor::new(["script", "style"]);
for tok in tokens {
+ preprocessor.update(&tok);
+ if preprocessor.is_inside_removed() {
+ continue;
+ }
+
match tok {
Token::StartTag(tag) if tag.name() == "meta" => {
metas.push(
@@ -271,7 +345,9 @@ impl<'a> Html<'a> {
Token::StartTag(_)
| Token::EndTag(_)
| Token::SelfTerminatingTag(_)
- | Token::Error => {}
+ | Token::Error
+ | Token::BeginComment
+ | Token::EndComment => {}
}
}