Skip to content

web search done right

[refactor] split-up some large pieces to make it easier to reason about the code. * split 'PatternQuery' into its 'Query', 'Weight' and 'Score' components. * split the html parsing into smaller logically similar functions. everything is still under 'impl Html {...}' so we don't need to import helper traits everywhere, but there are simply more 'impl Html's now. * extract some common functionality into helper functions for the stackoverflow sidebar.

Mikkel Denker1bb40b79e595
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
deleted file mode 100644
index eeb7b4da..00000000
--- a/core/src/query/pattern_query.rs
+++ /dev/null
@@ -1,911 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::sync::Arc;
-
-use optics::PatternPart;
-use tantivy::{
- fieldnorm::FieldNormReader,
- postings::SegmentPostings,
- query::{EmptyScorer, Explanation, Scorer},
- schema::IndexRecordOption,
- tokenizer::Tokenizer,
- DocId, DocSet, Postings, Score, SegmentReader, TantivyError, TERMINATED,
-};
-
-use crate::{
- fastfield_reader::{self, FastFieldReader},
- query::intersection::Intersection,
- ranking::bm25::Bm25Weight,
- schema::{FastField, Field, TextField, ALL_FIELDS},
-};
-
-#[derive(Clone)]
-pub struct PatternQuery {
- patterns: Vec<PatternPart>,
- can_optimize_site_domain: bool,
- field: tantivy::schema::Field,
- raw_terms: Vec<tantivy::Term>,
- fastfield_reader: FastFieldReader,
-}
-
-impl std::fmt::Debug for PatternQuery {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- f.debug_struct("PatternQuery")
- .field("patterns", &self.patterns)
- .field("field", &self.field)
- .field("raw_terms", &self.raw_terms)
- .finish()
- }
-}
-
-impl PatternQuery {
- pub fn new(
- patterns: Vec<PatternPart>,
- field: TextField,
- schema: &tantivy::schema::Schema,
- fastfield_reader: FastFieldReader,
- ) -> Self {
- let field = Field::Text(field);
- let tv_field = schema.get_field(field.name()).unwrap();
-
- if can_optimize_site_domain(&patterns, field) {
- if patterns.len() == 3 {
- let PatternPart::Raw(term) = &patterns[1] else {
- unreachable!()
- };
-
- return Self {
- patterns: Vec::new(),
- field: tv_field,
- can_optimize_site_domain: true,
- raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],
- fastfield_reader,
- };
- } else {
- let term: String = patterns
- .iter()
- .filter_map(|p| match p {
- PatternPart::Raw(s) => Some(s.clone()),
- PatternPart::Wildcard => None,
- PatternPart::Anchor => None,
- })
- .collect();
-
- return Self {
- patterns,
- field: tv_field,
- can_optimize_site_domain: true,
- raw_terms: vec![tantivy::Term::from_field_text(tv_field, &term)],
- fastfield_reader,
- };
- }
- }
-
- let mut raw_terms = Vec::with_capacity(patterns.len());
- let mut new_patterns = Vec::with_capacity(patterns.len());
-
- for pattern in &patterns {
- match pattern {
- PatternPart::Raw(text) => {
- let mut tokenizer = field.as_text().unwrap().indexing_tokenizer();
- let mut stream = tokenizer.token_stream(text);
-
- while let Some(token) = stream.next() {
- new_patterns.push(PatternPart::Raw(token.text.clone()));
- let term = tantivy::Term::from_field_text(tv_field, &token.text);
- raw_terms.push(term);
- }
- }
- PatternPart::Wildcard => new_patterns.push(PatternPart::Wildcard),
- PatternPart::Anchor => new_patterns.push(PatternPart::Anchor),
- }
- }
-
- raw_terms.shrink_to_fit();
-
- Self {
- patterns: new_patterns,
- field: tv_field,
- raw_terms,
- can_optimize_site_domain: false,
- fastfield_reader,
- }
- }
-}
-
-impl tantivy::query::Query for PatternQuery {
- fn weight(
- &self,
- scoring: tantivy::query::EnableScoring<'_>,
- ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
- let bm25_weight = match scoring {
- tantivy::query::EnableScoring::Enabled {
- searcher,
- statistics_provider: _,
- } => {
- if self.raw_terms.is_empty() {
- None
- } else {
- Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)
- }
- }
- tantivy::query::EnableScoring::Disabled { .. } => None,
- };
-
- if self.can_optimize_site_domain {
- return Ok(Box::new(FastSiteDomainPatternWeight {
- term: self.raw_terms[0].clone(),
- field: self.field,
- similarity_weight: bm25_weight,
- }));
- }
-
- Ok(Box::new(PatternWeight {
- similarity_weight: bm25_weight,
- raw_terms: self.raw_terms.clone(),
- patterns: self.patterns.clone(),
- field: self.field,
- fastfield_reader: self.fastfield_reader.clone(),
- }))
- }
-
- fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a tantivy::Term, bool)) {
- for term in &self.raw_terms {
- visitor(term, true);
- }
- }
-}
-
-#[derive(Debug)]
-enum SmallPatternPart {
- Term,
- Wildcard,
- Anchor,
-}
-
-/// if pattern is of form Site("|site|") or Domain("|domain|")
-/// we can use the field without tokenization to speed up the query significantly
-fn can_optimize_site_domain(patterns: &[PatternPart], field: Field) -> bool {
- patterns.len() >= 2
- && matches!(&patterns[0], PatternPart::Anchor)
- && matches!(&patterns[patterns.len() - 1], PatternPart::Anchor)
- && patterns[1..patterns.len() - 1]
- .iter()
- .all(|pattern| matches!(pattern, PatternPart::Raw(_)))
- && (matches!(field, Field::Text(TextField::UrlForSiteOperator))
- || matches!(field, Field::Text(TextField::Domain)))
-}
-
-struct FastSiteDomainPatternWeight {
- term: tantivy::Term,
- field: tantivy::schema::Field,
- similarity_weight: Option<Bm25Weight>,
-}
-
-impl FastSiteDomainPatternWeight {
- fn fieldnorm_reader(&self, reader: &SegmentReader) -> tantivy::Result<FieldNormReader> {
- if self.similarity_weight.is_some() {
- if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {
- return Ok(fieldnorm_reader);
- }
- }
- Ok(FieldNormReader::constant(reader.max_doc(), 1))
- }
-
- fn pattern_scorer(
- &self,
- reader: &SegmentReader,
- boost: Score,
- ) -> tantivy::Result<Option<FastSiteDomainPatternScorer>> {
- let similarity_weight = self
- .similarity_weight
- .as_ref()
- .map(|weight| weight.boost_by(boost));
-
- let fieldnorm_reader = self.fieldnorm_reader(reader)?;
-
- let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {
- Field::Text(TextField::UrlForSiteOperator) => Field::Text(TextField::SiteNoTokenizer),
- Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),
- _ => unreachable!(),
- };
-
- let tv_field = reader
- .schema()
- .get_field(field_no_tokenizer.name())
- .unwrap();
-
- let opt = match field_no_tokenizer {
- Field::Text(t) => t.index_option(),
- Field::Fast(_) => unreachable!(),
- };
-
- match reader
- .inverted_index(tv_field)?
- .read_postings(&self.term, opt)?
- {
- Some(posting) => Ok(Some(FastSiteDomainPatternScorer {
- similarity_weight,
- posting,
- fieldnorm_reader,
- })),
- None => Ok(None),
- }
- }
-}
-
-impl tantivy::query::Weight for FastSiteDomainPatternWeight {
- fn scorer(
- &self,
- reader: &tantivy::SegmentReader,
- boost: tantivy::Score,
- ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
- if let Some(scorer) = self.pattern_scorer(reader, boost)? {
- Ok(Box::new(PatternScorer::FastSiteDomain(scorer)))
- } else {
- Ok(Box::new(EmptyScorer))
- }
- }
-
- fn explain(
- &self,
- reader: &tantivy::SegmentReader,
- doc: tantivy::DocId,
- ) -> tantivy::Result<tantivy::query::Explanation> {
- let scorer_opt = self.pattern_scorer(reader, 1.0)?;
- if scorer_opt.is_none() {
- return Err(TantivyError::InvalidArgument(format!(
- "Document #({doc}) does not match (empty scorer)"
- )));
- }
- let mut scorer = scorer_opt.unwrap();
- if scorer.seek(doc) != doc {
- return Err(TantivyError::InvalidArgument(format!(
- "Document #({doc}) does not match"
- )));
- }
- let fieldnorm_reader = self.fieldnorm_reader(reader)?;
- let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
- let term_freq = scorer.term_freq();
- let mut explanation = Explanation::new("Pattern Scorer", scorer.score());
- explanation.add_detail(
- self.similarity_weight
- .as_ref()
- .unwrap()
- .explain(fieldnorm_id, term_freq),
- );
- Ok(explanation)
- }
-}
-
-struct PatternWeight {
- similarity_weight: Option<Bm25Weight>,
- patterns: Vec<PatternPart>,
- raw_terms: Vec<tantivy::Term>,
- field: tantivy::schema::Field,
- fastfield_reader: FastFieldReader,
-}
-
-impl PatternWeight {
- fn fieldnorm_reader(&self, reader: &SegmentReader) -> tantivy::Result<FieldNormReader> {
- if self.similarity_weight.is_some() {
- if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {
- return Ok(fieldnorm_reader);
- }
- }
- Ok(FieldNormReader::constant(reader.max_doc(), 1))
- }
-
- pub(crate) fn pattern_scorer(
- &self,
- reader: &SegmentReader,
- boost: Score,
- ) -> tantivy::Result<Option<PatternScorer>> {
- if self.patterns.is_empty() {
- return Ok(None);
- }
-
- let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {
- Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),
- Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),
- Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),
- Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
- Field::Text(TextField::UrlForSiteOperator) => {
- Ok(FastField::NumUrlForSiteOperatorTokens)
- }
- Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
- Field::Text(TextField::MicroformatTags) => Ok(FastField::NumMicroformatTagsTokens),
- Field::Text(TextField::FlattenedSchemaOrgJson) => {
- Ok(FastField::NumFlattenedSchemaTokens)
- }
- field => Err(TantivyError::InvalidArgument(format!(
- "{} is not supported in pattern query",
- field.name()
- ))),
- }?;
-
- // "*" matches everything
- if self.raw_terms.is_empty()
- && self
- .patterns
- .iter()
- .any(|p| matches!(p, PatternPart::Wildcard))
- {
- return Ok(Some(PatternScorer::Everything(AllScorer {
- doc: 0,
- max_doc: reader.max_doc(),
- })));
- }
-
- // "||" and "|" matches empty string
-
- if self.raw_terms.is_empty()
- && self
- .patterns
- .iter()
- .all(|p| matches!(p, PatternPart::Anchor))
- {
- return Ok(Some(PatternScorer::EmptyField(EmptyFieldScorer {
- num_tokens_fastfield,
- segment_reader: self.fastfield_reader.get_segment(&reader.segment_id()),
- all_scorer: AllScorer {
- doc: 0,
- max_doc: reader.max_doc(),
- },
- })));
- }
-
- let similarity_weight = self
- .similarity_weight
- .as_ref()
- .map(|weight| weight.boost_by(boost));
-
- let fieldnorm_reader = self.fieldnorm_reader(reader)?;
-
- let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());
- for term in &self.raw_terms {
- if let Some(postings) = reader
- .inverted_index(term.field())?
- .read_postings(term, IndexRecordOption::WithFreqsAndPositions)?
- {
- term_postings_list.push(postings);
- } else {
- return Ok(None);
- }
- }
-
- let small_patterns = self
- .patterns
- .iter()
- .map(|pattern| match pattern {
- PatternPart::Raw(_) => SmallPatternPart::Term,
- PatternPart::Wildcard => SmallPatternPart::Wildcard,
- PatternPart::Anchor => SmallPatternPart::Anchor,
- })
- .collect();
-
- Ok(Some(PatternScorer::Normal(NormalPatternScorer::new(
- similarity_weight,
- term_postings_list,
- fieldnorm_reader,
- small_patterns,
- reader.segment_id(),
- num_tokens_fastfield,
- self.fastfield_reader.clone(),
- ))))
- }
-}
-
-impl tantivy::query::Weight for PatternWeight {
- fn scorer(
- &self,
- reader: &tantivy::SegmentReader,
- boost: tantivy::Score,
- ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
- if let Some(scorer) = self.pattern_scorer(reader, boost)? {
- Ok(Box::new(scorer))
- } else {
- Ok(Box::new(EmptyScorer))
- }
- }
-
- fn explain(
- &self,
- reader: &tantivy::SegmentReader,
- doc: tantivy::DocId,
- ) -> tantivy::Result<tantivy::query::Explanation> {
- let scorer_opt = self.pattern_scorer(reader, 1.0)?;
- if scorer_opt.is_none() {
- return Err(TantivyError::InvalidArgument(format!(
- "Document #({doc}) does not match (empty scorer)"
- )));
- }
- let mut scorer = scorer_opt.unwrap();
- if scorer.seek(doc) != doc {
- return Err(TantivyError::InvalidArgument(format!(
- "Document #({doc}) does not match"
- )));
- }
- let fieldnorm_reader = self.fieldnorm_reader(reader)?;
- let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
- let term_freq = scorer.term_freq();
- let mut explanation = Explanation::new("Pattern Scorer", scorer.score());
- explanation.add_detail(
- self.similarity_weight
- .as_ref()
- .unwrap()
- .explain(fieldnorm_id, term_freq),
- );
- Ok(explanation)
- }
-}
-
-enum PatternScorer {
- Normal(NormalPatternScorer),
- FastSiteDomain(FastSiteDomainPatternScorer),
- Everything(AllScorer),
- EmptyField(EmptyFieldScorer),
-}
-
-impl Scorer for PatternScorer {
- fn score(&mut self) -> Score {
- match self {
- PatternScorer::Normal(scorer) => scorer.score(),
- PatternScorer::FastSiteDomain(scorer) => scorer.score(),
- PatternScorer::Everything(scorer) => scorer.score(),
- PatternScorer::EmptyField(scorer) => scorer.score(),
- }
- }
-}
-
-impl DocSet for PatternScorer {
- fn advance(&mut self) -> DocId {
- match self {
- PatternScorer::Normal(scorer) => scorer.advance(),
- PatternScorer::FastSiteDomain(scorer) => scorer.advance(),
- PatternScorer::Everything(scorer) => scorer.advance(),
- PatternScorer::EmptyField(scorer) => scorer.advance(),
- }
- }
-
- fn seek(&mut self, target: DocId) -> DocId {
- match self {
- PatternScorer::Normal(scorer) => scorer.seek(target),
- PatternScorer::FastSiteDomain(scorer) => scorer.seek(target),
- PatternScorer::Everything(scorer) => scorer.seek(target),
- PatternScorer::EmptyField(scorer) => scorer.seek(target),
- }
- }
-
- fn doc(&self) -> DocId {
- match self {
- PatternScorer::Normal(scorer) => scorer.doc(),
- PatternScorer::FastSiteDomain(scorer) => scorer.doc(),
- PatternScorer::Everything(scorer) => scorer.doc(),
- PatternScorer::EmptyField(scorer) => scorer.doc(),
- }
- }
-
- fn size_hint(&self) -> u32 {
- match self {
- PatternScorer::Normal(scorer) => scorer.size_hint(),
- PatternScorer::FastSiteDomain(scorer) => scorer.size_hint(),
- PatternScorer::Everything(scorer) => scorer.size_hint(),
- PatternScorer::EmptyField(scorer) => scorer.size_hint(),
- }
- }
-}
-
-impl PatternScorer {
- fn term_freq(&self) -> u32 {
- match self {
- PatternScorer::Normal(scorer) => scorer.phrase_count(),
- PatternScorer::FastSiteDomain(scorer) => scorer.term_freq(),
- PatternScorer::Everything(_) => 0,
- PatternScorer::EmptyField(_) => 0,
- }
- }
-}
-
-struct AllScorer {
- doc: DocId,
- max_doc: DocId,
-}
-
-impl DocSet for AllScorer {
- fn advance(&mut self) -> DocId {
- if self.doc + 1 >= self.max_doc {
- self.doc = TERMINATED;
- return TERMINATED;
- }
- self.doc += 1;
- self.doc
- }
-
- fn seek(&mut self, target: DocId) -> DocId {
- if target >= self.max_doc {
- self.doc = TERMINATED;
- return TERMINATED;
- }
- self.doc = target;
- self.doc
- }
-
- fn doc(&self) -> DocId {
- self.doc
- }
-
- fn size_hint(&self) -> u32 {
- self.max_doc
- }
-}
-
-impl Scorer for AllScorer {
- fn score(&mut self) -> Score {
- 1.0
- }
-}
-
-struct EmptyFieldScorer {
- segment_reader: Arc<fastfield_reader::SegmentReader>,
- num_tokens_fastfield: FastField,
- all_scorer: AllScorer,
-}
-
-impl EmptyFieldScorer {
- fn num_tokes(&self, doc: DocId) -> u64 {
- let s: Option<u64> = self
- .segment_reader
- .get_field_reader(&self.num_tokens_fastfield)
- .get(&doc)
- .into();
- s.unwrap_or_default()
- }
-}
-
-impl DocSet for EmptyFieldScorer {
- fn advance(&mut self) -> DocId {
- let mut doc = self.all_scorer.advance();
-
- while doc != TERMINATED && self.num_tokes(doc) > 0 {
- doc = self.all_scorer.advance();
- }
-
- doc
- }
-
- fn doc(&self) -> DocId {
- self.all_scorer.doc()
- }
-
- fn seek(&mut self, target: DocId) -> DocId {
- self.all_scorer.seek(target);
-
- if self.doc() != TERMINATED && self.num_tokes(self.all_scorer.doc()) > 0 {
- self.advance()
- } else {
- self.doc()
- }
- }
-
- fn size_hint(&self) -> u32 {
- self.all_scorer.size_hint()
- }
-}
-
-impl Scorer for EmptyFieldScorer {
- fn score(&mut self) -> Score {
- 1.0
- }
-}
-
-struct FastSiteDomainPatternScorer {
- similarity_weight: Option<Bm25Weight>,
- posting: SegmentPostings,
- fieldnorm_reader: FieldNormReader,
-}
-impl FastSiteDomainPatternScorer {
- fn term_freq(&self) -> u32 {
- self.posting.term_freq()
- }
-}
-
-impl Scorer for FastSiteDomainPatternScorer {
- fn score(&mut self) -> Score {
- self.similarity_weight
- .as_ref()
- .map(|w| {
- w.score(
- self.fieldnorm_reader.fieldnorm_id(self.doc()),
- self.posting.term_freq(),
- )
- })
- .unwrap_or_default()
- }
-}
-
-impl DocSet for FastSiteDomainPatternScorer {
- fn advance(&mut self) -> DocId {
- self.posting.advance()
- }
-
- fn doc(&self) -> DocId {
- self.posting.doc()
- }
-
- fn size_hint(&self) -> u32 {
- self.posting.size_hint()
- }
-}
-
-struct NormalPatternScorer {
- pattern_all_simple: bool,
- similarity_weight: Option<Bm25Weight>,
- fieldnorm_reader: FieldNormReader,
- intersection_docset: Intersection<SegmentPostings>,
- pattern: Vec<SmallPatternPart>,
- num_query_terms: usize,
- left: Vec<u32>,
- right: Vec<u32>,
- phrase_count: u32,
- num_tokens_field: FastField,
- segment_reader: Arc<fastfield_reader::SegmentReader>,
-}
-
-impl NormalPatternScorer {
- fn new(
- similarity_weight: Option<Bm25Weight>,
- term_postings_list: Vec<SegmentPostings>,
- fieldnorm_reader: FieldNormReader,
- pattern: Vec<SmallPatternPart>,
- segment: tantivy::SegmentId,
- num_tokens_field: FastField,
- fastfield_reader: FastFieldReader,
- ) -> Self {
- let num_query_terms = term_postings_list.len();
- let segment_reader = fastfield_reader.get_segment(&segment);
-
- let mut s = Self {
- pattern_all_simple: pattern.iter().all(|p| matches!(p, SmallPatternPart::Term)),
- intersection_docset: Intersection::new(term_postings_list),
- num_query_terms,
- similarity_weight,
- fieldnorm_reader,
- pattern,
- left: Vec::with_capacity(100),
- right: Vec::with_capacity(100),
- phrase_count: 0,
- num_tokens_field,
- segment_reader,
- };
-
- if !s.pattern_match() {
- s.advance();
- }
-
- s
- }
- fn phrase_count(&self) -> u32 {
- self.phrase_count
- }
-
- fn pattern_match(&mut self) -> bool {
- if self.num_query_terms == 1 && self.pattern_all_simple {
- // speedup for single term patterns
- self.phrase_count = self
- .intersection_docset
- .docset_mut_specialized(0)
- .term_freq();
- return self.phrase_count > 0;
- }
-
- self.phrase_count = self.perform_pattern_match() as u32;
-
- self.phrase_count > 0
- }
-
- fn perform_pattern_match(&mut self) -> usize {
- if self.intersection_docset.doc() == TERMINATED {
- return 0;
- }
-
- {
- self.intersection_docset
- .docset_mut_specialized(0)
- .positions(&mut self.left);
- }
-
- let mut intersection_len = self.left.len();
- let mut out = Vec::new();
-
- let mut current_right_term = 0;
- let mut slop = 1;
- let num_tokens_doc: Option<u64> = self
- .segment_reader
- .get_field_reader(&self.num_tokens_field)
- .get(&self.doc())
- .into();
- let num_tokens_doc = num_tokens_doc.unwrap();
-
- for (i, pattern_part) in self.pattern.iter().enumerate() {
- match pattern_part {
- SmallPatternPart::Term => {
- if current_right_term == 0 {
- current_right_term = 1;
- continue;
- }
-
- {
- self.intersection_docset
- .docset_mut_specialized(current_right_term)
- .positions(&mut self.right);
- }
- out.resize(self.left.len().max(self.right.len()), 0);
- intersection_len =
- intersection_with_slop(&self.left[..], &self.right[..], &mut out, slop);
-
- slop = 1;
-
- if intersection_len == 0 {
- return 0;
- }
-
- self.left = out[..intersection_len].to_vec();
- out = Vec::new();
- current_right_term += 1;
- }
- SmallPatternPart::Wildcard => {
- slop = u32::MAX;
- }
- SmallPatternPart::Anchor if i == 0 => {
- if let Some(pos) = self.left.first() {
- if *pos != 0 {
- return 0;
- }
- }
- }
- SmallPatternPart::Anchor if i == self.pattern.len() - 1 => {
- {
- self.intersection_docset
- .docset_mut_specialized(self.num_query_terms - 1)
- .positions(&mut self.right);
- }
-
- if let Some(pos) = self.right.last() {
- if *pos != (num_tokens_doc - 1) as u32 {
- return 0;
- }
- }
- }
- SmallPatternPart::Anchor => {}
- }
- }
-
- intersection_len
- }
-}
-
-impl Scorer for NormalPatternScorer {
- fn score(&mut self) -> Score {
- self.similarity_weight
- .as_ref()
- .map(|scorer| {
- let doc = self.doc();
- let fieldnorm_id = self.fieldnorm_reader.fieldnorm_id(doc);
- scorer.score(fieldnorm_id, self.phrase_count())
- })
- .unwrap_or_default()
- }
-}
-
-impl DocSet for NormalPatternScorer {
- fn advance(&mut self) -> DocId {
- loop {
- let doc = self.intersection_docset.advance();
- if doc == TERMINATED || self.pattern_match() {
- return doc;
- }
- }
- }
-
- fn doc(&self) -> tantivy::DocId {
- self.intersection_docset.doc()
- }
-
- fn size_hint(&self) -> u32 {
- self.intersection_docset.size_hint()
- }
-}
-
-/// Intersect twos sorted arrays `left` and `right` and outputs the
-/// resulting array in `out`. The positions in out are all positions from right where
-/// the distance to left_pos <= slop
-///
-/// Returns the length of the intersection
-fn intersection_with_slop(left: &[u32], right: &[u32], out: &mut [u32], slop: u32) -> usize {
- let mut left_index = 0;
- let mut right_index = 0;
- let mut count = 0;
- let left_len = left.len();
- let right_len = right.len();
- while left_index < left_len && right_index < right_len {
- let left_val = left[left_index];
- let right_val = right[right_index];
-
- // The three conditions are:
- // left_val < right_slop -> left index increment.
- // right_slop <= left_val <= right -> find the best match.
- // left_val > right -> right index increment.
- let right_slop = if right_val >= slop {
- right_val - slop
- } else {
- 0
- };
-
- if left_val < right_slop {
- left_index += 1;
- } else if right_slop <= left_val && left_val <= right_val {
- while left_index + 1 < left_len {
- // there could be a better match
- let next_left_val = left[left_index + 1];
- if next_left_val > right_val {
- // the next value is outside the range, so current one is the best.
- break;
- }
- // the next value is better.
- left_index += 1;
- }
- // store the match in left.
- out[count] = right_val;
- count += 1;
- right_index += 1;
- } else if left_val > right_val {
- right_index += 1;
- }
- }
- count
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- fn aux_intersection(left: &[u32], right: &[u32], expected: &[u32], slop: u32) {
- let mut out = vec![0; left.len().max(right.len())];
-
- let intersection_size = intersection_with_slop(left, right, &mut out, slop);
-
- assert_eq!(&out[..intersection_size], expected);
- }
-
- #[test]
- fn test_intersection_with_slop() {
- aux_intersection(&[20, 75, 77], &[18, 21, 60], &[21, 60], u32::MAX);
- aux_intersection(&[21, 60], &[50, 61], &[61], 1);
-
- aux_intersection(&[1, 2, 3], &[], &[], 1);
- aux_intersection(&[], &[1, 2, 3], &[], 1);
-
- aux_intersection(&[1, 2, 3], &[4, 5, 6], &[4], 1);
- aux_intersection(&[1, 2, 3], &[4, 5, 6], &[4, 5, 6], u32::MAX);
-
- aux_intersection(&[20, 75, 77], &[18, 21, 60], &[21, 60], u32::MAX);
- aux_intersection(&[21, 60], &[61, 62], &[61, 62], 2);
-
- aux_intersection(&[60], &[61, 62], &[61, 62], 2);
- }
-}
diff --git a/core/src/query/pattern_query/mod.rs b/core/src/query/pattern_query/mod.rs
new file mode 100644
index 00000000..b94674fa
--- /dev/null
+++ b/core/src/query/pattern_query/mod.rs
@@ -0,0 +1,187 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod scorer;
+mod weight;
+
+use optics::PatternPart;
+
+use tantivy::tokenizer::Tokenizer;
+
+use crate::{
+ fastfield_reader::FastFieldReader,
+ ranking::bm25::Bm25Weight,
+ schema::{Field, TextField},
+};
+
+use self::weight::{FastSiteDomainPatternWeight, PatternWeight};
+
+#[derive(Clone)]
+pub struct PatternQuery {
+ patterns: Vec<PatternPart>,
+ can_optimize_site_domain: bool,
+ field: tantivy::schema::Field,
+ raw_terms: Vec<tantivy::Term>,
+ fastfield_reader: FastFieldReader,
+}
+
+impl std::fmt::Debug for PatternQuery {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("PatternQuery")
+ .field("patterns", &self.patterns)
+ .field("field", &self.field)
+ .field("raw_terms", &self.raw_terms)
+ .finish()
+ }
+}
+
+impl PatternQuery {
+ pub fn new(
+ patterns: Vec<PatternPart>,
+ field: TextField,
+ schema: &tantivy::schema::Schema,
+ fastfield_reader: FastFieldReader,
+ ) -> Self {
+ let field = Field::Text(field);
+ let tv_field = schema.get_field(field.name()).unwrap();
+
+ if can_optimize_site_domain(&patterns, field) {
+ if patterns.len() == 3 {
+ let PatternPart::Raw(term) = &patterns[1] else {
+ unreachable!()
+ };
+
+ return Self {
+ patterns: Vec::new(),
+ field: tv_field,
+ can_optimize_site_domain: true,
+ raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],
+ fastfield_reader,
+ };
+ } else {
+ let term: String = patterns
+ .iter()
+ .filter_map(|p| match p {
+ PatternPart::Raw(s) => Some(s.clone()),
+ PatternPart::Wildcard => None,
+ PatternPart::Anchor => None,
+ })
+ .collect();
+
+ return Self {
+ patterns,
+ field: tv_field,
+ can_optimize_site_domain: true,
+ raw_terms: vec![tantivy::Term::from_field_text(tv_field, &term)],
+ fastfield_reader,
+ };
+ }
+ }
+
+ let mut raw_terms = Vec::with_capacity(patterns.len());
+ let mut new_patterns = Vec::with_capacity(patterns.len());
+
+ for pattern in &patterns {
+ match pattern {
+ PatternPart::Raw(text) => {
+ let mut tokenizer = field.as_text().unwrap().indexing_tokenizer();
+ let mut stream = tokenizer.token_stream(text);
+
+ while let Some(token) = stream.next() {
+ new_patterns.push(PatternPart::Raw(token.text.clone()));
+ let term = tantivy::Term::from_field_text(tv_field, &token.text);
+ raw_terms.push(term);
+ }
+ }
+ PatternPart::Wildcard => new_patterns.push(PatternPart::Wildcard),
+ PatternPart::Anchor => new_patterns.push(PatternPart::Anchor),
+ }
+ }
+
+ raw_terms.shrink_to_fit();
+
+ Self {
+ patterns: new_patterns,
+ field: tv_field,
+ raw_terms,
+ can_optimize_site_domain: false,
+ fastfield_reader,
+ }
+ }
+}
+
+impl tantivy::query::Query for PatternQuery {
+ fn weight(
+ &self,
+ scoring: tantivy::query::EnableScoring<'_>,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
+ let bm25_weight = match scoring {
+ tantivy::query::EnableScoring::Enabled {
+ searcher,
+ statistics_provider: _,
+ } => {
+ if self.raw_terms.is_empty() {
+ None
+ } else {
+ Some(Bm25Weight::for_terms(searcher, &self.raw_terms)?)
+ }
+ }
+ tantivy::query::EnableScoring::Disabled { .. } => None,
+ };
+
+ if self.can_optimize_site_domain {
+ return Ok(Box::new(FastSiteDomainPatternWeight {
+ term: self.raw_terms[0].clone(),
+ field: self.field,
+ similarity_weight: bm25_weight,
+ }));
+ }
+
+ Ok(Box::new(PatternWeight {
+ similarity_weight: bm25_weight,
+ raw_terms: self.raw_terms.clone(),
+ patterns: self.patterns.clone(),
+ field: self.field,
+ fastfield_reader: self.fastfield_reader.clone(),
+ }))
+ }
+
+ fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a tantivy::Term, bool)) {
+ for term in &self.raw_terms {
+ visitor(term, true);
+ }
+ }
+}
+
+#[derive(Debug)]
+enum SmallPatternPart {
+ Term,
+ Wildcard,
+ Anchor,
+}
+
+/// if pattern is of form Site("|site|") or Domain("|domain|")
+/// we can use the field without tokenization to speed up the query significantly
+fn can_optimize_site_domain(patterns: &[PatternPart], field: Field) -> bool {
+ patterns.len() >= 2
+ && matches!(&patterns[0], PatternPart::Anchor)
+ && matches!(&patterns[patterns.len() - 1], PatternPart::Anchor)
+ && patterns[1..patterns.len() - 1]
+ .iter()
+ .all(|pattern| matches!(pattern, PatternPart::Raw(_)))
+ && (matches!(field, Field::Text(TextField::UrlForSiteOperator))
+ || matches!(field, Field::Text(TextField::Domain)))
+}
diff --git a/core/src/query/pattern_query/scorer.rs b/core/src/query/pattern_query/scorer.rs
new file mode 100644
index 00000000..80e4c2aa
--- /dev/null
+++ b/core/src/query/pattern_query/scorer.rs
@@ -0,0 +1,487 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::sync::Arc;
+
+use tantivy::{
+ fieldnorm::FieldNormReader, postings::SegmentPostings, query::Scorer, DocId, DocSet, Postings,
+ Score, TERMINATED,
+};
+
+use crate::{
+ fastfield_reader::{self, FastFieldReader},
+ query::intersection::Intersection,
+ ranking::bm25::Bm25Weight,
+ schema::FastField,
+};
+
+use super::SmallPatternPart;
+
+pub enum PatternScorer {
+ Normal(NormalPatternScorer),
+ FastSiteDomain(FastSiteDomainPatternScorer),
+ Everything(AllScorer),
+ EmptyField(EmptyFieldScorer),
+}
+
+impl Scorer for PatternScorer {
+ fn score(&mut self) -> Score {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.score(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.score(),
+ PatternScorer::Everything(scorer) => scorer.score(),
+ PatternScorer::EmptyField(scorer) => scorer.score(),
+ }
+ }
+}
+
+impl DocSet for PatternScorer {
+ fn advance(&mut self) -> DocId {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.advance(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.advance(),
+ PatternScorer::Everything(scorer) => scorer.advance(),
+ PatternScorer::EmptyField(scorer) => scorer.advance(),
+ }
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.seek(target),
+ PatternScorer::FastSiteDomain(scorer) => scorer.seek(target),
+ PatternScorer::Everything(scorer) => scorer.seek(target),
+ PatternScorer::EmptyField(scorer) => scorer.seek(target),
+ }
+ }
+
+ fn doc(&self) -> DocId {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.doc(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.doc(),
+ PatternScorer::Everything(scorer) => scorer.doc(),
+ PatternScorer::EmptyField(scorer) => scorer.doc(),
+ }
+ }
+
+ fn size_hint(&self) -> u32 {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.size_hint(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.size_hint(),
+ PatternScorer::Everything(scorer) => scorer.size_hint(),
+ PatternScorer::EmptyField(scorer) => scorer.size_hint(),
+ }
+ }
+}
+
+impl PatternScorer {
+ pub fn term_freq(&self) -> u32 {
+ match self {
+ PatternScorer::Normal(scorer) => scorer.phrase_count(),
+ PatternScorer::FastSiteDomain(scorer) => scorer.term_freq(),
+ PatternScorer::Everything(_) => 0,
+ PatternScorer::EmptyField(_) => 0,
+ }
+ }
+}
+
+pub struct AllScorer {
+ pub doc: DocId,
+ pub max_doc: DocId,
+}
+
+impl DocSet for AllScorer {
+ fn advance(&mut self) -> DocId {
+ if self.doc + 1 >= self.max_doc {
+ self.doc = TERMINATED;
+ return TERMINATED;
+ }
+ self.doc += 1;
+ self.doc
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ if target >= self.max_doc {
+ self.doc = TERMINATED;
+ return TERMINATED;
+ }
+ self.doc = target;
+ self.doc
+ }
+
+ fn doc(&self) -> DocId {
+ self.doc
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.max_doc
+ }
+}
+
+impl Scorer for AllScorer {
+ fn score(&mut self) -> Score {
+ 1.0
+ }
+}
+
+pub struct EmptyFieldScorer {
+ pub segment_reader: Arc<fastfield_reader::SegmentReader>,
+ pub num_tokens_fastfield: FastField,
+ pub all_scorer: AllScorer,
+}
+
+impl EmptyFieldScorer {
+ fn num_tokes(&self, doc: DocId) -> u64 {
+ let s: Option<u64> = self
+ .segment_reader
+ .get_field_reader(&self.num_tokens_fastfield)
+ .get(&doc)
+ .into();
+ s.unwrap_or_default()
+ }
+}
+
+impl DocSet for EmptyFieldScorer {
+ fn advance(&mut self) -> DocId {
+ let mut doc = self.all_scorer.advance();
+
+ while doc != TERMINATED && self.num_tokes(doc) > 0 {
+ doc = self.all_scorer.advance();
+ }
+
+ doc
+ }
+
+ fn doc(&self) -> DocId {
+ self.all_scorer.doc()
+ }
+
+ fn seek(&mut self, target: DocId) -> DocId {
+ self.all_scorer.seek(target);
+
+ if self.doc() != TERMINATED && self.num_tokes(self.all_scorer.doc()) > 0 {
+ self.advance()
+ } else {
+ self.doc()
+ }
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.all_scorer.size_hint()
+ }
+}
+
+impl Scorer for EmptyFieldScorer {
+ fn score(&mut self) -> Score {
+ 1.0
+ }
+}
+
+pub struct FastSiteDomainPatternScorer {
+ pub similarity_weight: Option<Bm25Weight>,
+ pub posting: SegmentPostings,
+ pub fieldnorm_reader: FieldNormReader,
+}
+impl FastSiteDomainPatternScorer {
+ pub fn term_freq(&self) -> u32 {
+ self.posting.term_freq()
+ }
+}
+
+impl Scorer for FastSiteDomainPatternScorer {
+ fn score(&mut self) -> Score {
+ self.similarity_weight
+ .as_ref()
+ .map(|w| {
+ w.score(
+ self.fieldnorm_reader.fieldnorm_id(self.doc()),
+ self.posting.term_freq(),
+ )
+ })
+ .unwrap_or_default()
+ }
+}
+
+impl DocSet for FastSiteDomainPatternScorer {
+ fn advance(&mut self) -> DocId {
+ self.posting.advance()
+ }
+
+ fn doc(&self) -> DocId {
+ self.posting.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.posting.size_hint()
+ }
+}
+
+pub struct NormalPatternScorer {
+ pattern_all_simple: bool,
+ similarity_weight: Option<Bm25Weight>,
+ fieldnorm_reader: FieldNormReader,
+ intersection_docset: Intersection<SegmentPostings>,
+ pattern: Vec<SmallPatternPart>,
+ num_query_terms: usize,
+ left: Vec<u32>,
+ right: Vec<u32>,
+ phrase_count: u32,
+ num_tokens_field: FastField,
+ segment_reader: Arc<fastfield_reader::SegmentReader>,
+}
+
+impl NormalPatternScorer {
+ pub fn new(
+ similarity_weight: Option<Bm25Weight>,
+ term_postings_list: Vec<SegmentPostings>,
+ fieldnorm_reader: FieldNormReader,
+ pattern: Vec<SmallPatternPart>,
+ segment: tantivy::SegmentId,
+ num_tokens_field: FastField,
+ fastfield_reader: FastFieldReader,
+ ) -> Self {
+ let num_query_terms = term_postings_list.len();
+ let segment_reader = fastfield_reader.get_segment(&segment);
+
+ let mut s = Self {
+ pattern_all_simple: pattern.iter().all(|p| matches!(p, SmallPatternPart::Term)),
+ intersection_docset: Intersection::new(term_postings_list),
+ num_query_terms,
+ similarity_weight,
+ fieldnorm_reader,
+ pattern,
+ left: Vec::with_capacity(100),
+ right: Vec::with_capacity(100),
+ phrase_count: 0,
+ num_tokens_field,
+ segment_reader,
+ };
+
+ if !s.pattern_match() {
+ s.advance();
+ }
+
+ s
+ }
+ fn phrase_count(&self) -> u32 {
+ self.phrase_count
+ }
+
+ fn pattern_match(&mut self) -> bool {
+ if self.num_query_terms == 1 && self.pattern_all_simple {
+ // speedup for single term patterns
+ self.phrase_count = self
+ .intersection_docset
+ .docset_mut_specialized(0)
+ .term_freq();
+ return self.phrase_count > 0;
+ }
+
+ self.phrase_count = self.perform_pattern_match() as u32;
+
+ self.phrase_count > 0
+ }
+
+ fn perform_pattern_match(&mut self) -> usize {
+ if self.intersection_docset.doc() == TERMINATED {
+ return 0;
+ }
+
+ {
+ self.intersection_docset
+ .docset_mut_specialized(0)
+ .positions(&mut self.left);
+ }
+
+ let mut intersection_len = self.left.len();
+ let mut out = Vec::new();
+
+ let mut current_right_term = 0;
+ let mut slop = 1;
+ let num_tokens_doc: Option<u64> = self
+ .segment_reader
+ .get_field_reader(&self.num_tokens_field)
+ .get(&self.doc())
+ .into();
+ let num_tokens_doc = num_tokens_doc.unwrap();
+
+ for (i, pattern_part) in self.pattern.iter().enumerate() {
+ match pattern_part {
+ SmallPatternPart::Term => {
+ if current_right_term == 0 {
+ current_right_term = 1;
+ continue;
+ }
+
+ {
+ self.intersection_docset
+ .docset_mut_specialized(current_right_term)
+ .positions(&mut self.right);
+ }
+ out.resize(self.left.len().max(self.right.len()), 0);
+ intersection_len =
+ intersection_with_slop(&self.left[..], &self.right[..], &mut out, slop);
+
+ slop = 1;
+
+ if intersection_len == 0 {
+ return 0;
+ }
+
+ self.left = out[..intersection_len].to_vec();
+ out = Vec::new();
+ current_right_term += 1;
+ }
+ SmallPatternPart::Wildcard => {
+ slop = u32::MAX;
+ }
+ SmallPatternPart::Anchor if i == 0 => {
+ if let Some(pos) = self.left.first() {
+ if *pos != 0 {
+ return 0;
+ }
+ }
+ }
+ SmallPatternPart::Anchor if i == self.pattern.len() - 1 => {
+ {
+ self.intersection_docset
+ .docset_mut_specialized(self.num_query_terms - 1)
+ .positions(&mut self.right);
+ }
+
+ if let Some(pos) = self.right.last() {
+ if *pos != (num_tokens_doc - 1) as u32 {
+ return 0;
+ }
+ }
+ }
+ SmallPatternPart::Anchor => {}
+ }
+ }
+
+ intersection_len
+ }
+}
+
+impl Scorer for NormalPatternScorer {
+ fn score(&mut self) -> Score {
+ self.similarity_weight
+ .as_ref()
+ .map(|scorer| {
+ let doc = self.doc();
+ let fieldnorm_id = self.fieldnorm_reader.fieldnorm_id(doc);
+ scorer.score(fieldnorm_id, self.phrase_count())
+ })
+ .unwrap_or_default()
+ }
+}
+
+impl DocSet for NormalPatternScorer {
+ fn advance(&mut self) -> DocId {
+ loop {
+ let doc = self.intersection_docset.advance();
+ if doc == TERMINATED || self.pattern_match() {
+ return doc;
+ }
+ }
+ }
+
+ fn doc(&self) -> tantivy::DocId {
+ self.intersection_docset.doc()
+ }
+
+ fn size_hint(&self) -> u32 {
+ self.intersection_docset.size_hint()
+ }
+}
+
+/// Intersect twos sorted arrays `left` and `right` and outputs the
+/// resulting array in `out`. The positions in out are all positions from right where
+/// the distance to left_pos <= slop
+///
+/// Returns the length of the intersection
+fn intersection_with_slop(left: &[u32], right: &[u32], out: &mut [u32], slop: u32) -> usize {
+ let mut left_index = 0;
+ let mut right_index = 0;
+ let mut count = 0;
+ let left_len = left.len();
+ let right_len = right.len();
+ while left_index < left_len && right_index < right_len {
+ let left_val = left[left_index];
+ let right_val = right[right_index];
+
+ // The three conditions are:
+ // left_val < right_slop -> left index increment.
+ // right_slop <= left_val <= right -> find the best match.
+ // left_val > right -> right index increment.
+ let right_slop = if right_val >= slop {
+ right_val - slop
+ } else {
+ 0
+ };
+
+ if left_val < right_slop {
+ left_index += 1;
+ } else if right_slop <= left_val && left_val <= right_val {
+ while left_index + 1 < left_len {
+ // there could be a better match
+ let next_left_val = left[left_index + 1];
+ if next_left_val > right_val {
+ // the next value is outside the range, so current one is the best.
+ break;
+ }
+ // the next value is better.
+ left_index += 1;
+ }
+ // store the match in left.
+ out[count] = right_val;
+ count += 1;
+ right_index += 1;
+ } else if left_val > right_val {
+ right_index += 1;
+ }
+ }
+ count
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn aux_intersection(left: &[u32], right: &[u32], expected: &[u32], slop: u32) {
+ let mut out = vec![0; left.len().max(right.len())];
+
+ let intersection_size = intersection_with_slop(left, right, &mut out, slop);
+
+ assert_eq!(&out[..intersection_size], expected);
+ }
+
+ #[test]
+ fn test_intersection_with_slop() {
+ aux_intersection(&[20, 75, 77], &[18, 21, 60], &[21, 60], u32::MAX);
+ aux_intersection(&[21, 60], &[50, 61], &[61], 1);
+
+ aux_intersection(&[1, 2, 3], &[], &[], 1);
+ aux_intersection(&[], &[1, 2, 3], &[], 1);
+
+ aux_intersection(&[1, 2, 3], &[4, 5, 6], &[4], 1);
+ aux_intersection(&[1, 2, 3], &[4, 5, 6], &[4, 5, 6], u32::MAX);
+
+ aux_intersection(&[20, 75, 77], &[18, 21, 60], &[21, 60], u32::MAX);
+ aux_intersection(&[21, 60], &[61, 62], &[61, 62], 2);
+
+ aux_intersection(&[60], &[61, 62], &[61, 62], 2);
+ }
+}
diff --git a/core/src/query/pattern_query/weight.rs b/core/src/query/pattern_query/weight.rs
new file mode 100644
index 00000000..722da0a5
--- /dev/null
+++ b/core/src/query/pattern_query/weight.rs
@@ -0,0 +1,304 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use optics::PatternPart;
+use tantivy::{
+ fieldnorm::FieldNormReader,
+ query::{EmptyScorer, Explanation, Scorer},
+ schema::IndexRecordOption,
+ DocSet, TantivyError,
+};
+
+use crate::{
+ fastfield_reader::FastFieldReader,
+ ranking::bm25::Bm25Weight,
+ schema::{FastField, Field, TextField, ALL_FIELDS},
+};
+
+use super::scorer::{
+ AllScorer, EmptyFieldScorer, FastSiteDomainPatternScorer, NormalPatternScorer, PatternScorer,
+};
+use super::SmallPatternPart;
+
+pub struct FastSiteDomainPatternWeight {
+ pub term: tantivy::Term,
+ pub field: tantivy::schema::Field,
+ pub similarity_weight: Option<Bm25Weight>,
+}
+
+impl FastSiteDomainPatternWeight {
+ fn fieldnorm_reader(
+ &self,
+ reader: &tantivy::SegmentReader,
+ ) -> tantivy::Result<FieldNormReader> {
+ if self.similarity_weight.is_some() {
+ if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {
+ return Ok(fieldnorm_reader);
+ }
+ }
+ Ok(FieldNormReader::constant(reader.max_doc(), 1))
+ }
+
+ fn pattern_scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ boost: tantivy::Score,
+ ) -> tantivy::Result<Option<FastSiteDomainPatternScorer>> {
+ let similarity_weight = self
+ .similarity_weight
+ .as_ref()
+ .map(|weight| weight.boost_by(boost));
+
+ let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+
+ let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {
+ Field::Text(TextField::UrlForSiteOperator) => Field::Text(TextField::SiteNoTokenizer),
+ Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),
+ _ => unreachable!(),
+ };
+
+ let tv_field = reader
+ .schema()
+ .get_field(field_no_tokenizer.name())
+ .unwrap();
+
+ let opt = match field_no_tokenizer {
+ Field::Text(t) => t.index_option(),
+ Field::Fast(_) => unreachable!(),
+ };
+
+ match reader
+ .inverted_index(tv_field)?
+ .read_postings(&self.term, opt)?
+ {
+ Some(posting) => Ok(Some(FastSiteDomainPatternScorer {
+ similarity_weight,
+ posting,
+ fieldnorm_reader,
+ })),
+ None => Ok(None),
+ }
+ }
+}
+
+impl tantivy::query::Weight for FastSiteDomainPatternWeight {
+ fn scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ boost: tantivy::Score,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
+ if let Some(scorer) = self.pattern_scorer(reader, boost)? {
+ Ok(Box::new(PatternScorer::FastSiteDomain(scorer)))
+ } else {
+ Ok(Box::new(EmptyScorer))
+ }
+ }
+
+ fn explain(
+ &self,
+ reader: &tantivy::SegmentReader,
+ doc: tantivy::DocId,
+ ) -> tantivy::Result<tantivy::query::Explanation> {
+ let scorer_opt = self.pattern_scorer(reader, 1.0)?;
+ if scorer_opt.is_none() {
+ return Err(TantivyError::InvalidArgument(format!(
+ "Document #({doc}) does not match (empty scorer)"
+ )));
+ }
+ let mut scorer = scorer_opt.unwrap();
+ if scorer.seek(doc) != doc {
+ return Err(TantivyError::InvalidArgument(format!(
+ "Document #({doc}) does not match"
+ )));
+ }
+ let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+ let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+ let term_freq = scorer.term_freq();
+ let mut explanation = Explanation::new("Pattern Scorer", scorer.score());
+ explanation.add_detail(
+ self.similarity_weight
+ .as_ref()
+ .unwrap()
+ .explain(fieldnorm_id, term_freq),
+ );
+ Ok(explanation)
+ }
+}
+
+pub struct PatternWeight {
+ pub similarity_weight: Option<Bm25Weight>,
+ pub patterns: Vec<PatternPart>,
+ pub raw_terms: Vec<tantivy::Term>,
+ pub field: tantivy::schema::Field,
+ pub fastfield_reader: FastFieldReader,
+}
+
+impl PatternWeight {
+ fn fieldnorm_reader(
+ &self,
+ reader: &tantivy::SegmentReader,
+ ) -> tantivy::Result<FieldNormReader> {
+ if self.similarity_weight.is_some() {
+ if let Some(fieldnorm_reader) = reader.fieldnorms_readers().get_field(self.field)? {
+ return Ok(fieldnorm_reader);
+ }
+ }
+ Ok(FieldNormReader::constant(reader.max_doc(), 1))
+ }
+
+ pub(crate) fn pattern_scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ boost: tantivy::Score,
+ ) -> tantivy::Result<Option<PatternScorer>> {
+ if self.patterns.is_empty() {
+ return Ok(None);
+ }
+
+ let num_tokens_fastfield = match &ALL_FIELDS[self.field.field_id() as usize] {
+ Field::Text(TextField::Title) => Ok(FastField::NumTitleTokens),
+ Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),
+ Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),
+ Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
+ Field::Text(TextField::UrlForSiteOperator) => {
+ Ok(FastField::NumUrlForSiteOperatorTokens)
+ }
+ Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
+ Field::Text(TextField::MicroformatTags) => Ok(FastField::NumMicroformatTagsTokens),
+ Field::Text(TextField::FlattenedSchemaOrgJson) => {
+ Ok(FastField::NumFlattenedSchemaTokens)
+ }
+ field => Err(TantivyError::InvalidArgument(format!(
+ "{} is not supported in pattern query",
+ field.name()
+ ))),
+ }?;
+
+ // "*" matches everything
+ if self.raw_terms.is_empty()
+ && self
+ .patterns
+ .iter()
+ .any(|p| matches!(p, PatternPart::Wildcard))
+ {
+ return Ok(Some(PatternScorer::Everything(AllScorer {
+ doc: 0,
+ max_doc: reader.max_doc(),
+ })));
+ }
+
+ // "||" and "|" matches empty string
+
+ if self.raw_terms.is_empty()
+ && self
+ .patterns
+ .iter()
+ .all(|p| matches!(p, PatternPart::Anchor))
+ {
+ return Ok(Some(PatternScorer::EmptyField(EmptyFieldScorer {
+ num_tokens_fastfield,
+ segment_reader: self.fastfield_reader.get_segment(&reader.segment_id()),
+ all_scorer: AllScorer {
+ doc: 0,
+ max_doc: reader.max_doc(),
+ },
+ })));
+ }
+
+ let similarity_weight = self
+ .similarity_weight
+ .as_ref()
+ .map(|weight| weight.boost_by(boost));
+
+ let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+
+ let mut term_postings_list = Vec::with_capacity(self.raw_terms.len());
+ for term in &self.raw_terms {
+ if let Some(postings) = reader
+ .inverted_index(term.field())?
+ .read_postings(term, IndexRecordOption::WithFreqsAndPositions)?
+ {
+ term_postings_list.push(postings);
+ } else {
+ return Ok(None);
+ }
+ }
+
+ let small_patterns = self
+ .patterns
+ .iter()
+ .map(|pattern| match pattern {
+ PatternPart::Raw(_) => SmallPatternPart::Term,
+ PatternPart::Wildcard => SmallPatternPart::Wildcard,
+ PatternPart::Anchor => SmallPatternPart::Anchor,
+ })
+ .collect();
+
+ Ok(Some(PatternScorer::Normal(NormalPatternScorer::new(
+ similarity_weight,
+ term_postings_list,
+ fieldnorm_reader,
+ small_patterns,
+ reader.segment_id(),
+ num_tokens_fastfield,
+ self.fastfield_reader.clone(),
+ ))))
+ }
+}
+
+impl tantivy::query::Weight for PatternWeight {
+ fn scorer(
+ &self,
+ reader: &tantivy::SegmentReader,
+ boost: tantivy::Score,
+ ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
+ if let Some(scorer) = self.pattern_scorer(reader, boost)? {
+ Ok(Box::new(scorer))
+ } else {
+ Ok(Box::new(EmptyScorer))
+ }
+ }
+
+ fn explain(
+ &self,
+ reader: &tantivy::SegmentReader,
+ doc: tantivy::DocId,
+ ) -> tantivy::Result<tantivy::query::Explanation> {
+ let scorer_opt = self.pattern_scorer(reader, 1.0)?;
+ if scorer_opt.is_none() {
+ return Err(TantivyError::InvalidArgument(format!(
+ "Document #({doc}) does not match (empty scorer)"
+ )));
+ }
+ let mut scorer = scorer_opt.unwrap();
+ if scorer.seek(doc) != doc {
+ return Err(TantivyError::InvalidArgument(format!(
+ "Document #({doc}) does not match"
+ )));
+ }
+ let fieldnorm_reader = self.fieldnorm_reader(reader)?;
+ let fieldnorm_id = fieldnorm_reader.fieldnorm_id(doc);
+ let term_freq = scorer.term_freq();
+ let mut explanation = Explanation::new("Pattern Scorer", scorer.score());
+ explanation.add_detail(
+ self.similarity_weight
+ .as_ref()
+ .unwrap()
+ .explain(fieldnorm_id, term_freq),
+ );
+ Ok(explanation)
+ }
+}
diff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rs
index d5106209..23193671 100644
--- a/core/src/search_prettifier/stack_overflow.rs
+++ b/core/src/search_prettifier/stack_overflow.rs
@@ -134,6 +134,13 @@ fn schema_item_to_stackoverflow_answer(
}
}
+/// Limit the number of characters in a snippet
+/// to be less than `limit` characters. At least a single
+/// passage is returned, so the total number of characters
+/// in the returned passages may be more than `limit`.
+///
+/// This is useful for limiting the number of characters
+/// we send to the frontend.
fn limit_chars(passages: &[CodeOrText], limit: usize) -> Vec<CodeOrText> {
let mut res = Vec::new();
let mut taken_chars = 0;
@@ -180,20 +187,9 @@ pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
let mut answers = Vec::new();
- if let Some(ans) = item
- .properties
- .get("acceptedAnswer")
- .cloned()
- .and_then(|ans| ans.one())
- .and_then(|prop| prop.try_into_item())
- .and_then(|item| {
- schema_item_to_stackoverflow_answer(
- item,
- Url::parse(&webpage.url).unwrap(),
- true,
- )
- })
- {
+ let url = Url::parse(&webpage.url).unwrap();
+
+ if let Some(ans) = get_accepted_answer(&item, url) {
answers.push(ans);
}
@@ -251,15 +247,21 @@ pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: Url) -> Result<D
.and_then(|prop| prop.try_into_string())
.ok_or(Error::InvalidStackoverflowSchema)?;
- item.properties
- .get("acceptedAnswer")
- .cloned()
- .and_then(|ans| ans.one())
- .and_then(|prop| prop.try_into_item())
- .and_then(|item| schema_item_to_stackoverflow_answer(item, url, true))
+ get_accepted_answer(&item, url)
.map(|answer| DisplayedSidebar::StackOverflow { title, answer })
.ok_or(Error::InvalidStackoverflowSchema.into())
} else {
Err(Error::InvalidStackoverflowSchema.into())
}
}
+
+fn get_accepted_answer(item: &Item, url: Url) -> Option<StackOverflowAnswer> {
+ let item = item
+ .properties
+ .get("acceptedAnswer")
+ .cloned()?
+ .one()?
+ .try_into_item()?;
+
+ schema_item_to_stackoverflow_answer(item, url, true)
+}
diff --git a/core/src/webgraph/centrality/derived_harmonic.rs b/core/src/webgraph/centrality/derived_harmonic.rs
index 74d8db5d..94fedb4f 100644
--- a/core/src/webgraph/centrality/derived_harmonic.rs
+++ b/core/src/webgraph/centrality/derived_harmonic.rs
@@ -94,8 +94,10 @@ impl DerivedCentrality {
let non_normalized = RocksDbStore::open(output.as_ref().join("non_normalized"));
let norms: Mutex<BTreeMap<NodeID, f64>> = Mutex::new(BTreeMap::new());
+ let pb = indicatif::ProgressBar::new(num_nodes as u64);
- page_graph.node_ids().par_bridge().for_each(|(node, id)| {
+ page_graph.node_ids().for_each(|(node, id)| {
+ pb.inc(1);
if has_outgoing.contains(&id.bit_64()) {
let host_node = node.clone().into_host().id();
@@ -124,6 +126,8 @@ impl DerivedCentrality {
}
});
+ pb.finish_and_clear();
+
let norms = norms.into_inner().unwrap();
let db = RocksDbStore::open(output.as_ref());
diff --git a/core/src/webpage/html/into_tantivy.rs b/core/src/webpage/html/into_tantivy.rs
new file mode 100644
index 00000000..0ef4f5e7
--- /dev/null
+++ b/core/src/webpage/html/into_tantivy.rs
@@ -0,0 +1,568 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ ceil_char_boundary,
+ prehashed::hash,
+ schema::{FastField, TextField},
+ simhash, split_u128, tokenizer,
+ webpage::url_ext::UrlExt,
+ Error, Result,
+};
+use tantivy::{
+ tokenizer::{PreTokenizedString, Tokenizer},
+ TantivyDocument,
+};
+use whatlang::Lang;
+
+use super::{find_recipe_first_ingredient_tag_id, schema_org, Html};
+
+use crate::schema::{Field, ALL_FIELDS, FLOAT_SCALING};
+
+impl Html {
+ fn pretokenize_title(&self) -> Result<PreTokenizedString> {
+ let title = self.title();
+
+ if title.is_none() {
+ return Err(Error::EmptyField("title").into());
+ }
+ let title = title.unwrap();
+
+ Ok(self.pretokenize_string(title))
+ }
+
+ fn pretokenize_all_text(&self) -> Result<PreTokenizedString> {
+ let all_text = self.all_text();
+
+ if all_text.is_none() {
+ return Err(Error::EmptyField("all body").into());
+ }
+ let all_text = all_text.unwrap();
+
+ Ok(self.pretokenize_string(all_text))
+ }
+
+ fn pretokenize_clean_text(&self) -> PreTokenizedString {
+ let clean_text = self.clean_text().cloned().unwrap_or_default();
+ self.pretokenize_string(clean_text)
+ }
+
+ fn pretokenize_url(&self) -> PreTokenizedString {
+ let url = self.url().to_string();
+ self.pretokenize_string(url)
+ }
+
+ fn pretokenize_domain(&self) -> PreTokenizedString {
+ let domain = self.url().root_domain().unwrap_or_default().to_string();
+
+ self.pretokenize_string(domain)
+ }
+
+ fn pretokenize_site(&self) -> PreTokenizedString {
+ let site = self.url().normalized_host().unwrap_or_default().to_string();
+
+ self.pretokenize_string(site)
+ }
+
+ fn pretokenize_description(&self) -> PreTokenizedString {
+ let text = self.description().unwrap_or_default();
+
+ self.pretokenize_string(text)
+ }
+
+ fn pretokenize_microformats(&self) -> PreTokenizedString {
+ let mut text = String::new();
+
+ for microformat in self.microformats().iter() {
+ text.push_str(microformat.as_str());
+ text.push(' ');
+ }
+
+ self.pretokenize_string(text)
+ }
+
+ fn pretokenize_string(&self, text: String) -> PreTokenizedString {
+ self.pretokenize_string_with(text, tokenizer::Tokenizer::default())
+ }
+
+ fn pretokenize_string_with(
+ &self,
+ text: String,
+ tokenizer: tokenizer::Tokenizer,
+ ) -> PreTokenizedString {
+ let mut tokenizer = tokenizer;
+
+ let mut tokens = Vec::new();
+
+ {
+ let mut stream = tokenizer.token_stream(&text);
+ while let Some(token) = stream.next() {
+ tokens.push(token.clone());
+ }
+ }
+
+ PreTokenizedString { text, tokens }
+ }
+ pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
+ let mut doc = TantivyDocument::new();
+
+ let title = self.pretokenize_title()?;
+ let all_text = self.pretokenize_all_text()?;
+ let clean_text = self.pretokenize_clean_text();
+ let url = self.pretokenize_url();
+ let domain = self.pretokenize_domain();
+ let site = self.pretokenize_site();
+ let description = self.pretokenize_description();
+ let microformats = self.pretokenize_microformats();
+ let url_for_site_operator = self.pretokenize_string_with(
+ self.url().to_string(),
+ tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),
+ );
+
+ let domain_name = self
+ .url()
+ .root_domain()
+ .unwrap_or_default()
+ .find('.')
+ .map(|index| {
+ &domain.text[..ceil_char_boundary(&domain.text, index).min(domain.text.len())]
+ })
+ .unwrap_or_default()
+ .to_string();
+
+ let schemas: Vec<_> = self.schema_org();
+ let first_ingredient_tag_id =
+ find_recipe_first_ingredient_tag_id(&schemas, &self.root).unwrap_or_default();
+
+ let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
+
+ let pretokenized_schema_json = match schema_org::flattened_json(schemas) {
+ Ok(mut f) => {
+ let mut tokens = Vec::new();
+
+ {
+ let mut stream = f.token_stream();
+
+ while let Some(token) = stream.next() {
+ tokens.push(token.clone());
+ }
+ }
+
+ PreTokenizedString {
+ text: f.text().to_string(),
+ tokens,
+ }
+ }
+ Err(_) => PreTokenizedString {
+ text: String::new(),
+ tokens: Vec::new(),
+ },
+ };
+
+ let site_hash = split_u128(hash(self.url().normalized_host().unwrap_or_default()).0);
+
+ let mut url_without_query = self.url().clone();
+ url_without_query.set_query(None);
+
+ let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);
+ let url_hash = split_u128(hash(self.url().as_str()).0);
+
+ let tld = self.url().tld().unwrap_or_default();
+ let url_without_tld = self
+ .url()
+ .host_str()
+ .unwrap_or_default()
+ .trim_end_matches(&tld)
+ .to_string()
+ + "/"
+ + self.url().path()
+ + "?"
+ + self.url().query().unwrap_or_default();
+
+ let url_without_tld_hash = split_u128(hash(url_without_tld).0);
+
+ let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);
+ let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
+
+ for field in &ALL_FIELDS {
+ let tantivy_field = schema
+ .get_field(field.name())
+ .unwrap_or_else(|_| panic!("Unknown field: {}", field.name()));
+
+ match field {
+ Field::Text(TextField::Title) => {
+ doc.add_pre_tokenized_text(tantivy_field, title.clone())
+ }
+ Field::Text(TextField::StemmedTitle) => {
+ let mut tokens = title.tokens.clone();
+ stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
+
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: title.text.clone(),
+ tokens,
+ },
+ );
+ }
+ Field::Text(TextField::CleanBody) => {
+ doc.add_pre_tokenized_text(tantivy_field, clean_text.clone())
+ }
+ Field::Text(TextField::StemmedCleanBody) => {
+ let mut tokens = clean_text.tokens.clone();
+ stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
+
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: clean_text.text.clone(),
+ tokens,
+ },
+ );
+ }
+ Field::Text(TextField::CleanBodyBigrams) => {
+ doc.add_text(
+ tantivy_field,
+ self.clean_text().cloned().unwrap_or_default(),
+ );
+ }
+ Field::Text(TextField::CleanBodyTrigrams) => {
+ doc.add_text(
+ tantivy_field,
+ self.clean_text().cloned().unwrap_or_default(),
+ );
+ }
+ Field::Text(TextField::TitleBigrams) => {
+ doc.add_text(tantivy_field, title.text.clone());
+ }
+ Field::Text(TextField::TitleTrigrams) => {
+ doc.add_text(tantivy_field, title.text.clone());
+ }
+ Field::Text(TextField::Description) => {
+ doc.add_pre_tokenized_text(tantivy_field, description.clone());
+ }
+ Field::Text(TextField::Url) => {
+ doc.add_pre_tokenized_text(tantivy_field, url.clone())
+ }
+ Field::Text(TextField::UrlForSiteOperator) => {
+ doc.add_pre_tokenized_text(tantivy_field, url_for_site_operator.clone())
+ }
+ Field::Text(TextField::UrlNoTokenizer) => {
+ let url = self.url().to_string();
+
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: url.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: url.len(),
+ position: 0,
+ text: url,
+ position_length: 1,
+ }],
+ },
+ );
+ }
+ Field::Text(TextField::SiteWithout) => {
+ doc.add_pre_tokenized_text(tantivy_field, site.clone())
+ }
+ Field::Text(TextField::Domain) => {
+ doc.add_pre_tokenized_text(tantivy_field, domain.clone())
+ }
+ Field::Text(TextField::SiteNoTokenizer) => doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: site.text.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: site.text.len(),
+ position: 0,
+ text: site.text.clone(),
+ position_length: 1,
+ }],
+ },
+ ),
+ Field::Text(TextField::SiteIfHomepageNoTokenizer) => {
+ if self.is_homepage() {
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: site.text.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: site.text.len(),
+ position: 0,
+ text: site.text.clone(),
+ position_length: 1,
+ }],
+ },
+ )
+ } else {
+ doc.add_text(tantivy_field, "");
+ }
+ }
+ Field::Text(TextField::DomainNoTokenizer) => doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: domain.text.clone(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain.text.len(),
+ position: 0,
+ text: domain.text.clone(),
+ position_length: 1,
+ }],
+ },
+ ),
+ Field::Text(TextField::TitleIfHomepage) => {
+ if self.is_homepage() {
+ doc.add_pre_tokenized_text(tantivy_field, title.clone());
+ } else {
+ doc.add_text(tantivy_field, "");
+ }
+ }
+ Field::Text(TextField::DomainIfHomepage) => {
+ if self.is_homepage() {
+ doc.add_text(tantivy_field, domain.text.clone());
+ } else {
+ doc.add_text(tantivy_field, "");
+ }
+ }
+ Field::Text(TextField::DomainNameNoTokenizer) => {
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: domain_name.to_string(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain_name.len(),
+ position: 0,
+ text: domain_name.to_string(),
+ position_length: 1,
+ }],
+ },
+ );
+ }
+ Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
+ if self.is_homepage() {
+ doc.add_pre_tokenized_text(
+ tantivy_field,
+ PreTokenizedString {
+ text: domain_name.to_string(),
+ tokens: vec![tantivy::tokenizer::Token {
+ offset_from: 0,
+ offset_to: domain_name.len(),
+ position: 0,
+ text: domain_name.to_string(),
+ position_length: 1,
+ }],
+ },
+ );
+ } else {
+ doc.add_text(tantivy_field, "");
+ }
+ }
+ Field::Text(TextField::DomainIfHomepageNoTokenizer) => {
+ if self.is_homepage() {
+ doc.add_pre_tokenized_text(tantivy_field, domain.clone());
+ } else {
+ doc.add_text(tantivy_field, "");
+ }
+ }
+ Field::Text(TextField::AllBody) => {
+ doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
+ }
+ Field::Text(TextField::RecipeFirstIngredientTagId) => {
+ doc.add_text(tantivy_field, first_ingredient_tag_id.clone());
+ }
+ Field::Text(TextField::SchemaOrgJson) => {
+ doc.add_text(tantivy_field, schema_json.clone());
+ }
+ Field::Text(TextField::FlattenedSchemaOrgJson) => {
+ doc.add_pre_tokenized_text(tantivy_field, pretokenized_schema_json.clone());
+ }
+ Field::Text(TextField::MicroformatTags) => {
+ doc.add_pre_tokenized_text(tantivy_field, microformats.clone());
+ }
+ Field::Fast(FastField::IsHomepage) => {
+ doc.add_u64(tantivy_field, (self.is_homepage()).into());
+ }
+ Field::Fast(FastField::LastUpdated) => doc.add_u64(
+ tantivy_field,
+ self.updated_time()
+ .map_or(0, |time| time.timestamp().max(0) as u64),
+ ),
+ Field::Fast(FastField::TrackerScore) => {
+ doc.add_u64(tantivy_field, self.trackers().len() as u64)
+ }
+ Field::Fast(FastField::NumUrlTokens) => {
+ doc.add_u64(tantivy_field, url.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumMicroformatTagsTokens) => {
+ doc.add_u64(tantivy_field, microformats.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumTitleTokens) => {
+ doc.add_u64(tantivy_field, title.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumCleanBodyTokens) => {
+ doc.add_u64(tantivy_field, clean_text.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumDescriptionTokens) => {
+ doc.add_u64(tantivy_field, description.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {
+ doc.add_u64(tantivy_field, url_for_site_operator.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumDomainTokens) => {
+ doc.add_u64(tantivy_field, domain.tokens.len() as u64)
+ }
+ Field::Fast(FastField::NumFlattenedSchemaTokens) => {
+ doc.add_u64(tantivy_field, pretokenized_schema_json.tokens.len() as u64)
+ }
+ Field::Fast(FastField::SiteHash1) => {
+ doc.add_u64(tantivy_field, site_hash[0]);
+ }
+ Field::Fast(FastField::SiteHash2) => {
+ doc.add_u64(tantivy_field, site_hash[1]);
+ }
+ Field::Fast(FastField::UrlWithoutQueryHash1) => {
+ doc.add_u64(tantivy_field, url_without_query_hash[0]);
+ }
+ Field::Fast(FastField::UrlWithoutQueryHash2) => {
+ doc.add_u64(tantivy_field, url_without_query_hash[1]);
+ }
+ Field::Fast(FastField::UrlHash1) => {
+ doc.add_u64(tantivy_field, url_hash[0]);
+ }
+ Field::Fast(FastField::UrlHash2) => {
+ doc.add_u64(tantivy_field, url_hash[1]);
+ }
+ Field::Fast(FastField::UrlWithoutTldHash1) => {
+ doc.add_u64(tantivy_field, url_without_tld_hash[0]);
+ }
+ Field::Fast(FastField::UrlWithoutTldHash2) => {
+ doc.add_u64(tantivy_field, url_without_tld_hash[1]);
+ }
+ Field::Fast(FastField::DomainHash1) => {
+ doc.add_u64(tantivy_field, domain_hash[0]);
+ }
+ Field::Fast(FastField::DomainHash2) => {
+ doc.add_u64(tantivy_field, domain_hash[1]);
+ }
+ Field::Fast(FastField::TitleHash1) => {
+ doc.add_u64(tantivy_field, title_hash[0]);
+ }
+ Field::Fast(FastField::TitleHash2) => {
+ doc.add_u64(tantivy_field, title_hash[1]);
+ }
+ Field::Fast(FastField::SimHash) => {
+ let hash = if !clean_text.text.is_empty() {
+ simhash::hash(&clean_text.text)
+ } else {
+ 0
+ };
+ doc.add_u64(tantivy_field, hash);
+ }
+ Field::Fast(FastField::NumPathAndQuerySlashes) => {
+ let num_slashes = self
+ .url()
+ .path_segments()
+ .map(|segments| segments.count())
+ .unwrap_or(0);
+
+ doc.add_u64(tantivy_field, num_slashes as u64);
+ }
+ Field::Fast(FastField::NumPathAndQueryDigits) => {
+ let num_digits = self
+ .url()
+ .path()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count()
+ + self
+ .url()
+ .query()
+ .unwrap_or_default()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count();
+
+ doc.add_u64(tantivy_field, num_digits as u64);
+ }
+ Field::Fast(FastField::LikelyHasAds) => {
+ doc.add_u64(tantivy_field, self.likely_has_ads() as u64);
+ }
+ Field::Fast(FastField::LikelyHasPaywall) => {
+ doc.add_u64(tantivy_field, self.likely_has_paywall() as u64);
+ }
+ Field::Fast(FastField::LinkDensity) => {
+ doc.add_u64(
+ tantivy_field,
+ (self.link_density() * FLOAT_SCALING as f64) as u64,
+ );
+ }
+ Field::Text(TextField::BacklinkText)
+ | Field::Text(TextField::SafetyClassification)
+ | Field::Text(TextField::InsertionTimestamp)
+ | Field::Fast(FastField::HostCentrality)
+ | Field::Fast(FastField::PageCentrality)
+ | Field::Fast(FastField::FetchTimeMs)
+ | Field::Fast(FastField::PreComputedScore)
+ | Field::Fast(FastField::Region)
+ | Field::Fast(FastField::HostNodeID1)
+ | Field::Fast(FastField::HostNodeID2)
+ | Field::Text(TextField::DmozDescription) => {}
+ }
+ }
+
+ Ok(doc)
+ }
+}
+
+fn stemmer_from_lang(lang: &Lang) -> rust_stemmers::Stemmer {
+ match lang {
+ Lang::Ara => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Arabic),
+ Lang::Dan => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Danish),
+ Lang::Nld => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Dutch),
+ Lang::Fin => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Finnish),
+ Lang::Fra => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::French),
+ Lang::Deu => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::German),
+ Lang::Ell => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Greek),
+ Lang::Hun => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Hungarian),
+ Lang::Ita => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Italian),
+ Lang::Por => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Portuguese),
+ Lang::Ron => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Romanian),
+ Lang::Rus => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Russian),
+ Lang::Spa => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Spanish),
+ Lang::Swe => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Swedish),
+ Lang::Tam => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Tamil),
+ Lang::Tur => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Turkish),
+ _ => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::English),
+ }
+}
+
+fn stem_tokens(tokens: &mut [tantivy::tokenizer::Token], lang: Lang) {
+ let stemmer = stemmer_from_lang(&lang);
+ for token in tokens {
+ // TODO remove allocation
+ if let Ok(stemmed_str) = std::panic::catch_unwind(|| stemmer.stem(&token.text).into_owned())
+ {
+ token.text.clear();
+ token.text.push_str(&stemmed_str);
+ }
+ }
+}
diff --git a/core/src/webpage/html/links.rs b/core/src/webpage/html/links.rs
new file mode 100644
index 00000000..44ed75e1
--- /dev/null
+++ b/core/src/webpage/html/links.rs
@@ -0,0 +1,463 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use kuchiki::iter::NodeEdge;
+use url::Url;
+
+use crate::webpage::{url_ext::UrlExt, Link};
+
+use super::Html;
+
+#[derive(PartialEq, Eq, Debug)]
+pub struct FaviconLink {
+ pub link: Url,
+ width: Option<u32>,
+ height: Option<u32>,
+ image_type: Option<String>,
+}
+
+#[derive(PartialEq, Eq, Debug)]
+pub struct ImageLink {
+ pub url: Url,
+ pub title: Option<String>,
+ pub description: Option<String>,
+}
+
+impl Html {
+ pub fn favicon(&self) -> Option<FaviconLink> {
+ for node in self.root.select("link").unwrap() {
+ if !matches!(node.attributes.borrow().get("rel"), Some("icon")) {
+ continue;
+ }
+
+ if let Some(link) = node.attributes.borrow().get("href") {
+ let (width, height) = match node.attributes.borrow().get("sizes") {
+ Some(size) => {
+ if let Some((width, height)) = size.split_once('x') {
+ (width.parse().ok(), height.parse().ok())
+ } else {
+ (None, None)
+ }
+ }
+ _ => (None, None),
+ };
+
+ let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());
+ let link = Url::parse(link).or_else(|_| self.url().join(link)).ok()?;
+
+ let favicon = FaviconLink {
+ link,
+ width,
+ height,
+ image_type,
+ };
+
+ return Some(favicon);
+ }
+ }
+
+ None
+ }
+
+ fn og_image(&self) -> Option<ImageLink> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("og:image")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| {
+ metadata
+ .get("content")
+ .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
+ })
+ .map(|url| ImageLink {
+ url,
+ title: self.og_title(),
+ description: self.description(),
+ })
+ }
+
+ pub fn primary_image(&self) -> Option<ImageLink> {
+ self.og_image().or_else(|| {
+ self.schema_org_images()
+ .first()
+ .cloned()
+ .map(|url| ImageLink {
+ url,
+ title: self.og_title(),
+ description: self.description(),
+ })
+ })
+ }
+
+ pub fn link_density(&self) -> f64 {
+ (1.0 + self.anchor_links().len() as f64)
+ / (1.0
+ + self
+ .clean_text
+ .as_ref()
+ .map(|s| s.len())
+ .unwrap_or_default() as f64)
+ }
+
+ pub fn anchor_links(&self) -> Vec<Link> {
+ if self.is_no_follow() {
+ return Vec::new();
+ }
+
+ let mut links = Vec::new();
+ let mut open_links = Vec::new();
+
+ for edge in self.root.traverse() {
+ match edge {
+ NodeEdge::Start(node) => {
+ if let Some(element) = node.as_element() {
+ if &element.name.local == "a" {
+ open_links.push((String::new(), element.attributes.clone()));
+ }
+ }
+ }
+ NodeEdge::End(node) => {
+ if let Some(element) = node.as_element() {
+ if &element.name.local == "a" {
+ if let Some((text, attributes)) = open_links.pop() {
+ if let Some(dest) = attributes.borrow().get("href") {
+ if dest.starts_with("mailto:") || dest.starts_with("tel:") {
+ continue;
+ }
+
+ if let Ok(dest) =
+ Url::parse(dest).or_else(|_| self.url().join(dest))
+ {
+ links.push(Link {
+ source: self.url().clone(),
+ destination: dest,
+ text: text.trim().to_string(),
+ });
+ }
+ }
+ }
+ }
+ }
+
+ if let Some(text) = node.as_text() {
+ let raw_text = text.borrow();
+ let text = raw_text.trim();
+
+ if !text.is_empty() {
+ for (link_text, _) in &mut open_links {
+ link_text.push('\n');
+ link_text.push_str(text);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ while let Some((text, attributes)) = open_links.pop() {
+ if let Some(rel) = attributes.borrow().get("rel") {
+ if rel.contains("nofollow") || rel.contains("sponsored") || rel.contains("ugc") {
+ continue;
+ }
+ }
+
+ if let Some(dest) = attributes.borrow().get("href") {
+ if dest.starts_with("mailto:") || dest.starts_with("tel:") {
+ continue;
+ }
+
+ if let Ok(dest) = Url::parse(dest).or_else(|_| self.url().join(dest)) {
+ links.push(Link {
+ source: self.url().clone(),
+ destination: dest,
+ text: text.trim().to_string(),
+ });
+ }
+ }
+ }
+
+ links
+ }
+
+ fn links_tag(&self) -> Vec<Link> {
+ let mut links = Vec::new();
+
+ for node in self.root.select("link").unwrap() {
+ if let Some(element) = node.as_node().as_element() {
+ if let Some(href) = element.attributes.borrow().get("href") {
+ if let Ok(href) = Url::parse(href).or_else(|_| self.url().join(href)) {
+ links.push(Link {
+ source: self.url().clone(),
+ destination: href,
+ text: String::new(),
+ });
+ }
+ }
+ }
+ }
+
+ links
+ }
+
+ fn metadata_links(&self) -> Vec<Link> {
+ self.metadata()
+ .into_iter()
+ .filter_map(|metadata| {
+ // https://github.com/commoncrawl/cc-pyspark/blob/54918e85cf87d47e1f7278965ac04a0fc8e414a0/wat_extract_links.py#L54
+
+ if let Some(prop) = metadata.get("property") {
+ if matches!(
+ prop.as_str(),
+ "og:url"
+ | "og:image"
+ | "og:image:secure_url"
+ | "og:video"
+ | "og:video:url"
+ | "og:video:secure_url"
+ | "twitter:url"
+ | "twitter:image:src"
+ ) {
+ if let Some(content) = metadata.get("content") {
+ if let Ok(destination) = Url::parse(content.as_str())
+ .or_else(|_| self.url().join(content.as_str()))
+ {
+ return Some(Link {
+ source: self.url().clone(),
+ destination,
+ text: String::new(),
+ });
+ }
+ }
+ }
+ }
+
+ if let Some(name) = metadata.get("name") {
+ if matches!(
+ name.as_str(),
+ "twitter:image"
+ | "thumbnail"
+ | "application-url"
+ | "msapplication-starturl"
+ | "msapplication-TileImage"
+ | "vb_meta_bburl"
+ ) {
+ if let Some(content) = metadata.get("content") {
+ if let Ok(destination) = Url::parse(content.as_str())
+ .or_else(|_| self.url().join(content.as_str()))
+ {
+ return Some(Link {
+ source: self.url().clone(),
+ destination,
+ text: String::new(),
+ });
+ }
+ }
+ }
+ }
+
+ None
+ })
+ .collect()
+ }
+
+ pub fn all_links(&self) -> Vec<Link> {
+ let mut links = self.anchor_links();
+
+ links.extend(self.scripts().into_iter().filter_map(|script| {
+ match script.attributes.get("src") {
+ Some(url) => {
+ let script_url = Url::parse(url.as_str())
+ .or_else(|_| self.url().join(url.as_str()))
+ .ok()?;
+
+ if script_url.root_domain() != self.url().root_domain() {
+ Some(Link {
+ source: self.url().clone(),
+ destination: script_url,
+ text: String::new(),
+ })
+ } else {
+ None
+ }
+ }
+ None => None,
+ }
+ }));
+
+ links.extend(self.links_tag());
+ links.extend(self.metadata_links());
+
+ links
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn simple_favicon() {
+ let raw = r#"
+ <html>
+ <head>
+ <link rel="icon" sizes="192x192" href="https://example.com/favicon.png" />
+ </head>
+ </html>
+ "#
+ .to_string();
+
+ let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
+ assert_eq!(
+ webpage.favicon(),
+ Some(FaviconLink {
+ link: Url::parse("https://example.com/favicon.png").unwrap(),
+ width: Some(192),
+ height: Some(192),
+ image_type: None
+ })
+ );
+ }
+
+ fn full_link_favicon(href: &str, site_url: &str, expected: &str) {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <link rel="icon" sizes="192x192" href="{href}" />
+ </head>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, site_url).unwrap();
+ assert_eq!(
+ webpage.favicon(),
+ Some(FaviconLink {
+ link: Url::parse(expected).unwrap(),
+ width: Some(192),
+ height: Some(192),
+ image_type: None
+ })
+ );
+ }
+
+ #[test]
+ fn test_full_link_favicon_simple() {
+ full_link_favicon(
+ "/favicon.png",
+ "https://www.example.com/",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "/favicon.png",
+ "https://www.example.com",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "favicon.png",
+ "https://www.example.com",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "favicon.png",
+ "https://www.example.com/",
+ "https://www.example.com/favicon.png",
+ );
+ full_link_favicon(
+ "favicon.png",
+ "https://www.example.com/test/",
+ "https://www.example.com/test/favicon.png",
+ );
+ full_link_favicon(
+ "/favicon.png",
+ "https://www.example.com/test",
+ "https://www.example.com/favicon.png",
+ );
+ }
+
+ #[test]
+ fn primary_image() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:image" content="https://example.com/link_to_image.html" />
+ <meta property="og:description" content="desc" />
+ <meta property="og:title" content="title" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "https://example.com").unwrap();
+
+ assert_eq!(
+ html.primary_image(),
+ Some(ImageLink {
+ url: Url::parse("https://example.com/link_to_image.html").unwrap(),
+ title: Some("title".to_string()),
+ description: Some("desc".to_string())
+ })
+ );
+
+ let html = r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "https://example.com").unwrap();
+
+ assert_eq!(html.primary_image(), None);
+
+ let html = r#"
+ <html>
+ <head>
+ <script type="application/ld+json">
+ {
+ "@context": "https://schema.org",
+ "@type": "ImageObject",
+ "author": "Jane Doe",
+ "contentLocation": "Puerto Vallarta, Mexico",
+ "contentUrl": "mexico-beach.jpg",
+ "datePublished": "2008-01-25",
+ "description": "I took this picture while on vacation last year.",
+ "name": "Beach in Mexico"
+ }
+ </script>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "https://example.com").unwrap();
+
+ assert_eq!(
+ html.primary_image(),
+ Some(ImageLink {
+ url: Url::parse("https://example.com/mexico-beach.jpg").unwrap(),
+ title: None,
+ description: None
+ })
+ );
+ }
+}
diff --git a/core/src/webpage/html/microformats.rs b/core/src/webpage/html/microformats.rs
new file mode 100644
index 00000000..9ecba223
--- /dev/null
+++ b/core/src/webpage/html/microformats.rs
@@ -0,0 +1,138 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::{enum_map::EnumSet, Result};
+
+use super::Html;
+
+#[allow(clippy::enum_variant_names)]
+pub enum Microformat {
+ HCard,
+ HEvent,
+ HEntry,
+ HRecipe,
+ HReview,
+ HProduct,
+}
+
+pub const ALL_MICROFORMATS: [Microformat; 6] = [
+ Microformat::HCard,
+ Microformat::HEvent,
+ Microformat::HEntry,
+ Microformat::HRecipe,
+ Microformat::HReview,
+ Microformat::HProduct,
+];
+
+impl Microformat {
+ pub fn as_str(&self) -> &str {
+ match self {
+ Microformat::HCard => "h-card",
+ Microformat::HEvent => "h-event",
+ Microformat::HEntry => "h-entry",
+ Microformat::HRecipe => "h-recipe",
+ Microformat::HReview => "h-review",
+ Microformat::HProduct => "h-product",
+ }
+ }
+}
+
+impl From<Microformat> for usize {
+ fn from(value: Microformat) -> Self {
+ match value {
+ Microformat::HCard => 0,
+ Microformat::HEvent => 1,
+ Microformat::HEntry => 2,
+ Microformat::HRecipe => 3,
+ Microformat::HReview => 4,
+ Microformat::HProduct => 5,
+ }
+ }
+}
+
+impl TryFrom<usize> for Microformat {
+ type Error = anyhow::Error;
+
+ fn try_from(value: usize) -> Result<Self> {
+ match value {
+ 0 => Ok(Microformat::HCard),
+ 1 => Ok(Microformat::HEvent),
+ 2 => Ok(Microformat::HEntry),
+ 3 => Ok(Microformat::HRecipe),
+ 4 => Ok(Microformat::HReview),
+ 5 => Ok(Microformat::HProduct),
+ _ => Err(anyhow::anyhow!("Unknown microformat")),
+ }
+ }
+}
+
+impl Html {
+ pub fn microformats(&self) -> EnumSet<Microformat> {
+ let mut microformats = EnumSet::new();
+
+ for node in self.root.inclusive_descendants() {
+ if let Some(element) = node.as_element() {
+ if let Some(class) = element.attributes.borrow().get("class") {
+ for microformat in ALL_MICROFORMATS {
+ if class.to_lowercase().as_str() == microformat.as_str() {
+ microformats.insert(microformat);
+ }
+ }
+ }
+ }
+ }
+
+ microformats
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn microformats() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ <article class="h-entry">
+ <h1 class="p-name">Microformats are amazing</h1>
+ <p class="e-content">This is the content of the article</p>
+ <a class="u-url" href="https://example.com/microformats">Permalink</a>
+ <a class="u-author" href="https://example.com">Author</a>
+ <p class="search-product">substrings should not match</p>
+ <time class="dt-published" datetime="2021-01-01T00:00:00+00:00">2021-01-01</time>
+ </article>
+
+ <div class="h-RECIPE">
+ For some reason this site also has a recipe
+ </div>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/",
+ ).unwrap();
+
+ let microformats = html.microformats();
+
+ assert!(microformats.contains(Microformat::HEntry));
+ assert!(microformats.contains(Microformat::HRecipe));
+ assert!(!microformats.contains(Microformat::HCard));
+ assert!(!microformats.contains(Microformat::HProduct));
+ }
+}
diff --git a/core/src/webpage/html/mod.rs b/core/src/webpage/html/mod.rs
new file mode 100644
index 00000000..e0c60ccb
--- /dev/null
+++ b/core/src/webpage/html/mod.rs
@@ -0,0 +1,1081 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{enum_map::EnumSet, Result};
+use chrono::{DateTime, FixedOffset, Utc};
+use itertools::Itertools;
+use kuchiki::{traits::TendrilSink, NodeRef};
+use regex::Regex;
+use url::Url;
+use whatlang::Lang;
+
+use self::robots_meta::RobotsMeta;
+
+use super::{adservers::AD_SERVERS, schema_org, Meta, Script};
+
+use super::url_ext::UrlExt;
+
+mod into_tantivy;
+mod links;
+mod microformats;
+mod parse_text;
+mod robots_meta;
+
+pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
+ Regex::new(r"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b").unwrap()
+});
+
+#[derive(Debug)]
+pub struct Html {
+ pub(crate) url: Url,
+ pub(crate) root: NodeRef, // this is reference counted (cheap to clone)
+ pub(crate) all_text: Option<String>,
+ pub(crate) clean_text: Option<String>,
+ pub(crate) lang: Option<Lang>,
+ pub(crate) robots: Option<EnumSet<RobotsMeta>>,
+}
+
+impl Html {
+ pub fn parse(html: &str, url: &str) -> Result<Self> {
+ let mut html = Self::parse_without_text(html, url)?;
+
+ html.parse_text();
+
+ Ok(html)
+ }
+
+ #[cfg(test)]
+ pub fn set_clean_text(&mut self, text: String) {
+ self.clean_text = Some(text);
+ }
+
+ pub fn parse_without_text(html: &str, url: &str) -> Result<Self> {
+ let root = kuchiki::parse_html().one(html);
+
+ let mut url = Url::parse(url)?;
+ url.set_fragment(None); // remove fragment (e.g. #comments
+
+ let mut res = Self {
+ root,
+ all_text: None,
+ clean_text: None,
+ lang: None,
+ url,
+ robots: None,
+ };
+
+ let queries: Vec<_> = res
+ .url
+ .query_pairs()
+ .filter(|(key, _)| !key.starts_with("utm_"))
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect();
+
+ {
+ let mut query_mut = res.url.query_pairs_mut();
+ query_mut.clear();
+ if !queries.is_empty() {
+ query_mut.extend_pairs(queries);
+ }
+ }
+
+ if res.url.query().unwrap_or_default().is_empty() {
+ res.url.set_query(None);
+ }
+
+ res.robots = res.parse_robots_meta();
+
+ Ok(res)
+ }
+
+ pub fn lang(&self) -> Option<&'_ Lang> {
+ self.lang.as_ref()
+ }
+
+ pub fn canonical_url(&self) -> Option<Url> {
+ let mut canonical_url = None;
+
+ for node in self.root.select("link").unwrap() {
+ if let Some(element) = node.as_node().as_element() {
+ if let Some(rel) = element.attributes.borrow().get("rel") {
+ if rel == "canonical" {
+ if let Some(href) = element.attributes.borrow().get("href") {
+ match Url::parse(href) {
+ Ok(url) => canonical_url = Some(url),
+ Err(_) => {
+ if let Ok(url) = self.url().join(href) {
+ canonical_url = Some(url);
+ }
+ }
+ };
+ }
+ }
+ }
+ }
+ }
+
+ canonical_url
+ }
+
+ pub fn clean_text(&self) -> Option<&String> {
+ self.clean_text.as_ref()
+ }
+
+ fn all_text(&self) -> Option<String> {
+ self.all_text.clone()
+ }
+
+ pub fn empty_all_text(&self) -> bool {
+ match &self.all_text {
+ Some(text) => text.is_empty(),
+ None => true,
+ }
+ }
+
+ pub fn title(&self) -> Option<String> {
+ if let Some(title) = self.root.select_first("title") {
+ let title = title.text_contents().trim().to_string();
+ if title.is_empty() {
+ None
+ } else {
+ Some(title)
+ }
+ } else {
+ None
+ }
+ }
+
+ pub fn url(&self) -> &Url {
+ &self.url
+ }
+
+ pub fn metadata(&self) -> Vec<Meta> {
+ let mut metas = Vec::new();
+
+ for node in self.root.select("meta").unwrap() {
+ if let Some(element) = node.as_node().as_element() {
+ metas.push(
+ element
+ .attributes
+ .borrow()
+ .map
+ .iter()
+ .map(|(name, attr)| (name.local.to_string(), attr.value.to_string()))
+ .collect(),
+ );
+ }
+ }
+
+ metas
+ }
+
+ fn scripts(&self) -> Vec<Script> {
+ let mut scripts = Vec::new();
+
+ for node in self.root.select("script").unwrap() {
+ let content = node.text_contents().trim().to_string();
+ let attributes = node
+ .attributes
+ .borrow()
+ .map
+ .iter()
+ .map(|(name, attr)| (name.local.to_string(), attr.value.to_string()))
+ .collect();
+
+ scripts.push(Script {
+ attributes,
+ content,
+ });
+ }
+
+ scripts
+ }
+
+ pub fn schema_org(&self) -> Vec<schema_org::Item> {
+ schema_org::parse(self.root.clone())
+ }
+
+ pub fn trackers(&self) -> Vec<Url> {
+ let mut links: Vec<Url> = Vec::new();
+
+ for script in self.scripts() {
+ if let Some(link) = script
+ .attributes
+ .get("src")
+ .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
+ {
+ links.push(link);
+ }
+
+ for res in URL_REGEX.find_iter(&script.content) {
+ if let Ok(link) =
+ Url::parse(res.as_str()).or_else(|_| self.url().join(res.as_str()))
+ {
+ links.push(link);
+ }
+ }
+ }
+
+ for node in self.root.select("link").unwrap() {
+ if let Some(link) = node
+ .attributes
+ .borrow()
+ .get("href")
+ .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
+ {
+ links.push(link);
+ }
+ }
+
+ links
+ .into_iter()
+ .filter(|link| link.normalized_host().is_some())
+ .filter(|link| link.normalized_host() != self.url().normalized_host())
+ .unique_by(|link| link.normalized_host().unwrap().to_string())
+ .collect()
+ }
+
+ pub fn likely_has_ads(&self) -> bool {
+ for script in self.scripts() {
+ if let Some(url) = script
+ .attributes
+ .get("src")
+ .and_then(|url| Url::parse(url).ok())
+ {
+ if url.root_domain() == self.url().root_domain() {
+ continue;
+ }
+
+ if let Some(domain) = url.root_domain() {
+ if AD_SERVERS.is_adserver(domain) {
+ return true;
+ }
+ }
+
+ if let Some(host) = url.host_str() {
+ if AD_SERVERS.is_adserver(host) {
+ return true;
+ }
+ }
+ }
+ }
+
+ // check <link> tags
+ for node in self.root.select("link").unwrap() {
+ if let Some(url) = node
+ .attributes
+ .borrow()
+ .get("href")
+ .and_then(|url| Url::parse(url).ok())
+ {
+ if url.root_domain() == self.url().root_domain() {
+ continue;
+ }
+
+ if let Some(domain) = url.root_domain() {
+ if AD_SERVERS.is_adserver(domain) {
+ return true;
+ }
+ }
+
+ if let Some(host) = url.host_str() {
+ if AD_SERVERS.is_adserver(host) {
+ return true;
+ }
+ }
+ }
+ }
+
+ false
+ }
+
+ pub fn likely_has_paywall(&self) -> bool {
+ self.schema_org()
+ .into_iter()
+ .filter(|item| {
+ item.types_contains("NewsArticle")
+ || item.types_contains("Article")
+ || item.types_contains("BlogPosting")
+ || item.types_contains("WebPage")
+ || item.types_contains("WebPageElement")
+ })
+ .any(|item| {
+ item.properties
+ .get("isAccessibleForFree")
+ .and_then(|value| value.clone().one().and_then(|v| v.try_into_string()))
+ .map(|value| value.parse().ok().unwrap_or(false))
+ .unwrap_or(false)
+ })
+ }
+
+ fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("article:modified_time")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| {
+ metadata
+ .get("content")
+ .and_then(|time| DateTime::parse_from_rfc3339(time).ok())
+ })
+ }
+
+ fn og_updated_time(&self) -> Option<DateTime<FixedOffset>> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("og:updated_time")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| {
+ metadata
+ .get("content")
+ .and_then(|time| DateTime::parse_from_rfc3339(time).ok())
+ })
+ }
+
+ fn schema_org_images(&self) -> Vec<Url> {
+ self.schema_org()
+ .into_iter()
+ .filter(|item| item.types_contains("ImageObject"))
+ .filter_map(|item| {
+ item.properties.get("contentUrl").map(|content_url| {
+ content_url
+ .clone()
+ .many()
+ .into_iter()
+ .filter_map(|url| url.try_into_string())
+ .filter_map(|url| Url::parse(&url).or_else(|_| self.url().join(&url)).ok())
+ })
+ })
+ .flatten()
+ .collect()
+ }
+
+ pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {
+ if let Some(time) = self
+ .og_updated_time()
+ .or_else(|| self.article_modified_time())
+ {
+ let current_time = Utc::now();
+
+ if time > current_time {
+ None
+ } else {
+ Some(time)
+ }
+ } else {
+ None
+ }
+ }
+
+ pub fn og_description(&self) -> Option<String> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property.as_str() == "og:description"
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| metadata.get("content").cloned())
+ }
+
+ pub fn metadata_description(&self) -> Option<String> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(name) = metadata.get("name") {
+ name.as_str() == "description" || name.as_str() == "Description"
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| metadata.get("content").cloned())
+ }
+
+ pub fn description(&self) -> Option<String> {
+ self.og_description()
+ .or_else(|| self.metadata_description())
+ }
+
+ pub fn og_title(&self) -> Option<String> {
+ self.metadata()
+ .into_iter()
+ .find(|metadata| {
+ if let Some(property) = metadata.get("property") {
+ property == &String::from("og:title")
+ } else {
+ false
+ }
+ })
+ .and_then(|metadata| metadata.get("content").cloned())
+ }
+
+ pub fn is_homepage(&self) -> bool {
+ self.url().path() == "/" && self.url().query().is_none()
+ }
+}
+
+fn find_recipe_first_ingredient_tag_id(
+ schemas: &[schema_org::Item],
+ root: &NodeRef,
+) -> Option<String> {
+ schemas.iter().find_map(|schema| {
+ if let Some(ingredients) = schema.properties.get("recipeIngredient") {
+ if let Some(ingredient) = ingredients.clone().many().first() {
+ if let Some(ingredient) = ingredient.try_into_string() {
+ let ingredient = ingredient.trim();
+ // find first occurrence in html
+ if let Some(ingredient_node) = root
+ .select("body")
+ .unwrap()
+ .flat_map(|node| node.as_node().descendants())
+ .find(|node| {
+ if let Some(text) = node.as_text() {
+ text.borrow().trim() == ingredient
+ } else {
+ false
+ }
+ })
+ {
+ // find first parent that has an id
+ if let Some(id) = ingredient_node.ancestors().find_map(|node| {
+ node.as_element().and_then(|e| {
+ e.attributes.borrow().get("id").map(|s| s.to_string())
+ })
+ }) {
+ return Some(id.to_string());
+ }
+ }
+ }
+ }
+ }
+
+ None
+ })
+}
+
+#[cfg(test)]
+mod tests {
+ // TODO: make test macro to test both dom parsers
+
+ use std::collections::HashMap;
+
+ use crate::{
+ schema::create_schema,
+ webpage::{url_ext::UrlExt, Link},
+ };
+
+ use super::*;
+
+ const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
+
+ #[test]
+ fn simple() {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ </head>
+ <body>
+ <a href="https://example.com">Link to example</a>
+ <p>{CONTENT}</p>
+ <a href="mailto:[email protected]">Email me</a>
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
+
+ assert_eq!(webpage.title(), Some("Best website".to_string()));
+
+ assert_eq!(
+ webpage.anchor_links(),
+ vec![Link {
+ source: Url::parse("https://www.example.com/whatever").unwrap(),
+ destination: Url::parse("https://example.com").unwrap(),
+ text: "Link to example".to_string()
+ }]
+ );
+ assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
+
+ let mut expected_meta = HashMap::new();
+ expected_meta.insert("name".to_string(), "meta1".to_string());
+ expected_meta.insert("content".to_string(), "value".to_string());
+
+ assert_eq!(webpage.metadata(), vec![expected_meta]);
+ assert_eq!(
+ webpage.url().to_string().as_str(),
+ "https://www.example.com/whatever"
+ );
+ assert_eq!(webpage.url().host_str().unwrap(), "www.example.com");
+ }
+
+ #[test]
+ fn empty_title() {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <title></title>
+ </head>
+ <body>
+ <p>{CONTENT}</p>
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
+
+ assert_eq!(webpage.title(), None);
+ }
+
+ #[test]
+ fn text_raw_body() {
+ let raw = format!(
+ r#"
+ <html>
+ <body>
+ {CONTENT}
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
+
+ assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
+ }
+
+ #[test]
+ fn script_tags_text_ignored() {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ <script>this should not be extracted</script>
+ </head>
+ <body>
+ <script>this should not be extracted</script>
+ <p>{CONTENT}</p>
+ <div>
+ <script>this should not be extracted</script>
+ <p>This text should be the second text extracted</p>
+ </div>
+ <script>this should not be extracted</script>
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
+
+ assert!(!webpage.clean_text().unwrap().contains("not"));
+ }
+
+ #[test]
+ fn style_tags_text_ignored() {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ <style>this should not be extracted</style>
+ </head>
+ <body>
+ <style>this should not be extracted</style>
+ <p>{CONTENT}</p>
+ <div>
+ <style>this should not be extracted</style>
+ <p>This text should be the second text extracted</p>
+ </div>
+ <style>this should not be extracted</style>
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
+
+ assert!(!webpage.clean_text().unwrap().contains("not"));
+ }
+
+ #[test]
+ fn co_uk_domain() {
+ let raw = "";
+
+ let webpage = Html::parse(raw, "https://www.domain.co.uk").unwrap();
+ assert_eq!(
+ webpage.url().root_domain().unwrap_or_default(),
+ "domain.co.uk"
+ );
+ }
+
+ #[test]
+ fn is_homepage() {
+ let webpage = Html::parse("", "https://www.example.com").unwrap();
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://www.example.com/").unwrap();
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://www.example.com/test").unwrap();
+ assert!(!webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://example.com/test").unwrap();
+ assert!(!webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://example.com/").unwrap();
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "https://example.com").unwrap();
+ assert!(webpage.is_homepage());
+
+ let webpage = Html::parse("", "http://example.com").unwrap();
+ assert!(webpage.is_homepage());
+ }
+
+ #[test]
+ fn hard_parsing() {
+ let webpage = Html::parse(
+ include_str!("../../../testcases/parsing/yasudaya.html"),
+ "https://example.com",
+ )
+ .unwrap();
+ assert_eq!(
+ webpage.title(),
+ Some("パチンコ大当たり情報 - Pジューシーハニー3 大当たり詳細ページ - やすだひばりヶ丘店".to_string())
+ );
+ assert!(webpage.all_text().is_some());
+ assert!(!webpage.all_text().unwrap().is_empty());
+
+ let webpage = Html::parse(
+ include_str!("../../../testcases/parsing/5390001.html"),
+ "https://example.com",
+ )
+ .unwrap();
+ assert_eq!(
+ webpage.title(),
+ Some("特效烟机系列_山东壹线文化传播有限公司".to_string())
+ );
+ assert!(webpage.all_text().is_some());
+ assert!(!webpage.all_text().unwrap().is_empty());
+
+ let webpage = Html::parse(
+ include_str!("../../../testcases/parsing/77p2p-7.live-105.html"),
+ "https://example.com",
+ )
+ .unwrap();
+ assert_eq!(
+ webpage.title(),
+ Some("77p2pЅu¤WЖ[¬Э - ҐDјЅ :: іnєс".to_string())
+ );
+ assert!(webpage.all_text().is_some());
+ assert!(!webpage.all_text().unwrap().is_empty());
+ }
+
+ #[test]
+ fn reddit_comments() {
+ let webpage = Html::parse(
+ include_str!("../../../testcases/parsing/reddit.html"),
+ "https://reddit.com/",
+ )
+ .unwrap();
+
+ assert!(webpage.clean_text().is_some());
+ assert!(webpage.clean_text().unwrap().len() > 1000);
+ assert!(webpage
+ .all_text()
+ .unwrap()
+ .contains("They immediately moved outta striking range"));
+ }
+
+ #[test]
+ fn out_of_bounds_str() {
+ let webpage = Html::parse(
+ include_str!("../../../testcases/parsing/byte_index_out_of_bounds.html"),
+ "https://example.com",
+ )
+ .unwrap();
+ assert_eq!(webpage.title(), Some("Test".to_string()));
+ assert!(webpage.all_text().is_some());
+ assert!(!webpage.all_text().unwrap().is_empty());
+
+ let schema = create_schema();
+ webpage.into_tantivy(&schema).unwrap();
+ }
+
+ #[test]
+ fn metadata_updated_time() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "https://example.com").unwrap();
+
+ assert_eq!(
+ html.updated_time(),
+ Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())
+ );
+
+ let html = r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "https://example.com").unwrap();
+
+ assert_eq!(html.updated_time(), None);
+
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:whutwhut" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "https://example.com").unwrap();
+
+ assert_eq!(html.updated_time(), None);
+
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:updated_time" content="2ss022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "https://example.com").unwrap();
+
+ assert_eq!(html.updated_time(), None);
+ }
+
+ #[test]
+ fn future_updated_time_none() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:updated_time" content="2122-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "http://example.com").unwrap();
+
+ assert_eq!(html.updated_time(), None);
+ }
+
+ #[test]
+ fn description() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="og:description" content="This is a page description" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "http://example.com").unwrap();
+
+ assert_eq!(
+ html.description(),
+ Some("This is a page description".to_string())
+ );
+
+ let html = r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "http://example.com").unwrap();
+
+ assert_eq!(html.description(), None);
+ }
+
+ #[test]
+ fn article_modified_time() {
+ let html = r#"
+ <html>
+ <head>
+ <meta property="article:modified_time" content="2022-06-22T19:37:34+00:00" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "http://example.com").unwrap();
+
+ assert_eq!(
+ html.updated_time(),
+ Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())
+ );
+ }
+
+ #[test]
+ fn trackers() {
+ let html = r#"
+ <html>
+ <head>
+ <script>
+ !function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","once","off","on","addSourceMiddleware","addIntegrationMiddleware","setAnonymousId","addDestinationMiddleware"];analytics.factory=function(e){return function(){var t=Array.prototype.slice.call(arguments);t.unshift(e);analytics.push(t);return analytics}};for(var e=0;e<analytics.methods.length;e++){var key=analytics.methods[e];analytics[key]=analytics.factory(key)}analytics.load=function(key,e){var t=document.createElement("script");t.type="text/javascript";t.async=!0;t.src="https://cdn.segment.com/analytics.js/v1/" + key + "/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(t,n);analytics._loadOptions=e};analytics._writeKey="";analytics.SNIPPET_VERSION="4.13.2";
+ analytics.load("");
+ analytics.page();
+ }}();
+ </script>
+ <script>
+ (function(h,o,t,j,a,r){
+ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)};
+ a.appendChild(r);
+ })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
+ </script>
+ <script src="https://thirdparty.com/js"></script>
+ <script src="https://example.com/js"></script>
+ <link href='//securepubads.g.doubleclick.net' rel='preconnect'>
+ <script src="https://thirdparty.com/js"></script>
+ <script src="/js/file"></script>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#;
+ let html = Html::parse(html, "http://example.com").unwrap();
+
+ assert_eq!(
+ html.trackers()
+ .into_iter()
+ .map(|url| url.host_str().unwrap().to_string())
+ .collect::<Vec<_>>(),
+ vec![
+ "cdn.segment.com".to_string(),
+ "static.hotjar.com".to_string(),
+ "thirdparty.com".to_string(),
+ "securepubads.g.doubleclick.net".to_string()
+ ]
+ );
+ }
+
+ #[test]
+ fn parse_title_with_scripts() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <script>
+ !function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","once","off","on","addSourceMiddleware","addIntegrationMiddleware","setAnonymousId","addDestinationMiddleware"];analytics.factory=function(e){return function(){var t=Array.prototype.slice.call(arguments);t.unshift(e);analytics.push(t);return analytics}};for(var e=0;e<analytics.methods.length;e++){var key=analytics.methods[e];analytics[key]=analytics.factory(key)}analytics.load=function(key,e){var t=document.createElement("script");t.type="text/javascript";t.async=!0;t.src="https://cdn.segment.com/analytics.js/v1/" + key + "/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(t,n);analytics._loadOptions=e};analytics._writeKey="";analytics.SNIPPET_VERSION="4.13.2";
+ analytics.load("");
+ analytics.page();
+ }}();
+ </script>
+ <script>
+ (function(h,o,t,j,a,r){
+ h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)};
+ a.appendChild(r);
+ })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
+ </script>
+ <script src="https://thirdparty.com/js"></script>
+ <link href='//securepubads.g.doubleclick.net' rel='preconnect'>
+ <title>Test site</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "https://example.com",
+ ).unwrap();
+
+ assert_eq!(html.title(), Some("Test site".to_string()));
+ assert_eq!(html.all_text(), Some("test".to_string()));
+ }
+
+ #[test]
+ fn links() {
+ let raw = format!(
+ r#"
+ <html>
+ <head>
+ <title>Best website</title>
+ <meta name="meta1" content="value">
+ <link href="link.com" />
+ <script src="test.com"></script>
+ </head>
+ <body>
+ <a href="https://example.com">Link to example</a>
+ <p>{CONTENT}</p>
+ </body>
+ </html>
+ "#
+ );
+
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
+
+ assert_eq!(webpage.title(), Some("Best website".to_string()));
+
+ assert_eq!(
+ webpage.anchor_links(),
+ vec![Link {
+ source: Url::parse("https://www.example.com/whatever").unwrap(),
+ destination: Url::parse("https://example.com").unwrap(),
+ text: "Link to example".to_string()
+ },]
+ );
+ }
+
+ #[test]
+ fn stackoverflow_question_has_clean_text() {
+ let stackoverflow =
+ include_str!("../../../testcases/schema_org/stackoverflow_with_code.html");
+ let html = Html::parse(stackoverflow, "https://www.example.com").unwrap();
+
+ assert!(html.clean_text().is_some());
+ }
+
+ #[test]
+ fn canonical_url() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <link rel="canonical" href="https://example.com/canonical.html" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert_eq!(
+ html.canonical_url(),
+ Some(Url::parse("https://example.com/canonical.html").unwrap())
+ );
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert_eq!(html.canonical_url(), None);
+ assert_eq!(
+ html.url(),
+ &Url::parse("https://www.example.com/whatever").unwrap()
+ );
+ }
+
+ #[test]
+ fn recipe_first_ingredient_tag() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ <script type="application/ld+json">
+ {
+ "@context": "https://schema.org",
+ "@type": "Recipe",
+ "author": "John Smith",
+ "cookTime": "PT1H",
+ "datePublished": "2009-05-08",
+ "description": "This classic banana bread recipe comes from my mom -- the walnuts add a nice texture and flavor to the banana bread.",
+ "image": "bananabread.jpg",
+ "recipeIngredient": [
+ "3 or 4 ripe bananas, smashed",
+ "1 egg",
+ "3/4 cup of sugar"
+ ],
+ "interactionStatistic": {
+ "@type": "InteractionCounter",
+ "interactionType": "https://schema.org/Comment",
+ "userInteractionCount": "140"
+ },
+ "name": "Mom's World Famous Banana Bread",
+ "nutrition": {
+ "@type": "NutritionInformation",
+ "calories": "240 calories",
+ "fatContent": "9 grams fat"
+ },
+ "prepTime": "PT15M",
+ "recipeInstructions": "Preheat the oven to 350 degrees. Mix in the ingredients in a bowl. Add the flour last. Pour the mixture into a loaf pan and bake for one hour.",
+ "recipeYield": "1 loaf",
+ "suitableForDiet": "https://schema.org/LowFatDiet"
+ }
+ </script>
+
+ <div id="ingredients">
+ <h2>Ingredients</h2>
+ <ul>
+ <li>3 or 4 ripe bananas, smashed</li>
+ <li>1 egg</li>
+ <li>3/4 cup of sugar</li>
+ </ul>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/",
+ ).unwrap();
+
+ let schemas = html.schema_org();
+
+ assert_eq!(
+ find_recipe_first_ingredient_tag_id(&schemas, &html.root),
+ Some("ingredients".to_string())
+ );
+ }
+}
diff --git a/core/src/webpage/html/parse_text.rs b/core/src/webpage/html/parse_text.rs
new file mode 100644
index 00000000..63b5eeff
--- /dev/null
+++ b/core/src/webpage/html/parse_text.rs
@@ -0,0 +1,70 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use whatlang::Lang;
+
+use crate::webpage::just_text::{JustText, Paragraph};
+
+use super::Html;
+
+impl Html {
+ pub fn parse_text(&mut self) {
+ let paragraphs = JustText::paragraphs(self.root.clone());
+
+ self.lang = paragraphs
+ .iter()
+ .max_by_key(|paragraph| paragraph.text.len())
+ .and_then(|paragraph| {
+ whatlang::detect(&paragraph.text).and_then(|info| {
+ if info.is_reliable() && info.confidence() > 0.95 {
+ Some(info.lang())
+ } else {
+ None
+ }
+ })
+ });
+
+ self.all_text = Html::calculate_all_text(&paragraphs, &self.lang.unwrap_or(Lang::Eng));
+ self.clean_text = Html::calculate_clean_text(&paragraphs, &self.lang.unwrap_or(Lang::Eng));
+ }
+
+ fn calculate_clean_text(paragraphs: &[Paragraph], lang: &Lang) -> Option<String> {
+ let text = JustText::default().extract_from_paragraphs(paragraphs, lang);
+
+ if text.is_empty() {
+ None
+ } else {
+ Some(text)
+ }
+ }
+ fn calculate_all_text(paragraphs: &[Paragraph], lang: &Lang) -> Option<String> {
+ let text = JustText {
+ max_link_density: 20.0,
+ length_low: 0,
+ length_high: 0,
+ stopwords_low: -1.0,
+ stopwords_high: -1.0,
+ max_heading_distance: 10000,
+ }
+ .extract_from_paragraphs(paragraphs, lang);
+
+ if text.is_empty() {
+ None
+ } else {
+ Some(text)
+ }
+ }
+}
diff --git a/core/src/webpage/html/robots_meta.rs b/core/src/webpage/html/robots_meta.rs
new file mode 100644
index 00000000..08707624
--- /dev/null
+++ b/core/src/webpage/html/robots_meta.rs
@@ -0,0 +1,183 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::str::FromStr;
+
+use crate::{enum_map::EnumSet, Error, Result};
+
+use super::Html;
+
+#[derive(Debug)]
+pub enum RobotsMeta {
+ NoIndex,
+ NoFollow,
+}
+
+impl FromStr for RobotsMeta {
+ type Err = anyhow::Error;
+
+ fn from_str(s: &str) -> Result<Self> {
+ match s {
+ "noindex" => Ok(RobotsMeta::NoIndex),
+ "nofollow" => Ok(RobotsMeta::NoFollow),
+ _ => Err(Error::UnknownRobotsMetaTag.into()),
+ }
+ }
+}
+
+impl From<RobotsMeta> for usize {
+ fn from(val: RobotsMeta) -> Self {
+ match val {
+ RobotsMeta::NoIndex => 0,
+ RobotsMeta::NoFollow => 1,
+ }
+ }
+}
+
+impl Html {
+ pub fn parse_robots_meta(&self) -> Option<EnumSet<RobotsMeta>> {
+ let mut robots = EnumSet::new();
+
+ for node in self.root.select("meta").unwrap() {
+ if let Some(element) = node.as_node().as_element() {
+ if let Some(name) = element.attributes.borrow().get("name") {
+ if name == "robots" {
+ if let Some(content) = element.attributes.borrow().get("content") {
+ for part in content.split(',') {
+ let part = part.trim();
+ if let Ok(meta) = part.parse::<RobotsMeta>() {
+ robots.insert(meta);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if robots.is_empty() {
+ None
+ } else {
+ Some(robots)
+ }
+ }
+
+ pub fn is_no_index(&self) -> bool {
+ self.robots
+ .as_ref()
+ .map(|robots| robots.contains(RobotsMeta::NoIndex))
+ .unwrap_or(false)
+ }
+
+ pub fn is_no_follow(&self) -> bool {
+ self.robots
+ .as_ref()
+ .map(|robots| robots.contains(RobotsMeta::NoFollow))
+ .unwrap_or(false)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn robots_meta_tag() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noindex, nofollow" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert!(html.is_no_index());
+ assert!(html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noindex,nofollow" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert!(html.is_no_index());
+ assert!(html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noindex" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert!(html.is_no_index());
+ assert!(!html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="nofollow" />
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert!(!html.is_no_index());
+ assert!(html.is_no_follow());
+
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ </head>
+ <body>
+ </body>
+ </html>
+ "#,
+ "https://www.example.com/whatever",
+ )
+ .unwrap();
+
+ assert!(!html.is_no_index());
+ assert!(!html.is_no_follow());
+ }
+}
diff --git a/core/src/webpage/just_text.rs b/core/src/webpage/just_text.rs
index 607d3da1..9e7c5711 100644
--- a/core/src/webpage/just_text.rs
+++ b/core/src/webpage/just_text.rs
@@ -19,7 +19,54 @@ use std::collections::{HashMap, HashSet};
use kuchiki::{iter::NodeEdge, NodeRef};
use whatlang::Lang;
-use super::Preprocessor;
+pub struct Preprocessor<const N: usize> {
+ removed_tags: [&'static str; N],
+ num_open_tags: [i64; N],
+}
+
+impl<const N: usize> Preprocessor<N> {
+ pub fn new(removed_tags: [&'static str; N]) -> Self {
+ Self {
+ removed_tags,
+ num_open_tags: [0; N],
+ }
+ }
+
+ pub fn update(&mut self, edge: &NodeEdge<NodeRef>) {
+ match edge {
+ NodeEdge::Start(node) => {
+ if let Some(element) = node.as_element() {
+ let element_name: &str = &element.name.local;
+ if let Some((_, n)) = self
+ .removed_tags
+ .iter()
+ .zip(self.num_open_tags.iter_mut())
+ .find(|(name, _)| **name == element_name)
+ {
+ *n += 1;
+ }
+ }
+ }
+ NodeEdge::End(node) => {
+ if let Some(element) = node.as_element() {
+ let element_name: &str = &element.name.local;
+ if let Some((_, n)) = self
+ .removed_tags
+ .iter()
+ .zip(self.num_open_tags.iter_mut())
+ .find(|(name, _)| **name == element_name)
+ {
+ *n -= 1;
+ }
+ }
+ }
+ }
+ }
+
+ pub fn is_inside_removed(&self) -> bool {
+ self.num_open_tags.iter().any(|n| *n > 0)
+ }
+}
// implementation of the JustText algorithm described in this thesis: https://is.muni.cz/th/45523/fi_d/phdthesis.pdf
// reference implementation: https://github.com/miso-belica/jusText/blob/main/justext/core.py
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 0069b33f..7602719c 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -13,112 +13,33 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
use crate::{
- ceil_char_boundary,
- enum_map::EnumSet,
- prehashed::hash,
schema::{FastField, TextField},
- simhash, split_u128, tokenizer,
+ split_u128,
webgraph::NodeID,
- Error, Result,
-};
-use chrono::{DateTime, FixedOffset, Utc};
-use itertools::Itertools;
-use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
-use regex::Regex;
-use std::{collections::HashMap, panic, str::FromStr};
-use tantivy::{
- time::OffsetDateTime,
- tokenizer::{PreTokenizedString, Tokenizer},
- TantivyDocument,
+ Result,
};
+use chrono::{DateTime, Utc};
+
+#[cfg(test)]
+use kuchiki::traits::TendrilSink;
+use std::collections::HashMap;
+use tantivy::{time::OffsetDateTime, TantivyDocument};
use url::Url;
-use whatlang::Lang;
+
+use crate::schema::{Field, FLOAT_SCALING};
+
+use self::region::Region;
mod adservers;
+mod html;
mod just_text;
pub mod region;
pub mod safety_classifier;
pub mod schema_org;
pub mod url_ext;
-
-use self::adservers::AD_SERVERS;
-
-use crate::schema::{Field, ALL_FIELDS, FLOAT_SCALING};
-
-use self::{
- just_text::{JustText, Paragraph},
- region::Region,
- url_ext::UrlExt,
-};
-
-pub static URL_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::new(|| {
- Regex::new(r"(((http|ftp|https):/{2})+(([0-9a-z_-]+\.)+(aero|asia|biz|cat|com|coop|edu|gov|info|int|jobs|mil|mobi|museum|name|net|org|pro|tel|travel|ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cx|cy|cz|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mn|mn|mo|mp|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|nom|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ra|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sj|sk|sl|sm|sn|so|sr|st|su|sv|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|arpa)(:[0-9]+)?((/([~0-9a-zA-Z\#\+%@\./_-]+))?(\?[0-9a-zA-Z\+%@/&\[\];=_-]+)?)?))\b").unwrap()
-});
-
-#[derive(PartialEq, Eq, Debug)]
-pub struct FaviconLink {
- pub link: Url,
- width: Option<u32>,
- height: Option<u32>,
- image_type: Option<String>,
-}
-
-#[derive(PartialEq, Eq, Debug)]
-pub struct ImageLink {
- pub url: Url,
- pub title: Option<String>,
- pub description: Option<String>,
-}
-
-pub struct Preprocessor<const N: usize> {
- removed_tags: [&'static str; N],
- num_open_tags: [i64; N],
-}
-
-impl<const N: usize> Preprocessor<N> {
- pub fn new(removed_tags: [&'static str; N]) -> Self {
- Self {
- removed_tags,
- num_open_tags: [0; N],
- }
- }
-
- pub fn update(&mut self, edge: &NodeEdge<NodeRef>) {
- match edge {
- NodeEdge::Start(node) => {
- if let Some(element) = node.as_element() {
- let element_name: &str = &element.name.local;
- if let Some((_, n)) = self
- .removed_tags
- .iter()
- .zip(self.num_open_tags.iter_mut())
- .find(|(name, _)| **name == element_name)
- {
- *n += 1;
- }
- }
- }
- NodeEdge::End(node) => {
- if let Some(element) = node.as_element() {
- let element_name: &str = &element.name.local;
- if let Some((_, n)) = self
- .removed_tags
- .iter()
- .zip(self.num_open_tags.iter_mut())
- .find(|(name, _)| **name == element_name)
- {
- *n -= 1;
- }
- }
- }
- }
- }
-
- pub fn is_inside_removed(&self) -> bool {
- self.num_open_tags.iter().any(|n| *n > 0)
- }
-}
+pub use self::html::Html;
#[derive(Debug)]
pub struct Webpage {
@@ -324,2380 +245,73 @@ struct Script {
content: String,
}
-#[derive(Debug)]
-enum RobotsMeta {
- NoIndex,
- NoFollow,
-}
-
-impl FromStr for RobotsMeta {
- type Err = anyhow::Error;
-
- fn from_str(s: &str) -> Result<Self> {
- match s {
- "noindex" => Ok(RobotsMeta::NoIndex),
- "nofollow" => Ok(RobotsMeta::NoFollow),
- _ => Err(Error::UnknownRobotsMetaTag.into()),
- }
- }
-}
-
-impl From<RobotsMeta> for usize {
- fn from(val: RobotsMeta) -> Self {
- match val {
- RobotsMeta::NoIndex => 0,
- RobotsMeta::NoFollow => 1,
- }
- }
-}
-
-#[allow(clippy::enum_variant_names)]
-enum Microformat {
- HCard,
- HEvent,
- HEntry,
- HRecipe,
- HReview,
- HProduct,
-}
-
-impl Microformat {
- fn as_str(&self) -> &str {
- match self {
- Microformat::HCard => "h-card",
- Microformat::HEvent => "h-event",
- Microformat::HEntry => "h-entry",
- Microformat::HRecipe => "h-recipe",
- Microformat::HReview => "h-review",
- Microformat::HProduct => "h-product",
- }
- }
-}
-
-impl From<Microformat> for usize {
- fn from(value: Microformat) -> Self {
- match value {
- Microformat::HCard => 0,
- Microformat::HEvent => 1,
- Microformat::HEntry => 2,
- Microformat::HRecipe => 3,
- Microformat::HReview => 4,
- Microformat::HProduct => 5,
- }
- }
-}
-
-impl TryFrom<usize> for Microformat {
- type Error = anyhow::Error;
-
- fn try_from(value: usize) -> Result<Self> {
- match value {
- 0 => Ok(Microformat::HCard),
- 1 => Ok(Microformat::HEvent),
- 2 => Ok(Microformat::HEntry),
- 3 => Ok(Microformat::HRecipe),
- 4 => Ok(Microformat::HReview),
- 5 => Ok(Microformat::HProduct),
- _ => Err(anyhow::anyhow!("Unknown microformat")),
- }
- }
-}
-
-const ALL_MICROFORMATS: [Microformat; 6] = [
- Microformat::HCard,
- Microformat::HEvent,
- Microformat::HEntry,
- Microformat::HRecipe,
- Microformat::HReview,
- Microformat::HProduct,
-];
-
-#[derive(Debug)]
-pub struct Html {
- url: Url,
- root: NodeRef, // this is reference counted (cheap to clone)
- all_text: Option<String>,
- clean_text: Option<String>,
- lang: Option<Lang>,
- robots: Option<EnumSet<RobotsMeta>>,
+#[derive(Debug, PartialEq, Eq)]
+pub struct Link {
+ pub source: Url,
+ pub destination: Url,
+ pub text: String,
}
-impl Html {
- pub fn parse(html: &str, url: &str) -> Result<Self> {
- let mut html = Self::parse_without_text(html, url)?;
-
- html.parse_text();
-
- Ok(html)
- }
-
- #[cfg(test)]
- pub fn set_clean_text(&mut self, text: String) {
- self.clean_text = Some(text);
- }
+pub type Meta = HashMap<String, String>;
- pub fn parse_without_text(html: &str, url: &str) -> Result<Self> {
- let root = kuchiki::parse_html().one(html);
+#[cfg(test)]
+mod tests {
+ use super::*;
- let mut url = Url::parse(url)?;
- url.set_fragment(None); // remove fragment (e.g. #comments
+ #[test]
+ fn dmoz_description() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <title>Test site</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "https://example.com",
+ )
+ .unwrap();
- let mut res = Self {
- root,
- all_text: None,
- clean_text: None,
- lang: None,
- url,
- robots: None,
+ let webpage = Webpage {
+ html,
+ fetch_time_ms: 500,
+ dmoz_description: Some("dmoz description".to_string()),
+ ..Default::default()
};
- let queries: Vec<_> = res
- .url
- .query_pairs()
- .filter(|(key, _)| !key.starts_with("utm_"))
- .map(|(key, value)| (key.to_string(), value.to_string()))
- .collect();
-
- {
- let mut query_mut = res.url.query_pairs_mut();
- query_mut.clear();
- if !queries.is_empty() {
- query_mut.extend_pairs(queries);
- }
- }
-
- if res.url.query().unwrap_or_default().is_empty() {
- res.url.set_query(None);
- }
-
- res.robots = res.parse_robots_meta();
-
- Ok(res)
- }
-
- pub fn lang(&self) -> Option<&'_ Lang> {
- self.lang.as_ref()
- }
-
- pub fn link_density(&self) -> f64 {
- (1.0 + self.anchor_links().len() as f64)
- / (1.0
- + self
- .clean_text
- .as_ref()
- .map(|s| s.len())
- .unwrap_or_default() as f64)
- }
-
- pub fn canonical_url(&self) -> Option<Url> {
- let mut canonical_url = None;
-
- for node in self.root.select("link").unwrap() {
- if let Some(element) = node.as_node().as_element() {
- if let Some(rel) = element.attributes.borrow().get("rel") {
- if rel == "canonical" {
- if let Some(href) = element.attributes.borrow().get("href") {
- match Url::parse(href) {
- Ok(url) => canonical_url = Some(url),
- Err(_) => {
- if let Ok(url) = self.url().join(href) {
- canonical_url = Some(url);
- }
- }
- };
- }
- }
- }
- }
- }
-
- canonical_url
- }
-
- fn parse_robots_meta(&self) -> Option<EnumSet<RobotsMeta>> {
- let mut robots = EnumSet::new();
-
- for node in self.root.select("meta").unwrap() {
- if let Some(element) = node.as_node().as_element() {
- if let Some(name) = element.attributes.borrow().get("name") {
- if name == "robots" {
- if let Some(content) = element.attributes.borrow().get("content") {
- for part in content.split(',') {
- let part = part.trim();
- if let Ok(meta) = part.parse::<RobotsMeta>() {
- robots.insert(meta);
- }
- }
- }
- }
- }
- }
- }
-
- if robots.is_empty() {
- None
- } else {
- Some(robots)
- }
- }
-
- pub fn is_no_index(&self) -> bool {
- self.robots
- .as_ref()
- .map(|robots| robots.contains(RobotsMeta::NoIndex))
- .unwrap_or(false)
- }
-
- pub fn is_no_follow(&self) -> bool {
- self.robots
- .as_ref()
- .map(|robots| robots.contains(RobotsMeta::NoFollow))
- .unwrap_or(false)
- }
-
- pub fn parse_text(&mut self) {
- let paragraphs = JustText::paragraphs(self.root.clone());
-
- self.lang = paragraphs
- .iter()
- .max_by_key(|paragraph| paragraph.text.len())
- .and_then(|paragraph| {
- whatlang::detect(&paragraph.text).and_then(|info| {
- if info.is_reliable() && info.confidence() > 0.95 {
- Some(info.lang())
- } else {
- None
- }
- })
- });
-
- self.all_text = Html::calculate_all_text(&paragraphs, &self.lang.unwrap_or(Lang::Eng));
- self.clean_text = Html::calculate_clean_text(&paragraphs, &self.lang.unwrap_or(Lang::Eng));
- }
-
- pub fn anchor_links(&self) -> Vec<Link> {
- if self.is_no_follow() {
- return Vec::new();
- }
-
- let mut links = Vec::new();
- let mut open_links = Vec::new();
-
- for edge in self.root.traverse() {
- match edge {
- NodeEdge::Start(node) => {
- if let Some(element) = node.as_element() {
- if &element.name.local == "a" {
- open_links.push((String::new(), element.attributes.clone()));
- }
- }
- }
- NodeEdge::End(node) => {
- if let Some(element) = node.as_element() {
- if &element.name.local == "a" {
- if let Some((text, attributes)) = open_links.pop() {
- if let Some(dest) = attributes.borrow().get("href") {
- if dest.starts_with("mailto:") || dest.starts_with("tel:") {
- continue;
- }
-
- if let Ok(dest) =
- Url::parse(dest).or_else(|_| self.url().join(dest))
- {
- links.push(Link {
- source: self.url().clone(),
- destination: dest,
- text: text.trim().to_string(),
- });
- }
- }
- }
- }
- }
-
- if let Some(text) = node.as_text() {
- let raw_text = text.borrow();
- let text = raw_text.trim();
-
- if !text.is_empty() {
- for (link_text, _) in &mut open_links {
- link_text.push('\n');
- link_text.push_str(text);
- }
- }
- }
- }
- }
- }
-
- while let Some((text, attributes)) = open_links.pop() {
- if let Some(rel) = attributes.borrow().get("rel") {
- if rel.contains("nofollow") || rel.contains("sponsored") || rel.contains("ugc") {
- continue;
- }
- }
-
- if let Some(dest) = attributes.borrow().get("href") {
- if dest.starts_with("mailto:") || dest.starts_with("tel:") {
- continue;
- }
-
- if let Ok(dest) = Url::parse(dest).or_else(|_| self.url().join(dest)) {
- links.push(Link {
- source: self.url().clone(),
- destination: dest,
- text: text.trim().to_string(),
- });
- }
- }
- }
-
- links
- }
-
- fn links_tag(&self) -> Vec<Link> {
- let mut links = Vec::new();
-
- for node in self.root.select("link").unwrap() {
- if let Some(element) = node.as_node().as_element() {
- if let Some(href) = element.attributes.borrow().get("href") {
- if let Ok(href) = Url::parse(href).or_else(|_| self.url().join(href)) {
- links.push(Link {
- source: self.url().clone(),
- destination: href,
- text: String::new(),
- });
- }
- }
- }
- }
-
- links
- }
-
- fn metadata_links(&self) -> Vec<Link> {
- self.metadata()
- .into_iter()
- .filter_map(|metadata| {
- // https://github.com/commoncrawl/cc-pyspark/blob/54918e85cf87d47e1f7278965ac04a0fc8e414a0/wat_extract_links.py#L54
-
- if let Some(prop) = metadata.get("property") {
- if matches!(
- prop.as_str(),
- "og:url"
- | "og:image"
- | "og:image:secure_url"
- | "og:video"
- | "og:video:url"
- | "og:video:secure_url"
- | "twitter:url"
- | "twitter:image:src"
- ) {
- if let Some(content) = metadata.get("content") {
- if let Ok(destination) = Url::parse(content.as_str())
- .or_else(|_| self.url().join(content.as_str()))
- {
- return Some(Link {
- source: self.url().clone(),
- destination,
- text: String::new(),
- });
- }
- }
- }
- }
-
- if let Some(name) = metadata.get("name") {
- if matches!(
- name.as_str(),
- "twitter:image"
- | "thumbnail"
- | "application-url"
- | "msapplication-starturl"
- | "msapplication-TileImage"
- | "vb_meta_bburl"
- ) {
- if let Some(content) = metadata.get("content") {
- if let Ok(destination) = Url::parse(content.as_str())
- .or_else(|_| self.url().join(content.as_str()))
- {
- return Some(Link {
- source: self.url().clone(),
- destination,
- text: String::new(),
- });
- }
- }
- }
- }
-
- None
- })
- .collect()
- }
-
- pub fn all_links(&self) -> Vec<Link> {
- let mut links = self.anchor_links();
-
- links.extend(self.scripts().into_iter().filter_map(|script| {
- match script.attributes.get("src") {
- Some(url) => {
- let script_url = Url::parse(url.as_str())
- .or_else(|_| self.url().join(url.as_str()))
- .ok()?;
-
- if script_url.root_domain() != self.url().root_domain() {
- Some(Link {
- source: self.url().clone(),
- destination: script_url,
- text: String::new(),
- })
- } else {
- None
- }
- }
- None => None,
- }
- }));
-
- links.extend(self.links_tag());
- links.extend(self.metadata_links());
-
- links
- }
-
- pub fn favicon(&self) -> Option<FaviconLink> {
- for node in self.root.select("link").unwrap() {
- if !matches!(node.attributes.borrow().get("rel"), Some("icon")) {
- continue;
- }
-
- if let Some(link) = node.attributes.borrow().get("href") {
- let (width, height) = match node.attributes.borrow().get("sizes") {
- Some(size) => {
- if let Some((width, height)) = size.split_once('x') {
- (width.parse().ok(), height.parse().ok())
- } else {
- (None, None)
- }
- }
- _ => (None, None),
- };
-
- let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());
- let link = Url::parse(link).or_else(|_| self.url().join(link)).ok()?;
-
- let favicon = FaviconLink {
- link,
- width,
- height,
- image_type,
- };
-
- return Some(favicon);
- }
- }
-
- None
- }
-
- fn calculate_clean_text(paragraphs: &[Paragraph], lang: &Lang) -> Option<String> {
- let text = JustText::default().extract_from_paragraphs(paragraphs, lang);
-
- if text.is_empty() {
- None
- } else {
- Some(text)
- }
- }
-
- pub fn clean_text(&self) -> Option<&String> {
- self.clean_text.as_ref()
- }
-
- fn calculate_all_text(paragraphs: &[Paragraph], lang: &Lang) -> Option<String> {
- let text = JustText {
- max_link_density: 20.0,
- length_low: 0,
- length_high: 0,
- stopwords_low: -1.0,
- stopwords_high: -1.0,
- max_heading_distance: 10000,
- }
- .extract_from_paragraphs(paragraphs, lang);
-
- if text.is_empty() {
- None
- } else {
- Some(text)
- }
- }
-
- fn all_text(&self) -> Option<String> {
- self.all_text.clone()
- }
-
- pub fn empty_all_text(&self) -> bool {
- match &self.all_text {
- Some(text) => text.is_empty(),
- None => true,
- }
- }
-
- pub fn title(&self) -> Option<String> {
- if let Some(title) = self.root.select_first("title") {
- let title = title.text_contents().trim().to_string();
- if title.is_empty() {
- None
- } else {
- Some(title)
- }
- } else {
- None
- }
- }
-
- fn microformats(&self) -> EnumSet<Microformat> {
- let mut microformats = EnumSet::new();
-
- for node in self.root.inclusive_descendants() {
- if let Some(element) = node.as_element() {
- if let Some(class) = element.attributes.borrow().get("class") {
- for microformat in ALL_MICROFORMATS {
- if class.to_lowercase().as_str() == microformat.as_str() {
- microformats.insert(microformat);
- }
- }
- }
- }
- }
-
- microformats
- }
-
- pub fn url(&self) -> &Url {
- &self.url
- }
-
- pub fn metadata(&self) -> Vec<Meta> {
- let mut metas = Vec::new();
-
- for node in self.root.select("meta").unwrap() {
- if let Some(element) = node.as_node().as_element() {
- metas.push(
- element
- .attributes
- .borrow()
- .map
- .iter()
- .map(|(name, attr)| (name.local.to_string(), attr.value.to_string()))
- .collect(),
- );
- }
- }
-
- metas
+ assert_eq!(
+ webpage.dmoz_description(),
+ Some("dmoz description".to_string())
+ )
}
- fn pretokenize_title(&self) -> Result<PreTokenizedString> {
- let title = self.title();
-
- if title.is_none() {
- return Err(Error::EmptyField("title").into());
- }
- let title = title.unwrap();
-
- Ok(self.pretokenize_string(title))
- }
-
- fn pretokenize_all_text(&self) -> Result<PreTokenizedString> {
- let all_text = self.all_text();
-
- if all_text.is_none() {
- return Err(Error::EmptyField("all body").into());
- }
- let all_text = all_text.unwrap();
-
- Ok(self.pretokenize_string(all_text))
- }
-
- fn pretokenize_clean_text(&self) -> PreTokenizedString {
- let clean_text = self.clean_text().cloned().unwrap_or_default();
- self.pretokenize_string(clean_text)
- }
-
- fn pretokenize_url(&self) -> PreTokenizedString {
- let url = self.url().to_string();
- self.pretokenize_string(url)
- }
-
- fn pretokenize_domain(&self) -> PreTokenizedString {
- let domain = self.url().root_domain().unwrap_or_default().to_string();
-
- self.pretokenize_string(domain)
- }
-
- fn pretokenize_site(&self) -> PreTokenizedString {
- let site = self.url().normalized_host().unwrap_or_default().to_string();
-
- self.pretokenize_string(site)
- }
-
- fn pretokenize_description(&self) -> PreTokenizedString {
- let text = self.description().unwrap_or_default();
-
- self.pretokenize_string(text)
- }
-
- fn pretokenize_microformats(&self) -> PreTokenizedString {
- let mut text = String::new();
-
- for microformat in self.microformats().iter() {
- text.push_str(microformat.as_str());
- text.push(' ');
- }
-
- self.pretokenize_string(text)
- }
-
- fn pretokenize_string(&self, text: String) -> PreTokenizedString {
- self.pretokenize_string_with(text, tokenizer::Tokenizer::default())
- }
-
- fn pretokenize_string_with(
- &self,
- text: String,
- tokenizer: tokenizer::Tokenizer,
- ) -> PreTokenizedString {
- let mut tokenizer = tokenizer;
-
- let mut tokens = Vec::new();
-
- {
- let mut stream = tokenizer.token_stream(&text);
- while let Some(token) = stream.next() {
- tokens.push(token.clone());
- }
- }
-
- PreTokenizedString { text, tokens }
- }
-
- pub fn into_tantivy(self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {
- let mut doc = TantivyDocument::new();
-
- let title = self.pretokenize_title()?;
- let all_text = self.pretokenize_all_text()?;
- let clean_text = self.pretokenize_clean_text();
- let url = self.pretokenize_url();
- let domain = self.pretokenize_domain();
- let site = self.pretokenize_site();
- let description = self.pretokenize_description();
- let microformats = self.pretokenize_microformats();
- let url_for_site_operator = self.pretokenize_string_with(
- self.url().to_string(),
- tokenizer::Tokenizer::SiteOperator(tokenizer::SiteOperatorUrlTokenizer),
- );
-
- let domain_name = self
- .url()
- .root_domain()
- .unwrap_or_default()
- .find('.')
- .map(|index| {
- &domain.text[..ceil_char_boundary(&domain.text, index).min(domain.text.len())]
- })
- .unwrap_or_default()
- .to_string();
-
- let schemas: Vec<_> = self.schema_org();
- let first_ingredient_tag_id =
- find_recipe_first_ingredient_tag_id(&schemas, &self.root).unwrap_or_default();
-
- let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
-
- let pretokenized_schema_json = match schema_org::flattened_json(schemas) {
- Ok(mut f) => {
- let mut tokens = Vec::new();
-
- {
- let mut stream = f.token_stream();
-
- while let Some(token) = stream.next() {
- tokens.push(token.clone());
- }
- }
-
- PreTokenizedString {
- text: f.text().to_string(),
- tokens,
- }
- }
- Err(_) => PreTokenizedString {
- text: String::new(),
- tokens: Vec::new(),
- },
- };
-
- let site_hash = split_u128(hash(self.url().normalized_host().unwrap_or_default()).0);
-
- let mut url_without_query = self.url().clone();
- url_without_query.set_query(None);
-
- let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);
- let url_hash = split_u128(hash(self.url().as_str()).0);
-
- let tld = self.url().tld().unwrap_or_default();
- let url_without_tld = self
- .url()
- .host_str()
- .unwrap_or_default()
- .trim_end_matches(&tld)
- .to_string()
- + "/"
- + self.url().path()
- + "?"
- + self.url().query().unwrap_or_default();
-
- let url_without_tld_hash = split_u128(hash(url_without_tld).0);
-
- let domain_hash = split_u128(hash(self.url().root_domain().unwrap_or_default()).0);
- let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
-
- for field in &ALL_FIELDS {
- let tantivy_field = schema
- .get_field(field.name())
- .unwrap_or_else(|_| panic!("Unknown field: {}", field.name()));
-
- match field {
- Field::Text(TextField::Title) => {
- doc.add_pre_tokenized_text(tantivy_field, title.clone())
- }
- Field::Text(TextField::StemmedTitle) => {
- let mut tokens = title.tokens.clone();
- stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
-
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: title.text.clone(),
- tokens,
- },
- );
- }
- Field::Text(TextField::CleanBody) => {
- doc.add_pre_tokenized_text(tantivy_field, clean_text.clone())
- }
- Field::Text(TextField::StemmedCleanBody) => {
- let mut tokens = clean_text.tokens.clone();
- stem_tokens(&mut tokens, self.lang.unwrap_or(Lang::Eng));
-
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: clean_text.text.clone(),
- tokens,
- },
- );
- }
- Field::Text(TextField::CleanBodyBigrams) => {
- doc.add_text(
- tantivy_field,
- self.clean_text().cloned().unwrap_or_default(),
- );
- }
- Field::Text(TextField::CleanBodyTrigrams) => {
- doc.add_text(
- tantivy_field,
- self.clean_text().cloned().unwrap_or_default(),
- );
- }
- Field::Text(TextField::TitleBigrams) => {
- doc.add_text(tantivy_field, title.text.clone());
- }
- Field::Text(TextField::TitleTrigrams) => {
- doc.add_text(tantivy_field, title.text.clone());
- }
- Field::Text(TextField::Description) => {
- doc.add_pre_tokenized_text(tantivy_field, description.clone());
- }
- Field::Text(TextField::Url) => {
- doc.add_pre_tokenized_text(tantivy_field, url.clone())
- }
- Field::Text(TextField::UrlForSiteOperator) => {
- doc.add_pre_tokenized_text(tantivy_field, url_for_site_operator.clone())
- }
- Field::Text(TextField::UrlNoTokenizer) => {
- let url = self.url().to_string();
-
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: url.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: url.len(),
- position: 0,
- text: url,
- position_length: 1,
- }],
- },
- );
- }
- Field::Text(TextField::SiteWithout) => {
- doc.add_pre_tokenized_text(tantivy_field, site.clone())
- }
- Field::Text(TextField::Domain) => {
- doc.add_pre_tokenized_text(tantivy_field, domain.clone())
- }
- Field::Text(TextField::SiteNoTokenizer) => doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: site.text.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: site.text.len(),
- position: 0,
- text: site.text.clone(),
- position_length: 1,
- }],
- },
- ),
- Field::Text(TextField::SiteIfHomepageNoTokenizer) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: site.text.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: site.text.len(),
- position: 0,
- text: site.text.clone(),
- position_length: 1,
- }],
- },
- )
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainNoTokenizer) => doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: domain.text.clone(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: domain.text.len(),
- position: 0,
- text: domain.text.clone(),
- position_length: 1,
- }],
- },
- ),
- Field::Text(TextField::TitleIfHomepage) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(tantivy_field, title.clone());
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainIfHomepage) => {
- if self.is_homepage() {
- doc.add_text(tantivy_field, domain.text.clone());
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainNameNoTokenizer) => {
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: domain_name.to_string(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: domain_name.len(),
- position: 0,
- text: domain_name.to_string(),
- position_length: 1,
- }],
- },
- );
- }
- Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(
- tantivy_field,
- PreTokenizedString {
- text: domain_name.to_string(),
- tokens: vec![tantivy::tokenizer::Token {
- offset_from: 0,
- offset_to: domain_name.len(),
- position: 0,
- text: domain_name.to_string(),
- position_length: 1,
- }],
- },
- );
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::DomainIfHomepageNoTokenizer) => {
- if self.is_homepage() {
- doc.add_pre_tokenized_text(tantivy_field, domain.clone());
- } else {
- doc.add_text(tantivy_field, "");
- }
- }
- Field::Text(TextField::AllBody) => {
- doc.add_pre_tokenized_text(tantivy_field, all_text.clone())
- }
- Field::Text(TextField::RecipeFirstIngredientTagId) => {
- doc.add_text(tantivy_field, first_ingredient_tag_id.clone());
- }
- Field::Text(TextField::SchemaOrgJson) => {
- doc.add_text(tantivy_field, schema_json.clone());
- }
- Field::Text(TextField::FlattenedSchemaOrgJson) => {
- doc.add_pre_tokenized_text(tantivy_field, pretokenized_schema_json.clone());
- }
- Field::Text(TextField::MicroformatTags) => {
- doc.add_pre_tokenized_text(tantivy_field, microformats.clone());
- }
- Field::Fast(FastField::IsHomepage) => {
- doc.add_u64(tantivy_field, (self.is_homepage()).into());
- }
- Field::Fast(FastField::LastUpdated) => doc.add_u64(
- tantivy_field,
- self.updated_time()
- .map_or(0, |time| time.timestamp().max(0) as u64),
- ),
- Field::Fast(FastField::TrackerScore) => {
- doc.add_u64(tantivy_field, self.trackers().len() as u64)
- }
- Field::Fast(FastField::NumUrlTokens) => {
- doc.add_u64(tantivy_field, url.tokens.len() as u64)
- }
- Field::Fast(FastField::NumMicroformatTagsTokens) => {
- doc.add_u64(tantivy_field, microformats.tokens.len() as u64)
- }
- Field::Fast(FastField::NumTitleTokens) => {
- doc.add_u64(tantivy_field, title.tokens.len() as u64)
- }
- Field::Fast(FastField::NumCleanBodyTokens) => {
- doc.add_u64(tantivy_field, clean_text.tokens.len() as u64)
- }
- Field::Fast(FastField::NumDescriptionTokens) => {
- doc.add_u64(tantivy_field, description.tokens.len() as u64)
- }
- Field::Fast(FastField::NumUrlForSiteOperatorTokens) => {
- doc.add_u64(tantivy_field, url_for_site_operator.tokens.len() as u64)
- }
- Field::Fast(FastField::NumDomainTokens) => {
- doc.add_u64(tantivy_field, domain.tokens.len() as u64)
- }
- Field::Fast(FastField::NumFlattenedSchemaTokens) => {
- doc.add_u64(tantivy_field, pretokenized_schema_json.tokens.len() as u64)
- }
- Field::Fast(FastField::SiteHash1) => {
- doc.add_u64(tantivy_field, site_hash[0]);
- }
- Field::Fast(FastField::SiteHash2) => {
- doc.add_u64(tantivy_field, site_hash[1]);
- }
- Field::Fast(FastField::UrlWithoutQueryHash1) => {
- doc.add_u64(tantivy_field, url_without_query_hash[0]);
- }
- Field::Fast(FastField::UrlWithoutQueryHash2) => {
- doc.add_u64(tantivy_field, url_without_query_hash[1]);
- }
- Field::Fast(FastField::UrlHash1) => {
- doc.add_u64(tantivy_field, url_hash[0]);
- }
- Field::Fast(FastField::UrlHash2) => {
- doc.add_u64(tantivy_field, url_hash[1]);
- }
- Field::Fast(FastField::UrlWithoutTldHash1) => {
- doc.add_u64(tantivy_field, url_without_tld_hash[0]);
- }
- Field::Fast(FastField::UrlWithoutTldHash2) => {
- doc.add_u64(tantivy_field, url_without_tld_hash[1]);
- }
- Field::Fast(FastField::DomainHash1) => {
- doc.add_u64(tantivy_field, domain_hash[0]);
- }
- Field::Fast(FastField::DomainHash2) => {
- doc.add_u64(tantivy_field, domain_hash[1]);
- }
- Field::Fast(FastField::TitleHash1) => {
- doc.add_u64(tantivy_field, title_hash[0]);
- }
- Field::Fast(FastField::TitleHash2) => {
- doc.add_u64(tantivy_field, title_hash[1]);
- }
- Field::Fast(FastField::SimHash) => {
- let hash = if !clean_text.text.is_empty() {
- simhash::hash(&clean_text.text)
- } else {
- 0
- };
- doc.add_u64(tantivy_field, hash);
- }
- Field::Fast(FastField::NumPathAndQuerySlashes) => {
- let num_slashes = self
- .url()
- .path_segments()
- .map(|segments| segments.count())
- .unwrap_or(0);
-
- doc.add_u64(tantivy_field, num_slashes as u64);
- }
- Field::Fast(FastField::NumPathAndQueryDigits) => {
- let num_digits = self
- .url()
- .path()
- .chars()
- .filter(|c| c.is_ascii_digit())
- .count()
- + self
- .url()
- .query()
- .unwrap_or_default()
- .chars()
- .filter(|c| c.is_ascii_digit())
- .count();
-
- doc.add_u64(tantivy_field, num_digits as u64);
- }
- Field::Fast(FastField::LikelyHasAds) => {
- doc.add_u64(tantivy_field, self.likely_has_ads() as u64);
- }
- Field::Fast(FastField::LikelyHasPaywall) => {
- doc.add_u64(tantivy_field, self.likely_has_paywall() as u64);
- }
- Field::Fast(FastField::LinkDensity) => {
- doc.add_u64(
- tantivy_field,
- (self.link_density() * FLOAT_SCALING as f64) as u64,
- );
- }
- Field::Text(TextField::BacklinkText)
- | Field::Text(TextField::SafetyClassification)
- | Field::Text(TextField::InsertionTimestamp)
- | Field::Fast(FastField::HostCentrality)
- | Field::Fast(FastField::PageCentrality)
- | Field::Fast(FastField::FetchTimeMs)
- | Field::Fast(FastField::PreComputedScore)
- | Field::Fast(FastField::Region)
- | Field::Fast(FastField::HostNodeID1)
- | Field::Fast(FastField::HostNodeID2)
- | Field::Text(TextField::DmozDescription) => {}
- }
- }
-
- Ok(doc)
- }
-
- fn scripts(&self) -> Vec<Script> {
- let mut scripts = Vec::new();
-
- for node in self.root.select("script").unwrap() {
- let content = node.text_contents().trim().to_string();
- let attributes = node
- .attributes
- .borrow()
- .map
- .iter()
- .map(|(name, attr)| (name.local.to_string(), attr.value.to_string()))
- .collect();
-
- scripts.push(Script {
- attributes,
- content,
- });
- }
-
- scripts
- }
-
- pub fn schema_org(&self) -> Vec<schema_org::Item> {
- schema_org::parse(self.root.clone())
- }
-
- pub fn trackers(&self) -> Vec<Url> {
- let mut links: Vec<Url> = Vec::new();
-
- for script in self.scripts() {
- if let Some(link) = script
- .attributes
- .get("src")
- .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
- {
- links.push(link);
- }
-
- for res in URL_REGEX.find_iter(&script.content) {
- if let Ok(link) =
- Url::parse(res.as_str()).or_else(|_| self.url().join(res.as_str()))
- {
- links.push(link);
- }
- }
- }
-
- for node in self.root.select("link").unwrap() {
- if let Some(link) = node
- .attributes
- .borrow()
- .get("href")
- .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
- {
- links.push(link);
- }
- }
-
- links
- .into_iter()
- .filter(|link| link.normalized_host().is_some())
- .filter(|link| link.normalized_host() != self.url().normalized_host())
- .unique_by(|link| link.normalized_host().unwrap().to_string())
- .collect()
- }
-
- pub fn likely_has_ads(&self) -> bool {
- for script in self.scripts() {
- if let Some(url) = script
- .attributes
- .get("src")
- .and_then(|url| Url::parse(url).ok())
- {
- if url.root_domain() == self.url().root_domain() {
- continue;
- }
-
- if let Some(domain) = url.root_domain() {
- if AD_SERVERS.is_adserver(domain) {
- return true;
- }
- }
-
- if let Some(host) = url.host_str() {
- if AD_SERVERS.is_adserver(host) {
- return true;
- }
- }
- }
- }
-
- // check <link> tags
- for node in self.root.select("link").unwrap() {
- if let Some(url) = node
- .attributes
- .borrow()
- .get("href")
- .and_then(|url| Url::parse(url).ok())
- {
- if url.root_domain() == self.url().root_domain() {
- continue;
- }
-
- if let Some(domain) = url.root_domain() {
- if AD_SERVERS.is_adserver(domain) {
- return true;
- }
- }
-
- if let Some(host) = url.host_str() {
- if AD_SERVERS.is_adserver(host) {
- return true;
- }
- }
- }
- }
-
- false
- }
-
- pub fn likely_has_paywall(&self) -> bool {
- self.schema_org()
- .into_iter()
- .filter(|item| {
- item.types_contains("NewsArticle")
- || item.types_contains("Article")
- || item.types_contains("BlogPosting")
- || item.types_contains("WebPage")
- || item.types_contains("WebPageElement")
- })
- .any(|item| {
- item.properties
- .get("isAccessibleForFree")
- .and_then(|value| value.clone().one().and_then(|v| v.try_into_string()))
- .map(|value| value.parse().ok().unwrap_or(false))
- .unwrap_or(false)
- })
- }
-
- fn article_modified_time(&self) -> Option<DateTime<FixedOffset>> {
- self.metadata()
- .into_iter()
- .find(|metadata| {
- if let Some(property) = metadata.get("property") {
- property == &String::from("article:modified_time")
- } else {
- false
- }
- })
- .and_then(|metadata| {
- metadata
- .get("content")
- .and_then(|time| DateTime::parse_from_rfc3339(time).ok())
- })
- }
-
- fn og_updated_time(&self) -> Option<DateTime<FixedOffset>> {
- self.metadata()
- .into_iter()
- .find(|metadata| {
- if let Some(property) = metadata.get("property") {
- property == &String::from("og:updated_time")
- } else {
- false
- }
- })
- .and_then(|metadata| {
- metadata
- .get("content")
- .and_then(|time| DateTime::parse_from_rfc3339(time).ok())
- })
- }
-
- fn og_image(&self) -> Option<ImageLink> {
- self.metadata()
- .into_iter()
- .find(|metadata| {
- if let Some(property) = metadata.get("property") {
- property == &String::from("og:image")
- } else {
- false
- }
- })
- .and_then(|metadata| {
- metadata
- .get("content")
- .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
- })
- .map(|url| ImageLink {
- url,
- title: self.og_title(),
- description: self.description(),
- })
- }
-
- #[allow(unreachable_patterns)]
- fn schema_org_images(&self) -> Vec<Url> {
- self.schema_org()
- .into_iter()
- .filter(|item| item.types_contains("ImageObject"))
- .filter_map(|item| {
- item.properties.get("contentUrl").map(|content_url| {
- content_url
- .clone()
- .many()
- .into_iter()
- .filter_map(|url| url.try_into_string())
- .filter_map(|url| Url::parse(&url).or_else(|_| self.url().join(&url)).ok())
- })
- })
- .flatten()
- .collect()
- }
-
- pub fn updated_time(&self) -> Option<DateTime<FixedOffset>> {
- if let Some(time) = self
- .og_updated_time()
- .or_else(|| self.article_modified_time())
- {
- let current_time = Utc::now();
-
- if time > current_time {
- None
- } else {
- Some(time)
- }
- } else {
- None
- }
- }
-
- pub fn primary_image(&self) -> Option<ImageLink> {
- self.og_image().or_else(|| {
- self.schema_org_images()
- .first()
- .cloned()
- .map(|url| ImageLink {
- url,
- title: self.og_title(),
- description: self.description(),
- })
- })
- }
-
- pub fn og_description(&self) -> Option<String> {
- self.metadata()
- .into_iter()
- .find(|metadata| {
- if let Some(property) = metadata.get("property") {
- property.as_str() == "og:description"
- } else {
- false
- }
- })
- .and_then(|metadata| metadata.get("content").cloned())
- }
-
- pub fn metadata_description(&self) -> Option<String> {
- self.metadata()
- .into_iter()
- .find(|metadata| {
- if let Some(name) = metadata.get("name") {
- name.as_str() == "description" || name.as_str() == "Description"
- } else {
- false
- }
- })
- .and_then(|metadata| metadata.get("content").cloned())
- }
-
- pub fn description(&self) -> Option<String> {
- self.og_description()
- .or_else(|| self.metadata_description())
- }
-
- pub fn og_title(&self) -> Option<String> {
- self.metadata()
- .into_iter()
- .find(|metadata| {
- if let Some(property) = metadata.get("property") {
- property == &String::from("og:title")
- } else {
- false
- }
- })
- .and_then(|metadata| metadata.get("content").cloned())
- }
-
- pub fn is_homepage(&self) -> bool {
- self.url().path() == "/" && self.url().query().is_none()
- }
-}
-
-fn stemmer_from_lang(lang: &Lang) -> rust_stemmers::Stemmer {
- match lang {
- Lang::Ara => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Arabic),
- Lang::Dan => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Danish),
- Lang::Nld => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Dutch),
- Lang::Fin => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Finnish),
- Lang::Fra => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::French),
- Lang::Deu => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::German),
- Lang::Ell => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Greek),
- Lang::Hun => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Hungarian),
- Lang::Ita => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Italian),
- Lang::Por => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Portuguese),
- Lang::Ron => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Romanian),
- Lang::Rus => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Russian),
- Lang::Spa => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Spanish),
- Lang::Swe => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Swedish),
- Lang::Tam => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Tamil),
- Lang::Tur => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::Turkish),
- _ => rust_stemmers::Stemmer::create(rust_stemmers::Algorithm::English),
- }
-}
-
-fn find_recipe_first_ingredient_tag_id(
- schemas: &[schema_org::Item],
- root: &NodeRef,
-) -> Option<String> {
- schemas
- .iter()
- .filter_map(|schema| {
- if let Some(ingredients) = schema.properties.get("recipeIngredient") {
- if let Some(ingredient) = ingredients.clone().many().first() {
- if let Some(ingredient) = ingredient.try_into_string() {
- let ingredient = ingredient.trim();
- // find first occurrence in html
- if let Some(ingredient_node) = root
- .select("body")
- .unwrap()
- .flat_map(|node| node.as_node().descendants())
- .find(|node| {
- if let Some(text) = node.as_text() {
- text.borrow().trim() == ingredient
- } else {
- false
- }
- })
- {
- // find first parent that has an id
- if let Some(id) = ingredient_node
- .ancestors()
- .filter_map(|node| {
- node.as_element().and_then(|e| {
- e.attributes.borrow().get("id").map(|s| s.to_string())
- })
- })
- .next()
- {
- return Some(id.to_string());
- }
- }
- }
- }
- }
-
- None
- })
- .next()
-}
-
-fn stem_tokens(tokens: &mut [tantivy::tokenizer::Token], lang: Lang) {
- let stemmer = stemmer_from_lang(&lang);
- for token in tokens {
- // TODO remove allocation
- if let Ok(stemmed_str) = panic::catch_unwind(|| stemmer.stem(&token.text).into_owned()) {
- token.text.clear();
- token.text.push_str(&stemmed_str);
- }
- }
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct Link {
- pub source: Url,
- pub destination: Url,
- pub text: String,
-}
-
-pub type Meta = HashMap<String, String>;
-
-#[cfg(test)]
-mod tests {
- // TODO: make test macro to test both dom parsers
-
- use crate::{schema::create_schema, webpage::url_ext::UrlExt};
-
- use super::*;
-
- const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";
-
- #[test]
- fn simple() {
- let raw = format!(
- r#"
- <html>
- <head>
- <title>Best website</title>
- <meta name="meta1" content="value">
- </head>
- <body>
- <a href="https://example.com">Link to example</a>
- <p>{CONTENT}</p>
- <a href="mailto:[email protected]">Email me</a>
- </body>
- </html>
- "#
- );
-
- let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
-
- assert_eq!(webpage.title(), Some("Best website".to_string()));
-
- assert_eq!(
- webpage.anchor_links(),
- vec![Link {
- source: Url::parse("https://www.example.com/whatever").unwrap(),
- destination: Url::parse("https://example.com").unwrap(),
- text: "Link to example".to_string()
- }]
- );
- assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
-
- let mut expected_meta = HashMap::new();
- expected_meta.insert("name".to_string(), "meta1".to_string());
- expected_meta.insert("content".to_string(), "value".to_string());
-
- assert_eq!(webpage.metadata(), vec![expected_meta]);
- assert_eq!(
- webpage.url().to_string().as_str(),
- "https://www.example.com/whatever"
- );
- assert_eq!(webpage.url().host_str().unwrap(), "www.example.com");
- }
-
- #[test]
- fn empty_title() {
- let raw = format!(
- r#"
- <html>
- <head>
- <title></title>
- </head>
- <body>
- <p>{CONTENT}</p>
- </body>
- </html>
- "#
- );
-
- let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
-
- assert_eq!(webpage.title(), None);
- }
-
- #[test]
- fn text_raw_body() {
- let raw = format!(
- r#"
- <html>
- <body>
- {CONTENT}
- </body>
- </html>
- "#
- );
-
- let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
-
- assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
- }
-
- #[test]
- fn script_tags_text_ignored() {
- let raw = format!(
- r#"
- <html>
- <head>
- <title>Best website</title>
- <meta name="meta1" content="value">
- <script>this should not be extracted</script>
- </head>
- <body>
- <script>this should not be extracted</script>
- <p>{CONTENT}</p>
- <div>
- <script>this should not be extracted</script>
- <p>This text should be the second text extracted</p>
- </div>
- <script>this should not be extracted</script>
- </body>
- </html>
- "#
- );
-
- let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
-
- assert!(!webpage.clean_text().unwrap().contains("not"));
- }
-
- #[test]
- fn style_tags_text_ignored() {
- let raw = format!(
- r#"
- <html>
- <head>
- <title>Best website</title>
- <meta name="meta1" content="value">
- <style>this should not be extracted</style>
- </head>
- <body>
- <style>this should not be extracted</style>
- <p>{CONTENT}</p>
- <div>
- <style>this should not be extracted</style>
- <p>This text should be the second text extracted</p>
- </div>
- <style>this should not be extracted</style>
- </body>
- </html>
- "#
- );
-
- let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
-
- assert!(!webpage.clean_text().unwrap().contains("not"));
- }
-
- #[test]
- fn co_uk_domain() {
- let raw = "";
-
- let webpage = Html::parse(raw, "https://www.domain.co.uk").unwrap();
- assert_eq!(
- webpage.url().root_domain().unwrap_or_default(),
- "domain.co.uk"
- );
- }
-
- #[test]
- fn is_homepage() {
- let webpage = Html::parse("", "https://www.example.com").unwrap();
- assert!(webpage.is_homepage());
-
- let webpage = Html::parse("", "https://www.example.com/").unwrap();
- assert!(webpage.is_homepage());
-
- let webpage = Html::parse("", "https://www.example.com/test").unwrap();
- assert!(!webpage.is_homepage());
-
- let webpage = Html::parse("", "https://example.com/test").unwrap();
- assert!(!webpage.is_homepage());
-
- let webpage = Html::parse("", "https://example.com/").unwrap();
- assert!(webpage.is_homepage());
-
- let webpage = Html::parse("", "https://example.com").unwrap();
- assert!(webpage.is_homepage());
-
- let webpage = Html::parse("", "http://example.com").unwrap();
- assert!(webpage.is_homepage());
- }
-
- #[test]
- fn hard_parsing() {
- let webpage = Html::parse(
- include_str!("../../testcases/parsing/yasudaya.html"),
- "https://example.com",
- )
- .unwrap();
- assert_eq!(
- webpage.title(),
- Some("パチンコ大当たり情報 - Pジューシーハニー3 大当たり詳細ページ - やすだひばりヶ丘店".to_string())
- );
- assert!(webpage.all_text().is_some());
- assert!(!webpage.all_text().unwrap().is_empty());
-
- let webpage = Html::parse(
- include_str!("../../testcases/parsing/5390001.html"),
- "https://example.com",
- )
- .unwrap();
- assert_eq!(
- webpage.title(),
- Some("特效烟机系列_山东壹线文化传播有限公司".to_string())
- );
- assert!(webpage.all_text().is_some());
- assert!(!webpage.all_text().unwrap().is_empty());
-
- let webpage = Html::parse(
- include_str!("../../testcases/parsing/77p2p-7.live-105.html"),
- "https://example.com",
- )
- .unwrap();
- assert_eq!(
- webpage.title(),
- Some("77p2pЅu¤WЖ[¬Э - ҐDјЅ :: іnєс".to_string())
- );
- assert!(webpage.all_text().is_some());
- assert!(!webpage.all_text().unwrap().is_empty());
- }
-
- #[test]
- fn reddit_comments() {
- let webpage = Html::parse(
- include_str!("../../testcases/parsing/reddit.html"),
- "https://reddit.com/",
- )
- .unwrap();
-
- assert!(webpage.clean_text().is_some());
- assert!(webpage.clean_text().unwrap().len() > 1000);
- assert!(webpage
- .all_text()
- .unwrap()
- .contains("They immediately moved outta striking range"));
- }
-
- #[test]
- fn out_of_bounds_str() {
- let webpage = Html::parse(
- include_str!("../../testcases/parsing/byte_index_out_of_bounds.html"),
- "https://example.com",
- )
- .unwrap();
- assert_eq!(webpage.title(), Some("Test".to_string()));
- assert!(webpage.all_text().is_some());
- assert!(!webpage.all_text().unwrap().is_empty());
-
- let schema = create_schema();
- webpage.into_tantivy(&schema).unwrap();
- }
-
- #[test]
- fn simple_favicon() {
- let raw = r#"
- <html>
- <head>
- <link rel="icon" sizes="192x192" href="https://example.com/favicon.png" />
- </head>
- </html>
- "#
- .to_string();
-
- let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
- assert_eq!(
- webpage.favicon(),
- Some(FaviconLink {
- link: Url::parse("https://example.com/favicon.png").unwrap(),
- width: Some(192),
- height: Some(192),
- image_type: None
- })
- );
- }
-
- fn full_link_favicon(href: &str, site_url: &str, expected: &str) {
- let raw = format!(
- r#"
- <html>
- <head>
- <link rel="icon" sizes="192x192" href="{href}" />
- </head>
- </html>
- "#
- );
-
- let webpage = Html::parse(&raw, site_url).unwrap();
- assert_eq!(
- webpage.favicon(),
- Some(FaviconLink {
- link: Url::parse(expected).unwrap(),
- width: Some(192),
- height: Some(192),
- image_type: None
- })
- );
- }
-
- #[test]
- fn test_full_link_favicon_simple() {
- full_link_favicon(
- "/favicon.png",
- "https://www.example.com/",
- "https://www.example.com/favicon.png",
- );
- full_link_favicon(
- "/favicon.png",
- "https://www.example.com",
- "https://www.example.com/favicon.png",
- );
- full_link_favicon(
- "favicon.png",
- "https://www.example.com",
- "https://www.example.com/favicon.png",
- );
- full_link_favicon(
- "favicon.png",
- "https://www.example.com/",
- "https://www.example.com/favicon.png",
- );
- full_link_favicon(
- "favicon.png",
- "https://www.example.com/test/",
- "https://www.example.com/test/favicon.png",
- );
- full_link_favicon(
- "/favicon.png",
- "https://www.example.com/test",
- "https://www.example.com/favicon.png",
- );
- }
-
- #[test]
- fn metadata_updated_time() {
- let html = r#"
- <html>
- <head>
- <meta property="og:updated_time" content="2022-06-22T19:37:34+00:00" />
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "https://example.com").unwrap();
-
- assert_eq!(
- html.updated_time(),
- Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())
- );
-
- let html = r#"
- <html>
- <head>
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "https://example.com").unwrap();
-
- assert_eq!(html.updated_time(), None);
-
- let html = r#"
- <html>
- <head>
- <meta property="og:whutwhut" content="2022-06-22T19:37:34+00:00" />
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "https://example.com").unwrap();
-
- assert_eq!(html.updated_time(), None);
-
- let html = r#"
- <html>
- <head>
- <meta property="og:updated_time" content="2ss022-06-22T19:37:34+00:00" />
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "https://example.com").unwrap();
-
- assert_eq!(html.updated_time(), None);
- }
-
- #[test]
- fn primary_image() {
- let html = r#"
- <html>
- <head>
- <meta property="og:image" content="https://example.com/link_to_image.html" />
- <meta property="og:description" content="desc" />
- <meta property="og:title" content="title" />
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "https://example.com").unwrap();
-
- assert_eq!(
- html.primary_image(),
- Some(ImageLink {
- url: Url::parse("https://example.com/link_to_image.html").unwrap(),
- title: Some("title".to_string()),
- description: Some("desc".to_string())
- })
- );
-
- let html = r#"
- <html>
- <head>
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "https://example.com").unwrap();
-
- assert_eq!(html.primary_image(), None);
-
- let html = r#"
- <html>
- <head>
- <script type="application/ld+json">
- {
- "@context": "https://schema.org",
- "@type": "ImageObject",
- "author": "Jane Doe",
- "contentLocation": "Puerto Vallarta, Mexico",
- "contentUrl": "mexico-beach.jpg",
- "datePublished": "2008-01-25",
- "description": "I took this picture while on vacation last year.",
- "name": "Beach in Mexico"
- }
- </script>
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "https://example.com").unwrap();
-
- assert_eq!(
- html.primary_image(),
- Some(ImageLink {
- url: Url::parse("https://example.com/mexico-beach.jpg").unwrap(),
- title: None,
- description: None
- })
- );
- }
-
- #[test]
- fn future_updated_time_none() {
- let html = r#"
- <html>
- <head>
- <meta property="og:updated_time" content="2122-06-22T19:37:34+00:00" />
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "http://example.com").unwrap();
-
- assert_eq!(html.updated_time(), None);
- }
-
- #[test]
- fn description() {
- let html = r#"
- <html>
- <head>
- <meta property="og:description" content="This is a page description" />
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "http://example.com").unwrap();
-
- assert_eq!(
- html.description(),
- Some("This is a page description".to_string())
- );
-
- let html = r#"
- <html>
- <head>
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "http://example.com").unwrap();
-
- assert_eq!(html.description(), None);
- }
-
- #[test]
- fn article_modified_time() {
- let html = r#"
- <html>
- <head>
- <meta property="article:modified_time" content="2022-06-22T19:37:34+00:00" />
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "http://example.com").unwrap();
-
- assert_eq!(
- html.updated_time(),
- Some(DateTime::parse_from_rfc3339("2022-06-22T19:37:34+00:00").unwrap())
- );
- }
-
- #[test]
- fn trackers() {
- let html = r#"
- <html>
- <head>
- <script>
- !function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","once","off","on","addSourceMiddleware","addIntegrationMiddleware","setAnonymousId","addDestinationMiddleware"];analytics.factory=function(e){return function(){var t=Array.prototype.slice.call(arguments);t.unshift(e);analytics.push(t);return analytics}};for(var e=0;e<analytics.methods.length;e++){var key=analytics.methods[e];analytics[key]=analytics.factory(key)}analytics.load=function(key,e){var t=document.createElement("script");t.type="text/javascript";t.async=!0;t.src="https://cdn.segment.com/analytics.js/v1/" + key + "/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(t,n);analytics._loadOptions=e};analytics._writeKey="";analytics.SNIPPET_VERSION="4.13.2";
- analytics.load("");
- analytics.page();
- }}();
- </script>
- <script>
- (function(h,o,t,j,a,r){
- h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)};
- a.appendChild(r);
- })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
- </script>
- <script src="https://thirdparty.com/js"></script>
- <script src="https://example.com/js"></script>
- <link href='//securepubads.g.doubleclick.net' rel='preconnect'>
- <script src="https://thirdparty.com/js"></script>
- <script src="/js/file"></script>
- </head>
- <body>
- </body>
- </html>
- "#;
- let html = Html::parse(html, "http://example.com").unwrap();
-
- assert_eq!(
- html.trackers()
- .into_iter()
- .map(|url| url.host_str().unwrap().to_string())
- .collect::<Vec<_>>(),
- vec![
- "cdn.segment.com".to_string(),
- "static.hotjar.com".to_string(),
- "thirdparty.com".to_string(),
- "securepubads.g.doubleclick.net".to_string()
- ]
- );
- }
-
- #[test]
- fn parse_title_with_scripts() {
- let html = Html::parse(
- r#"
- <html>
- <head>
- <script>
- !function(){var analytics=window.analytics=window.analytics||[];if(!analytics.initialize)if(analytics.invoked)window.console&&console.error&&console.error("Segment snippet included twice.");else{analytics.invoked=!0;analytics.methods=["trackSubmit","trackClick","trackLink","trackForm","pageview","identify","reset","group","track","ready","alias","debug","page","once","off","on","addSourceMiddleware","addIntegrationMiddleware","setAnonymousId","addDestinationMiddleware"];analytics.factory=function(e){return function(){var t=Array.prototype.slice.call(arguments);t.unshift(e);analytics.push(t);return analytics}};for(var e=0;e<analytics.methods.length;e++){var key=analytics.methods[e];analytics[key]=analytics.factory(key)}analytics.load=function(key,e){var t=document.createElement("script");t.type="text/javascript";t.async=!0;t.src="https://cdn.segment.com/analytics.js/v1/" + key + "/analytics.min.js";var n=document.getElementsByTagName("script")[0];n.parentNode.insertBefore(t,n);analytics._loadOptions=e};analytics._writeKey="";analytics.SNIPPET_VERSION="4.13.2";
- analytics.load("");
- analytics.page();
- }}();
- </script>
- <script>
- (function(h,o,t,j,a,r){
- h.hj=h.hj||function(){(h.hj.q=h.hj.q||[]).push(arguments)};
- a.appendChild(r);
- })(window,document,'https://static.hotjar.com/c/hotjar-','.js?sv=');
- </script>
- <script src="https://thirdparty.com/js"></script>
- <link href='//securepubads.g.doubleclick.net' rel='preconnect'>
- <title>Test site</title>
- </head>
- <body>
- test
- </body>
- </html>
- "#,
- "https://example.com",
- ).unwrap();
-
- assert_eq!(html.title(), Some("Test site".to_string()));
- assert_eq!(html.all_text(), Some("test".to_string()));
- }
-
- #[test]
- fn dmoz_description() {
- let html = Html::parse(
- r#"
- <html>
- <head>
- <title>Test site</title>
- </head>
- <body>
- test
- </body>
- </html>
- "#,
- "https://example.com",
- )
- .unwrap();
-
- let webpage = Webpage {
- html,
- fetch_time_ms: 500,
- dmoz_description: Some("dmoz description".to_string()),
- ..Default::default()
- };
-
- assert_eq!(
- webpage.dmoz_description(),
- Some("dmoz description".to_string())
- )
- }
-
- #[test]
- fn noodp_ignores_dmoz() {
- let html = Html::parse(
- r#"
- <html>
- <head>
- <meta name="robots" content="noodp" />
- <title>Test site</title>
- </head>
- <body>
- test
- </body>
- </html>
- "#,
- "http://example.com",
- )
- .unwrap();
- let webpage = Webpage {
- html,
- fetch_time_ms: 500,
- dmoz_description: Some("dmoz description".to_string()),
- ..Default::default()
- };
+ #[test]
+ fn noodp_ignores_dmoz() {
+ let html = Html::parse(
+ r#"
+ <html>
+ <head>
+ <meta name="robots" content="noodp" />
+ <title>Test site</title>
+ </head>
+ <body>
+ test
+ </body>
+ </html>
+ "#,
+ "http://example.com",
+ )
+ .unwrap();
+ let webpage = Webpage {
+ html,
+ fetch_time_ms: 500,
+ dmoz_description: Some("dmoz description".to_string()),
+ ..Default::default()
+ };
assert_eq!(webpage.dmoz_description(), None)
}
-
- #[test]
- fn links() {
- let raw = format!(
- r#"
- <html>
- <head>
- <title>Best website</title>
- <meta name="meta1" content="value">
- <link href="link.com" />
- <script src="test.com"></script>
- </head>
- <body>
- <a href="https://example.com">Link to example</a>
- <p>{CONTENT}</p>
- </body>
- </html>
- "#
- );
-
- let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
-
- assert_eq!(webpage.title(), Some("Best website".to_string()));
-
- assert_eq!(
- webpage.anchor_links(),
- vec![Link {
- source: Url::parse("https://www.example.com/whatever").unwrap(),
- destination: Url::parse("https://example.com").unwrap(),
- text: "Link to example".to_string()
- },]
- );
- }
-
- #[test]
- fn stackoverflow_question_has_clean_text() {
- let stackoverflow = include_str!("../../testcases/schema_org/stackoverflow_with_code.html");
- let html = Html::parse(stackoverflow, "https://www.example.com").unwrap();
-
- assert!(html.clean_text().is_some());
- }
-
- #[test]
- fn canonical_url() {
- let html = Html::parse(
- r#"
- <html>
- <head>
- <link rel="canonical" href="https://example.com/canonical.html" />
- </head>
- <body>
- </body>
- </html>
- "#,
- "https://www.example.com/whatever",
- )
- .unwrap();
-
- assert_eq!(
- html.canonical_url(),
- Some(Url::parse("https://example.com/canonical.html").unwrap())
- );
-
- let html = Html::parse(
- r#"
- <html>
- <head>
- </head>
- <body>
- </body>
- </html>
- "#,
- "https://www.example.com/whatever",
- )
- .unwrap();
-
- assert_eq!(html.canonical_url(), None);
- assert_eq!(
- html.url(),
- &Url::parse("https://www.example.com/whatever").unwrap()
- );
- }
-
- #[test]
- fn robots_meta_tag() {
- let html = Html::parse(
- r#"
- <html>
- <head>
- <meta name="robots" content="noindex, nofollow" />
- </head>
- <body>
- </body>
- </html>
- "#,
- "https://www.example.com/whatever",
- )
- .unwrap();
-
- assert!(html.is_no_index());
- assert!(html.is_no_follow());
-
- let html = Html::parse(
- r#"
- <html>
- <head>
- <meta name="robots" content="noindex,nofollow" />
- </head>
- <body>
- </body>
- </html>
- "#,
- "https://www.example.com/whatever",
- )
- .unwrap();
-
- assert!(html.is_no_index());
- assert!(html.is_no_follow());
-
- let html = Html::parse(
- r#"
- <html>
- <head>
- <meta name="robots" content="noindex" />
- </head>
- <body>
- </body>
- </html>
- "#,
- "https://www.example.com/whatever",
- )
- .unwrap();
-
- assert!(html.is_no_index());
- assert!(!html.is_no_follow());
-
- let html = Html::parse(
- r#"
- <html>
- <head>
- <meta name="robots" content="nofollow" />
- </head>
- <body>
- </body>
- </html>
- "#,
- "https://www.example.com/whatever",
- )
- .unwrap();
-
- assert!(!html.is_no_index());
- assert!(html.is_no_follow());
-
- let html = Html::parse(
- r#"
- <html>
- <head>
- </head>
- <body>
- </body>
- </html>
- "#,
- "https://www.example.com/whatever",
- )
- .unwrap();
-
- assert!(!html.is_no_index());
- assert!(!html.is_no_follow());
- }
-
- #[test]
- fn microformats() {
- let html = Html::parse(
- r#"
- <html>
- <head>
- </head>
- <body>
- <article class="h-entry">
- <h1 class="p-name">Microformats are amazing</h1>
- <p class="e-content">This is the content of the article</p>
- <a class="u-url" href="https://example.com/microformats">Permalink</a>
- <a class="u-author" href="https://example.com">Author</a>
- <p class="search-product">substrings should not match</p>
- <time class="dt-published" datetime="2021-01-01T00:00:00+00:00">2021-01-01</time>
- </article>
-
- <div class="h-RECIPE">
- For some reason this site also has a recipe
- </div>
- </body>
- </html>
- "#,
- "https://www.example.com/",
- ).unwrap();
-
- let microformats = html.microformats();
-
- assert!(microformats.contains(Microformat::HEntry));
- assert!(microformats.contains(Microformat::HRecipe));
- assert!(!microformats.contains(Microformat::HCard));
- assert!(!microformats.contains(Microformat::HProduct));
- }
-
- #[test]
- fn recipe_first_ingredient_tag() {
- let html = Html::parse(
- r#"
- <html>
- <head>
- </head>
- <body>
- <script type="application/ld+json">
- {
- "@context": "https://schema.org",
- "@type": "Recipe",
- "author": "John Smith",
- "cookTime": "PT1H",
- "datePublished": "2009-05-08",
- "description": "This classic banana bread recipe comes from my mom -- the walnuts add a nice texture and flavor to the banana bread.",
- "image": "bananabread.jpg",
- "recipeIngredient": [
- "3 or 4 ripe bananas, smashed",
- "1 egg",
- "3/4 cup of sugar"
- ],
- "interactionStatistic": {
- "@type": "InteractionCounter",
- "interactionType": "https://schema.org/Comment",
- "userInteractionCount": "140"
- },
- "name": "Mom's World Famous Banana Bread",
- "nutrition": {
- "@type": "NutritionInformation",
- "calories": "240 calories",
- "fatContent": "9 grams fat"
- },
- "prepTime": "PT15M",
- "recipeInstructions": "Preheat the oven to 350 degrees. Mix in the ingredients in a bowl. Add the flour last. Pour the mixture into a loaf pan and bake for one hour.",
- "recipeYield": "1 loaf",
- "suitableForDiet": "https://schema.org/LowFatDiet"
- }
- </script>
-
- <div id="ingredients">
- <h2>Ingredients</h2>
- <ul>
- <li>3 or 4 ripe bananas, smashed</li>
- <li>1 egg</li>
- <li>3/4 cup of sugar</li>
- </ul>
- </body>
- </html>
- "#,
- "https://www.example.com/",
- ).unwrap();
-
- let schemas = html.schema_org();
-
- assert_eq!(
- find_recipe_first_ingredient_tag_id(&schemas, &html.root),
- Some("ingredients".to_string())
- );
- }
}