Skip to content

web search done right

5.9 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

mod scorer;
mod weight;

use lending_iter::LendingIterator;
use optics::PatternPart;

use tantivy::tokenizer::Tokenizer;

use crate::{
    numericalfield_reader::NumericalFieldReader,
    schema::{text_field::TextField, Field, TextFieldEnum},
};

use self::weight::{FastSiteDomainPatternWeight, PatternWeight};

#[derive(Clone)]
pub struct PatternQuery {
    patterns: Vec<PatternPart>,
    can_optimize_site_domain: bool,
    field: tantivy::schema::Field,
    raw_terms: Vec<tantivy::Term>,
    columnfield_reader: NumericalFieldReader,
}

impl std::fmt::Debug for PatternQuery {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("PatternQuery")
            .field("patterns", &self.patterns)
            .field("field", &self.field)
            .field("raw_terms", &self.raw_terms)
            .finish()
    }
}

impl PatternQuery {
    pub fn new(
        patterns: Vec<PatternPart>,
        field: TextFieldEnum,
        schema: &tantivy::schema::Schema,
        columnfield_reader: NumericalFieldReader,
    ) -> Self {
        let field = Field::Text(field);
        let tv_field = schema.get_field(field.name()).unwrap();

        if can_optimize_site_domain(&patterns, field) {
            if patterns.len() == 3 {
                let PatternPart::Raw(term) = &patterns[1] else {
                    unreachable!()
                };

                return Self {
                    patterns: Vec::new(),
                    field: tv_field,
                    can_optimize_site_domain: true,
                    raw_terms: vec![tantivy::Term::from_field_text(tv_field, term.as_str())],
                    columnfield_reader,
                };
            } else {
                let term: String = patterns
                    .iter()
                    .filter_map(|p| match p {
                        PatternPart::Raw(s) => Some(s.clone()),
                        PatternPart::Wildcard => None,
                        PatternPart::Anchor => None,
                    })
                    .collect();

                return Self {
                    patterns,
                    field: tv_field,
                    can_optimize_site_domain: true,
                    raw_terms: vec![tantivy::Term::from_field_text(tv_field, &term)],
                    columnfield_reader,
                };
            }
        }

        let mut raw_terms = Vec::with_capacity(patterns.len());
        let mut new_patterns = Vec::with_capacity(patterns.len());

        for pattern in &patterns {
            match pattern {
                PatternPart::Raw(text) => {
                    let mut tokenizer = field
                        .as_text()
                        .unwrap()
                        .tokenizer(Some(&whatlang::Lang::Eng));
                    let mut stream = tokenizer.token_stream(text);
                    let mut it = tantivy::tokenizer::TokenStream::iter(&mut stream);

                    while let Some(token) = it.next() {
                        new_patterns.push(PatternPart::Raw(token.text.clone()));
                        let term = tantivy::Term::from_field_text(tv_field, &token.text);
                        raw_terms.push(term);
                    }
                }
                PatternPart::Wildcard => new_patterns.push(PatternPart::Wildcard),
                PatternPart::Anchor => new_patterns.push(PatternPart::Anchor),
            }
        }

        raw_terms.shrink_to_fit();

        Self {
            patterns: new_patterns,
            field: tv_field,
            raw_terms,
            can_optimize_site_domain: false,
            columnfield_reader,
        }
    }
}

impl tantivy::query::Query for PatternQuery {
    fn weight(
        &self,
        _scoring: tantivy::query::EnableScoring<'_>,
    ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
        if self.can_optimize_site_domain {
            return Ok(Box::new(FastSiteDomainPatternWeight {
                term: self.raw_terms[0].clone(),
                field: self.field,
            }));
        }

        Ok(Box::new(PatternWeight {
            raw_terms: self.raw_terms.clone(),
            patterns: self.patterns.clone(),
            field: self.field,
            columnfield_reader: self.columnfield_reader.clone(),
        }))
    }

    fn query_terms<'a>(&'a self, visitor: &mut dyn FnMut(&'a tantivy::Term, bool)) {
        for term in &self.raw_terms {
            visitor(term, true);
        }
    }
}

#[derive(Debug)]
pub enum SmallPatternPart {
    Term,
    Wildcard,
    Anchor,
}

/// if pattern is of form Site("|site|") or Domain("|domain|")
/// we can use the field without tokenization to speed up the query significantly
fn can_optimize_site_domain(patterns: &[PatternPart], field: Field) -> bool {
    patterns.len() >= 2
        && matches!(&patterns[0], PatternPart::Anchor)
        && matches!(&patterns[patterns.len() - 1], PatternPart::Anchor)
        && patterns[1..patterns.len() - 1]
            .iter()
            .all(|pattern| matches!(pattern, PatternPart::Raw(_)))
        && (matches!(field, Field::Text(TextFieldEnum::UrlForSiteOperator(_)))
            || matches!(field, Field::Text(TextFieldEnum::Domain(_))))
}