Skip to content

web search done right

3.0 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

pub mod url;

use enum_dispatch::enum_dispatch;
use strum::{EnumDiscriminants, VariantArray};
use tantivy::tokenizer::{BoxTokenStream, Tokenizer as _};
use url::UrlTokenStream;

#[enum_dispatch]
pub trait Tokenizer: Into<TokenizerEnum> + Clone {
    fn name(&self) -> &'static str;
    fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a>;

    fn into_tantivy(self) -> TantivyBridge {
        TantivyBridge::from(self.into())
    }
}

#[derive(Clone, Default)]
pub struct SimpleTokenizer(tantivy::tokenizer::SimpleTokenizer);

impl Tokenizer for SimpleTokenizer {
    fn name(&self) -> &'static str {
        "simple"
    }

    fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
        BoxTokenStream::new(self.0.token_stream(text))
    }
}

#[derive(Clone, Default)]
pub struct UrlTokenizer;

impl Tokenizer for UrlTokenizer {
    fn name(&self) -> &'static str {
        "url"
    }

    fn token_stream<'a>(&'a mut self, text: &'a str) -> BoxTokenStream<'a> {
        BoxTokenStream::new(UrlTokenStream::new(text))
    }
}

#[enum_dispatch(Tokenizer)]
#[derive(Clone, EnumDiscriminants)]
#[strum_discriminants(derive(VariantArray))]
pub enum TokenizerEnum {
    SimpleTokenizer,
    UrlTokenizer,
}

impl TokenizerEnum {
    pub fn iter() -> impl Iterator<Item = Self> {
        TokenizerEnumDiscriminants::VARIANTS
            .iter()
            .map(|d| (*d).into())
    }
}

impl Default for TokenizerEnum {
    fn default() -> Self {
        TokenizerEnum::SimpleTokenizer(SimpleTokenizer::default())
    }
}

impl From<TokenizerEnumDiscriminants> for TokenizerEnum {
    fn from(value: TokenizerEnumDiscriminants) -> Self {
        match value {
            TokenizerEnumDiscriminants::SimpleTokenizer => SimpleTokenizer::default().into(),
            TokenizerEnumDiscriminants::UrlTokenizer => UrlTokenizer.into(),
        }
    }
}

#[derive(Clone)]
pub struct TantivyBridge(TokenizerEnum);

impl<T> From<T> for TantivyBridge
where
    T: Tokenizer + Sync + Send + 'static,
{
    fn from(value: T) -> Self {
        Self(value.into())
    }
}

impl tantivy::tokenizer::Tokenizer for TantivyBridge {
    type TokenStream<'a> = BoxTokenStream<'a>;

    fn token_stream<'a>(&'a mut self, text: &'a str) -> Self::TokenStream<'a> {
        self.0.token_stream(text)
    }
}