Skip to content

web search done right

4.0 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use super::{script::Script, Token};

/// A segment is a part of a text where the entire segment has the same script and langage.
#[derive(Clone)]
pub struct Segment<'a> {
    full_text: &'a str,
    span: std::ops::Range<usize>,
    script: Script,
}

impl<'a> Segment<'a> {
    pub fn text(&self) -> &'a str {
        &self.full_text[self.span.clone()]
    }

    pub fn tokenize(&self) -> impl Iterator<Item = Token<'a>> + 'a {
        let offset = self.span.start;
        let script = self.script;

        script
            .tokenizer()
            .tokenize(self.text())
            .map(move |mut token| {
                token.offset(offset);
                token
            })
    }
}

pub trait Segmenter {
    fn segments(&self) -> SegmentIterator;
}

impl Segmenter for str {
    fn segments(&self) -> SegmentIterator<'_> {
        SegmentIterator::new(self)
    }
}

impl Segmenter for String {
    fn segments(&self) -> SegmentIterator<'_> {
        SegmentIterator::new(self)
    }
}

pub struct SegmentIterator<'a> {
    prev_end: usize,
    input: &'a str,
}

impl<'a> SegmentIterator<'a> {
    pub fn new(input: &'a str) -> Self {
        Self { input, prev_end: 0 }
    }
}

impl<'a> Iterator for SegmentIterator<'a> {
    type Item = Segment<'a>;

    fn next(&mut self) -> Option<Self::Item> {
        if self.prev_end >= self.input.len() {
            return None;
        }

        let start = self.prev_end;
        let mut end = start;
        let mut script = None;

        while end < self.input.len() {
            let c = self.input[end..].chars().next().unwrap();
            let next_script = Script::from(c);

            if let Some(script) = &script {
                if &next_script != script && next_script != Script::Other {
                    break;
                }
            } else {
                script = Some(next_script);
            }

            end += c.len_utf8();
        }

        self.prev_end = end;

        Some(Segment {
            script: script.unwrap_or_default(),
            full_text: self.input,
            span: start..end,
        })
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use proptest::prelude::*;

    #[test]
    fn test_segments() {
        let txt = "Hello, world! This is a test.";
        let segments: Vec<_> = txt.segments().collect();
        assert_eq!(segments.len(), 1);
        assert_eq!(segments[0].text(), txt);
        assert_eq!(segments[0].script, Script::Latin);

        let txt = "こんにちは、世界!";
        let segments: Vec<_> = txt.segments().collect();

        assert_eq!(segments.len(), 1);
        assert_eq!(segments[0].text(), txt);
        assert_eq!(segments[0].script, Script::Other);

        let txt = "Hello, こんにちは、世界!";
        let segments: Vec<_> = txt.segments().collect();

        // TODO: this should be split into multiple segments
        // when we have more script tokenizers than just latin
        assert_eq!(segments.len(), 1);
        assert_eq!(segments[0].text(), "Hello, こんにちは、世界!");
        assert_eq!(segments[0].script, Script::Latin);
    }

    proptest! {
        #[test]
        fn proptest_byte_offsets(txt in ".*") {
            for segment in txt.segments() {
                assert!(!segment.text().is_empty());
            }
        }
    }
}