Skip to content

web search done right

6.1 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use itertools::Itertools;

use utoipa::ToSchema;

#[derive(Debug)]
pub struct Entity {
    pub article_url: String,
    pub is_disambiguation: bool,
    pub title: String,
    pub page_abstract: Span,
    pub info: Vec<(String, Span)>,
    pub image: Option<String>,
}

#[derive(
    Debug,
    Default,
    Clone,
    PartialEq,
    Eq,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
)]
pub struct Span {
    pub text: String,
    pub links: Vec<Link>,
}

impl Span {
    #[cfg(test)]
    pub fn new(text: &str) -> Self {
        Self {
            text: text.to_string(),
            links: vec![],
        }
    }

    pub fn merge(&mut self, other_span: Span) {
        let orig_end = self.text.len();
        self.text.push_str(&other_span.text);

        for link in other_span.links {
            self.links.push(Link {
                start: orig_end + link.start,
                end: orig_end + link.end,
                target: link.target,
            });
        }
    }

    pub fn add_text(&mut self, text: &str) {
        self.text.push_str(text);
    }

    pub fn add_link(&mut self, text: &str, target: String) {
        let link = Link {
            target,
            start: self.text.len(),
            end: self.text.len() + text.len(),
        };
        self.links.push(link);
        self.text.push_str(text);
    }

    pub fn trim_end(&mut self) {
        self.text = self.text.trim_end().to_string();

        while let Some(last_link) = self.links.last() {
            if last_link.end > self.text.len() {
                self.links.pop();
            } else {
                break;
            }
        }
    }

    pub fn text(&self) -> &str {
        &self.text
    }
}

#[derive(
    Debug,
    PartialEq,
    Eq,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
    Clone,
)]
pub struct Link {
    pub start: usize,
    pub end: usize,
    pub target: String,
}

#[derive(
    Debug,
    Clone,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
    PartialEq,
    ToSchema,
)]
#[serde(rename_all = "camelCase", tag = "kind")]
pub enum EntitySnippetFragment {
    Normal { text: String },
    Link { text: String, href: String },
}

#[derive(
    Default,
    Debug,
    Clone,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
    PartialEq,
    ToSchema,
)]
#[serde(rename_all = "camelCase")]
pub struct EntitySnippet {
    pub fragments: Vec<EntitySnippetFragment>,
}

impl EntitySnippetFragment {
    pub fn text(&self) -> &str {
        match self {
            EntitySnippetFragment::Normal { text } | EntitySnippetFragment::Link { text, .. } => {
                text
            }
        }
    }
    pub fn text_mut(&mut self) -> &mut String {
        match self {
            EntitySnippetFragment::Normal { text } | EntitySnippetFragment::Link { text, .. } => {
                text
            }
        }
    }
}

impl EntitySnippet {
    pub fn from_span(span: &Span, truncate_to: usize) -> Self {
        let (s, maybe_ellipsis) = if span.text.len() > truncate_to {
            let truncate_to = crate::floor_char_boundary(&span.text, truncate_to);
            (&span.text[0..truncate_to], "...")
        } else {
            (&*span.text, "")
        };

        let mut last_end = 0;
        let mut fragments = span
            .links
            .iter()
            .filter(|link| s.len() > link.start)
            .flat_map(|link| {
                let end = link.end.min(s.len());
                let split = std::mem::replace(&mut last_end, end);
                [
                    EntitySnippetFragment::Normal {
                        text: s[split..link.start].to_string(),
                    },
                    EntitySnippetFragment::Link {
                        text: s[link.start..end].to_string(),
                        href: format!(
                            "https://en.wikipedia.org/wiki/{}",
                            link.target.replace(' ', "_"),
                        ),
                    },
                ]
            })
            .filter(|s| !s.text().is_empty())
            .collect_vec();

        let remainder = s[last_end..].to_string() + maybe_ellipsis;

        if !remainder.is_empty() {
            match fragments.last_mut() {
                Some(EntitySnippetFragment::Normal { text }) => *text += &remainder,
                _ => fragments.push(EntitySnippetFragment::Normal { text: remainder }),
            }
        }

        EntitySnippet { fragments }
    }

    #[cfg(test)]
    pub fn to_md(&self, strip_href_prefix: Option<&str>) -> String {
        self.fragments
            .iter()
            .map(|s| match (s, strip_href_prefix) {
                (EntitySnippetFragment::Normal { text }, _) => text.clone(),
                (EntitySnippetFragment::Link { text, href }, Some(prefix))
                    if &href.trim_start_matches(prefix).replace('_', " ") == text =>
                {
                    format!("[[{}]]", text)
                }
                (EntitySnippetFragment::Link { text, href }, Some(prefix)) => {
                    format!("[{}]({})", text, href.trim_start_matches(prefix))
                }
                (EntitySnippetFragment::Link { text, href }, None) => {
                    format!("[{}]({href})", text)
                }
            })
            .join("")
    }
}