Skip to content

web search done right

7.8 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2023 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use url::Url;
use utoipa::ToSchema;

use crate::{
    inverted_index::RetrievedWebpage,
    webpage::schema_org::{self, Item, Property},
    Error, OneOrMany,
};

use super::DisplayedSidebar;
use crate::Result;

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,
)]
#[serde(rename_all = "camelCase")]
pub struct StackOverflowAnswer {
    pub body: Vec<CodeOrText>,
    pub date: String,
    pub url: String,
    pub upvotes: u32,
    pub accepted: bool,
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,
)]
#[serde(rename_all = "camelCase")]
pub struct StackOverflowQuestion {
    pub body: Vec<CodeOrText>,
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,
)]
#[serde(tag = "_type", content = "value", rename_all = "camelCase")]
pub enum CodeOrText {
    Code(String),
    Text(String),
}

fn parse_code(item: schema_org::Item) -> Option<CodeOrText> {
    let s = item
        .properties
        .get("text")
        .and_then(|p| p.clone().one())
        .and_then(|prop| prop.try_into_string())
        .unwrap_or_default()
        .to_string();

    if s.is_empty() {
        return None;
    }

    Some(CodeOrText::Code(s))
}

fn parse_so_answer(
    text: OneOrMany<Property>,
    date: OneOrMany<Property>,
    upvotes: OneOrMany<Property>,
    url: OneOrMany<Property>,
    webpage_url: Url,
    accepted: bool,
) -> Option<StackOverflowAnswer> {
    let text: Vec<_> = text
        .many()
        .into_iter()
        .filter_map(|prop| match prop {
            Property::String(s) => Some(CodeOrText::Text(s)),
            Property::Item(item) => parse_code(item),
        })
        .collect();

    let date = chrono::NaiveDateTime::parse_from_str(
        date.one()
            .and_then(|prop| prop.try_into_string())
            .unwrap_or_default()
            .as_str(),
        "%Y-%m-%dT%H:%M:%S",
    )
    .ok()?;

    let upvotes = upvotes
        .one()
        .and_then(|prop| prop.try_into_string())
        .and_then(|s| s.parse().ok())?;

    let url = url
        .one()
        .and_then(|prop| prop.try_into_string())
        .and_then(|s| Url::parse(&s).or_else(|_| webpage_url.join(&s)).ok())?;

    Some(StackOverflowAnswer {
        body: text,
        date: format!("{}", date.date().format("%b %d, %Y")),
        upvotes,
        url: url.to_string(),
        accepted,
    })
}

fn schema_item_to_stackoverflow_answer(
    item: schema_org::Item,
    url: Url,
    accepted: bool,
) -> Option<StackOverflowAnswer> {
    match (
        item.properties.get("text"),
        item.properties.get("dateCreated"),
        item.properties.get("upvoteCount"),
        item.properties.get("url"),
    ) {
        (Some(text), Some(date), Some(upvotes), Some(answer_url)) => parse_so_answer(
            text.clone(),
            date.clone(),
            upvotes.clone(),
            answer_url.clone(),
            url,
            accepted,
        ),
        _ => None,
    }
}

/// Limit the number of characters in a snippet
/// to be less than `limit` characters. At least a single
/// passage is returned, so the total number of characters
/// in the returned passages may be more than `limit`.
///
/// This is useful for limiting the number of characters
/// we send to the frontend.
fn limit_chars(passages: &[CodeOrText], limit: usize) -> Vec<CodeOrText> {
    let mut res = Vec::new();
    let mut taken_chars = 0;

    for p in passages {
        res.push(p.clone());
        let s = match p {
            CodeOrText::Code(s) => s,
            CodeOrText::Text(s) => s,
        };

        if taken_chars + s.len() > limit {
            break;
        }

        taken_chars += s.len();
    }

    res
}

pub fn stackoverflow_snippet(
    webpage: &RetrievedWebpage,
) -> Result<(StackOverflowQuestion, Vec<StackOverflowAnswer>)> {
    let schema_org = &webpage.schema_org;

    match schema_org
        .iter()
        .find(|item| item.types_contains("QAPage"))
        .and_then(|item| item.properties.get("mainEntity"))
        .and_then(|properties| properties.clone().one())
        .and_then(|property| property.try_into_item())
    {
        Some(item) => {
            let question: Vec<CodeOrText> = item
                .properties
                .get("text")
                .map(|item| item.clone().many())
                .unwrap_or_default()
                .into_iter()
                .map(|prop| match prop {
                    Property::String(s) => CodeOrText::Text(s),
                    Property::Item(item) => parse_code(item).unwrap(),
                })
                .collect();

            let mut answers = Vec::new();

            let url = Url::parse(&webpage.url).unwrap();

            if let Some(ans) = get_accepted_answer(&item, url) {
                answers.push(ans);
            }

            for answer in item
                .properties
                .get("suggestedAnswer")
                .cloned()
                .map(|answers| answers.many())
                .unwrap_or_default()
                .into_iter()
                .filter_map(|prop| prop.try_into_item())
                .filter_map(|item| {
                    schema_item_to_stackoverflow_answer(
                        item,
                        Url::parse(&webpage.url).unwrap(),
                        false,
                    )
                })
            {
                answers.push(answer);
            }

            let question = StackOverflowQuestion {
                body: limit_chars(&question, 512),
            };

            let answers: Vec<_> = answers
                .into_iter()
                .map(|answer| StackOverflowAnswer {
                    body: limit_chars(&answer.body, 512),
                    ..answer
                })
                .take(3)
                .collect();

            Ok((question, answers))
        }
        None => Err(Error::InvalidStackoverflowSchema.into()),
    }
}

pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: Url) -> Result<DisplayedSidebar> {
    if let Some(item) = schema_org
        .into_iter()
        .find(|item| item.types_contains("QAPage"))
        .and_then(|item| item.properties.get("mainEntity").cloned())
        .and_then(|properties| properties.one())
        .and_then(|property| property.try_into_item())
    {
        let title = item
            .properties
            .get("name")
            .cloned()
            .and_then(|prop| prop.one())
            .and_then(|prop| prop.try_into_string())
            .ok_or(Error::InvalidStackoverflowSchema)?;

        get_accepted_answer(&item, url)
            .map(|answer| DisplayedSidebar::StackOverflow { title, answer })
            .ok_or(Error::InvalidStackoverflowSchema.into())
    } else {
        Err(Error::InvalidStackoverflowSchema.into())
    }
}

fn get_accepted_answer(item: &Item, url: Url) -> Option<StackOverflowAnswer> {
    let item = item
        .properties
        .get("acceptedAnswer")
        .cloned()?
        .one()?
        .try_into_item()?;

    schema_item_to_stackoverflow_answer(item, url, true)
}