Skip to content

web search done right

12.6 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::{
    config::defaults,
    enum_map::EnumMap,
    ranking::{SignalCoefficients, SignalEnum, SignalEnumDiscriminants},
    search_prettifier::{DisplayedSidebar, HighlightedSpellCorrection},
    widgets::Widget,
};
use http::StatusCode;
use optics::{HostRankings, Optic};
use std::{collections::HashMap, sync::Arc};
use utoipa::ToSchema;

use axum::Json;
use axum_macros::debug_handler;

use crate::{
    bangs::BangHit,
    searcher::{self, SearchQuery, SearchResult, WebsitesResult},
    webpage::region::Region,
};

use super::State;

use axum::{extract, response::IntoResponse};

#[derive(
    Clone,
    Copy,
    Debug,
    serde::Serialize,
    serde::Deserialize,
    bincode::Encode,
    bincode::Decode,
    ToSchema,
)]
#[serde(tag = "_type", content = "value", rename_all = "camelCase")]
pub enum ReturnBody {
    All,
    Truncated(usize),
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,
)]
#[serde(rename_all = "camelCase")]
#[schema(title = "SearchQuery", example = json!({"query": "hello world"}))]
pub struct ApiSearchQuery {
    /// The search query string
    pub query: String,

    /// The page number to return
    pub page: Option<usize>,

    /// The number of results to return per page (max: 100)
    pub num_results: Option<usize>,

    /// Prioritize results for a specific geographic region
    pub selected_region: Option<Region>,

    /// Apply an `optic` to the query
    pub optic: Option<String>,

    /// Custom host ranking preferences
    pub host_rankings: Option<HostRankings>,

    /// Enable/disable nsfw filtering
    pub safe_search: Option<bool>,

    /// Custom weights for ranking signals
    pub signal_coefficients: Option<HashMap<SignalEnumDiscriminants, f64>>,

    /// Include ranking signal scores in results
    #[serde(default = "defaults::SearchQuery::return_ranking_signals")]
    pub return_ranking_signals: bool,

    /// Return flattened result format
    #[serde(default = "defaults::SearchQuery::flatten_response")]
    pub flatten_response: bool,

    /// Get exact vs estimated result count
    #[serde(default = "defaults::SearchQuery::count_results_exact")]
    pub count_results_exact: bool,

    /// Include structured schema.org data in results
    #[serde(default = "defaults::SearchQuery::return_structured_data")]
    pub return_structured_data: bool,

    /// Control whether or not the page content is returned
    #[cfg(feature = "return_body")]
    pub return_body: Option<ReturnBody>,
}

impl TryFrom<ApiSearchQuery> for SearchQuery {
    type Error = anyhow::Error;

    fn try_from(api: ApiSearchQuery) -> Result<Self, Self::Error> {
        let optic = if let Some(optic) = &api.optic {
            Some(Optic::parse(optic)?)
        } else {
            None
        };

        let signal_coefficients: Option<SignalCoefficients> =
            api.signal_coefficients.map(|coefficients| {
                coefficients
                    .into_iter()
                    .map(|(signal, coefficient)| (signal.into(), coefficient))
                    .collect::<EnumMap<SignalEnum, f64>>()
                    .into()
            });

        let default = SearchQuery::default();

        Ok(SearchQuery {
            query: api.query,
            page: api.page.unwrap_or(default.page),
            num_results: api.num_results.unwrap_or(default.num_results),
            selected_region: api.selected_region,
            optic,
            host_rankings: api.host_rankings,
            return_ranking_signals: api.return_ranking_signals,
            safe_search: api.safe_search.unwrap_or(default.safe_search),
            count_results_exact: api.count_results_exact,
            signal_coefficients: signal_coefficients.unwrap_or(default.signal_coefficients),
            #[cfg(feature = "return_body")]
            return_body: api.return_body,
            #[cfg(not(feature = "return_body"))]
            return_body: None,
            return_structured_data: api.return_structured_data,
        })
    }
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,
)]
#[serde(tag = "_type", rename_all = "camelCase")]
pub enum ApiSearchResult {
    Websites(WebsitesResult),
    Bang(Box<BangHit>),
}

impl From<SearchResult> for ApiSearchResult {
    fn from(result: SearchResult) -> Self {
        match result {
            SearchResult::Websites(result) => ApiSearchResult::Websites(result),
            SearchResult::Bang(result) => ApiSearchResult::Bang(result),
        }
    }
}

/// Web Search
///
/// The main search endpoint that powers Stract's web search functionality. It performs a full-text search
/// across all pages in the index and returns the most relevant results.
///
/// The endpoint supports the [optic syntax](https://github.com/StractOrg/sample-optics/blob/main/quickstart.optic)
/// that can be used to customize the search results and perform advanced filtering.
/// If the query matches the bang prefix `!`, the result will be a redirect to the bang target.
/// For example, `!w cats` redirects to Wikipedia's page on cats.
///
/// Results are paginated and can be flattened into a simpler format if desired. The response includes
/// rich metadata like snippets, titles, and ranking signal scores (when enabled).
#[debug_handler]
#[utoipa::path(
    post,
    path = "/beta/api/search",
    request_body(content = ApiSearchQuery),
    responses(
        (status = 200, description = "Search results", body = ApiSearchResult),
    )
)]
pub async fn search(
    extract::State(state): extract::State<Arc<State>>,
    extract::Json(query): extract::Json<ApiSearchQuery>,
) -> Result<impl IntoResponse, StatusCode> {
    tracing::debug!(?query);
    let flatten_result = query.flatten_response;
    let query = SearchQuery::try_from(query);

    if let Err(err) = query {
        tracing::error!("{:?}", err);
        return Err(StatusCode::BAD_REQUEST);
    }
    let mut query = query.unwrap();

    query.num_results = query.num_results.min(100);

    match state.searcher.search(&query).await {
        Ok(result) => {
            if flatten_result {
                Ok(Json(ApiSearchResult::from(result)).into_response())
            } else {
                Ok(Json(result).into_response())
            }
        }

        Err(err) => match err.downcast_ref() {
            Some(searcher::distributed::Error::EmptyQuery) => {
                Ok(searcher::distributed::Error::EmptyQuery
                    .to_string()
                    .into_response())
            }
            _ => {
                tracing::error!("{:?}", err);
                Err(StatusCode::INTERNAL_SERVER_ERROR)
            }
        },
    }
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,
)]
#[schema(title = "WidgetQuery", example = json!({"query": "2+2"}))]
pub struct WidgetQuery {
    /// The query string to search for
    pub query: String,
}

/// Widgets
///
/// The widget endpoint returns a widget that matches the query. Widgets are special UI components
/// that provide direct answers or interactive functionality for certain types of queries at the top of the search results.
/// A widget could for example be a calculator for math expressions, or a thesaurus for word definitions.
///
/// The endpoint returns an empty response if no widget matches the given query. When a widget does match,
/// it returns a structured response containing the widget type and its specific data needed to render it.
#[debug_handler]
#[utoipa::path(
    post,
    path = "/beta/api/search/widget",
    request_body(content = WidgetQuery),
    responses(
        (status = 200, description = "The resulting widget if one matches the query", body = Option<Widget>),
    )
)]
pub async fn widget(
    extract::State(state): extract::State<Arc<State>>,
    extract::Json(req): extract::Json<WidgetQuery>,
) -> impl IntoResponse {
    Json(state.searcher.widget(&req.query).await)
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,
)]
#[schema(title = "SidebarQuery", example = json!({"query": "Aristotle"}))]
pub struct SidebarQuery {
    /// The query string to search for
    pub query: String,
}

/// Sidebar
///
/// The sidebar endpoint returns a sidebar that matches the query. The sidebar is a UI component
/// that provides additional information or navigation options related to the query at the top of the search results.
///
/// The endpoint returns an empty response if no relevant sidebar content is found for the given query.
/// When content is found, it returns a structured response containing the sidebar data needed
/// to render the component.
///
/// For example, a query for "Aristotle" might return a sidebar with biographical information,
/// key philosophical concepts, and links to related ancient Greek philosophers.
#[debug_handler]
#[utoipa::path(
    post,
    path = "/beta/api/search/sidebar",
    request_body(content = SidebarQuery),
    responses(
        (status = 200, description = "The sidebar if one matches the query", body = Option<DisplayedSidebar>),
    )
)]
pub async fn sidebar(
    extract::State(state): extract::State<Arc<State>>,
    extract::Json(req): extract::Json<SidebarQuery>,
) -> impl IntoResponse {
    Json(state.searcher.sidebar(&req.query).await)
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,
)]
#[schema(title = "SpellcheckQuery", example = json!({"query": "hwllo eorld"}))]
pub struct SpellcheckQuery {
    /// The query string to possibly correct
    pub query: String,
}

/// Spellcheck
///
/// The spellcheck endpoint checks a query string for spelling errors and returns a corrected version if needed.
/// The correction is returned with highlighting to show what was changed.
///
/// For example, given the query "hwllo eorld", it might return "hello world" with "hello" and "world"
/// highlighted to show the corrections.
///
/// The corrections aim to preserve the user's intent while fixing obvious mistakes.
///
/// The correction dictionary has been trained on common words and phrases on the web, so it might make mistakes itself.
#[debug_handler]
#[utoipa::path(
    post,
    path = "/beta/api/search/spellcheck",
    request_body(content = SpellcheckQuery),
    responses(
        (status = 200, description = "The corrected string or an empty response if there is no correction to be made.", body = Option<HighlightedSpellCorrection>),
    )
)]
pub async fn spellcheck(
    extract::State(state): extract::State<Arc<State>>,
    extract::Json(req): extract::Json<SpellcheckQuery>,
) -> impl IntoResponse {
    Json(state.searcher.spell_check(&req.query))
}

#[derive(
    Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,
)]
#[serde(rename_all = "camelCase")]
pub struct EntityImageParams {
    pub image_id: String,
    pub max_width: Option<u64>,
    pub max_height: Option<u64>,
}

#[utoipa::path(
    post,
    path = "/beta/api/entity_image",
    request_body(content = ApiSearchQuery),
    responses(
        (status = 200, description = "Search results", body = ApiSearchResult),
    )
)]
pub async fn entity_image(
    extract::Query(query): extract::Query<EntityImageParams>,
    extract::State(state): extract::State<Arc<State>>,
) -> Result<impl IntoResponse, StatusCode> {
    match state
        .searcher
        .get_entity_image(&query.image_id, query.max_height, query.max_width)
        .await
    {
        Ok(Some(result)) => {
            let bytes = result.as_raw_bytes();

            Ok((
                ([(axum::http::header::CONTENT_TYPE, "image/webp")]),
                axum::response::AppendHeaders([(axum::http::header::CONTENT_TYPE, "image/webp")]),
                bytes,
            ))
        }
        Ok(None) => Err(StatusCode::NOT_FOUND),
        Err(err) => {
            tracing::error!("{:?}", err);
            Err(StatusCode::INTERNAL_SERVER_ERROR)
        }
    }
}