Skip to content

web search done right

6.3 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::{
    generic_query::{self, GenericQuery},
    index::Index,
    inverted_index,
    ranking::{LocalRanker, SignalComputer},
    searcher::InitialWebsiteResult,
    Result,
};
use std::sync::Arc;
use tokio::sync::{OwnedRwLockReadGuard, RwLock};

use crate::{
    config::CollectorConfig, models::dual_encoder::DualEncoder, query::Query,
    ranking::models::linear::LinearRegression, search_ctx::Ctx, searcher::SearchQuery,
};

use super::InvertedIndexResult;

pub struct InnerLocalSearcher {
    index: Arc<RwLock<Index>>,
    linear_regression: Option<Arc<LinearRegression>>,
    dual_encoder: Option<Arc<DualEncoder>>,
    collector_config: CollectorConfig,
}

impl InnerLocalSearcher {
    pub fn new(index: Arc<RwLock<Index>>) -> Self {
        Self {
            index,
            linear_regression: None,
            dual_encoder: None,
            collector_config: CollectorConfig::default(),
        }
    }

    pub async fn guard(&self) -> OwnedRwLockReadGuard<Index> {
        self.index.clone().read_owned().await
    }

    pub fn set_linear_model(&mut self, model: LinearRegression) {
        self.linear_regression = Some(Arc::new(model));
    }

    pub fn set_dual_encoder(&mut self, dual_encoder: DualEncoder) {
        self.dual_encoder = Some(Arc::new(dual_encoder));
    }

    pub fn set_collector_config(&mut self, config: CollectorConfig) {
        self.collector_config = config;
    }

    fn parse_query(
        &self,
        ctx: &Ctx,
        guard: &OwnedRwLockReadGuard<Index>,
        query: &SearchQuery,
    ) -> Result<Query> {
        Query::parse(ctx, query, guard.inverted_index())
    }

    fn ranker(
        &self,
        query: &Query,
        guard: &OwnedRwLockReadGuard<Index>,
        de_rank_similar: bool,
        computer: SignalComputer,
    ) -> Result<LocalRanker> {
        let mut ranker = LocalRanker::new(
            computer,
            guard.inverted_index().columnfield_reader(),
            self.collector_config.clone(),
        );

        ranker.de_rank_similar(de_rank_similar);

        Ok(ranker
            .with_max_docs(
                self.collector_config.max_docs_considered,
                guard.inverted_index().num_segments(),
            )
            .with_num_results(query.num_results())
            .with_offset(query.offset()))
    }

    fn search_inverted_index(
        &self,
        ctx: &Ctx,
        guard: &OwnedRwLockReadGuard<Index>,
        query: &SearchQuery,
        de_rank_similar: bool,
    ) -> Result<InvertedIndexResult> {
        let parsed_query = self.parse_query(ctx, guard, query)?;

        let mut computer = SignalComputer::new(Some(&parsed_query));

        computer.set_region_count(
            guard
                .region_count()
                .lock()
                .unwrap_or_else(|e| e.into_inner())
                .clone(),
        );

        if let Some(model) = self.linear_regression.as_ref() {
            computer.set_linear_model(model.clone());
        }

        let ranker = self.ranker(&parsed_query, guard, de_rank_similar, computer)?;

        let res = guard.inverted_index().search_initial(
            &parsed_query,
            ctx,
            ranker.collector(ctx.clone()),
        )?;

        let columnfield_reader = guard.inverted_index().columnfield_reader();

        let ranking_websites = guard.inverted_index().retrieve_ranking_websites(
            ctx,
            res.top_websites,
            ranker.computer(),
            &columnfield_reader,
        )?;

        Ok(InvertedIndexResult {
            webpages: ranking_websites,
            num_hits: res.num_websites,
        })
    }

    pub fn search_initial(
        &self,
        query: &SearchQuery,
        guard: &OwnedRwLockReadGuard<Index>,
        de_rank_similar: bool,
    ) -> Result<InitialWebsiteResult> {
        let query = query.clone();

        let ctx = guard.inverted_index().local_search_ctx();
        let inverted_index_result =
            self.search_inverted_index(&ctx, guard, &query, de_rank_similar)?;

        Ok(InitialWebsiteResult {
            websites: inverted_index_result.webpages,
            num_websites: inverted_index_result.num_hits,
        })
    }

    pub fn retrieve_websites(
        &self,
        websites: &[inverted_index::WebpagePointer],
        query: &str,
        guard: &OwnedRwLockReadGuard<Index>,
    ) -> Result<Vec<inverted_index::RetrievedWebpage>> {
        let ctx = guard.inverted_index().local_search_ctx();
        let query = SearchQuery {
            query: query.to_string(),
            ..Default::default()
        };
        let query = Query::parse(&ctx, &query, guard.inverted_index())?;

        guard.inverted_index().retrieve_websites(websites, &query)
    }

    pub fn search_initial_generic<Q: GenericQuery>(
        &self,
        query: &Q,
        guard: &OwnedRwLockReadGuard<Index>,
    ) -> Result<<Q::Collector as generic_query::Collector>::Fruit> {
        guard.inverted_index().search_initial_generic(query)
    }

    pub fn retrieve_generic<Q: GenericQuery>(
        &self,
        query: &Q,
        fruit: <Q::Collector as generic_query::Collector>::Fruit,
        guard: &OwnedRwLockReadGuard<Index>,
    ) -> Result<Q::IntermediateOutput> {
        guard.inverted_index().retrieve_generic(query, fruit)
    }

    pub fn search_generic<Q: GenericQuery>(
        &self,
        query: Q,
        guard: &OwnedRwLockReadGuard<Index>,
    ) -> Result<Q::Output> {
        let fruit = self.search_initial_generic(&query, guard)?;
        Ok(Q::merge_results(vec![
            self.retrieve_generic(&query, fruit, guard)?
        ]))
    }
}