Skip to content

web search done right

4.4 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

pub mod indexable_webpage;
pub mod job;
pub mod worker;

use rayon::prelude::*;
use std::thread;

use itertools::Itertools;

pub use crate::entrypoint::indexer::indexable_webpage::IndexableWebpage;
pub use crate::entrypoint::indexer::job::{Job, JobSettings};
pub use crate::entrypoint::indexer::worker::IndexingWorker;

use crate::config::{self, WarcSource};
use crate::index::Index;
use crate::Result;

#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]
pub struct IndexPointer(String);

impl From<String> for IndexPointer {
    fn from(path: String) -> Self {
        IndexPointer(path)
    }
}

pub fn run(config: &config::IndexerConfig) -> Result<()> {
    let warc_paths = config.warc_source.paths()?;

    let job_config: WarcSource = config.warc_source.clone();

    let worker = crate::block_on(IndexingWorker::new(config.clone().into()));

    let thread_pool = rayon::ThreadPoolBuilder::new()
        .num_threads(usize::from(std::thread::available_parallelism()?))
        .stack_size(160_000_000)
        .thread_name(move |num| format!("indexer-worker-{num}"))
        .build()?;

    let indexes = thread_pool.install(|| {
        warc_paths
            .into_par_iter()
            .skip(config.skip_warc_files.unwrap_or(0))
            .take(config.limit_warc_files.unwrap_or(usize::MAX))
            .map(|warc_path| Job {
                source_config: job_config.clone(),
                warc_path,
                base_path: config.output_path.clone(),
                settings: JobSettings {
                    host_centrality_threshold: config.host_centrality_threshold,
                    minimum_clean_words: config.minimum_clean_words,
                    batch_size: config.batch_size,
                    autocommit_after_num_inserts: config.autocommit_after_num_inserts,
                },
            })
            .map(|job| {
                IndexPointer(
                    job.process(&worker)
                        .path()
                        .as_os_str()
                        .to_str()
                        .unwrap()
                        .to_string(),
                )
            })
            .collect()
    });

    let index = merge(indexes)?;
    crate::mv(index.path(), &config.output_path)?;

    Ok(())
}

pub fn merge(indexes: Vec<IndexPointer>) -> Result<Index> {
    let num_indexes = indexes.len();
    let mut it = indexes.into_iter();
    let num_cores = usize::from(std::thread::available_parallelism()?);

    let mut threads = Vec::new();

    for _ in 0..(num_cores + 1) {
        let indexes = it
            .by_ref()
            .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)
            .collect_vec();

        if indexes.is_empty() {
            break;
        }

        threads.push(thread::spawn(move || {
            let mut it = indexes.into_iter();
            let mut index = Index::open(it.next().unwrap().0).unwrap();

            for other in it {
                let other_path = other.0;
                let other = Index::open(&other_path).unwrap();

                index = index.merge(other);

                std::fs::remove_dir_all(other_path).unwrap();
            }

            index.inverted_index.merge_into_max_segments(1).unwrap();

            index
        }));
    }

    let mut indexes = Vec::new();
    for thread in threads {
        indexes.push(thread.join().unwrap());
    }

    let mut it = indexes.into_iter();
    let mut index = it.next().unwrap();

    for other in it {
        let other_path = other.path();
        index = index.merge(other);
        std::fs::remove_dir_all(other_path).unwrap();
    }

    index.inverted_index.merge_into_max_segments(1).unwrap();

    Ok(index)
}