Skip to content

web search done right

7.8 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use tokio::fs::File;
use tokio::io;
use tokio_stream::StreamExt;
use tracing::info;

use crate::config::{
    defaults, IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LocalConfig,
    WebSpellConfig,
};
use crate::entrypoint::indexer;
use crate::entrypoint::indexer::JobSettings;
use crate::Result;
use std::fs::{self};
use std::path::Path;

use super::{webgraph, Centrality, EntityIndexer};

const DATA_PATH: &str = "data";
const BUCKET_NAME: &str = "public";

fn download_files() {
    tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .unwrap()
        .block_on(async {
            for name in [
                "sample.warc.gz",
                "bangs.json",
                "english-wordnet-2022-subset.ttl",
                "lambdamart.txt",
                "test.zim",
                "internet_archive.warc.gz",
            ] {
                let path = Path::new(DATA_PATH).join(name);

                if path.exists() {
                    info!("Skipping {}", name);
                    continue;
                }

                info!("Downloading {}", name);
                let body = reqwest::get(format!("http://s3.stract.com/{BUCKET_NAME}/{name}"))
                    .await
                    .unwrap();

                let mut file = File::create(path).await.unwrap();
                let mut bytes = body.bytes_stream();

                while let Some(item) = bytes.next().await {
                    io::copy(&mut item.unwrap().as_ref(), &mut file)
                        .await
                        .unwrap();
                }
            }
        });
}

fn build_spellchecker() -> Result<()> {
    info!("Building spellchecker");
    let spellchecker_path = Path::new(DATA_PATH).join("web_spell");

    if !spellchecker_path.exists() {
        crate::entrypoint::web_spell::run(WebSpellConfig {
            languages: vec![whatlang::Lang::Eng],
            output_path: spellchecker_path.to_str().unwrap().to_string(),
            warc_source: crate::config::WarcSource::Local(LocalConfig {
                folder: ".".to_string(),
                names: vec![Path::new(DATA_PATH)
                    .join("sample.warc.gz")
                    .to_str()
                    .unwrap()
                    .to_string()],
            }),
            limit_warc_files: None,
            skip_warc_files: None,
        })?;
    }

    Ok(())
}

fn create_webgraph() -> Result<()> {
    info!("Creating webgraph");
    let out_path = Path::new(DATA_PATH).join("webgraph");

    if out_path.exists() {
        std::fs::remove_dir_all(&out_path)?;
    }

    let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");

    let job = webgraph::Job {
        config: webgraph::JobConfig::Local(crate::config::LocalConfig {
            folder: ".".to_string(),
            names: vec![warc_path.to_str().unwrap().to_string()],
        }),
        warc_paths: vec![warc_path.to_str().unwrap().to_string()],
    };

    let mut worker = webgraph::WebgraphWorker {
        graph: crate::webgraph::Webgraph::open(&out_path, 0u64.into()).unwrap(),
        host_centrality_store: None,
        host_rank_store: None,
        canonical_index: None,
    };

    worker.process_job(&job);
    worker.graph.optimize_read().unwrap();

    Ok(())
}

fn calculate_centrality() {
    info!("Calculating centrality");
    let webgraph_path = Path::new(DATA_PATH).join("webgraph");
    let out_path = Path::new(DATA_PATH).join("centrality");

    if !out_path.exists() {
        Centrality::build_harmonic(&webgraph_path, &out_path);
    }

    let out_path_page = Path::new(DATA_PATH).join("centrality_page");

    if !out_path_page.exists() {
        Centrality::build_approx_harmonic(webgraph_path, out_path_page).unwrap();
    }
}

fn create_inverted_index() -> Result<()> {
    info!("Creating inverted index");
    let out_path = Path::new(DATA_PATH).join("index");

    if out_path.exists() {
        std::fs::remove_dir_all(&out_path)?;
    }

    let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");

    let job = indexer::Job {
        source_config: crate::config::WarcSource::Local(crate::config::LocalConfig {
            folder: ".".to_string(),
            names: vec![warc_path.to_str().unwrap().to_string()],
        }),
        warc_path: warc_path.to_str().unwrap().to_string(),
        base_path: out_path.to_str().unwrap().to_string(),
        settings: JobSettings {
            host_centrality_threshold: None,
            minimum_clean_words: None,
            batch_size: defaults::Indexing::batch_size(),
            autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),
        },
    };

    let webgraph_path = Path::new(DATA_PATH).join("webgraph");
    let centrality_path = Path::new(DATA_PATH).join("centrality");
    let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");
    let dual_encoder_path = Path::new(DATA_PATH).join("dual_encoder");
    let dual_encoder_path = if !dual_encoder_path.exists() {
        None
    } else {
        Some(dual_encoder_path)
    };

    let worker = crate::block_on(indexer::IndexingWorker::new(
        IndexerConfig {
            host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),
            page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),
            page_webgraph: Some(IndexerGraphConfig::Local {
                path: webgraph_path.to_str().unwrap().to_string(),
            }),
            output_path: out_path.to_str().unwrap().to_string(),
            limit_warc_files: None,
            skip_warc_files: None,
            warc_source: job.source_config.clone(),
            host_centrality_threshold: None,
            safety_classifier_path: None,
            minimum_clean_words: None,
            batch_size: defaults::Indexing::batch_size(),
            autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),
            dual_encoder: dual_encoder_path.map(|p| IndexerDualEncoderConfig {
                model_path: p.to_str().unwrap().to_string(),
                page_centrality_rank_threshold: Some(100_000),
            }),
        }
        .into(),
    ));

    let index = job.process(&worker);
    crate::mv(index.path(), &out_path)?;

    Ok(())
}

fn create_entity_index() -> Result<()> {
    info!("Creating entity index");
    let out_path = Path::new(DATA_PATH).join("entity");
    if out_path.exists() {
        std::fs::remove_dir_all(&out_path)?;
    }

    let wiki_path = Path::new(DATA_PATH).join("test.zim");

    EntityIndexer::run(
        wiki_path.to_str().unwrap().to_string(),
        out_path.to_str().unwrap().to_string(),
    )?;

    Ok(())
}

fn index_files() -> Result<()> {
    create_webgraph()?;
    calculate_centrality();
    create_inverted_index()?;
    create_entity_index()?;
    build_spellchecker()?;

    Ok(())
}

pub fn run(skip_download: bool) -> Result<()> {
    let p = Path::new(DATA_PATH);

    if !p.exists() {
        fs::create_dir_all(p)?;
    }

    if !skip_download {
        download_files();
    }

    index_files()?;

    Ok(())
}