Skip to content

web search done right

17.8 KB
Raw Blame
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use anyhow::{Context, Result};
use clap::{Parser, Subcommand};
use serde::de::DeserializeOwned;
use std::fs;
use std::net::SocketAddr;
use std::path::Path;
use stract::config;

#[cfg(feature = "dev")]
use stract::entrypoint::configure;

use stract::entrypoint::{
    self, api, entity_search_server, safety_classifier, search_server, webgraph_server,
};
use stract::webgraph::WebgraphBuilder;
use tracing_subscriber::prelude::*;

#[cfg(not(target_env = "msvc"))]
use tikv_jemallocator::Jemalloc;

#[cfg(not(target_env = "msvc"))]
#[global_allocator]
static GLOBAL: Jemalloc = Jemalloc;

#[derive(Parser)]
#[clap(author, version, about, long_about = None)]
#[clap(propagate_version = true)]
struct Args {
    #[clap(subcommand)]
    command: Commands,
}

#[derive(Subcommand)]
enum Commands {
    /// Build an index.
    Indexer {
        #[clap(subcommand)]
        options: IndexingOptions,
    },

    /// Calculate centrality metrics that estimates a websites importance. These metrics are used to rank search results.
    Centrality {
        #[clap(subcommand)]
        mode: CentralityMode,
    },

    /// Webgraph specific commands.
    Webgraph {
        #[clap(subcommand)]
        options: WebgraphOptions,
    },

    /// Deploy the search server.
    SearchServer { config_path: String },

    /// Deploy the entity search server.
    EntitySearchServer { config_path: String },

    /// Deploy the json http api. The api interacts with
    /// the search servers, webgraph servers etc. to provide the necesarry functionality.
    Api { config_path: String },

    /// Deploy the crawler.
    Crawler {
        #[clap(subcommand)]
        options: Crawler,
    },

    /// Train or run inference on the classifier that predicts if a webpage is NSFW or SFW.
    SafetyClassifier {
        #[clap(subcommand)]
        options: SafetyClassifierOptions,
    },

    /// Setup dev environment.
    #[cfg(feature = "dev")]
    Configure {
        #[clap(long)]
        skip_download: bool,
        #[clap(long)]
        ml: bool,
    },

    /// Commands for the live index.
    LiveIndex {
        #[clap(subcommand)]
        options: LiveIndex,
    },

    /// Build spell correction model.
    WebSpell { config_path: String },

    /// Compute statistics for sites.
    SiteStats { config_path: String },

    /// Commands to compute distributed graph algorithms.
    Ampc {
        #[clap(subcommand)]
        options: AmpcOptions,
    },

    /// Commands for the admin interface to manage stract.
    Admin {
        #[clap(subcommand)]
        options: AdminOptions,
    },
}

#[derive(Subcommand)]
enum AmpcOptions {
    /// Start a node for the distributed hash table (DHT).
    Dht { config_path: String },

    /// Start a worker to compute the harmonic centrality of a graph.
    HarmonicWorker { config_path: String },

    /// Start a coordinator to distribute the harmonic centrality computation.
    /// Workers needs to be started before the coordinator.
    HarmonicCoordinator { config_path: String },

    /// Start a coordinator to distribute the approximation of the harmonic centrality computation.
    /// Uses shortest path workers. Workers needs to be started before the coordinator.
    ApproxHarmonicCoordinator { config_path: String },

    /// Start a worker to compute the shortest paths of a graph.
    ShortestPathWorker { config_path: String },

    /// Start a coordinator to distribute the shortest path computation.
    /// Workers needs to be started before the coordinator.
    ShortestPathCoordinator { config_path: String },
}

#[derive(Subcommand)]
enum AdminOptions {
    /// Create the admin config file. Run this before any other admin commands so the client knows where to connect.
    Init { host: SocketAddr },

    /// Print the reachable cluster members and which service they are running.
    Status,

    /// Export the top most common phrases in the index.
    TopKeyphrases { top: usize },

    /// Get statistics about the index.
    #[clap(subcommand)]
    IndexStats(AdminIndexStatsOptions),
}

#[derive(Subcommand)]
enum AdminIndexStatsOptions {
    /// Get the size of the index
    Size,
}

#[derive(Subcommand)]
enum LiveIndex {
    /// Serve the live index.
    Serve { config_path: String },

    /// Start the live index crawler.
    Crawler { config_path: String },
}

#[derive(Subcommand)]
enum Crawler {
    /// Deploy the crawl worker. The worker is responsible for downloading webpages, saving them to S3,
    /// and sending newly discovered urls back to the crawl coordinator.
    Worker { config_path: String },

    /// Deploy the crawl coordinator. The crawl coordinator is responsible for
    /// distributing crawl jobs to the crawles and deciding which urls to crawl next.
    Coordinator { config_path: String },

    /// Deploy the crawl router. The crawl router is responsible for routing job responses and requests
    /// from the workers to the correct crawl coordinators.
    Router { config_path: String },

    /// Create a crawl plan.
    Plan { config_path: String },
}

/// Commands to train or run inference on the classifier that predicts if a webpage is NSFW or SFW.
#[derive(Subcommand)]
enum SafetyClassifierOptions {
    /// Train the classifier
    Train {
        dataset_path: String,
        output_path: String,
    },

    /// Run a single prediction to test the model
    Predict { model_path: String, text: String },
}

#[derive(Subcommand)]
enum CentralityMode {
    /// Calculate harmonic centrality for the webgraph.
    Harmonic {
        webgraph_path: String,
        output_path: String,
    },
    /// Calculate approximate harmonic centrality.
    ApproxHarmonic {
        webgraph_path: String,
        output_path: String,
    },

    /// Calculate harmonic centrality nearest neighbor that uses
    /// the harmonic centrality of the highest neighbors node
    /// as a seed node proxy for the centrality of that node (with a discount factor).
    HarmonicNearestSeed { config_path: String },
}

#[derive(Subcommand)]
enum WebgraphOptions {
    /// Create a new webgraph.
    Create { config_path: String },

    /// Merge multiple webgraphs into a single graph.
    Merge {
        #[clap(required = true)]
        paths: Vec<String>,
    },

    /// Deploy the webgraph server. The webgraph server is responsible for serving the webgraph to the search servers.
    /// This is e.g. used to find similar sites etc.
    Server { config_path: String },
}

#[derive(Subcommand)]
enum IndexingOptions {
    /// Create the search index.
    Search { config_path: String },

    /// Merge multiple search indexes into a single index.
    MergeSearch {
        #[clap(required = true)]
        paths: Vec<String>,
    },

    /// Create the entity index. Used in the sidebar of the search UI.
    Entity {
        wikipedia_dump_path: String,
        output_path: String,
    },

    /// Create an index of canonical urls.
    Canonical { config_path: String },
}

fn load_toml_config<T: DeserializeOwned, P: AsRef<Path>>(path: P) -> T {
    let path = path.as_ref();
    let raw_config = fs::read_to_string(path)
        .with_context(|| format!("Failed to read config: '{}'", path.display()))
        .unwrap();
    toml::from_str(&raw_config)
        .with_context(|| format!("Failed to parse config: '{}'", path.display()))
        .unwrap()
}

fn main() -> Result<()> {
    tracing_subscriber::fmt()
        .with_max_level(tracing::Level::INFO)
        .with_env_filter(
            tracing_subscriber::EnvFilter::builder()
                .with_default_directive("stract=info".parse().unwrap())
                .from_env_lossy(),
        )
        .without_time()
        .with_target(false)
        .finish()
        .init();

    let args = Args::parse();

    match args.command {
        Commands::Indexer { options } => match options {
            IndexingOptions::Search { config_path } => {
                let config = load_toml_config(config_path);
                entrypoint::indexer::run(&config)?;
            }
            IndexingOptions::Entity {
                wikipedia_dump_path,
                output_path,
            } => entrypoint::EntityIndexer::run(wikipedia_dump_path, output_path)?,
            IndexingOptions::MergeSearch { paths } => {
                let pointers = paths
                    .into_iter()
                    .map(entrypoint::indexer::IndexPointer::from)
                    .collect::<Vec<_>>();
                entrypoint::indexer::merge(pointers)?;
            }
            IndexingOptions::Canonical { config_path } => {
                let config: config::CanonicalIndexConfig = load_toml_config(config_path);
                entrypoint::canonical::create(config)?;
            }
        },
        Commands::Centrality { mode } => match mode {
            CentralityMode::Harmonic {
                webgraph_path,
                output_path,
            } => {
                entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);
            }
            CentralityMode::ApproxHarmonic {
                webgraph_path,
                output_path,
            } => entrypoint::Centrality::build_approx_harmonic(webgraph_path, output_path)?,
            CentralityMode::HarmonicNearestSeed { config_path } => {
                let config: config::HarmonicNearestSeedConfig = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::Centrality::harmonic_nearest_seed(config))?;
            }
        },
        Commands::Webgraph { options } => match options {
            WebgraphOptions::Create { config_path } => {
                let config = load_toml_config(config_path);
                entrypoint::Webgraph::run(&config)?;
            }
            WebgraphOptions::Merge { mut paths } => {
                let mut webgraph = WebgraphBuilder::new(paths.remove(0), 0u64.into()).open()?;

                for other_path in paths {
                    let other = WebgraphBuilder::new(&other_path, 0u64.into()).open()?;
                    webgraph.merge(other)?;
                }

                webgraph.optimize_read()?;
            }
            WebgraphOptions::Server { config_path } => {
                let config: config::WebgraphServerConfig = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(webgraph_server::run(config))?;
            }
        },
        Commands::Api { config_path } => {
            let config: config::ApiConfig = load_toml_config(config_path);

            tokio::runtime::Builder::new_multi_thread()
                .enable_all()
                .build()?
                .block_on(api::run(config))?;
        }
        Commands::SearchServer { config_path } => {
            let config: config::SearchServerConfig = load_toml_config(config_path);

            tokio::runtime::Builder::new_multi_thread()
                .enable_all()
                .build()?
                .block_on(search_server::run(config))?;
        }
        Commands::EntitySearchServer { config_path } => {
            let config: config::EntitySearchServerConfig = load_toml_config(config_path);

            tokio::runtime::Builder::new_multi_thread()
                .enable_all()
                .build()?
                .block_on(entity_search_server::run(config))?;
        }
        #[cfg(feature = "dev")]
        Commands::Configure {
            skip_download,
            ml: _,
        } => {
            configure::run(skip_download)?;
        }
        Commands::Crawler { options } => match options {
            Crawler::Worker { config_path } => {
                let config: config::CrawlerConfig = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::crawler::worker(config))?;
            }
            Crawler::Coordinator { config_path } => {
                let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::crawler::coordinator(config))?;
            }
            Crawler::Router { config_path } => {
                let config: config::CrawlRouterConfig = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::crawler::router(config))?;
            }
            Crawler::Plan { config_path } => {
                let config: config::CrawlPlannerConfig = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::crawler::planner(config))?;
            }
        },
        Commands::SafetyClassifier { options } => match options {
            SafetyClassifierOptions::Train {
                dataset_path,
                output_path,
            } => safety_classifier::train(dataset_path, output_path)?,
            SafetyClassifierOptions::Predict { model_path, text } => {
                safety_classifier::predict(model_path, &text)?;
            }
        },
        Commands::LiveIndex { options } => match options {
            LiveIndex::Serve { config_path } => {
                let config = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::live_index::search_server::serve(config))?;
            }
            LiveIndex::Crawler { config_path } => {
                let config = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::live_index::crawler::run(config))?;
            }
        },
        Commands::WebSpell { config_path } => {
            let config: config::WebSpellConfig = load_toml_config(config_path);
            entrypoint::web_spell::run(config)?;
        }
        Commands::SiteStats { config_path } => {
            let config: config::SiteStatsConfig = load_toml_config(config_path);
            entrypoint::site_stats::run(config)?;
        }
        Commands::Ampc { options } => match options {
            AmpcOptions::Dht { config_path } => {
                let config: config::DhtConfig = load_toml_config(config_path);

                tokio::runtime::Builder::new_multi_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::ampc::dht::run(config))?;
            }

            AmpcOptions::HarmonicWorker { config_path } => {
                let config: config::HarmonicWorkerConfig = load_toml_config(config_path);
                entrypoint::ampc::harmonic_centrality::worker::run(config)?;
            }

            AmpcOptions::HarmonicCoordinator { config_path } => {
                let config: config::HarmonicCoordinatorConfig = load_toml_config(config_path);
                entrypoint::ampc::harmonic_centrality::coordinator::run(config)?;
            }

            AmpcOptions::ApproxHarmonicCoordinator { config_path } => {
                let config: config::ApproxHarmonicCoordinatorConfig = load_toml_config(config_path);
                entrypoint::ampc::approximated_harmonic_centrality::coordinator::run(config)?;
            }

            AmpcOptions::ShortestPathWorker { config_path } => {
                let config: config::ShortestPathWorkerConfig = load_toml_config(config_path);
                entrypoint::ampc::shortest_path::worker::run(config)?;
            }

            AmpcOptions::ShortestPathCoordinator { config_path } => {
                let config: config::ShortestPathCoordinatorConfig = load_toml_config(config_path);
                entrypoint::ampc::shortest_path::coordinator::run(config)?;
            }
        },

        Commands::Admin { options } => match options {
            AdminOptions::Init { host } => {
                entrypoint::admin::init(host)?;
            }

            AdminOptions::Status => {
                tokio::runtime::Builder::new_current_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::admin::status())?;
            }

            AdminOptions::TopKeyphrases { top } => {
                tokio::runtime::Builder::new_current_thread()
                    .enable_all()
                    .build()?
                    .block_on(entrypoint::admin::top_keyphrases(top))?;
            }

            AdminOptions::IndexStats(index_options) => match index_options {
                AdminIndexStatsOptions::Size => {
                    tokio::runtime::Builder::new_current_thread()
                        .enable_all()
                        .build()?
                        .block_on(entrypoint::admin::index_size())?;
                }
            },
        },
    }

    Ok(())
}