web search done right
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
pub mod defaults;
pub use web_spell::CorrectionConfig;
use super::Result;
use crate::ampc::dht;
use crate::distributed::member::ShardId;
use std::fs::File;
use std::io::{self, BufRead};
use std::net::SocketAddr;
use std::path::PathBuf;
use std::time::Duration;
pub fn parse_duration<'de, D: serde::de::Deserializer<'de>>(
deserializer: D,
) -> Result<Duration, D::Error> {
let err = <D::Error as serde::de::Error>::custom;
let s: String = serde::de::Deserialize::deserialize(deserializer)?;
let num_part = s.trim_end_matches(|c: char| !c.is_numeric());
let suffix = &s[num_part.len()..];
let num: u64 = num_part
.parse()
.map_err(|_| err("invalid number".to_string()))?;
let ret = match suffix.trim() {
"s" => Duration::from_secs(num),
"sec" => Duration::from_secs(num),
"secs" => Duration::from_secs(num),
"ms" => Duration::from_millis(num),
"millis" => Duration::from_millis(num),
"milliseconds" => Duration::from_millis(num),
"m" => Duration::from_secs(num * 60),
"mins" => Duration::from_secs(num * 60),
"minutes" => Duration::from_secs(num * 60),
"h" => Duration::from_secs(num * 60 * 60),
"hours" => Duration::from_secs(num * 60 * 60),
"d" => Duration::from_secs(num * 24 * 60 * 60),
"day" => Duration::from_secs(num * 24 * 60 * 60),
"days" => Duration::from_secs(num * 24 * 60 * 60),
other => return Err(err(format!("invalid suffix {other}"))),
};
Ok(ret)
}
#[derive(Debug, serde::Deserialize, Clone)]
pub struct IndexerConfig {
pub output_path: String,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
pub warc_source: WarcSource,
pub page_webgraph: Option<IndexerGraphConfig>,
pub host_centrality_threshold: Option<f64>,
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
pub safety_classifier_path: Option<String>,
pub minimum_clean_words: Option<usize>,
#[serde(default = "defaults::Indexing::batch_size")]
pub batch_size: usize,
#[serde(default = "defaults::Indexing::autocommit_after_num_inserts")]
pub autocommit_after_num_inserts: usize,
pub dual_encoder: Option<IndexerDualEncoderConfig>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
#[serde(tag = "type")]
pub enum IndexerGraphConfig {
Local { path: String },
Remote { gossip: GossipConfig },
}
#[derive(Debug, serde::Deserialize, Clone)]
pub struct IndexerDualEncoderConfig {
pub model_path: String,
/// Only compute embeddings for pages that has a
/// centrality rank less than this threshold
pub page_centrality_rank_threshold: Option<u64>,
}
#[derive(Debug, serde::Deserialize, Clone)]
pub struct WebgraphConstructConfig {
pub graph_base_path: String,
pub shard: ShardId,
pub host_centrality_store_path: String,
pub host_rank_store_path: String,
pub warc_source: WarcSource,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
pub batch_size: Option<usize>,
pub canonical_index_path: Option<String>,
#[serde(default = "defaults::Webgraph::merge_all_segments")]
pub merge_all_segments: bool,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
#[serde(tag = "type")]
pub enum WarcSource {
HTTP(HttpConfig),
Local(LocalConfig),
S3(S3Config),
}
impl WarcSource {
pub fn paths(&self) -> Result<Vec<String>> {
let mut warc_paths = Vec::new();
match &self {
WarcSource::HTTP(config) => {
let file = File::open(&config.warc_paths_file)?;
for line in io::BufReader::new(file).lines() {
warc_paths.push(line?);
}
}
WarcSource::Local(config) => {
warc_paths.clone_from(&config.names);
}
WarcSource::S3(config) => {
let bucket = s3::Bucket::new(
&config.bucket,
s3::Region::Custom {
region: "".to_string(),
endpoint: config.endpoint.clone(),
},
s3::creds::Credentials {
access_key: Some(config.access_key.clone()),
secret_key: Some(config.secret_key.clone()),
security_token: None,
session_token: None,
expiration: None,
},
)?
.with_path_style();
let mut folder = config.folder.clone();
if !folder.ends_with('/') {
folder.push('/');
}
let pages = bucket.list_blocking(folder, Some("/".to_string()))?;
let objects = pages
.into_iter()
.flat_map(|p| p.contents.into_iter())
.collect::<Vec<_>>();
for p in objects.into_iter().filter_map(|o| {
if o.key.ends_with("warc.gz") {
Some(o.key)
} else {
None
}
}) {
warc_paths.push(p);
}
}
}
Ok(warc_paths)
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
pub struct LocalConfig {
pub folder: String,
pub names: Vec<String>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
pub struct HttpConfig {
pub base_url: String,
pub warc_paths_file: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]
pub struct S3Config {
pub bucket: String,
pub folder: String,
pub access_key: String,
pub secret_key: String,
pub endpoint: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CollectorConfig {
#[serde(default = "defaults::Collector::site_penalty")]
pub site_penalty: f64,
#[serde(default = "defaults::Collector::title_penalty")]
pub title_penalty: f64,
#[serde(default = "defaults::Collector::url_penalty")]
pub url_penalty: f64,
#[serde(default = "defaults::Collector::url_without_tld_penalty")]
pub url_without_tld_penalty: f64,
#[serde(default = "defaults::Collector::max_docs_considered")]
pub max_docs_considered: usize,
}
impl Default for CollectorConfig {
fn default() -> Self {
Self {
site_penalty: defaults::Collector::site_penalty(),
title_penalty: defaults::Collector::title_penalty(),
url_penalty: defaults::Collector::url_penalty(),
url_without_tld_penalty: defaults::Collector::url_without_tld_penalty(),
max_docs_considered: defaults::Collector::max_docs_considered(),
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct ApiThresholds {
#[serde(default = "defaults::Api::stackoverflow")]
pub stackoverflow: f64,
#[serde(default = "defaults::Api::entity_sidebar")]
pub entity_sidebar: f64,
}
impl Default for ApiThresholds {
fn default() -> Self {
Self {
stackoverflow: defaults::Api::stackoverflow(),
entity_sidebar: defaults::Api::entity_sidebar(),
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct ApiSpellCheck {
pub path: String,
#[serde(default)]
pub correction_config: CorrectionConfig,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct QueryStoreConfig {
pub host: String,
pub username: String,
pub password: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct ApiConfig {
pub host: SocketAddr,
pub prometheus_host: SocketAddr,
pub crossencoder_model_path: Option<String>,
pub lambda_model_path: Option<String>,
pub dual_encoder_model_path: Option<String>,
pub bangs_path: Option<String>,
pub query_store_db: Option<QueryStoreConfig>,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub management_host: SocketAddr,
#[serde(default = "defaults::Api::max_similar_hosts")]
pub max_similar_hosts: usize,
#[serde(default = "defaults::Api::top_phrases_for_autosuggest")]
pub top_phrases_for_autosuggest: usize,
pub spell_check: Option<ApiSpellCheck>,
#[serde(default)]
pub thresholds: ApiThresholds,
#[serde(default)]
pub widgets: WidgetsConfig,
#[serde(default)]
pub collector: CollectorConfig,
#[serde(default = "defaults::Api::max_concurrent_searches")]
pub max_concurrent_searches: Option<usize>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SnippetConfig {
#[serde(default = "defaults::Snippet::desired_num_chars")]
pub desired_num_chars: usize,
#[serde(default = "defaults::Snippet::delta_num_chars")]
pub delta_num_chars: usize,
#[serde(default = "defaults::Snippet::min_passage_width")]
pub min_passage_width: usize,
pub max_considered_words: Option<usize>,
pub num_words_for_lang_detection: Option<usize>,
#[serde(default = "defaults::Snippet::empty_query_snippet_words")]
pub empty_query_snippet_words: usize,
#[serde(default = "defaults::Snippet::min_description_words")]
pub min_description_words: usize,
#[serde(default = "defaults::Snippet::min_body_length")]
pub min_body_length: usize,
#[serde(default = "defaults::Snippet::min_body_length_homepage")]
pub min_body_length_homepage: usize,
}
impl Default for SnippetConfig {
fn default() -> Self {
Self {
desired_num_chars: defaults::Snippet::desired_num_chars(),
delta_num_chars: defaults::Snippet::delta_num_chars(),
min_passage_width: defaults::Snippet::min_passage_width(),
max_considered_words: None,
num_words_for_lang_detection: None,
empty_query_snippet_words: defaults::Snippet::empty_query_snippet_words(),
min_description_words: defaults::Snippet::min_description_words(),
min_body_length: defaults::Snippet::min_body_length(),
min_body_length_homepage: defaults::Snippet::min_body_length_homepage(),
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct SearchServerConfig {
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub shard: u64,
pub index_path: String,
pub linear_model_path: Option<String>,
pub dual_encoder_model_path: Option<String>,
pub host: SocketAddr,
#[serde(default)]
pub collector: CollectorConfig,
#[serde(default)]
pub snippet: SnippetConfig,
}
impl From<LiveIndexConfig> for SearchServerConfig {
fn from(config: LiveIndexConfig) -> Self {
Self {
host: config.search_host,
gossip_seed_nodes: config.gossip_seed_nodes,
gossip_addr: config.gossip_addr,
shard: config.shard_id,
index_path: config.index_path,
linear_model_path: config.linear_model_path,
dual_encoder_model_path: config.lambda_model_path,
collector: config.collector,
snippet: config.snippet,
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct EntitySearchServerConfig {
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub index_path: String,
pub host: SocketAddr,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlCoordinatorConfig {
pub job_queue: String,
pub host: SocketAddr,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct UserAgent {
pub full: String,
pub token: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlerConfig {
pub num_worker_threads: usize,
pub user_agent: UserAgent,
#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
pub robots_txt_cache_sec: u64,
#[serde(default = "defaults::Crawler::min_politeness_factor")]
pub min_politeness_factor: u32,
#[serde(default = "defaults::Crawler::start_politeness_factor")]
pub start_politeness_factor: u32,
#[serde(default = "defaults::Crawler::min_crawl_delay_ms")]
pub min_crawl_delay_ms: u64,
#[serde(default = "defaults::Crawler::max_crawl_delay_ms")]
pub max_crawl_delay_ms: u64,
#[serde(default = "defaults::Crawler::max_politeness_factor")]
pub max_politeness_factor: u32,
#[serde(default = "defaults::Crawler::max_url_slowdown_retry")]
pub max_url_slowdown_retry: u8,
pub timeout_seconds: u64,
pub s3: S3Config,
pub router_hosts: Vec<String>,
}
#[cfg(test)]
impl CrawlerConfig {
pub fn for_tests() -> Self {
Self {
num_worker_threads: 1,
user_agent: UserAgent {
full: "test".to_string(),
token: "test".to_string(),
},
robots_txt_cache_sec: defaults::Crawler::robots_txt_cache_sec(),
min_politeness_factor: defaults::Crawler::min_politeness_factor(),
start_politeness_factor: defaults::Crawler::start_politeness_factor(),
min_crawl_delay_ms: defaults::Crawler::min_crawl_delay_ms(),
max_crawl_delay_ms: defaults::Crawler::max_crawl_delay_ms(),
max_politeness_factor: defaults::Crawler::max_politeness_factor(),
max_url_slowdown_retry: defaults::Crawler::max_url_slowdown_retry(),
timeout_seconds: defaults::Crawler::timeout_seconds(),
s3: S3Config {
bucket: "".to_string(),
folder: "".to_string(),
access_key: "".to_string(),
secret_key: "".to_string(),
endpoint: "".to_string(),
},
router_hosts: Vec::new(),
}
}
}
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
pub struct DailyLiveIndexCrawlerBudget {
#[serde(default = "defaults::LiveCrawler::blogs_budget")]
pub blogs: u64,
#[serde(default = "defaults::LiveCrawler::news_budget")]
pub news: u64,
#[serde(default = "defaults::LiveCrawler::remaining_budget")]
pub remaining: u64,
}
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
pub struct CheckIntervals {
#[serde(
deserialize_with = "parse_duration",
default = "defaults::LiveCrawler::feeds_crawl_interval"
)]
pub feeds: Duration,
#[serde(
deserialize_with = "parse_duration",
default = "defaults::LiveCrawler::sitemap_crawl_interval"
)]
pub sitemap: Duration,
#[serde(
deserialize_with = "parse_duration",
default = "defaults::LiveCrawler::frontpage_crawl_interval"
)]
pub frontpage: Duration,
}
impl Default for CheckIntervals {
fn default() -> Self {
Self {
feeds: defaults::LiveCrawler::feeds_crawl_interval(),
sitemap: defaults::LiveCrawler::sitemap_crawl_interval(),
frontpage: defaults::LiveCrawler::frontpage_crawl_interval(),
}
}
}
impl Default for DailyLiveIndexCrawlerBudget {
fn default() -> Self {
Self {
blogs: defaults::LiveCrawler::blogs_budget(),
news: defaults::LiveCrawler::news_budget(),
remaining: defaults::LiveCrawler::remaining_budget(),
}
}
}
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
pub struct LiveCrawlerConfig {
pub crawled_db_path: PathBuf,
pub gossip: GossipConfig,
pub site_stats_path: PathBuf,
pub host_centrality_path: PathBuf,
pub user_agent: UserAgent,
pub num_worker_threads: usize,
#[serde(default)]
pub check_intervals: CheckIntervals,
#[serde(default)]
pub daily_budget: DailyLiveIndexCrawlerBudget,
#[serde(default = "defaults::LiveCrawler::init_crawl_db")]
pub init_crawl_db: bool,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlRouterConfig {
pub host: SocketAddr,
pub coordinator_addrs: Vec<SocketAddr>,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
#[serde(tag = "type", content = "args", rename_all = "snake_case")]
pub enum AcceleratorDevice {
Cpu,
Cuda(usize),
Mps,
}
#[derive(
serde::Serialize,
serde::Deserialize,
bincode::Encode,
bincode::Decode,
PartialEq,
Eq,
Hash,
Clone,
Copy,
Debug,
)]
#[serde(rename_all = "snake_case")]
pub enum WebgraphGranularity {
Host,
Page,
}
impl std::fmt::Display for WebgraphGranularity {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
WebgraphGranularity::Host => write!(f, "host"),
WebgraphGranularity::Page => write!(f, "page"),
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct WebgraphServerConfig {
pub host: SocketAddr,
pub shard: ShardId,
pub graph_path: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct WidgetsConfig {
pub thesaurus_paths: Vec<String>,
#[serde(default = "defaults::Widgets::calculator_fetch_currencies_exchange")]
pub calculator_fetch_currencies_exchange: bool,
}
impl Default for WidgetsConfig {
fn default() -> Self {
Self {
thesaurus_paths: Vec::new(),
calculator_fetch_currencies_exchange:
defaults::Widgets::calculator_fetch_currencies_exchange(),
}
}
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct TopHostsBudgetBoostConfig {
pub top_hosts: usize,
pub reserved_budget_fraction: f64,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlPlannerDomainBoost {
pub domain: String,
pub boost: f64,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct CrawlPlannerConfig {
pub host_harmonic_path: String,
pub page_harmonic_path: String,
pub host_centrality_rank_store_path: String,
pub output_path: String,
pub num_job_queues: usize,
pub crawl_budget: usize,
pub wander_fraction: f64,
pub top_host_fraction: f64,
pub excluded_domains: Option<Vec<String>>,
pub domain_boosts: Option<Vec<CrawlPlannerDomainBoost>>,
pub top_hosts_budget_boost: Option<TopHostsBudgetBoostConfig>,
pub gossip: GossipConfig,
}
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct LiveIndexConfig {
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
pub safety_classifier_path: Option<String>,
pub host_centrality_threshold: Option<f64>,
pub minimum_clean_words: Option<usize>,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
pub shard_id: u64,
pub index_path: String,
pub linear_model_path: Option<String>,
pub lambda_model_path: Option<String>,
pub host: SocketAddr,
pub search_host: SocketAddr,
#[serde(default)]
pub collector: CollectorConfig,
#[serde(default)]
pub snippet: SnippetConfig,
}
#[derive(Debug, serde::Deserialize, Clone)]
pub struct WebSpellConfig {
pub output_path: String,
pub warc_source: WarcSource,
pub languages: Vec<whatlang::Lang>,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
}
#[derive(Debug, serde::Deserialize, Clone)]
pub struct SiteStatsConfig {
pub output_path: String,
pub host_centrality_path: String,
pub top_sites: usize,
pub warc_source: WarcSource,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct GossipConfig {
pub seed_nodes: Option<Vec<SocketAddr>>,
pub addr: SocketAddr,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct DhtConfig {
pub node_id: dht::NodeId,
pub host: SocketAddr,
pub shard: ShardId,
pub seed_node: Option<SocketAddr>,
pub gossip: GossipConfig,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct HarmonicCoordinatorConfig {
pub gossip: GossipConfig,
pub host: SocketAddr,
pub output_path: String,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct ShortestPathCoordinatorConfig {
pub source: String,
pub gossip: GossipConfig,
pub host: SocketAddr,
pub output_path: String,
pub max_distance: Option<u64>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct HarmonicWorkerConfig {
pub gossip: GossipConfig,
pub shard: ShardId,
pub graph_path: String,
pub host: SocketAddr,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct ShortestPathWorkerConfig {
pub gossip: GossipConfig,
pub shard: ShardId,
pub graph_path: String,
pub host: SocketAddr,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct ApproxHarmonicCoordinatorConfig {
pub gossip: GossipConfig,
pub host: SocketAddr,
pub output_path: String,
#[serde(default = "defaults::ApproxHarmonic::sample_rate")]
pub sample_rate: f64,
#[serde(default = "defaults::ApproxHarmonic::max_distance")]
pub max_distance: u8,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct CanonicalIndexConfig {
pub output_path: String,
pub warc_source: WarcSource,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
}
#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]
pub struct HarmonicNearestSeedConfig {
pub gossip: GossipConfig,
pub original_centrality_path: PathBuf,
pub output_path: PathBuf,
#[serde(default = "defaults::HarmonicNearestSeed::discount_factor")]
pub discount_factor: f64,
}