web search done right
Moved some hardcoded constants into configuration files. Also a bunch of other stuff; I should really have split this commit into multiple parts...
Mikkel Denker4a2ca1472d0a
diff --git a/Cargo.lock b/Cargo.lockindex 86970877..6c13afcf 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -95,12 +95,6 @@ dependencies = ["memchr",]-[[package]]-name = "allocator-api2"-version = "0.2.14"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c4f263788a35611fba42eb41ff811c5d0360c58b97402570312a350736e2542e"-[[package]]name = "android_system_properties"version = "0.1.5"@@ -350,7 +344,7 @@ checksum = "113713495a32dd0ab52baf5c10044725aa3aec00b31beda84218e469029b72a3"dependencies = ["async-trait","axum-core",- "bitflags 1.3.2",+ "bitflags","bytes","futures-util","http",@@ -472,7 +466,7 @@ version = "0.64.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"dependencies = [- "bitflags 1.3.2",+ "bitflags","cexpr","clang-sys","lazy_static",@@ -513,12 +507,6 @@ version = "1.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"-[[package]]-name = "bitflags"-version = "2.3.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded"-[[package]]name = "bitpacking"version = "0.8.4"@@ -752,7 +740,7 @@ version = "2.34.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"dependencies = [- "bitflags 1.3.2",+ "bitflags","textwrap 0.11.0","unicode-width",]@@ -764,7 +752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"dependencies = ["atty",- "bitflags 1.3.2",+ "bitflags","clap_derive 3.2.18","clap_lex 0.2.4","indexmap",@@ -793,7 +781,7 @@ checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"dependencies = ["anstream","anstyle",- "bitflags 1.3.2",+ "bitflags","clap_lex 0.4.1","strsim",]@@ -1415,18 +1403,6 @@ dependencies = ["rand 0.8.5",]-[[package]]-name = "fallible-iterator"-version = "0.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"--[[package]]-name = "fallible-streaming-iterator"-version = "0.1.9"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"-[[package]]name = "fastdivide"version = "0.4.0"@@ -1801,25 +1777,6 @@ dependencies = ["ahash 0.8.3",]-[[package]]-name = "hashbrown"-version = "0.14.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"-dependencies = [- "ahash 0.8.3",- "allocator-api2",-]--[[package]]-name = "hashlink"-version = "0.8.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f"-dependencies = [- "hashbrown 0.14.0",-]-[[package]]name = "heck"version = "0.3.3"@@ -2302,18 +2259,6 @@ dependencies = ["zstd-sys",]-[[package]]-name = "libsqlite3-sys"-version = "0.26.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"-dependencies = [- "bindgen",- "cc",- "pkg-config",- "vcpkg",-]-[[package]]name = "libz-sys"version = "1.1.8"@@ -2846,7 +2791,7 @@ version = "6.4.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"dependencies = [- "bitflags 1.3.2",+ "bitflags","libc","once_cell","onig_sys",@@ -2880,7 +2825,7 @@ version = "0.10.51"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "97ea2d98598bf9ada7ea6ee8a30fb74f9156b63bbe495d64ec2b87c269d2dda3"dependencies = [- "bitflags 1.3.2",+ "bitflags","cfg-if","foreign-types","libc",@@ -3212,7 +3157,7 @@ version = "0.17.8"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa"dependencies = [- "bitflags 1.3.2",+ "bitflags","crc32fast","fdeflate","flate2",@@ -3497,7 +3442,7 @@ version = "0.2.16"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"dependencies = [- "bitflags 1.3.2",+ "bitflags",][[package]]@@ -3506,7 +3451,7 @@ version = "0.3.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"dependencies = [- "bitflags 1.3.2",+ "bitflags",][[package]]@@ -3657,26 +3602,6 @@ dependencies = ["librocksdb-sys",]-[[package]]-name = "rusqlite"-version = "0.29.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"-dependencies = [- "bitflags 2.3.2",- "chrono",- "csv",- "fallible-iterator",- "fallible-streaming-iterator",- "hashlink",- "libsqlite3-sys",- "serde_json",- "smallvec",- "time 0.3.20",- "url",- "uuid",-]-[[package]]name = "rust-ini"version = "0.18.0"@@ -3752,7 +3677,7 @@ version = "0.37.14"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f"dependencies = [- "bitflags 1.3.2",+ "bitflags","errno","io-lifetimes","libc",@@ -3910,7 +3835,7 @@ version = "2.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"dependencies = [- "bitflags 1.3.2",+ "bitflags","core-foundation","core-foundation-sys","libc",@@ -3933,7 +3858,7 @@ version = "0.22.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"dependencies = [- "bitflags 1.3.2",+ "bitflags","cssparser","derive_more","fxhash",@@ -4226,7 +4151,6 @@ dependencies = ["rkyv","robotstxt-with-cache","rocksdb",- "rusqlite","rust-s3","rust-stemmers","safetensors",@@ -4800,7 +4724,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"dependencies = ["async-compression",- "bitflags 1.3.2",+ "bitflags","bytes","futures-core","futures-util",diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 920e73ab..a092d90b 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -90,7 +90,6 @@ reqwest-eventsource = { workspace = true }publicsuffix = { workspace = true }robotstxt-with-cache = { workspace = true }rust-s3 = { workspace = true }-rusqlite = { workspace = true }async-channel = { workspace = true }lru = { workspace = true }@@ -131,4 +130,4 @@ harness = false[[bench]]name = "similar-sites"-harness = false\ No newline at end of file+harness = falsediff --git a/core/benches/similar-sites.rs b/core/benches/similar-sites.rsindex d9ecd090..9f1bf9fd 100644--- a/core/benches/similar-sites.rs+++ b/core/benches/similar-sites.rs@@ -13,7 +13,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {let webgraph = Arc::new(WebgraphBuilder::new(WEBGRAPH_PATH).open());let inbound = InboundSimilarity::open(INBOUND_SIMILARITY_PATH).unwrap();- let finder = SimilarSitesFinder::new(webgraph, inbound);+ let finder = SimilarSitesFinder::new(+ webgraph,+ inbound,+ stract::config::defaults::WebgraphServer::max_similar_sites(),+ );for _ in 0..10 {c.bench_function("similar_sites", |b| {diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rsindex 9547bec9..dc05faeb 100644--- a/core/examples/print_similar_sites.rs+++ b/core/examples/print_similar_sites.rs@@ -42,7 +42,11 @@ pub fn main() {let graph = WebgraphBuilder::new("data/webgraph").open();let store = CentralityStore::open("data/centrality");- let similarity_finder = SimilarSitesFinder::new(graph.into(), store.inbound_similarity);+ let similarity_finder = SimilarSitesFinder::new(+ graph.into(),+ store.inbound_similarity,+ stract::config::defaults::WebgraphServer::max_similar_sites(),+ );print_top_nodes(&[diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex 5141d6fd..60ee2612 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -41,12 +41,14 @@ use tch::Tensor;use crate::{api::search::ApiSearchQuery,+ config::AcceleratorDevice,+ config::AcceleratorDtype,+ config::AliceAcceleratorConfig,llm_utils::ClonableTensor,search_prettifier::DisplayedWebpage,searcher::{SearchResult, WebsitesResult},summarizer::{self, ExtractiveSummarizer},webpage::Url,- AcceleratorDevice, AcceleratorDtype, AliceAcceleratorConfig,};use self::{diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 08d6edb9..445c1955 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -26,6 +26,7 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};use crate::{autosuggest::Autosuggest,bangs::Bangs,+ config::FrontendConfig,distributed::{cluster::Cluster,member::{Member, Service},@@ -37,7 +38,6 @@ use crate::{ranking::models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},searcher::frontend::FrontendSearcher,summarizer::Summarizer,- FrontendConfig,};use anyhow::Result;use std::sync::Arc;@@ -150,8 +150,15 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route.await?,);let remote_webgraph = RemoteWebgraph::new(cluster.clone());- let searcher =- FrontendSearcher::new(cluster.clone(), crossencoder, lambda_model, qa_model, bangs);+ let searcher = FrontendSearcher::new(+ cluster.clone(),+ crossencoder,+ lambda_model,+ qa_model,+ bangs,+ config.collector.clone(),+ config.thresholds.clone(),+ );let state = Arc::new(State {config: config.clone(),diff --git a/core/src/collector.rs b/core/src/collector.rsindex 1f91246b..36ffe6f3 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -24,7 +24,9 @@ use tantivy::{};use crate::{- combine_u64s, fastfield_reader,+ combine_u64s,+ config::CollectorConfig,+ fastfield_reader,inverted_index::{DocAddress, WebsitePointer},prehashed::Prehashed,ranking::initial::Score,@@ -32,11 +34,6 @@ use crate::{simhash,};-// lower scale -> higher penalty-const SITE_SCALE: f64 = 14.0;-const TITLE_SCALE: f64 = 6.0;-const URL_SCALE: f64 = 0.1;-#[derive(Clone)]pub struct MaxDocsConsidered {pub total_docs: usize,@@ -52,7 +49,7 @@ pub struct Hashes {}pub trait Doc: Clone {- fn score(&self) -> &f64;+ fn score(&self) -> f64;fn id(&self) -> &DocId;fn hashes(&self) -> Hashes;}@@ -63,6 +60,7 @@ pub struct TopDocs {max_docs: Option<MaxDocsConsidered>,fastfield_reader: fastfield_reader::FastFieldReader,de_rank_similar: bool,+ collector_config: CollectorConfig,}impl TopDocs {@@ -73,6 +71,7 @@ impl TopDocs {max_docs: None,de_rank_similar: false,fastfield_reader,+ collector_config: CollectorConfig::default(),}}@@ -83,13 +82,16 @@ impl TopDocs {pub fn and_max_docs(mut self, max_docs: MaxDocsConsidered) -> Self {self.max_docs = Some(max_docs);-self}pub fn and_de_rank_similar(mut self) -> Self {self.de_rank_similar = true;+ self+ }+ pub fn and_collector_config(mut self, collector_config: CollectorConfig) -> Self {+ self.collector_config = collector_config;self}@@ -121,7 +123,10 @@ impl TopDocs {max_docs,num_docs_taken: 0,segment_ord: segment_local_id,- bucket_collector: BucketCollector::new(self.top_n + self.offset),+ bucket_collector: BucketCollector::new(+ self.top_n + self.offset,+ self.collector_config.clone(),+ ),})}}@@ -185,7 +190,7 @@ impl TopSegmentCollector {}fn harvest(self) -> Vec<SegmentDoc> {- self.bucket_collector.into_sorted_vec(false)+ self.bucket_collector.into_sorted_vec(true)}}@@ -217,34 +222,39 @@ impl<T: Doc> Eq for ScoredDoc<T> {}impl<T: Doc> From<T> for ScoredDoc<T> {fn from(doc: T) -> Self {Self {- adjusted_score: *doc.score(),+ adjusted_score: doc.score(),doc,}}}-#[derive(Default)]struct BucketCount {+ config: CollectorConfig,buckets: HashMap<Prehashed, usize>,}impl BucketCount {+ fn new(config: CollectorConfig) -> Self {+ Self {+ config,+ buckets: HashMap::new(),+ }+ }+pub fn adjust_score<T: Doc>(&self, doc: &mut ScoredDoc<T>) {let hashes = doc.doc.hashes();- let mut adjuster = 1.0;+ let taken_sites = *self.buckets.get(&hashes.site).unwrap_or(&0);+ let taken_urls = *self.buckets.get(&hashes.url).unwrap_or(&0);+ let taken_titles = *self.buckets.get(&hashes.title).unwrap_or(&0);- let taken_sites = self.buckets.get(&hashes.site).unwrap_or(&0);+ let adjuster = 1.0+ / (1.0+ + taken_sites as f64 * self.config.site_penalty+ + taken_urls as f64 * self.config.url_penalty+ + taken_titles as f64 * self.config.title_penalty);- adjuster *= SITE_SCALE / (SITE_SCALE + (*taken_sites as f64));-- let taken_urls = self.buckets.get(&hashes.url).unwrap_or(&0);- adjuster *= URL_SCALE / (URL_SCALE + (*taken_urls as f64));-- let taken_titles = self.buckets.get(&hashes.title).unwrap_or(&0);- adjuster *= TITLE_SCALE / (TITLE_SCALE + (*taken_titles as f64));-- doc.adjusted_score = *doc.doc.score() * adjuster;+ doc.adjusted_score = doc.doc.score() * adjuster;}fn update_counts<T: Doc>(&mut self, doc: &ScoredDoc<T>) {@@ -263,27 +273,28 @@ pub struct BucketCollector<T: Doc> {}impl<T: Doc> BucketCollector<T> {- pub fn new(top_n: usize) -> Self {+ pub fn new(top_n: usize, config: CollectorConfig) -> Self {assert!(top_n > 0);Self {top_n,- count: BucketCount::default(),+ count: BucketCount::new(config),documents: MinMaxHeap::with_capacity(top_n + 1),}}pub fn insert(&mut self, doc: T) {- let scored_doc = doc.into();+ let mut scored_doc: ScoredDoc<T> = doc.into();+ self.count.adjust_score(&mut scored_doc);- self.documents.push(scored_doc);+ if self.documents.len() > self.top_n {+ let mut min_scored_doc = self.documents.peek_min_mut().unwrap();- self.prune_buckets()- }-- fn prune_buckets(&mut self) {- while self.documents.len() > self.top_n + 1 {- self.documents.pop_min().unwrap();+ if scored_doc.adjusted_score > min_scored_doc.adjusted_score {+ *min_scored_doc = scored_doc;+ }+ } else {+ self.documents.push(scored_doc);}}@@ -341,8 +352,8 @@ pub struct SegmentDoc {}impl Doc for SegmentDoc {- fn score(&self) -> &f64 {- &self.score.total+ fn score(&self) -> f64 {+ self.score.total}fn id(&self) -> &DocId {@@ -404,7 +415,10 @@ where&self,segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,) -> tantivy::Result<Self::Fruit> {- let mut collector = BucketCollector::new(self.top_docs.top_n + self.top_docs.offset);+ let mut collector = BucketCollector::new(+ self.top_docs.top_n + self.top_docs.offset,+ self.top_docs.collector_config.clone(),+ );for docs in segment_fruits {for doc in docs {@@ -458,7 +472,7 @@ mod tests {use super::*;fn test(top_n: usize, docs: &[(Hashes, DocId, f64)], expected: &[(f64, DocId)]) {- let mut collector = BucketCollector::new(top_n);+ let mut collector = BucketCollector::new(top_n, CollectorConfig::default());for doc in docs {collector.insert(SegmentDoc {diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsnew file mode 100644index 00000000..a8d8ba9b--- /dev/null+++ b/core/src/config/defaults.rs@@ -0,0 +1,95 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub struct Collector;++impl Collector {+ pub fn site_penalty() -> f64 {+ 0.3+ }++ pub fn title_penalty() -> f64 {+ 2.0+ }++ pub fn url_penalty() -> f64 {+ 20.0+ }+}++pub struct Frontend;++impl Frontend {+ pub fn stackoverflow() -> f64 {+ 0.5+ }++ pub fn discussions_widget() -> f64 {+ 0.1+ }+}++pub struct Snippet;++impl Snippet {+ pub fn desired_num_chars() -> usize {+ 275+ }++ pub fn delta_num_chars() -> usize {+ 50+ }++ pub fn min_passage_width() -> usize {+ 20+ }+}++pub struct Crawler;++impl Crawler {+ pub fn politeness_factor() -> f32 {+ 1.0+ }++ pub fn min_crawl_delay_ms() -> u64 {+ 200+ }++ pub fn max_crawl_delay_ms() -> u64 {+ 30_000+ }++ pub fn max_politeness_factor() -> f32 {+ 1024.0+ }++ pub fn min_politeness_factor() -> f32 {+ 1.0+ }++ pub fn max_url_slowdown_retry() -> u8 {+ 3+ }+}++pub struct WebgraphServer;++impl WebgraphServer {+ pub fn max_similar_sites() -> usize {+ 1_000+ }+}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsnew file mode 100644index 00000000..bc204257--- /dev/null+++ b/core/src/config/mod.rs@@ -0,0 +1,343 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub mod defaults;++use super::Result;+use crate::searcher::ShardId;+use serde::{Deserialize, Serialize};+use std::fs::File;+use std::io::{self, BufRead};+use std::net::SocketAddr;++#[derive(Debug, Deserialize, Clone)]+pub struct IndexingLocalConfig {+ pub(crate) limit_warc_files: Option<usize>,+ pub(crate) skip_warc_files: Option<usize>,+ pub(crate) warc_source: WarcSource,+ pub(crate) batch_size: Option<usize>,+ pub(crate) page_webgraph_path: Option<String>,+ pub(crate) output_path: Option<String>,+ pub(crate) host_centrality_threshold: Option<f64>,+ pub(crate) topics_path: Option<String>,+ pub(crate) host_centrality_store_path: String,+ pub(crate) page_centrality_store_path: Option<String>,+ pub(crate) minimum_clean_words: Option<usize>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+#[serde(rename_all = "lowercase")]+pub enum WebgraphLevel {+ Page,+ Host,+}++#[derive(Debug, Deserialize, Clone)]+pub struct WebgraphConstructConfig {+ pub(crate) level: WebgraphLevel,+ pub(crate) redirect_db_path: Option<String>,+ pub(crate) limit_warc_files: Option<usize>,+ pub(crate) warc_source: WarcSource,+ pub(crate) graph_base_path: Option<String>,+ pub(crate) batch_size: Option<usize>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+#[serde(tag = "type")]+pub enum WarcSource {+ HTTP(HttpConfig),+ Local(LocalConfig),+ S3(S3Config),+}++impl WarcSource {+ pub fn paths(&self) -> Result<Vec<String>> {+ let mut warc_paths = Vec::new();+ match &self {+ WarcSource::HTTP(config) => {+ let file = File::open(&config.warc_paths_file)?;+ for line in io::BufReader::new(file).lines() {+ warc_paths.push(line?);+ }+ }+ WarcSource::Local(config) => {+ warc_paths = config.names.clone();+ }+ WarcSource::S3(config) => {+ let bucket = s3::Bucket::new(+ &config.bucket,+ s3::Region::Custom {+ region: "".to_string(),+ endpoint: config.endpoint.clone(),+ },+ s3::creds::Credentials {+ access_key: Some(config.access_key.clone()),+ secret_key: Some(config.secret_key.clone()),+ security_token: None,+ session_token: None,+ expiration: None,+ },+ )?+ .with_path_style();++ let mut folder = config.folder.clone();++ if !folder.ends_with('/') {+ folder.push('/');+ }++ let pages = bucket.list_blocking(folder, Some("/".to_string()))?;++ let objects = pages+ .into_iter()+ .flat_map(|p| p.contents.into_iter())+ .collect::<Vec<_>>();++ for p in objects.into_iter().filter_map(|o| {+ if o.key.ends_with("warc.gz") {+ Some(o.key)+ } else {+ None+ }+ }) {+ warc_paths.push(p);+ }+ }+ }++ Ok(warc_paths)+ }+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct LocalConfig {+ pub folder: String,+ pub names: Vec<String>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct HttpConfig {+ pub(crate) base_url: String,+ pub(crate) warc_paths_file: String,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct S3Config {+ pub bucket: String,+ pub folder: String,+ pub access_key: String,+ pub secret_key: String,+ pub endpoint: String,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct CollectorConfig {+ #[serde(default = "defaults::Collector::site_penalty")]+ pub site_penalty: f64,++ #[serde(default = "defaults::Collector::title_penalty")]+ pub title_penalty: f64,++ #[serde(default = "defaults::Collector::url_penalty")]+ pub url_penalty: f64,+}++impl Default for CollectorConfig {+ fn default() -> Self {+ Self {+ site_penalty: defaults::Collector::site_penalty(),+ title_penalty: defaults::Collector::title_penalty(),+ url_penalty: defaults::Collector::url_penalty(),+ }+ }+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct FrontendThresholds {+ #[serde(default = "defaults::Frontend::stackoverflow")]+ pub stackoverflow: f64,++ #[serde(default = "defaults::Frontend::discussions_widget")]+ pub discussions_widget: f64,+}++impl Default for FrontendThresholds {+ fn default() -> Self {+ Self {+ stackoverflow: defaults::Frontend::stackoverflow(),+ discussions_widget: defaults::Frontend::discussions_widget(),+ }+ }+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct FrontendConfig {+ pub queries_csv_path: String,+ pub host: SocketAddr,+ pub prometheus_host: SocketAddr,+ pub crossencoder_model_path: String,+ pub lambda_model_path: Option<String>,+ pub qa_model_path: Option<String>,+ pub with_alice: Option<bool>,+ pub bangs_path: String,+ pub summarizer_path: String,+ pub fact_check_model_path: String,+ pub query_store_db_host: Option<String>,+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,++ #[serde(default)]+ pub collector: CollectorConfig,++ #[serde(default)]+ pub thresholds: FrontendThresholds,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct SnippetConfig {+ #[serde(default = "defaults::Snippet::desired_num_chars")]+ pub desired_num_chars: usize,++ #[serde(default = "defaults::Snippet::delta_num_chars")]+ pub delta_num_chars: usize,++ #[serde(default = "defaults::Snippet::min_passage_width")]+ pub min_passage_width: usize,+}++impl Default for SnippetConfig {+ fn default() -> Self {+ Self {+ desired_num_chars: defaults::Snippet::desired_num_chars(),+ delta_num_chars: defaults::Snippet::delta_num_chars(),+ min_passage_width: defaults::Snippet::min_passage_width(),+ }+ }+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct SearchServerConfig {+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,+ pub shard_id: ShardId,+ pub index_path: String,+ pub entity_index_path: Option<String>,+ pub host_centrality_store_path: Option<String>,+ pub linear_model_path: Option<String>,+ pub lambda_model_path: Option<String>,+ pub host: SocketAddr,++ #[serde(default)]+ pub collector: CollectorConfig,++ #[serde(default)]+ pub snippet: SnippetConfig,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct CrawlCoordinatorConfig {+ pub crawldb_folder: String,+ pub host: SocketAddr,+ pub num_urls_to_crawl: u64,+ pub seed_urls: Vec<String>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct UserAgent {+ pub full: String,+ pub token: String,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct CrawlerConfig {+ pub num_workers: usize,+ pub user_agent: UserAgent,++ #[serde(default = "defaults::Crawler::politeness_factor")]+ pub politeness_factor: f32,++ #[serde(default = "defaults::Crawler::min_crawl_delay_ms")]+ pub min_crawl_delay_ms: u64,++ #[serde(default = "defaults::Crawler::max_crawl_delay_ms")]+ pub max_crawl_delay_ms: u64,++ #[serde(default = "defaults::Crawler::max_politeness_factor")]+ pub max_politeness_factor: f32,++ #[serde(default = "defaults::Crawler::min_politeness_factor")]+ pub min_politeness_factor: f32,++ #[serde(default = "defaults::Crawler::max_url_slowdown_retry")]+ pub max_url_slowdown_retry: u8,++ pub timeout_seconds: u64,+ pub s3: S3Config,+ pub coordinator_host: String,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+#[serde(tag = "type", content = "args", rename_all = "snake_case")]+pub enum AcceleratorDevice {+ Cpu,+ Cuda(usize),+ Mps,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+#[serde(tag = "type", rename_all = "snake_case")]+pub enum AcceleratorDtype {+ Float,+ Bf16,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct AliceAcceleratorConfig {+ pub layer_fraction: f64,+ /// percentage of layers on accelerator to quantize+ pub quantize_fraction: f64,+ pub device: AcceleratorDevice,+ pub dtype: AcceleratorDtype,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct AliceLocalConfig {+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,+ pub host: SocketAddr,+ pub summarizer_path: String,++ pub alice_path: String,+ pub accelerator: Option<AliceAcceleratorConfig>,+ /// base64 encoded+ pub encryption_key: String,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct WebgraphServerConfig {+ pub host: SocketAddr,+ pub graph_path: String,+ pub inbound_similarity_path: String,+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,++ #[serde(default = "defaults::WebgraphServer::max_similar_sites")]+ pub max_similar_sites: usize,+}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex 77e4f786..db6576d4 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -22,7 +22,7 @@ use std::{use tokio::sync::Mutex;-use crate::{webpage::Url, CrawlerConfig};+use crate::{config::CrawlerConfig, webpage::Url};use self::{warc_writer::{WarcWriter, WarcWriterMessage},@@ -37,8 +37,6 @@ mod worker;pub use coordinator::CrawlCoordinator;-const DEFAULT_POLITENESS_FACTOR: f32 = 1.0;-#[derive(thiserror::Error, Debug)]pub enum Error {#[error("reqwest error: {0}")]@@ -59,9 +57,6 @@ pub enum Error {#[error("invalid politeness factor")]InvalidPolitenessFactor,- #[error("sqlite error")]- Sqlite(#[from] rusqlite::Error),-#[error("addr parse error: {0}")]AddrParse(#[from] std::net::AddrParseError),@@ -170,7 +165,7 @@ pub struct Crawler {impl Crawler {pub async fn new(config: CrawlerConfig) -> Result<Self> {let pending_commands = Arc::new(Mutex::new(VecDeque::new()));- let writer = Arc::new(WarcWriter::new(config.s3));+ let writer = Arc::new(WarcWriter::new(config.s3.clone()));let timeout = Duration::from_secs(config.timeout_seconds);let mut handles = Vec::new();let coordinator_host = config.coordinator_host.parse()?;@@ -179,13 +174,9 @@ impl Crawler {let worker = Worker::new(Arc::clone(&pending_commands),Arc::clone(&writer),- config.user_agent.clone(),- config- .politeness_factor- .unwrap_or(DEFAULT_POLITENESS_FACTOR),+ config.clone(),timeout,coordinator_host,- config.num_workers,)?;handles.push(tokio::spawn(async move {diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex 30f6ca90..0c1fa821 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -16,7 +16,10 @@use std::time::Duration;-use crate::{warc, S3Config};+use crate::{+ config::{self, S3Config},+ warc,+};use super::{CrawlDatum, Result};@@ -32,7 +35,7 @@ pub enum WarcWriterMessage {Finish,}-async fn commit(writer: warc::WarcWriter, s3: S3Config) {+async fn commit(writer: warc::WarcWriter, s3: config::S3Config) {let filename = format!("{}_{}.warc.gz",chrono::Utc::now().to_rfc3339(),diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rsindex 8dbc8b1a..c8b19609 100644--- a/core/src/crawler/worker.rs+++ b/core/src/crawler/worker.rs@@ -26,10 +26,10 @@ use std::{use tokio::sync::Mutex;use crate::{+ config::CrawlerConfig,crawler::{JobResponse, Request, Response},distributed::{retry_strategy::ExponentialBackoff, sonic},webpage::{Html, Url},- UserAgent,};use super::{@@ -37,19 +37,12 @@ use super::{WarcWriter, WorkerJob,};-const MIN_CRAWL_DELAY_MS: u64 = 200;-const MAX_CRAWL_DELAY_MS: u64 = 30_000;-const MAX_POLITENESS_FACTOR: f32 = 1024.0;-const MIN_POLITENESS_FACTOR: f32 = 1.0;-const MAX_URL_SLOWDOWN_RETRY: u8 = 3;-pub struct Worker {current_job: Option<WorkerJob>,pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,client: reqwest::Client,- user_agent: UserAgent,- default_politeness_factor: f32,+ config: CrawlerConfig,politeness_factor: f32,coordinator_host: SocketAddr,num_jobs_per_fetch: usize,@@ -60,13 +53,11 @@ impl Worker {pub fn new(pending_commands: Arc<Mutex<VecDeque<Command>>>,writer: Arc<WarcWriter>,- user_agent: UserAgent,- politeness_factor: f32,+ config: CrawlerConfig,timeout: Duration,coordinator_host: SocketAddr,- num_jobs_per_fetch: usize,) -> Result<Self> {- if politeness_factor < MIN_POLITENESS_FACTOR {+ if config.politeness_factor < config.min_politeness_factor {return Err(Error::InvalidPolitenessFactor);}@@ -74,7 +65,7 @@ impl Worker {.timeout(timeout).connect_timeout(timeout).http2_keep_alive_interval(None)- .user_agent(&user_agent.full)+ .user_agent(&config.user_agent.full).build()?;let robotstxt = RobotsTxtManager::new(client.clone());@@ -84,11 +75,10 @@ impl Worker {client,current_job: None,pending_commands,- user_agent,- politeness_factor,- default_politeness_factor: politeness_factor,+ num_jobs_per_fetch: config.num_workers,+ politeness_factor: config.politeness_factor,+ config,coordinator_host,- num_jobs_per_fetch,robotstxt,})}@@ -159,7 +149,7 @@ impl Worker {}async fn process_job(&mut self) {- self.politeness_factor = self.default_politeness_factor;+ self.politeness_factor = self.config.politeness_factor;let mut job = self.current_job.take().unwrap();tracing::info!("Processing job: {:?}", job.domain);@@ -170,7 +160,7 @@ impl Worker {let mut crawled_sitemaps: HashSet<Site> = HashSet::new();while let Some(retryable_url) = job.urls.pop_front() {- if retryable_url.retries > MAX_URL_SLOWDOWN_RETRY {+ if retryable_url.retries > self.config.max_url_slowdown_retry {continue;}@@ -180,7 +170,7 @@ impl Worker {if !self.robotstxt- .is_allowed(&retryable_url.url, &self.user_agent.token)+ .is_allowed(&retryable_url.url, &self.config.user_agent.token).await{continue;@@ -200,12 +190,12 @@ impl Worker {let mut delay = res.fetch_time.mul_f32(self.politeness_factor);- if delay < Duration::from_millis(MIN_CRAWL_DELAY_MS) {- delay = Duration::from_millis(MIN_CRAWL_DELAY_MS);+ if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {+ delay = Duration::from_millis(self.config.min_crawl_delay_ms);}- if delay > Duration::from_millis(MAX_CRAWL_DELAY_MS) {- delay = Duration::from_millis(MAX_CRAWL_DELAY_MS);+ if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {+ delay = Duration::from_millis(self.config.max_crawl_delay_ms);}tokio::time::sleep(delay).await;@@ -308,8 +298,8 @@ impl Worker {if datum.status_code == 429 {self.politeness_factor *= 2.0;- if self.politeness_factor > MAX_POLITENESS_FACTOR {- self.politeness_factor = MAX_POLITENESS_FACTOR;+ if self.politeness_factor > self.config.max_politeness_factor {+ self.politeness_factor = self.config.max_politeness_factor;}tracing::warn!(@@ -480,7 +470,10 @@ impl Worker {.full().into();- tokio::time::sleep(Duration::from_millis(MIN_CRAWL_DELAY_MS)).await;+ tokio::time::sleep(Duration::from_millis(+ self.config.min_crawl_delay_ms,+ ))+ .await;urls.append(&mut self.urls_from_sitemap(url, depth + 1, max_depth).await,diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rsindex 9cec9839..28cdd524 100644--- a/core/src/entrypoint/alice.rs+++ b/core/src/entrypoint/alice.rs@@ -33,12 +33,12 @@ use tracing::info;use crate::{alice::{Alice, EncodedEncryptedState, EncryptedState},+ config::AliceLocalConfig,distributed::{cluster::Cluster,member::{Member, Service},},ttl_cache::TTLCache,- AliceLocalConfig,};pub struct State {diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex afa53ed1..00928dee 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -74,13 +74,13 @@ fn create_webgraph() -> Result<()> {let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");let job = webgraph::Job {- config: webgraph::JobConfig::Local(crate::LocalConfig {+ config: webgraph::JobConfig::Local(crate::config::LocalConfig {folder: ".".to_string(),names: vec![warc_path.to_str().unwrap().to_string()],}),warc_paths: vec![warc_path.to_str().unwrap().to_string()],graph_base_path: out_path_tmp.to_str().unwrap().to_string(),- level: crate::WebgraphLevel::Host,+ level: crate::config::WebgraphLevel::Host,};let worker = webgraph::WebgraphWorker { redirect: None };@@ -115,7 +115,7 @@ fn create_inverted_index() -> Result<()> {let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");let job = indexer::Job {- source_config: indexer::JobConfig::Local(crate::LocalConfig {+ source_config: indexer::JobConfig::Local(crate::config::LocalConfig {folder: ".".to_string(),names: vec![warc_path.to_str().unwrap().to_string()],}),diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex 2b4a290d..83336833 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -17,12 +17,13 @@use std::{net::SocketAddr, sync::Arc};use crate::{+ config,crawler::{CrawlCoordinator, Crawler},distributed::sonic,- CrawlCoordinatorConfig, CrawlerConfig, Result,+ Result,};-pub async fn worker(config: CrawlerConfig) -> Result<()> {+pub async fn worker(config: config::CrawlerConfig) -> Result<()> {let crawler = Crawler::new(config).await?;crawler.wait().await;@@ -30,7 +31,7 @@ pub async fn worker(config: CrawlerConfig) -> Result<()> {Ok(())}-pub async fn coordinator(config: CrawlCoordinatorConfig) -> Result<()> {+pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {let coordinator = Arc::new(CrawlCoordinator::new(config.crawldb_folder,config.num_urls_to_crawl,diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rsindex c21823a1..90aac536 100644--- a/core/src/entrypoint/frontend.rs+++ b/core/src/entrypoint/frontend.rs@@ -18,11 +18,11 @@ use anyhow::Result;use crate::{api::{metrics_router, router, Counters},+ config,metrics::Label,- FrontendConfig,};-pub async fn run(config: FrontendConfig) -> Result<()> {+pub async fn run(config: config::FrontendConfig) -> Result<()> {let search_counter_success = crate::metrics::Counter::default();let search_counter_fail = crate::metrics::Counter::default();let explore_counter = crate::metrics::Counter::default();diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex ca065eec..5f083365 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize};use tokio::pin;use tracing::{debug, info, trace, warn};+use crate::config;use crate::entrypoint::download_all_warc_files;use crate::executor::Executor;use crate::index::{FrozenIndex, Index};@@ -31,36 +32,33 @@ use crate::ranking::SignalAggregator;use crate::warc::WarcFile;use crate::webgraph::{Node, Webgraph, WebgraphBuilder};use crate::webpage::{Html, Link, Webpage};-use crate::{- human_website_annotations, HttpConfig, IndexingLocalConfig, LocalConfig, Result, S3Config,- WarcSource,-};+use crate::{human_website_annotations, Result};pub struct Indexer {}#[derive(Debug, Serialize, Deserialize, Clone)]pub enum JobConfig {- Http(HttpConfig),- Local(LocalConfig),- S3(S3Config),+ Http(config::HttpConfig),+ Local(config::LocalConfig),+ S3(config::S3Config),}-impl From<WarcSource> for JobConfig {- fn from(value: WarcSource) -> Self {+impl From<config::WarcSource> for JobConfig {+ fn from(value: config::WarcSource) -> Self {match value {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- WarcSource::S3(config) => JobConfig::S3(config),+ config::WarcSource::HTTP(config) => JobConfig::Http(config),+ config::WarcSource::Local(config) => JobConfig::Local(config),+ config::WarcSource::S3(config) => JobConfig::S3(config),}}}-impl From<JobConfig> for WarcSource {+impl From<JobConfig> for config::WarcSource {fn from(value: JobConfig) -> Self {match value {- JobConfig::Http(config) => WarcSource::HTTP(config),- JobConfig::Local(config) => WarcSource::Local(config),- JobConfig::S3(config) => WarcSource::S3(config),+ JobConfig::Http(config) => config::WarcSource::HTTP(config),+ JobConfig::Local(config) => config::WarcSource::Local(config),+ JobConfig::S3(config) => config::WarcSource::S3(config),}}}@@ -108,7 +106,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();- let source: WarcSource = job.source_config.clone().into();+ let source: config::WarcSource = job.source_config.clone().into();let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);pin!(warc_files);@@ -320,7 +318,7 @@ impl Reduce<Index> for Index {}impl Indexer {- pub fn run(config: &IndexingLocalConfig) -> Result<()> {+ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;let job_config: JobConfig = config.warc_source.clone().into();diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 0056c487..04440029 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -37,11 +37,11 @@ pub use indexer::Indexer;use tracing::{debug, log::error};pub use webgraph::Webgraph;-use crate::{warc::WarcFile, WarcSource};+use crate::{config, warc::WarcFile};fn download_all_warc_files<'a>(warc_paths: &'a [String],- source: &'a WarcSource,+ source: &'a config::WarcSource,base_path: &'a str,) -> impl Iterator<Item = String> + 'a {let download_path = Path::new(base_path).join("warc_files");diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 83255c63..d5139d90 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -19,6 +19,7 @@ use std::{collections::HashMap, net::SocketAddr};use tracing::info;use crate::{+ config,distributed::{cluster::Cluster,member::{Member, Service},@@ -33,10 +34,10 @@ use crate::{},searcher::{self, LocalSearcher},webpage::Url,- Result, SearchServerConfig,+ Result,};-pub async fn run(config: SearchServerConfig) -> Result<()> {+pub async fn run(config: config::SearchServerConfig) -> Result<()> {let addr: SocketAddr = config.host;let server = sonic::Server::bind(addr).await.unwrap();@@ -66,6 +67,9 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {local_searcher.set_lambda_model(LambdaMART::open(model_path)?);}+ local_searcher.set_collector_config(config.collector);+ local_searcher.set_snippet_config(config.snippet);+// dropping the handle leaves the clusterlet _cluster_handle = Cluster::join(Member {@@ -100,7 +104,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {}}searcher::Request::Search(query) => {- match local_searcher.search_initial(query, false) {+ match local_searcher.search_initial(query, true) {Ok(result) => {req.respond(sonic::Response::Content(result)).await.ok();}diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 3a9b8646..a4928652 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -14,13 +14,16 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ config::WarcSource,+ config::WebgraphLevel,+ config::{self, WebgraphConstructConfig},crawler::crawl_db::RedirectDb,entrypoint::download_all_warc_files,mapreduce::{Map, Reduce, Worker},warc::WarcFile,webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,- HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphConstructConfig, WebgraphLevel,+ Result,};use itertools::Itertools;use rayon::prelude::*;@@ -36,26 +39,26 @@ struct GraphPointer {#[derive(Debug, Serialize, Deserialize, Clone)]pub enum JobConfig {- Http(HttpConfig),- Local(LocalConfig),- S3(S3Config),+ Http(config::HttpConfig),+ Local(config::LocalConfig),+ S3(config::S3Config),}-impl From<WarcSource> for JobConfig {- fn from(value: WarcSource) -> Self {+impl From<config::WarcSource> for JobConfig {+ fn from(value: config::WarcSource) -> Self {match value {- WarcSource::HTTP(config) => JobConfig::Http(config),- WarcSource::Local(config) => JobConfig::Local(config),- WarcSource::S3(config) => JobConfig::S3(config),+ config::WarcSource::HTTP(config) => JobConfig::Http(config),+ config::WarcSource::Local(config) => JobConfig::Local(config),+ config::WarcSource::S3(config) => JobConfig::S3(config),}}}-impl From<JobConfig> for WarcSource {+impl From<JobConfig> for config::WarcSource {fn from(value: JobConfig) -> Self {match value {- JobConfig::Http(config) => WarcSource::HTTP(config),- JobConfig::Local(config) => WarcSource::Local(config),+ JobConfig::Http(config) => config::WarcSource::HTTP(config),+ JobConfig::Local(config) => config::WarcSource::Local(config),JobConfig::S3(config) => WarcSource::S3(config),}}diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex df90ca69..aecc196a 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -22,6 +22,7 @@ use serde::Deserialize;use serde::Serialize;use tracing::info;+use crate::config;use crate::distributed::cluster::Cluster;use crate::distributed::member::Member;use crate::distributed::member::Service;@@ -33,7 +34,6 @@ use crate::webgraph::Node;use crate::webgraph::WebgraphBuilder;use crate::webpage::Url;use crate::Result;-use crate::WebgraphServerConfig;#[derive(Serialize, Deserialize)]pub enum Request {@@ -50,7 +50,7 @@ pub struct ScoredSite {const MAX_SITES: usize = 20;-pub async fn run(config: WebgraphServerConfig) -> Result<()> {+pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {let addr: SocketAddr = config.host;// dropping the handle leaves the cluster@@ -70,7 +70,11 @@ pub async fn run(config: WebgraphServerConfig) -> Result<()> {let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;- let similar_sites_finder = SimilarSitesFinder::new(Arc::clone(&graph), inbound_similarity);+ let similar_sites_finder = SimilarSitesFinder::new(+ Arc::clone(&graph),+ inbound_similarity,+ config.max_similar_sites,+ );let server = sonic::Server::bind(addr).await.unwrap();diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 017ef8e1..5ae6d04c 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -35,6 +35,7 @@ use tantivy::tokenizer::TokenizerManager;use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};use crate::collector::Hashes;+use crate::config::SnippetConfig;use crate::fastfield_reader::FastFieldReader;use crate::query::Query;use crate::ranking::initial::Score;@@ -102,6 +103,7 @@ pub struct InvertedIndex {writer: IndexWriter,reader: IndexReader,schema: Arc<Schema>,+ snippet_config: SnippetConfig,}impl InvertedIndex {@@ -165,9 +167,14 @@ impl InvertedIndex {schema: Arc::new(schema),path: path.as_ref().to_str().unwrap().to_string(),tantivy_index,+ snippet_config: SnippetConfig::default(),})}+ pub fn set_snippet_config(&mut self, config: SnippetConfig) {+ self.snippet_config = config;+ }+pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {FastFieldReader::new(tv_searcher)}@@ -305,7 +312,8 @@ impl InvertedIndex {.collect();for page in &mut webpages {- page.snippet = snippet::generate(query, &page.body, &page.region);+ page.snippet =+ snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());}Ok(webpages)@@ -649,6 +657,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");@@ -678,6 +687,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -724,6 +734,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -768,6 +779,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -812,6 +824,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -886,6 +899,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let mut result =@@ -940,6 +954,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -983,6 +998,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -1051,6 +1067,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -1078,6 +1095,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -1108,6 +1126,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");@@ -1152,6 +1171,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -1216,6 +1236,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =@@ -1266,6 +1287,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),+ Default::default(),);let result =diff --git a/core/src/lib.rs b/core/src/lib.rsindex 5cb425e4..5ddbc29e 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -21,11 +21,7 @@#![allow(clippy::cast_precision_loss)]#![allow(clippy::missing_errors_doc)]-use searcher::ShardId;-use serde::{Deserialize, Serialize};-use std::fs::File;-use std::io::{self, BufRead};-use std::net::SocketAddr;+use std::io;use std::num::ParseIntError;use std::path::PathBuf;use tantivy::TantivyError;@@ -42,6 +38,7 @@ mod autosuggest;mod bangs;mod call_counter;mod collector;+pub mod config;pub mod crawler;mod directory;mod distributed;@@ -84,229 +81,6 @@ pub mod webgraph;pub mod webpage;mod widgets;-#[derive(Debug, Deserialize, Clone)]-pub struct IndexingLocalConfig {- limit_warc_files: Option<usize>,- skip_warc_files: Option<usize>,- warc_source: WarcSource,- batch_size: Option<usize>,- page_webgraph_path: Option<String>,- output_path: Option<String>,- host_centrality_threshold: Option<f64>,- topics_path: Option<String>,- host_centrality_store_path: String,- page_centrality_store_path: Option<String>,- minimum_clean_words: Option<usize>,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-#[serde(rename_all = "lowercase")]-pub enum WebgraphLevel {- Page,- Host,-}--#[derive(Debug, Deserialize, Clone)]-pub struct WebgraphConstructConfig {- level: WebgraphLevel,- redirect_db_path: Option<String>,- limit_warc_files: Option<usize>,- warc_source: WarcSource,- graph_base_path: Option<String>,- batch_size: Option<usize>,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-#[serde(tag = "type")]-pub enum WarcSource {- HTTP(HttpConfig),- Local(LocalConfig),- S3(S3Config),-}--impl WarcSource {- pub fn paths(&self) -> Result<Vec<String>> {- let mut warc_paths = Vec::new();- match &self {- WarcSource::HTTP(config) => {- let file = File::open(&config.warc_paths_file)?;- for line in io::BufReader::new(file).lines() {- warc_paths.push(line?);- }- }- WarcSource::Local(config) => {- warc_paths = config.names.clone();- }- WarcSource::S3(config) => {- let bucket = s3::Bucket::new(- &config.bucket,- s3::Region::Custom {- region: "".to_string(),- endpoint: config.endpoint.clone(),- },- s3::creds::Credentials {- access_key: Some(config.access_key.clone()),- secret_key: Some(config.secret_key.clone()),- security_token: None,- session_token: None,- expiration: None,- },- )?- .with_path_style();-- let mut folder = config.folder.clone();-- if !folder.ends_with('/') {- folder.push('/');- }-- let pages = bucket.list_blocking(folder, Some("/".to_string()))?;-- let objects = pages- .into_iter()- .flat_map(|p| p.contents.into_iter())- .collect::<Vec<_>>();-- for p in objects.into_iter().filter_map(|o| {- if o.key.ends_with("warc.gz") {- Some(o.key)- } else {- None- }- }) {- warc_paths.push(p);- }- }- }-- Ok(warc_paths)- }-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct LocalConfig {- pub folder: String,- pub names: Vec<String>,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct HttpConfig {- base_url: String,- warc_paths_file: String,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct S3Config {- pub bucket: String,- pub folder: String,- pub access_key: String,- pub secret_key: String,- pub endpoint: String,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct FrontendConfig {- pub queries_csv_path: String,- pub host: SocketAddr,- pub prometheus_host: SocketAddr,- pub crossencoder_model_path: String,- pub lambda_model_path: Option<String>,- pub qa_model_path: Option<String>,- pub with_alice: Option<bool>,- pub bangs_path: String,- pub summarizer_path: String,- pub fact_check_model_path: String,- pub query_store_db_host: Option<String>,- pub cluster_id: String,- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,- pub gossip_addr: SocketAddr,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct SearchServerConfig {- pub cluster_id: String,- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,- pub gossip_addr: SocketAddr,- pub shard_id: ShardId,- pub index_path: String,- pub entity_index_path: Option<String>,- pub host_centrality_store_path: Option<String>,- pub linear_model_path: Option<String>,- pub lambda_model_path: Option<String>,- pub host: SocketAddr,-}-#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct CrawlCoordinatorConfig {- pub crawldb_folder: String,- pub host: SocketAddr,- pub num_urls_to_crawl: u64,- pub seed_urls: Vec<String>,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct UserAgent {- pub full: String,- pub token: String,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct CrawlerConfig {- pub num_workers: usize,- pub user_agent: UserAgent,- pub politeness_factor: Option<f32>,- pub timeout_seconds: u64,- pub s3: S3Config,- pub coordinator_host: String,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-#[serde(tag = "type", content = "args", rename_all = "snake_case")]-pub enum AcceleratorDevice {- Cpu,- Cuda(usize),- Mps,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-#[serde(tag = "type", rename_all = "snake_case")]-pub enum AcceleratorDtype {- Float,- Bf16,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct AliceAcceleratorConfig {- pub layer_fraction: f64,- /// percentage of layers on accelerator to quantize- pub quantize_fraction: f64,- pub device: AcceleratorDevice,- pub dtype: AcceleratorDtype,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct AliceLocalConfig {- pub cluster_id: String,- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,- pub gossip_addr: SocketAddr,- pub host: SocketAddr,- pub summarizer_path: String,-- pub alice_path: String,- pub accelerator: Option<AliceAcceleratorConfig>,- /// base64 encoded- pub encryption_key: String,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct WebgraphServerConfig {- pub host: SocketAddr,- pub graph_path: String,- pub inbound_similarity_path: String,- pub cluster_id: String,- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,- pub gossip_addr: SocketAddr,-}-#[derive(Error, Debug)]pub enum Error {#[error("Failed to download object from HTTP")]diff --git a/core/src/main.rs b/core/src/main.rsindex f95a06a2..2cd24313 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -18,16 +18,15 @@ use clap::{Parser, Subcommand};use serde::de::DeserializeOwned;use std::fs;use std::path::Path;+use stract::config;use stract::entrypoint::autosuggest_scrape::{self, Gl};+#[cfg(feature = "dev")]use stract::entrypoint::configure;+use stract::entrypoint::indexer::IndexPointer;use stract::entrypoint::{self, frontend, search_server, webgraph_server};use stract::webgraph::WebgraphBuilder;-use stract::{- AliceLocalConfig, CrawlCoordinatorConfig, CrawlerConfig, FrontendConfig, SearchServerConfig,- WebgraphServerConfig,-};use tracing::Level;use tracing_subscriber::FmtSubscriber;@@ -231,7 +230,7 @@ fn main() -> Result<()> {}}WebgraphOptions::Server { config_path } => {- let config: WebgraphServerConfig = load_toml_config(config_path);+ let config: config::WebgraphServerConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()@@ -240,7 +239,7 @@ fn main() -> Result<()> {}},Commands::Frontend { config_path } => {- let config: FrontendConfig = load_toml_config(config_path);+ let config: config::FrontendConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()@@ -248,7 +247,7 @@ fn main() -> Result<()> {.block_on(frontend::run(config))?}Commands::SearchServer { config_path } => {- let config: SearchServerConfig = load_toml_config(config_path);+ let config: config::SearchServerConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()@@ -280,7 +279,7 @@ fn main() -> Result<()> {} => entrypoint::dmoz_parser::run(dmoz_file, output_path).unwrap(),Commands::Alice { options } => match options {AliceOptions::Serve { config_path } => {- let config: AliceLocalConfig = load_toml_config(config_path);+ let config: config::AliceLocalConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()@@ -290,7 +289,7 @@ fn main() -> Result<()> {AliceOptions::GenerateKey => entrypoint::alice::generate_key(),},Commands::CrawlCoordinator { config_path } => {- let config: CrawlCoordinatorConfig = load_toml_config(config_path);+ let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()@@ -298,7 +297,7 @@ fn main() -> Result<()> {.block_on(entrypoint::crawler::coordinator(config))?}Commands::Crawler { config_path } => {- let config: CrawlerConfig = load_toml_config(config_path);+ let config: config::CrawlerConfig = load_toml_config(config_path);tokio::runtime::Builder::new_multi_thread().enable_all()diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 09b77ea0..bae1dbc7 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -34,7 +34,7 @@ use crate::{inverted_index,search_ctx::Ctx,searcher::NUM_RESULTS_PER_PAGE,- webpage::region::Region,+ webpage::region::Region, config::CollectorConfig,};pub use self::signal::*;@@ -47,10 +47,11 @@ pub struct Ranker {fastfield_reader: FastFieldReader,de_rank_similar: bool,num_results: Option<usize>,+ collector_config: CollectorConfig,}impl Ranker {- pub fn new(aggregator: SignalAggregator, fastfield_reader: FastFieldReader) -> Self {+ pub fn new(aggregator: SignalAggregator, fastfield_reader: FastFieldReader, collector_config: CollectorConfig) -> Self {Ranker {offset: None,aggregator,@@ -58,6 +59,7 @@ impl Ranker {de_rank_similar: true,fastfield_reader,num_results: None,+ collector_config,}}@@ -118,6 +120,8 @@ impl Ranker {collector = collector.and_max_docs(max_docs.clone());}+ collector = collector.and_collector_config(self.collector_config.clone());+collector.tweak_score(score_tweaker)}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 78093885..59b2d502 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -20,6 +20,7 @@ use serde::{Deserialize, Serialize};use crate::{collector::{self, BucketCollector},+ config::CollectorConfig,enum_map::EnumMap,inverted_index::WebsitePointer,searcher::SearchQuery,@@ -40,8 +41,8 @@ impl<T> collector::Doc for TwhereT: AsRankingWebsite,{- fn score(&self) -> &f64 {- &self.as_ranking().score+ fn score(&self) -> f64 {+ self.as_ranking().score}fn id(&self) -> &tantivy::DocId {@@ -234,7 +235,13 @@ struct RankingStage<T: AsRankingWebsite> {}impl<T: AsRankingWebsite> RankingStage<T> {- fn apply(&self, websites: Vec<T>, top_n: usize, offset: usize) -> Vec<T> {+ fn apply(+ &self,+ websites: Vec<T>,+ top_n: usize,+ offset: usize,+ collector_config: CollectorConfig,+ ) -> Vec<T> {let mut websites = websites.into_iter().skip(offset)@@ -251,7 +258,8 @@ impl<T: AsRankingWebsite> RankingStage<T> {}}- let mut collector = BucketCollector::new(self.stage_top_n.max(top_n) + offset);+ let mut collector =+ BucketCollector::new(self.stage_top_n.max(top_n) + offset, collector_config);for website in websites {collector.insert(website);@@ -269,23 +277,26 @@ pub struct RankingPipeline<T: AsRankingWebsite> {stage: RankingStage<T>,page: usize,pub top_n: usize,+ collector_config: CollectorConfig,}impl<T: AsRankingWebsite> RankingPipeline<T> {fn create_reranking<M: CrossEncoder + 'static>(crossencoder: Arc<M>,lambda: Option<Arc<LambdaMART>>,+ collector_config: CollectorConfig,) -> Result<Self> {let stage = RankingStage {scorer: Box::new(ReRanker::new(crossencoder, lambda)),stage_top_n: 20,- derank_similar: false,+ derank_similar: true,};Ok(Self {stage,page: 0,top_n: 0,+ collector_config,})}@@ -293,14 +304,15 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {query: &mut SearchQuery,crossencoder: Arc<M>,lambda: Option<Arc<LambdaMART>>,+ collector_config: CollectorConfig,) -> Result<Self> {- let mut pipeline = Self::create_reranking(crossencoder, lambda)?;+ let mut pipeline = Self::create_reranking(crossencoder, lambda, collector_config)?;pipeline.set_query_info(query);Ok(pipeline)}- fn create_ltr(model: Option<Arc<LambdaMART>>) -> Self {+ fn create_ltr(model: Option<Arc<LambdaMART>>, collector_config: CollectorConfig) -> Self {let last_stage = RankingStage {scorer: Box::new(Initial {model,@@ -314,11 +326,16 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {stage: last_stage,page: 0,top_n: 0,+ collector_config,}}- pub fn ltr_for_query(query: &mut SearchQuery, model: Option<Arc<LambdaMART>>) -> Self {- let mut pipeline = Self::create_ltr(model);+ pub fn ltr_for_query(+ query: &mut SearchQuery,+ model: Option<Arc<LambdaMART>>,+ collector_config: CollectorConfig,+ ) -> Self {+ let mut pipeline = Self::create_ltr(model, collector_config);pipeline.set_query_info(query);pipeline@@ -338,7 +355,12 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}pub fn apply(self, websites: Vec<T>) -> Vec<T> {- self.stage.apply(websites, self.top_n, self.offset())+ self.stage.apply(+ websites,+ self.top_n,+ self.offset(),+ self.collector_config.clone(),+ )}pub fn collector_top_n(&self) -> usize {@@ -399,6 +421,7 @@ mod tests {},Arc::new(DummyCrossEncoder {}),None,+ CollectorConfig::default(),).unwrap();assert_eq!(pipeline.collector_top_n(), 20);@@ -429,6 +452,7 @@ mod tests {},Arc::new(DummyCrossEncoder {}),None,+ CollectorConfig::default(),).unwrap();@@ -462,6 +486,7 @@ mod tests {},Arc::new(DummyCrossEncoder {}),None,+ CollectorConfig::default(),).unwrap();@@ -475,6 +500,7 @@ mod tests {},Arc::new(DummyCrossEncoder {}),None,+ CollectorConfig::default(),).unwrap();diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 94b15541..a22e443b 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -250,30 +250,30 @@ impl Signal {pub fn default_coefficient(&self) -> f64 {match self {Signal::Bm25 => 0.0005,- Signal::Bm25Title => 0.004,+ Signal::Bm25Title => 0.005,Signal::Bm25TitleBigrams => 0.005,- Signal::Bm25TitleTrigrams => 0.01,- Signal::Bm25CleanBody => 0.004,+ Signal::Bm25TitleTrigrams => 0.005,+ Signal::Bm25CleanBody => 0.005,Signal::Bm25CleanBodyBigrams => 0.005,- Signal::Bm25CleanBodyTrigrams => 0.01,- Signal::Bm25StemmedTitle => 0.001,+ Signal::Bm25CleanBodyTrigrams => 0.005,+ Signal::Bm25StemmedTitle => 0.0005,Signal::Bm25StemmedCleanBody => 0.0005,Signal::Bm25AllBody => 0.0,- Signal::Bm25Url => 0.0001,- Signal::Bm25Site => 0.0001,- Signal::Bm25Domain => 0.0001,- Signal::Bm25SiteNoTokenizer => 0.0001,- Signal::Bm25DomainNoTokenizer => 0.0001,- Signal::Bm25DomainIfHomepage => 0.0001,+ Signal::Bm25Url => 0.00005,+ Signal::Bm25Site => 0.00005,+ Signal::Bm25Domain => 0.00005,+ Signal::Bm25SiteNoTokenizer => 0.00005,+ Signal::Bm25DomainNoTokenizer => 0.00005,+ Signal::Bm25DomainIfHomepage => 0.00005,Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0001,- Signal::Bm25TitleIfHomepage => 0.0001,- Signal::Bm25BacklinkText => 0.01,+ Signal::Bm25TitleIfHomepage => 0.00001,+ Signal::Bm25BacklinkText => 0.001,Signal::Bm25Description => 5.99679347128802e-05,- Signal::ProximitySlop0 => 0.0005,- Signal::ProximitySlop1 => 0.0005,- Signal::ProximitySlop2 => 0.0005,- Signal::ProximitySlop4 => 0.0005,- Signal::ProximitySlop8 => 0.0005,+ Signal::ProximitySlop0 => 0.01,+ Signal::ProximitySlop1 => 0.01,+ Signal::ProximitySlop2 => 0.01,+ Signal::ProximitySlop4 => 0.01,+ Signal::ProximitySlop8 => 0.01,Signal::CrossEncoder => 0.16801589371906178,Signal::HostCentrality => 1.0,Signal::PageCentrality => 2.0,@@ -446,21 +446,19 @@ impl Signal {| Signal::ProximitySlop2| Signal::ProximitySlop4| Signal::ProximitySlop8 => {- signal_aggregator+ let reader = signal_aggregator.segment_reader.as_mut()- .and_then(|reader| {- reader.proximity_scorers.get_mut(self).map(|scorer| {- let docset = &mut scorer.docset;- if docset.doc() == doc- || (docset.doc() < doc && docset.seek(doc) == doc)- {- docset.score() as f64- } else {- 0.0- }- })- })+ .expect("set segment reader before computing signals");++ reader.proximity_scorers.get_mut(self).map(|scorer| {+ let docset = &mut scorer.docset;+ if docset.doc() == doc || (docset.doc() < doc && docset.seek(doc) == doc) {+ docset.score() as f64+ } else {+ 0.0+ }+ })}Signal::CrossEncoder => None, // this is calculated in a later stepSignal::LambdaMART => None,diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rsindex 1546df33..77be5be9 100644--- a/core/src/searcher/frontend.rs+++ b/core/src/searcher/frontend.rs@@ -24,6 +24,7 @@ use itertools::intersperse;use optics::Optic;use crate::bangs::{Bang, BangHit};+use crate::config::{CollectorConfig, FrontendThresholds};use crate::inverted_index::RetrievedWebpage;use crate::ranking::ALL_SIGNALS;use crate::search_prettifier::{@@ -48,15 +49,14 @@ use super::{SearchResult, WebsitesResult,};-const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 0.1;-const DISCUSSIONS_WIDGET_THRESHOLD: f64 = 0.1;-pub struct FrontendSearcher {distributed_searcher: DistributedSearcher,cross_encoder: Arc<CrossEncoderModel>,lambda_model: Option<Arc<LambdaMART>>,qa_model: Option<Arc<QaModel>>,bangs: Bangs,+ collector_config: CollectorConfig,+ thresholds: FrontendThresholds,}impl FrontendSearcher {@@ -66,6 +66,8 @@ impl FrontendSearcher {lambda_model: Option<LambdaMART>,qa_model: Option<QaModel>,bangs: Bangs,+ collector_config: CollectorConfig,+ thresholds: FrontendThresholds,) -> Self {Self {distributed_searcher: DistributedSearcher::new(cluster),@@ -73,6 +75,8 @@ impl FrontendSearcher {lambda_model: lambda_model.map(Arc::new),qa_model: qa_model.map(Arc::new),bangs,+ collector_config,+ thresholds,}}@@ -81,7 +85,8 @@ impl FrontendSearcher {initial_results: Vec<InitialSearchResultShard>,pipeline: RankingPipeline<ScoredWebsitePointer>,) -> (Vec<ScoredWebsitePointer>, bool) {- let mut collector = BucketCollector::new(pipeline.collector_top_n());+ let mut collector =+ BucketCollector::new(pipeline.collector_top_n(), self.collector_config.clone());let mut num_sites: usize = 0;for result in initial_results {@@ -136,7 +141,7 @@ impl FrontendSearcher {results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));if let Some((shard, website)) = results.pop() {- if website.score > STACKOVERFLOW_SIDEBAR_THRESHOLD {+ if website.score > self.thresholds.stackoverflow {let scored_websites = vec![ScoredWebsitePointer { website, shard }];let mut retrieved = self.distributed_searcher@@ -228,6 +233,7 @@ impl FrontendSearcher {&mut query,self.cross_encoder.clone(),self.lambda_model.clone(),+ self.collector_config.clone(),)?;let initial_results = self.distributed_searcher.search_initial(&query).await;@@ -258,7 +264,7 @@ impl FrontendSearcher {scores[scores.len() / 2]};- if median < DISCUSSIONS_WIDGET_THRESHOLD {+ if median < self.thresholds.discussions_widget {return Ok(None);}@@ -308,6 +314,7 @@ impl FrontendSearcher {&mut search_query,self.cross_encoder.clone(),self.lambda_model.clone(),+ self.collector_config.clone(),)?;let initial_results = selfdiff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 659efff6..ea7f8eb4 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -17,6 +17,7 @@use std::collections::HashMap;use std::sync::Arc;+use crate::config::{CollectorConfig, SnippetConfig};use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;@@ -45,6 +46,7 @@ pub struct LocalSearcher {centrality_store: Option<SearchCentralityStore>,linear_regression: Option<Arc<LinearRegression>>,lambda_model: Option<Arc<LambdaMART>>,+ collector_config: CollectorConfig,}impl From<Index> for LocalSearcher {@@ -69,6 +71,7 @@ impl LocalSearcher {centrality_store: None,linear_regression: None,lambda_model: None,+ collector_config: CollectorConfig::default(),}}@@ -88,6 +91,14 @@ impl LocalSearcher {self.lambda_model = Some(Arc::new(model));}+ pub fn set_collector_config(&mut self, config: CollectorConfig) {+ self.collector_config = config;+ }++ pub fn set_snippet_config(&mut self, config: SnippetConfig) {+ self.index.inverted_index.set_snippet_config(config);+ }+fn parse_query(&self, ctx: &Ctx, query: &SearchQuery) -> Result<Query> {let parsed_query = Query::parse(ctx, query, &self.index.inverted_index)?;@@ -110,6 +121,7 @@ impl LocalSearcher {let mut ranker = Ranker::new(aggregator,self.index.inverted_index.fastfield_reader(&ctx.tv_searcher),+ self.collector_config.clone(),);if let Some(region) = query.region() {@@ -155,8 +167,11 @@ impl LocalSearcher {de_rank_similar: bool,) -> Result<InvertedIndexResult> {let mut query = query.clone();- let pipeline: RankingPipeline<RankingWebsite> =- RankingPipeline::ltr_for_query(&mut query, self.lambda_model.clone());+ let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::ltr_for_query(+ &mut query,+ self.lambda_model.clone(),+ self.collector_config.clone(),+ );let parsed_query = self.parse_query(ctx, &query)?;let mut aggregator = SignalAggregator::new(Some(&parsed_query));@@ -298,13 +313,17 @@ impl LocalSearcher {let mut search_query = query.clone();let pipeline = match CrossEncoderModel::open("data/cross_encoder") {- Ok(model) => {- RankingPipeline::reranking_for_query(&mut search_query, Arc::new(model), None)?- }+ Ok(model) => RankingPipeline::reranking_for_query(+ &mut search_query,+ Arc::new(model),+ None,+ self.collector_config.clone(),+ )?,Err(_) => RankingPipeline::reranking_for_query(&mut search_query,Arc::new(DummyCrossEncoder {}),None,+ self.collector_config.clone(),)?,};diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 4d3c7377..6d593546 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -22,8 +22,6 @@ use crate::{webgraph::{Node, NodeID, Webgraph},};-const MAX_SIMILAR_SITES: usize = 1_000;-#[derive(serde::Serialize, serde::Deserialize, Debug)]pub struct ScoredNode {pub node: Node,@@ -58,18 +56,24 @@ impl Eq for ScoredNodeID {}pub struct SimilarSitesFinder {webgraph: Arc<Webgraph>,inbound_similarity: InboundSimilarity,+ max_similar_sites: usize,}impl SimilarSitesFinder {- pub fn new(webgraph: Arc<Webgraph>, inbound_similarity: InboundSimilarity) -> Self {+ pub fn new(+ webgraph: Arc<Webgraph>,+ inbound_similarity: InboundSimilarity,+ max_similar_sites: usize,+ ) -> Self {Self {webgraph,inbound_similarity,+ max_similar_sites,}}pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {- let limit = limit.min(MAX_SIMILAR_SITES);+ let limit = limit.min(self.max_similar_sites);let nodes: Vec<_> = nodes.iter()diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex 7543c6fd..8a7d5426 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::config::SnippetConfig;use crate::query::Query;use crate::search_prettifier::html_escape;use crate::spell::sentence_ranges;@@ -34,10 +35,6 @@ use whatlang::Lang;/// This might require us to store each paragraph of the webpage separately to get adequate performance (maybe we can split passages online with adequate performance/// but we need to test this).-const DESIRED_NUM_CHARS: usize = 275;-const DELTA_NUM_CHARS: usize = 50;-const MIN_PASSAGE_WIDTH: usize = 20;-const K1: f64 = 1.2;const B: f64 = 0.75;@@ -90,7 +87,12 @@ impl SnippetString {}}-fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> SnippetString {+fn snippet_string(+ text: &str,+ terms: &[String],+ lang: whatlang::Lang,+ config: SnippetConfig,+) -> SnippetString {let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));let terms: HashSet<String> = terms@@ -109,7 +111,7 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippetlet mut passages: Vec<_> = sentence_ranges(text).into_iter()- .filter(|offset| offset.end - offset.start > MIN_PASSAGE_WIDTH)+ .filter(|offset| offset.end - offset.start > config.min_passage_width).map(|offset| {let sentence = text[offset].to_string();@@ -132,7 +134,7 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippetif passages.is_empty() {let mut snippet = SnippetString {- fragment: text.chars().take(DESIRED_NUM_CHARS).collect(),+ fragment: text.chars().take(config.desired_num_chars).collect(),highlighted: Vec::new(),};@@ -191,18 +193,18 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippethighlighted: Vec::new(),};- if snippet.fragment.len() > DESIRED_NUM_CHARS + DELTA_NUM_CHARS {- // TODO: find 'DESIRED_NUM_CHARS' sized window that contains most highlights+ if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {+ // TODO: find 'desired_num_chars' sized window that contains most highlights// instead of taking the prefix of the passage as a snippetsnippet.fragment = snippet.fragment.chars()- .take(DESIRED_NUM_CHARS + DELTA_NUM_CHARS)+ .take(config.desired_num_chars + config.delta_num_chars).collect();} else {let mut next_passage_idx = best_idx + 1;- while snippet.fragment.len() < DESIRED_NUM_CHARS - DELTA_NUM_CHARS+ while snippet.fragment.len() < config.desired_num_chars - config.delta_num_chars&& next_passage_idx < passages.len(){snippet.fragment += " ";@@ -210,11 +212,11 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippetnext_passage_idx += 1;}- if snippet.fragment.len() > DESIRED_NUM_CHARS + DELTA_NUM_CHARS {+ if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {snippet.fragment = snippet.fragment.chars()- .take(DESIRED_NUM_CHARS + DELTA_NUM_CHARS)+ .take(config.desired_num_chars + config.delta_num_chars).collect();}}@@ -223,13 +225,13 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippetsnippet}-pub fn generate(query: &Query, text: &str, region: &Region) -> String {+pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfig) -> String {let lang = match region.lang() {Some(lang) => lang,None => whatlang::detect_lang(text).unwrap_or(Lang::Eng),};- let snippet = snippet_string(text, query.simple_terms(), lang);+ let snippet = snippet_string(text, query.simple_terms(), lang, config);snippet.to_html()}@@ -380,9 +382,14 @@ Survey in 2016, 2017, and 2018."#;#[test]fn empty_query() {assert_eq!(- snippet_string("this is a test", &[], whatlang::Lang::Eng)- .fragment- .as_str(),+ snippet_string(+ "this is a test",+ &[],+ whatlang::Lang::Eng,+ SnippetConfig::default()+ )+ .fragment+ .as_str(),"this is a test");}@@ -390,14 +397,19 @@ Survey in 2016, 2017, and 2018."#;#[test]fn empty_text() {assert_eq!(- snippet_string("", &["test".to_string()], whatlang::Lang::Eng)- .fragment- .as_str(),+ snippet_string(+ "",+ &["test".to_string()],+ whatlang::Lang::Eng,+ SnippetConfig::default()+ )+ .fragment+ .as_str(),"");assert_eq!(- snippet_string("", &[], whatlang::Lang::Eng)+ snippet_string("", &[], whatlang::Lang::Eng, SnippetConfig::default()).fragment.as_str(),""diff --git a/core/src/warc.rs b/core/src/warc.rsindex ec425833..5e882d34 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::distributed::retry_strategy::ExponentialBackoff;-use crate::{Error, Result, S3Config, WarcSource};+use crate::{config::S3Config, config::WarcSource, Error, Result};use std::collections::BTreeMap;use std::fs::File;use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};