Skip to content

web search done right

Moved some hardcoded constants into configuration files. Also a bunch of other stuff; I should really have split this commit into multiple parts...

Mikkel Denker4a2ca1472d0a
diff --git a/Cargo.lock b/Cargo.lock
index 86970877..6c13afcf 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -95,12 +95,6 @@ dependencies = [
"memchr",
]
-[[package]]
-name = "allocator-api2"
-version = "0.2.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4f263788a35611fba42eb41ff811c5d0360c58b97402570312a350736e2542e"
-
[[package]]
name = "android_system_properties"
version = "0.1.5"
@@ -350,7 +344,7 @@ checksum = "113713495a32dd0ab52baf5c10044725aa3aec00b31beda84218e469029b72a3"
dependencies = [
"async-trait",
"axum-core",
- "bitflags 1.3.2",
+ "bitflags",
"bytes",
"futures-util",
"http",
@@ -472,7 +466,7 @@ version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"cexpr",
"clang-sys",
"lazy_static",
@@ -513,12 +507,6 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-[[package]]
-name = "bitflags"
-version = "2.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded"
-
[[package]]
name = "bitpacking"
version = "0.8.4"
@@ -752,7 +740,7 @@ version = "2.34.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"textwrap 0.11.0",
"unicode-width",
]
@@ -764,7 +752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
dependencies = [
"atty",
- "bitflags 1.3.2",
+ "bitflags",
"clap_derive 3.2.18",
"clap_lex 0.2.4",
"indexmap",
@@ -793,7 +781,7 @@ checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749"
dependencies = [
"anstream",
"anstyle",
- "bitflags 1.3.2",
+ "bitflags",
"clap_lex 0.4.1",
"strsim",
]
@@ -1415,18 +1403,6 @@ dependencies = [
"rand 0.8.5",
]
-[[package]]
-name = "fallible-iterator"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
-
-[[package]]
-name = "fallible-streaming-iterator"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
-
[[package]]
name = "fastdivide"
version = "0.4.0"
@@ -1801,25 +1777,6 @@ dependencies = [
"ahash 0.8.3",
]
-[[package]]
-name = "hashbrown"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
-dependencies = [
- "ahash 0.8.3",
- "allocator-api2",
-]
-
-[[package]]
-name = "hashlink"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f"
-dependencies = [
- "hashbrown 0.14.0",
-]
-
[[package]]
name = "heck"
version = "0.3.3"
@@ -2302,18 +2259,6 @@ dependencies = [
"zstd-sys",
]
-[[package]]
-name = "libsqlite3-sys"
-version = "0.26.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326"
-dependencies = [
- "bindgen",
- "cc",
- "pkg-config",
- "vcpkg",
-]
-
[[package]]
name = "libz-sys"
version = "1.1.8"
@@ -2846,7 +2791,7 @@ version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"libc",
"once_cell",
"onig_sys",
@@ -2880,7 +2825,7 @@ version = "0.10.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97ea2d98598bf9ada7ea6ee8a30fb74f9156b63bbe495d64ec2b87c269d2dda3"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"cfg-if",
"foreign-types",
"libc",
@@ -3212,7 +3157,7 @@ version = "0.17.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aaeebc51f9e7d2c150d3f3bfeb667f2aa985db5ef1e3d212847bdedb488beeaa"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"crc32fast",
"fdeflate",
"flate2",
@@ -3497,7 +3442,7 @@ version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
]
[[package]]
@@ -3506,7 +3451,7 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
]
[[package]]
@@ -3657,26 +3602,6 @@ dependencies = [
"librocksdb-sys",
]
-[[package]]
-name = "rusqlite"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2"
-dependencies = [
- "bitflags 2.3.2",
- "chrono",
- "csv",
- "fallible-iterator",
- "fallible-streaming-iterator",
- "hashlink",
- "libsqlite3-sys",
- "serde_json",
- "smallvec",
- "time 0.3.20",
- "url",
- "uuid",
-]
-
[[package]]
name = "rust-ini"
version = "0.18.0"
@@ -3752,7 +3677,7 @@ version = "0.37.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"errno",
"io-lifetimes",
"libc",
@@ -3910,7 +3835,7 @@ version = "2.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
@@ -3933,7 +3858,7 @@ version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
dependencies = [
- "bitflags 1.3.2",
+ "bitflags",
"cssparser",
"derive_more",
"fxhash",
@@ -4226,7 +4151,6 @@ dependencies = [
"rkyv",
"robotstxt-with-cache",
"rocksdb",
- "rusqlite",
"rust-s3",
"rust-stemmers",
"safetensors",
@@ -4800,7 +4724,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d1d42a9b3f3ec46ba828e8d376aec14592ea199f70a06a548587ecd1c4ab658"
dependencies = [
"async-compression",
- "bitflags 1.3.2",
+ "bitflags",
"bytes",
"futures-core",
"futures-util",
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 920e73ab..a092d90b 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -90,7 +90,6 @@ reqwest-eventsource = { workspace = true }
publicsuffix = { workspace = true }
robotstxt-with-cache = { workspace = true }
rust-s3 = { workspace = true }
-rusqlite = { workspace = true }
async-channel = { workspace = true }
lru = { workspace = true }
@@ -131,4 +130,4 @@ harness = false
[[bench]]
name = "similar-sites"
-harness = false
\ No newline at end of file
+harness = false
diff --git a/core/benches/similar-sites.rs b/core/benches/similar-sites.rs
index d9ecd090..9f1bf9fd 100644
--- a/core/benches/similar-sites.rs
+++ b/core/benches/similar-sites.rs
@@ -13,7 +13,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {
let webgraph = Arc::new(WebgraphBuilder::new(WEBGRAPH_PATH).open());
let inbound = InboundSimilarity::open(INBOUND_SIMILARITY_PATH).unwrap();
- let finder = SimilarSitesFinder::new(webgraph, inbound);
+ let finder = SimilarSitesFinder::new(
+ webgraph,
+ inbound,
+ stract::config::defaults::WebgraphServer::max_similar_sites(),
+ );
for _ in 0..10 {
c.bench_function("similar_sites", |b| {
diff --git a/core/examples/print_similar_sites.rs b/core/examples/print_similar_sites.rs
index 9547bec9..dc05faeb 100644
--- a/core/examples/print_similar_sites.rs
+++ b/core/examples/print_similar_sites.rs
@@ -42,7 +42,11 @@ pub fn main() {
let graph = WebgraphBuilder::new("data/webgraph").open();
let store = CentralityStore::open("data/centrality");
- let similarity_finder = SimilarSitesFinder::new(graph.into(), store.inbound_similarity);
+ let similarity_finder = SimilarSitesFinder::new(
+ graph.into(),
+ store.inbound_similarity,
+ stract::config::defaults::WebgraphServer::max_similar_sites(),
+ );
print_top_nodes(
&[
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index 5141d6fd..60ee2612 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -41,12 +41,14 @@ use tch::Tensor;
use crate::{
api::search::ApiSearchQuery,
+ config::AcceleratorDevice,
+ config::AcceleratorDtype,
+ config::AliceAcceleratorConfig,
llm_utils::ClonableTensor,
search_prettifier::DisplayedWebpage,
searcher::{SearchResult, WebsitesResult},
summarizer::{self, ExtractiveSummarizer},
webpage::Url,
- AcceleratorDevice, AcceleratorDtype, AliceAcceleratorConfig,
};
use self::{
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 08d6edb9..445c1955 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -26,6 +26,7 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};
use crate::{
autosuggest::Autosuggest,
bangs::Bangs,
+ config::FrontendConfig,
distributed::{
cluster::Cluster,
member::{Member, Service},
@@ -37,7 +38,6 @@ use crate::{
ranking::models::{cross_encoder::CrossEncoderModel, lambdamart::LambdaMART},
searcher::frontend::FrontendSearcher,
summarizer::Summarizer,
- FrontendConfig,
};
use anyhow::Result;
use std::sync::Arc;
@@ -150,8 +150,15 @@ pub async fn router(config: &FrontendConfig, counters: Counters) -> Result<Route
.await?,
);
let remote_webgraph = RemoteWebgraph::new(cluster.clone());
- let searcher =
- FrontendSearcher::new(cluster.clone(), crossencoder, lambda_model, qa_model, bangs);
+ let searcher = FrontendSearcher::new(
+ cluster.clone(),
+ crossencoder,
+ lambda_model,
+ qa_model,
+ bangs,
+ config.collector.clone(),
+ config.thresholds.clone(),
+ );
let state = Arc::new(State {
config: config.clone(),
diff --git a/core/src/collector.rs b/core/src/collector.rs
index 1f91246b..36ffe6f3 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -24,7 +24,9 @@ use tantivy::{
};
use crate::{
- combine_u64s, fastfield_reader,
+ combine_u64s,
+ config::CollectorConfig,
+ fastfield_reader,
inverted_index::{DocAddress, WebsitePointer},
prehashed::Prehashed,
ranking::initial::Score,
@@ -32,11 +34,6 @@ use crate::{
simhash,
};
-// lower scale -> higher penalty
-const SITE_SCALE: f64 = 14.0;
-const TITLE_SCALE: f64 = 6.0;
-const URL_SCALE: f64 = 0.1;
-
#[derive(Clone)]
pub struct MaxDocsConsidered {
pub total_docs: usize,
@@ -52,7 +49,7 @@ pub struct Hashes {
}
pub trait Doc: Clone {
- fn score(&self) -> &f64;
+ fn score(&self) -> f64;
fn id(&self) -> &DocId;
fn hashes(&self) -> Hashes;
}
@@ -63,6 +60,7 @@ pub struct TopDocs {
max_docs: Option<MaxDocsConsidered>,
fastfield_reader: fastfield_reader::FastFieldReader,
de_rank_similar: bool,
+ collector_config: CollectorConfig,
}
impl TopDocs {
@@ -73,6 +71,7 @@ impl TopDocs {
max_docs: None,
de_rank_similar: false,
fastfield_reader,
+ collector_config: CollectorConfig::default(),
}
}
@@ -83,13 +82,16 @@ impl TopDocs {
pub fn and_max_docs(mut self, max_docs: MaxDocsConsidered) -> Self {
self.max_docs = Some(max_docs);
-
self
}
pub fn and_de_rank_similar(mut self) -> Self {
self.de_rank_similar = true;
+ self
+ }
+ pub fn and_collector_config(mut self, collector_config: CollectorConfig) -> Self {
+ self.collector_config = collector_config;
self
}
@@ -121,7 +123,10 @@ impl TopDocs {
max_docs,
num_docs_taken: 0,
segment_ord: segment_local_id,
- bucket_collector: BucketCollector::new(self.top_n + self.offset),
+ bucket_collector: BucketCollector::new(
+ self.top_n + self.offset,
+ self.collector_config.clone(),
+ ),
})
}
}
@@ -185,7 +190,7 @@ impl TopSegmentCollector {
}
fn harvest(self) -> Vec<SegmentDoc> {
- self.bucket_collector.into_sorted_vec(false)
+ self.bucket_collector.into_sorted_vec(true)
}
}
@@ -217,34 +222,39 @@ impl<T: Doc> Eq for ScoredDoc<T> {}
impl<T: Doc> From<T> for ScoredDoc<T> {
fn from(doc: T) -> Self {
Self {
- adjusted_score: *doc.score(),
+ adjusted_score: doc.score(),
doc,
}
}
}
-#[derive(Default)]
struct BucketCount {
+ config: CollectorConfig,
buckets: HashMap<Prehashed, usize>,
}
impl BucketCount {
+ fn new(config: CollectorConfig) -> Self {
+ Self {
+ config,
+ buckets: HashMap::new(),
+ }
+ }
+
pub fn adjust_score<T: Doc>(&self, doc: &mut ScoredDoc<T>) {
let hashes = doc.doc.hashes();
- let mut adjuster = 1.0;
+ let taken_sites = *self.buckets.get(&hashes.site).unwrap_or(&0);
+ let taken_urls = *self.buckets.get(&hashes.url).unwrap_or(&0);
+ let taken_titles = *self.buckets.get(&hashes.title).unwrap_or(&0);
- let taken_sites = self.buckets.get(&hashes.site).unwrap_or(&0);
+ let adjuster = 1.0
+ / (1.0
+ + taken_sites as f64 * self.config.site_penalty
+ + taken_urls as f64 * self.config.url_penalty
+ + taken_titles as f64 * self.config.title_penalty);
- adjuster *= SITE_SCALE / (SITE_SCALE + (*taken_sites as f64));
-
- let taken_urls = self.buckets.get(&hashes.url).unwrap_or(&0);
- adjuster *= URL_SCALE / (URL_SCALE + (*taken_urls as f64));
-
- let taken_titles = self.buckets.get(&hashes.title).unwrap_or(&0);
- adjuster *= TITLE_SCALE / (TITLE_SCALE + (*taken_titles as f64));
-
- doc.adjusted_score = *doc.doc.score() * adjuster;
+ doc.adjusted_score = doc.doc.score() * adjuster;
}
fn update_counts<T: Doc>(&mut self, doc: &ScoredDoc<T>) {
@@ -263,27 +273,28 @@ pub struct BucketCollector<T: Doc> {
}
impl<T: Doc> BucketCollector<T> {
- pub fn new(top_n: usize) -> Self {
+ pub fn new(top_n: usize, config: CollectorConfig) -> Self {
assert!(top_n > 0);
Self {
top_n,
- count: BucketCount::default(),
+ count: BucketCount::new(config),
documents: MinMaxHeap::with_capacity(top_n + 1),
}
}
pub fn insert(&mut self, doc: T) {
- let scored_doc = doc.into();
+ let mut scored_doc: ScoredDoc<T> = doc.into();
+ self.count.adjust_score(&mut scored_doc);
- self.documents.push(scored_doc);
+ if self.documents.len() > self.top_n {
+ let mut min_scored_doc = self.documents.peek_min_mut().unwrap();
- self.prune_buckets()
- }
-
- fn prune_buckets(&mut self) {
- while self.documents.len() > self.top_n + 1 {
- self.documents.pop_min().unwrap();
+ if scored_doc.adjusted_score > min_scored_doc.adjusted_score {
+ *min_scored_doc = scored_doc;
+ }
+ } else {
+ self.documents.push(scored_doc);
}
}
@@ -341,8 +352,8 @@ pub struct SegmentDoc {
}
impl Doc for SegmentDoc {
- fn score(&self) -> &f64 {
- &self.score.total
+ fn score(&self) -> f64 {
+ self.score.total
}
fn id(&self) -> &DocId {
@@ -404,7 +415,10 @@ where
&self,
segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
) -> tantivy::Result<Self::Fruit> {
- let mut collector = BucketCollector::new(self.top_docs.top_n + self.top_docs.offset);
+ let mut collector = BucketCollector::new(
+ self.top_docs.top_n + self.top_docs.offset,
+ self.top_docs.collector_config.clone(),
+ );
for docs in segment_fruits {
for doc in docs {
@@ -458,7 +472,7 @@ mod tests {
use super::*;
fn test(top_n: usize, docs: &[(Hashes, DocId, f64)], expected: &[(f64, DocId)]) {
- let mut collector = BucketCollector::new(top_n);
+ let mut collector = BucketCollector::new(top_n, CollectorConfig::default());
for doc in docs {
collector.insert(SegmentDoc {
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
new file mode 100644
index 00000000..a8d8ba9b
--- /dev/null
+++ b/core/src/config/defaults.rs
@@ -0,0 +1,95 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub struct Collector;
+
+impl Collector {
+ pub fn site_penalty() -> f64 {
+ 0.3
+ }
+
+ pub fn title_penalty() -> f64 {
+ 2.0
+ }
+
+ pub fn url_penalty() -> f64 {
+ 20.0
+ }
+}
+
+pub struct Frontend;
+
+impl Frontend {
+ pub fn stackoverflow() -> f64 {
+ 0.5
+ }
+
+ pub fn discussions_widget() -> f64 {
+ 0.1
+ }
+}
+
+pub struct Snippet;
+
+impl Snippet {
+ pub fn desired_num_chars() -> usize {
+ 275
+ }
+
+ pub fn delta_num_chars() -> usize {
+ 50
+ }
+
+ pub fn min_passage_width() -> usize {
+ 20
+ }
+}
+
+pub struct Crawler;
+
+impl Crawler {
+ pub fn politeness_factor() -> f32 {
+ 1.0
+ }
+
+ pub fn min_crawl_delay_ms() -> u64 {
+ 200
+ }
+
+ pub fn max_crawl_delay_ms() -> u64 {
+ 30_000
+ }
+
+ pub fn max_politeness_factor() -> f32 {
+ 1024.0
+ }
+
+ pub fn min_politeness_factor() -> f32 {
+ 1.0
+ }
+
+ pub fn max_url_slowdown_retry() -> u8 {
+ 3
+ }
+}
+
+pub struct WebgraphServer;
+
+impl WebgraphServer {
+ pub fn max_similar_sites() -> usize {
+ 1_000
+ }
+}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
new file mode 100644
index 00000000..bc204257
--- /dev/null
+++ b/core/src/config/mod.rs
@@ -0,0 +1,343 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+pub mod defaults;
+
+use super::Result;
+use crate::searcher::ShardId;
+use serde::{Deserialize, Serialize};
+use std::fs::File;
+use std::io::{self, BufRead};
+use std::net::SocketAddr;
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct IndexingLocalConfig {
+ pub(crate) limit_warc_files: Option<usize>,
+ pub(crate) skip_warc_files: Option<usize>,
+ pub(crate) warc_source: WarcSource,
+ pub(crate) batch_size: Option<usize>,
+ pub(crate) page_webgraph_path: Option<String>,
+ pub(crate) output_path: Option<String>,
+ pub(crate) host_centrality_threshold: Option<f64>,
+ pub(crate) topics_path: Option<String>,
+ pub(crate) host_centrality_store_path: String,
+ pub(crate) page_centrality_store_path: Option<String>,
+ pub(crate) minimum_clean_words: Option<usize>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "lowercase")]
+pub enum WebgraphLevel {
+ Page,
+ Host,
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct WebgraphConstructConfig {
+ pub(crate) level: WebgraphLevel,
+ pub(crate) redirect_db_path: Option<String>,
+ pub(crate) limit_warc_files: Option<usize>,
+ pub(crate) warc_source: WarcSource,
+ pub(crate) graph_base_path: Option<String>,
+ pub(crate) batch_size: Option<usize>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(tag = "type")]
+pub enum WarcSource {
+ HTTP(HttpConfig),
+ Local(LocalConfig),
+ S3(S3Config),
+}
+
+impl WarcSource {
+ pub fn paths(&self) -> Result<Vec<String>> {
+ let mut warc_paths = Vec::new();
+ match &self {
+ WarcSource::HTTP(config) => {
+ let file = File::open(&config.warc_paths_file)?;
+ for line in io::BufReader::new(file).lines() {
+ warc_paths.push(line?);
+ }
+ }
+ WarcSource::Local(config) => {
+ warc_paths = config.names.clone();
+ }
+ WarcSource::S3(config) => {
+ let bucket = s3::Bucket::new(
+ &config.bucket,
+ s3::Region::Custom {
+ region: "".to_string(),
+ endpoint: config.endpoint.clone(),
+ },
+ s3::creds::Credentials {
+ access_key: Some(config.access_key.clone()),
+ secret_key: Some(config.secret_key.clone()),
+ security_token: None,
+ session_token: None,
+ expiration: None,
+ },
+ )?
+ .with_path_style();
+
+ let mut folder = config.folder.clone();
+
+ if !folder.ends_with('/') {
+ folder.push('/');
+ }
+
+ let pages = bucket.list_blocking(folder, Some("/".to_string()))?;
+
+ let objects = pages
+ .into_iter()
+ .flat_map(|p| p.contents.into_iter())
+ .collect::<Vec<_>>();
+
+ for p in objects.into_iter().filter_map(|o| {
+ if o.key.ends_with("warc.gz") {
+ Some(o.key)
+ } else {
+ None
+ }
+ }) {
+ warc_paths.push(p);
+ }
+ }
+ }
+
+ Ok(warc_paths)
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct LocalConfig {
+ pub folder: String,
+ pub names: Vec<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct HttpConfig {
+ pub(crate) base_url: String,
+ pub(crate) warc_paths_file: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct S3Config {
+ pub bucket: String,
+ pub folder: String,
+ pub access_key: String,
+ pub secret_key: String,
+ pub endpoint: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CollectorConfig {
+ #[serde(default = "defaults::Collector::site_penalty")]
+ pub site_penalty: f64,
+
+ #[serde(default = "defaults::Collector::title_penalty")]
+ pub title_penalty: f64,
+
+ #[serde(default = "defaults::Collector::url_penalty")]
+ pub url_penalty: f64,
+}
+
+impl Default for CollectorConfig {
+ fn default() -> Self {
+ Self {
+ site_penalty: defaults::Collector::site_penalty(),
+ title_penalty: defaults::Collector::title_penalty(),
+ url_penalty: defaults::Collector::url_penalty(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct FrontendThresholds {
+ #[serde(default = "defaults::Frontend::stackoverflow")]
+ pub stackoverflow: f64,
+
+ #[serde(default = "defaults::Frontend::discussions_widget")]
+ pub discussions_widget: f64,
+}
+
+impl Default for FrontendThresholds {
+ fn default() -> Self {
+ Self {
+ stackoverflow: defaults::Frontend::stackoverflow(),
+ discussions_widget: defaults::Frontend::discussions_widget(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct FrontendConfig {
+ pub queries_csv_path: String,
+ pub host: SocketAddr,
+ pub prometheus_host: SocketAddr,
+ pub crossencoder_model_path: String,
+ pub lambda_model_path: Option<String>,
+ pub qa_model_path: Option<String>,
+ pub with_alice: Option<bool>,
+ pub bangs_path: String,
+ pub summarizer_path: String,
+ pub fact_check_model_path: String,
+ pub query_store_db_host: Option<String>,
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+
+ #[serde(default)]
+ pub collector: CollectorConfig,
+
+ #[serde(default)]
+ pub thresholds: FrontendThresholds,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct SnippetConfig {
+ #[serde(default = "defaults::Snippet::desired_num_chars")]
+ pub desired_num_chars: usize,
+
+ #[serde(default = "defaults::Snippet::delta_num_chars")]
+ pub delta_num_chars: usize,
+
+ #[serde(default = "defaults::Snippet::min_passage_width")]
+ pub min_passage_width: usize,
+}
+
+impl Default for SnippetConfig {
+ fn default() -> Self {
+ Self {
+ desired_num_chars: defaults::Snippet::desired_num_chars(),
+ delta_num_chars: defaults::Snippet::delta_num_chars(),
+ min_passage_width: defaults::Snippet::min_passage_width(),
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct SearchServerConfig {
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+ pub shard_id: ShardId,
+ pub index_path: String,
+ pub entity_index_path: Option<String>,
+ pub host_centrality_store_path: Option<String>,
+ pub linear_model_path: Option<String>,
+ pub lambda_model_path: Option<String>,
+ pub host: SocketAddr,
+
+ #[serde(default)]
+ pub collector: CollectorConfig,
+
+ #[serde(default)]
+ pub snippet: SnippetConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CrawlCoordinatorConfig {
+ pub crawldb_folder: String,
+ pub host: SocketAddr,
+ pub num_urls_to_crawl: u64,
+ pub seed_urls: Vec<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct UserAgent {
+ pub full: String,
+ pub token: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct CrawlerConfig {
+ pub num_workers: usize,
+ pub user_agent: UserAgent,
+
+ #[serde(default = "defaults::Crawler::politeness_factor")]
+ pub politeness_factor: f32,
+
+ #[serde(default = "defaults::Crawler::min_crawl_delay_ms")]
+ pub min_crawl_delay_ms: u64,
+
+ #[serde(default = "defaults::Crawler::max_crawl_delay_ms")]
+ pub max_crawl_delay_ms: u64,
+
+ #[serde(default = "defaults::Crawler::max_politeness_factor")]
+ pub max_politeness_factor: f32,
+
+ #[serde(default = "defaults::Crawler::min_politeness_factor")]
+ pub min_politeness_factor: f32,
+
+ #[serde(default = "defaults::Crawler::max_url_slowdown_retry")]
+ pub max_url_slowdown_retry: u8,
+
+ pub timeout_seconds: u64,
+ pub s3: S3Config,
+ pub coordinator_host: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(tag = "type", content = "args", rename_all = "snake_case")]
+pub enum AcceleratorDevice {
+ Cpu,
+ Cuda(usize),
+ Mps,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum AcceleratorDtype {
+ Float,
+ Bf16,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AliceAcceleratorConfig {
+ pub layer_fraction: f64,
+ /// percentage of layers on accelerator to quantize
+ pub quantize_fraction: f64,
+ pub device: AcceleratorDevice,
+ pub dtype: AcceleratorDtype,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AliceLocalConfig {
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+ pub host: SocketAddr,
+ pub summarizer_path: String,
+
+ pub alice_path: String,
+ pub accelerator: Option<AliceAcceleratorConfig>,
+ /// base64 encoded
+ pub encryption_key: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct WebgraphServerConfig {
+ pub host: SocketAddr,
+ pub graph_path: String,
+ pub inbound_similarity_path: String,
+ pub cluster_id: String,
+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
+ pub gossip_addr: SocketAddr,
+
+ #[serde(default = "defaults::WebgraphServer::max_similar_sites")]
+ pub max_similar_sites: usize,
+}
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index 77e4f786..db6576d4 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -22,7 +22,7 @@ use std::{
use tokio::sync::Mutex;
-use crate::{webpage::Url, CrawlerConfig};
+use crate::{config::CrawlerConfig, webpage::Url};
use self::{
warc_writer::{WarcWriter, WarcWriterMessage},
@@ -37,8 +37,6 @@ mod worker;
pub use coordinator::CrawlCoordinator;
-const DEFAULT_POLITENESS_FACTOR: f32 = 1.0;
-
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("reqwest error: {0}")]
@@ -59,9 +57,6 @@ pub enum Error {
#[error("invalid politeness factor")]
InvalidPolitenessFactor,
- #[error("sqlite error")]
- Sqlite(#[from] rusqlite::Error),
-
#[error("addr parse error: {0}")]
AddrParse(#[from] std::net::AddrParseError),
@@ -170,7 +165,7 @@ pub struct Crawler {
impl Crawler {
pub async fn new(config: CrawlerConfig) -> Result<Self> {
let pending_commands = Arc::new(Mutex::new(VecDeque::new()));
- let writer = Arc::new(WarcWriter::new(config.s3));
+ let writer = Arc::new(WarcWriter::new(config.s3.clone()));
let timeout = Duration::from_secs(config.timeout_seconds);
let mut handles = Vec::new();
let coordinator_host = config.coordinator_host.parse()?;
@@ -179,13 +174,9 @@ impl Crawler {
let worker = Worker::new(
Arc::clone(&pending_commands),
Arc::clone(&writer),
- config.user_agent.clone(),
- config
- .politeness_factor
- .unwrap_or(DEFAULT_POLITENESS_FACTOR),
+ config.clone(),
timeout,
coordinator_host,
- config.num_workers,
)?;
handles.push(tokio::spawn(async move {
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index 30f6ca90..0c1fa821 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -16,7 +16,10 @@
use std::time::Duration;
-use crate::{warc, S3Config};
+use crate::{
+ config::{self, S3Config},
+ warc,
+};
use super::{CrawlDatum, Result};
@@ -32,7 +35,7 @@ pub enum WarcWriterMessage {
Finish,
}
-async fn commit(writer: warc::WarcWriter, s3: S3Config) {
+async fn commit(writer: warc::WarcWriter, s3: config::S3Config) {
let filename = format!(
"{}_{}.warc.gz",
chrono::Utc::now().to_rfc3339(),
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 8dbc8b1a..c8b19609 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -26,10 +26,10 @@ use std::{
use tokio::sync::Mutex;
use crate::{
+ config::CrawlerConfig,
crawler::{JobResponse, Request, Response},
distributed::{retry_strategy::ExponentialBackoff, sonic},
webpage::{Html, Url},
- UserAgent,
};
use super::{
@@ -37,19 +37,12 @@ use super::{
WarcWriter, WorkerJob,
};
-const MIN_CRAWL_DELAY_MS: u64 = 200;
-const MAX_CRAWL_DELAY_MS: u64 = 30_000;
-const MAX_POLITENESS_FACTOR: f32 = 1024.0;
-const MIN_POLITENESS_FACTOR: f32 = 1.0;
-const MAX_URL_SLOWDOWN_RETRY: u8 = 3;
-
pub struct Worker {
current_job: Option<WorkerJob>,
pending_commands: Arc<Mutex<VecDeque<Command>>>,
writer: Arc<WarcWriter>,
client: reqwest::Client,
- user_agent: UserAgent,
- default_politeness_factor: f32,
+ config: CrawlerConfig,
politeness_factor: f32,
coordinator_host: SocketAddr,
num_jobs_per_fetch: usize,
@@ -60,13 +53,11 @@ impl Worker {
pub fn new(
pending_commands: Arc<Mutex<VecDeque<Command>>>,
writer: Arc<WarcWriter>,
- user_agent: UserAgent,
- politeness_factor: f32,
+ config: CrawlerConfig,
timeout: Duration,
coordinator_host: SocketAddr,
- num_jobs_per_fetch: usize,
) -> Result<Self> {
- if politeness_factor < MIN_POLITENESS_FACTOR {
+ if config.politeness_factor < config.min_politeness_factor {
return Err(Error::InvalidPolitenessFactor);
}
@@ -74,7 +65,7 @@ impl Worker {
.timeout(timeout)
.connect_timeout(timeout)
.http2_keep_alive_interval(None)
- .user_agent(&user_agent.full)
+ .user_agent(&config.user_agent.full)
.build()?;
let robotstxt = RobotsTxtManager::new(client.clone());
@@ -84,11 +75,10 @@ impl Worker {
client,
current_job: None,
pending_commands,
- user_agent,
- politeness_factor,
- default_politeness_factor: politeness_factor,
+ num_jobs_per_fetch: config.num_workers,
+ politeness_factor: config.politeness_factor,
+ config,
coordinator_host,
- num_jobs_per_fetch,
robotstxt,
})
}
@@ -159,7 +149,7 @@ impl Worker {
}
async fn process_job(&mut self) {
- self.politeness_factor = self.default_politeness_factor;
+ self.politeness_factor = self.config.politeness_factor;
let mut job = self.current_job.take().unwrap();
tracing::info!("Processing job: {:?}", job.domain);
@@ -170,7 +160,7 @@ impl Worker {
let mut crawled_sitemaps: HashSet<Site> = HashSet::new();
while let Some(retryable_url) = job.urls.pop_front() {
- if retryable_url.retries > MAX_URL_SLOWDOWN_RETRY {
+ if retryable_url.retries > self.config.max_url_slowdown_retry {
continue;
}
@@ -180,7 +170,7 @@ impl Worker {
if !self
.robotstxt
- .is_allowed(&retryable_url.url, &self.user_agent.token)
+ .is_allowed(&retryable_url.url, &self.config.user_agent.token)
.await
{
continue;
@@ -200,12 +190,12 @@ impl Worker {
let mut delay = res.fetch_time.mul_f32(self.politeness_factor);
- if delay < Duration::from_millis(MIN_CRAWL_DELAY_MS) {
- delay = Duration::from_millis(MIN_CRAWL_DELAY_MS);
+ if delay < Duration::from_millis(self.config.min_crawl_delay_ms) {
+ delay = Duration::from_millis(self.config.min_crawl_delay_ms);
}
- if delay > Duration::from_millis(MAX_CRAWL_DELAY_MS) {
- delay = Duration::from_millis(MAX_CRAWL_DELAY_MS);
+ if delay > Duration::from_millis(self.config.max_crawl_delay_ms) {
+ delay = Duration::from_millis(self.config.max_crawl_delay_ms);
}
tokio::time::sleep(delay).await;
@@ -308,8 +298,8 @@ impl Worker {
if datum.status_code == 429 {
self.politeness_factor *= 2.0;
- if self.politeness_factor > MAX_POLITENESS_FACTOR {
- self.politeness_factor = MAX_POLITENESS_FACTOR;
+ if self.politeness_factor > self.config.max_politeness_factor {
+ self.politeness_factor = self.config.max_politeness_factor;
}
tracing::warn!(
@@ -480,7 +470,10 @@ impl Worker {
.full()
.into();
- tokio::time::sleep(Duration::from_millis(MIN_CRAWL_DELAY_MS)).await;
+ tokio::time::sleep(Duration::from_millis(
+ self.config.min_crawl_delay_ms,
+ ))
+ .await;
urls.append(
&mut self.urls_from_sitemap(url, depth + 1, max_depth).await,
diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rs
index 9cec9839..28cdd524 100644
--- a/core/src/entrypoint/alice.rs
+++ b/core/src/entrypoint/alice.rs
@@ -33,12 +33,12 @@ use tracing::info;
use crate::{
alice::{Alice, EncodedEncryptedState, EncryptedState},
+ config::AliceLocalConfig,
distributed::{
cluster::Cluster,
member::{Member, Service},
},
ttl_cache::TTLCache,
- AliceLocalConfig,
};
pub struct State {
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index afa53ed1..00928dee 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -74,13 +74,13 @@ fn create_webgraph() -> Result<()> {
let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");
let job = webgraph::Job {
- config: webgraph::JobConfig::Local(crate::LocalConfig {
+ config: webgraph::JobConfig::Local(crate::config::LocalConfig {
folder: ".".to_string(),
names: vec![warc_path.to_str().unwrap().to_string()],
}),
warc_paths: vec![warc_path.to_str().unwrap().to_string()],
graph_base_path: out_path_tmp.to_str().unwrap().to_string(),
- level: crate::WebgraphLevel::Host,
+ level: crate::config::WebgraphLevel::Host,
};
let worker = webgraph::WebgraphWorker { redirect: None };
@@ -115,7 +115,7 @@ fn create_inverted_index() -> Result<()> {
let warc_path = Path::new(DATA_PATH).join("sample.warc.gz");
let job = indexer::Job {
- source_config: indexer::JobConfig::Local(crate::LocalConfig {
+ source_config: indexer::JobConfig::Local(crate::config::LocalConfig {
folder: ".".to_string(),
names: vec![warc_path.to_str().unwrap().to_string()],
}),
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index 2b4a290d..83336833 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -17,12 +17,13 @@
use std::{net::SocketAddr, sync::Arc};
use crate::{
+ config,
crawler::{CrawlCoordinator, Crawler},
distributed::sonic,
- CrawlCoordinatorConfig, CrawlerConfig, Result,
+ Result,
};
-pub async fn worker(config: CrawlerConfig) -> Result<()> {
+pub async fn worker(config: config::CrawlerConfig) -> Result<()> {
let crawler = Crawler::new(config).await?;
crawler.wait().await;
@@ -30,7 +31,7 @@ pub async fn worker(config: CrawlerConfig) -> Result<()> {
Ok(())
}
-pub async fn coordinator(config: CrawlCoordinatorConfig) -> Result<()> {
+pub async fn coordinator(config: config::CrawlCoordinatorConfig) -> Result<()> {
let coordinator = Arc::new(CrawlCoordinator::new(
config.crawldb_folder,
config.num_urls_to_crawl,
diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rs
index c21823a1..90aac536 100644
--- a/core/src/entrypoint/frontend.rs
+++ b/core/src/entrypoint/frontend.rs
@@ -18,11 +18,11 @@ use anyhow::Result;
use crate::{
api::{metrics_router, router, Counters},
+ config,
metrics::Label,
- FrontendConfig,
};
-pub async fn run(config: FrontendConfig) -> Result<()> {
+pub async fn run(config: config::FrontendConfig) -> Result<()> {
let search_counter_success = crate::metrics::Counter::default();
let search_counter_fail = crate::metrics::Counter::default();
let explore_counter = crate::metrics::Counter::default();
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index ca065eec..5f083365 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -22,6 +22,7 @@ use serde::{Deserialize, Serialize};
use tokio::pin;
use tracing::{debug, info, trace, warn};
+use crate::config;
use crate::entrypoint::download_all_warc_files;
use crate::executor::Executor;
use crate::index::{FrozenIndex, Index};
@@ -31,36 +32,33 @@ use crate::ranking::SignalAggregator;
use crate::warc::WarcFile;
use crate::webgraph::{Node, Webgraph, WebgraphBuilder};
use crate::webpage::{Html, Link, Webpage};
-use crate::{
- human_website_annotations, HttpConfig, IndexingLocalConfig, LocalConfig, Result, S3Config,
- WarcSource,
-};
+use crate::{human_website_annotations, Result};
pub struct Indexer {}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub enum JobConfig {
- Http(HttpConfig),
- Local(LocalConfig),
- S3(S3Config),
+ Http(config::HttpConfig),
+ Local(config::LocalConfig),
+ S3(config::S3Config),
}
-impl From<WarcSource> for JobConfig {
- fn from(value: WarcSource) -> Self {
+impl From<config::WarcSource> for JobConfig {
+ fn from(value: config::WarcSource) -> Self {
match value {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- WarcSource::S3(config) => JobConfig::S3(config),
+ config::WarcSource::HTTP(config) => JobConfig::Http(config),
+ config::WarcSource::Local(config) => JobConfig::Local(config),
+ config::WarcSource::S3(config) => JobConfig::S3(config),
}
}
}
-impl From<JobConfig> for WarcSource {
+impl From<JobConfig> for config::WarcSource {
fn from(value: JobConfig) -> Self {
match value {
- JobConfig::Http(config) => WarcSource::HTTP(config),
- JobConfig::Local(config) => WarcSource::Local(config),
- JobConfig::S3(config) => WarcSource::S3(config),
+ JobConfig::Http(config) => config::WarcSource::HTTP(config),
+ JobConfig::Local(config) => config::WarcSource::Local(config),
+ JobConfig::S3(config) => config::WarcSource::S3(config),
}
}
}
@@ -108,7 +106,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();
- let source: WarcSource = job.source_config.clone().into();
+ let source: config::WarcSource = job.source_config.clone().into();
let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);
pin!(warc_files);
@@ -320,7 +318,7 @@ impl Reduce<Index> for Index {
}
impl Indexer {
- pub fn run(config: &IndexingLocalConfig) -> Result<()> {
+ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {
let warc_paths = config.warc_source.paths()?;
let job_config: JobConfig = config.warc_source.clone().into();
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index 0056c487..04440029 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -37,11 +37,11 @@ pub use indexer::Indexer;
use tracing::{debug, log::error};
pub use webgraph::Webgraph;
-use crate::{warc::WarcFile, WarcSource};
+use crate::{config, warc::WarcFile};
fn download_all_warc_files<'a>(
warc_paths: &'a [String],
- source: &'a WarcSource,
+ source: &'a config::WarcSource,
base_path: &'a str,
) -> impl Iterator<Item = String> + 'a {
let download_path = Path::new(base_path).join("warc_files");
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 83255c63..d5139d90 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -19,6 +19,7 @@ use std::{collections::HashMap, net::SocketAddr};
use tracing::info;
use crate::{
+ config,
distributed::{
cluster::Cluster,
member::{Member, Service},
@@ -33,10 +34,10 @@ use crate::{
},
searcher::{self, LocalSearcher},
webpage::Url,
- Result, SearchServerConfig,
+ Result,
};
-pub async fn run(config: SearchServerConfig) -> Result<()> {
+pub async fn run(config: config::SearchServerConfig) -> Result<()> {
let addr: SocketAddr = config.host;
let server = sonic::Server::bind(addr).await.unwrap();
@@ -66,6 +67,9 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
local_searcher.set_lambda_model(LambdaMART::open(model_path)?);
}
+ local_searcher.set_collector_config(config.collector);
+ local_searcher.set_snippet_config(config.snippet);
+
// dropping the handle leaves the cluster
let _cluster_handle = Cluster::join(
Member {
@@ -100,7 +104,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
}
}
searcher::Request::Search(query) => {
- match local_searcher.search_initial(query, false) {
+ match local_searcher.search_initial(query, true) {
Ok(result) => {
req.respond(sonic::Response::Content(result)).await.ok();
}
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 3a9b8646..a4928652 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -14,13 +14,16 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ config::WarcSource,
+ config::WebgraphLevel,
+ config::{self, WebgraphConstructConfig},
crawler::crawl_db::RedirectDb,
entrypoint::download_all_warc_files,
mapreduce::{Map, Reduce, Worker},
warc::WarcFile,
webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
- HttpConfig, LocalConfig, Result, S3Config, WarcSource, WebgraphConstructConfig, WebgraphLevel,
+ Result,
};
use itertools::Itertools;
use rayon::prelude::*;
@@ -36,26 +39,26 @@ struct GraphPointer {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub enum JobConfig {
- Http(HttpConfig),
- Local(LocalConfig),
- S3(S3Config),
+ Http(config::HttpConfig),
+ Local(config::LocalConfig),
+ S3(config::S3Config),
}
-impl From<WarcSource> for JobConfig {
- fn from(value: WarcSource) -> Self {
+impl From<config::WarcSource> for JobConfig {
+ fn from(value: config::WarcSource) -> Self {
match value {
- WarcSource::HTTP(config) => JobConfig::Http(config),
- WarcSource::Local(config) => JobConfig::Local(config),
- WarcSource::S3(config) => JobConfig::S3(config),
+ config::WarcSource::HTTP(config) => JobConfig::Http(config),
+ config::WarcSource::Local(config) => JobConfig::Local(config),
+ config::WarcSource::S3(config) => JobConfig::S3(config),
}
}
}
-impl From<JobConfig> for WarcSource {
+impl From<JobConfig> for config::WarcSource {
fn from(value: JobConfig) -> Self {
match value {
- JobConfig::Http(config) => WarcSource::HTTP(config),
- JobConfig::Local(config) => WarcSource::Local(config),
+ JobConfig::Http(config) => config::WarcSource::HTTP(config),
+ JobConfig::Local(config) => config::WarcSource::Local(config),
JobConfig::S3(config) => WarcSource::S3(config),
}
}
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
index df90ca69..aecc196a 100644
--- a/core/src/entrypoint/webgraph_server.rs
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -22,6 +22,7 @@ use serde::Deserialize;
use serde::Serialize;
use tracing::info;
+use crate::config;
use crate::distributed::cluster::Cluster;
use crate::distributed::member::Member;
use crate::distributed::member::Service;
@@ -33,7 +34,6 @@ use crate::webgraph::Node;
use crate::webgraph::WebgraphBuilder;
use crate::webpage::Url;
use crate::Result;
-use crate::WebgraphServerConfig;
#[derive(Serialize, Deserialize)]
pub enum Request {
@@ -50,7 +50,7 @@ pub struct ScoredSite {
const MAX_SITES: usize = 20;
-pub async fn run(config: WebgraphServerConfig) -> Result<()> {
+pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
let addr: SocketAddr = config.host;
// dropping the handle leaves the cluster
@@ -70,7 +70,11 @@ pub async fn run(config: WebgraphServerConfig) -> Result<()> {
let graph = Arc::new(WebgraphBuilder::new(config.graph_path).open());
let inbound_similarity = InboundSimilarity::open(config.inbound_similarity_path)?;
- let similar_sites_finder = SimilarSitesFinder::new(Arc::clone(&graph), inbound_similarity);
+ let similar_sites_finder = SimilarSitesFinder::new(
+ Arc::clone(&graph),
+ inbound_similarity,
+ config.max_similar_sites,
+ );
let server = sonic::Server::bind(addr).await.unwrap();
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 017ef8e1..5ae6d04c 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -35,6 +35,7 @@ use tantivy::tokenizer::TokenizerManager;
use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
use crate::collector::Hashes;
+use crate::config::SnippetConfig;
use crate::fastfield_reader::FastFieldReader;
use crate::query::Query;
use crate::ranking::initial::Score;
@@ -102,6 +103,7 @@ pub struct InvertedIndex {
writer: IndexWriter,
reader: IndexReader,
schema: Arc<Schema>,
+ snippet_config: SnippetConfig,
}
impl InvertedIndex {
@@ -165,9 +167,14 @@ impl InvertedIndex {
schema: Arc::new(schema),
path: path.as_ref().to_str().unwrap().to_string(),
tantivy_index,
+ snippet_config: SnippetConfig::default(),
})
}
+ pub fn set_snippet_config(&mut self, config: SnippetConfig) {
+ self.snippet_config = config;
+ }
+
pub fn fastfield_reader(&self, tv_searcher: &tantivy::Searcher) -> FastFieldReader {
FastFieldReader::new(tv_searcher)
}
@@ -305,7 +312,8 @@ impl InvertedIndex {
.collect();
for page in &mut webpages {
- page.snippet = snippet::generate(query, &page.body, &page.region);
+ page.snippet =
+ snippet::generate(query, &page.body, &page.region, self.snippet_config.clone());
}
Ok(webpages)
@@ -649,6 +657,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
@@ -678,6 +687,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -724,6 +734,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -768,6 +779,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -812,6 +824,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -886,6 +899,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let mut result =
@@ -940,6 +954,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -983,6 +998,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -1051,6 +1067,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -1078,6 +1095,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -1108,6 +1126,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
@@ -1152,6 +1171,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -1216,6 +1236,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
@@ -1266,6 +1287,7 @@ mod tests {
let ranker = Ranker::new(
SignalAggregator::new(Some(&query)),
ctx.fastfield_reader.clone(),
+ Default::default(),
);
let result =
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 5cb425e4..5ddbc29e 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -21,11 +21,7 @@
#![allow(clippy::cast_precision_loss)]
#![allow(clippy::missing_errors_doc)]
-use searcher::ShardId;
-use serde::{Deserialize, Serialize};
-use std::fs::File;
-use std::io::{self, BufRead};
-use std::net::SocketAddr;
+use std::io;
use std::num::ParseIntError;
use std::path::PathBuf;
use tantivy::TantivyError;
@@ -42,6 +38,7 @@ mod autosuggest;
mod bangs;
mod call_counter;
mod collector;
+pub mod config;
pub mod crawler;
mod directory;
mod distributed;
@@ -84,229 +81,6 @@ pub mod webgraph;
pub mod webpage;
mod widgets;
-#[derive(Debug, Deserialize, Clone)]
-pub struct IndexingLocalConfig {
- limit_warc_files: Option<usize>,
- skip_warc_files: Option<usize>,
- warc_source: WarcSource,
- batch_size: Option<usize>,
- page_webgraph_path: Option<String>,
- output_path: Option<String>,
- host_centrality_threshold: Option<f64>,
- topics_path: Option<String>,
- host_centrality_store_path: String,
- page_centrality_store_path: Option<String>,
- minimum_clean_words: Option<usize>,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "lowercase")]
-pub enum WebgraphLevel {
- Page,
- Host,
-}
-
-#[derive(Debug, Deserialize, Clone)]
-pub struct WebgraphConstructConfig {
- level: WebgraphLevel,
- redirect_db_path: Option<String>,
- limit_warc_files: Option<usize>,
- warc_source: WarcSource,
- graph_base_path: Option<String>,
- batch_size: Option<usize>,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(tag = "type")]
-pub enum WarcSource {
- HTTP(HttpConfig),
- Local(LocalConfig),
- S3(S3Config),
-}
-
-impl WarcSource {
- pub fn paths(&self) -> Result<Vec<String>> {
- let mut warc_paths = Vec::new();
- match &self {
- WarcSource::HTTP(config) => {
- let file = File::open(&config.warc_paths_file)?;
- for line in io::BufReader::new(file).lines() {
- warc_paths.push(line?);
- }
- }
- WarcSource::Local(config) => {
- warc_paths = config.names.clone();
- }
- WarcSource::S3(config) => {
- let bucket = s3::Bucket::new(
- &config.bucket,
- s3::Region::Custom {
- region: "".to_string(),
- endpoint: config.endpoint.clone(),
- },
- s3::creds::Credentials {
- access_key: Some(config.access_key.clone()),
- secret_key: Some(config.secret_key.clone()),
- security_token: None,
- session_token: None,
- expiration: None,
- },
- )?
- .with_path_style();
-
- let mut folder = config.folder.clone();
-
- if !folder.ends_with('/') {
- folder.push('/');
- }
-
- let pages = bucket.list_blocking(folder, Some("/".to_string()))?;
-
- let objects = pages
- .into_iter()
- .flat_map(|p| p.contents.into_iter())
- .collect::<Vec<_>>();
-
- for p in objects.into_iter().filter_map(|o| {
- if o.key.ends_with("warc.gz") {
- Some(o.key)
- } else {
- None
- }
- }) {
- warc_paths.push(p);
- }
- }
- }
-
- Ok(warc_paths)
- }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct LocalConfig {
- pub folder: String,
- pub names: Vec<String>,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct HttpConfig {
- base_url: String,
- warc_paths_file: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct S3Config {
- pub bucket: String,
- pub folder: String,
- pub access_key: String,
- pub secret_key: String,
- pub endpoint: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct FrontendConfig {
- pub queries_csv_path: String,
- pub host: SocketAddr,
- pub prometheus_host: SocketAddr,
- pub crossencoder_model_path: String,
- pub lambda_model_path: Option<String>,
- pub qa_model_path: Option<String>,
- pub with_alice: Option<bool>,
- pub bangs_path: String,
- pub summarizer_path: String,
- pub fact_check_model_path: String,
- pub query_store_db_host: Option<String>,
- pub cluster_id: String,
- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
- pub gossip_addr: SocketAddr,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct SearchServerConfig {
- pub cluster_id: String,
- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
- pub gossip_addr: SocketAddr,
- pub shard_id: ShardId,
- pub index_path: String,
- pub entity_index_path: Option<String>,
- pub host_centrality_store_path: Option<String>,
- pub linear_model_path: Option<String>,
- pub lambda_model_path: Option<String>,
- pub host: SocketAddr,
-}
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct CrawlCoordinatorConfig {
- pub crawldb_folder: String,
- pub host: SocketAddr,
- pub num_urls_to_crawl: u64,
- pub seed_urls: Vec<String>,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct UserAgent {
- pub full: String,
- pub token: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct CrawlerConfig {
- pub num_workers: usize,
- pub user_agent: UserAgent,
- pub politeness_factor: Option<f32>,
- pub timeout_seconds: u64,
- pub s3: S3Config,
- pub coordinator_host: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(tag = "type", content = "args", rename_all = "snake_case")]
-pub enum AcceleratorDevice {
- Cpu,
- Cuda(usize),
- Mps,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(tag = "type", rename_all = "snake_case")]
-pub enum AcceleratorDtype {
- Float,
- Bf16,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct AliceAcceleratorConfig {
- pub layer_fraction: f64,
- /// percentage of layers on accelerator to quantize
- pub quantize_fraction: f64,
- pub device: AcceleratorDevice,
- pub dtype: AcceleratorDtype,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct AliceLocalConfig {
- pub cluster_id: String,
- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
- pub gossip_addr: SocketAddr,
- pub host: SocketAddr,
- pub summarizer_path: String,
-
- pub alice_path: String,
- pub accelerator: Option<AliceAcceleratorConfig>,
- /// base64 encoded
- pub encryption_key: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct WebgraphServerConfig {
- pub host: SocketAddr,
- pub graph_path: String,
- pub inbound_similarity_path: String,
- pub cluster_id: String,
- pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
- pub gossip_addr: SocketAddr,
-}
-
#[derive(Error, Debug)]
pub enum Error {
#[error("Failed to download object from HTTP")]
diff --git a/core/src/main.rs b/core/src/main.rs
index f95a06a2..2cd24313 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -18,16 +18,15 @@ use clap::{Parser, Subcommand};
use serde::de::DeserializeOwned;
use std::fs;
use std::path::Path;
+use stract::config;
use stract::entrypoint::autosuggest_scrape::{self, Gl};
+
#[cfg(feature = "dev")]
use stract::entrypoint::configure;
+
use stract::entrypoint::indexer::IndexPointer;
use stract::entrypoint::{self, frontend, search_server, webgraph_server};
use stract::webgraph::WebgraphBuilder;
-use stract::{
- AliceLocalConfig, CrawlCoordinatorConfig, CrawlerConfig, FrontendConfig, SearchServerConfig,
- WebgraphServerConfig,
-};
use tracing::Level;
use tracing_subscriber::FmtSubscriber;
@@ -231,7 +230,7 @@ fn main() -> Result<()> {
}
}
WebgraphOptions::Server { config_path } => {
- let config: WebgraphServerConfig = load_toml_config(config_path);
+ let config: config::WebgraphServerConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -240,7 +239,7 @@ fn main() -> Result<()> {
}
},
Commands::Frontend { config_path } => {
- let config: FrontendConfig = load_toml_config(config_path);
+ let config: config::FrontendConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -248,7 +247,7 @@ fn main() -> Result<()> {
.block_on(frontend::run(config))?
}
Commands::SearchServer { config_path } => {
- let config: SearchServerConfig = load_toml_config(config_path);
+ let config: config::SearchServerConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -280,7 +279,7 @@ fn main() -> Result<()> {
} => entrypoint::dmoz_parser::run(dmoz_file, output_path).unwrap(),
Commands::Alice { options } => match options {
AliceOptions::Serve { config_path } => {
- let config: AliceLocalConfig = load_toml_config(config_path);
+ let config: config::AliceLocalConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -290,7 +289,7 @@ fn main() -> Result<()> {
AliceOptions::GenerateKey => entrypoint::alice::generate_key(),
},
Commands::CrawlCoordinator { config_path } => {
- let config: CrawlCoordinatorConfig = load_toml_config(config_path);
+ let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
@@ -298,7 +297,7 @@ fn main() -> Result<()> {
.block_on(entrypoint::crawler::coordinator(config))?
}
Commands::Crawler { config_path } => {
- let config: CrawlerConfig = load_toml_config(config_path);
+ let config: config::CrawlerConfig = load_toml_config(config_path);
tokio::runtime::Builder::new_multi_thread()
.enable_all()
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 09b77ea0..bae1dbc7 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -34,7 +34,7 @@ use crate::{
inverted_index,
search_ctx::Ctx,
searcher::NUM_RESULTS_PER_PAGE,
- webpage::region::Region,
+ webpage::region::Region, config::CollectorConfig,
};
pub use self::signal::*;
@@ -47,10 +47,11 @@ pub struct Ranker {
fastfield_reader: FastFieldReader,
de_rank_similar: bool,
num_results: Option<usize>,
+ collector_config: CollectorConfig,
}
impl Ranker {
- pub fn new(aggregator: SignalAggregator, fastfield_reader: FastFieldReader) -> Self {
+ pub fn new(aggregator: SignalAggregator, fastfield_reader: FastFieldReader, collector_config: CollectorConfig) -> Self {
Ranker {
offset: None,
aggregator,
@@ -58,6 +59,7 @@ impl Ranker {
de_rank_similar: true,
fastfield_reader,
num_results: None,
+ collector_config,
}
}
@@ -118,6 +120,8 @@ impl Ranker {
collector = collector.and_max_docs(max_docs.clone());
}
+ collector = collector.and_collector_config(self.collector_config.clone());
+
collector.tweak_score(score_tweaker)
}
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 78093885..59b2d502 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -20,6 +20,7 @@ use serde::{Deserialize, Serialize};
use crate::{
collector::{self, BucketCollector},
+ config::CollectorConfig,
enum_map::EnumMap,
inverted_index::WebsitePointer,
searcher::SearchQuery,
@@ -40,8 +41,8 @@ impl<T> collector::Doc for T
where
T: AsRankingWebsite,
{
- fn score(&self) -> &f64 {
- &self.as_ranking().score
+ fn score(&self) -> f64 {
+ self.as_ranking().score
}
fn id(&self) -> &tantivy::DocId {
@@ -234,7 +235,13 @@ struct RankingStage<T: AsRankingWebsite> {
}
impl<T: AsRankingWebsite> RankingStage<T> {
- fn apply(&self, websites: Vec<T>, top_n: usize, offset: usize) -> Vec<T> {
+ fn apply(
+ &self,
+ websites: Vec<T>,
+ top_n: usize,
+ offset: usize,
+ collector_config: CollectorConfig,
+ ) -> Vec<T> {
let mut websites = websites
.into_iter()
.skip(offset)
@@ -251,7 +258,8 @@ impl<T: AsRankingWebsite> RankingStage<T> {
}
}
- let mut collector = BucketCollector::new(self.stage_top_n.max(top_n) + offset);
+ let mut collector =
+ BucketCollector::new(self.stage_top_n.max(top_n) + offset, collector_config);
for website in websites {
collector.insert(website);
@@ -269,23 +277,26 @@ pub struct RankingPipeline<T: AsRankingWebsite> {
stage: RankingStage<T>,
page: usize,
pub top_n: usize,
+ collector_config: CollectorConfig,
}
impl<T: AsRankingWebsite> RankingPipeline<T> {
fn create_reranking<M: CrossEncoder + 'static>(
crossencoder: Arc<M>,
lambda: Option<Arc<LambdaMART>>,
+ collector_config: CollectorConfig,
) -> Result<Self> {
let stage = RankingStage {
scorer: Box::new(ReRanker::new(crossencoder, lambda)),
stage_top_n: 20,
- derank_similar: false,
+ derank_similar: true,
};
Ok(Self {
stage,
page: 0,
top_n: 0,
+ collector_config,
})
}
@@ -293,14 +304,15 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
query: &mut SearchQuery,
crossencoder: Arc<M>,
lambda: Option<Arc<LambdaMART>>,
+ collector_config: CollectorConfig,
) -> Result<Self> {
- let mut pipeline = Self::create_reranking(crossencoder, lambda)?;
+ let mut pipeline = Self::create_reranking(crossencoder, lambda, collector_config)?;
pipeline.set_query_info(query);
Ok(pipeline)
}
- fn create_ltr(model: Option<Arc<LambdaMART>>) -> Self {
+ fn create_ltr(model: Option<Arc<LambdaMART>>, collector_config: CollectorConfig) -> Self {
let last_stage = RankingStage {
scorer: Box::new(Initial {
model,
@@ -314,11 +326,16 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
stage: last_stage,
page: 0,
top_n: 0,
+ collector_config,
}
}
- pub fn ltr_for_query(query: &mut SearchQuery, model: Option<Arc<LambdaMART>>) -> Self {
- let mut pipeline = Self::create_ltr(model);
+ pub fn ltr_for_query(
+ query: &mut SearchQuery,
+ model: Option<Arc<LambdaMART>>,
+ collector_config: CollectorConfig,
+ ) -> Self {
+ let mut pipeline = Self::create_ltr(model, collector_config);
pipeline.set_query_info(query);
pipeline
@@ -338,7 +355,12 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
pub fn apply(self, websites: Vec<T>) -> Vec<T> {
- self.stage.apply(websites, self.top_n, self.offset())
+ self.stage.apply(
+ websites,
+ self.top_n,
+ self.offset(),
+ self.collector_config.clone(),
+ )
}
pub fn collector_top_n(&self) -> usize {
@@ -399,6 +421,7 @@ mod tests {
},
Arc::new(DummyCrossEncoder {}),
None,
+ CollectorConfig::default(),
)
.unwrap();
assert_eq!(pipeline.collector_top_n(), 20);
@@ -429,6 +452,7 @@ mod tests {
},
Arc::new(DummyCrossEncoder {}),
None,
+ CollectorConfig::default(),
)
.unwrap();
@@ -462,6 +486,7 @@ mod tests {
},
Arc::new(DummyCrossEncoder {}),
None,
+ CollectorConfig::default(),
)
.unwrap();
@@ -475,6 +500,7 @@ mod tests {
},
Arc::new(DummyCrossEncoder {}),
None,
+ CollectorConfig::default(),
)
.unwrap();
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index 94b15541..a22e443b 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -250,30 +250,30 @@ impl Signal {
pub fn default_coefficient(&self) -> f64 {
match self {
Signal::Bm25 => 0.0005,
- Signal::Bm25Title => 0.004,
+ Signal::Bm25Title => 0.005,
Signal::Bm25TitleBigrams => 0.005,
- Signal::Bm25TitleTrigrams => 0.01,
- Signal::Bm25CleanBody => 0.004,
+ Signal::Bm25TitleTrigrams => 0.005,
+ Signal::Bm25CleanBody => 0.005,
Signal::Bm25CleanBodyBigrams => 0.005,
- Signal::Bm25CleanBodyTrigrams => 0.01,
- Signal::Bm25StemmedTitle => 0.001,
+ Signal::Bm25CleanBodyTrigrams => 0.005,
+ Signal::Bm25StemmedTitle => 0.0005,
Signal::Bm25StemmedCleanBody => 0.0005,
Signal::Bm25AllBody => 0.0,
- Signal::Bm25Url => 0.0001,
- Signal::Bm25Site => 0.0001,
- Signal::Bm25Domain => 0.0001,
- Signal::Bm25SiteNoTokenizer => 0.0001,
- Signal::Bm25DomainNoTokenizer => 0.0001,
- Signal::Bm25DomainIfHomepage => 0.0001,
+ Signal::Bm25Url => 0.00005,
+ Signal::Bm25Site => 0.00005,
+ Signal::Bm25Domain => 0.00005,
+ Signal::Bm25SiteNoTokenizer => 0.00005,
+ Signal::Bm25DomainNoTokenizer => 0.00005,
+ Signal::Bm25DomainIfHomepage => 0.00005,
Signal::Bm25DomainNameIfHomepageNoTokenizer => 0.0001,
- Signal::Bm25TitleIfHomepage => 0.0001,
- Signal::Bm25BacklinkText => 0.01,
+ Signal::Bm25TitleIfHomepage => 0.00001,
+ Signal::Bm25BacklinkText => 0.001,
Signal::Bm25Description => 5.99679347128802e-05,
- Signal::ProximitySlop0 => 0.0005,
- Signal::ProximitySlop1 => 0.0005,
- Signal::ProximitySlop2 => 0.0005,
- Signal::ProximitySlop4 => 0.0005,
- Signal::ProximitySlop8 => 0.0005,
+ Signal::ProximitySlop0 => 0.01,
+ Signal::ProximitySlop1 => 0.01,
+ Signal::ProximitySlop2 => 0.01,
+ Signal::ProximitySlop4 => 0.01,
+ Signal::ProximitySlop8 => 0.01,
Signal::CrossEncoder => 0.16801589371906178,
Signal::HostCentrality => 1.0,
Signal::PageCentrality => 2.0,
@@ -446,21 +446,19 @@ impl Signal {
| Signal::ProximitySlop2
| Signal::ProximitySlop4
| Signal::ProximitySlop8 => {
- signal_aggregator
+ let reader = signal_aggregator
.segment_reader
.as_mut()
- .and_then(|reader| {
- reader.proximity_scorers.get_mut(self).map(|scorer| {
- let docset = &mut scorer.docset;
- if docset.doc() == doc
- || (docset.doc() < doc && docset.seek(doc) == doc)
- {
- docset.score() as f64
- } else {
- 0.0
- }
- })
- })
+ .expect("set segment reader before computing signals");
+
+ reader.proximity_scorers.get_mut(self).map(|scorer| {
+ let docset = &mut scorer.docset;
+ if docset.doc() == doc || (docset.doc() < doc && docset.seek(doc) == doc) {
+ docset.score() as f64
+ } else {
+ 0.0
+ }
+ })
}
Signal::CrossEncoder => None, // this is calculated in a later step
Signal::LambdaMART => None,
diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rs
index 1546df33..77be5be9 100644
--- a/core/src/searcher/frontend.rs
+++ b/core/src/searcher/frontend.rs
@@ -24,6 +24,7 @@ use itertools::intersperse;
use optics::Optic;
use crate::bangs::{Bang, BangHit};
+use crate::config::{CollectorConfig, FrontendThresholds};
use crate::inverted_index::RetrievedWebpage;
use crate::ranking::ALL_SIGNALS;
use crate::search_prettifier::{
@@ -48,15 +49,14 @@ use super::{
SearchResult, WebsitesResult,
};
-const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 0.1;
-const DISCUSSIONS_WIDGET_THRESHOLD: f64 = 0.1;
-
pub struct FrontendSearcher {
distributed_searcher: DistributedSearcher,
cross_encoder: Arc<CrossEncoderModel>,
lambda_model: Option<Arc<LambdaMART>>,
qa_model: Option<Arc<QaModel>>,
bangs: Bangs,
+ collector_config: CollectorConfig,
+ thresholds: FrontendThresholds,
}
impl FrontendSearcher {
@@ -66,6 +66,8 @@ impl FrontendSearcher {
lambda_model: Option<LambdaMART>,
qa_model: Option<QaModel>,
bangs: Bangs,
+ collector_config: CollectorConfig,
+ thresholds: FrontendThresholds,
) -> Self {
Self {
distributed_searcher: DistributedSearcher::new(cluster),
@@ -73,6 +75,8 @@ impl FrontendSearcher {
lambda_model: lambda_model.map(Arc::new),
qa_model: qa_model.map(Arc::new),
bangs,
+ collector_config,
+ thresholds,
}
}
@@ -81,7 +85,8 @@ impl FrontendSearcher {
initial_results: Vec<InitialSearchResultShard>,
pipeline: RankingPipeline<ScoredWebsitePointer>,
) -> (Vec<ScoredWebsitePointer>, bool) {
- let mut collector = BucketCollector::new(pipeline.collector_top_n());
+ let mut collector =
+ BucketCollector::new(pipeline.collector_top_n(), self.collector_config.clone());
let mut num_sites: usize = 0;
for result in initial_results {
@@ -136,7 +141,7 @@ impl FrontendSearcher {
results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));
if let Some((shard, website)) = results.pop() {
- if website.score > STACKOVERFLOW_SIDEBAR_THRESHOLD {
+ if website.score > self.thresholds.stackoverflow {
let scored_websites = vec![ScoredWebsitePointer { website, shard }];
let mut retrieved = self
.distributed_searcher
@@ -228,6 +233,7 @@ impl FrontendSearcher {
&mut query,
self.cross_encoder.clone(),
self.lambda_model.clone(),
+ self.collector_config.clone(),
)?;
let initial_results = self.distributed_searcher.search_initial(&query).await;
@@ -258,7 +264,7 @@ impl FrontendSearcher {
scores[scores.len() / 2]
};
- if median < DISCUSSIONS_WIDGET_THRESHOLD {
+ if median < self.thresholds.discussions_widget {
return Ok(None);
}
@@ -308,6 +314,7 @@ impl FrontendSearcher {
&mut search_query,
self.cross_encoder.clone(),
self.lambda_model.clone(),
+ self.collector_config.clone(),
)?;
let initial_results = self
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 659efff6..ea7f8eb4 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -17,6 +17,7 @@
use std::collections::HashMap;
use std::sync::Arc;
+use crate::config::{CollectorConfig, SnippetConfig};
use crate::entity_index::{EntityIndex, StoredEntity};
use crate::image_store::Image;
use crate::index::Index;
@@ -45,6 +46,7 @@ pub struct LocalSearcher {
centrality_store: Option<SearchCentralityStore>,
linear_regression: Option<Arc<LinearRegression>>,
lambda_model: Option<Arc<LambdaMART>>,
+ collector_config: CollectorConfig,
}
impl From<Index> for LocalSearcher {
@@ -69,6 +71,7 @@ impl LocalSearcher {
centrality_store: None,
linear_regression: None,
lambda_model: None,
+ collector_config: CollectorConfig::default(),
}
}
@@ -88,6 +91,14 @@ impl LocalSearcher {
self.lambda_model = Some(Arc::new(model));
}
+ pub fn set_collector_config(&mut self, config: CollectorConfig) {
+ self.collector_config = config;
+ }
+
+ pub fn set_snippet_config(&mut self, config: SnippetConfig) {
+ self.index.inverted_index.set_snippet_config(config);
+ }
+
fn parse_query(&self, ctx: &Ctx, query: &SearchQuery) -> Result<Query> {
let parsed_query = Query::parse(ctx, query, &self.index.inverted_index)?;
@@ -110,6 +121,7 @@ impl LocalSearcher {
let mut ranker = Ranker::new(
aggregator,
self.index.inverted_index.fastfield_reader(&ctx.tv_searcher),
+ self.collector_config.clone(),
);
if let Some(region) = query.region() {
@@ -155,8 +167,11 @@ impl LocalSearcher {
de_rank_similar: bool,
) -> Result<InvertedIndexResult> {
let mut query = query.clone();
- let pipeline: RankingPipeline<RankingWebsite> =
- RankingPipeline::ltr_for_query(&mut query, self.lambda_model.clone());
+ let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::ltr_for_query(
+ &mut query,
+ self.lambda_model.clone(),
+ self.collector_config.clone(),
+ );
let parsed_query = self.parse_query(ctx, &query)?;
let mut aggregator = SignalAggregator::new(Some(&parsed_query));
@@ -298,13 +313,17 @@ impl LocalSearcher {
let mut search_query = query.clone();
let pipeline = match CrossEncoderModel::open("data/cross_encoder") {
- Ok(model) => {
- RankingPipeline::reranking_for_query(&mut search_query, Arc::new(model), None)?
- }
+ Ok(model) => RankingPipeline::reranking_for_query(
+ &mut search_query,
+ Arc::new(model),
+ None,
+ self.collector_config.clone(),
+ )?,
Err(_) => RankingPipeline::reranking_for_query(
&mut search_query,
Arc::new(DummyCrossEncoder {}),
None,
+ self.collector_config.clone(),
)?,
};
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index 4d3c7377..6d593546 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -22,8 +22,6 @@ use crate::{
webgraph::{Node, NodeID, Webgraph},
};
-const MAX_SIMILAR_SITES: usize = 1_000;
-
#[derive(serde::Serialize, serde::Deserialize, Debug)]
pub struct ScoredNode {
pub node: Node,
@@ -58,18 +56,24 @@ impl Eq for ScoredNodeID {}
pub struct SimilarSitesFinder {
webgraph: Arc<Webgraph>,
inbound_similarity: InboundSimilarity,
+ max_similar_sites: usize,
}
impl SimilarSitesFinder {
- pub fn new(webgraph: Arc<Webgraph>, inbound_similarity: InboundSimilarity) -> Self {
+ pub fn new(
+ webgraph: Arc<Webgraph>,
+ inbound_similarity: InboundSimilarity,
+ max_similar_sites: usize,
+ ) -> Self {
Self {
webgraph,
inbound_similarity,
+ max_similar_sites,
}
}
pub fn find_similar_sites(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {
- let limit = limit.min(MAX_SIMILAR_SITES);
+ let limit = limit.min(self.max_similar_sites);
let nodes: Vec<_> = nodes
.iter()
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 7543c6fd..8a7d5426 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use crate::config::SnippetConfig;
use crate::query::Query;
use crate::search_prettifier::html_escape;
use crate::spell::sentence_ranges;
@@ -34,10 +35,6 @@ use whatlang::Lang;
/// This might require us to store each paragraph of the webpage separately to get adequate performance (maybe we can split passages online with adequate performance
/// but we need to test this).
-const DESIRED_NUM_CHARS: usize = 275;
-const DELTA_NUM_CHARS: usize = 50;
-const MIN_PASSAGE_WIDTH: usize = 20;
-
const K1: f64 = 1.2;
const B: f64 = 0.75;
@@ -90,7 +87,12 @@ impl SnippetString {
}
}
-fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> SnippetString {
+fn snippet_string(
+ text: &str,
+ terms: &[String],
+ lang: whatlang::Lang,
+ config: SnippetConfig,
+) -> SnippetString {
let tokenizer = Tokenizer::Stemmed(Stemmed::with_forced_language(lang));
let terms: HashSet<String> = terms
@@ -109,7 +111,7 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
let mut passages: Vec<_> = sentence_ranges(text)
.into_iter()
- .filter(|offset| offset.end - offset.start > MIN_PASSAGE_WIDTH)
+ .filter(|offset| offset.end - offset.start > config.min_passage_width)
.map(|offset| {
let sentence = text[offset].to_string();
@@ -132,7 +134,7 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
if passages.is_empty() {
let mut snippet = SnippetString {
- fragment: text.chars().take(DESIRED_NUM_CHARS).collect(),
+ fragment: text.chars().take(config.desired_num_chars).collect(),
highlighted: Vec::new(),
};
@@ -191,18 +193,18 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
highlighted: Vec::new(),
};
- if snippet.fragment.len() > DESIRED_NUM_CHARS + DELTA_NUM_CHARS {
- // TODO: find 'DESIRED_NUM_CHARS' sized window that contains most highlights
+ if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {
+ // TODO: find 'desired_num_chars' sized window that contains most highlights
// instead of taking the prefix of the passage as a snippet
snippet.fragment = snippet
.fragment
.chars()
- .take(DESIRED_NUM_CHARS + DELTA_NUM_CHARS)
+ .take(config.desired_num_chars + config.delta_num_chars)
.collect();
} else {
let mut next_passage_idx = best_idx + 1;
- while snippet.fragment.len() < DESIRED_NUM_CHARS - DELTA_NUM_CHARS
+ while snippet.fragment.len() < config.desired_num_chars - config.delta_num_chars
&& next_passage_idx < passages.len()
{
snippet.fragment += " ";
@@ -210,11 +212,11 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
next_passage_idx += 1;
}
- if snippet.fragment.len() > DESIRED_NUM_CHARS + DELTA_NUM_CHARS {
+ if snippet.fragment.len() > config.desired_num_chars + config.delta_num_chars {
snippet.fragment = snippet
.fragment
.chars()
- .take(DESIRED_NUM_CHARS + DELTA_NUM_CHARS)
+ .take(config.desired_num_chars + config.delta_num_chars)
.collect();
}
}
@@ -223,13 +225,13 @@ fn snippet_string(text: &str, terms: &[String], lang: whatlang::Lang) -> Snippet
snippet
}
-pub fn generate(query: &Query, text: &str, region: &Region) -> String {
+pub fn generate(query: &Query, text: &str, region: &Region, config: SnippetConfig) -> String {
let lang = match region.lang() {
Some(lang) => lang,
None => whatlang::detect_lang(text).unwrap_or(Lang::Eng),
};
- let snippet = snippet_string(text, query.simple_terms(), lang);
+ let snippet = snippet_string(text, query.simple_terms(), lang, config);
snippet.to_html()
}
@@ -380,9 +382,14 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn empty_query() {
assert_eq!(
- snippet_string("this is a test", &[], whatlang::Lang::Eng)
- .fragment
- .as_str(),
+ snippet_string(
+ "this is a test",
+ &[],
+ whatlang::Lang::Eng,
+ SnippetConfig::default()
+ )
+ .fragment
+ .as_str(),
"this is a test"
);
}
@@ -390,14 +397,19 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn empty_text() {
assert_eq!(
- snippet_string("", &["test".to_string()], whatlang::Lang::Eng)
- .fragment
- .as_str(),
+ snippet_string(
+ "",
+ &["test".to_string()],
+ whatlang::Lang::Eng,
+ SnippetConfig::default()
+ )
+ .fragment
+ .as_str(),
""
);
assert_eq!(
- snippet_string("", &[], whatlang::Lang::Eng)
+ snippet_string("", &[], whatlang::Lang::Eng, SnippetConfig::default())
.fragment
.as_str(),
""
diff --git a/core/src/warc.rs b/core/src/warc.rs
index ec425833..5e882d34 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::distributed::retry_strategy::ExponentialBackoff;
-use crate::{Error, Result, S3Config, WarcSource};
+use crate::{config::S3Config, config::WarcSource, Error, Result};
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};