web search done right
Semantic embeddings (#179)
Mikkel Denker32bbbc63ab5a
diff --git a/.gitignore b/.gitignoreindex 1e96e4da..8fc1d80a 100644--- a/.gitignore+++ b/.gitignore@@ -15,3 +15,4 @@ ltr_scores.jsonproptest-regressions*.pending-snap.ipynb_checkpoints+.zeddiff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex 3c07e48f..03e82d67 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -34,8 +34,9 @@ if they take our code without making theirs open source, which would breach the# Development## Setup-* Install rust by following the steps outlined [here](https://www.rust-lang.org/tools/install)-* Install clang and npm+* Install rust by following the steps outlined [here](https://www.rust-lang.org/tools/install).+* Install clang and npm.+* Install liburing. If you're using a Debian based Linux, you can install it by running `sudo apt install liburing-dev`.* Update ulimit. RocksDB tends to exceed the max number of allowed open files, so you will have to run `ulimit -n 10240` to increase the allowed max number of open files.* Install [just](https://github.com/casey/just) by running `cargo install just`. This allows you to run the scripts in the `justfile` file. A justfile is basically a simple Makefile.* Install [wasm-pack](https://rustwasm.github.io/wasm-pack/installer/) by running `cargo install wasm-pack`. This will allow for generation and packaging of client side WebAssembly code.diff --git a/Cargo.lock b/Cargo.lockindex 22bb18d5..3ba9853e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -531,6 +531,12 @@ version = "3.15.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b"+[[package]]+name = "bytecount"+version = "0.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205"+[[package]]name = "bytemuck"version = "1.14.3"@@ -4462,6 +4468,7 @@ dependencies = ["base64 0.21.7","bincode","bitvec",+ "bytecount","bytemuck","byteorder 1.5.0","bzip2",diff --git a/Cargo.toml b/Cargo.tomlindex b2a78c4a..284a21cd 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -26,6 +26,7 @@ base64 = "0.21.4"bincode = "1.3.3"bindgen = "0.69.2"bitvec = "1.0.1"+bytecount = "0.6.7"bytemuck = {version = "1.13.1", features = ["derive"]}byteorder = "1.4.3"bzip2 = "0.4.4"diff --git a/assets/licenses.html b/assets/licenses.htmlindex 0329dce2..a2e6042f 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,7 +44,7 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (389)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (390)</li><li><a href="#MIT">MIT License</a> (175)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (8)</li>@@ -3238,6 +3238,7 @@<ul class="license-used-by"><li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle-parse 0.2.3</a></li><li><a href=" https://github.com/phil-opp/rust-bit-field ">bit_field 0.10.2</a></li>+ <li><a href=" https://github.com/llogiq/bytecount ">bytecount 0.6.7</a></li><li><a href=" https://github.com/retep998/winapi-rs ">winapi 0.3.9</a></li></ul><pre class="license-text"> Apache Licensediff --git a/configs/api.toml b/configs/api.tomlindex 3bf1797f..9c4322c1 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -4,6 +4,7 @@ gossip_addr = "0.0.0.0:3005"gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]host = "0.0.0.0:3000"# lambda_model_path = "data/lambdamart.txt"+dual_encoder_model_path = "data/summarizer/dual_encoder"prometheus_host = "0.0.0.0:3001"queries_csv_path = "data/queries_us.csv"spell_checker_path = "data/web_spell/checker"diff --git a/configs/search_server.toml b/configs/search_server.tomlindex aee278bc..323c047f 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -6,6 +6,7 @@ host = "0.0.0.0:3002"host_centrality_store_path = "data/centrality"index_path = "data/index"shard_id = 0+dual_encoder_model_path = "data/summarizer/dual_encoder"# linear_model_path = "data/linear_model.json"# lambda_model_path = "data/lambdamart.txt"diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 64097cfe..8e65aaa6 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -30,6 +30,7 @@ axum-macros = {workspace = true}base64 = {workspace = true}bincode = {workspace = true}bitvec = {workspace = true}+bytecount = {workspace = true}bytemuck = {workspace = true}byteorder = {workspace = true}bzip2 = {workspace = true}@@ -144,3 +145,6 @@ name = "search_preindexed"[[example]]name = "bitvec_similarity"++[[example]]+name = "indexer"diff --git a/crates/core/benches/bitvec_similarity.rs b/crates/core/benches/bitvec_similarity.rsindex f7cc4fbf..16a8de5e 100644--- a/crates/core/benches/bitvec_similarity.rs+++ b/crates/core/benches/bitvec_similarity.rs@@ -11,7 +11,7 @@ fn random_bitvec(max_len: usize, max_id: usize) -> BitVec {ranks.push(rng.gen_range(0..max_id) as u64);}- ranks.sort();+ ranks.sort_unstable();ranks.dedup();BitVec::new(ranks)@@ -26,7 +26,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {for _ in 0..10_000 {a.sim(&b);}- })+ });}});@@ -38,7 +38,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {for _ in 0..10_000 {a.sim(&b);}- })+ });}});@@ -50,7 +50,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {for _ in 0..10_000 {a.sim(&b);}- })+ });}});}diff --git a/crates/core/benches/build_similarity.rs b/crates/core/benches/build_similarity.rsindex fbac03c1..7670d75c 100644--- a/crates/core/benches/build_similarity.rs+++ b/crates/core/benches/build_similarity.rs@@ -9,7 +9,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {c.bench_function("Inbound similarity creation", |b| {b.iter(|| {InboundSimilarity::build(&graph);- })+ });});}diff --git a/crates/core/benches/harmonic_centrality.rs b/crates/core/benches/harmonic_centrality.rsindex 33a6393d..ba4c4dda 100644--- a/crates/core/benches/harmonic_centrality.rs+++ b/crates/core/benches/harmonic_centrality.rs@@ -10,7 +10,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {for _ in 0..10 {HarmonicCentrality::calculate(&webgraph);}- })+ });});}diff --git a/crates/core/benches/hyperloglog.rs b/crates/core/benches/hyperloglog.rsindex 2a1ca1d5..7cef3493 100644--- a/crates/core/benches/hyperloglog.rs+++ b/crates/core/benches/hyperloglog.rs@@ -7,13 +7,13 @@ pub fn criterion_benchmark(c: &mut Criterion) {let mut log: HyperLogLog<128> = HyperLogLog::default();for i in 0..10_000_000 {log.add(i);- log.size();+ let _ = log.size();}for _ in 0..1_000_000_000 {- log.size();+ let _ = log.size();}- })+ });});}diff --git a/crates/core/benches/naive_bayes.rs b/crates/core/benches/naive_bayes.rsindex 7d55c7b2..9ff46817 100644--- a/crates/core/benches/naive_bayes.rs+++ b/crates/core/benches/naive_bayes.rs@@ -68,7 +68,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {"{:.2} ms/pred",start.elapsed().as_millis() as f32 / num_samples as f32);- })+ });});}diff --git a/crates/core/benches/similar_hosts.rs b/crates/core/benches/similar_hosts.rsindex 3bcb2d81..37cdd22e 100644--- a/crates/core/benches/similar_hosts.rs+++ b/crates/core/benches/similar_hosts.rs@@ -21,7 +21,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {for _ in 0..10 {c.bench_function("similar_hosts", |b| {- b.iter(|| finder.find_similar_hosts(&["google.com".to_string()], 100))+ b.iter(|| finder.find_similar_hosts(&["google.com".to_string()], 100));});}}diff --git a/crates/core/examples/bitvec_similarity.rs b/crates/core/examples/bitvec_similarity.rsindex e0d1a158..a5d86ad7 100644--- a/crates/core/examples/bitvec_similarity.rs+++ b/crates/core/examples/bitvec_similarity.rs@@ -10,7 +10,7 @@ fn random_bitvec(max_len: usize, max_id: usize) -> BitVec {ranks.push(rng.gen_range(0..max_id) as u64);}- ranks.sort();+ ranks.sort_unstable();ranks.dedup();BitVec::new(ranks)diff --git a/crates/core/examples/indexer.rs b/crates/core/examples/indexer.rsnew file mode 100644index 00000000..c7aacd41--- /dev/null+++ b/crates/core/examples/indexer.rs@@ -0,0 +1,67 @@+use std::path::Path;++use clap::Parser;+use stract::config::IndexingLocalConfig;++#[derive(Parser)]+struct Args {+ dual_encoder_path: Option<String>,+}++fn main() -> anyhow::Result<()> {+ let args = Args::parse();++ let data_path = Path::new("data");+ let path = data_path.join("example_index");++ if path.exists() {+ std::fs::remove_dir_all(&path)?;+ }++ println!("Indexing...");+ let start = std::time::Instant::now();+ stract::entrypoint::indexer::run(&IndexingLocalConfig {+ output_path: path.to_str().unwrap().to_string(),+ limit_warc_files: None,+ skip_warc_files: None,+ warc_source: stract::config::WarcSource::Local(stract::config::LocalConfig {+ folder: ".".to_string(),+ names: vec![data_path+ .join("sample.warc.gz")+ .to_str()+ .unwrap()+ .to_string()],+ }),+ page_webgraph_path: Some(+ data_path+ .join("webgraph_page")+ .to_str()+ .unwrap()+ .to_string(),+ ),+ host_centrality_threshold: None,+ topics_path: None,+ host_centrality_store_path: data_path.join("centrality/").to_str().unwrap().to_string(),+ page_centrality_store_path: Some(+ data_path+ .join("centrality_page")+ .to_str()+ .unwrap()+ .to_string(),+ ),+ safety_classifier_path: None,+ minimum_clean_words: None,+ batch_size: 512,+ dual_encoder: args+ .dual_encoder_path+ .map(|p| stract::config::IndexingDualEncoderConfig {+ model_path: p,+ page_centrality_rank_threshold: Some(1_000_000),+ }),+ })?;++ println!("Indexing took {:?}", start.elapsed());++ std::fs::remove_dir(path)?;+ Ok(())+}diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex be26c240..1c04cb84 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -9,7 +9,7 @@ use stract::{image_store::Image,index::Index,inverted_index::RetrievedWebpage,- ranking::{inbound_similarity::InboundSimilarity, pipeline::RetrievedWebpageRanking},+ ranking::{inbound_similarity::InboundSimilarity, pipeline::PrecisionRankingWebpage},searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery, ShardId},Result,};@@ -30,9 +30,9 @@ impl stract::searcher::distributed::SearchClient for Searcher {async fn retrieve_webpages(&self,- top_websites: &[(usize, stract::searcher::ScoredWebsitePointer)],+ top_websites: &[(usize, stract::searcher::ScoredWebpagePointer)],query: &str,- ) -> Vec<(usize, stract::ranking::pipeline::RetrievedWebpageRanking)> {+ ) -> Vec<(usize, stract::ranking::pipeline::PrecisionRankingWebpage)> {let pointers = top_websites.iter().map(|(_, p)| p.website.pointer.clone())@@ -44,7 +44,7 @@ impl stract::searcher::distributed::SearchClient for Searcher {.unwrap().into_iter().zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))- .map(|(ret, (i, ran))| (i, RetrievedWebpageRanking::new(ret, ran)))+ .map(|(ret, (i, ran))| (i, PrecisionRankingWebpage::new(ret, ran))).collect::<Vec<_>>();res@@ -99,6 +99,7 @@ pub async fn main() {prometheus_host: "0.0.0.0:8001".parse().unwrap(),crossencoder_model_path: None,lambda_model_path: None,+ dual_encoder_model_path: None,spell_checker_path: Some("data/web_spell".to_string()),bangs_path: "data/bangs.json".to_string(),summarizer_path: "data/summarizer".to_string(),@@ -142,14 +143,14 @@ pub async fn main() {let searcher = Searcher(searcher);let searcher: ApiSearcher<Searcher, LiveSearcher> =- ApiSearcher::new(searcher, None, None, None, bangs, config);+ ApiSearcher::new(searcher, None, None, None, None, bangs, config);for query in &queries {let mut desc = "search '".to_string();desc.push_str(query);desc.push('\'');- println!("{}", desc);+ println!("{desc}");searcher.search(&SearchQuery {diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rsindex 89a08e31..6cd02e29 100644--- a/crates/core/src/api/docs.rs+++ b/crates/core/src/api/docs.rs@@ -106,10 +106,10 @@ struct ApiModifier;impl Modify for ApiModifier {fn modify(&self, openapi: &mut utoipa::openapi::OpenApi) {openapi.info.description = Some(- r#"Stract is an open source web search engine. The API is totally free while in beta, but some endpoints will most likely be paid by consumption in the future.+ "Stract is an open source web search engine. The API is totally free while in beta, but some endpoints will most likely be paid by consumption in the future.The API might also change quite a bit during the beta period, but we will try to keep it as stable as possible. We look forward to see what you will build!-Remember to always give proper attributions to the sources you use from the search results."#.to_string(),+Remember to always give proper attributions to the sources you use from the search results.".to_string(),);}}diff --git a/crates/core/src/api/improvement.rs b/crates/core/src/api/improvement.rsindex fb8cd112..18851ba5 100644--- a/crates/core/src/api/improvement.rs+++ b/crates/core/src/api/improvement.rs@@ -45,7 +45,7 @@ pub async fn click(q.lock().await.push(ImprovementEvent::Click {qid: params.qid,idx: params.click,- })+ });}}diff --git a/crates/core/src/api/metrics.rs b/crates/core/src/api/metrics.rsindex c2425559..9ee20f89 100644--- a/crates/core/src/api/metrics.rs+++ b/crates/core/src/api/metrics.rs@@ -24,5 +24,5 @@ use axum_macros::debug_handler;pub async fn route(extract::State(registry): extract::State<Arc<crate::metrics::PrometheusRegistry>>,) -> impl IntoResponse {- format!("{}", registry)+ format!("{registry}");}diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex 37ba6b10..cc5bcbd0 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -31,6 +31,7 @@ use crate::{},improvement::{store_improvements_loop, ImprovementEvent},leaky_queue::LeakyQueue,+ models::dual_encoder::DualEncoder,ranking::models::lambdamart::LambdaMART,searcher::{api::ApiSearcher, live::LiveSearcher, DistributedSearcher},};@@ -90,6 +91,58 @@ pub async fn favicon() -> impl IntoResponse {.unwrap()}+fn build_router(state: Arc<State>) -> Router {+ Router::new()+ .merge(+ Router::new()+ .route("/beta/api/search", post(search::search))+ .route_layer(middleware::from_fn_with_state(state.clone(), search_metric))+ .layer(cors_layer()),+ )+ .route("/favicon.ico", get(favicon))+ .merge(+ Router::new()+ .route("/improvement/click", post(improvement::click))+ .route("/improvement/store", post(improvement::store))+ .layer(cors_layer()),+ )+ .layer(CompressionLayer::new())+ .merge(docs::router())+ .nest(+ "/beta",+ Router::new()+ .route("/api/search/widget", post(search::widget))+ .route("/api/search/sidebar", post(search::sidebar))+ .route("/api/search/spellcheck", post(search::spellcheck))+ .route("/api/autosuggest", post(autosuggest::route))+ .route("/api/autosuggest/browser", get(autosuggest::browser))+ .route("/api/summarize", get(summarize::summarize_route))+ .route("/api/webgraph/host/similar", post(webgraph::host::similar))+ .route("/api/webgraph/host/knows", post(webgraph::host::knows))+ .route(+ "/api/webgraph/host/ingoing",+ post(webgraph::host::ingoing_hosts),+ )+ .route(+ "/api/webgraph/host/outgoing",+ post(webgraph::host::outgoing_hosts),+ )+ .route(+ "/api/webgraph/page/ingoing",+ post(webgraph::page::ingoing_pages),+ )+ .route(+ "/api/webgraph/page/outgoing",+ post(webgraph::page::outgoing_pages),+ )+ .route("/api/hosts/export", post(hosts::hosts_export_optic))+ .route("/api/explore/export", post(explore::explore_export_optic))+ .route("/api/entity_image", get(search::entity_image))+ .layer(cors_layer()),+ )+ .with_state(state)+}+pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {let autosuggest = Autosuggest::load_csv(&config.queries_csv_path)?;@@ -98,6 +151,11 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {None => None,};+ let dual_encoder_model = match &config.dual_encoder_model_path {+ Some(path) => Some(DualEncoder::open(path)?),+ None => None,+ };+let query_store_queue = config.query_store_db_host.clone().map(|db_host| {let query_store_queue = Arc::new(Mutex::new(LeakyQueue::new(10_000)));tokio::spawn(store_improvements_loop(query_store_queue.clone(), db_host));@@ -134,6 +192,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {Some(live_searcher),cross_encoder,lambda_model,+ dual_encoder_model,bangs,config.clone(),);@@ -155,55 +214,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {})};- Ok(Router::new()- .merge(- Router::new()- .route("/beta/api/search", post(search::search))- .route_layer(middleware::from_fn_with_state(state.clone(), search_metric))- .layer(cors_layer()),- )- .route("/favicon.ico", get(favicon))- .merge(- Router::new()- .route("/improvement/click", post(improvement::click))- .route("/improvement/store", post(improvement::store))- .layer(cors_layer()),- )- .layer(CompressionLayer::new())- .merge(docs::router())- .nest(- "/beta",- Router::new()- .route("/api/search/widget", post(search::widget))- .route("/api/search/sidebar", post(search::sidebar))- .route("/api/search/spellcheck", post(search::spellcheck))- .route("/api/autosuggest", post(autosuggest::route))- .route("/api/autosuggest/browser", get(autosuggest::browser))- .route("/api/summarize", get(summarize::summarize_route))- .route("/api/webgraph/host/similar", post(webgraph::host::similar))- .route("/api/webgraph/host/knows", post(webgraph::host::knows))- .route(- "/api/webgraph/host/ingoing",- post(webgraph::host::ingoing_hosts),- )- .route(- "/api/webgraph/host/outgoing",- post(webgraph::host::outgoing_hosts),- )- .route(- "/api/webgraph/page/ingoing",- post(webgraph::page::ingoing_pages),- )- .route(- "/api/webgraph/page/outgoing",- post(webgraph::page::outgoing_pages),- )- .route("/api/hosts/export", post(hosts::hosts_export_optic))- .route("/api/explore/export", post(explore::explore_export_optic))- .route("/api/entity_image", get(search::entity_image))- .layer(cors_layer()),- )- .with_state(state))+ Ok(build_router(state))}/// Enables CORS for development where the API and frontend are ondiff --git a/crates/core/src/bloom.rs b/crates/core/src/bloom.rsindex 7a56427e..2bcfaed1 100644--- a/crates/core/src/bloom.rs+++ b/crates/core/src/bloom.rs@@ -35,16 +35,16 @@ impl BloomFilter {((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64}- fn hash(item: &u64) -> usize {+ fn hash(item: u64) -> usize {item.wrapping_mul(11400714819323198549) as usize}pub fn insert(&mut self, item: u64) {- let h = Self::hash(&item);+ let h = Self::hash(item);self.bit_vec.set(h % self.num_bits as usize, true);}- pub fn contains(&self, item: &u64) -> bool {+ pub fn contains(&self, item: u64) -> bool {let h = Self::hash(item);self.bit_vec[h % self.num_bits as usize]}diff --git a/crates/core/src/collector.rs b/crates/core/src/collector.rsindex 19d2a84b..bd921c0b 100644--- a/crates/core/src/collector.rs+++ b/crates/core/src/collector.rs@@ -27,7 +27,7 @@ use crate::{combine_u64s,config::CollectorConfig,fastfield_reader,- inverted_index::{DocAddress, WebsitePointer},+ inverted_index::{DocAddress, WebpagePointer},prehashed::Prehashed,ranking::initial::{InitialScoreTweaker, Score},schema::FastField,@@ -139,10 +139,13 @@ pub struct TopSegmentCollector {}impl TopSegmentCollector {- fn get_hash(&self, doc: &DocId, field1: &FastField, field2: &FastField) -> Prehashed {+ fn get_hash(&self, doc: DocId, field1: FastField, field2: FastField) -> Prehashed {let field_reader = self.fastfield_segment_reader.get_field_reader(doc);- let hash = [field_reader.get(field1), field_reader.get(field2)];+ let hash = [+ field_reader.get(field1).unwrap().as_u64().unwrap(),+ field_reader.get(field2).unwrap().as_u64().unwrap(),+ ];combine_u64s(hash).into()}}@@ -165,19 +168,20 @@ impl TopSegmentCollector {let simhash: Option<u64> = self.fastfield_segment_reader- .get_field_reader(&doc)- .get(&FastField::SimHash)+ .get_field_reader(doc)+ .get(FastField::SimHash)+ .unwrap().into();self.bucket_collector.insert(SegmentDoc {hashes: Hashes {- site: self.get_hash(&doc, &FastField::SiteHash1, &FastField::SiteHash2),- title: self.get_hash(&doc, &FastField::TitleHash1, &FastField::TitleHash2),- url: self.get_hash(&doc, &FastField::UrlHash1, &FastField::UrlHash2),+ site: self.get_hash(doc, FastField::SiteHash1, FastField::SiteHash2),+ title: self.get_hash(doc, FastField::TitleHash1, FastField::TitleHash2),+ url: self.get_hash(doc, FastField::UrlHash1, FastField::UrlHash2),url_without_tld: self.get_hash(- &doc,- &FastField::UrlWithoutTldHash1,- &FastField::UrlWithoutTldHash2,+ doc,+ FastField::UrlWithoutTldHash1,+ FastField::UrlWithoutTldHash2,),simhash: simhash.unwrap(),},@@ -382,7 +386,7 @@ impl<TScoreTweaker> Collector for TweakedScoreTopCollector<TScoreTweaker>whereTScoreTweaker: ScoreTweaker<Score> + Send + Sync,{- type Fruit = Vec<WebsitePointer>;+ type Fruit = Vec<WebpagePointer>;type Child = TopTweakedScoreSegmentCollector<TScoreTweaker::Child>;@@ -426,7 +430,7 @@ where.into_sorted_vec(self.top_docs.de_rank_similar).into_iter().skip(self.top_docs.offset)- .map(|doc| WebsitePointer {+ .map(|doc| WebpagePointer {score: doc.score,hashes: doc.hashes,address: DocAddress {diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex 58a305a4..3815ab11 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -169,3 +169,11 @@ impl Widgets {true}}++pub struct Indexing;++impl Indexing {+ pub fn batch_size() -> usize {+ 512+ }+}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex ffd4a7a1..cafecd78 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -30,7 +30,6 @@ pub struct IndexingLocalConfig {pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,pub warc_source: WarcSource,- pub batch_size: Option<usize>,pub page_webgraph_path: Option<String>,pub host_centrality_threshold: Option<f64>,pub topics_path: Option<String>,@@ -38,6 +37,20 @@ pub struct IndexingLocalConfig {pub page_centrality_store_path: Option<String>,pub safety_classifier_path: Option<String>,pub minimum_clean_words: Option<usize>,++ #[serde(default = "defaults::Indexing::batch_size")]+ pub batch_size: usize,++ pub dual_encoder: Option<IndexingDualEncoderConfig>,+}++#[derive(Debug, Deserialize, Clone)]+pub struct IndexingDualEncoderConfig {+ pub model_path: String,++ /// Only compute embeddings for pages that has a+ /// centrality rank less than this threshold+ pub page_centrality_rank_threshold: Option<u64>,}#[derive(Debug, Deserialize, Clone)]@@ -201,6 +214,7 @@ pub struct ApiConfig {pub prometheus_host: SocketAddr,pub crossencoder_model_path: Option<String>,pub lambda_model_path: Option<String>,+ pub dual_encoder_model_path: Option<String>,pub spell_checker_path: Option<String>,pub bangs_path: String,pub query_store_db_host: Option<String>,@@ -272,6 +286,7 @@ pub struct SearchServerConfig {pub host_centrality_store_path: Option<String>,pub linear_model_path: Option<String>,pub lambda_model_path: Option<String>,+ pub dual_encoder_model_path: Option<String>,pub host: SocketAddr,#[serde(default)]diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rsindex acb1d44f..b9e3a893 100644--- a/crates/core/src/entrypoint/centrality.rs+++ b/crates/core/src/entrypoint/centrality.rs@@ -63,7 +63,8 @@ impl Centrality {}store.flush();- let rank_store = RocksDbStore::open(base_output.as_ref().join("harmonic_rank"));+ let rank_store: RocksDbStore<crate::webgraph::NodeID, u64> =+ RocksDbStore::open(base_output.as_ref().join("harmonic_rank"));let mut top_harmonics = Vec::new();for (rank, node, centrality) in ExternalSorter::new().with_chunk_size(100_000_000)@@ -78,7 +79,7 @@ impl Centrality {(rank, node_id, centrality)}){- rank_store.insert(node, rank as f64);+ rank_store.insert(node, rank as u64);if top_harmonics.len() < 1_000_000 {top_harmonics.push((graph.id2node(&node).unwrap(), centrality));@@ -110,7 +111,8 @@ impl Centrality {let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();let approx = ApproxHarmonic::build(&graph, base_output.as_ref().join("approx_harmonic"));- let approx_rank = RocksDbStore::open(base_output.as_ref().join("approx_harmonic_rank"));+ let approx_rank: RocksDbStore<crate::webgraph::NodeID, u64> =+ RocksDbStore::open(base_output.as_ref().join("approx_harmonic_rank"));let mut top_nodes = Vec::new();@@ -126,7 +128,7 @@ impl Centrality {(rank, node_id, centrality)}){- approx_rank.insert(node, rank as f64);+ approx_rank.insert(node, rank as u64);if top_nodes.len() < 1_000_000 {top_nodes.push((graph.id2node(&node).unwrap(), centrality));}diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex 1dac8629..1551979d 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -19,7 +19,9 @@ use tokio::io;use tokio_stream::StreamExt;use tracing::{debug, info};-use crate::config::{LocalConfig, WebSpellConfig};+use crate::config::{+ defaults, IndexingDualEncoderConfig, IndexingLocalConfig, LocalConfig, WebSpellConfig,+};use crate::entrypoint::indexer::JobSettings;use crate::entrypoint::{dmoz_parser, indexer};use crate::Result;@@ -169,33 +171,46 @@ fn create_inverted_index() -> Result<()> {folder: ".".to_string(),names: vec![warc_path.to_str().unwrap().to_string()],}),- warc_paths: vec![warc_path.to_str().unwrap().to_string()],+ warc_path: warc_path.to_str().unwrap().to_string(),base_path: out_path_tmp.to_str().unwrap().to_string(),settings: JobSettings {host_centrality_threshold: None,minimum_clean_words: None,+ batch_size: defaults::Indexing::batch_size(),},};let webgraph_path = Path::new(DATA_PATH).join("webgraph_page");let centrality_path = Path::new(DATA_PATH).join("centrality");let page_centrality_path = Path::new(DATA_PATH).join("centrality_page");+ let dual_encoder_path = Path::new(DATA_PATH).join("summarizer").join("dual_encoder");- let worker = indexer::IndexingWorker::new(- centrality_path.to_str().unwrap().to_string(),- Some(page_centrality_path.to_str().unwrap().to_string()),- Some(webgraph_path.to_str().unwrap().to_string()),- Some(+ let worker = indexer::IndexingWorker::new(IndexingLocalConfig {+ host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),+ page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),+ page_webgraph_path: Some(webgraph_path.to_str().unwrap().to_string()),+ topics_path: Some(Path::new(DATA_PATH).join("human_annotations").to_str().unwrap().to_string(),),- None,- );+ output_path: out_path_tmp.to_str().unwrap().to_string(),+ limit_warc_files: None,+ skip_warc_files: None,+ warc_source: job.source_config.clone(),+ host_centrality_threshold: None,+ safety_classifier_path: None,+ minimum_clean_words: None,+ batch_size: defaults::Indexing::batch_size(),+ dual_encoder: Some(IndexingDualEncoderConfig {+ model_path: dual_encoder_path.to_str().unwrap().to_string(),+ page_centrality_rank_threshold: Some(100_000),+ }),+ });- let index = indexer::process_job(&job, &worker);+ let index = job.process(&worker);std::fs::rename(index.path, out_path)?;std::fs::remove_dir_all(&out_path_tmp)?;diff --git a/crates/core/src/entrypoint/indexer.rs b/crates/core/src/entrypoint/indexer.rsdeleted file mode 100644index d63b5f9a..00000000--- a/crates/core/src/entrypoint/indexer.rs+++ /dev/null@@ -1,460 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.-use anyhow::anyhow;-use chrono::Utc;-use rayon::prelude::*;-use std::path::Path;-use std::thread;--use itertools::Itertools;-use serde::{Deserialize, Serialize};-use tokio::pin;-use tracing::{debug, info, trace, warn};--use crate::config::{self, WarcSource};-use crate::entrypoint::download_all_warc_files;-use crate::index::Index;-use crate::kv::rocksdb_store::RocksDbStore;-use crate::kv::Kv;-use crate::mapreduce::{Map, Reduce, Worker};-use crate::ranking::SignalAggregator;-use crate::warc::PayloadType;-use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};-use crate::webpage::{safety_classifier, Html, Webpage};-use crate::{human_website_annotations, Result};--#[derive(Debug, Serialize, Deserialize)]-pub struct Job {- pub source_config: config::WarcSource,- pub warc_paths: Vec<String>,- pub base_path: String,- pub settings: JobSettings,-}--#[derive(Debug, Clone, Copy, Serialize, Deserialize)]-pub struct JobSettings {- pub host_centrality_threshold: Option<f64>,- pub minimum_clean_words: Option<usize>,-}--pub struct IndexingWorker {- host_centrality_store: RocksDbStore<NodeID, f64>,- host_centrality_rank_store: RocksDbStore<NodeID, f64>,- page_centrality_store: Option<RocksDbStore<NodeID, f64>>,- page_centrality_rank_store: Option<RocksDbStore<NodeID, f64>>,- page_webgraph: Option<Webgraph>,- topics: Option<human_website_annotations::Mapper>,- safety_classifier: Option<safety_classifier::Model>,- job_settings: Option<JobSettings>,-}--impl IndexingWorker {- pub fn new(- host_centrality_store_path: String,- page_centrality_store_path: Option<String>,- page_webgraph_path: Option<String>,- topics_path: Option<String>,- safety_classifier_path: Option<String>,- ) -> Self {- Self {- host_centrality_store: RocksDbStore::open(- Path::new(&host_centrality_store_path).join("harmonic"),- ),- host_centrality_rank_store: RocksDbStore::open(- Path::new(&host_centrality_store_path).join("harmonic_rank"),- ),- page_centrality_store: page_centrality_store_path- .as_ref()- .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic"))),- page_centrality_rank_store: page_centrality_store_path- .as_ref()- .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic_rank"))),- page_webgraph: page_webgraph_path- .map(|path| WebgraphBuilder::new(path).single_threaded().open()),- topics: topics_path.map(|path| human_website_annotations::Mapper::open(path).unwrap()),- safety_classifier: safety_classifier_path- .map(|path| safety_classifier::Model::open(path).unwrap()),- job_settings: None,- }- }-- pub fn set_job_settings(&mut self, job_settings: JobSettings) {- self.job_settings = Some(job_settings);- }-- pub fn prepare_webpage(&self, body: &str, url: &str, fetch_time_ms: u64) -> Result<Webpage> {- let mut html = match Html::parse_without_text(body, url) {- Ok(html) => html,- Err(err) => {- debug!("error parsing html: {:?}", err);- return Err(anyhow!("error parsing html: {:?}", err));- }- };-- if html.is_no_index() {- return Err(anyhow!("noindex"));- }-- let title = html.title().unwrap_or_default();- if title.is_empty() || title.chars().all(|c| c.is_whitespace()) {- return Err(anyhow!("empty title"));- }-- let node = Node::from(html.url());- let host_node_id = node.clone().into_host().id();-- let mut host_centrality = self- .host_centrality_store- .get(&host_node_id)- .unwrap_or_default();-- let mut host_centrality_rank = self- .host_centrality_rank_store- .get(&host_node_id)- .unwrap_or(u64::MAX as f64);-- if let Some(host_centrality_threshold) =- self.job_settings.and_then(|s| s.host_centrality_threshold)- {- if host_centrality < host_centrality_threshold {- debug!("skipping due to low host_centrality value");- return Err(anyhow!("low host_centrality value"));- }- }-- html.parse_text();-- if html.empty_all_text() {- return Err(anyhow!("empty all text"));- }-- if let Some(minimum_clean_words) = self.job_settings.and_then(|s| s.minimum_clean_words) {- match html.clean_text() {- Some(clean_text) => {- if clean_text.split_whitespace().count() < minimum_clean_words {- return Err(anyhow!("too few clean words"));- }- }- None => return Err(anyhow!("no clean text")),- }- }-- let backlink_labels: Vec<String> = self- .page_webgraph- .as_ref()- .map(|webgraph| {- webgraph- .raw_ingoing_edges_with_labels(&Node::from(html.url()).id())- .into_iter()- .map(|edge| edge.label)- .filter(|label| !label.is_empty())- .filter(|label| {- let label = label.to_lowercase();- let stopwords = [- "click",- "click here",- "here",- "link",- "website",- "webpage",- "page",- "site",- "url",- "web",- "visit",- "more",- "info",- "information",- "read",- "read more",- ];-- !stopwords.contains(&label.as_str())- })- .collect()- })- .unwrap_or_default();-- let mut page_centrality = 0.0;-- if let Some(store) = self.page_centrality_store.as_ref() {- let node_id = node.id();-- page_centrality = store.get(&node_id).unwrap_or_default();- }-- let mut page_centrality_rank = u64::MAX as f64;-- if let Some(store) = self.page_centrality_rank_store.as_ref() {- let node_id = node.id();-- page_centrality_rank = store.get(&node_id).unwrap_or(u64::MAX as f64);- }-- if !page_centrality.is_finite() {- page_centrality = 0.0;- }-- if !host_centrality.is_finite() {- host_centrality = 0.0;- }-- if !page_centrality_rank.is_finite() {- page_centrality_rank = u64::MAX as f64;- }-- if !host_centrality_rank.is_finite() {- host_centrality_rank = u64::MAX as f64;- }-- let mut dmoz_description = None;-- if let Some(mapper) = self.topics.as_ref() {- if let Some(info) = mapper.get(&html.url().host_str().unwrap_or_default().to_string()) {- dmoz_description = Some(info.description.clone())- }- }-- let mut webpage = Webpage {- html,- backlink_labels,- page_centrality,- page_centrality_rank,- host_centrality,- host_centrality_rank,- fetch_time_ms,- pre_computed_score: 0.0,- node_id: Some(host_node_id),- dmoz_description,- safety_classification: None,- inserted_at: Utc::now(),- };-- if let Some(model) = self.safety_classifier.as_ref() {- webpage.safety_classification = Some(model.predict(&webpage).label);- }-- let mut signal_aggregator = SignalAggregator::new(None);- signal_aggregator.set_current_timestamp(Utc::now().timestamp().max(0) as usize);-- webpage.pre_computed_score = signal_aggregator.precompute_score(&webpage);-- Ok(webpage)- }-}--pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {- let name = job.warc_paths.first().unwrap().split('/').last().unwrap();-- let mut has_host_centrality = false;- let mut has_page_centrality = false;- let mut has_backlinks = false;-- info!("processing {}", name);-- let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();- index.prepare_writer().unwrap();-- let warc_files = download_all_warc_files(&job.warc_paths, &job.source_config);- pin!(warc_files);-- for file in warc_files.by_ref() {- for record in- file.records()- .flatten()- .filter(|record| match &record.response.payload_type {- Some(payload_type) => matches!(payload_type, PayloadType::Html),- None => true,- })- {- if let Ok(webpage) = worker.prepare_webpage(- &record.response.body,- &record.request.url,- record.metadata.fetch_time_ms,- ) {- if webpage.host_centrality > 0.0 {- has_host_centrality = true;- }-- if webpage.page_centrality > 0.0 {- has_page_centrality = true;- }-- if !webpage.backlink_labels.is_empty() {- has_backlinks = true;- }- trace!("inserting webpage: {:?}", webpage.html.url());- trace!("title = {:?}", webpage.html.title());- trace!("text = {:?}", webpage.html.clean_text());-- if let Err(err) = index.insert(webpage) {- warn!("{:?}", err);- panic!();- }- }- }-- index.commit().unwrap();- }-- if !has_host_centrality {- warn!("no host centrality values found in {}", name);- }-- if !has_page_centrality && worker.page_centrality_store.is_some() {- warn!("no page centrality values found in {}", name);- }-- if !has_backlinks && worker.page_webgraph.is_some() {- warn!("no backlinks found in {}", name);- }-- index.inverted_index.merge_into_max_segments(1).unwrap();-- info!("{} done", name);-- index-}--#[derive(Debug, Serialize, Deserialize)]-pub struct IndexPointer(String);--impl From<String> for IndexPointer {- fn from(path: String) -> Self {- IndexPointer(path)- }-}--impl Worker for IndexingWorker {}--impl Map<IndexingWorker, IndexPointer> for Job {- fn map(&self, worker: &IndexingWorker) -> IndexPointer {- let index = process_job(self, worker);- IndexPointer(index.path)- }-}--impl Reduce<Index> for Index {- fn reduce(self, element: Index) -> Self {- let other = element;- let other_path = other.path.clone();-- let res = self.merge(other);-- std::fs::remove_dir_all(other_path).unwrap();-- res- }-}--pub struct Indexer {}-impl Indexer {- pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {- let warc_paths = config.warc_source.paths()?;-- let job_config: WarcSource = config.warc_source.clone();-- let worker = IndexingWorker::new(- config.host_centrality_store_path.clone(),- config.page_centrality_store_path.clone(),- config.page_webgraph_path.clone(),- config.topics_path.clone(),- config.safety_classifier_path.clone(),- );-- let indexes = warc_paths- .into_iter()- .skip(config.skip_warc_files.unwrap_or(0))- .take(config.limit_warc_files.unwrap_or(usize::MAX))- .chunks(config.batch_size.unwrap_or(1))- .into_iter()- .map(|paths| paths.collect_vec())- .collect_vec()- .into_par_iter()- .map(|warc_paths| Job {- source_config: job_config.clone(),- warc_paths,- base_path: config.output_path.clone(),- settings: JobSettings {- host_centrality_threshold: config.host_centrality_threshold,- minimum_clean_words: config.minimum_clean_words,- },- })- .map(|job| {- let pointer: IndexPointer = job.map(&worker);- pointer- })- .collect();-- Self::merge(indexes)?;- Ok(())- }-- pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {- let num_indexes = indexes.len();- let mut it = indexes.into_iter();- let num_cores = num_cpus::get();-- let mut threads = Vec::new();-- for _ in 0..(num_cores + 1) {- let indexes = it- .by_ref()- .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)- .collect_vec();-- if indexes.is_empty() {- break;- }-- threads.push(thread::spawn(move || {- let mut it = indexes.into_iter();- let mut index = Index::open(it.next().unwrap().0).unwrap();-- for other in it {- let other_path = other.0;- let other = Index::open(&other_path).unwrap();-- index = index.merge(other);-- std::fs::remove_dir_all(other_path).unwrap();- }-- index.inverted_index.merge_into_max_segments(1).unwrap();-- index- }));- }-- let mut indexes = Vec::new();- for thread in threads {- indexes.push(thread.join().unwrap());- }-- let mut it = indexes.into_iter();- let mut index = it.next().unwrap();-- for other in it {- let other_path = other.path.clone();- index = index.merge(other);- std::fs::remove_dir_all(other_path).unwrap();- }-- index.inverted_index.merge_into_max_segments(1).unwrap();-- Ok(())- }-}diff --git a/crates/core/src/entrypoint/indexer/indexable_webpage.rs b/crates/core/src/entrypoint/indexer/indexable_webpage.rsnew file mode 100644index 00000000..290d8ff9--- /dev/null+++ b/crates/core/src/entrypoint/indexer/indexable_webpage.rs@@ -0,0 +1,44 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::crawler::CrawlDatum;+use crate::warc::WarcRecord;++pub struct IndexableWebpage {+ pub url: String,+ pub body: String,+ pub fetch_time_ms: u64,+}++impl From<CrawlDatum> for IndexableWebpage {+ fn from(datum: CrawlDatum) -> Self {+ Self {+ url: datum.url.to_string(),+ body: datum.body,+ fetch_time_ms: datum.fetch_time_ms,+ }+ }+}++impl From<WarcRecord> for IndexableWebpage {+ fn from(record: WarcRecord) -> Self {+ Self {+ url: record.request.url,+ body: record.response.body,+ fetch_time_ms: record.metadata.fetch_time_ms,+ }+ }+}diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rsnew file mode 100644index 00000000..a75b5dc0--- /dev/null+++ b/crates/core/src/entrypoint/indexer/job.rs@@ -0,0 +1,127 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::path::Path;++use itertools::Itertools;+use serde::{Deserialize, Serialize};+use tokio::pin;+use tracing::{info, trace, warn};++use crate::config;+use crate::entrypoint::download_all_warc_files;+use crate::index::Index;+use crate::warc::PayloadType;++use super::{IndexableWebpage, IndexingWorker};++#[derive(Debug, Serialize, Deserialize)]+pub struct Job {+ pub source_config: config::WarcSource,+ pub warc_path: String,+ pub base_path: String,+ pub settings: JobSettings,+}++#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+pub struct JobSettings {+ pub host_centrality_threshold: Option<f64>,+ pub minimum_clean_words: Option<usize>,+ pub batch_size: usize,+}++impl Job {+ pub fn process(&self, worker: &IndexingWorker) -> Index {+ let name = self.warc_path.split('/').last().unwrap();++ let mut has_host_centrality = false;+ let mut has_page_centrality = false;+ let mut has_backlinks = false;++ info!("processing {}", name);++ let mut index = Index::open(Path::new(&self.base_path).join(name)).unwrap();+ index.prepare_writer().unwrap();++ let paths = vec![self.warc_path.clone()];+ let warc_files = download_all_warc_files(&paths, &self.source_config);+ pin!(warc_files);++ for file in warc_files.by_ref() {+ let mut batch = Vec::with_capacity(self.settings.batch_size);++ for chunk in &file+ .records()+ .flatten()+ .filter(|record| match &record.response.payload_type {+ Some(payload_type) => matches!(payload_type, PayloadType::Html),+ None => true,+ })+ .chunks(self.settings.batch_size)+ {+ batch.clear();++ for record in chunk {+ batch.push(IndexableWebpage::from(record));+ }++ let prepared = worker.prepare_webpages(&batch);++ for webpage in &prepared {+ if webpage.host_centrality > 0.0 {+ has_host_centrality = true;+ }++ if webpage.page_centrality > 0.0 {+ has_page_centrality = true;+ }++ if !webpage.backlink_labels.is_empty() {+ has_backlinks = true;+ }+ trace!("inserting webpage: {:?}", webpage.html.url());+ trace!("title = {:?}", webpage.html.title());+ trace!("text = {:?}", webpage.html.clean_text());++ if let Err(err) = index.insert(webpage) {+ warn!("{:?}", err);+ panic!();+ }+ }+ }++ index.commit().unwrap();+ }++ if !has_host_centrality {+ warn!("no host centrality values found in {}", name);+ }++ if !has_page_centrality && worker.page_centrality_store().is_some() {+ warn!("no page centrality values found in {}", name);+ }++ if !has_backlinks && worker.page_webgraph().is_some() {+ warn!("no backlinks found in {}", name);+ }++ index.inverted_index.merge_into_max_segments(1).unwrap();++ info!("{} done", name);++ index+ }+}diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rsnew file mode 100644index 00000000..9693cea0--- /dev/null+++ b/crates/core/src/entrypoint/indexer/mod.rs@@ -0,0 +1,151 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub mod indexable_webpage;+pub mod job;+pub mod worker;++use rayon::prelude::*;+use std::thread;++use itertools::Itertools;+use serde::{Deserialize, Serialize};++pub use crate::entrypoint::indexer::indexable_webpage::IndexableWebpage;+pub use crate::entrypoint::indexer::job::{Job, JobSettings};+pub use crate::entrypoint::indexer::worker::IndexingWorker;++use crate::config::{self, WarcSource};+use crate::index::Index;+use crate::mapreduce::{Map, Reduce, Worker};+use crate::Result;++#[derive(Debug, Serialize, Deserialize)]+pub struct IndexPointer(String);++impl From<String> for IndexPointer {+ fn from(path: String) -> Self {+ IndexPointer(path)+ }+}++impl Worker for IndexingWorker {}++impl Map<IndexingWorker, IndexPointer> for Job {+ fn map(&self, worker: &IndexingWorker) -> IndexPointer {+ let index = self.process(worker);+ IndexPointer(index.path)+ }+}++impl Reduce<Index> for Index {+ fn reduce(self, element: Index) -> Self {+ let other = element;+ let other_path = other.path.clone();++ let res = self.merge(other);++ std::fs::remove_dir_all(other_path).unwrap();++ res+ }+}++pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {+ let warc_paths = config.warc_source.paths()?;++ let job_config: WarcSource = config.warc_source.clone();++ let worker = IndexingWorker::new(config.clone());++ let indexes = warc_paths+ .into_par_iter()+ .skip(config.skip_warc_files.unwrap_or(0))+ .take(config.limit_warc_files.unwrap_or(usize::MAX))+ .map(|warc_path| Job {+ source_config: job_config.clone(),+ warc_path,+ base_path: config.output_path.clone(),+ settings: JobSettings {+ host_centrality_threshold: config.host_centrality_threshold,+ minimum_clean_words: config.minimum_clean_words,+ batch_size: config.batch_size,+ },+ })+ .map(|job| {+ let pointer: IndexPointer = job.map(&worker);+ pointer+ })+ .collect();++ merge(indexes)?;+ Ok(())+}++pub fn merge(indexes: Vec<IndexPointer>) -> Result<()> {+ let num_indexes = indexes.len();+ let mut it = indexes.into_iter();+ let num_cores = num_cpus::get();++ let mut threads = Vec::new();++ for _ in 0..(num_cores + 1) {+ let indexes = it+ .by_ref()+ .take(((num_indexes as f64) / (num_cores as f64)).ceil() as usize)+ .collect_vec();++ if indexes.is_empty() {+ break;+ }++ threads.push(thread::spawn(move || {+ let mut it = indexes.into_iter();+ let mut index = Index::open(it.next().unwrap().0).unwrap();++ for other in it {+ let other_path = other.0;+ let other = Index::open(&other_path).unwrap();++ index = index.merge(other);++ std::fs::remove_dir_all(other_path).unwrap();+ }++ index.inverted_index.merge_into_max_segments(1).unwrap();++ index+ }));+ }++ let mut indexes = Vec::new();+ for thread in threads {+ indexes.push(thread.join().unwrap());+ }++ let mut it = indexes.into_iter();+ let mut index = it.next().unwrap();++ for other in it {+ let other_path = other.path.clone();+ index = index.merge(other);+ std::fs::remove_dir_all(other_path).unwrap();+ }++ index.inverted_index.merge_into_max_segments(1).unwrap();++ Ok(())+}diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsnew file mode 100644index 00000000..acbc6235--- /dev/null+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -0,0 +1,608 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use chrono::Utc;+use std::path::Path;+use url::Url;++use tracing::debug;++pub use super::indexable_webpage::IndexableWebpage;+pub use super::job::{Job, JobSettings};+use crate::config::{IndexingDualEncoderConfig, IndexingLocalConfig, LiveIndexConfig};+use crate::models::dual_encoder::DualEncoder as DualEncoderModel;+use crate::Result;++use crate::human_website_annotations;+use crate::index::Index;+use crate::kv::rocksdb_store::RocksDbStore;+use crate::kv::Kv;+use crate::rake::RakeModel;+use crate::ranking::SignalAggregator;+use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};+use crate::webpage::{safety_classifier, Html, Webpage};++pub struct Config {+ pub host_centrality_store_path: String,+ pub page_centrality_store_path: Option<String>,+ pub page_webgraph_path: Option<String>,+ pub topics_path: Option<String>,+ pub safety_classifier_path: Option<String>,+ pub dual_encoder: Option<IndexingDualEncoderConfig>,+}++impl From<IndexingLocalConfig> for Config {+ fn from(config: IndexingLocalConfig) -> Self {+ Self {+ host_centrality_store_path: config.host_centrality_store_path,+ page_centrality_store_path: config.page_centrality_store_path,+ page_webgraph_path: config.page_webgraph_path,+ topics_path: config.topics_path,+ safety_classifier_path: config.safety_classifier_path,+ dual_encoder: config.dual_encoder,+ }+ }+}++impl From<LiveIndexConfig> for Config {+ fn from(config: LiveIndexConfig) -> Self {+ Self {+ host_centrality_store_path: config.host_centrality_store_path,+ page_centrality_store_path: config.page_centrality_store_path,+ page_webgraph_path: config.page_webgraph_path,+ topics_path: None,+ safety_classifier_path: config.safety_classifier_path,+ dual_encoder: None,+ }+ }+}++struct DualEncoder {+ model: DualEncoderModel,+ page_centrality_rank_threshold: Option<u64>,+}++pub struct IndexingWorker {+ host_centrality_store: RocksDbStore<NodeID, f64>,+ host_centrality_rank_store: RocksDbStore<NodeID, u64>,+ page_centrality_store: Option<RocksDbStore<NodeID, f64>>,+ page_centrality_rank_store: Option<RocksDbStore<NodeID, u64>>,+ page_webgraph: Option<Webgraph>,+ topics: Option<human_website_annotations::Mapper>,+ safety_classifier: Option<safety_classifier::Model>,+ job_settings: Option<JobSettings>,+ rake: RakeModel,+ dual_encoder: Option<DualEncoder>,+}++impl IndexingWorker {+ pub fn new<C>(config: C) -> Self+ where+ Config: From<C>,+ {+ let config = Config::from(config);++ Self {+ host_centrality_store: RocksDbStore::open(+ Path::new(&config.host_centrality_store_path).join("harmonic"),+ ),+ host_centrality_rank_store: RocksDbStore::open(+ Path::new(&config.host_centrality_store_path).join("harmonic_rank"),+ ),+ page_centrality_store: config+ .page_centrality_store_path+ .as_ref()+ .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic"))),+ page_centrality_rank_store: config+ .page_centrality_store_path+ .as_ref()+ .map(|p| RocksDbStore::open(Path::new(&p).join("approx_harmonic_rank"))),+ page_webgraph: config+ .page_webgraph_path+ .as_ref()+ .map(|path| WebgraphBuilder::new(path).single_threaded().open()),+ topics: config+ .topics_path+ .as_ref()+ .map(|path| human_website_annotations::Mapper::open(path).unwrap()),+ safety_classifier: config+ .safety_classifier_path+ .as_ref()+ .map(|path| safety_classifier::Model::open(path).unwrap()),+ job_settings: None,+ rake: RakeModel::default(),+ dual_encoder: config.dual_encoder.as_ref().map(|dual_encoder| {+ let model =+ DualEncoderModel::open(&dual_encoder.model_path).unwrap_or_else(|err| {+ panic!("failed to open dual encoder model: {}", err);+ });++ DualEncoder {+ model,+ page_centrality_rank_threshold: dual_encoder.page_centrality_rank_threshold,+ }+ }),+ }+ }++ pub(super) fn page_centrality_store(&self) -> Option<&RocksDbStore<NodeID, f64>> {+ self.page_centrality_store.as_ref()+ }++ pub(super) fn page_webgraph(&self) -> Option<&Webgraph> {+ self.page_webgraph.as_ref()+ }++ pub fn process(&self, job: &Job) -> Index {+ job.process(self)+ }++ pub fn set_job_settings(&mut self, job_settings: JobSettings) {+ self.job_settings = Some(job_settings);+ }++ fn prepare(&self, page: &IndexableWebpage) -> Result<Webpage> {+ let html = match Html::parse_without_text(&page.body, &page.url) {+ Ok(html) => html,+ Err(err) => {+ return Err(anyhow::anyhow!("error parsing html: {:?}", err));+ }+ };++ if html.is_no_index() {+ return Err(anyhow::anyhow!("noindex"));+ }++ let title = html.title().unwrap_or_default();+ if title.is_empty() || title.chars().all(|c| c.is_whitespace()) {+ return Err(anyhow::anyhow!("empty title"));+ }++ Ok(Webpage::from(html))+ }++ fn set_host_centrality(&self, page: &mut Webpage) -> Result<()> {+ let node = Node::from(page.html.url());+ let host_node_id = node.clone().into_host().id();++ let host_centrality = self+ .host_centrality_store+ .get(&host_node_id)+ .unwrap_or_default();++ let host_centrality_rank = self+ .host_centrality_rank_store+ .get(&host_node_id)+ .unwrap_or(u64::MAX);++ if let Some(host_centrality_threshold) =+ self.job_settings.and_then(|s| s.host_centrality_threshold)+ {+ if host_centrality < host_centrality_threshold {+ return Err(anyhow::anyhow!("low host_centrality value"));+ }+ }++ page.node_id = Some(host_node_id);++ page.host_centrality = host_centrality;+ page.host_centrality_rank = host_centrality_rank;++ if !page.host_centrality.is_finite() {+ page.host_centrality = 0.0;+ }++ self.parse_text(page)?;++ Ok(())+ }++ fn parse_text(&self, page: &mut Webpage) -> Result<()> {+ page.html.parse_text();++ if page.html.empty_all_text() {+ return Err(anyhow::anyhow!("empty all text"));+ }++ if let Some(minimum_clean_words) = self.job_settings.and_then(|s| s.minimum_clean_words) {+ match page.html.clean_text() {+ Some(clean_text) => {+ if clean_text.split_whitespace().count() < minimum_clean_words {+ return Err(anyhow::anyhow!("too few clean words"));+ }+ }+ None => {+ return Err(anyhow::anyhow!("no clean text"));+ }+ }+ }++ Ok(())+ }++ fn backlink_labels(&self, page: &Url) -> Vec<String> {+ self.page_webgraph+ .as_ref()+ .map(|webgraph| {+ webgraph+ .raw_ingoing_edges_with_labels(&Node::from(page).id())+ .into_iter()+ .map(|edge| edge.label)+ .filter(|label| !label.is_empty())+ .filter(|label| {+ let label = label.to_lowercase();+ let stopwords = [+ "click",+ "click here",+ "here",+ "link",+ "website",+ "webpage",+ "page",+ "site",+ "url",+ "web",+ "visit",+ "more",+ "info",+ "information",+ "read",+ "read more",+ ];++ !stopwords.contains(&label.as_str())+ })+ .collect()+ })+ .unwrap_or_default()+ }++ fn set_page_centralities(&self, page: &mut Webpage) {+ let node = Node::from(page.html.url());++ if let Some(store) = self.page_centrality_store.as_ref() {+ let node_id = node.id();++ page.page_centrality = store.get(&node_id).unwrap_or_default();+ }++ page.page_centrality_rank = u64::MAX;++ if let Some(store) = self.page_centrality_rank_store.as_ref() {+ let node_id = node.id();++ page.page_centrality_rank = store.get(&node_id).unwrap_or(u64::MAX);+ }++ if !page.page_centrality.is_finite() {+ page.page_centrality = 0.0;+ }+ }++ fn set_dmoz_description(&self, page: &mut Webpage) {+ if let Some(mapper) = self.topics.as_ref() {+ if let Some(info) =+ mapper.get(&page.html.url().host_str().unwrap_or_default().to_string())+ {+ page.dmoz_description = Some(info.description.clone())+ }+ }+ }++ fn set_keywords(&self, page: &mut Webpage) {+ page.keywords = page.html.keywords(&self.rake);+ }++ fn set_safety_classification(&self, page: &mut Webpage) {+ if let Some(model) = self.safety_classifier.as_ref() {+ page.safety_classification = Some(model.predict(page).label);+ }+ }++ pub fn set_title_embeddings(&self, pages: &mut [Webpage]) {+ if let Some(dual_encoder) = self.dual_encoder.as_ref() {+ let (page_indexes, titles): (Vec<_>, Vec<_>) = pages+ .iter()+ .enumerate()+ .filter(|(_, w)| {+ dual_encoder+ .page_centrality_rank_threshold+ .map(|thresh| w.page_centrality_rank <= thresh)+ .unwrap_or(true)+ })+ .map(|(i, w)| (i, w.html.title().unwrap_or_default()))+ .unzip();++ let title_emb = dual_encoder+ .model+ .embed(&titles)+ .ok()+ .and_then(|t| t.to_dtype(candle_core::DType::BF16).ok());++ if let Some(title_emb) = title_emb {+ for (i, page_index) in page_indexes.into_iter().enumerate() {+ if let Ok(emb) = title_emb.get(i) {+ pages[page_index].title_embedding = Some(emb);+ }+ }+ }+ }+ }++ pub fn set_keyword_embeddings(&self, pages: &mut [Webpage]) {+ if let Some(dual_encoder) = self.dual_encoder.as_ref() {+ let (page_indexes, keywords): (Vec<_>, Vec<_>) = pages+ .iter()+ .enumerate()+ .filter(|(_, w)| {+ dual_encoder+ .page_centrality_rank_threshold+ .map(|thresh| w.page_centrality_rank <= thresh)+ .unwrap_or(true)+ })+ .map(|(i, w)| (i, w.keywords.join("\n")))+ .unzip();++ let keyword_emb = dual_encoder+ .model+ .embed(&keywords)+ .ok()+ .and_then(|t| t.to_dtype(candle_core::DType::BF16).ok());++ if let Some(keyword_emb) = keyword_emb {+ for (i, page_index) in page_indexes.into_iter().enumerate() {+ if let Ok(emb) = keyword_emb.get(i) {+ pages[page_index].keyword_embedding = Some(emb);+ }+ }+ }+ }+ }++ pub fn prepare_webpages(&self, batch: &[IndexableWebpage]) -> Vec<Webpage> {+ let mut res = Vec::with_capacity(batch.len());+ let mut signal_aggregator = SignalAggregator::new(None);++ for page in batch {+ let mut prepared = match self.prepare(page) {+ Ok(html) => html,+ Err(err) => {+ debug!("skipping webpage: {}", err);+ continue;+ }+ };+ if let Err(e) = self.set_host_centrality(&mut prepared) {+ debug!("skipping webpage: {}", e);+ continue;+ }++ prepared.backlink_labels = self.backlink_labels(prepared.html.url());++ self.set_page_centralities(&mut prepared);+ self.set_dmoz_description(&mut prepared);+ self.set_keywords(&mut prepared);+ self.set_safety_classification(&mut prepared);++ // make sure we remember to set everything+ let mut webpage = Webpage {+ html: prepared.html,+ backlink_labels: prepared.backlink_labels,+ page_centrality: prepared.page_centrality,+ page_centrality_rank: prepared.page_centrality_rank,+ host_centrality: prepared.host_centrality,+ host_centrality_rank: prepared.host_centrality_rank,+ fetch_time_ms: page.fetch_time_ms,+ pre_computed_score: 0.0,+ node_id: prepared.node_id,+ dmoz_description: prepared.dmoz_description,+ safety_classification: prepared.safety_classification,+ inserted_at: Utc::now(),+ keywords: prepared.keywords,+ title_embedding: None, // set later+ keyword_embedding: None, // set later+ };++ signal_aggregator.set_current_timestamp(Utc::now().timestamp().max(0) as usize);+ webpage.pre_computed_score = signal_aggregator.precompute_score(&webpage);++ res.push(webpage);+ }++ self.set_title_embeddings(&mut res);+ self.set_keyword_embeddings(&mut res);++ res+ }+}++#[cfg(test)]+mod tests {+ use crate::config::WarcSource;++ use super::*;++ fn setup_worker(data_path: &Path, threshold: Option<u64>) -> IndexingWorker {+ IndexingWorker::new(IndexingLocalConfig {+ host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ page_centrality_store_path: None,+ page_webgraph_path: None,+ topics_path: None,+ safety_classifier_path: None,+ dual_encoder: Some(IndexingDualEncoderConfig {+ model_path: data_path.to_str().unwrap().to_string(),+ page_centrality_rank_threshold: threshold,+ }),+ output_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ limit_warc_files: None,+ skip_warc_files: None,+ warc_source: WarcSource::Local(crate::config::LocalConfig {+ folder: crate::gen_temp_path().to_str().unwrap().to_string(),+ names: vec!["".to_string()],+ }),+ host_centrality_threshold: None,+ minimum_clean_words: None,+ batch_size: 10,+ })+ }++ #[test]+ fn title_embeddings() {+ let data_path = Path::new("../../data/summarizer/dual_encoder");+ if !data_path.exists() {+ // Skip the test if the test data is not available+ return;+ }+ let worker = setup_worker(data_path, None);++ let webpages = vec![+ IndexableWebpage {+ url: "https://a.com".to_string(),+ body: "<html><head><title>Homemade Heart Brownie Recipe</title></head><body>Example</body></html>"+ .to_string(),+ fetch_time_ms: 0,+ },+ IndexableWebpage {+ url: "https://b.com".to_string(),+ body: "<html><head><title>How To Use an iMac as a Monitor for a PC</title></head><body>Example</body></html>"+ .to_string(),+ fetch_time_ms: 0,+ },+ ];++ let webpages = worker.prepare_webpages(&webpages);++ assert_eq!(webpages.len(), 2);++ assert_eq!(+ webpages[0].html.title(),+ Some("Homemade Heart Brownie Recipe".to_string())+ );+ assert_eq!(+ webpages[1].html.title(),+ Some("How To Use an iMac as a Monitor for a PC".to_string())+ );++ assert!(webpages.iter().all(|w| w.title_embedding.is_some()));++ let emb1 = webpages[0].title_embedding.as_ref().unwrap();+ let emb2 = webpages[1].title_embedding.as_ref().unwrap();++ let query = "best chocolate cake";+ let query_emb = worker+ .dual_encoder+ .as_ref()+ .unwrap()+ .model+ .embed(&[query.to_string()])+ .unwrap()+ .to_dtype(candle_core::DType::F16)+ .unwrap()+ .get(0)+ .unwrap();++ let sim1: f32 = query_emb+ .unsqueeze(0)+ .unwrap()+ .matmul(+ &emb1+ .to_dtype(candle_core::DType::F16)+ .unwrap()+ .unsqueeze(0)+ .unwrap()+ .t()+ .unwrap(),+ )+ .unwrap()+ .get(0)+ .unwrap()+ .squeeze(0)+ .unwrap()+ .to_dtype(candle_core::DType::F32)+ .unwrap()+ .to_vec0()+ .unwrap();++ let sim2: f32 = query_emb+ .unsqueeze(0)+ .unwrap()+ .matmul(+ &emb2+ .to_dtype(candle_core::DType::F16)+ .unwrap()+ .unsqueeze(0)+ .unwrap()+ .t()+ .unwrap(),+ )+ .unwrap()+ .get(0)+ .unwrap()+ .squeeze(0)+ .unwrap()+ .to_dtype(candle_core::DType::F32)+ .unwrap()+ .to_vec0()+ .unwrap();++ assert!(sim1 > sim2);+ }++ #[test]+ fn title_embedding_ranks() {+ let data_path = Path::new("../../data/summarizer/dual_encoder");+ if !data_path.exists() {+ // Skip the test if the test data is not available+ return;+ }+ let worker = setup_worker(data_path, Some(100_000));++ let mut a = Webpage::test_parse("<html><head><title>Homemade Heart Brownie Recipe</title></head><body>Example</body></html>", "https://a.com").unwrap();+ a.page_centrality_rank = 1;++ let mut b = Webpage::test_parse("<html><head><title>How To Use an iMac as a Monitor for a PC</title></head><body>Example</body></html>", "https://b.com").unwrap();+ b.page_centrality_rank = 1_000_000;++ let mut webpages = vec![a, b];+ worker.set_title_embeddings(&mut webpages);++ assert_eq!(webpages.len(), 2);+ assert_eq!(+ webpages[0].html.title(),+ Some("Homemade Heart Brownie Recipe".to_string())+ );++ assert!(webpages[0].title_embedding.is_some());+ assert!(webpages[1].title_embedding.is_none());++ let mut a = Webpage::test_parse("<html><head><title>Homemade Heart Brownie Recipe</title></head><body>Example</body></html>", "https://a.com").unwrap();+ a.page_centrality_rank = 1_000_000;++ let mut b = Webpage::test_parse("<html><head><title>How To Use an iMac as a Monitor for a PC</title></head><body>Example</body></html>", "https://b.com").unwrap();+ b.page_centrality_rank = 1;++ let mut webpages = vec![a, b];+ worker.set_title_embeddings(&mut webpages);++ assert_eq!(webpages.len(), 2);+ assert_eq!(+ webpages[0].html.title(),+ Some("Homemade Heart Brownie Recipe".to_string())+ );++ assert!(webpages[0].title_embedding.is_none());+ assert!(webpages[1].title_embedding.is_some());+ }+}diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rsindex 3622c20b..8f63e4b6 100644--- a/crates/core/src/entrypoint/mod.rs+++ b/crates/core/src/entrypoint/mod.rs@@ -34,7 +34,6 @@ pub mod webgraph_server;pub use centrality::Centrality;pub use entity::EntityIndexer;-pub use indexer::Indexer;use tracing::{debug, log::error};pub use webgraph::Webgraph;pub mod live_index;diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex b3bab405..864dc12b 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -29,6 +29,7 @@ use crate::{},index::Index,inverted_index::{self, RetrievedWebpage},+ models::dual_encoder::DualEncoder,ranking::{inbound_similarity::InboundSimilarity,models::{lambdamart::LambdaMART, linear::LinearRegression},@@ -75,6 +76,10 @@ impl SearchService {local_searcher.set_lambda_model(LambdaMART::open(model_path)?);}+ if let Some(model_path) = config.dual_encoder_model_path {+ local_searcher.set_dual_encoder(DualEncoder::open(model_path)?);+ }+local_searcher.set_collector_config(config.collector);local_searcher.set_snippet_config(config.snippet);@@ -100,7 +105,7 @@ impl SearchService {#[derive(Debug, Clone, Serialize, Deserialize)]pub struct RetrieveWebsites {- pub websites: Vec<inverted_index::WebsitePointer>,+ pub websites: Vec<inverted_index::WebpagePointer>,pub query: String,}impl sonic::service::Message<SearchService> for RetrieveWebsites {diff --git a/crates/core/src/executor.rs b/crates/core/src/executor.rsindex 597ae069..4a87f251 100644--- a/crates/core/src/executor.rs+++ b/crates/core/src/executor.rs@@ -119,7 +119,7 @@ mod tests {}#[test]- #[should_panic] //< unfortunately the panic message is not propagated+ #[should_panic(expected = "panic should propagate")]fn test_panic_propagates_multi_thread() {let _result: Vec<usize> = Executor::with_threads(1, "search-test").unwrap()diff --git a/crates/core/src/external_sort.rs b/crates/core/src/external_sort.rsindex 1837775b..31f7bc67 100644--- a/crates/core/src/external_sort.rs+++ b/crates/core/src/external_sort.rs@@ -344,7 +344,7 @@ mod tests {data.push(item);}- expected.sort();+ expected.sort_unstable();let sorted = ExternalSorter::new().with_chunk_size(100_000)diff --git a/crates/core/src/fastfield_reader.rs b/crates/core/src/fastfield_reader.rsindex 15de12d6..3098cd12 100644--- a/crates/core/src/fastfield_reader.rs+++ b/crates/core/src/fastfield_reader.rs@@ -37,6 +37,10 @@ impl FastFieldReader {pub fn get_segment(&self, segment: &SegmentId) -> Arc<SegmentReader> {Arc::clone(self.inner.segments.get(segment).unwrap())}++ pub fn borrow_segment(&self, segment: &SegmentId) -> &SegmentReader {+ self.inner.segments.get(segment).unwrap()+ }}impl FastFieldReader {@@ -46,41 +50,28 @@ impl FastFieldReader {for reader in tv_searcher.segment_readers() {let fastfield_readers = reader.fast_fields();- let mut segment_reader_data = Vec::new();-- let mut tv_readers = EnumMap::new();+ let mut u64s = EnumMap::new();+ let mut bytes = EnumMap::new();- for field in Field::all().filter_map(|field| field.as_fast()) {+ for field in Field::all().filter_map(Field::as_fast) {match field.data_type() {DataType::U64 => {- let reader = fastfield_readers.u64(field.name()).unwrap();- tv_readers.insert(field, reader);+ if let Ok(reader) = fastfield_readers.u64(field.name()) {+ u64s.insert(field, reader);+ }}- };- }-- for doc in 0..reader.max_doc() as usize {- let mut data = Vec::new();-- for field in Field::all().filter_map(|field| field.as_fast()) {- match field.data_type() {- DataType::U64 => {- let reader = tv_readers.get(field).unwrap();- data.push((field as usize, reader.values.get_val(doc as u32)));+ DataType::Bytes => {+ if let Some(reader) = fastfield_readers.bytes(field.name()).ok().flatten() {+ bytes.insert(field, reader);}- };- }-- data.sort_by_key(|(a, _)| *a);-- segment_reader_data.extend(data.into_iter().map(|(_, val)| val));+ }+ };}segments.insert(reader.segment_id(),Arc::new(SegmentReader {- data: segment_reader_data,- num_fields: tv_readers.len(),+ field_readers: AllReaders { u64s, bytes },}),);}@@ -91,25 +82,112 @@ impl FastFieldReader {}}+struct AllReaders {+ u64s: EnumMap<FastField, tantivy::columnar::Column<u64>>,+ bytes: EnumMap<FastField, tantivy::columnar::BytesColumn>,+}++pub enum Value {+ U64(u64),+ Bytes(Vec<u8>),+}++impl Value {+ pub fn as_u64(&self) -> Option<u64> {+ match self {+ Value::U64(val) => Some(*val),+ _ => None,+ }+ }++ pub fn as_bytes(&self) -> Option<&[u8]> {+ match self {+ Value::Bytes(val) => Some(val),+ _ => None,+ }+ }+}++impl From<u64> for Value {+ fn from(val: u64) -> Self {+ Value::U64(val)+ }+}++impl From<Vec<u8>> for Value {+ fn from(val: Vec<u8>) -> Self {+ Value::Bytes(val)+ }+}++impl From<Value> for Option<u64> {+ fn from(val: Value) -> Self {+ val.as_u64()+ }+}++impl<'a> From<&'a Value> for Option<&'a [u8]> {+ fn from(val: &'a Value) -> Self {+ val.as_bytes()+ }+}++impl From<Value> for Option<Vec<u8>> {+ fn from(val: Value) -> Self {+ match val {+ Value::Bytes(val) => Some(val),+ _ => None,+ }+ }+}+pub struct FieldReader<'a> {- data: &'a [u64],+ readers: &'a AllReaders,+ doc: DocId,}impl<'a> FieldReader<'a> {- pub fn get(&self, field: &FastField) -> u64 {- self.data[*field as usize]+ pub fn get(&self, field: FastField) -> Option<Value> {+ match field.data_type() {+ DataType::U64 => Some(+ self.readers+ .u64s+ .get(field)?+ .values+ .get_val(self.doc)+ .into(),+ ),++ DataType::Bytes => {+ let reader = self.readers.bytes.get(field)?;+ let ord = reader.ords().values.get_val(self.doc);++ if ord > reader.num_terms() as u64 || reader.num_terms() == 0 {+ return None;+ }++ let mut bytes = Vec::new();+ reader.ord_to_bytes(ord, &mut bytes).unwrap();++ if bytes.is_empty() {+ None+ } else {+ Some(bytes.into())+ }+ }+ }}}pub struct SegmentReader {- data: Vec<u64>,- num_fields: usize,+ field_readers: AllReaders,}impl SegmentReader {- pub fn get_field_reader(&self, doc: &DocId) -> FieldReader<'_> {- let data =- &self.data[(*doc as usize) * self.num_fields..(*doc as usize + 1) * self.num_fields];- FieldReader { data }+ pub fn get_field_reader(&self, doc: DocId) -> FieldReader<'_> {+ FieldReader {+ readers: &self.field_readers,+ doc,+ }}}diff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rsindex cb59e606..4fdfd668 100644--- a/crates/core/src/hyperloglog.rs+++ b/crates/core/src/hyperloglog.rs@@ -456,12 +456,12 @@ pub(crate) const RAW_ESTIMATE_DATA_VEC: &[&[f64]] = &[97665.679,98442.68,99229.3002,- 100011.0994,- 100790.6386,- 101580.1564,- 102377.7484,- 103152.1392,- 103944.2712,+ 100_011.099_4,+ 100_790.638_6,+ 101_580.156_4,+ 102_377.748_4,+ 103_152.139_2,+ 103_944.271_2,104730.216,105528.6336,106324.9398,@@ -4362,12 +4362,12 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {}#[inline]- fn b(&self) -> usize {+ fn b() -> usize {(N as f64).log2() as usize}pub fn add(&mut self, item: u64) {- let b = self.b();+ let b = Self::b();let hash = H::hash(item) as usize;let j = hash >> (64 - b);@@ -4383,14 +4383,16 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {self.registers.iter_mut().for_each(|r| *r = 0);}+ #[must_use]pub fn estimate_bias(&self, e: f64, b: usize) -> f64 {+ const K: usize = 6;+// binary search first nearest neighborlet lookup_array = RAW_ESTIMATE_DATA_VEC[b - 1 - RAW_ESTIMATE_DATA_OFFSET];- let mut idx_left = match lookup_array.binary_search_by(|v| v.partial_cmp(&e).unwrap()) {- Ok(i) => Some(i), // exact match+ let mut idx_left = match lookup_array.binary_search_by(|v| v.total_cmp(&e)) {Err(i) if i == lookup_array.len() => Some(i - 1), // no match, i points to end of array- Err(i) => Some(i), // no match, i points to left neighbor+ Ok(i) | Err(i) => Some(i), // exact match};let mut idx_right = match idx_left {@@ -4405,10 +4407,9 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {};// collect k nearest neighbors- const K: usize = 6;debug_assert!(lookup_array.len() >= K);let mut neighbors = [0; K];- for neighbor in neighbors.iter_mut() {+ for neighbor in &mut neighbors {let (right_instead_left, idx) = match (idx_left, idx_right) {(Some(i_left), Some(i_right)) => {// 2 candidates, find better one@@ -4428,7 +4429,7 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {// just right one is there, use it(true, i_right)}- _ => panic!("neighborhood search failed, this is bug!"),+ _ => unreachable!("neighborhood search failed, this is bug!"),};*neighbor = idx;if right_instead_left {@@ -4454,13 +4455,15 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {m * (m / (v as f64)).ln()}- fn threshold(&self, b: usize) -> usize {+ fn threshold(b: usize) -> usize {THRESHOLD_DATA_VEC[b - THRESHOLD_DATA_OFFSET]}+ #[must_use]+ #[allow(clippy::many_single_char_names)] // variables refer directly to the paperpub fn size(&self) -> usize {let m = self.registers.len() as f64;- let b = self.b();+ let b = Self::b();let sum: f64 = self.registers@@ -4478,24 +4481,26 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {e};- let v = self.registers.iter().filter(|r| **r == 0).count();+ let v = bytecount::count(&self.registers, 0);let h = if v != 0 {self.linear_counting(v)} else {e_star};- if h <= (self.threshold(b) as f64) {+ if h <= (Self::threshold(b) as f64) {h as usize} else {e_star as usize}}+ #[must_use]pub fn relative_error(&self) -> f64 {1.04 / (self.registers.len() as f64).sqrt()}+ #[must_use]pub fn size_bounds(&self) -> (usize, usize) {let size = self.size();let delta = (self.relative_error() * 2.0 * (size as f64)) as usize;@@ -4515,6 +4520,7 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {}}+ #[must_use]pub fn registers(&self) -> &[u8] {self.registers.as_ref()}@@ -4580,7 +4586,7 @@ mod tests {set.add(item);counter += 1;- assert!((set.size() as f64 - counter as f64).abs() <= 10.0);+ assert!((set.size() as f64 - f64::from(counter)).abs() <= 10.0);}}}diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rsindex 41649b73..be0c789c 100644--- a/crates/core/src/image_store.rs+++ b/crates/core/src/image_store.rs@@ -111,7 +111,7 @@ impl BaseImageStore {image = filter.transform(image);}- self.store.insert(key, image)+ self.store.insert(key, image);}fn flush(&self) {@@ -122,7 +122,7 @@ impl BaseImageStore {self.store.get(key)}- fn merge(&mut self, other: BaseImageStore) {+ fn merge(&mut self, other: &Self) {for (key, image) in other.store.iter() {self.insert(key, image);}@@ -179,11 +179,11 @@ impl ImageStore<String> for EntityImageStore {}fn merge(&mut self, other: Self) {- self.store.merge(other.store)+ self.store.merge(&other.store);}fn flush(&self) {- self.store.flush()+ self.store.flush();}}@@ -221,10 +221,12 @@ impl Image {bytes}+ #[must_use]pub fn resize_max(self, width: u32, height: u32) -> Self {MaxSizeFilter { width, height }.transform(self)}+ #[must_use]pub fn empty(width: u32, height: u32) -> Self {Self(image::DynamicImage::new_rgb8(width, height))}diff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex 73cf0906..f14ae105 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -76,8 +76,8 @@ impl Index {Ok(s)}- pub fn insert(&self, webpage: Webpage) -> Result<()> {- if let Ok(region) = Region::guess_from(&webpage) {+ pub fn insert(&self, webpage: &Webpage) -> Result<()> {+ if let Ok(region) = Region::guess_from(webpage) {let mut reg = self.region_count.lock().unwrap_or_else(|e| e.into_inner());reg.increment(®ion);}@@ -121,14 +121,14 @@ impl Index {pub fn retrieve_websites(&self,- websites: &[inverted_index::WebsitePointer],+ websites: &[inverted_index::WebpagePointer],query: &Query,) -> Result<Vec<inverted_index::RetrievedWebpage>> {self.inverted_index.retrieve_websites(websites, query)}pub fn merge(self, other: Self) -> Self {- self.inverted_index.merge(other.inverted_index);+ let _ = self.inverted_index.merge(other.inverted_index);let mut self_region_count = self.region_count@@ -165,7 +165,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -186,7 +186,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -207,7 +207,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>diff --git a/crates/core/src/inverted_index.rs b/crates/core/src/inverted_index.rsindex 3a13552f..c1901c36 100644--- a/crates/core/src/inverted_index.rs+++ b/crates/core/src/inverted_index.rs@@ -41,9 +41,8 @@ use crate::config::SnippetConfig;use crate::fastfield_reader::FastFieldReader;use crate::query::shortcircuit::ShortCircuitQuery;use crate::query::Query;-use crate::rake::RakeModel;use crate::ranking::initial::Score;-use crate::ranking::pipeline::RankingWebsite;+use crate::ranking::pipeline::RecallRankingWebpage;use crate::ranking::SignalAggregator;use crate::schema::{FastField, Field, TextField};use crate::search_ctx::Ctx;@@ -66,11 +65,11 @@ use std::sync::Arc;#[derive(Debug, Serialize, Deserialize)]pub struct InitialSearchResult {pub num_websites: Option<usize>,- pub top_websites: Vec<WebsitePointer>,+ pub top_websites: Vec<WebpagePointer>,}#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]-pub struct WebsitePointer {+pub struct WebpagePointer {pub score: Score,pub hashes: Hashes,pub address: DocAddress,@@ -165,7 +164,6 @@ pub struct InvertedIndex {schema: Arc<Schema>,snippet_config: SnippetConfig,fastfield_reader: FastFieldReader,- rake: RakeModel,}impl InvertedIndex {@@ -236,7 +234,6 @@ impl InvertedIndex {tantivy_index,snippet_config: SnippetConfig::default(),fastfield_reader,- rake: RakeModel::default(),})}@@ -287,11 +284,11 @@ impl InvertedIndex {Ok(s)}- pub fn insert(&self, webpage: Webpage) -> Result<()> {+ pub fn insert(&self, webpage: &Webpage) -> Result<()> {self.writer.as_ref().expect("writer has not been prepared")- .add_document(webpage.into_tantivy(&self.schema, &self.rake)?)?;+ .add_document(webpage.as_tantivy(&self.schema)?)?;Ok(())}@@ -374,10 +371,10 @@ impl InvertedIndex {pub fn retrieve_ranking_websites(&self,ctx: &Ctx,- pointers: Vec<WebsitePointer>,+ pointers: Vec<WebpagePointer>,mut aggregator: SignalAggregator,fastfield_reader: &FastFieldReader,- ) -> Result<Vec<RankingWebsite>> {+ ) -> Result<Vec<RecallRankingWebpage>> {let mut top_websites = Vec::new();let mut pointers: Vec<_> = pointers.into_iter().enumerate().collect();@@ -396,14 +393,21 @@ impl InvertedIndex {_ => false,};+ let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);if update_segment {- let segment_reader = ctx.tv_searcher.segment_reader(pointer.address.segment);aggregator.register_segment(&ctx.tv_searcher, segment_reader, fastfield_reader)?;}prev_segment = Some(pointer.address.segment);- top_websites.push((orig_index, RankingWebsite::new(pointer, &mut aggregator)));+ top_websites.push((+ orig_index,+ RecallRankingWebpage::new(+ pointer,+ fastfield_reader.borrow_segment(&segment_reader.segment_id()),+ &mut aggregator,+ ),+ ));}top_websites.sort_by(|a, b| a.0.cmp(&b.0));@@ -414,7 +418,7 @@ impl InvertedIndex {.collect())}- pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {+ pub fn website_host_node(&self, website: &WebpagePointer) -> Result<Option<NodeID>> {let searcher = self.reader.searcher();let doc: TantivyDocument = searcher.doc(website.address.into())?;@@ -434,7 +438,7 @@ impl InvertedIndex {pub fn retrieve_websites(&self,- websites: &[WebsitePointer],+ websites: &[WebpagePointer],query: &Query,) -> Result<Vec<RetrievedWebpage>> {let tv_searcher = self.reader.searcher();@@ -509,6 +513,7 @@ impl InvertedIndex {Ok(webpages)}+ #[allow(clippy::missing_panics_doc)] // cannot panic as writer is preparedpub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {self.prepare_writer()?;let base_path = Path::new(&self.path);@@ -538,6 +543,7 @@ impl InvertedIndex {Ok(RetrievedWebpage::from(doc))}+ #[must_use]pub fn merge(mut self, mut other: InvertedIndex) -> Self {self.prepare_writer().expect("failed to prepare writer");other.prepare_writer().expect("failed to prepare writer");@@ -700,6 +706,15 @@ impl RetrievedWebpage {}}+fn str_value(field: TextField, value: &tantivy::schema::FieldValue) -> String {+ value+ .value()+ .as_value()+ .as_str()+ .unwrap_or_else(|| panic!("{} field should be text", field.name()))+ .to_string()+}+impl From<TantivyDocument> for RetrievedWebpage {fn from(doc: TantivyDocument) -> Self {let mut webpage = RetrievedWebpage::default();@@ -707,38 +722,17 @@ impl From<TantivyDocument> for RetrievedWebpage {for value in doc.field_values() {match Field::get(value.field.field_id() as usize).copied() {Some(Field::Text(TextField::Title)) => {- webpage.title = value- .value()- .as_value()- .as_str()- .expect("Title field should be text")- .to_string();+ webpage.title = str_value(TextField::Title, value);}Some(Field::Text(TextField::StemmedCleanBody)) => {- webpage.body = value- .value()- .as_value()- .as_str()- .expect("Body field should be text")- .to_string();+ webpage.body = str_value(TextField::StemmedCleanBody, value);}Some(Field::Text(TextField::Description)) => {- let desc = value- .value()- .as_value()- .as_str()- .expect("Description field should be text")- .to_string();-+ let desc = str_value(TextField::Description, value);webpage.description = if desc.is_empty() { None } else { Some(desc) }}Some(Field::Text(TextField::Url)) => {- webpage.url = value- .value()- .as_value()- .as_str()- .expect("Url field should be text")- .to_string();+ webpage.url = str_value(TextField::Url, value);}Some(Field::Fast(FastField::LastUpdated)) => {webpage.updated_time = {@@ -751,12 +745,7 @@ impl From<TantivyDocument> for RetrievedWebpage {}}Some(Field::Text(TextField::AllBody)) => {- webpage.dirty_body = value- .value()- .as_value()- .as_str()- .expect("All body field should be text")- .to_string();+ webpage.dirty_body = str_value(TextField::AllBody, value);}Some(Field::Fast(FastField::Region)) => {webpage.region = {@@ -765,23 +754,11 @@ impl From<TantivyDocument> for RetrievedWebpage {}}Some(Field::Text(TextField::DmozDescription)) => {- let desc = value- .value()- .as_value()- .as_str()- .expect("Dmoz description field should be text")- .to_string();-+ let desc = str_value(TextField::DmozDescription, value);webpage.dmoz_description = if desc.is_empty() { None } else { Some(desc) }}Some(Field::Text(TextField::SchemaOrgJson)) => {- let json = value- .value()- .as_value()- .as_str()- .expect("Schema.org json field should be stored as text")- .to_string();-+ let json = str_value(TextField::SchemaOrgJson, value);webpage.schema_org = serde_json::from_str(&json).unwrap_or_default();}Some(Field::Fast(FastField::LikelyHasAds)) => {@@ -793,25 +770,13 @@ impl From<TantivyDocument> for RetrievedWebpage {value.value().as_value().as_u64().unwrap_or_default() != 0;}Some(Field::Text(TextField::RecipeFirstIngredientTagId)) => {- let tag_id = value- .value()- .as_value()- .as_str()- .expect("Recipe first ingredient tag id field should be stored as text")- .to_string();-+ let tag_id = str_value(TextField::RecipeFirstIngredientTagId, value);if !tag_id.is_empty() {webpage.recipe_first_ingredient_tag_id = Some(tag_id);}}Some(Field::Text(TextField::Keywords)) => {- let keywords = value- .value()- .as_value()- .as_str()- .expect("Keywords field should be stored as text")- .to_string();-+ let keywords = str_value(TextField::Keywords, value);webpage.keywords = keywords.split('\n').map(|s| s.to_string()).collect();}_ => {}@@ -824,9 +789,11 @@ impl From<TantivyDocument> for RetrievedWebpage {#[cfg(test)]mod tests {+ use candle_core::Tensor;use maplit::hashmap;use crate::{+ config::CollectorConfig,ranking::{Ranker, SignalAggregator},searcher::SearchQuery,webpage::Html,@@ -872,7 +839,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");@@ -880,7 +847,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -904,7 +871,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -919,7 +886,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -953,7 +920,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -967,7 +934,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1000,7 +967,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1015,7 +982,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1048,7 +1015,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1063,7 +1030,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1081,7 +1048,7 @@ mod tests {).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1120,7 +1087,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let mut result =@@ -1143,7 +1110,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1178,7 +1145,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1192,7 +1159,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1225,7 +1192,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1240,7 +1207,7 @@ mod tests {index1.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1264,7 +1231,7 @@ mod tests {index2.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1301,7 +1268,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1328,7 +1295,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1337,7 +1304,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1361,7 +1328,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");@@ -1375,7 +1342,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1408,7 +1375,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1422,7 +1389,7 @@ mod tests {let mut index = InvertedIndex::temporary().expect("Unable to open index");index- .insert(Webpage::new(+ .insert(&Webpage::test_parse(&format!(r#"<html>@@ -1459,7 +1426,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let result =@@ -1501,7 +1468,7 @@ mod tests {let mut index = InvertedIndex::temporary().expect("Unable to open index");index- .insert(Webpage::new(+ .insert(&Webpage::test_parse(&format!(r#"<html>@@ -1535,7 +1502,7 @@ mod tests {let mut index = InvertedIndex::temporary().expect("Unable to open index");index- .insert(Webpage::new(+ .insert(&Webpage::test_parse(&format!(r#"<html>@@ -1617,8 +1584,8 @@ mod tests {..Default::default()};- index.insert(a).unwrap();- index.insert(b).unwrap();+ index.insert(&a).unwrap();+ index.insert(&b).unwrap();index.commit().expect("failed to commit index");@@ -1636,7 +1603,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);let mut result =@@ -1656,7 +1623,7 @@ mod tests {let ranker = Ranker::new(SignalAggregator::new(Some(&query)),ctx.fastfield_reader.clone(),- Default::default(),+ CollectorConfig::default(),);result =@@ -1664,4 +1631,92 @@ mod tests {assert_eq!(result.documents.len(), 1);}++ #[test]+ fn test_title_embeddings_stored() {+ let mut index = InvertedIndex::temporary().expect("Unable to open index");++ let mut webpage = Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website A</title>+ </head>+ <body>+ {CONTENT} test+ </body>+ </html>+ "#,+ CONTENT = crate::rand_words(100)+ ),+ "https://www.a.com",+ )+ .unwrap();++ webpage.title_embedding =+ Some(Tensor::rand(0.0, 1.0, &[2, 2], &candle_core::Device::Cpu).unwrap());++ index.insert(&webpage).unwrap();++ let mut webpage = Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Test website B</title>+ </head>+ <body>+ {CONTENT} test+ </body>+ </html>+ "#,+ CONTENT = crate::rand_words(100)+ ),+ "https://www.b.com",+ )+ .unwrap();++ webpage.title_embedding = None;++ index.insert(&webpage).unwrap();+ index.commit().expect("failed to commit index");++ let ctx = index.local_search_ctx();++ let query = Query::parse(+ &ctx,+ &SearchQuery {+ query: "test".to_string(),+ ..Default::default()+ },+ &index,+ )+ .expect("Failed to parse query");++ let ranker = Ranker::new(+ SignalAggregator::new(Some(&query)),+ ctx.fastfield_reader.clone(),+ CollectorConfig::default(),+ );++ let res = index+ .search_initial(&query, &ctx, ranker.collector(ctx.clone()))+ .unwrap();++ let fastfield_reader = index.fastfield_reader();++ let ranking_websites = index+ .retrieve_ranking_websites(+ &ctx,+ res.top_websites,+ ranker.aggregator(),+ &fastfield_reader,+ )+ .unwrap();++ assert_eq!(ranking_websites.len(), 2);+ assert!(ranking_websites[0].title_embedding.is_some());+ assert!(ranking_websites[1].title_embedding.is_none());+ }}diff --git a/crates/core/src/kv/rocksdb_store.rs b/crates/core/src/kv/rocksdb_store.rsindex 4c306199..e0d7cc07 100644--- a/crates/core/src/kv/rocksdb_store.rs+++ b/crates/core/src/kv/rocksdb_store.rs@@ -51,7 +51,7 @@ whereoptions.set_level_compaction_dynamic_level_bytes(true);options.set_bytes_per_sync(1048576);let mut block_options = BlockBasedOptions::default();- block_options.set_block_size(1024 * 1024 * 1024); // 1 GB+ block_options.set_block_size(32 * 1024); // 32 kbblock_options.set_format_version(5);block_options.set_cache_index_and_filter_blocks(true);block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 1d32c8c0..caf64a94 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -17,11 +17,16 @@//! Main library for Stract.#![doc(html_logo_url = "https://stract.com/images/biglogo.svg")]+// #![warn(clippy::too_many_lines)]+// pedantic stuff// #![warn(clippy::pedantic)]-// #![warn(missing_docs)]-// #![warn(clippy::missing_docs_in_private_items)]-#![allow(clippy::cast_precision_loss)]-#![allow(clippy::missing_errors_doc)]+// #![allow(clippy::unreadable_literal)]+// #![allow(clippy::missing_fields_in_debug)]+// #![allow(clippy::cast_possible_truncation)]+// #![allow(clippy::cast_precision_loss)]+// #![allow(clippy::cast_sign_loss)]+// #![allow(clippy::module_name_repetitions)] // maybe we should remove this later+// #![allow(clippy::missing_errors_doc)]use serde::{Deserialize, Serialize};use std::path::PathBuf;diff --git a/crates/core/src/live_index.rs b/crates/core/src/live_index.rsindex e659dc8e..db0d1312 100644--- a/crates/core/src/live_index.rs+++ b/crates/core/src/live_index.rs@@ -17,7 +17,7 @@use anyhow::Result;use std::{path::Path,- sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},+ sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard},time::{Duration, SystemTime},};@@ -29,7 +29,7 @@ use crate::{crawler::{reqwest_client, CrawlDatum, DatumStream, JobExecutor, RetrieableUrl, WeightedUrl, WorkerJob,},- entrypoint::indexer::IndexingWorker,+ entrypoint::indexer::{self, IndexingWorker},feed::{self,scheduler::{Domain, DomainFeeds, Split},@@ -41,6 +41,7 @@ const PRUNE_INTERVAL: Duration = Duration::from_secs(60 * 60); // 1 hourconst FEED_CHECK_INTERVAL: Duration = Duration::from_secs(60 * 10); // 10 minutesconst AUTO_COMMIT_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 minutesconst EVENT_LOOP_INTERVAL: Duration = Duration::from_secs(5);+const BATCH_SIZE: usize = 512;#[derive(Debug, Clone)]struct Feeds {@@ -108,6 +109,7 @@ impl DownloadedDb {struct Indexer {search_index: Arc<RwLock<crate::index::Index>>,worker: IndexingWorker,+ write_batch: Arc<Mutex<Vec<indexer::IndexableWebpage>>>,}enum CrawlResults {@@ -221,6 +223,38 @@ impl Crawler {}}+impl Indexer {+ fn maybe_write_batch_to_index(&self) {+ let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());++ if batch.len() < BATCH_SIZE {+ return;+ }++ let prepared = self.worker.prepare_webpages(&batch);++ let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());+ for webpage in &prepared {+ search_index.insert(webpage).ok();+ }+ }++ fn write_batch_to_index(&self) {+ let batch = self.write_batch.lock().unwrap_or_else(|e| e.into_inner());++ if batch.is_empty() {+ return;+ }++ let prepared = self.worker.prepare_webpages(&batch);++ let search_index = self.search_index.write().unwrap_or_else(|e| e.into_inner());+ for webpage in &prepared {+ search_index.insert(webpage).ok();+ }+ }+}+pub struct Index {search_index: Arc<RwLock<crate::index::Index>>,}@@ -267,21 +301,19 @@ impl Index {impl DatumStream for Indexer {async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {- let webpage = self.worker.prepare_webpage(- &crawl_datum.body,- crawl_datum.url.as_str(),- crawl_datum.fetch_time_ms,- )?;-- self.search_index- .read()+ self.write_batch+ .lock().unwrap_or_else(|e| e.into_inner())- .insert(webpage)?;+ .push(crawl_datum.into());++ self.maybe_write_batch_to_index();Ok(())}async fn finish(&self) -> Result<()> {+ self.write_batch_to_index();+self.search_index.write().unwrap_or_else(|e| e.into_inner())@@ -328,13 +360,8 @@ impl IndexManager {let index = Index::new(&config.index_path)?;let indexer = Arc::new(Indexer {search_index: index.clone_inner_index(),- worker: IndexingWorker::new(- config.host_centrality_store_path.clone(),- config.page_centrality_store_path.clone(),- config.page_webgraph_path.clone(),- None,- config.safety_classifier_path.clone(),- ),+ worker: IndexingWorker::new(config.clone()),+ write_batch: Arc::new(Mutex::new(Vec::new())),});let crawler_config = CrawlerConfig::from(&config);diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 30d739db..d531c5d7 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -243,7 +243,7 @@ fn main() -> Result<()> {Commands::Indexer { options } => match options {IndexingOptions::Search { config_path } => {let config = load_toml_config(config_path);- entrypoint::Indexer::run(&config)?;+ entrypoint::indexer::run(&config)?;}IndexingOptions::Entity {wikipedia_dump_path,@@ -258,7 +258,7 @@ fn main() -> Result<()> {.into_iter().map(entrypoint::indexer::IndexPointer::from).collect::<Vec<_>>();- entrypoint::indexer::Indexer::merge(pointers)?;+ entrypoint::indexer::merge(pointers)?;}},Commands::Centrality { mode } => {@@ -299,7 +299,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(webgraph_server::run(config))?+ .block_on(webgraph_server::run(config))?;}},Commands::Api { config_path } => {@@ -308,7 +308,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(api::run(config))?+ .block_on(api::run(config))?;}Commands::SearchServer { config_path } => {let config: config::SearchServerConfig = load_toml_config(config_path);@@ -316,7 +316,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(search_server::run(config))?+ .block_on(search_server::run(config))?;}Commands::EntitySearchServer { config_path } => {let config: config::EntitySearchServerConfig = load_toml_config(config_path);@@ -324,7 +324,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(entity_search_server::run(config))?+ .block_on(entity_search_server::run(config))?;}Commands::AutosuggestScrape {num_queries: queries_to_scrape,@@ -349,7 +349,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(entrypoint::crawler::worker(config))?+ .block_on(entrypoint::crawler::worker(config))?;}Crawler::Coordinator { config_path } => {let config: config::CrawlCoordinatorConfig = load_toml_config(config_path);@@ -357,7 +357,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(entrypoint::crawler::coordinator(config))?+ .block_on(entrypoint::crawler::coordinator(config))?;}Crawler::Router { config_path } => {let config: config::CrawlRouterConfig = load_toml_config(config_path);@@ -365,7 +365,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(entrypoint::crawler::router(config))?+ .block_on(entrypoint::crawler::router(config))?;}Crawler::Plan { config_path } => {let config: config::CrawlPlannerConfig = load_toml_config(config_path);@@ -379,7 +379,7 @@ fn main() -> Result<()> {output_path,} => safety_classifier::train(dataset_path, output_path)?,SafetyClassifierOptions::Predict { model_path, text } => {- safety_classifier::predict(model_path, &text)?+ safety_classifier::predict(model_path, &text)?;}},Commands::LiveIndex { options } => match options {@@ -393,7 +393,7 @@ fn main() -> Result<()> {tokio::runtime::Builder::new_multi_thread().enable_all().build()?- .block_on(entrypoint::live_index::serve(config))?+ .block_on(entrypoint::live_index::serve(config))?;}},Commands::WebSpell { config_path } => {diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rsindex 9d2c7096..ab952904 100644--- a/crates/core/src/models/bert.rs+++ b/crates/core/src/models/bert.rs@@ -78,7 +78,7 @@ impl Default for Config {num_hidden_layers: 12,num_attention_heads: 12,intermediate_size: 3072,- hidden_act: HiddenAct::Gelu,+ hidden_act: HiddenAct::GeluApproximate,hidden_dropout_prob: 0.1,max_position_embeddings: 512,type_vocab_size: 2,@@ -102,7 +102,7 @@ impl Config {num_hidden_layers: 6,num_attention_heads: 12,intermediate_size: 1536,- hidden_act: HiddenAct::Gelu,+ hidden_act: HiddenAct::GeluApproximate,hidden_dropout_prob: 0.1,max_position_embeddings: 512,type_vocab_size: 2,diff --git a/crates/core/src/models/dual_encoder.rs b/crates/core/src/models/dual_encoder.rsnew file mode 100644index 00000000..1b9a66db--- /dev/null+++ b/crates/core/src/models/dual_encoder.rs@@ -0,0 +1,118 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use anyhow::anyhow;+use candle_core::{Device, Tensor};+use candle_nn::VarBuilder;+use std::path::Path;++use crate::{+ models::bert::{self, BertModel},+ Result,+};+use tokenizers::{PaddingParams, TruncationParams};++pub struct DualEncoder {+ model: BertModel,+ tokenizer: tokenizers::Tokenizer,+ device: Device,+ dtype: candle_core::DType,+ config: bert::Config,+}++impl DualEncoder {+ pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {+ let device = Device::Cpu;+ let dtype = candle_core::DType::F16;++ let truncation = TruncationParams {+ max_length: 256,+ ..Default::default()+ };++ let padding = PaddingParams {+ ..Default::default()+ };++ let mut tokenizer =+ tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json")).unwrap();++ tokenizer.with_truncation(Some(truncation)).unwrap();+ tokenizer.with_padding(Some(padding));++ let vb = unsafe {+ VarBuilder::from_mmaped_safetensors(+ &[folder.as_ref().join("model.safetensors")],+ dtype,+ &device,+ )?+ };+ let config = std::fs::read_to_string(folder.as_ref().join("config.json"))?;+ let mut config: bert::Config = serde_json::from_str(&config)?;+ config.hidden_act = bert::HiddenAct::GeluApproximate;++ // all tensors can be loaded with (useful for debugging):+ // candle_core::safetensors::load(folder.as_ref().join("model.safetensors"), &device)++ let mut model = BertModel::load(vb, &config)?;+ model.set_pooler(None); // model should use mean pooling++ Ok(Self {+ model,+ tokenizer,+ device,+ dtype,+ config,+ })+ }++ pub fn embed(&self, texts: &[String]) -> Result<Tensor> {+ let enc = self+ .tokenizer+ .encode_batch(texts.to_vec(), true)+ .map_err(|e| anyhow!(e))?;++ let ids = enc+ .iter()+ .map(|enc| Tensor::new(enc.get_ids(), &self.device).map_err(|e| anyhow!(e)))+ .collect::<Result<Vec<_>>>()?;++ let input_ids = Tensor::stack(&ids, 0)?;++ let token_type_ids = input_ids.zeros_like()?;++ let attention_mask = enc+ .iter()+ .map(|enc| Tensor::new(enc.get_attention_mask(), &self.device).map_err(|e| anyhow!(e)))+ .collect::<Result<Vec<_>>>()?;+ let attention_mask = Tensor::stack(&attention_mask, 0)?.to_dtype(self.dtype)?;++ let emb = self+ .model+ .forward(&input_ids, &token_type_ids, &attention_mask)?;++ let (_n_sentence, n_tokens, _hidden_size) = emb.dims3()?;++ let emb = (emb.sum(1)? / (n_tokens as f64))?; // mean pooling+ let emb = emb.broadcast_div(&emb.sqr()?.sum_keepdim(1)?.sqrt()?)?; // l2 normalization++ Ok(emb)+ }++ pub fn hidden_size(&self) -> usize {+ self.config.hidden_size+ }+}diff --git a/crates/core/src/models/mod.rs b/crates/core/src/models/mod.rsindex 01b31e25..17bec51d 100644--- a/crates/core/src/models/mod.rs+++ b/crates/core/src/models/mod.rs@@ -1 +1,18 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+pub mod bert;+pub mod dual_encoder;diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 589605bf..d769ad09 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -396,7 +396,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -414,7 +414,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -444,7 +444,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -462,7 +462,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -512,7 +512,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -530,7 +530,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -564,7 +564,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -582,7 +582,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(r#"<html><head>@@ -652,7 +652,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -673,7 +673,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -725,7 +725,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -746,7 +746,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -790,7 +790,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -812,7 +812,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -855,7 +855,7 @@ mod tests {#[test]fn safe_search() {let mut index = Index::temporary().expect("Unable to open index");- let mut webpage = Webpage::new(+ let mut webpage = Webpage::test_parse(&format!(r#"<html>@@ -876,9 +876,9 @@ mod tests {webpage.safety_classification = Some(safety_classifier::Label::SFW);webpage.html.set_clean_text("sfw".to_string());- index.insert(webpage).expect("failed to insert webpage");+ index.insert(&webpage).expect("failed to insert webpage");- let mut webpage = Webpage::new(+ let mut webpage = Webpage::test_parse(&format!(r#"<html>@@ -899,7 +899,7 @@ mod tests {webpage.safety_classification = Some(safety_classifier::Label::NSFW);webpage.html.set_clean_text("nsfw".to_string());- index.insert(webpage).expect("failed to insert webpage");+ index.insert(&webpage).expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);@@ -931,7 +931,7 @@ mod tests {index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -952,7 +952,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -973,7 +973,7 @@ mod tests {.expect("failed to insert webpage");index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -1027,6 +1027,7 @@ mod tests {query: "test site:www.first.com".to_string(),..Default::default()};+let result = searcher.search(&query).expect("Search failed");assert_eq!(result.webpages.len(), 1);}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex e51e6907..040cf05f 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -283,7 +283,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -307,7 +307,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -408,7 +408,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -431,7 +431,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -505,7 +505,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -529,7 +529,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -553,7 +553,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -653,7 +653,7 @@ mod tests {let graph = writer.finalize();index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -678,7 +678,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -703,7 +703,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -764,7 +764,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -800,7 +800,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r##"@@ -943,7 +943,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1009,7 +1009,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1033,7 +1033,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1095,7 +1095,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1251,7 +1251,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1319,7 +1319,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1374,7 +1374,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1398,7 +1398,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -1516,7 +1516,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);@@ -1589,7 +1589,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");let mut page = Webpage {html: Html::parse(@@ -1612,7 +1612,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);@@ -1668,7 +1668,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");let mut page = Webpage {html: Html::parse(@@ -1691,7 +1691,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);@@ -1760,7 +1760,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");let mut page = Webpage {html: Html::parse(@@ -1782,7 +1782,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");let mut page = Webpage {html: Html::parse(@@ -1804,7 +1804,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");@@ -1849,7 +1849,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");let mut page = Webpage {html: Html::parse(@@ -1871,7 +1871,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");let mut page = Webpage {html: Html::parse(@@ -1893,7 +1893,7 @@ mod tests {page.html.set_clean_text("".to_string());- index.insert(page).expect("failed to insert webpage");+ index.insert(&page).expect("failed to insert webpage");index.commit().expect("failed to commit index");diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rsindex 46243f4b..3efafe66 100644--- a/crates/core/src/query/pattern_query/scorer.rs+++ b/crates/core/src/query/pattern_query/scorer.rs@@ -134,9 +134,9 @@ impl EmptyFieldScorer {fn num_tokes(&self, doc: DocId) -> u64 {let s: Option<u64> = self.segment_reader- .get_field_reader(&doc)- .get(&self.num_tokens_fastfield)- .into();+ .get_field_reader(doc)+ .get(self.num_tokens_fastfield)+ .and_then(|v| v.as_u64());s.unwrap_or_default()}}@@ -277,9 +277,9 @@ impl NormalPatternScorer {let mut slop = 1;let num_tokens_doc: Option<u64> = self.segment_reader- .get_field_reader(&self.doc())- .get(&self.num_tokens_field)- .into();+ .get_field_reader(self.doc())+ .get(self.num_tokens_field)+ .and_then(|v| v.as_u64());let num_tokens_doc = num_tokens_doc.unwrap();for (i, pattern_part) in self.pattern.iter().enumerate() {diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 3d0646d0..1f54f75d 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -400,7 +400,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -424,7 +424,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 6082f789..01c847d9 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -126,13 +126,18 @@ impl Ranker {#[cfg(test)]mod tests {+ use std::path::Path;+use optics::{ast::{RankingCoeff, RankingTarget},Optic,};use crate::{+ config::{IndexingDualEncoderConfig, IndexingLocalConfig, WarcSource},+ entrypoint::indexer::IndexingWorker,index::Index,+ models::dual_encoder::DualEncoder,searcher::{LocalSearcher, SearchQuery},webpage::{Html, Webpage},};@@ -145,7 +150,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -169,7 +174,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -211,7 +216,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -235,7 +240,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -278,7 +283,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -303,7 +308,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -353,7 +358,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html>@@ -373,7 +378,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html><head>@@ -423,7 +428,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html>@@ -444,7 +449,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html>@@ -483,7 +488,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html>@@ -505,7 +510,7 @@ mod tests {.expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html>@@ -527,7 +532,7 @@ mod tests {.expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html>@@ -596,7 +601,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -619,7 +624,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -669,7 +674,7 @@ mod tests {let mut index = Index::temporary().expect("Unable to open index");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -693,7 +698,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -717,7 +722,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -767,4 +772,198 @@ mod tests {assert_eq!(result.webpages[1].url, "https://www.second.com/one/two");assert_eq!(result.webpages[2].url, "https://www.third.com/one/two123");}++ fn setup_worker(data_path: &Path) -> IndexingWorker {+ IndexingWorker::new(IndexingLocalConfig {+ host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ page_centrality_store_path: None,+ page_webgraph_path: None,+ topics_path: None,+ safety_classifier_path: None,+ dual_encoder: Some(IndexingDualEncoderConfig {+ model_path: data_path.to_str().unwrap().to_string(),+ page_centrality_rank_threshold: None,+ }),+ output_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ limit_warc_files: None,+ skip_warc_files: None,+ warc_source: WarcSource::Local(crate::config::LocalConfig {+ folder: crate::gen_temp_path().to_str().unwrap().to_string(),+ names: vec!["".to_string()],+ }),+ host_centrality_threshold: None,+ minimum_clean_words: None,+ batch_size: 10,+ })+ }++ #[test]+ fn title_embeddings() {+ let data_path = Path::new("../../data/summarizer/dual_encoder");+ if !data_path.exists() {+ // Skip the test if the test data is not available+ return;+ }++ let worker = setup_worker(data_path);++ let mut index = Index::temporary().expect("Unable to open index");++ let mut pages = vec![+ Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Homemade Heart Brownie Recipe</title>+ </head>+ <body>+ best chocolate cake {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.a.com/",+ )+ .unwrap(),+ Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>How To Best Use an iMac as a Monitor for a PC</title>+ </head>+ <body>+ best chocolate cake {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.b.com/",+ )+ .unwrap(),+ ];++ worker.set_title_embeddings(&mut pages);+ assert!(pages.iter().all(|p| p.title_embedding.is_some()));++ for page in pages {+ index.insert(&page).expect("failed to insert webpage");+ }++ index.commit().expect("failed to commit index");++ let mut searcher = LocalSearcher::new(index);+ searcher+ .set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));++ let result = searcher+ .search(&SearchQuery {+ query: "best chocolate cake".to_string(),+ optic: Some(Optic {+ rankings: vec![RankingCoeff {+ target: RankingTarget::Signal("title_embedding_similarity".to_string()),+ value: 100_000.0,+ }],+ ..Default::default()+ }),+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.a.com/");+ }++ #[test]+ fn keyword_embeddings() {+ let data_path = Path::new("../../data/summarizer/dual_encoder");+ if !data_path.exists() {+ // Skip the test if the test data is not available+ return;+ }++ let worker = setup_worker(data_path);++ let mut index = Index::temporary().expect("Unable to open index");++ let mut a = Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>Homemade Heart Brownie Recipe</title>+ </head>+ <body>+ best chocolate cake {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.a.com/",+ )+ .unwrap();++ a.keywords = vec![+ "chocolate".to_string(),+ "cake".to_string(),+ "recipe".to_string(),+ ];++ let mut b = Webpage::test_parse(+ &format!(+ r#"+ <html>+ <head>+ <title>How To Best Use an iMac as a Monitor for a PC</title>+ </head>+ <body>+ best chocolate cake {CONTENT} {}+ </body>+ </html>+ "#,+ crate::rand_words(100)+ ),+ "https://www.b.com/",+ )+ .unwrap();++ b.keywords = vec!["imac".to_string()];++ let mut pages = vec![a, b];++ worker.set_keyword_embeddings(&mut pages);++ assert!(pages.iter().all(|p| p.keyword_embedding.is_some()));++ for page in pages {+ index.insert(&page).expect("failed to insert webpage");+ }++ index.commit().expect("failed to commit index");++ let mut searcher = LocalSearcher::new(index);+ searcher+ .set_dual_encoder(DualEncoder::open(data_path).expect("failed to open dual encoder"));++ let result = searcher+ .search(&SearchQuery {+ query: "best chocolate cake".to_string(),+ optic: Some(Optic {+ rankings: vec![RankingCoeff {+ target: RankingTarget::Signal("keyword_embedding_similarity".to_string()),+ value: 100_000.0,+ }],+ ..Default::default()+ }),+ ..Default::default()+ })+ .expect("Search failed");++ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.a.com/");+ }}diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex f1169576..537828e8 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -85,7 +85,7 @@ mod tests {let graph = wrt.finalize();index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -110,7 +110,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"@@ -135,7 +135,7 @@ mod tests {}).expect("failed to insert webpage");index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(&format!(r#"diff --git a/crates/core/src/ranking/pipeline.rs b/crates/core/src/ranking/pipeline.rsdeleted file mode 100644index 528fa865..00000000--- a/crates/core/src/ranking/pipeline.rs+++ /dev/null@@ -1,613 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::sync::Arc;--use serde::{Deserialize, Serialize};--use crate::{- collector::{self, BucketCollector},- config::CollectorConfig,- enum_map::EnumMap,- inverted_index::{RetrievedWebpage, WebsitePointer},- searcher::SearchQuery,- Result,-};--use super::{- models::lambdamart::{self, LambdaMART},- Signal, SignalAggregator, SignalCoefficient, SignalScore,-};--use super::models::cross_encoder::CrossEncoder;--pub trait AsRankingWebsite: Clone {- fn as_ranking(&self) -> &RankingWebsite;- fn as_mut_ranking(&mut self) -> &mut RankingWebsite;-}--impl<T> collector::Doc for T-where- T: AsRankingWebsite,-{- fn score(&self) -> f64 {- self.as_ranking().score- }-- fn hashes(&self) -> collector::Hashes {- self.as_ranking().pointer.hashes- }-}--impl AsRankingWebsite for RankingWebsite {- fn as_ranking(&self) -> &RankingWebsite {- self- }-- fn as_mut_ranking(&mut self) -> &mut RankingWebsite {- self- }-}--impl lambdamart::AsValue for SignalScore {- fn as_value(&self) -> f64 {- self.value- }-}--#[derive(Clone, Debug, Serialize, Deserialize)]-pub struct RetrievedWebpageRanking {- retrieved_webpage: RetrievedWebpage,- ranking: RankingWebsite,-}--impl RetrievedWebpageRanking {- pub fn new(retrieved_webpage: RetrievedWebpage, ranking: RankingWebsite) -> Self {- let mut ranking = ranking;- ranking.snippet = Some(retrieved_webpage.snippet.unhighlighted_string());-- Self {- retrieved_webpage,- ranking,- }- }-- pub fn into_retrieved_webpage(self) -> RetrievedWebpage {- self.retrieved_webpage- }-}--impl AsRankingWebsite for RetrievedWebpageRanking {- fn as_ranking(&self) -> &RankingWebsite {- &self.ranking- }-- fn as_mut_ranking(&mut self) -> &mut RankingWebsite {- &mut self.ranking- }-}--#[derive(Clone, Debug, Serialize, Deserialize)]-pub struct RankingWebsite {- pub pointer: WebsitePointer,- pub signals: EnumMap<Signal, SignalScore>,- pub title: Option<String>,- pub snippet: Option<String>,- pub optic_boost: Option<f64>,- pub score: f64,-}--impl RankingWebsite {- pub fn new(pointer: WebsitePointer, aggregator: &mut SignalAggregator) -> Self {- let mut res = RankingWebsite {- signals: EnumMap::new(),- title: None,- score: pointer.score.total,- optic_boost: None,- snippet: None,- pointer: pointer.clone(),- };-- for computed_signal in aggregator.compute_signals(pointer.address.doc_id).flatten() {- res.signals- .insert(computed_signal.signal, computed_signal.score);- }-- if let Some(boost) = aggregator.boosts(pointer.address.doc_id) {- res.optic_boost = Some(boost);- }-- res- }-}--trait Scorer<T>: Send + Sync {- fn score(&self, websites: &mut [T]);- fn set_query_info(&mut self, _query: &SearchQuery) {}-}--struct ReRanker<M: CrossEncoder> {- crossencoder: Arc<M>,- lambda_mart: Option<Arc<LambdaMART>>,- query: Option<SearchQuery>,- signal_coefficients: Option<SignalCoefficient>,-}--impl<M: CrossEncoder> ReRanker<M> {- fn new(crossencoder: Arc<M>, lambda: Option<Arc<LambdaMART>>) -> Self {- Self {- crossencoder,- lambda_mart: lambda,- query: None,- signal_coefficients: None,- }- }-- fn crossencoder_score_websites<T: AsRankingWebsite>(&self, websites: &mut [T]) {- let mut snippets = Vec::with_capacity(websites.len());- let mut titles = Vec::with_capacity(websites.len());-- for website in websites.iter_mut() {- titles.push(website.as_mut_ranking().title.clone().unwrap_or_default());- snippets.push(website.as_mut_ranking().snippet.clone().unwrap_or_default());- }-- let query = &self.query.as_ref().unwrap().query;- let snippet_scores = self.crossencoder.run(query, &snippets);- let title_scores = self.crossencoder.run(query, &titles);-- for ((website, snippet), title) in websites.iter_mut().zip(snippet_scores).zip(title_scores)- {- let website = website.as_mut_ranking();- website.signals.insert(- Signal::CrossEncoderSnippet,- SignalScore {- coefficient: self.crossencoder_snippet_coeff(),- value: snippet,- },- );-- website.signals.insert(- Signal::CrossEncoderTitle,- SignalScore {- coefficient: self.crossencoder_title_coeff(),- value: title,- },- );- }- }-- fn crossencoder_snippet_coeff(&self) -> f64 {- self.signal_coefficients- .as_ref()- .and_then(|coeffs| coeffs.get(&Signal::CrossEncoderSnippet))- .unwrap_or(Signal::CrossEncoderSnippet.default_coefficient())- }-- fn crossencoder_title_coeff(&self) -> f64 {- self.signal_coefficients- .as_ref()- .and_then(|coeffs| coeffs.get(&Signal::CrossEncoderSnippet))- .unwrap_or(Signal::CrossEncoderSnippet.default_coefficient())- }-}--impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {- fn score(&self, websites: &mut [T]) {- self.crossencoder_score_websites(websites);-- for website in websites.iter_mut() {- let website = website.as_mut_ranking();- website.score = calculate_score(- &self.lambda_mart,- &self.signal_coefficients,- &website.signals,- );- }- }-- fn set_query_info(&mut self, query: &SearchQuery) {- self.query = Some(query.clone());-- self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);- }-}--struct IdentityScorer;--impl<T: AsRankingWebsite> Scorer<T> for IdentityScorer {- fn score(&self, _websites: &mut [T]) {}-}--fn calculate_score(- model: &Option<Arc<LambdaMART>>,- signal_coefficients: &Option<SignalCoefficient>,- signals: &EnumMap<Signal, SignalScore>,-) -> f64 {- let lambda_score = match model {- Some(model) => match signal_coefficients {- Some(coefficients) => match coefficients.get(&Signal::LambdaMART) {- Some(coeff) => {- if coeff == 0.0 {- signals- .values()- .map(|score| score.coefficient * score.value)- .sum()- } else {- coeff * model.predict(signals)- }- }- None => Signal::LambdaMART.default_coefficient() * model.predict(signals),- },- None => Signal::LambdaMART.default_coefficient() * model.predict(signals),- },- None => signals- .values()- .map(|score| score.coefficient * score.value)- .sum(),- };-- lambda_score-}--#[derive(Default)]-struct Initial {- model: Option<Arc<LambdaMART>>,- signal_coefficients: Option<SignalCoefficient>,-}--impl<T: AsRankingWebsite> Scorer<T> for Initial {- fn score(&self, websites: &mut [T]) {- for website in websites {- let website = website.as_mut_ranking();- website.score =- calculate_score(&self.model, &self.signal_coefficients, &website.signals);- }- }-- fn set_query_info(&mut self, query: &SearchQuery) {- self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);- }-}--struct RankingStage<T> {- scorer: Box<dyn Scorer<T>>,- stage_top_n: usize,- derank_similar: bool,-}--impl<T: AsRankingWebsite> RankingStage<T> {- fn apply(- &self,- websites: Vec<T>,- top_n: usize,- offset: usize,- collector_config: CollectorConfig,- ) -> Vec<T> {- let mut websites = websites- .into_iter()- .skip(offset)- .take(self.stage_top_n.max(top_n))- .collect::<Vec<_>>();-- self.scorer.score(&mut websites);- for website in websites.iter_mut() {- let boost = website.as_ranking().optic_boost;- if let Some(boost) = boost {- if boost != 0.0 {- website.as_mut_ranking().score *= boost;- }- }- }-- let mut collector =- BucketCollector::new(self.stage_top_n.max(top_n) + offset, collector_config);-- for website in websites {- collector.insert(website);- }-- collector- .into_sorted_vec(self.derank_similar)- .into_iter()- .take(top_n)- .collect()- }-- fn set_query_info(&mut self, query: &SearchQuery) {- self.scorer.set_query_info(query);- }-}--pub struct RankingPipeline<T> {- stage: RankingStage<T>,- page: usize,- pub top_n: usize,- collector_config: CollectorConfig,-}--impl<T: AsRankingWebsite> RankingPipeline<T> {- fn create_reranking_stage<M: CrossEncoder + 'static>(- crossencoder: Option<Arc<M>>,- lambda: Option<Arc<LambdaMART>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Result<Self> {- let scorer = match crossencoder {- Some(cross_encoder) => {- Box::new(ReRanker::new(cross_encoder, lambda)) as Box<dyn Scorer<T>>- }- None => Box::new(IdentityScorer) as Box<dyn Scorer<T>>,- };-- let stage = RankingStage {- scorer,- stage_top_n: top_n_considered,- derank_similar: true,- };-- Ok(Self {- stage,- page: 0,- top_n: 0,- collector_config,- })- }-- pub fn reranker<M: CrossEncoder + 'static>(- query: &mut SearchQuery,- crossencoder: Option<Arc<M>>,- lambda: Option<Arc<LambdaMART>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Result<Self> {- let mut pipeline =- Self::create_reranking_stage(crossencoder, lambda, collector_config, top_n_considered)?;- pipeline.set_query_info(query);-- Ok(pipeline)- }-- fn create_recall_stage(- model: Option<Arc<LambdaMART>>,- collector_config: CollectorConfig,- stage_top_n: usize,- ) -> Self {- let last_stage = RankingStage {- scorer: Box::new(Initial {- model,- signal_coefficients: None,- }),- stage_top_n,- derank_similar: true,- };-- Self {- stage: last_stage,- page: 0,- top_n: 0,- collector_config,- }- }-- pub fn recall_stage(- query: &mut SearchQuery,- model: Option<Arc<LambdaMART>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Self {- let mut pipeline = Self::create_recall_stage(model, collector_config, top_n_considered);- pipeline.set_query_info(query);-- pipeline- }-- fn set_query_info(&mut self, query: &mut SearchQuery) {- self.stage.set_query_info(query);- self.page = query.page;- self.top_n = query.num_results;-- query.num_results = self.collector_top_n();- query.page = 0;- }-- pub fn offset(&self) -> usize {- self.top_n * self.page- }-- pub fn apply(self, websites: Vec<T>) -> Vec<T> {- if websites.len() <= 1 {- return websites;- }-- self.stage.apply(- websites,- self.top_n,- self.offset(),- self.collector_config.clone(),- )- }-- pub fn collector_top_n(&self) -> usize {- (self.initial_top_n().max(self.top_n) + self.top_n * self.page) + 1- }-- pub fn initial_top_n(&self) -> usize {- self.stage.stage_top_n.max(self.top_n)- }-}--#[cfg(test)]-mod tests {- use itertools::Itertools;-- use crate::ranking::models::cross_encoder::DummyCrossEncoder;- use crate::{- collector::Hashes, inverted_index::DocAddress, prehashed::Prehashed,- ranking::initial::Score,- };-- use super::*;-- fn sample_websites(n: usize) -> Vec<RankingWebsite> {- (0..n)- .map(|i| -> RankingWebsite {- let mut signals = EnumMap::new();- signals.insert(- Signal::HostCentrality,- SignalScore {- coefficient: 1.0,- value: 1.0 / i as f64,- },- );- RankingWebsite {- pointer: WebsitePointer {- score: Score { total: 0.0 },- hashes: Hashes {- site: Prehashed(0),- title: Prehashed(0),- url: Prehashed(0),- url_without_tld: Prehashed(0),- simhash: 0,- },- address: DocAddress {- segment: 0,- doc_id: i as u32,- },- },- signals,- optic_boost: None,- title: None,- snippet: None,- score: 1.0 / i as f64,- }- })- .collect()- }-- #[test]- fn simple() {- let pipeline = RankingPipeline::reranker(- &mut SearchQuery {- ..Default::default()- },- Some(Arc::new(DummyCrossEncoder {})),- None,- CollectorConfig::default(),- 20,- )- .unwrap();- assert_eq!(pipeline.collector_top_n(), 20 + 1);-- let sample = sample_websites(pipeline.collector_top_n());- let res: Vec<_> = pipeline- .apply(sample)- .into_iter()- .map(|w| w.pointer.address)- .collect();-- let expected: Vec<_> = sample_websites(100)- .into_iter()- .take(20)- .map(|w| w.pointer.address)- .collect();-- assert_eq!(res, expected);- }-- #[test]- fn top_n() {- let num_results = 100;- let pipeline = RankingPipeline::reranker(- &mut SearchQuery {- num_results,- ..Default::default()- },- Some(Arc::new(DummyCrossEncoder {})),- None,- CollectorConfig::default(),- num_results,- )- .unwrap();-- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());-- let expected: Vec<_> = sample- .clone()- .into_iter()- .take(num_results)- .map(|w| w.pointer.address)- .collect();-- let res = pipeline- .apply(sample)- .into_iter()- .map(|w| w.pointer.address)- .collect_vec();-- assert_eq!(res.len(), num_results);- assert_eq!(res, expected);- }-- #[test]- fn offsets() {- let num_results = 20;- let pipeline = RankingPipeline::reranker(- &mut SearchQuery {- page: 0,- num_results,- ..Default::default()- },- Some(Arc::new(DummyCrossEncoder {})),- None,- CollectorConfig::default(),- num_results,- )- .unwrap();-- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());- let mut prev: Vec<_> = pipeline.apply(sample);- for p in 1..1_000 {- let pipeline = RankingPipeline::reranker(- &mut SearchQuery {- page: p,- ..Default::default()- },- Some(Arc::new(DummyCrossEncoder {})),- None,- CollectorConfig::default(),- num_results,- )- .unwrap();-- let sample: Vec<_> = sample_websites(pipeline.collector_top_n());- let res: Vec<_> = pipeline.apply(sample).into_iter().collect();-- assert_eq!(- res.len(),- num_results,- "Every page should have {num_results} results"- );-- assert!(!prev- .iter()- .zip_eq(res.iter())- .any(|(p, r)| p.pointer.address.doc_id == r.pointer.address.doc_id));-- prev = res;- }- }-}diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsnew file mode 100644index 00000000..2a2b5ff8--- /dev/null+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -0,0 +1,339 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use crate::{+ collector::{self, BucketCollector},+ config::CollectorConfig,+ models::dual_encoder::DualEncoder,+ searcher::SearchQuery,+};++use super::{+ models::lambdamart::{self, LambdaMART},+ SignalScore,+};++mod scorers;+mod stages;++pub use scorers::{ReRanker, Recall, Scorer};+pub use stages::{PrecisionRankingWebpage, RecallRankingWebpage};++pub trait RankableWebpage: collector::Doc + Send + Sync {+ fn set_score(&mut self, score: f64);+ fn boost(&self) -> Option<f64>;++ fn boost_score(&mut self) {+ if let Some(boost) = self.boost() {+ if boost != 0.0 {+ let score = self.score() * boost;+ self.set_score(score);+ }+ }+ }+}++impl lambdamart::AsValue for SignalScore {+ fn as_value(&self) -> f64 {+ self.value+ }+}++struct RankingStage<T> {+ scorer: Box<dyn Scorer<T>>,+ stage_top_n: usize,+ derank_similar: bool,+}++impl<T: RankableWebpage> RankingStage<T> {+ fn apply(+ &self,+ websites: Vec<T>,+ top_n: usize,+ offset: usize,+ collector_config: CollectorConfig,+ ) -> Vec<T> {+ let mut websites = websites+ .into_iter()+ .skip(offset)+ .take(self.stage_top_n.max(top_n))+ .collect::<Vec<_>>();++ self.scorer.score(&mut websites);++ let mut collector =+ BucketCollector::new(self.stage_top_n.max(top_n) + offset, collector_config);++ for mut website in websites {+ website.boost_score();+ collector.insert(website);+ }++ collector+ .into_sorted_vec(self.derank_similar)+ .into_iter()+ .take(top_n)+ .collect()+ }++ fn set_query_info(&mut self, query: &SearchQuery) {+ self.scorer.set_query_info(query);+ }+}++pub struct RankingPipeline<T> {+ stage: RankingStage<T>,+ page: usize,+ pub top_n: usize,+ collector_config: CollectorConfig,+}++impl RankingPipeline<crate::searcher::api::ScoredWebpagePointer> {+ fn create_recall_stage(+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,+ stage_top_n: usize,+ ) -> Self {+ let last_stage = RankingStage {+ scorer: Box::new(Recall::<crate::searcher::api::ScoredWebpagePointer>::new(+ lambdamart,+ dual_encoder,+ )),+ stage_top_n,+ derank_similar: true,+ };++ Self {+ stage: last_stage,+ page: 0,+ top_n: 0,+ collector_config,+ }+ }++ pub fn recall_stage(+ query: &mut SearchQuery,+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,+ top_n_considered: usize,+ ) -> Self {+ let mut pipeline =+ Self::create_recall_stage(lambdamart, dual_encoder, collector_config, top_n_considered);+ pipeline.set_query_info(query);++ pipeline+ }+}++impl<T: RankableWebpage> RankingPipeline<T> {+ fn set_query_info(&mut self, query: &mut SearchQuery) {+ self.stage.set_query_info(query);+ self.page = query.page;+ self.top_n = query.num_results;++ query.num_results = self.collector_top_n();+ query.page = 0;+ }++ pub fn offset(&self) -> usize {+ self.top_n * self.page+ }++ pub fn apply(self, websites: Vec<T>) -> Vec<T> {+ if websites.len() <= 1 {+ return websites;+ }++ self.stage.apply(+ websites,+ self.top_n,+ self.offset(),+ self.collector_config.clone(),+ )+ }++ pub fn collector_top_n(&self) -> usize {+ (self.initial_top_n().max(self.top_n) + self.top_n * self.page) + 1+ }++ pub fn initial_top_n(&self) -> usize {+ self.stage.stage_top_n.max(self.top_n)+ }+}++#[cfg(test)]+mod tests {+ use itertools::Itertools;++ use crate::{+ collector::Hashes,+ enum_map::EnumMap,+ inverted_index::{DocAddress, WebpagePointer},+ prehashed::Prehashed,+ ranking::{initial::Score, Signal},+ };++ use super::*;++ fn sample_websites(n: usize) -> Vec<RecallRankingWebpage> {+ (0..n)+ .map(|i| -> RecallRankingWebpage {+ let mut signals = EnumMap::new();+ signals.insert(+ Signal::HostCentrality,+ SignalScore {+ coefficient: 1.0,+ value: 1.0 / i as f64,+ },+ );+ RecallRankingWebpage {+ pointer: WebpagePointer {+ score: Score { total: 0.0 },+ hashes: Hashes {+ site: Prehashed(0),+ title: Prehashed(0),+ url: Prehashed(0),+ url_without_tld: Prehashed(0),+ simhash: 0,+ },+ address: DocAddress {+ segment: 0,+ doc_id: i as u32,+ },+ },+ signals,+ optic_boost: None,+ title_embedding: None,+ keyword_embedding: None,+ score: 1.0 / i as f64,+ }+ })+ .collect()+ }++ #[test]+ fn simple() {+ let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ &mut SearchQuery {+ ..Default::default()+ },+ None,+ None,+ CollectorConfig::default(),+ 20,+ );+ assert_eq!(pipeline.collector_top_n(), 20 + 1);++ let sample = sample_websites(pipeline.collector_top_n());+ let res: Vec<_> = pipeline+ .apply(sample)+ .into_iter()+ .map(|w| w.pointer.address)+ .collect();++ let expected: Vec<_> = sample_websites(100)+ .into_iter()+ .take(20)+ .map(|w| w.pointer.address)+ .collect();++ assert_eq!(res, expected);+ }++ #[test]+ fn top_n() {+ let num_results = 100;+ let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ &mut SearchQuery {+ num_results,+ ..Default::default()+ },+ None,+ None,+ CollectorConfig::default(),+ num_results,+ );++ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());++ let expected: Vec<_> = sample+ .clone()+ .into_iter()+ .take(num_results)+ .map(|w| w.pointer.address)+ .collect();++ let res = pipeline+ .apply(sample)+ .into_iter()+ .map(|w| w.pointer.address)+ .collect_vec();++ assert_eq!(res.len(), num_results);+ assert_eq!(res, expected);+ }++ #[test]+ fn offsets() {+ let num_results = 20;+ let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ &mut SearchQuery {+ page: 0,+ num_results,+ ..Default::default()+ },+ None,+ None,+ CollectorConfig::default(),+ num_results,+ );++ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());+ let mut prev: Vec<_> = pipeline.apply(sample);+ for p in 1..1_000 {+ let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ &mut SearchQuery {+ page: p,+ ..Default::default()+ },+ None,+ None,+ CollectorConfig::default(),+ num_results,+ );++ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());+ let res: Vec<_> = pipeline.apply(sample).into_iter().collect();++ assert_eq!(+ res.len(),+ num_results,+ "Every page should have {num_results} results"+ );++ assert!(!prev+ .iter()+ .zip_eq(res.iter())+ .any(|(p, r)| p.pointer.address.doc_id == r.pointer.address.doc_id));++ prev = res;+ }+ }+}diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rsnew file mode 100644index 00000000..966b9442--- /dev/null+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs@@ -0,0 +1,292 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use candle_core::Tensor;++use crate::{+ models::dual_encoder::DualEncoder,+ ranking::{+ pipeline::{stages::StoredEmbeddings, RankableWebpage, RecallRankingWebpage},+ Signal, SignalCoefficient, SignalScore,+ },+ searcher::{api::ScoredWebpagePointer, SearchQuery},+ Result,+};++use super::Scorer;++pub struct Embedding(Tensor);++impl Embedding {+ pub fn dot(&self, other: &Self) -> Result<f64> {+ Ok(self+ .0+ .to_dtype(candle_core::DType::F16)?+ .unsqueeze(0)?+ .matmul(+ &other+ .0+ .to_dtype(candle_core::DType::F16)?+ .unsqueeze(0)?+ .t()?,+ )?+ .get(0)?+ .squeeze(0)?+ .to_dtype(candle_core::DType::F64)?+ .to_vec0()?)+ }++ pub fn size(&self) -> usize {+ self.0.dims1().unwrap()+ }+}++impl Embedding {+ fn new(stored: &StoredEmbeddings, size: usize) -> Result<Self> {+ let tensor = Tensor::from_raw_buffer(+ stored.as_slice(),+ candle_core::DType::BF16,+ &[size],+ &candle_core::Device::Cpu,+ )?;++ Ok(Self(tensor))+ }+}++pub struct EmbeddingScorer<W, E: EmbeddingSignal<W>> {+ dual_encoder: Option<Arc<DualEncoder>>,+ signal_coefficients: Option<SignalCoefficient>,+ query: Option<String>,+ _marker: std::marker::PhantomData<(E, W)>,+}++impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {+ pub fn new(dual_encoder: Option<Arc<DualEncoder>>) -> Self {+ Self {+ dual_encoder,+ signal_coefficients: None,+ query: None,+ _marker: std::marker::PhantomData,+ }+ }+}++impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {+ pub fn query_emb(&self) -> Option<Embedding> {+ self.dual_encoder.as_ref().and_then(|dual_encoder| {+ self.query.as_ref().and_then(|query| {+ dual_encoder+ .embed(&[query.to_string()])+ .ok()+ .and_then(|d| d.squeeze(0).ok())+ .map(Embedding)+ })+ })+ }++ fn query_emb_and_coefficient(&self, coeff_signal: Signal) -> Option<(Embedding, f64)> {+ self.query_emb().and_then(|query_emb| {+ self.signal_coefficients+ .as_ref()+ .map(|c| c.get(&coeff_signal))+ .map(|coefficient| (query_emb, coefficient))+ })+ }+}++impl RecallRankingWebpage {+ fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {+ self.title_embedding+ .as_ref()+ .and_then(|s| Embedding::new(s, hidden_size).ok())+ }++ fn keyword_emb(&self, hidden_size: usize) -> Option<Embedding> {+ self.keyword_embedding+ .as_ref()+ .and_then(|s| Embedding::new(s, hidden_size).ok())+ }+}++impl ScoredWebpagePointer {+ fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {+ self.as_ranking()+ .title_embedding+ .as_ref()+ .and_then(|s| Embedding::new(s, hidden_size).ok())+ }++ fn keyword_emb(&self, hidden_size: usize) -> Option<Embedding> {+ self.as_ranking()+ .keyword_embedding+ .as_ref()+ .and_then(|s| Embedding::new(s, hidden_size).ok())+ }+}++impl<W: RankableWebpage, E: EmbeddingSignal<W>> Scorer<W> for EmbeddingScorer<W, E> {+ fn score(&self, webpages: &mut [W]) {+ if !webpages.iter().any(E::has_embedding) {+ return;+ }++ if let Some((query_emb, coefficient)) = self.query_emb_and_coefficient(E::signal()) {+ let hidden_size = query_emb.size();+ for webpage in webpages.iter_mut() {+ if let Some(emb) = E::embedding(webpage, hidden_size) {+ let sim = query_emb.dot(&emb).unwrap_or_default();+ E::insert_signal(webpage, sim, coefficient);+ }+ }+ }+ }++ fn set_query_info(&mut self, query: &SearchQuery) {+ self.query = Some(query.query.clone());+ self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);+ }+}++pub struct TitleEmbeddings;+pub struct KeywordEmbeddings;++pub trait EmbeddingSignal<W>: Send + Sync {+ fn signal() -> Signal;+ fn has_embedding(webpage: &W) -> bool;+ fn embedding(webpage: &W, hidden_size: usize) -> Option<Embedding>;+ fn insert_signal(webpage: &mut W, score: f64, coefficient: f64);+}++impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {+ fn signal() -> Signal {+ Signal::TitleEmbeddingSimilarity+ }++ fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {+ webpage.as_ranking().title_embedding.is_some()+ }++ fn embedding(webpage: &ScoredWebpagePointer, hidden_size: usize) -> Option<Embedding> {+ webpage.title_emb(hidden_size)+ }++ fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64, coefficient: f64) {+ let sig = <TitleEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();+ webpage.as_ranking_mut().signals.insert(+ sig,+ SignalScore {+ coefficient,+ value: score,+ },+ );+ }+}++impl EmbeddingSignal<RecallRankingWebpage> for TitleEmbeddings {+ fn signal() -> Signal {+ Signal::TitleEmbeddingSimilarity+ }++ fn has_embedding(webpage: &RecallRankingWebpage) -> bool {+ webpage.title_embedding.is_some()+ }++ fn embedding(webpage: &RecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {+ webpage.title_emb(hidden_size)+ }++ fn insert_signal(webpage: &mut RecallRankingWebpage, score: f64, coefficient: f64) {+ let sig = <TitleEmbeddings as EmbeddingSignal<RecallRankingWebpage>>::signal();++ webpage.signals.insert(+ sig,+ SignalScore {+ coefficient,+ value: score,+ },+ );+ }+}++impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {+ fn signal() -> Signal {+ Signal::KeywordEmbeddingSimilarity+ }++ fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {+ webpage.as_ranking().keyword_embedding.is_some()+ }++ fn embedding(webpage: &ScoredWebpagePointer, hidden_size: usize) -> Option<Embedding> {+ webpage.keyword_emb(hidden_size)+ }++ fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64, coefficient: f64) {+ let sig = <KeywordEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();+ webpage.as_ranking_mut().signals.insert(+ sig,+ SignalScore {+ coefficient,+ value: score,+ },+ );+ }+}++impl EmbeddingSignal<RecallRankingWebpage> for KeywordEmbeddings {+ fn signal() -> Signal {+ Signal::KeywordEmbeddingSimilarity+ }++ fn has_embedding(webpage: &RecallRankingWebpage) -> bool {+ webpage.keyword_embedding.is_some()+ }++ fn embedding(webpage: &RecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {+ webpage.keyword_emb(hidden_size)+ }++ fn insert_signal(webpage: &mut RecallRankingWebpage, score: f64, coefficient: f64) {+ let sig = <KeywordEmbeddings as EmbeddingSignal<RecallRankingWebpage>>::signal();+ webpage.signals.insert(+ sig,+ SignalScore {+ coefficient,+ value: score,+ },+ );+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn embedding_dot() {+ let emb1 = Embedding(+ Tensor::from_vec(vec![1.0, 2.0, 3.0], &[3], &candle_core::Device::Cpu).unwrap(),+ );+ let emb2 = Embedding(+ Tensor::from_vec(vec![4.0, 5.0, 6.0], &[3], &candle_core::Device::Cpu).unwrap(),+ );++ assert_eq!(emb1.dot(&emb2).unwrap(), 32.0);+ }+}diff --git a/crates/core/src/ranking/pipeline/scorers/mod.rs b/crates/core/src/ranking/pipeline/scorers/mod.rsnew file mode 100644index 00000000..1c0ffc53--- /dev/null+++ b/crates/core/src/ranking/pipeline/scorers/mod.rs@@ -0,0 +1,93 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub mod embedding;+pub mod recall;+pub mod reranker;++use std::sync::Arc;++pub use recall::Recall;+pub use reranker::ReRanker;++use crate::{+ enum_map::EnumMap,+ ranking::{models::lambdamart::LambdaMART, Signal, SignalCoefficient, SignalScore},+ searcher::SearchQuery,+};++use super::RankableWebpage;++pub trait Scorer<T: RankableWebpage>: Send + Sync {+ fn score(&self, webpages: &mut [T]);+ fn set_query_info(&mut self, _query: &SearchQuery) {}+}++pub struct IdentityScorer;++impl<T: RankableWebpage> Scorer<T> for IdentityScorer {+ fn score(&self, _webpages: &mut [T]) {}+}++fn calculate_score(+ model: &Option<Arc<LambdaMART>>,+ coefficients: SignalCoefficient,+ signals: &EnumMap<Signal, SignalScore>,+) -> f64 {+ let lambda_score = match model {+ Some(model) => {+ let coeff = coefficients.get(&Signal::LambdaMART);+ if coeff == 0.0 {+ signals+ .values()+ .map(|score| score.coefficient * score.value)+ .sum()+ } else {+ coeff * model.predict(signals)+ }+ }+ None => signals+ .values()+ .map(|score| score.coefficient * score.value)+ .sum(),+ };++ lambda_score+}++pub struct MultiScorer<T: RankableWebpage> {+ scorers: Vec<Box<dyn Scorer<T>>>,+}++impl<T: RankableWebpage> MultiScorer<T> {+ pub fn new(scorers: Vec<Box<dyn Scorer<T>>>) -> Self {+ Self { scorers }+ }+}++impl<T: RankableWebpage> Scorer<T> for MultiScorer<T> {+ fn score(&self, webpages: &mut [T]) {+ for scorer in &self.scorers {+ scorer.score(webpages);+ }+ }++ fn set_query_info(&mut self, query: &SearchQuery) {+ for scorer in &mut self.scorers {+ scorer.set_query_info(query);+ }+ }+}diff --git a/crates/core/src/ranking/pipeline/scorers/recall.rs b/crates/core/src/ranking/pipeline/scorers/recall.rsnew file mode 100644index 00000000..5dbdc3ef--- /dev/null+++ b/crates/core/src/ranking/pipeline/scorers/recall.rs@@ -0,0 +1,120 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use crate::{+ models::dual_encoder::DualEncoder,+ ranking::{+ models::lambdamart::LambdaMART,+ pipeline::{RankableWebpage, RecallRankingWebpage},+ SignalCoefficient,+ },+ searcher::{api::ScoredWebpagePointer, SearchQuery},+};++use super::{+ calculate_score,+ embedding::{EmbeddingScorer, KeywordEmbeddings, TitleEmbeddings},+ MultiScorer, Scorer,+};++pub struct Recall<T: RankableWebpage> {+ sub_scorers: MultiScorer<T>,+ lambdamart: Option<Arc<LambdaMART>>,+ signal_coefficients: Option<SignalCoefficient>,+}++impl Recall<RecallRankingWebpage> {+ pub fn new(+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ ) -> Self {+ Self {+ lambdamart,+ signal_coefficients: None,+ sub_scorers: MultiScorer::new(vec![+ Box::new(+ EmbeddingScorer::<RecallRankingWebpage, TitleEmbeddings>::new(+ dual_encoder.clone(),+ ),+ ),+ Box::new(+ EmbeddingScorer::<RecallRankingWebpage, KeywordEmbeddings>::new(dual_encoder),+ ),+ ]),+ }+ }+}++impl Recall<ScoredWebpagePointer> {+ pub fn new(+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ ) -> Self {+ Self {+ lambdamart,+ signal_coefficients: None,+ sub_scorers: MultiScorer::new(vec![+ Box::new(+ EmbeddingScorer::<ScoredWebpagePointer, TitleEmbeddings>::new(+ dual_encoder.clone(),+ ),+ ),+ Box::new(+ EmbeddingScorer::<ScoredWebpagePointer, KeywordEmbeddings>::new(dual_encoder),+ ),+ ]),+ }+ }+}++impl Scorer<RecallRankingWebpage> for Recall<RecallRankingWebpage> {+ fn score(&self, webpages: &mut [RecallRankingWebpage]) {+ self.sub_scorers.score(webpages);++ for webpage in webpages {+ webpage.set_score(calculate_score(+ &self.lambdamart,+ self.signal_coefficients.clone().unwrap_or_default(),+ &webpage.signals,+ ));+ }+ }++ fn set_query_info(&mut self, query: &SearchQuery) {+ self.sub_scorers.set_query_info(query);+ self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);+ }+}++impl Scorer<ScoredWebpagePointer> for Recall<ScoredWebpagePointer> {+ fn score(&self, webpages: &mut [crate::searcher::api::ScoredWebpagePointer]) {+ self.sub_scorers.score(webpages);++ for webpage in webpages {+ webpage.set_score(calculate_score(+ &self.lambdamart,+ self.signal_coefficients.clone().unwrap_or_default(),+ &webpage.as_ranking().signals,+ ));+ }+ }++ fn set_query_info(&mut self, query: &SearchQuery) {+ self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);+ }+}diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rsnew file mode 100644index 00000000..b450201d--- /dev/null+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs@@ -0,0 +1,113 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use crate::{+ ranking::{+ models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},+ Signal, SignalCoefficient, SignalScore,+ },+ searcher::SearchQuery,+};++use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, Scorer};++use super::calculate_score;++pub struct ReRanker<M: CrossEncoder> {+ crossencoder: Arc<M>,+ lambda_mart: Option<Arc<LambdaMART>>,+ query: Option<SearchQuery>,+ signal_coefficients: Option<SignalCoefficient>,+}++impl<M: CrossEncoder> ReRanker<M> {+ pub fn new(crossencoder: Arc<M>, lambda: Option<Arc<LambdaMART>>) -> Self {+ Self {+ crossencoder,+ lambda_mart: lambda,+ query: None,+ signal_coefficients: None,+ }+ }++ fn crossencoder_snippet_coeff(&self) -> f64 {+ self.signal_coefficients+ .as_ref()+ .map(|coeff| coeff.get(&Signal::CrossEncoderSnippet))+ .unwrap_or_else(|| Signal::CrossEncoderSnippet.default_coefficient())+ }++ fn crossencoder_title_coeff(&self) -> f64 {+ self.signal_coefficients+ .as_ref()+ .map(|coeff| coeff.get(&Signal::CrossEncoderTitle))+ .unwrap_or_else(|| Signal::CrossEncoderTitle.default_coefficient())+ }++ fn crossencoder_score_webpages(&self, webpage: &mut [PrecisionRankingWebpage]) {+ let mut snippets = Vec::with_capacity(webpage.len());+ let mut titles = Vec::with_capacity(webpage.len());++ for webpage in webpage.iter_mut() {+ titles.push(webpage.retrieved_webpage.title.clone());+ snippets.push(webpage.retrieved_webpage.snippet.unhighlighted_string());+ }++ let query = &self.query.as_ref().unwrap().query;+ let snippet_scores = self.crossencoder.run(query, &snippets);+ let title_scores = self.crossencoder.run(query, &titles);++ for ((webpage, snippet), title) in webpage.iter_mut().zip(snippet_scores).zip(title_scores)+ {+ webpage.ranking.signals.insert(+ Signal::CrossEncoderSnippet,+ SignalScore {+ coefficient: self.crossencoder_snippet_coeff(),+ value: snippet,+ },+ );++ webpage.ranking.signals.insert(+ Signal::CrossEncoderTitle,+ SignalScore {+ coefficient: self.crossencoder_title_coeff(),+ value: title,+ },+ );+ }+ }+}++impl<M: CrossEncoder> Scorer<PrecisionRankingWebpage> for ReRanker<M> {+ fn score(&self, webpages: &mut [PrecisionRankingWebpage]) {+ self.crossencoder_score_webpages(webpages);++ for webpage in webpages.iter_mut() {+ webpage.set_score(calculate_score(+ &self.lambda_mart,+ self.signal_coefficients.clone().unwrap_or_default(),+ &webpage.ranking.signals,+ ));+ }+ }++ fn set_query_info(&mut self, query: &SearchQuery) {+ self.query = Some(query.clone());+ self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);+ }+}diff --git a/crates/core/src/ranking/pipeline/stages/mod.rs b/crates/core/src/ranking/pipeline/stages/mod.rsnew file mode 100644index 00000000..c821db4a--- /dev/null+++ b/crates/core/src/ranking/pipeline/stages/mod.rs@@ -0,0 +1,21 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod precision;+mod recall;++pub use precision::PrecisionRankingWebpage;+pub use recall::{RecallRankingWebpage, StoredEmbeddings};diff --git a/crates/core/src/ranking/pipeline/stages/precision.rs b/crates/core/src/ranking/pipeline/stages/precision.rsnew file mode 100644index 00000000..5d9f011d--- /dev/null+++ b/crates/core/src/ranking/pipeline/stages/precision.rs@@ -0,0 +1,117 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use serde::{Deserialize, Serialize};++use crate::{+ collector,+ config::CollectorConfig,+ inverted_index::RetrievedWebpage,+ ranking::{+ models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},+ pipeline::{+ scorers::IdentityScorer, RankableWebpage, RankingPipeline, RankingStage, ReRanker,+ Scorer,+ },+ },+ searcher::SearchQuery,+ Result,+};++use super::RecallRankingWebpage;++#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct PrecisionRankingWebpage {+ pub retrieved_webpage: RetrievedWebpage,+ pub ranking: RecallRankingWebpage,+}++impl collector::Doc for PrecisionRankingWebpage {+ fn score(&self) -> f64 {+ self.ranking.score+ }++ fn hashes(&self) -> collector::Hashes {+ self.ranking.pointer.hashes+ }+}++impl RankableWebpage for PrecisionRankingWebpage {+ fn set_score(&mut self, score: f64) {+ self.ranking.score = score;+ }++ fn boost(&self) -> Option<f64> {+ self.ranking.optic_boost+ }+}++impl PrecisionRankingWebpage {+ pub fn new(retrieved_webpage: RetrievedWebpage, ranking: RecallRankingWebpage) -> Self {+ Self {+ retrieved_webpage,+ ranking,+ }+ }++ pub fn into_retrieved_webpage(self) -> RetrievedWebpage {+ self.retrieved_webpage+ }+}++impl RankingPipeline<PrecisionRankingWebpage> {+ fn create_reranking_stage<M: CrossEncoder + 'static>(+ crossencoder: Option<Arc<M>>,+ lambda: Option<Arc<LambdaMART>>,+ collector_config: CollectorConfig,+ top_n_considered: usize,+ ) -> Result<Self> {+ let scorer = match crossencoder {+ Some(cross_encoder) => Box::new(ReRanker::new(cross_encoder, lambda))+ as Box<dyn Scorer<PrecisionRankingWebpage>>,+ None => Box::new(IdentityScorer) as Box<dyn Scorer<PrecisionRankingWebpage>>,+ };++ let stage = RankingStage {+ scorer,+ stage_top_n: top_n_considered,+ derank_similar: true,+ };++ Ok(Self {+ stage,+ page: 0,+ top_n: 0,+ collector_config,+ })+ }++ pub fn reranker<M: CrossEncoder + 'static>(+ query: &mut SearchQuery,+ crossencoder: Option<Arc<M>>,+ lambda: Option<Arc<LambdaMART>>,+ collector_config: CollectorConfig,+ top_n_considered: usize,+ ) -> Result<Self> {+ let mut pipeline =+ Self::create_reranking_stage(crossencoder, lambda, collector_config, top_n_considered)?;+ pipeline.set_query_info(query);++ Ok(pipeline)+ }+}diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsnew file mode 100644index 00000000..56a9f93b--- /dev/null+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -0,0 +1,150 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Arc;++use serde::{Deserialize, Serialize};++use crate::{+ collector,+ config::CollectorConfig,+ enum_map::EnumMap,+ fastfield_reader,+ inverted_index::WebpagePointer,+ models::dual_encoder::DualEncoder,+ ranking::{+ models::lambdamart::LambdaMART,+ pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},+ Signal, SignalAggregator, SignalScore,+ },+ schema::FastField,+ searcher::SearchQuery,+};++#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct StoredEmbeddings(Vec<u8>);++impl StoredEmbeddings {+ pub fn as_slice(&self) -> &[u8] {+ &self.0+ }+}++#[derive(Clone, Debug, Serialize, Deserialize)]+pub struct RecallRankingWebpage {+ pub pointer: WebpagePointer,+ pub signals: EnumMap<Signal, SignalScore>,+ pub optic_boost: Option<f64>,+ pub title_embedding: Option<StoredEmbeddings>,+ pub keyword_embedding: Option<StoredEmbeddings>,+ pub score: f64,+}++impl RecallRankingWebpage {+ pub fn new(+ pointer: WebpagePointer,+ fastfield_reader: &fastfield_reader::SegmentReader,+ aggregator: &mut SignalAggregator,+ ) -> Self {+ let fastfields = fastfield_reader.get_field_reader(pointer.address.doc_id);++ let title_embedding: Option<Vec<u8>> = fastfields+ .get(FastField::TitleEmbeddings)+ .and_then(|v| v.into());++ let keyword_embedding: Option<Vec<u8>> = fastfields+ .get(FastField::KeywordEmbeddings)+ .and_then(|v| v.into());++ let mut res = RecallRankingWebpage {+ signals: EnumMap::new(),+ score: pointer.score.total,+ optic_boost: None,+ pointer: pointer.clone(),+ title_embedding: title_embedding.map(StoredEmbeddings),+ keyword_embedding: keyword_embedding.map(StoredEmbeddings),+ };++ for computed_signal in aggregator.compute_signals(pointer.address.doc_id).flatten() {+ res.signals+ .insert(computed_signal.signal, computed_signal.score);+ }++ if let Some(boost) = aggregator.boosts(pointer.address.doc_id) {+ res.optic_boost = Some(boost);+ }++ res+ }+}+impl RankableWebpage for RecallRankingWebpage {+ fn set_score(&mut self, score: f64) {+ self.score = score;+ }++ fn boost(&self) -> Option<f64> {+ self.optic_boost+ }+}++impl collector::Doc for RecallRankingWebpage {+ fn score(&self) -> f64 {+ self.score+ }++ fn hashes(&self) -> collector::Hashes {+ self.pointer.hashes+ }+}++impl RankingPipeline<RecallRankingWebpage> {+ fn create_recall_stage(+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,+ stage_top_n: usize,+ ) -> Self {+ let last_stage = RankingStage {+ scorer: Box::new(Recall::<RecallRankingWebpage>::new(+ lambdamart,+ dual_encoder,+ )) as Box<dyn Scorer<RecallRankingWebpage>>,+ stage_top_n,+ derank_similar: true,+ };++ Self {+ stage: last_stage,+ page: 0,+ top_n: 0,+ collector_config,+ }+ }++ pub fn recall_stage(+ query: &mut SearchQuery,+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,+ top_n_considered: usize,+ ) -> Self {+ let mut pipeline =+ Self::create_recall_stage(lambdamart, dual_encoder, collector_config, top_n_considered);+ pipeline.set_query_info(query);++ pipeline+ }+}diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rsindex fb79110e..0900d39c 100644--- a/crates/core/src/ranking/signal.rs+++ b/crates/core/src/ranking/signal.rs@@ -134,6 +134,10 @@ pub enum Signal {UrlSlashes,#[serde(rename = "link_density")]LinkDensity,+ #[serde(rename = "title_embedding_similarity")]+ TitleEmbeddingSimilarity,+ #[serde(rename = "keyword_embedding_similarity")]+ KeywordEmbeddingSimilarity,}impl From<Signal> for usize {@@ -142,7 +146,7 @@ impl From<Signal> for usize {}}-pub const ALL_SIGNALS: [Signal; 38] = [+pub const ALL_SIGNALS: [Signal; 40] = [Signal::Bm25Title,Signal::Bm25TitleBigrams,Signal::Bm25TitleTrigrams,@@ -181,6 +185,8 @@ pub const ALL_SIGNALS: [Signal; 38] = [Signal::UrlDigits,Signal::UrlSlashes,Signal::LinkDensity,+ Signal::TitleEmbeddingSimilarity,+ Signal::KeywordEmbeddingSimilarity,];fn score_timestamp(timestamp: usize, signal_aggregator: &SignalAggregator) -> f64 {@@ -288,6 +294,10 @@ fn idf_sum(field: &mut TextFieldData, doc: DocId) -> f64 {impl Signal {fn is_computable_before_search(&self) -> bool {self.as_fastfield().is_some()+ && !matches!(+ self,+ Signal::TitleEmbeddingSimilarity | Signal::KeywordEmbeddingSimilarity+ )}pub fn default_coefficient(&self) -> f64 {@@ -329,7 +339,9 @@ impl Signal {Signal::LambdaMART => 10.0,Signal::UrlSlashes => 0.01,Signal::UrlDigits => 0.01,- Signal::LinkDensity => 0.00,+ Signal::LinkDensity => 0.0,+ Signal::TitleEmbeddingSimilarity => 0.01,+ Signal::KeywordEmbeddingSimilarity => 0.01,}}@@ -344,9 +356,13 @@ impl Signal {.as_ref().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(&doc);+ let fastfield_reader = seg_reader.fastfield_reader.get_field_reader(doc);++ let node_id = fastfield_reader+ .get(FastField::HostNodeID)+ .and_then(|n| n.as_u64())+ .unwrap();- let node_id = fastfield_reader.get(&FastField::HostNodeID);let host_id: Option<NodeID> = if node_id == u64::MAX {None} else {@@ -355,23 +371,38 @@ impl Signal {let value: Option<f64> = match self {Signal::HostCentrality | Signal::PageCentrality => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();Some(val as f64 / FLOAT_SCALING as f64)}Signal::HostCentralityRank | Signal::PageCentralityRank => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();Some(score_rank(val as f64))}Signal::IsHomepage => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();Some(val as f64)}Signal::LinkDensity => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();Some(score_link_density(val as f64 / FLOAT_SCALING as f64))}Signal::FetchTimeMs => {- let fetch_time_ms = fastfield_reader.get(&self.as_fastfield().unwrap()) as usize;+ let fetch_time_ms = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap() as usize;if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {Some(0.0)@@ -380,24 +411,39 @@ impl Signal {}}Signal::UpdateTimestamp => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap()) as usize;+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap() as usize;Some(score_timestamp(val, signal_aggregator))}Signal::TrackerScore => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();Some(score_trackers(val as f64))}Signal::UrlDigits => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();Some(score_digits(val as f64))}Signal::UrlSlashes => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();Some(score_slashes(val as f64))}Signal::Region => {- let val = fastfield_reader.get(&self.as_fastfield().unwrap());+ let val = fastfield_reader+ .get(self.as_fastfield().unwrap())+ .and_then(|v| v.as_u64())+ .unwrap();let region = Region::from_id(val);Some(score_region(region, signal_aggregator))}@@ -436,8 +482,11 @@ impl Signal {.get_mut(self.as_textfield().unwrap()).map(|field| idf_sum(field, doc)),- Signal::CrossEncoderSnippet => None, // this is calculated in a later step- Signal::CrossEncoderTitle => None, // this is calculated in a later step+ Signal::CrossEncoderSnippet+ | Signal::CrossEncoderTitle+ | Signal::TitleEmbeddingSimilarity+ | Signal::KeywordEmbeddingSimilarity => None, // these are calculated in a later step+Signal::LambdaMART => None,};@@ -458,9 +507,9 @@ impl Signal {let value = match self {Signal::HostCentrality => Some(webpage.host_centrality),- Signal::HostCentralityRank => Some(webpage.host_centrality_rank),Signal::PageCentrality => Some(webpage.page_centrality),- Signal::PageCentralityRank => Some(webpage.page_centrality_rank),+ Signal::HostCentralityRank => Some(score_rank(webpage.host_centrality_rank as f64)),+ Signal::PageCentralityRank => Some(score_rank(webpage.page_centrality_rank as f64)),Signal::IsHomepage => Some(webpage.html.is_homepage().into()),Signal::FetchTimeMs => {let fetch_time_ms = webpage.fetch_time_ms as usize;@@ -544,6 +593,8 @@ impl Signal {| Signal::CrossEncoderTitle| Signal::InboundSimilarity| Signal::LambdaMART+ | Signal::TitleEmbeddingSimilarity+ | Signal::KeywordEmbeddingSimilarity| Signal::QueryCentrality => {tracing::error!("signal {self:?} cannot be precomputed");None@@ -573,6 +624,8 @@ impl Signal {Signal::UrlSlashes => Some(FastField::NumPathAndQuerySlashes),Signal::UrlDigits => Some(FastField::NumPathAndQueryDigits),Signal::LinkDensity => Some(FastField::LinkDensity),+ Signal::TitleEmbeddingSimilarity => Some(FastField::TitleEmbeddings),+ Signal::KeywordEmbeddingSimilarity => Some(FastField::KeywordEmbeddings),_ => None,}}@@ -625,8 +678,11 @@ pub struct SignalCoefficient {}impl SignalCoefficient {- pub fn get(&self, signal: &Signal) -> Option<f64> {- self.map.get(*signal).copied()+ pub fn get(&self, signal: &Signal) -> f64 {+ self.map+ .get(*signal)+ .copied()+ .unwrap_or(signal.default_coefficient())}pub fn new(coefficients: impl Iterator<Item = (Signal, f64)>) -> Self {@@ -651,7 +707,7 @@ impl SignalCoefficient {pub fn merge_into(&mut self, coeffs: SignalCoefficient) {for signal in ALL_SIGNALS {- if let Some(coeff) = coeffs.get(&signal) {+ if let Some(coeff) = coeffs.map.get(signal).copied() {match self.map.get_mut(signal) {Some(existing_coeff) => *existing_coeff += coeff,None => {@@ -997,7 +1053,7 @@ impl SignalAggregator {pub fn coefficient(&self, signal: &Signal) -> f64 {self.query_signal_coefficients.as_ref()- .and_then(|coefficients| coefficients.get(signal))+ .map(|coefficients| coefficients.get(signal)).or_else(|| {self.linear_regression.as_ref()diff --git a/crates/core/src/schema.rs b/crates/core/src/schema.rsindex 253822cd..c4c4228b 100644--- a/crates/core/src/schema.rs+++ b/crates/core/src/schema.rs@@ -14,7 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::schema::{IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions};+use tantivy::{+ schema::{BytesOptions, IndexRecordOption, NumericOptions, TextFieldIndexing, TextOptions},+ DateOptions,+};use crate::tokenizer::{BigramTokenizer, Identity, JsonField, SiteOperatorUrlTokenizer, Tokenizer, TrigramTokenizer,@@ -316,6 +319,8 @@ pub enum FastField {LikelyHasAds,LikelyHasPaywall,LinkDensity,+ TitleEmbeddings,+ KeywordEmbeddings,}impl FastField {@@ -358,6 +363,8 @@ impl FastField {FastField::LikelyHasAds => "likely_has_ads",FastField::LikelyHasPaywall => "likely_has_paywall",FastField::LinkDensity => "link_density",+ FastField::TitleEmbeddings => "title_embeddings",+ FastField::KeywordEmbeddings => "keyword_embeddings",}}}@@ -374,7 +381,7 @@ pub enum Field {Text(TextField),}-static ALL_FIELDS: [Field; 67] = [+static ALL_FIELDS: [Field; 70] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -443,6 +450,9 @@ static ALL_FIELDS: [Field; 67] = [Field::Fast(FastField::NumPathAndQueryDigits),Field::Fast(FastField::LikelyHasAds),Field::Fast(FastField::LikelyHasPaywall),+ Field::Fast(FastField::LinkDensity),+ Field::Fast(FastField::TitleEmbeddings),+ Field::Fast(FastField::KeywordEmbeddings),];impl Field {@@ -703,6 +713,12 @@ impl Field {Field::Fast(FastField::LinkDensity) => {IndexingOption::Integer(NumericOptions::default().set_fast().set_stored())}+ Field::Fast(FastField::TitleEmbeddings) => {+ IndexingOption::Bytes(BytesOptions::default().set_fast())+ }+ Field::Fast(FastField::KeywordEmbeddings) => {+ IndexingOption::Bytes(BytesOptions::default().set_fast())+ }}}@@ -768,6 +784,7 @@ pub fn create_schema() -> tantivy::schema::Schema {IndexingOption::Text(options) => builder.add_text_field(field.name(), options),IndexingOption::Integer(options) => builder.add_u64_field(field.name(), options),IndexingOption::DateTime(options) => builder.add_date_field(field.name(), options),+ IndexingOption::Bytes(options) => builder.add_bytes_field(field.name(), options),};}@@ -775,13 +792,15 @@ pub fn create_schema() -> tantivy::schema::Schema {}pub enum IndexingOption {- Text(tantivy::schema::TextOptions),- Integer(tantivy::schema::NumericOptions),- DateTime(tantivy::schema::DateOptions),+ Text(TextOptions),+ Integer(NumericOptions),+ DateTime(DateOptions),+ Bytes(BytesOptions),}pub enum DataType {U64,+ Bytes,}impl FastField {@@ -824,6 +843,8 @@ impl FastField {FastField::LikelyHasAds => DataType::U64,FastField::LikelyHasPaywall => DataType::U64,FastField::LinkDensity => DataType::U64,+ FastField::TitleEmbeddings => DataType::Bytes,+ FastField::KeywordEmbeddings => DataType::Bytes,}}}diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 99ba2f64..9ad8846a 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -25,12 +25,13 @@ use itertools::{intersperse, Itertools};use url::Url;use crate::bangs::{Bang, BangHit};-use crate::collector::Doc;+use crate::collector::{self, Doc};use crate::config::{ApiConfig, CollectorConfig};use crate::image_store::Image;use crate::inverted_index::RetrievedWebpage;+use crate::models::dual_encoder::DualEncoder;use crate::ranking::models::cross_encoder::CrossEncoderModel;-use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite, RetrievedWebpageRanking};+use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};use crate::ranking::ALL_SIGNALS;use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};use crate::web_spell::SpellChecker;@@ -48,33 +49,53 @@ use self::widget::WidgetManager;use super::{distributed, live, SearchQuery, SearchResult, WebsitesResult};#[derive(Clone)]-pub enum ScoredWebsitePointer {- Normal(distributed::ScoredWebsitePointer),- Live(live::ScoredWebsitePointer),+pub enum ScoredWebpagePointer {+ Normal(distributed::ScoredWebpagePointer),+ Live(live::ScoredWebpagePointer),}-impl AsRankingWebsite for ScoredWebsitePointer {- fn as_ranking(&self) -> &RankingWebsite {+impl ScoredWebpagePointer {+ pub fn as_ranking(&self) -> &RecallRankingWebpage {match self {- ScoredWebsitePointer::Normal(p) => &p.website,- ScoredWebsitePointer::Live(p) => &p.website,+ ScoredWebpagePointer::Normal(p) => &p.website,+ ScoredWebpagePointer::Live(p) => &p.website,}}- fn as_mut_ranking(&mut self) -> &mut RankingWebsite {+ pub fn as_ranking_mut(&mut self) -> &mut RecallRankingWebpage {match self {- ScoredWebsitePointer::Normal(p) => &mut p.website,- ScoredWebsitePointer::Live(p) => &mut p.website,+ ScoredWebpagePointer::Normal(p) => &mut p.website,+ ScoredWebpagePointer::Live(p) => &mut p.website,}}}+impl RankableWebpage for ScoredWebpagePointer {+ fn set_score(&mut self, score: f64) {+ self.as_ranking_mut().set_score(score);+ }++ fn boost(&self) -> Option<f64> {+ self.as_ranking().boost()+ }+}++impl collector::Doc for ScoredWebpagePointer {+ fn score(&self) -> f64 {+ self.as_ranking().score()+ }++ fn hashes(&self) -> collector::Hashes {+ self.as_ranking().hashes()+ }+}+pub fn combine_results(collector_config: CollectorConfig,initial_results: Vec<distributed::InitialSearchResultShard>,live_results: Vec<live::InitialSearchResultSplit>,- pipeline: RankingPipeline<ScoredWebsitePointer>,-) -> (Vec<ScoredWebsitePointer>, bool) {+ pipeline: RankingPipeline<ScoredWebpagePointer>,+) -> (Vec<ScoredWebpagePointer>, bool) {let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);let mut has_more = false;@@ -84,12 +105,12 @@ pub fn combine_results(}for website in result.local_result.websites {- let pointer = distributed::ScoredWebsitePointer {+ let pointer = distributed::ScoredWebpagePointer {website,shard: result.shard,};- let pointer = ScoredWebsitePointer::Normal(pointer);+ let pointer = ScoredWebpagePointer::Normal(pointer);collector.insert(pointer);}@@ -101,12 +122,12 @@ pub fn combine_results(}for website in result.local_result.websites {- let pointer = live::ScoredWebsitePointer {+ let pointer = live::ScoredWebpagePointer {website,split_id: result.split_id.clone(),};- let pointer = ScoredWebsitePointer::Live(pointer);+ let pointer = ScoredWebpagePointer::Live(pointer);collector.insert(pointer);}@@ -122,7 +143,7 @@ pub fn combine_results((res, has_more)}-pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[ScoredWebsitePointer]) {+pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[ScoredWebpagePointer]) {for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {let mut signals = HashMap::with_capacity(ALL_SIGNALS.len());@@ -142,6 +163,7 @@ pub struct ApiSearcher<S, L> {live_searcher: Option<L>,cross_encoder: Option<Arc<CrossEncoderModel>>,lambda_model: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,bangs: Bangs,collector_config: CollectorConfig,widget_manager: WidgetManager,@@ -158,6 +180,7 @@ wherelive_searcher: Option<L>,cross_encoder: Option<CrossEncoderModel>,lambda_model: Option<LambdaMART>,+ dual_encoder: Option<DualEncoder>,bangs: Bangs,config: ApiConfig,) -> Self {@@ -166,6 +189,7 @@ whereSidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());let lambda_model = lambda_model.map(Arc::new);+ let dual_encoder = dual_encoder.map(Arc::new);let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());@@ -175,6 +199,7 @@ wherelive_searcher,cross_encoder: cross_encoder.map(Arc::new),lambda_model,+ dual_encoder,bangs,collector_config: config.collector,widget_manager,@@ -290,13 +315,13 @@ whereasync fn retrieve_webpages(&self,query: &str,- top_websites: &[ScoredWebsitePointer],- ) -> Vec<RetrievedWebpageRanking> {+ top_websites: &[ScoredWebpagePointer],+ ) -> Vec<PrecisionRankingWebpage> {let normal: Vec<_> = top_websites.iter().enumerate().filter_map(|(i, pointer)| {- if let ScoredWebsitePointer::Normal(p) = pointer {+ if let ScoredWebpagePointer::Normal(p) = pointer {Some((i, p.clone()))} else {None@@ -308,7 +333,7 @@ where.iter().enumerate().filter_map(|(i, pointer)| {- if let ScoredWebsitePointer::Live(p) = pointer {+ if let ScoredWebpagePointer::Live(p) = pointer {Some((i, p.clone()))} else {None@@ -343,9 +368,9 @@ whereasync fn retrieve_webpages_from_live(&self,- pointers: &[(usize, live::ScoredWebsitePointer)],+ pointers: &[(usize, live::ScoredWebpagePointer)],query: &str,- ) -> Vec<(usize, RetrievedWebpageRanking)> {+ ) -> Vec<(usize, PrecisionRankingWebpage)> {match &self.live_searcher {Some(searcher) => searcher.retrieve_webpages(pointers, query).await,None => vec![],@@ -364,12 +389,14 @@ where// This pipeline should be created before the first search is performed// so the query knows how many results to fetch from the indices- let recall_pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::recall_stage(- &mut search_query,- self.lambda_model.clone(),- self.collector_config.clone(),- top_n,- );+ let recall_pipeline: RankingPipeline<ScoredWebpagePointer> =+ RankingPipeline::<ScoredWebpagePointer>::recall_stage(+ &mut search_query,+ self.lambda_model.clone(),+ self.dual_encoder.clone(),+ self.collector_config.clone(),+ top_n,+ );let (initial_results, live_results) = tokio::join!(self.distributed_searcher.search_initial(&search_query),@@ -397,8 +424,8 @@ where..query.clone()};- let reranking_pipeline: RankingPipeline<RetrievedWebpageRanking> =- RankingPipeline::reranker(+ let reranking_pipeline: RankingPipeline<PrecisionRankingWebpage> =+ RankingPipeline::<PrecisionRankingWebpage>::reranker(&mut search_query,self.cross_encoder.clone(),self.lambda_model.clone(),diff --git a/crates/core/src/searcher/api/sidebar.rs b/crates/core/src/searcher/api/sidebar.rsindex fbc9ad0f..1f009fd3 100644--- a/crates/core/src/searcher/api/sidebar.rs+++ b/crates/core/src/searcher/api/sidebar.rs@@ -70,7 +70,7 @@ wheretracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");if website.score > self.thresholds.stackoverflow {let scored_websites =- vec![(0, distributed::ScoredWebsitePointer { website, shard })];+ vec![(0, distributed::ScoredWebpagePointer { website, shard })];let mut retrieved = self.distributed_searcher.retrieve_webpages(&scored_websites, &query.query)diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex cd40317e..6888a51b 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -29,8 +29,8 @@ use crate::{search_server::{self, SearchService},},image_store::Image,- inverted_index::{RetrievedWebpage, WebsitePointer},- ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},+ inverted_index::{RetrievedWebpage, WebpagePointer},+ ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},Result,};@@ -59,8 +59,8 @@ pub enum Error {}#[derive(Clone, Debug)]-pub struct ScoredWebsitePointer {- pub website: RankingWebsite,+pub struct ScoredWebpagePointer {+ pub website: RecallRankingWebpage,pub shard: ShardId,}@@ -126,9 +126,9 @@ impl DistributedSearcher {shard: ShardId,client: &ShardedClient<SearchService, ShardId>,query: &str,- pointers: Vec<(usize, WebsitePointer)>,+ pointers: Vec<(usize, WebpagePointer)>,) -> Vec<(usize, RetrievedWebpage)> {- let (idxs, pointers): (Vec<usize>, Vec<WebsitePointer>) = pointers.into_iter().unzip();+ let (idxs, pointers): (Vec<usize>, Vec<WebpagePointer>) = pointers.into_iter().unzip();match client.send(@@ -184,9 +184,9 @@ impl SearchClient for DistributedSearcher {async fn retrieve_webpages(&self,- top_websites: &[(usize, ScoredWebsitePointer)],+ top_websites: &[(usize, ScoredWebpagePointer)],query: &str,- ) -> Vec<(usize, RetrievedWebpageRanking)> {+ ) -> Vec<(usize, PrecisionRankingWebpage)> {let mut rankings = FnvHashMap::default();let mut pointers: HashMap<_, Vec<_>> = HashMap::new();@@ -209,7 +209,7 @@ impl SearchClient for DistributedSearcher {for pages in join_all(futures).await {for (i, page) in pages {retrieved_webpages- .push((i, RetrievedWebpageRanking::new(page, rankings[&i].clone())));+ .push((i, PrecisionRankingWebpage::new(page, rankings[&i].clone())));}}@@ -308,9 +308,9 @@ pub trait SearchClient {fn retrieve_webpages(&self,- top_websites: &[(usize, ScoredWebsitePointer)],+ top_websites: &[(usize, ScoredWebpagePointer)],query: &str,- ) -> impl Future<Output = Vec<(usize, RetrievedWebpageRanking)>> + Send;+ ) -> impl Future<Output = Vec<(usize, PrecisionRankingWebpage)>> + Send;fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;diff --git a/crates/core/src/searcher/live.rs b/crates/core/src/searcher/live.rsindex b3b44eec..5d25be94 100644--- a/crates/core/src/searcher/live.rs+++ b/crates/core/src/searcher/live.rs@@ -25,8 +25,8 @@ use crate::{},entrypoint::search_server::{self, SearchService},feed::scheduler::SplitId,- inverted_index::{RetrievedWebpage, WebsitePointer},- ranking::pipeline::{RankingWebsite, RetrievedWebpageRanking},+ inverted_index::{RetrievedWebpage, WebpagePointer},+ ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},};use std::future::Future;@@ -39,8 +39,8 @@ use itertools::Itertools;use super::{InitialWebsiteResult, SearchQuery};#[derive(Clone, Debug)]-pub struct ScoredWebsitePointer {- pub website: RankingWebsite,+pub struct ScoredWebpagePointer {+ pub website: RecallRankingWebpage,pub split_id: SplitId,}@@ -86,9 +86,9 @@ impl LiveSearcher {split: SplitId,client: &ShardedClient<SearchService, SplitId>,query: &str,- pointers: Vec<(usize, WebsitePointer)>,+ pointers: Vec<(usize, WebpagePointer)>,) -> Vec<(usize, RetrievedWebpage)> {- let (idxs, pointers): (Vec<usize>, Vec<WebsitePointer>) = pointers.into_iter().unzip();+ let (idxs, pointers): (Vec<usize>, Vec<WebpagePointer>) = pointers.into_iter().unzip();match client.send(@@ -144,9 +144,9 @@ impl SearchClient for LiveSearcher {async fn retrieve_webpages(&self,- top_websites: &[(usize, ScoredWebsitePointer)],+ top_websites: &[(usize, ScoredWebpagePointer)],query: &str,- ) -> Vec<(usize, RetrievedWebpageRanking)> {+ ) -> Vec<(usize, PrecisionRankingWebpage)> {let mut rankings = FnvHashMap::default();let mut pointers: HashMap<_, Vec<_>> = HashMap::new();@@ -169,7 +169,7 @@ impl SearchClient for LiveSearcher {for pages in join_all(futures).await {for (i, page) in pages {retrieved_webpages- .push((i, RetrievedWebpageRanking::new(page, rankings[&i].clone())));+ .push((i, PrecisionRankingWebpage::new(page, rankings[&i].clone())));}}@@ -188,7 +188,7 @@ pub trait SearchClient {) -> impl Future<Output = Vec<InitialSearchResultSplit>> + Send;fn retrieve_webpages(&self,- top_websites: &[(usize, ScoredWebsitePointer)],+ top_websites: &[(usize, ScoredWebpagePointer)],query: &str,- ) -> impl Future<Output = Vec<(usize, RetrievedWebpageRanking)>> + Send;+ ) -> impl Future<Output = Vec<(usize, PrecisionRankingWebpage)>> + Send;}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex 90f98e3a..42dde1b4 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -17,16 +17,18 @@use std::collections::HashMap;use std::sync::{Arc, RwLockReadGuard};+use itertools::Itertools;use url::Url;use crate::config::{CollectorConfig, SnippetConfig};use crate::index::Index;use crate::inverted_index::{InvertedIndex, RetrievedWebpage};+use crate::models::dual_encoder::DualEncoder;use crate::query::Query;use crate::ranking::inbound_similarity::InboundSimilarity;use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;-use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};+use crate::ranking::pipeline::{PrecisionRankingWebpage, RankingPipeline, RecallRankingWebpage};use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS};use crate::search_ctx::Ctx;use crate::search_prettifier::DisplayedWebpage;@@ -103,6 +105,7 @@ pub struct LocalSearcher<I: SearchableIndex> {inbound_similarity: Option<InboundSimilarity>,linear_regression: Option<Arc<LinearRegression>>,lambda_model: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,collector_config: CollectorConfig,}@@ -116,7 +119,7 @@ where}struct InvertedIndexResult {- webpages: Vec<RankingWebsite>,+ webpages: Vec<RecallRankingWebpage>,num_hits: Option<usize>,has_more: bool,}@@ -131,6 +134,7 @@ whereinbound_similarity: None,linear_regression: None,lambda_model: None,+ dual_encoder: None,collector_config: CollectorConfig::default(),}}@@ -147,6 +151,10 @@ whereself.lambda_model = Some(Arc::new(model));}+ pub fn set_dual_encoder(&mut self, dual_encoder: DualEncoder) {+ self.dual_encoder = Some(Arc::new(dual_encoder));+ }+pub fn set_collector_config(&mut self, config: CollectorConfig) {self.collector_config = config;}@@ -226,12 +234,14 @@ wherede_rank_similar: bool,) -> Result<InvertedIndexResult> {let mut query = query.clone();- let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::recall_stage(- &mut query,- self.lambda_model.clone(),- self.collector_config.clone(),- 100,- );+ let pipeline: RankingPipeline<RecallRankingWebpage> =+ RankingPipeline::<RecallRankingWebpage>::recall_stage(+ &mut query,+ self.lambda_model.clone(),+ self.dual_encoder.clone(),+ self.collector_config.clone(),+ 100,+ );let parsed_query = self.parse_query(ctx, guard, &query)?;let mut aggregator = SignalAggregator::new(Some(&parsed_query));@@ -318,7 +328,7 @@ wherepub fn retrieve_websites(&self,- websites: &[inverted_index::WebsitePointer],+ websites: &[inverted_index::WebpagePointer],query: &str,) -> Result<Vec<inverted_index::RetrievedWebpage>> {let guard = self.index.guard();@@ -365,15 +375,28 @@ wherelet search_result = self.search_initial(&search_query, true)?;- let search_len = search_result.websites.len();+ let pointers: Vec<_> = search_result+ .websites+ .iter()+ .map(|website| website.pointer.clone())+ .collect();++ let websites: Vec<_> = self+ .retrieve_websites(&pointers, &query.query)?+ .into_iter()+ .zip_eq(search_result.websites)+ .map(|(webpage, ranking)| PrecisionRankingWebpage::new(webpage, ranking))+ .collect();++ let search_len = websites.len();- let top_websites = pipeline.apply(search_result.websites);+ let top_websites = pipeline.apply(websites);let has_more_results = search_len != top_websites.len();let pointers: Vec<_> = top_websites.iter()- .map(|website| website.pointer.clone())+ .map(|website| website.ranking.pointer.clone()).collect();let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;@@ -387,7 +410,7 @@ wherelet mut ranking_signals = HashMap::new();for signal in ALL_SIGNALS {- if let Some(score) = ranking.signals.get(signal) {+ if let Some(score) = ranking.ranking.signals.get(signal) {ranking_signals.insert(signal, *score);}}@@ -430,7 +453,7 @@ mod tests {for i in 0..NUM_WEBSITES {index- .insert(Webpage {+ .insert(&Webpage {html: Html::parse(r#"<html>diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex 0a50dd58..f5a49fef 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -26,7 +26,7 @@ use serde::{Deserialize, Serialize};use utoipa::ToSchema;use crate::{- bangs::BangHit, config::defaults, ranking::pipeline::RankingWebsite,+ bangs::BangHit, config::defaults, ranking::pipeline::RecallRankingWebpage,search_prettifier::DisplayedWebpage, webpage::region::Region,};@@ -63,7 +63,7 @@ pub struct SearchQuery {#[derive(Debug, Clone, Serialize, Deserialize)]pub struct InitialWebsiteResult {pub num_websites: Option<usize>,- pub websites: Vec<RankingWebsite>,+ pub websites: Vec<RecallRankingWebpage>,pub has_more: bool,}diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex 1da0ef53..1875d8f7 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -388,7 +388,7 @@ Survey in 2016, 2017, and 2018."#;index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -427,7 +427,7 @@ Survey in 2016, 2017, and 2018."#;index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>@@ -466,7 +466,7 @@ Survey in 2016, 2017, and 2018."#;index.insert(- Webpage::new(+ &Webpage::test_parse(&format!(r#"<html>diff --git a/crates/core/src/summarizer.rs b/crates/core/src/summarizer.rsindex cb4b2591..8c489dc2 100644--- a/crates/core/src/summarizer.rs+++ b/crates/core/src/summarizer.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,9 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use anyhow::anyhow;-use candle_core::{Device, Tensor};-use candle_nn::VarBuilder;+use candle_core::Tensor;use futures::stream::Stream;use std::{cmp::Reverse,@@ -26,13 +24,8 @@ use std::{};use tokio_stream::StreamExt;-use crate::{- llm_utils::OpenAiApi,- models::bert::{self, BertModel},- Result,-};+use crate::{llm_utils::OpenAiApi, models::dual_encoder::DualEncoder, Result};use itertools::{intersperse, Itertools};-use tokenizers::{PaddingParams, TruncationParams};use crate::ceil_char_boundary;@@ -289,103 +282,17 @@ impl Summarizer {}}-pub struct DualEncoder {- model: BertModel,- tokenizer: tokenizers::Tokenizer,- device: Device,- dtype: candle_core::DType,-}--impl DualEncoder {- pub fn open<P: AsRef<Path>>(folder: P) -> Result<Self> {- let device = Device::Cpu;- let dtype = candle_core::DType::F16;-- let truncation = TruncationParams {- max_length: 256,- ..Default::default()- };-- let padding = PaddingParams {- ..Default::default()- };-- let mut tokenizer =- tokenizers::Tokenizer::from_file(folder.as_ref().join("tokenizer.json")).unwrap();-- tokenizer.with_truncation(Some(truncation)).unwrap();- tokenizer.with_padding(Some(padding));-- let vb = unsafe {- VarBuilder::from_mmaped_safetensors(- &[folder.as_ref().join("model.safetensors")],- dtype,- &device,- )?- };- let config = std::fs::read_to_string(folder.as_ref().join("config.json"))?;- let mut config: bert::Config = serde_json::from_str(&config)?;- config.hidden_act = bert::HiddenAct::GeluApproximate;-- // all tensors can be loaded with (useful for debugging):- // candle_core::safetensors::load(folder.as_ref().join("model.safetensors"), &device)-- let mut model = BertModel::load(vb, &config)?;- model.set_pooler(None); // model should use mean pooling-- Ok(Self {- model,- tokenizer,- device,- dtype,- })- }-- pub fn embed(&self, texts: &[&str]) -> Result<Tensor> {- let enc = self- .tokenizer- .encode_batch(texts.to_vec(), true)- .map_err(|e| anyhow!(e))?;-- let ids = enc- .iter()- .map(|enc| Tensor::new(enc.get_ids(), &self.device).map_err(|e| anyhow!(e)))- .collect::<Result<Vec<_>>>()?;-- let input_ids = Tensor::stack(&ids, 0)?;-- let token_type_ids = input_ids.zeros_like()?;-- let attention_mask = enc- .iter()- .map(|enc| Tensor::new(enc.get_attention_mask(), &self.device).map_err(|e| anyhow!(e)))- .collect::<Result<Vec<_>>>()?;- let attention_mask = Tensor::stack(&attention_mask, 0)?.to_dtype(self.dtype)?;-- let emb = self- .model- .forward(&input_ids, &token_type_ids, &attention_mask)?;-- let (_n_sentence, n_tokens, _hidden_size) = emb.dims3()?;-- let emb = (emb.sum(1)? / (n_tokens as f64))?; // mean pooling- let emb = emb.broadcast_div(&emb.sqr()?.sum_keepdim(1)?.sqrt()?)?; // l2 normalization-- Ok(emb)- }-}-impl PassageScorer for DualEncoder {type QueryEmbedding = Tensor;type PassageEmbedding = Tensor;fn embed_query(&self, query: &str) -> Option<Self::QueryEmbedding> {- self.embed(&[query]).ok()+ self.embed(&[query.to_string()]).ok()}fn embed_passage(&self, passage: &str) -> Option<Self::PassageEmbedding> {- self.embed(&[passage]).ok()+ self.embed(&[passage.to_string()]).ok()}fn score(&self, query: &Self::QueryEmbedding, passage: &Self::PassageEmbedding) -> f32 {diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rsindex cffeff1c..7035ce0f 100644--- a/crates/core/src/warc.rs+++ b/crates/core/src/warc.rs@@ -47,12 +47,11 @@ fn decode_string(raw: &[u8]) -> String {res} else {let mut detector = chardetng::EncodingDetector::new();- detector.feed(raw, true);+ let end = std::cmp::min(64, raw.len());+ detector.feed(&raw[..end], false);let (enc, conf) = detector.guess_assess(None, true);if conf {- dbg!(enc.name());-let (cow, _, had_errors) = enc.decode(raw);if !had_errors {return cow.to_string();diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rsindex 9523a3f8..cc03160f 100644--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs+++ b/crates/core/src/webgraph/centrality/derived_harmonic.rs@@ -98,7 +98,7 @@ impl DerivedCentrality {page_graph.node_ids().for_each(|(node, id)| {pb.inc(1);- if has_outgoing.contains(&id.as_u64()) {+ if has_outgoing.contains(id.as_u64()) {let host_node = node.clone().into_host().id();if let Some(harmonic) = host_harmonic.get(&host_node) {diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex 8218b21b..f7d7fec2 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -106,7 +106,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {} else {exact_changed_nodes = BTreeSet::default();graph.edges().for_each(|edge| {- if changed_nodes.contains(&edge.from.as_u64()) {+ if changed_nodes.contains(edge.from.as_u64()) {if let (Some(counter_to), Some(counter_from)) =(new_counters.get_mut(&edge.to), counters.get(&edge.from)){diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 6fa51926..7cc19220 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -485,22 +485,24 @@ impl Id2NodeDb {opts.set_target_file_size_base(512 * 1024 * 1024); // 512 MBopts.set_target_file_size_multiplier(10);- opts.set_compression_type(rocksdb::DBCompressionType::Lz4);+ opts.set_compression_type(rocksdb::DBCompressionType::None);let mut block_opts = rocksdb::BlockBasedOptions::default();- let cache = rocksdb::Cache::new_lru_cache(8 * 1024 * 1024 * 1024); // 8 gb- opts.set_block_based_table_factory(&block_opts);+ let cache = rocksdb::Cache::new_lru_cache(1024 * 1024 * 1024); // 1 gb+ block_opts.set_ribbon_filter(10.0);// some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)opts.set_level_compaction_dynamic_level_bytes(true);opts.set_bytes_per_sync(1048576);- block_opts.set_block_size(16 * 1024);+ block_opts.set_block_size(32 * 1024); // 32 kbblock_opts.set_format_version(5);block_opts.set_cache_index_and_filter_blocks(true);block_opts.set_pin_l0_filter_and_index_blocks_in_cache(true);block_opts.set_block_cache(&cache);+ opts.set_block_based_table_factory(&block_opts);+let db = rocksdb::DB::open(&opts, path).unwrap();Self { db, _cache: cache }diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 55333416..7db3ad5d 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -314,7 +314,7 @@ impl EdgeStore {options.set_level_compaction_dynamic_level_bytes(true);options.set_bytes_per_sync(1048576);let mut block_options = BlockBasedOptions::default();- block_options.set_block_size(16 * 1024);+ block_options.set_block_size(32 * 1024);block_options.set_format_version(5);block_options.set_cache_index_and_filter_blocks(true);block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);diff --git a/crates/core/src/webpage/html/into_tantivy.rs b/crates/core/src/webpage/html/into_tantivy.rsindex 443d5ff7..3263e75f 100644--- a/crates/core/src/webpage/html/into_tantivy.rs+++ b/crates/core/src/webpage/html/into_tantivy.rs@@ -129,11 +129,7 @@ impl Html {.unwrap_or_default()}- pub fn into_tantivy(- self,- schema: &tantivy::schema::Schema,- rake: &RakeModel,- ) -> Result<TantivyDocument> {+ pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {let mut doc = TantivyDocument::new();let title = self.pretokenize_title()?;@@ -415,10 +411,6 @@ impl Html {doc.add_text(tantivy_field, "");}}- Field::Text(TextField::Keywords) => {- let rake_keywords = self.keywords(rake);- doc.add_text(tantivy_field, rake_keywords.join("\n"));- }Field::Text(TextField::AllBody) => {doc.add_pre_tokenized_text(tantivy_field, all_text.clone())}@@ -562,6 +554,9 @@ impl Html {| Field::Fast(FastField::PreComputedScore)| Field::Fast(FastField::Region)| Field::Fast(FastField::HostNodeID)+ | Field::Fast(FastField::TitleEmbeddings)+ | Field::Fast(FastField::KeywordEmbeddings)+ | Field::Text(TextField::Keywords)| Field::Text(TextField::DmozDescription) => {}}}diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex 68f89979..59175ad4 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -493,7 +493,6 @@ mod tests {use std::collections::HashMap;use crate::{- rake::RakeModel,schema::create_schema,webpage::{url_ext::UrlExt, Link},};@@ -741,9 +740,7 @@ mod tests {assert!(!webpage.all_text().unwrap().is_empty());let schema = create_schema();- webpage- .into_tantivy(&schema, &RakeModel::default())- .unwrap();+ webpage.as_tantivy(&schema).unwrap();}#[test]diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex 4ebd2c46..03c88a8b 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -15,11 +15,11 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- rake::RakeModel,schema::{FastField, TextField},webgraph::NodeID,Result,};+use candle_core::Tensor;use chrono::{DateTime, Utc};use std::collections::HashMap;@@ -44,15 +44,18 @@ pub struct Webpage {pub html: Html,pub backlink_labels: Vec<String>,pub host_centrality: f64,- pub host_centrality_rank: f64,+ pub host_centrality_rank: u64,pub page_centrality: f64,- pub page_centrality_rank: f64,+ pub page_centrality_rank: u64,pub fetch_time_ms: u64,pub pre_computed_score: f64,pub node_id: Option<NodeID>,pub dmoz_description: Option<String>,pub safety_classification: Option<safety_classifier::Label>,pub inserted_at: DateTime<Utc>,+ pub keywords: Vec<String>,+ pub title_embedding: Option<Tensor>,+ pub keyword_embedding: Option<Tensor>,}#[cfg(test)]@@ -62,22 +65,47 @@ impl Default for Webpage {html: Html::parse_without_text("<html></html>", "https://example.com/").unwrap(),backlink_labels: Default::default(),host_centrality: Default::default(),- host_centrality_rank: u64::MAX as f64,+ host_centrality_rank: u64::MAX,page_centrality: Default::default(),- page_centrality_rank: u64::MAX as f64,+ page_centrality_rank: u64::MAX,fetch_time_ms: Default::default(),pre_computed_score: Default::default(),node_id: Default::default(),dmoz_description: Default::default(),safety_classification: Default::default(),inserted_at: Utc::now(),+ keywords: Default::default(),+ title_embedding: Default::default(),+ keyword_embedding: Default::default(),+ }+ }+}++impl From<Html> for Webpage {+ fn from(html: Html) -> Self {+ Self {+ html,+ backlink_labels: Default::default(),+ host_centrality: Default::default(),+ host_centrality_rank: u64::MAX,+ page_centrality: Default::default(),+ page_centrality_rank: u64::MAX,+ fetch_time_ms: Default::default(),+ pre_computed_score: Default::default(),+ node_id: Default::default(),+ dmoz_description: Default::default(),+ safety_classification: Default::default(),+ inserted_at: Utc::now(),+ keywords: Default::default(),+ title_embedding: Default::default(),+ keyword_embedding: Default::default(),}}}impl Webpage {#[cfg(test)]- pub fn new(html: &str, url: &str) -> Result<Self> {+ pub fn test_parse(html: &str, url: &str) -> Result<Self> {let html = Html::parse(html, url)?;Ok(Self {@@ -102,16 +130,12 @@ impl Webpage {})}- pub fn into_tantivy(- self,- schema: &tantivy::schema::Schema,- rake: &RakeModel,- ) -> Result<TantivyDocument> {- let region = Region::guess_from(&self);+ pub fn as_tantivy(&self, schema: &tantivy::schema::Schema) -> Result<TantivyDocument> {+ let region = Region::guess_from(self);let dmoz_description = self.dmoz_description();- let mut doc = self.html.into_tantivy(schema, rake)?;+ let mut doc = self.html.as_tantivy(schema)?;if let Ok(region) = region {doc.add_u64(@@ -130,7 +154,7 @@ impl Webpage {}let backlink_text: String =- itertools::intersperse(self.backlink_labels, "\n".to_string()).collect();+ itertools::intersperse(self.backlink_labels.clone(), "\n".to_string()).collect();doc.add_text(schema@@ -139,6 +163,13 @@ impl Webpage {backlink_text,);+ doc.add_text(+ schema+ .get_field(Field::Text(TextField::Keywords).name())+ .expect("Failed to get keywords field"),+ self.keywords.join("\n"),+ );+doc.add_date(schema.get_field(Field::Text(TextField::InsertionTimestamp).name())@@ -171,7 +202,7 @@ impl Webpage {schema.get_field(Field::Fast(FastField::HostCentralityRank).name()).expect("Failed to get host_centrality_rank field"),- self.host_centrality_rank as u64,+ self.host_centrality_rank,);doc.add_u64(@@ -185,7 +216,7 @@ impl Webpage {schema.get_field(Field::Fast(FastField::PageCentralityRank).name()).expect("Failed to get page_centrality_rank field"),- self.page_centrality_rank as u64,+ self.page_centrality_rank,);doc.add_u64(@@ -202,6 +233,44 @@ impl Webpage {(self.pre_computed_score * FLOAT_SCALING as f64) as u64,);+ if let Some(emb) = &self.title_embedding {+ let mut serialized = Vec::new();+ emb.write_bytes(&mut serialized)?;++ doc.add_bytes(+ schema+ .get_field(Field::Fast(FastField::TitleEmbeddings).name())+ .expect("Failed to get title_embeddings field"),+ serialized,+ );+ } else {+ doc.add_bytes(+ schema+ .get_field(Field::Fast(FastField::TitleEmbeddings).name())+ .expect("Failed to get title_embeddings field"),+ Vec::new(),+ );+ }++ if let Some(emb) = &self.keyword_embedding {+ let mut serialized = Vec::new();+ emb.write_bytes(&mut serialized)?;++ doc.add_bytes(+ schema+ .get_field(Field::Fast(FastField::KeywordEmbeddings).name())+ .expect("Failed to get keyword_embeddings field"),+ serialized,+ );+ } else {+ doc.add_bytes(+ schema+ .get_field(Field::Fast(FastField::KeywordEmbeddings).name())+ .expect("Failed to get keyword_embeddings field"),+ Vec::new(),+ );+ }+match &self.node_id {Some(node_id) => {doc.add_u64(diff --git a/crates/kuchiki/examples/find_matches.rs b/crates/kuchiki/examples/find_matches.rsindex b702eb7a..f7be3f9b 100644--- a/crates/kuchiki/examples/find_matches.rs+++ b/crates/kuchiki/examples/find_matches.rs@@ -1,6 +1,6 @@extern crate kuchiki;-use kuchiki::traits::*;+use kuchiki::traits::TendrilSink;fn main() {let html = r"@@ -43,6 +43,6 @@ fn main() {//// "Hello, world!"// "I love HTML"- println!("{:?}", text);+ println!("{text:?}");}}diff --git a/crates/kuchiki/examples/stack-overflow.rs b/crates/kuchiki/examples/stack-overflow.rsindex 535b702a..e874868f 100644--- a/crates/kuchiki/examples/stack-overflow.rs+++ b/crates/kuchiki/examples/stack-overflow.rs@@ -11,7 +11,7 @@ fn main() {node = parent;}- println!("Trying to drop {} nodes...", depth);+ println!("Trying to drop {depth} nodes...");// Without an explicit `impl Drop for Node`,// depth = 20_000 causes "thread '<main>' has overflowed its stack"// on my machine (Linux x86_64).diff --git a/crates/kuchiki/src/attributes.rs b/crates/kuchiki/src/attributes.rsindex 655585e4..c1a68ab3 100644--- a/crates/kuchiki/src/attributes.rs+++ b/crates/kuchiki/src/attributes.rs@@ -37,31 +37,31 @@ pub struct Attribute {}impl Attributes {- /// Like IndexMap::contains+ /// Like `IndexMap::contains`pub fn contains<A: Into<LocalName>>(&self, local_name: A) -> bool {self.map.contains_key(&ExpandedName::new(ns!(), local_name))}- /// Like IndexMap::get+ /// Like `IndexMap::get`pub fn get<A: Into<LocalName>>(&self, local_name: A) -> Option<&str> {self.map.get(&ExpandedName::new(ns!(), local_name)).map(|attr| &*attr.value)}- /// Like IndexMap::get_mut+ /// Like `IndexMap::get_mut`pub fn get_mut<A: Into<LocalName>>(&mut self, local_name: A) -> Option<&mut String> {self.map.get_mut(&ExpandedName::new(ns!(), local_name)).map(|attr| &mut attr.value)}- /// Like IndexMap::entry+ /// Like `IndexMap::entry`pub fn entry<A: Into<LocalName>>(&mut self, local_name: A) -> Entry<ExpandedName, Attribute> {self.map.entry(ExpandedName::new(ns!(), local_name))}- /// Like IndexMap::insert+ /// Like `IndexMap::insert`pub fn insert<A: Into<LocalName>>(&mut self,local_name: A,@@ -76,7 +76,7 @@ impl Attributes {)}- /// Like IndexMap::remove+ /// Like `IndexMap::remove`pub fn remove<A: Into<LocalName>>(&mut self, local_name: A) -> Option<Attribute> {self.map.remove(&ExpandedName::new(ns!(), local_name))}diff --git a/crates/kuchiki/src/iter.rs b/crates/kuchiki/src/iter.rsindex 5ae9713c..9af15bb1 100644--- a/crates/kuchiki/src/iter.rs+++ b/crates/kuchiki/src/iter.rs@@ -12,13 +12,13 @@ use crate::Result;impl NodeRef {/// Return an iterator of references to this node and its ancestors.#[inline]- pub fn inclusive_ancestors(&self) -> Ancestors {+ #[must_use] pub fn inclusive_ancestors(&self) -> Ancestors {Ancestors(Some(self.clone()))}/// Return an iterator of references to this node’s ancestors.#[inline]- pub fn ancestors(&self) -> Ancestors {+ #[must_use] pub fn ancestors(&self) -> Ancestors {Ancestors(self.parent())}@@ -63,7 +63,7 @@ impl NodeRef {/// Return an iterator of references to this node and the siblings after it.#[inline]- pub fn inclusive_following_siblings(&self) -> Siblings {+ #[must_use] pub fn inclusive_following_siblings(&self) -> Siblings {match self.parent() {Some(parent) => {let last_sibling = parent.last_child().unwrap();@@ -85,7 +85,7 @@ impl NodeRef {/// Return an iterator of references to this node’s siblings after it.#[inline]- pub fn following_siblings(&self) -> Siblings {+ #[must_use] pub fn following_siblings(&self) -> Siblings {match (self.parent(), self.next_sibling()) {(Some(parent), Some(next_sibling)) => {let last_sibling = parent.last_child().unwrap();@@ -100,7 +100,7 @@ impl NodeRef {/// Return an iterator of references to this node’s children.#[inline]- pub fn children(&self) -> Siblings {+ #[must_use] pub fn children(&self) -> Siblings {match (self.first_child(), self.last_child()) {(Some(first_child), Some(last_child)) => Siblings(Some(State {next: first_child,@@ -117,7 +117,7 @@ impl NodeRef {////// Note: this is the `NodeEdge::Start` items from `traverse()`.#[inline]- pub fn inclusive_descendants(&self) -> Descendants {+ #[must_use] pub fn inclusive_descendants(&self) -> Descendants {Descendants(self.traverse_inclusive())}@@ -127,14 +127,14 @@ impl NodeRef {////// Note: this is the `NodeEdge::Start` items from `traverse()`.#[inline]- pub fn descendants(&self) -> Descendants {+ #[must_use] pub fn descendants(&self) -> Descendants {Descendants(self.traverse())}/// Return an iterator of the start and end edges of this node and its descendants,/// in tree order.#[inline]- pub fn traverse_inclusive(&self) -> Traverse {+ #[must_use] pub fn traverse_inclusive(&self) -> Traverse {Traverse(Some(State {next: NodeEdge::Start(self.clone()),next_back: NodeEdge::End(self.clone()),@@ -144,7 +144,7 @@ impl NodeRef {/// Return an iterator of the start and end edges of this node’s descendants,/// in tree order.#[inline]- pub fn traverse(&self) -> Traverse {+ #[must_use] pub fn traverse(&self) -> Traverse {match (self.first_child(), self.last_child()) {(Some(first_child), Some(last_child)) => Traverse(Some(State {next: NodeEdge::Start(first_child),@@ -163,7 +163,7 @@ impl NodeRef {/// Return the first inclusive descendants element that match the given selector list.#[inline]- pub fn select_first(&self, selectors: &str) -> Option<NodeDataRef<ElementData>> {+ #[must_use] pub fn select_first(&self, selectors: &str) -> Option<NodeDataRef<ElementData>> {let mut elements = self.select(selectors).ok()?;elements.next()}diff --git a/crates/kuchiki/src/node_data_ref.rs b/crates/kuchiki/src/node_data_ref.rsindex 41be520e..a5f5cbaa 100644--- a/crates/kuchiki/src/node_data_ref.rs+++ b/crates/kuchiki/src/node_data_ref.rs@@ -61,7 +61,7 @@ impl<T> NodeDataRef<T> {whereF: FnOnce(&Node) -> Option<&T>,{- f(&rc).map(|r| r as *const T).map(move |r| NodeDataRef {+ f(&rc).map(|r| std::ptr::from_ref::<T>(r)).map(move |r| NodeDataRef {_reference: r,_keep_alive: rc,})@@ -69,7 +69,7 @@ impl<T> NodeDataRef<T> {/// Access the corresponding node.#[inline]- pub fn as_node(&self) -> &NodeRef {+ #[must_use] pub fn as_node(&self) -> &NodeRef {&self._keep_alive}}@@ -110,7 +110,7 @@ impl<T: fmt::Debug> fmt::Debug for NodeDataRef<T> {impl NodeDataRef<ElementData> {/// Return the concatenation of all text nodes in this subtree.- pub fn text_contents(&self) -> String {+ #[must_use] pub fn text_contents(&self) -> String {self.as_node().text_contents()}}diff --git a/crates/kuchiki/src/parser.rs b/crates/kuchiki/src/parser.rsindex f160a6a2..1397d006 100644--- a/crates/kuchiki/src/parser.rs+++ b/crates/kuchiki/src/parser.rs@@ -20,12 +20,12 @@ pub struct ParseOpts {}/// Parse an HTML document with html5ever and the default configuration.-pub fn parse_html() -> html5ever::Parser<Sink> {+#[must_use] pub fn parse_html() -> html5ever::Parser<Sink> {parse_html_with_options(ParseOpts::default())}/// Parse an HTML document with html5ever with custom configuration.-pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {+#[must_use] pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {let sink = Sink {document_node: NodeRef::new_document(),on_parse_error: opts.on_parse_error,@@ -38,7 +38,7 @@ pub fn parse_html_with_options(opts: ParseOpts) -> html5ever::Parser<Sink> {}/// Parse an HTML fragment with html5ever and the default configuration.-pub fn parse_fragment(ctx_name: QualName, ctx_attr: Vec<Attribute>) -> html5ever::Parser<Sink> {+#[must_use] pub fn parse_fragment(ctx_name: QualName, ctx_attr: Vec<Attribute>) -> html5ever::Parser<Sink> {parse_fragment_with_options(ParseOpts::default(), ctx_name, ctx_attr)}@@ -77,7 +77,7 @@ impl TreeSink for Sink {#[inline]fn parse_error(&mut self, message: Cow<'static, str>) {if let Some(ref mut handler) = self.on_parse_error {- handler(message)+ handler(message);}}@@ -92,7 +92,7 @@ impl TreeSink for Sink {.as_document().unwrap()._quirks_mode- .set(mode)+ .set(mode);}#[inline]@@ -149,7 +149,7 @@ impl TreeSink for Sink {return;}}- parent.append(NodeRef::new_text(text))+ parent.append(NodeRef::new_text(text));}}}@@ -165,7 +165,7 @@ impl TreeSink for Sink {return;}}- sibling.insert_before(NodeRef::new_text(text))+ sibling.insert_before(NodeRef::new_text(text));}}}@@ -178,7 +178,7 @@ impl TreeSink for Sink {system_id: StrTendril,) {self.document_node- .append(NodeRef::new_doctype(name, public_id, system_id))+ .append(NodeRef::new_doctype(name, public_id, system_id));}#[inline]@@ -203,7 +203,7 @@ impl TreeSink for Sink {#[inline]fn remove_from_parent(&mut self, target: &NodeRef) {- target.detach()+ target.detach();}#[inline]@@ -211,7 +211,7 @@ impl TreeSink for Sink {// FIXME: Can this be done more effciently in rctree,// by moving the whole linked list of children at once?for child in node.children() {- new_parent.append(child)+ new_parent.append(child);}}@@ -237,9 +237,9 @@ impl TreeSink for Sink {child: NodeOrText<NodeRef>,) {if element.parent().is_some() {- self.append_before_sibling(element, child)+ self.append_before_sibling(element, child);} else {- self.append(prev_element, child)+ self.append(prev_element, child);}}}diff --git a/crates/kuchiki/src/select.rs b/crates/kuchiki/src/select.rsindex 7c0b8c92..9c1a7c73 100644--- a/crates/kuchiki/src/select.rs+++ b/crates/kuchiki/src/select.rs@@ -50,7 +50,7 @@ impl<'i> Parser<'i> for KuchikiParser {location: SourceLocation,name: CowRcStr<'i>,) -> std::result::Result<PseudoClass, ParseError<'i, SelectorParseErrorKind<'i>>> {- use self::PseudoClass::*;+ use self::PseudoClass::{Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited};if name.eq_ignore_ascii_case("any-link") {Ok(AnyLink)} else if name.eq_ignore_ascii_case("link") {@@ -314,7 +314,7 @@ impl selectors::Element for NodeDataRef<ElementData> {whereF: FnMut(&Self, matching::ElementSelectorFlags),{- use self::PseudoClass::*;+ use self::PseudoClass::{Active, AnyLink, Checked, Disabled, Enabled, Focus, Hover, Indeterminate, Link, Visited};match *pseudo {Active | Focus | Hover | Enabled | Disabled | Checked | Indeterminate | Visited => {false@@ -359,7 +359,7 @@ impl Selectors {/// Returns whether the given element matches this list of selectors.#[inline]- pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {+ #[must_use] pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {self.0.iter().any(|s| s.matches(element))}@@ -379,7 +379,7 @@ impl Selectors {impl Selector {/// Returns whether the given element matches this selector.#[inline]- pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {+ #[must_use] pub fn matches(&self, element: &NodeDataRef<ElementData>) -> bool {let mut context = matching::MatchingContext::new(matching::MatchingMode::Normal,None,@@ -390,7 +390,7 @@ impl Selector {}/// Return the specificity of this selector.- pub fn specificity(&self) -> Specificity {+ #[must_use] pub fn specificity(&self) -> Specificity {Specificity(self.0.specificity())}}diff --git a/crates/kuchiki/src/serializer.rs b/crates/kuchiki/src/serializer.rsindex 1edfac92..615ef59d 100644--- a/crates/kuchiki/src/serializer.rs+++ b/crates/kuchiki/src/serializer.rs@@ -1,4 +1,4 @@-use html5ever::serialize::TraversalScope::*;+use html5ever::serialize::TraversalScope::{ChildrenOnly, IncludeNode};use html5ever::serialize::{serialize, Serialize, SerializeOpts, Serializer, TraversalScope};use html5ever::QualName;use std::fmt::Display;@@ -38,22 +38,22 @@ impl Serialize for NodeRef {serializer.start_elem(element.name.clone(),attrs.iter().map(|&(ref name, value)| (name, &**value)),- )?+ )?;}for child in self.children() {- Serialize::serialize(&child, serializer, IncludeNode)?+ Serialize::serialize(&child, serializer, IncludeNode)?;}if *scope == IncludeNode {- serializer.end_elem(element.name.clone())?+ serializer.end_elem(element.name.clone())?;}Ok(())}- (_, &NodeData::DocumentFragment) | (_, &NodeData::Document(_)) => {+ (_, &NodeData::DocumentFragment | &NodeData::Document(_)) => {for child in self.children() {- Serialize::serialize(&child, serializer, IncludeNode)?+ Serialize::serialize(&child, serializer, IncludeNode)?;}Ok(())}diff --git a/crates/kuchiki/src/tests.rs b/crates/kuchiki/src/tests.rsindex 3180d37a..83edbb6c 100644--- a/crates/kuchiki/src/tests.rs+++ b/crates/kuchiki/src/tests.rs@@ -182,7 +182,7 @@ fn specificity() {let specificities = selectors.0.iter()- .map(|s| s.specificity())+ .map(super::select::Selector::specificity).collect::<Vec<_>>();assert_eq!(specificities.len(), 3);assert!(specificities[0] == specificities[1]);diff --git a/crates/kuchiki/src/tree.rs b/crates/kuchiki/src/tree.rsindex e3bcc3ea..555e8861 100644--- a/crates/kuchiki/src/tree.rs+++ b/crates/kuchiki/src/tree.rs@@ -6,7 +6,7 @@ use std::ops::Deref;use std::rc::{Rc, Weak};use crate::attributes::{Attribute, Attributes, ExpandedName};-use crate::cell_extras::*;+use crate::cell_extras::{CellOption, CellOptionRc, CellOptionWeak};use crate::iter::NodeIterator;/// Node data specific to the node type.@@ -126,7 +126,7 @@ pub struct Node {impl fmt::Debug for Node {#[inline]fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {- write!(f, "{:?} @ {:?}", self.data, self as *const Node)+ write!(f, "{:?} @ {:?}", self.data, std::ptr::from_ref::<Node>(self))}}@@ -267,14 +267,14 @@ impl NodeRef {/// Create a new document node.#[inline]- pub fn new_document() -> NodeRef {+ #[must_use] pub fn new_document() -> NodeRef {NodeRef::new(NodeData::Document(DocumentData {_quirks_mode: Cell::new(QuirksMode::NoQuirks),}))}/// Return the concatenation of all text nodes in this subtree.- pub fn text_contents(&self) -> String {+ #[must_use] pub fn text_contents(&self) -> String {let mut s = String::new();for text_node in self.inclusive_descendants().text_nodes() {s.push_str(&text_node.borrow());@@ -375,7 +375,7 @@ impl Node {let previous_sibling_opt = previous_sibling_weak.as_ref()- .and_then(|weak| weak.upgrade());+ .and_then(std::rc::Weak::upgrade);if let Some(next_sibling_ref) = next_sibling_strong.as_ref() {next_sibling_refdiff --git a/crates/optics/src/ast.rs b/crates/optics/src/ast.rsindex 0ee972c2..c70b2021 100644--- a/crates/optics/src/ast.rs+++ b/crates/optics/src/ast.rs@@ -45,7 +45,7 @@ impl Display for RankingCoeff {match &self.target {RankingTarget::Signal(signal) => {- write!(f, "Signal(\"{}\")", signal)?;+ write!(f, "Signal(\"{signal}\")")?;}}@@ -77,12 +77,7 @@ impl From<Vec<RawOpticBlock>> for RawOptic {}}- RawOptic {- rankings,- rules,- host_preferences,- discard_non_matching,- }+ RawOptic { rules, rankings, host_preferences, discard_non_matching }}}diff --git a/crates/optics/src/lexer.rs b/crates/optics/src/lexer.rsindex dbcdab25..c65142bc 100644--- a/crates/optics/src/lexer.rs+++ b/crates/optics/src/lexer.rs@@ -236,7 +236,7 @@ impl<'source> Iterator for LexerBridge<'source> {}self.lexer = inner.morph();- tok = self.lex_next()+ tok = self.lex_next();}Outer::StartLineComment => {let mut inner: Lexer<LineComment> = self.lexer.clone().morph();@@ -247,7 +247,7 @@ impl<'source> Iterator for LexerBridge<'source> {}self.lexer = inner.morph();- tok = self.lex_next()+ tok = self.lex_next();}_ => break,}@@ -359,7 +359,7 @@ mod tests {let lexer = LexerBridge::new(s);- let result: Vec<Token> = lexer.filter_map(|r| r.ok()).map(|(_, t, _)| t).collect();+ let result: Vec<Token> = lexer.filter_map(std::result::Result::ok).map(|(_, t, _)| t).collect();let expected = vec![Token::Ranking,@@ -394,7 +394,7 @@ mod tests {Token::CloseBracket,];- assert_eq!(result, expected)+ assert_eq!(result, expected);}#[test]@@ -405,7 +405,7 @@ mod tests {let lexer = LexerBridge::new(s);- let result: Vec<Token> = lexer.filter_map(|r| r.ok()).map(|(_, t, _)| t).collect();+ let result: Vec<Token> = lexer.filter_map(std::result::Result::ok).map(|(_, t, _)| t).collect();let expected = vec![Token::Ranking,@@ -419,15 +419,15 @@ mod tests {Token::CloseParenthesis,];- assert_eq!(result, expected)+ assert_eq!(result, expected);}#[test]fn empty_program() {- let s = r#""#;+ let s = r"";let lexer = LexerBridge::new(s);- assert_eq!(lexer.filter_map(|r| r.ok()).count(), 0);+ assert_eq!(lexer.filter_map(std::result::Result::ok).count(), 0);}}diff --git a/crates/optics/src/lib.rs b/crates/optics/src/lib.rsindex 06be67d7..783e8892 100644--- a/crates/optics/src/lib.rs+++ b/crates/optics/src/lib.rs@@ -131,7 +131,7 @@ impl TryFrom<RawRule> for Rule {Ok(Rule {matches,- action: action.map(Action::from).unwrap_or(Action::Boost(0)),+ action: action.map_or(Action::Boost(0), Action::from),})}}@@ -164,10 +164,10 @@ impl Display for Matching {MatchLocation::MicroformatTag => "MicroformatTag",MatchLocation::Schema => "Schema",};- write!(f, "{}(\"", s)?;+ write!(f, "{s}(\"")?;for part in &self.pattern {- write!(f, "{}", part)?;+ write!(f, "{part}")?;}write!(f, "\")")@@ -279,7 +279,7 @@ pub enum PatternPart {impl Display for PatternPart {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {match self {- PatternPart::Raw(s) => write!(f, "{}", s),+ PatternPart::Raw(s) => write!(f, "{s}"),PatternPart::Wildcard => write!(f, "*"),PatternPart::Anchor => write!(f, "|"),}@@ -310,8 +310,8 @@ impl Display for Action {write!(f, "Action(")?;match self {- Action::Boost(b) => write!(f, "Boost({})", b)?,- Action::Downrank(d) => write!(f, "Downrank({})", d)?,+ Action::Boost(b) => write!(f, "Boost({b})")?,+ Action::Downrank(d) => write!(f, "Downrank({d})")?,Action::Discard => write!(f, "Discard")?,}@@ -340,11 +340,11 @@ impl Display for Optic {}for rule in &self.rules {- write!(f, "{}", rule)?;+ write!(f, "{rule}")?;}for ranking in &self.rankings {- writeln!(f, "{};", ranking)?;+ writeln!(f, "{ranking};")?;}write!(f, "{}", self.host_rankings)@@ -402,7 +402,7 @@ impl Display for Rule {for matcher in &self.matches {writeln!(f, "\tMatches {{")?;for m in matcher {- writeln!(f, "\t\t{},", m)?;+ writeln!(f, "\t\t{m},")?;}writeln!(f, "\t}},")?;}@@ -423,11 +423,11 @@ pub struct HostRankings {impl Display for HostRankings {fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {for liked in &self.liked {- writeln!(f, "Like(Site(\"{}\"));", liked)?;+ writeln!(f, "Like(Site(\"{liked}\"));")?;}for disliked in &self.disliked {- writeln!(f, "Dislike(Site(\"{}\"));", disliked)?;+ writeln!(f, "Dislike(Site(\"{disliked}\"));")?;}let matches = self@@ -435,7 +435,7 @@ impl Display for HostRankings {.iter().map(|host| {host.strip_prefix("www.")- .map(|host| host.to_string())+ .map(std::string::ToString::to_string).unwrap_or(host.clone())}).map(|host| {@@ -456,7 +456,7 @@ impl Display for HostRankings {action: Action::Discard,};- write!(f, "{}", rule)?;+ write!(f, "{rule}")?;}Ok(())@@ -464,13 +464,13 @@ impl Display for HostRankings {}impl HostRankings {- pub fn rules(&self) -> Rule {+ #[must_use] pub fn rules(&self) -> Rule {let matches: Vec<_> = self.blocked.iter().map(|host| {host.strip_prefix("www.")- .map(|host| host.to_string())+ .map(std::string::ToString::to_string).unwrap_or(host.clone())}).map(|host| {@@ -491,7 +491,7 @@ impl HostRankings {}}- pub fn into_optic(self) -> Optic {+ #[must_use] pub fn into_optic(self) -> Optic {Optic {host_rankings: self,..Default::default()@@ -562,7 +562,7 @@ mod tests {let exported = optic.to_string();- println!("{:}", exported);+ println!("{exported:}");let parsed = Optic::parse(&exported).unwrap();diff --git a/crates/zimba/src/lib.rs b/crates/zimba/src/lib.rsindex c718b83f..16b8ea74 100644--- a/crates/zimba/src/lib.rs+++ b/crates/zimba/src/lib.rs@@ -119,7 +119,7 @@ impl Header {]),};- if header.magic != 72173914 {+ if header.magic != 72_173_914 {return Err(Error::InvalidMagicNumber);}@@ -412,12 +412,12 @@ impl Cluster {match size {OffsetSize::U32 => {blob_offsets.push(ClusterOffset {- offset: u32::from_le_bytes([+ offset: u64::from(u32::from_le_bytes([reader.next().ok_or(Error::UnexpectedEndOfBytes)??,reader.next().ok_or(Error::UnexpectedEndOfBytes)??,reader.next().ok_or(Error::UnexpectedEndOfBytes)??,reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- ]) as u64,+ ])),});}OffsetSize::U64 => {@@ -445,12 +445,12 @@ impl Cluster {match size {OffsetSize::U32 => {blob_offsets.push(ClusterOffset {- offset: u32::from_le_bytes([+ offset: u64::from(u32::from_le_bytes([reader.next().ok_or(Error::UnexpectedEndOfBytes)??,reader.next().ok_or(Error::UnexpectedEndOfBytes)??,reader.next().ok_or(Error::UnexpectedEndOfBytes)??,reader.next().ok_or(Error::UnexpectedEndOfBytes)??,- ]) as u64,+ ])),});}OffsetSize::U64 => {@@ -490,7 +490,7 @@ impl Cluster {})}- pub fn get_blob(&self, blob_number: usize) -> Option<&[u8]> {+ #[must_use] pub fn get_blob(&self, blob_number: usize) -> Option<&[u8]> {if self.blob_offsets.is_empty() {return None;}@@ -524,7 +524,7 @@ impl ZimFile {let header = Header::from_bytes(&mmap)?;- if header.magic != 72173914 {+ if header.magic != 72_173_914 {return Err(Error::InvalidMagicNumber);}@@ -574,19 +574,19 @@ impl ZimFile {Ok(Some(Cluster::from_bytes(&self.mmap[pointer..])?))}- pub fn mime_types(&self) -> &MimeTypes {+ #[must_use] pub fn mime_types(&self) -> &MimeTypes {&self.mime_types}- pub fn url_pointers(&self) -> &UrlPointerList {+ #[must_use] pub fn url_pointers(&self) -> &UrlPointerList {&self.url_pointers}- pub fn title_pointers(&self) -> &TitlePointerList {+ #[must_use] pub fn title_pointers(&self) -> &TitlePointerList {&self.title_pointers}- pub fn dir_entries(&self) -> DirEntryIterator<'_> {+ #[must_use] pub fn dir_entries(&self) -> DirEntryIterator<'_> {DirEntryIterator::new(&self.mmap, &self.url_pointers)}@@ -644,7 +644,7 @@ mod tests {let zim = ZimFile::open(data_path).unwrap();- assert_eq!(zim.header.magic, 72173914);+ assert_eq!(zim.header.magic, 72_173_914);assert_eq!(zim.header.major_version, 5);assert_eq!(zim.header.minor_version, 0);diff --git a/crates/zimba/src/wiki.rs b/crates/zimba/src/wiki.rsindex f317e105..390d18ae 100644--- a/crates/zimba/src/wiki.rs+++ b/crates/zimba/src/wiki.rs@@ -64,7 +64,7 @@ impl<'a> ArticleIterator<'a> {title,};- let key = cluster_number as u64;+ let key = u64::from(cluster_number);articles.entry(key).or_insert(Vec::new()).push(article_ref);}@@ -180,7 +180,7 @@ impl<'a> ImageIterator<'a> {mime_type: zim.mime_types()[mime_type].clone(),};- let key = cluster_number as u64;+ let key = u64::from(cluster_number);images.entry(key).or_insert(Vec::new()).push(image_ref);}@@ -210,7 +210,7 @@ pub struct Image {pub content: Vec<u8>,}impl Image {- pub fn bytes(&self) -> &[u8] {+ #[must_use] pub fn bytes(&self) -> &[u8] {&self.content}}