web search done right
Use remote webgraph instead of local (#196)
Mikkel Denkerec8b9a0786eb
diff --git a/configs/api.toml b/configs/api.tomlindex 9c4322c1..5ebd9acd 100644--- a/configs/api.toml+++ b/configs/api.toml@@ -1,5 +1,5 @@cluster_id = "dev_api"-crossencoder_model_path = "data/cross_encoder"+# crossencoder_model_path = "data/cross_encoder"gossip_addr = "0.0.0.0:3005"gossip_seed_nodes = ["0.0.0.0:3006", "0.0.0.0:3007"]host = "0.0.0.0:3000"@@ -7,10 +7,17 @@ host = "0.0.0.0:3000"dual_encoder_model_path = "data/summarizer/dual_encoder"prometheus_host = "0.0.0.0:3001"queries_csv_path = "data/queries_us.csv"-spell_checker_path = "data/web_spell/checker"bangs_path = "data/bangs.json"summarizer_path = "data/summarizer"+[spell_check]+path = "data/web_spell/checker"++[spell_check.correction_config]+correction_threshold = 3.0+lm_prob_weight = 1.0++[thresholds]entity_sidebar = 0.0stackoverflow = 0.0@@ -18,10 +25,6 @@ stackoverflow = 0.0[widgets]thesaurus_paths = ["data/english-wordnet-2022-subset.ttl"]-[correction_config]-correction_threshold = 3.0-lm_prob_weight = 1.0-[llm]api_base = "http://localhost:4000/v1"model = "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf"diff --git a/configs/search_server.toml b/configs/search_server.tomlindex 323c047f..51cfa902 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -3,9 +3,8 @@ entity_index_path = "data/entity"gossip_addr = "0.0.0.0:3006"gossip_seed_nodes = ["0.0.0.0:3005"]host = "0.0.0.0:3002"-host_centrality_store_path = "data/centrality"index_path = "data/index"-shard_id = 0+shard = 0dual_encoder_model_path = "data/summarizer/dual_encoder"# linear_model_path = "data/linear_model.json"# lambda_model_path = "data/lambdamart.txt"diff --git a/configs/webgraph/host_server.toml b/configs/webgraph/host_server.tomlindex d21e5b2a..f59efb26 100644--- a/configs/webgraph/host_server.toml+++ b/configs/webgraph/host_server.toml@@ -4,4 +4,5 @@ gossip_seed_nodes = ["0.0.0.0:3005"]granularity = "host"graph_path = "data/webgraph_host"host = "0.0.0.0:3003"+shard = 0inbound_similarity_path = "data/centrality/inbound_similarity"diff --git a/configs/webgraph/page_server.toml b/configs/webgraph/page_server.tomlindex 46a58021..7c934091 100644--- a/configs/webgraph/page_server.toml+++ b/configs/webgraph/page_server.toml@@ -4,4 +4,5 @@ gossip_seed_nodes = ["0.0.0.0:3005"]granularity = "page"graph_path = "data/webgraph_page"host = "0.0.0.0:3011"+shard = 0inbound_similarity_path = "data/centrality/inbound_similarity"diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 123fc076..48564c24 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -133,14 +133,6 @@ tracing-test = { workspace = true }harness = falsename = "harmonic_centrality"-[[bench]]-harness = false-name = "build_similarity"--[[bench]]-harness = false-name = "similar_hosts"-[[bench]]harness = falsename = "hyperloglog"diff --git a/crates/core/benches/build_similarity.rs b/crates/core/benches/build_similarity.rsdeleted file mode 100644index 7670d75c..00000000--- a/crates/core/benches/build_similarity.rs+++ /dev/null@@ -1,17 +0,0 @@-use criterion::{criterion_group, criterion_main, Criterion};-use stract::{ranking::inbound_similarity::InboundSimilarity, webgraph::WebgraphBuilder};--const WEBGRAPH_PATH: &str = "data/webgraph";--pub fn criterion_benchmark(c: &mut Criterion) {- let graph = WebgraphBuilder::new(WEBGRAPH_PATH).open();-- c.bench_function("Inbound similarity creation", |b| {- b.iter(|| {- InboundSimilarity::build(&graph);- });- });-}--criterion_group!(benches, criterion_benchmark);-criterion_main!(benches);diff --git a/crates/core/benches/similar_hosts.rs b/crates/core/benches/similar_hosts.rsdeleted file mode 100644index 37cdd22e..00000000--- a/crates/core/benches/similar_hosts.rs+++ /dev/null@@ -1,30 +0,0 @@-use std::sync::Arc;--use criterion::{criterion_group, criterion_main, Criterion};-use stract::{- ranking::inbound_similarity::InboundSimilarity, similar_hosts::SimilarHostsFinder,- webgraph::WebgraphBuilder,-};--const WEBGRAPH_PATH: &str = "data/webgraph_host";-const INBOUND_SIMILARITY_PATH: &str = "data/centrality/inbound_similarity";--pub fn criterion_benchmark(c: &mut Criterion) {- let webgraph = Arc::new(WebgraphBuilder::new(WEBGRAPH_PATH).open());- let inbound = InboundSimilarity::open(INBOUND_SIMILARITY_PATH).unwrap();-- let finder = SimilarHostsFinder::new(- webgraph,- inbound,- stract::config::defaults::WebgraphServer::max_similar_hosts(),- );-- for _ in 0..10 {- c.bench_function("similar_hosts", |b| {- b.iter(|| finder.find_similar_hosts(&["google.com".to_string()], 100));- });- }-}--criterion_group!(benches, criterion_benchmark);-criterion_main!(benches);diff --git a/crates/core/examples/indexer.rs b/crates/core/examples/indexer.rsindex c9854fcb..19373be5 100644--- a/crates/core/examples/indexer.rs+++ b/crates/core/examples/indexer.rs@@ -1,7 +1,7 @@use std::path::Path;use clap::Parser;-use stract::config::IndexingLocalConfig;+use stract::config::{IndexingGraphConfig, IndexingLocalConfig};#[derive(Parser)]struct Args {@@ -32,13 +32,13 @@ fn main() -> anyhow::Result<()> {.unwrap().to_string()],}),- page_webgraph_path: Some(- data_path+ page_webgraph: Some(IndexingGraphConfig::Local {+ path: data_path.join("webgraph_page").to_str().unwrap().to_string(),- ),+ }),host_centrality_threshold: None,topics_path: None,host_centrality_store_path: data_path.join("centrality/").to_str().unwrap().to_string(),@@ -52,6 +52,8 @@ fn main() -> anyhow::Result<()> {safety_classifier_path: None,minimum_clean_words: None,batch_size: 512,+ autocommit_after_num_inserts:+ stract::config::defaults::Indexing::autocommit_after_num_inserts(),dual_encoder: args.dual_encoder_path.map(|p| stract::config::IndexingDualEncoderConfig {diff --git a/crates/core/examples/search_preindexed.rs b/crates/core/examples/search_preindexed.rsindex 44072038..b8c0b38a 100644--- a/crates/core/examples/search_preindexed.rs+++ b/crates/core/examples/search_preindexed.rs@@ -6,85 +6,12 @@ use stract::{defaults, ApiConfig, ApiThresholds, CollectorConfig, CorrectionConfig, LLMConfig,SnippetConfig, WidgetsConfig,},- distributed::member::ShardId,- image_store::Image,index::Index,- inverted_index::RetrievedWebpage,- ranking::{inbound_similarity::InboundSimilarity, pipeline::PrecisionRankingWebpage},- searcher::{api::ApiSearcher, live::LiveSearcher, LocalSearcher, SearchQuery},- Result,+ searcher::{+ api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,+ },+ webgraph::Webgraph,};-struct Searcher(LocalSearcher<Index>);--impl stract::searcher::distributed::SearchClient for Searcher {- async fn search_initial(- &self,- query: &SearchQuery,- ) -> Vec<stract::searcher::InitialSearchResultShard> {- let res = self.0.search_initial(query, true).unwrap();-- vec![stract::searcher::InitialSearchResultShard {- local_result: res,- shard: ShardId::new(0),- }]- }-- async fn retrieve_webpages(- &self,- top_websites: &[(usize, stract::searcher::ScoredWebpagePointer)],- query: &str,- ) -> Vec<(usize, stract::ranking::pipeline::PrecisionRankingWebpage)> {- let pointers = top_websites- .iter()- .map(|(_, p)| p.website.pointer.clone())- .collect::<Vec<_>>();-- let res = self- .0- .retrieve_websites(&pointers, query)- .unwrap()- .into_iter()- .zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))- .map(|(ret, (i, ran))| (i, PrecisionRankingWebpage::new(ret, ran)))- .collect::<Vec<_>>();-- res- }-- async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {- Ok(self.0.get_webpage(url))- }-- async fn get_homepage_descriptions(- &self,- urls: &[url::Url],- ) -> std::collections::HashMap<url::Url, String> {- let mut res = std::collections::HashMap::new();-- for url in urls {- if let Some(homepage) = self.0.get_homepage(url) {- if let Some(desc) = homepage.description() {- res.insert(url.clone(), desc.clone());- }- }- }-- res- }-- async fn get_entity_image(- &self,- _image_id: &str,- _max_height: Option<u64>,- _max_width: Option<u64>,- ) -> Result<Option<Image>> {- Ok(None)- }-- async fn search_entity(&self, _query: &str) -> Option<stract::entity_index::EntityMatch> {- None- }-}#[tokio::main]pub async fn main() {@@ -101,7 +28,6 @@ pub async fn main() {crossencoder_model_path: None,lambda_model_path: None,dual_encoder_model_path: None,- spell_checker_path: Some("data/web_spell".to_string()),bangs_path: "data/bangs.json".to_string(),summarizer_path: "data/summarizer".to_string(),query_store_db_host: None,@@ -114,13 +40,17 @@ pub async fn main() {thesaurus_paths: vec!["data/english-wordnet-2022-subset.ttl".to_string()],calculator_fetch_currencies_exchange: false,},- correction_config: CorrectionConfig::default(),+ spell_check: Some(stract::config::ApiSpellCheck {+ path: "data/web_spell".to_string(),+ correction_config: CorrectionConfig::default(),+ }),llm: LLMConfig {api_base: "http://localhost:4000/v1".to_string(),model: "data/mistral-7b-instruct-v0.2.Q4_K_M.gguf".to_string(),api_key: None,},max_concurrent_searches: defaults::Api::max_concurrent_searches(),+ max_similar_hosts: defaults::Api::max_similar_hosts(),};let mut queries = stract::autosuggest::Autosuggest::load_csv(&config.queries_csv_path)@@ -131,9 +61,6 @@ pub async fn main() {queries.shuffle(&mut rand::thread_rng());let mut searcher = LocalSearcher::new(index);- searcher.set_inbound_similarity(- InboundSimilarity::open("data/centrality/inbound_similarity").unwrap(),- );searcher.set_collector_config(collector_conf);searcher.set_snippet_config(SnippetConfig {num_words_for_lang_detection: Some(250),@@ -142,10 +69,10 @@ pub async fn main() {});let bangs = Bangs::from_path(&config.bangs_path);- let searcher = Searcher(searcher);+ let searcher = stract::searcher::LocalSearchClient::from(searcher);- let searcher: ApiSearcher<Searcher, LiveSearcher> =- ApiSearcher::new(searcher, None, None, None, None, bangs, config);+ let searcher: ApiSearcher<LocalSearchClient, LiveSearcher, Webgraph> =+ ApiSearcher::new(searcher, bangs, config);for query in queries {let mut query = query;diff --git a/crates/core/src/ampc/dht_conn.rs b/crates/core/src/ampc/dht_conn.rsindex c34c489d..63637334 100644--- a/crates/core/src/ampc/dht_conn.rs+++ b/crates/core/src/ampc/dht_conn.rs@@ -14,12 +14,10 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use crate::block_on;use std::{collections::BTreeMap, net::SocketAddr, pin::Pin};-use super::{- block_on,- dht::{self, upsert::UpsertEnum, UpsertAction},-};+use super::dht::{self, upsert::UpsertEnum, UpsertAction};use crate::Result;@@ -51,11 +49,11 @@ macro_rules! impl_dht_tables {fn cleanup_prev_tables(&self) {$(- let tables = $crate::ampc::block_on(self.$field.client().all_tables()).unwrap();+ let tables = $crate::block_on(self.$field.client().all_tables()).unwrap();for table in tables {if table.as_str().starts_with(&self.$field.table().prefix()) {- $crate::ampc::block_on(self.$field.client().drop_table(table)).unwrap();+ $crate::block_on(self.$field.client().drop_table(table)).unwrap();}})*diff --git a/crates/core/src/ampc/mod.rs b/crates/core/src/ampc/mod.rsindex a2e58c23..9add597b 100644--- a/crates/core/src/ampc/mod.rs+++ b/crates/core/src/ampc/mod.rs@@ -36,18 +36,6 @@ pub use dht_conn::{DefaultDhtTable, DhtConn, DhtTable, DhtTables, Table};pub use server::Server;pub use worker::{Message, RequestWrapper, Worker};-static TOKIO_RUNTIME: once_cell::sync::Lazy<tokio::runtime::Runtime> =- once_cell::sync::Lazy::new(|| {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- });--pub fn block_on<F: std::future::Future>(f: F) -> F::Output {- TOKIO_RUNTIME.block_on(f)-}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub enum CoordReq<J, M, T> {CurrentJob,diff --git a/crates/core/src/ampc/server.rs b/crates/core/src/ampc/server.rsindex 1cc1ab74..5da586a5 100644--- a/crates/core/src/ampc/server.rs+++ b/crates/core/src/ampc/server.rs@@ -21,7 +21,8 @@ use tokio::net::ToSocketAddrs;use crate::distributed::sonic;use crate::Result;-use super::{block_on, CoordReq, CoordResp, JobDht, JobReq, JobResp, Mapper, Req, Resp, Worker};+use super::{CoordReq, CoordResp, JobDht, JobReq, JobResp, Mapper, Req, Resp, Worker};+use crate::block_on;pub struct Server<W>wherediff --git a/crates/core/src/ampc/worker.rs b/crates/core/src/ampc/worker.rsindex 7bf825e4..05fedacd 100644--- a/crates/core/src/ampc/worker.rs+++ b/crates/core/src/ampc/worker.rs@@ -16,9 +16,8 @@use std::net::SocketAddr;-use super::{- block_on, CoordReq, CoordResp, DhtConn, Job, JobConn, JobReq, JobResp, Req, Resp, Server,-};+use super::{CoordReq, CoordResp, DhtConn, Job, JobConn, JobReq, JobResp, Req, Resp, Server};+use crate::block_on;use crate::Result;use anyhow::anyhow;use tokio::net::ToSocketAddrs;diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex 0b5177c6..a76717f2 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -35,6 +35,8 @@ use crate::{models::dual_encoder::DualEncoder,ranking::models::lambdamart::LambdaMART,searcher::{api::ApiSearcher, live::LiveSearcher, DistributedSearcher},+ similar_hosts::SimilarHostsFinder,+ webgraph::remote::RemoteWebgraph,};use crate::{ranking::models::cross_encoder::CrossEncoderModel, summarizer::Summarizer};@@ -52,8 +54,6 @@ use axum::{routing::post,};-use self::webgraph::RemoteWebgraph;-mod autosuggest;mod docs;mod explore;@@ -74,13 +74,15 @@ pub struct Counters {pub struct State {pub config: ApiConfig,- pub searcher: ApiSearcher<DistributedSearcher, LiveSearcher>,- pub remote_webgraph: RemoteWebgraph,+ pub searcher: Arc<ApiSearcher<DistributedSearcher, LiveSearcher, Arc<RemoteWebgraph>>>,+ pub page_webgraph: Arc<RemoteWebgraph>,+ pub host_webgraph: Arc<RemoteWebgraph>,pub autosuggest: Autosuggest,pub counters: Counters,pub summarizer: Arc<Summarizer>,pub improvement_queue: Option<Arc<Mutex<LeakyQueue<ImprovementEvent>>>>,- pub cluster: Arc<Cluster>,+ pub _cluster: Arc<Cluster>,+ pub similar_hosts: SimilarHostsFinder,}pub async fn favicon() -> impl IntoResponse {@@ -180,7 +182,11 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {).await?,);- let remote_webgraph = RemoteWebgraph::new(cluster.clone());++ let host_webgraph =+ RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Host);+ let page_webgraph =+ RemoteWebgraph::new(cluster.clone(), crate::config::WebgraphGranularity::Page);let dist_searcher = DistributedSearcher::new(Arc::clone(&cluster));let live_searcher = LiveSearcher::new(Arc::clone(&cluster));@@ -192,22 +198,34 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {cross_encoder = Some(CrossEncoderModel::open(path)?);}- let searcher = ApiSearcher::new(- dist_searcher,- Some(live_searcher),- cross_encoder,- lambda_model,- dual_encoder_model,- bangs,- config.clone(),- );+ let mut searcher =+ ApiSearcher::new(dist_searcher, bangs, config.clone()).with_live(live_searcher);++ if let Some(cross_encoder) = cross_encoder {+ searcher = searcher.with_cross_encoder(cross_encoder);+ }++ if let Some(lambda) = lambda_model {+ searcher = searcher.with_lambda_model(lambda);+ }++ if let Some(dual_encoder_model) = dual_encoder_model {+ searcher = searcher.with_dual_encoder(dual_encoder_model);+ }++ let host_webgraph = Arc::new(host_webgraph);+ let page_webgraph = Arc::new(page_webgraph);++ let similar_hosts =+ SimilarHostsFinder::new(Arc::clone(&host_webgraph), config.max_similar_hosts);Arc::new(State {config: config.clone(),- searcher,+ searcher: Arc::new(searcher),autosuggest,counters,- remote_webgraph,+ host_webgraph,+ page_webgraph,summarizer: Arc::new(Summarizer::new(&config.summarizer_path,config.llm.api_base.clone(),@@ -215,7 +233,8 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {config.llm.api_key.clone(),)?),improvement_queue: query_store_queue,- cluster,+ _cluster: cluster,+ similar_hosts,})};diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex 4db9628f..428afa73 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{net::SocketAddr, sync::Arc, time::Duration};+use std::sync::Arc;use axum::{extract, response::IntoResponse, Json};use http::StatusCode;@@ -22,36 +22,16 @@ use utoipa::{IntoParams, ToSchema};use crate::{config::WebgraphGranularity,- distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff, sonic},webgraph::{FullEdge, Node},};use super::State;-pub struct RemoteWebgraph {- cluster: Arc<Cluster>,-}--impl RemoteWebgraph {- pub fn new(cluster: Arc<Cluster>) -> Self {- Self { cluster }- }-- async fn host(&self, level: WebgraphGranularity) -> Option<SocketAddr> {- self.cluster- .members()- .await- .iter()- .find_map(|member| match member.service {- Service::Webgraph { host, granularity } if granularity == level => Some(host),- _ => None,- })- }-}-pub mod host {use url::Url;+ use crate::entrypoint::webgraph_server::ScoredHost;+use super::*;#[derive(serde::Deserialize, ToSchema)]@@ -85,40 +65,22 @@ pub mod host {extract::Json(params): extract::Json<SimilarHostsParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {state.counters.explore_counter.inc();- let host = state- .remote_webgraph- .host(WebgraphGranularity::Host)- .await- .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;- let retry = ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5);-- let conn = sonic::service::Connection::create_with_timeout_retry(- host,- Duration::from_secs(30),- retry,- )- .await- .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;-- match conn- .send_with_timeout(- &crate::entrypoint::webgraph_server::SimilarHosts {- hosts: params.hosts,- top_n: params.top_n,- },- Duration::from_secs(60),- )- .await- {- Ok(nodes) => Ok(Json(nodes)),- Err(err) => {- tracing::error!("Failed to send request to webgraph: {}", err);- Err(StatusCode::INTERNAL_SERVER_ERROR)- }- }+ let hosts: Vec<_> = params.hosts.into_iter().take(8).collect();++ Ok(Json(+ state+ .similar_hosts+ .find_similar_hosts(&hosts, params.top_n)+ .await+ .into_iter()+ .map(|node| ScoredHost {+ host: node.node.as_str().to_string(),+ score: node.score,+ description: None,+ })+ .collect::<Vec<_>>(),+ ))}#[utoipa::path(post,@@ -132,31 +94,7 @@ pub mod host {extract::State(state): extract::State<Arc<State>>,extract::Query(params): extract::Query<KnowsHostParams>,) -> std::result::Result<impl IntoResponse, StatusCode> {- let host = state- .remote_webgraph- .host(WebgraphGranularity::Host)- .await- .ok_or(StatusCode::INTERNAL_SERVER_ERROR)?;-- let retry = ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5);-- let conn = sonic::service::Connection::create_with_timeout_retry(- host,- Duration::from_secs(30),- retry,- )- .await- .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;-- match conn- .send_with_timeout(- &crate::entrypoint::webgraph_server::Knows { host: params.host },- Duration::from_secs(60),- )- .await- {+ match state.host_webgraph.knows(params.host).await {Ok(Some(node)) => Ok(Json(KnowsHost::Known {host: node.as_str().to_string(),})),@@ -276,28 +214,12 @@ async fn ingoing_links(node: Node,level: WebgraphGranularity,) -> anyhow::Result<Vec<FullEdge>> {- let host = state- .remote_webgraph- .host(level)- .await- .ok_or(anyhow::anyhow!(- "no remote webgraph for granularity {level:?}"- ))?;-- let retry = ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5);+ let graph = match level {+ WebgraphGranularity::Host => &state.host_webgraph,+ WebgraphGranularity::Page => &state.page_webgraph,+ };- let conn =- sonic::service::Connection::create_with_timeout_retry(host, Duration::from_secs(30), retry)- .await?;-- Ok(conn- .send_with_timeout(- &crate::entrypoint::webgraph_server::IngoingLinks { node },- Duration::from_secs(60),- )- .await?)+ graph.ingoing_edges(node).await}async fn outgoing_links(@@ -305,28 +227,12 @@ async fn outgoing_links(node: Node,level: WebgraphGranularity,) -> anyhow::Result<Vec<FullEdge>> {- let host = state- .remote_webgraph- .host(level)- .await- .ok_or(anyhow::anyhow!(- "no remote webgraph for granularity {level:?}"- ))?;+ let graph = match level {+ WebgraphGranularity::Host => &state.host_webgraph,+ WebgraphGranularity::Page => &state.page_webgraph,+ };- let retry = ExponentialBackoff::from_millis(30)- .with_limit(Duration::from_millis(200))- .take(5);-- let conn =- sonic::service::Connection::create_with_timeout_retry(host, Duration::from_secs(30), retry)- .await?;-- Ok(conn- .send_with_timeout(- &crate::entrypoint::webgraph_server::OutgoingLinks { node },- Duration::from_secs(60),- )- .await?)+ graph.outgoing_edges(node).await}#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]diff --git a/crates/core/src/bangs.rs b/crates/core/src/bangs.rsindex 05d9a76c..fb366299 100644--- a/crates/core/src/bangs.rs+++ b/crates/core/src/bangs.rs@@ -122,6 +122,13 @@ pub struct Bangs {}impl Bangs {+ #[cfg(test)]+ pub fn empty() -> Self {+ Self {+ bangs: HashMap::new(),+ }+ }+pub fn from_path<P: AsRef<Path>>(path: P) -> Self {let json = fs::read_to_string(path).unwrap();diff --git a/crates/core/src/config/defaults.rs b/crates/core/src/config/defaults.rsindex db8ffc6c..982d747a 100644--- a/crates/core/src/config/defaults.rs+++ b/crates/core/src/config/defaults.rs@@ -52,6 +52,10 @@ impl Api {pub fn max_concurrent_searches() -> Option<usize> {None}++ pub fn max_similar_hosts() -> usize {+ 1_000+ }}pub struct Snippet;@@ -122,14 +126,6 @@ impl Crawler {}}-pub struct WebgraphServer;--impl WebgraphServer {- pub fn max_similar_hosts() -> usize {- 1_000- }-}-pub struct SearchQuery;impl SearchQuery {@@ -180,4 +176,8 @@ impl Indexing {pub fn batch_size() -> usize {512}++ pub fn autocommit_after_num_inserts() -> usize {+ 25_000+ }}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex fdf5b5da..89f341ab 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -31,7 +31,7 @@ pub struct IndexingLocalConfig {pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,pub warc_source: WarcSource,- pub page_webgraph_path: Option<String>,+ pub page_webgraph: Option<IndexingGraphConfig>,pub host_centrality_threshold: Option<f64>,pub topics_path: Option<String>,pub host_centrality_store_path: String,@@ -42,9 +42,19 @@ pub struct IndexingLocalConfig {#[serde(default = "defaults::Indexing::batch_size")]pub batch_size: usize,+ #[serde(default = "defaults::Indexing::autocommit_after_num_inserts")]+ pub autocommit_after_num_inserts: usize,+pub dual_encoder: Option<IndexingDualEncoderConfig>,}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]+#[serde(tag = "type")]+pub enum IndexingGraphConfig {+ Local { path: String },+ Remote { gossip: GossipConfig },+}+#[derive(Debug, serde::Deserialize, Clone)]pub struct IndexingDualEncoderConfig {pub model_path: String,@@ -208,6 +218,14 @@ pub struct LLMConfig {pub api_key: Option<String>,}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]+pub struct ApiSpellCheck {+ pub path: String,++ #[serde(default)]+ pub correction_config: CorrectionConfig,+}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct ApiConfig {pub summarizer_path: String,@@ -217,17 +235,18 @@ pub struct ApiConfig {pub crossencoder_model_path: Option<String>,pub lambda_model_path: Option<String>,pub dual_encoder_model_path: Option<String>,- pub spell_checker_path: Option<String>,pub bangs_path: String,pub query_store_db_host: Option<String>,pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,- pub llm: LLMConfig,+ #[serde(default = "defaults::Api::max_similar_hosts")]+ pub max_similar_hosts: usize,- #[serde(default)]- pub collector: CollectorConfig,+ pub spell_check: Option<ApiSpellCheck>,++ pub llm: LLMConfig,#[serde(default)]pub thresholds: ApiThresholds,@@ -235,7 +254,7 @@ pub struct ApiConfig {pub widgets: WidgetsConfig,#[serde(default)]- pub correction_config: CorrectionConfig,+ pub collector: CollectorConfig,#[serde(default = "defaults::Api::max_concurrent_searches")]pub max_concurrent_searches: Option<usize>,@@ -286,9 +305,8 @@ pub struct SearchServerConfig {pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,- pub shard_id: ShardId,+ pub shard: ShardId,pub index_path: String,- pub host_centrality_store_path: Option<String>,pub linear_model_path: Option<String>,pub lambda_model_path: Option<String>,pub dual_encoder_model_path: Option<String>,@@ -391,16 +409,13 @@ pub enum WebgraphGranularity {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct WebgraphServerConfig {pub host: SocketAddr,+ pub shard: ShardId,pub graph_path: String,pub granularity: WebgraphGranularity,- pub inbound_similarity_path: Option<String>,pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,-- #[serde(default = "defaults::WebgraphServer::max_similar_hosts")]- pub max_similar_hosts: usize,}#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]@@ -411,6 +426,16 @@ pub struct WidgetsConfig {pub calculator_fetch_currencies_exchange: bool,}+impl Default for WidgetsConfig {+ fn default() -> Self {+ Self {+ thesaurus_paths: Vec::new(),+ calculator_fetch_currencies_exchange:+ defaults::Widgets::calculator_fetch_currencies_exchange(),+ }+ }+}+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlPlannerConfig {pub page_harmonic_path: String,@@ -454,7 +479,7 @@ pub struct LiveIndexConfig {// indexerpub host_centrality_store_path: String,- pub page_webgraph_path: Option<String>,+ pub page_webgraph: Option<IndexingGraphConfig>,pub page_centrality_store_path: Option<String>,pub safety_classifier_path: Option<String>,pub host_centrality_threshold: Option<f64>,diff --git a/crates/core/src/distributed/cluster.rs b/crates/core/src/distributed/cluster.rsindex c5df507a..7a9a4eed 100644--- a/crates/core/src/distributed/cluster.rs+++ b/crates/core/src/distributed/cluster.rs@@ -14,11 +14,11 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use chitchat::{- spawn_chitchat, transport::UdpTransport, ChitchatConfig, ChitchatHandle, FailureDetectorConfig,- NodeId,+ spawn_chitchat, transport::UdpTransport, Chitchat, ChitchatConfig, ChitchatHandle,+ FailureDetectorConfig, NodeId,};use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration};-use tokio::sync::RwLock;+use tokio::sync::{Mutex, RwLock};use tokio_stream::StreamExt;use tracing::error;@@ -30,6 +30,57 @@ const SERVICE_KEY: &str = "service";type Result<T> = std::result::Result<T, anyhow::Error>;+fn alive_nodes_updater(chitchat: Arc<Mutex<Chitchat>>) -> Arc<RwLock<HashSet<Member>>> {+ let alive_nodes = Arc::new(RwLock::new(HashSet::new()));+ let alive_nodes_ref = alive_nodes.clone();++ tokio::spawn(async move {+ let mut node_change_receiver = chitchat.lock().await.ready_nodes_watcher();++ while let Some(members_set) = node_change_receiver.next().await {+ let alive_node_ids: HashSet<_> = alive_nodes_ref+ .read()+ .await+ .iter()+ .map(|member: &Member| member.id.clone())+ .collect();++ let member_ids: HashSet<_> =+ members_set.iter().map(|member| member.id.clone()).collect();++ if alive_node_ids != member_ids {+ let snapshot = chitchat.lock().await.state_snapshot();+ let mut new_members = Vec::new();+ for member in members_set {+ if let Some(state) = snapshot.node_states.get(&member.id) {+ if let Some(service) = state.get(SERVICE_KEY) {+ let service: Service = serde_json::from_str(service).unwrap();+ new_members.push(Member {+ service,+ id: member.id,+ });+ } else {+ error!("failed to get service");+ }+ } else {+ error!("no state found for node")+ }+ }++ tracing::info!("new members: {:#?}", new_members);+ let mut write = alive_nodes_ref.write().await;+ write.clear();++ for member in new_members {+ write.insert(member);+ }+ }+ }+ });++ alive_nodes+}+pub struct Cluster {alive_nodes: Arc<RwLock<HashSet<Member>>>,// dropping the handle leaves the cluster@@ -42,7 +93,6 @@ impl Cluster {gossip_addr: SocketAddr,seed_addrs: Vec<SocketAddr>,) -> Result<Self> {- let transport = UdpTransport;let failure_detector_config = FailureDetectorConfig {initial_interval: GOSSIP_INTERVAL,..Default::default()@@ -54,7 +104,6 @@ impl Cluster {id: format!("{}_{}", self_node.id, uuid),gossip_public_address: gossip_addr,};-let config = ChitchatConfig {node_id,cluster_id: CLUSTER_ID.to_string(),@@ -68,63 +117,58 @@ impl Cluster {is_ready_predicate: None,};- let chitchat_handle = spawn_chitchat(+ Self::join_with_config(config,vec![(SERVICE_KEY.to_string(),serde_json::to_string(&self_node.service)?,)],- &transport,)- .await?;- let chitchat = chitchat_handle.chitchat();+ .await+ }- let alive_nodes = Arc::new(RwLock::new(HashSet::new()));- let alive_nodes_ref = alive_nodes.clone();-- tokio::spawn(async move {- let mut node_change_receiver = chitchat.lock().await.ready_nodes_watcher();-- while let Some(members_set) = node_change_receiver.next().await {- let alive_node_ids: HashSet<_> = alive_nodes_ref- .read()- .await- .iter()- .map(|member: &Member| member.id.clone())- .collect();-- let member_ids: HashSet<_> =- members_set.iter().map(|member| member.id.clone()).collect();-- if alive_node_ids != member_ids {- let snapshot = chitchat.lock().await.state_snapshot();- let mut new_members = Vec::new();- for member in members_set {- if let Some(state) = snapshot.node_states.get(&member.id) {- if let Some(service) = state.get(SERVICE_KEY) {- let service: Service = serde_json::from_str(service).unwrap();- new_members.push(Member {- service,- id: member.id,- });- } else {- error!("failed to get service");- }- } else {- error!("no state found for node")- }- }+ pub async fn join_as_spectator(+ cluster_id: String,+ gossip_addr: SocketAddr,+ seed_addrs: Vec<SocketAddr>,+ ) -> Result<Self> {+ let failure_detector_config = FailureDetectorConfig {+ initial_interval: GOSSIP_INTERVAL,+ ..Default::default()+ };- tracing::info!("new members: {:#?}", new_members);- let mut write = alive_nodes_ref.write().await;- write.clear();+ let uuid = uuid::Uuid::new_v4().to_string();- for member in new_members {- write.insert(member);- }- }- }- });+ let node_id = NodeId {+ id: format!("{}_{}", cluster_id, uuid),+ gossip_public_address: gossip_addr,+ };+ let config = ChitchatConfig {+ node_id,+ cluster_id: CLUSTER_ID.to_string(),+ gossip_interval: GOSSIP_INTERVAL,+ listen_addr: gossip_addr,+ seed_nodes: seed_addrs+ .into_iter()+ .map(|addr| addr.to_string())+ .collect(),+ failure_detector_config,+ is_ready_predicate: None,+ };++ Self::join_with_config(config, vec![]).await+ }++ async fn join_with_config(+ config: ChitchatConfig,+ key_values: Vec<(String, String)>,+ ) -> Result<Self> {+ let transport = UdpTransport;++ let chitchat_handle = spawn_chitchat(config, key_values, &transport).await?;+ let chitchat = chitchat_handle.chitchat();++ let alive_nodes = alive_nodes_updater(chitchat);Ok(Self {alive_nodes,diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex 87c074f1..ec10a189 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -80,6 +80,7 @@ pub enum Service {},Webgraph {host: SocketAddr,+ shard: ShardId,granularity: WebgraphGranularity,},Dht {diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rsindex 7e07f26a..bfafb881 100644--- a/crates/core/src/distributed/sonic/replication.rs+++ b/crates/core/src/distributed/sonic/replication.rs@@ -113,6 +113,18 @@ where.await}+ pub async fn batch_send<R: sonic::service::Wrapper<S>>(+ &self,+ reqs: &[R],+ ) -> Result<Vec<R::Response>> {+ self.batch_send_with_timeout_retry(+ reqs,+ Duration::from_secs(60),+ ExponentialBackoff::from_millis(500).with_limit(Duration::from_secs(3)),+ )+ .await+ }+pub async fn send_with_timeout<R: sonic::service::Wrapper<S>>(&self,req: &R,@@ -122,6 +134,15 @@ whereconn.send_with_timeout(req, timeout).await}+ pub async fn batch_send_with_timeout<R: sonic::service::Wrapper<S>>(+ &self,+ reqs: &[R],+ timeout: Duration,+ ) -> Result<Vec<R::Response>> {+ let conn = self.conn().await?;+ conn.batch_send_with_timeout(reqs, timeout).await+ }+pub async fn send_with_timeout_retry<R: sonic::service::Wrapper<S>>(&self,req: &R,@@ -142,6 +163,27 @@ whereErr(er.unwrap())}++ pub async fn batch_send_with_timeout_retry<R: sonic::service::Wrapper<S>>(+ &self,+ reqs: &[R],+ timeout: Duration,+ retry: impl Iterator<Item = Duration>,+ ) -> Result<Vec<R::Response>> {+ let mut er = None;+ for backoff in retry {+ match self.batch_send_with_timeout(reqs, timeout).await {+ Ok(r) => return Ok(r),+ Err(e) => {+ tracing::error!("Failed to send request: {:?}", e);+ er = Some(e);+ tokio::time::sleep(backoff).await;+ }+ }+ }++ Err(er.unwrap())+ }}pub trait ReplicaSelector<S: sonic::service::Service> {@@ -183,14 +225,67 @@ whereSelf { clients }}- pub async fn send<Req, Sel>(&self, req: &Req, selector: &Sel) -> Result<Vec<Req::Response>>+ async fn send_single<Req>(+ &self,+ req: &Req,+ client: &RemoteClient<S>,+ ) -> Result<(SocketAddr, Req::Response)>+ where+ Req: sonic::service::Wrapper<S>,+ {+ Ok((client.addr(), client.send(req).await?))+ }++ pub async fn send<Req, Sel>(+ &self,+ req: &Req,+ selector: &Sel,+ ) -> Result<Vec<(SocketAddr, Req::Response)>>whereReq: sonic::service::Wrapper<S>,Sel: ReplicaSelector<S>,{let mut futures = Vec::new();for client in selector.select(&self.clients) {- futures.push(client.send(req));+ futures.push(self.send_single(req, client));+ }++ let mut results = Vec::new();+ for r in join_all(futures).await {+ match r {+ Ok(r) => results.push(r),+ Err(e) => {+ tracing::error!("Failed to send request: {:?}", e);+ }+ }+ }++ Ok(results)+ }++ async fn batch_send_single<Req>(+ &self,+ reqs: &[Req],+ client: &RemoteClient<S>,+ ) -> Result<(SocketAddr, Vec<Req::Response>)>+ where+ Req: sonic::service::Wrapper<S>,+ {+ Ok((client.addr(), client.batch_send(reqs).await?))+ }++ pub async fn batch_send<Req, Sel>(+ &self,+ reqs: &[Req],+ selector: &Sel,+ ) -> Result<Vec<(SocketAddr, Vec<Req::Response>)>>+ where+ Req: sonic::service::Wrapper<S>,+ Sel: ReplicaSelector<S>,+ {+ let mut futures = Vec::new();+ for client in selector.select(&self.clients) {+ futures.push(self.batch_send_single(reqs, client));}let mut results = Vec::new();@@ -283,7 +378,7 @@ wherereq: &Req,shard: &Shard<S, Id>,replica_selector: &Sel,- ) -> Result<(Id, Vec<Req::Response>)>+ ) -> Result<(Id, Vec<(SocketAddr, Req::Response)>)>whereReq: sonic::service::Wrapper<S>,Sel: ReplicaSelector<S>,@@ -299,7 +394,7 @@ wherereq: &Req,shard_selector: &SSel,replica_selector: &RSel,- ) -> Result<Vec<(Id, Vec<Req::Response>)>>+ ) -> Result<Vec<(Id, Vec<(SocketAddr, Req::Response)>)>>whereReq: sonic::service::Wrapper<S>,SSel: ShardSelector<S, Id>,@@ -322,4 +417,49 @@ whereOk(results)}++ async fn batch_send_single<Req, Sel>(+ &self,+ reqs: &[Req],+ shard: &Shard<S, Id>,+ replica_selector: &Sel,+ ) -> Result<(Id, Vec<(SocketAddr, Vec<Req::Response>)>)>+ where+ Req: sonic::service::Wrapper<S>,+ Sel: ReplicaSelector<S>,+ {+ Ok((+ shard.id.clone(),+ shard.replicas.batch_send(reqs, replica_selector).await?,+ ))+ }++ pub async fn batch_send<Req, SSel, RSel>(+ &self,+ reqs: &[Req],+ shard_selector: &SSel,+ replica_selector: &RSel,+ ) -> Result<Vec<(Id, Vec<(SocketAddr, Vec<Req::Response>)>)>>+ where+ Req: sonic::service::Wrapper<S>,+ SSel: ShardSelector<S, Id>,+ RSel: ReplicaSelector<S>,+ {+ let mut futures = Vec::new();+ for shard in shard_selector.select(&self.shards) {+ futures.push(self.batch_send_single(reqs, shard, replica_selector));+ }++ let mut results = Vec::new();+ for r in join_all(futures).await {+ match r {+ Ok(r) => results.push(r),+ Err(e) => {+ tracing::error!("Failed to send request: {:?}", e);+ }+ }+ }++ Ok(results)+ }}diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rsindex 5cba0b51..2857b2f2 100644--- a/crates/core/src/distributed/sonic/service.rs+++ b/crates/core/src/distributed/sonic/service.rs@@ -18,6 +18,8 @@ use std::{sync::Arc, time::Duration};use tokio::net::ToSocketAddrs;+use crate::OneOrMany;+use super::Result;pub trait Service: Sized + Send + Sync + 'static {@@ -41,7 +43,7 @@ pub trait Wrapper<S: Service>: Message<S> {}pub struct Server<S: Service> {- inner: super::Server<S::Request, S::Response>,+ inner: super::Server<OneOrMany<S::Request>, OneOrMany<S::Response>>,service: Arc<S>,}@@ -57,9 +59,25 @@ impl<S: Service> Server<S> {let service = Arc::clone(&self.service);tokio::spawn(async move {- let res = S::handle(req.take_body(), &service).await;- if let Err(e) = req.respond(res).await {- tracing::error!("failed to respond to request: {}", e);+ match req.take_body() {+ OneOrMany::One(body) => {+ let res = S::handle(body, &service).await;++ if let Err(e) = req.respond(OneOrMany::One(res)).await {+ tracing::error!("failed to respond to request: {}", e);+ }+ }+ OneOrMany::Many(bodies) => {+ let mut res = Vec::new();++ for req in bodies {+ res.push(S::handle(req, &service).await);+ }++ if let Err(e) = req.respond(OneOrMany::Many(res)).await {+ tracing::error!("failed to respond to request: {}", e);+ }+ }}});@@ -68,18 +86,16 @@ impl<S: Service> Server<S> {}pub struct Connection<'a, S: Service> {- inner: super::Connection<S::RequestRef<'a>, S::Response>,+ inner: super::Connection<OneOrMany<S::RequestRef<'a>>, OneOrMany<S::Response>>,}impl<'a, S: Service> Connection<'a, S> {- #[allow(dead_code)]pub async fn create(server: impl ToSocketAddrs) -> Result<Connection<'a, S>> {Ok(Connection {inner: super::Connection::create(server).await?,})}- #[allow(dead_code)]pub async fn create_with_timeout(server: impl ToSocketAddrs,timeout: Duration,@@ -99,19 +115,26 @@ impl<'a, S: Service> Connection<'a, S> {})}- #[allow(dead_code)]pub async fn send_without_timeout<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {Ok(R::unwrap_response(self.inner- .send_without_timeout(&R::wrap_request_ref(request))- .await?,+ .send_without_timeout(&OneOrMany::One(R::wrap_request_ref(request)))+ .await?+ .one()+ .expect("response is missing"),).unwrap())}- #[allow(dead_code)]pub async fn send<R: Wrapper<S>>(self, request: &'a R) -> Result<R::Response> {- Ok(R::unwrap_response(self.inner.send(&R::wrap_request_ref(request)).await?).unwrap())+ Ok(R::unwrap_response(+ self.inner+ .send(&OneOrMany::One(R::wrap_request_ref(request)))+ .await?+ .one()+ .expect("response is missing"),+ )+ .unwrap())}pub async fn send_with_timeout<R: Wrapper<S>>(@@ -121,11 +144,36 @@ impl<'a, S: Service> Connection<'a, S> {) -> Result<R::Response> {Ok(R::unwrap_response(self.inner- .send_with_timeout(&R::wrap_request_ref(request), timeout)- .await?,+ .send_with_timeout(&OneOrMany::One(R::wrap_request_ref(request)), timeout)+ .await?+ .one()+ .expect("response is missing"),).unwrap())}++ pub async fn batch_send_with_timeout<R: Wrapper<S>>(+ self,+ requests: &'a [R],+ timeout: Duration,+ ) -> Result<Vec<R::Response>> {+ Ok(self+ .inner+ .send_with_timeout(+ &OneOrMany::Many(+ requests+ .iter()+ .map(|req| R::wrap_request_ref(req))+ .collect::<Vec<_>>(),+ ),+ timeout,+ )+ .await?+ .many()+ .into_iter()+ .map(|res| R::unwrap_response(res).unwrap())+ .collect())+ }}macro_rules! sonic_service {diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rsindex a8e4aa2e..34fb932a 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs@@ -19,10 +19,7 @@ use bloom::U64BloomFilter;use crate::{ampc::prelude::*,config::HarmonicWorkerConfig,- distributed::{- cluster::Cluster,- member::{Member, Service, ShardId},- },+ distributed::member::{Service, ShardId},webgraph::{self, Webgraph},Result,};@@ -151,42 +148,16 @@ impl RemoteWorker for RemoteCentralityWorker {}}-fn start_gossip_cluster_thread(config: HarmonicWorkerConfig) {- std::thread::spawn(move || {- let rt = tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap();-- rt.block_on(async {- let _cluster = Cluster::join(- Member {- id: config.gossip.cluster_id,- service: Service::HarmonicWorker {- host: config.host,- shard: config.shard,- },- },- config.gossip.addr,- config.gossip.seed_nodes.unwrap_or_default(),- )- .await;-- // need to keep tokio runtime alive- // otherwise the spawned task in Cluster::join will be dropped- loop {- tokio::time::sleep(std::time::Duration::from_secs(1)).await;- }- });- });-}-pub fn run(config: HarmonicWorkerConfig) -> Result<()> {let tokio_conf = config.clone();let graph = Webgraph::builder(config.graph_path).open();let worker = CentralityWorker::new(config.shard, graph);- start_gossip_cluster_thread(tokio_conf);+ let service = Service::HarmonicWorker {+ host: tokio_conf.host,+ shard: tokio_conf.shard,+ };+ crate::start_gossip_cluster_thread(tokio_conf.gossip, Some(service));worker.run(config.host)?;diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rsindex 4ff9a43d..248bdba0 100644--- a/crates/core/src/entrypoint/centrality.rs+++ b/crates/core/src/entrypoint/centrality.rs@@ -19,7 +19,6 @@ use std::{cmp::Reverse, path::Path};use crate::{external_sort::ExternalSorter,- ranking::inbound_similarity::InboundSimilarity,webgraph::{centrality::{approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality, store_csv,@@ -54,19 +53,6 @@ impl Centrality {store_csv(top_harmonics, base_output.as_ref().join("harmonic.csv"));}- pub fn build_similarity<P: AsRef<Path>>(webgraph_path: P, base_output: P) {- tracing::info!(- "Building inbound similarity for {}",- webgraph_path.as_ref().to_str().unwrap()- );- let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();-- let sim = InboundSimilarity::build(&graph);-- sim.save(base_output.as_ref().join("inbound_similarity"))- .unwrap();- }-pub fn build_approx_harmonic<P: AsRef<Path>>(webgraph_path: P, base_output: P) -> Result<()> {tracing::info!("Building approximated harmonic centrality for {}",diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex 5ce10484..c6fb7d92 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -20,7 +20,8 @@ use tokio_stream::StreamExt;use tracing::{debug, info};use crate::config::{- defaults, IndexingDualEncoderConfig, IndexingLocalConfig, LocalConfig, WebSpellConfig,+ defaults, IndexingDualEncoderConfig, IndexingGraphConfig, IndexingLocalConfig, LocalConfig,+ WebSpellConfig,};use crate::entrypoint::indexer::JobSettings;use crate::entrypoint::{dmoz_parser, indexer};@@ -142,7 +143,6 @@ fn calculate_centrality() {if !out_path.exists() {Centrality::build_harmonic(&webgraph_path, &out_path);- Centrality::build_similarity(&webgraph_path, &out_path);}let webgraph_page = Path::new(DATA_PATH).join("webgraph_page");@@ -178,6 +178,7 @@ fn create_inverted_index() -> Result<()> {host_centrality_threshold: None,minimum_clean_words: None,batch_size: defaults::Indexing::batch_size(),+ autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),},};@@ -189,7 +190,9 @@ fn create_inverted_index() -> Result<()> {let worker = indexer::IndexingWorker::new(IndexingLocalConfig {host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),- page_webgraph_path: Some(webgraph_path.to_str().unwrap().to_string()),+ page_webgraph: Some(IndexingGraphConfig::Local {+ path: webgraph_path.to_str().unwrap().to_string(),+ }),topics_path: Some(Path::new(DATA_PATH).join("human_annotations")@@ -205,6 +208,7 @@ fn create_inverted_index() -> Result<()> {safety_classifier_path: None,minimum_clean_words: None,batch_size: defaults::Indexing::batch_size(),+ autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),dual_encoder: Some(IndexingDualEncoderConfig {model_path: dual_encoder_path.to_str().unwrap().to_string(),page_centrality_rank_threshold: Some(100_000),diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rsindex 9e2d7de7..f5a3d9e8 100644--- a/crates/core/src/entrypoint/indexer/job.rs+++ b/crates/core/src/entrypoint/indexer/job.rs@@ -43,6 +43,7 @@ pub struct JobSettings {pub host_centrality_threshold: Option<f64>,pub minimum_clean_words: Option<usize>,pub batch_size: usize,+ pub autocommit_after_num_inserts: usize,}impl Job {@@ -62,6 +63,8 @@ impl Job {let warc_files = download_all_warc_files(&paths, &self.source_config);pin!(warc_files);+ let mut num_inserts_since_commit = 0;+for file in warc_files.by_ref() {let mut batch = Vec::with_capacity(self.settings.batch_size);@@ -102,11 +105,17 @@ impl Job {warn!("{:?}", err);panic!();}++ num_inserts_since_commit += 1;}- }- index.commit().unwrap();+ if num_inserts_since_commit >= self.settings.autocommit_after_num_inserts {+ index.commit().unwrap();+ num_inserts_since_commit = 0;+ }+ }}+ index.commit().unwrap();if !has_host_centrality {warn!("no host centrality values found in {}", name);diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rsindex b6491cf9..04a75fe5 100644--- a/crates/core/src/entrypoint/indexer/mod.rs+++ b/crates/core/src/entrypoint/indexer/mod.rs@@ -59,6 +59,7 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {host_centrality_threshold: config.host_centrality_threshold,minimum_clean_words: config.minimum_clean_words,batch_size: config.batch_size,+ autocommit_after_num_inserts: config.autocommit_after_num_inserts,},}).map(|job| IndexPointer(job.process(&worker).path))diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex de0456c9..d8e7b2a6 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -15,28 +15,32 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use chrono::Utc;+use itertools::Itertools;use std::path::Path;-use url::Url;use tracing::debug;pub use super::indexable_webpage::IndexableWebpage;pub use super::job::{Job, JobSettings};-use crate::config::{IndexingDualEncoderConfig, IndexingLocalConfig, LiveIndexConfig};+use crate::config::{+ IndexingDualEncoderConfig, IndexingGraphConfig, IndexingLocalConfig, LiveIndexConfig,+ WebgraphGranularity,+};use crate::models::dual_encoder::DualEncoder as DualEncoderModel;+use crate::webgraph::remote::RemoteWebgraph;use crate::Result;use crate::human_website_annotations;use crate::index::Index;use crate::rake::RakeModel;use crate::ranking::SignalComputer;-use crate::webgraph::{Node, NodeID, Webgraph, WebgraphBuilder};+use crate::webgraph::{self, Node, NodeID};use crate::webpage::{safety_classifier, Html, Webpage};pub struct Config {pub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,- pub page_webgraph_path: Option<String>,+ pub page_webgraph: Option<IndexingGraphConfig>,pub topics_path: Option<String>,pub safety_classifier_path: Option<String>,pub dual_encoder: Option<IndexingDualEncoderConfig>,@@ -47,7 +51,7 @@ impl From<IndexingLocalConfig> for Config {Self {host_centrality_store_path: config.host_centrality_store_path,page_centrality_store_path: config.page_centrality_store_path,- page_webgraph_path: config.page_webgraph_path,+ page_webgraph: config.page_webgraph,topics_path: config.topics_path,safety_classifier_path: config.safety_classifier_path,dual_encoder: config.dual_encoder,@@ -60,7 +64,7 @@ impl From<LiveIndexConfig> for Config {Self {host_centrality_store_path: config.host_centrality_store_path,page_centrality_store_path: config.page_centrality_store_path,- page_webgraph_path: config.page_webgraph_path,+ page_webgraph: config.page_webgraph,topics_path: None,safety_classifier_path: config.safety_classifier_path,dual_encoder: None,@@ -73,6 +77,81 @@ struct DualEncoder {page_centrality_rank_threshold: Option<u64>,}+pub(super) enum Webgraph {+ Remote(RemoteWebgraph),+ Local(webgraph::Webgraph),+}++impl Webgraph {+ fn batch_raw_ingoing_edges_with_labels(&self, ids: Vec<NodeID>) -> Vec<Vec<String>> {+ let edges = match self {+ Self::Remote(webgraph) => {+ crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids))+ .unwrap_or_default()+ }+ Self::Local(webgraph) => {+ let mut res = Vec::new();++ for id in ids {+ res.push(webgraph.raw_ingoing_edges_with_labels(&id));+ }++ res+ }+ };++ edges+ .into_iter()+ .map(|edges| {+ edges+ .into_iter()+ .map(|edge| edge.label)+ .filter(|label| !label.is_empty())+ .filter(|label| {+ let label = label.to_lowercase();+ let stopwords = [+ "click",+ "click here",+ "here",+ "link",+ "website",+ "webpage",+ "page",+ "site",+ "url",+ "web",+ "visit",+ "more",+ "info",+ "information",+ "read",+ "read more",+ ];++ !stopwords.contains(&label.as_str())+ })+ .collect::<Vec<_>>()+ })+ .collect()+ }+}++impl Webgraph {+ fn new(config: &IndexingGraphConfig) -> Self {+ match config {+ IndexingGraphConfig::Local { path } => Self::Local(+ webgraph::WebgraphBuilder::new(path)+ .single_threaded()+ .open(),+ ),+ IndexingGraphConfig::Remote { gossip } => {+ let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);+ Self::Remote(RemoteWebgraph::new(cluster, WebgraphGranularity::Page))+ }+ }+ }+}+pub struct IndexingWorker {host_centrality_store: speedy_kv::Db<NodeID, f64>,host_centrality_rank_store: speedy_kv::Db<NodeID, u64>,@@ -108,10 +187,7 @@ impl IndexingWorker {page_centrality_rank_store: config.page_centrality_store_path.as_ref().map(|p| {speedy_kv::Db::open_or_create(Path::new(&p).join("approx_harmonic_rank")).unwrap()}),- page_webgraph: config- .page_webgraph_path- .as_ref()- .map(|path| WebgraphBuilder::new(path).single_threaded().open()),+ page_webgraph: config.page_webgraph.as_ref().map(Webgraph::new),topics: config.topics_path.as_ref()@@ -233,43 +309,6 @@ impl IndexingWorker {Ok(())}- fn backlink_labels(&self, page: &Url) -> Vec<String> {- self.page_webgraph- .as_ref()- .map(|webgraph| {- webgraph- .raw_ingoing_edges_with_labels(&Node::from(page).id())- .into_iter()- .map(|edge| edge.label)- .filter(|label| !label.is_empty())- .filter(|label| {- let label = label.to_lowercase();- let stopwords = [- "click",- "click here",- "here",- "link",- "website",- "webpage",- "page",- "site",- "url",- "web",- "visit",- "more",- "info",- "information",- "read",- "read more",- ];-- !stopwords.contains(&label.as_str())- })- .collect()- })- .unwrap_or_default()- }-fn set_page_centralities(&self, page: &mut Webpage) {let node = Node::from(page.html.url());@@ -342,6 +381,21 @@ impl IndexingWorker {}}+ pub fn set_backlink_labels(&self, pages: &mut [Webpage]) {+ if let Some(graph) = self.page_webgraph() {+ let ids = pages+ .iter()+ .map(|w| Node::from(w.html.url()).id())+ .collect::<Vec<_>>();++ let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids);++ for (page, backlink) in pages.iter_mut().zip_eq(backlinks) {+ page.backlink_labels = backlink;+ }+ }+ }+pub fn set_keyword_embeddings(&self, pages: &mut [Webpage]) {if let Some(dual_encoder) = self.dual_encoder.as_ref() {let (page_indexes, keywords): (Vec<_>, Vec<_>) = pages@@ -389,8 +443,6 @@ impl IndexingWorker {continue;}- prepared.backlink_labels = self.backlink_labels(prepared.html.url());-self.set_page_centralities(&mut prepared);self.set_dmoz_description(&mut prepared);self.set_keywords(&mut prepared);@@ -423,6 +475,7 @@ impl IndexingWorker {self.set_title_embeddings(&mut res);self.set_keyword_embeddings(&mut res);+ self.set_backlink_labels(&mut res);res}@@ -438,7 +491,7 @@ mod tests {IndexingWorker::new(IndexingLocalConfig {host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),page_centrality_store_path: None,- page_webgraph_path: None,+ page_webgraph: None,topics_path: None,safety_classifier_path: None,dual_encoder: Some(IndexingDualEncoderConfig {@@ -455,6 +508,8 @@ mod tests {host_centrality_threshold: None,minimum_clean_words: None,batch_size: 10,+ autocommit_after_num_inserts:+ crate::config::defaults::Indexing::autocommit_after_num_inserts(),})}diff --git a/crates/core/src/entrypoint/live_index.rs b/crates/core/src/entrypoint/live_index.rsindex c5393e63..d46ccaf1 100644--- a/crates/core/src/entrypoint/live_index.rs+++ b/crates/core/src/entrypoint/live_index.rs@@ -26,7 +26,6 @@ use crate::{feed::{self, index::FeedIndex},inverted_index,live_index::{Index, IndexManager},- ranking::inbound_similarity::InboundSimilarity,searcher::{InitialWebsiteResult, LocalSearcher},webgraph::WebgraphBuilder,};@@ -46,14 +45,8 @@ pub struct SearchService {impl SearchService {async fn new(config: LiveIndexConfig) -> Result<Self> {- let inbound_similarity = InboundSimilarity::open(- Path::new(&config.host_centrality_store_path).join("inbound_similarity"),- )?;-let manager = IndexManager::new(config.clone())?;- let mut local_searcher = LocalSearcher::new(manager.index());-- local_searcher.set_inbound_similarity(inbound_similarity);+ let local_searcher = LocalSearcher::new(manager.index());tokio::task::spawn(manager.run());diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 94223c86..cdcf444a 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::HashMap, path::Path};+use std::collections::HashMap;use tracing::info;use url::Url;@@ -30,10 +30,7 @@ use crate::{index::Index,inverted_index::{self, RetrievedWebpage},models::dual_encoder::DualEncoder,- ranking::{- inbound_similarity::InboundSimilarity,- models::{lambdamart::LambdaMART, linear::LinearRegression},- },+ ranking::models::{lambdamart::LambdaMART, linear::LinearRegression},searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},Result,};@@ -57,17 +54,10 @@ pub struct SearchService {impl SearchService {async fn new(config: config::SearchServerConfig) -> Result<Self> {- let centrality_store = config- .host_centrality_store_path- .map(|p| InboundSimilarity::open(Path::new(&p).join("inbound_similarity")).unwrap());let search_index = Index::open(config.index_path)?;let mut local_searcher = LocalSearcher::new(search_index);- if let Some(centrality_store) = centrality_store {- local_searcher.set_inbound_similarity(centrality_store);- }-if let Some(model_path) = config.linear_model_path {local_searcher.set_linear_model(LinearRegression::open(model_path)?);}@@ -88,7 +78,7 @@ impl SearchService {id: config.cluster_id,service: Service::Searcher {host: config.host,- shard: config.shard_id,+ shard: config.shard,},},config.gossip_addr,diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 04edd45e..a41db1f9 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -17,25 +17,20 @@use std::net::SocketAddr;use std::sync::Arc;-use itertools::Itertools;use tracing::info;-use url::Url;use utoipa::ToSchema;use crate::config;-use crate::config::WebgraphGranularity;use crate::distributed::cluster::Cluster;use crate::distributed::member::Member;use crate::distributed::member::Service;use crate::distributed::sonic::service::sonic_service;use crate::distributed::sonic::service::Message;-use crate::ranking::inbound_similarity::InboundSimilarity;-use crate::searcher::DistributedSearcher;-use crate::searcher::SearchClient;-use crate::similar_hosts::SimilarHostsFinder;use crate::webgraph::Compression;+use crate::webgraph::Edge;use crate::webgraph::FullEdge;use crate::webgraph::Node;+use crate::webgraph::NodeID;use crate::webgraph::Webgraph;use crate::webgraph::WebgraphBuilder;use crate::Result;@@ -48,136 +43,111 @@ pub struct ScoredHost {pub description: Option<String>,}-const MAX_HOSTS: usize = 20;-pub struct WebGraphService {- granularity: WebgraphGranularity,- searcher: DistributedSearcher,- similar_hosts_finder: Option<SimilarHostsFinder>,graph: Arc<Webgraph>,}sonic_service!(WebGraphService,- [SimilarHosts, Knows, IngoingLinks, OutgoingLinks]+ [+ GetNode,+ IngoingEdges,+ OutgoingEdges,+ RawIngoingEdges,+ RawOutgoingEdges,+ RawIngoingEdgesWithLabels,+ RawOutgoingEdgesWithLabels+ ]);#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct SimilarHosts {- pub hosts: Vec<String>,- pub top_n: usize,+pub struct GetNode {+ pub node: NodeID,+}++impl Message<WebGraphService> for GetNode {+ type Response = Option<Node>;++ async fn handle(self, server: &WebGraphService) -> Self::Response {+ server.graph.id2node(&self.node)+ }}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct Knows {- pub host: String,+pub struct IngoingEdges {+ pub node: Node,}-impl Message<WebGraphService> for SimilarHosts {- type Response = Vec<ScoredHost>;+impl Message<WebGraphService> for IngoingEdges {+ type Response = Vec<FullEdge>;async fn handle(self, server: &WebGraphService) -> Self::Response {- if server.similar_hosts_finder.is_none() {- return vec![];- }+ server.graph.ingoing_edges(self.node)+ }+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct OutgoingEdges {+ pub node: Node,+}++impl Message<WebGraphService> for OutgoingEdges {+ type Response = Vec<FullEdge>;- let sites = &self.hosts[..std::cmp::min(self.hosts.len(), MAX_HOSTS)];- let similar_hosts = server- .similar_hosts_finder- .as_ref()- .unwrap()- .find_similar_hosts(sites, self.top_n);-- let urls = similar_hosts- .iter()- .filter_map(|s| Url::parse(&("http://".to_string() + s.node.as_str())).ok())- .collect_vec();-- let descriptions = server.searcher.get_homepage_descriptions(&urls).await;-- let similar_hosts = similar_hosts- .into_iter()- .map(|site| {- let description = Url::parse(&("http://".to_string() + site.node.as_str()))- .ok()- .and_then(|url| descriptions.get(&url).cloned());-- ScoredHost {- host: site.node.as_str().to_string(),- score: site.score,- description,- }- })- .collect_vec();-- similar_hosts+ async fn handle(self, server: &WebGraphService) -> Self::Response {+ server.graph.outgoing_edges(self.node)}}-impl Message<WebGraphService> for Knows {- type Response = Option<Node>;+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct RawIngoingEdges {+ pub node: NodeID,+}- async fn handle(mut self, server: &WebGraphService) -> Self::Response {- if let Some(suf) = self.host.strip_prefix("http://") {- self.host = suf.to_string();- }+impl Message<WebGraphService> for RawIngoingEdges {+ type Response = Vec<Edge<()>>;- if let Some(suf) = self.host.strip_prefix("https://") {- self.host = suf.to_string();- }+ async fn handle(self, server: &WebGraphService) -> Self::Response {+ server.graph.raw_ingoing_edges(&self.node)+ }+}- let url = Url::parse(&("http://".to_string() + self.host.as_str())).ok()?;+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct RawOutgoingEdges {+ pub node: NodeID,+}- let node = Node::from(url).into_host();+impl Message<WebGraphService> for RawOutgoingEdges {+ type Response = Vec<Edge<()>>;- if server- .similar_hosts_finder- .as_ref()- .map(|finder| finder.knows_about(&node))- .unwrap_or(false)- {- Some(node)- } else {- None- }+ async fn handle(self, server: &WebGraphService) -> Self::Response {+ server.graph.raw_outgoing_edges(&self.node)}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct IngoingLinks {- pub node: Node,+pub struct RawIngoingEdgesWithLabels {+ pub node: NodeID,}-impl Message<WebGraphService> for IngoingLinks {- type Response = Vec<FullEdge>;+impl Message<WebGraphService> for RawIngoingEdgesWithLabels {+ type Response = Vec<Edge<String>>;async fn handle(self, server: &WebGraphService) -> Self::Response {- match server.granularity {- WebgraphGranularity::Host => {- let node = self.node.into_host();- server.graph.ingoing_edges(node)- }- WebgraphGranularity::Page => server.graph.ingoing_edges(self.node),- }+ server.graph.raw_ingoing_edges_with_labels(&self.node)}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]-pub struct OutgoingLinks {- pub node: Node,+pub struct RawOutgoingEdgesWithLabels {+ pub node: NodeID,}-impl Message<WebGraphService> for OutgoingLinks {- type Response = Vec<FullEdge>;+impl Message<WebGraphService> for RawOutgoingEdgesWithLabels {+ type Response = Vec<Edge<String>>;async fn handle(self, server: &WebGraphService) -> Self::Response {- match server.granularity {- WebgraphGranularity::Host => {- let node = self.node.into_host();- server.graph.outgoing_edges(node)- }- WebgraphGranularity::Page => server.graph.outgoing_edges(self.node),- }+ server.graph.raw_outgoing_edges_with_labels(&self.node)}}@@ -185,13 +155,14 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {let addr: SocketAddr = config.host;// dropping the handle leaves the cluster- let cluster = Arc::new(+ let _cluster = Arc::new(Cluster::join(Member {id: config.cluster_id,service: Service::Webgraph {host: addr,granularity: config.granularity,+ shard: config.shard,},},config.gossip_addr,@@ -199,7 +170,6 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {).await?,);- let searcher = DistributedSearcher::new(cluster);let graph = Arc::new(WebgraphBuilder::new(config.graph_path)@@ -207,24 +177,7 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {.open(),);- let similar_hosts_finder = config.inbound_similarity_path.map(|path| {- let inbound_similarity = InboundSimilarity::open(path).unwrap();- SimilarHostsFinder::new(- Arc::clone(&graph),- inbound_similarity,- config.max_similar_hosts,- )- });-- let server = WebGraphService {- graph,- searcher,- similar_hosts_finder,- granularity: config.granularity,- }- .bind(addr)- .await- .unwrap();+ let server = WebGraphService { graph }.bind(addr).await.unwrap();info!("webgraph server is ready to accept requests on {}", addr);diff --git a/crates/core/src/enum_map.rs b/crates/core/src/enum_map.rsindex 6ac1e345..28998a27 100644--- a/crates/core/src/enum_map.rs+++ b/crates/core/src/enum_map.rs@@ -106,6 +106,14 @@ where.enumerate().filter_map(|(key, value)| value.as_ref().and_then(|_| K::from_usize(key)))}++ pub fn iter(&self) -> impl Iterator<Item = (K, &V)> + '_ {+ self.inner.iter().enumerate().filter_map(|(key, value)| {+ value+ .as_ref()+ .map(|value| (K::from_usize(key).unwrap(), value))+ })+ }}impl<K, V> FromIterator<(K, V)> for EnumMap<K, V>diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 7343b43e..c4c39a97 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -346,6 +346,7 @@ mod tests {search_ctx::Ctx,searcher::SearchQuery,webpage::{Html, Webpage},+ OneOrMany,};use super::*;@@ -987,25 +988,25 @@ mod tests {assert_eq!(schema[0].itemtype,- Some(schema_org::OneOrMany::One("LiveBlogPosting".to_string()))+ Some(OneOrMany::One("LiveBlogPosting".to_string())));assert_eq!(schema[0].properties.get("coverageStartTime"),- Some(&schema_org::OneOrMany::One(schema_org::Property::String(+ Some(&OneOrMany::One(schema_org::Property::String("2022-11-14T23:45:00+00:00".to_string()))));assert_eq!(schema[1].itemtype,- Some(schema_org::OneOrMany::One("NewsArticle".to_string()))+ Some(OneOrMany::One("NewsArticle".to_string())));assert_eq!(schema[1].properties.get("publisher"),- Some(&schema_org::OneOrMany::One(schema_org::Property::Item(+ Some(&OneOrMany::One(schema_org::Property::Item(schema_org::Item {- itemtype: Some(schema_org::OneOrMany::One("Organization".to_string())),+ itemtype: Some(OneOrMany::One("Organization".to_string())),properties: hashmap! {- "name".to_string() => schema_org::OneOrMany::One(schema_org::Property::String("DR".to_string()))+ "name".to_string() => OneOrMany::One(schema_org::Property::String("DR".to_string()))}})))@@ -1260,7 +1261,7 @@ mod tests {.unwrap();assert_eq!(ranking_websites.len(), 2);- assert!(ranking_websites[0].title_embedding.is_some());- assert!(ranking_websites[1].title_embedding.is_none());+ assert!(ranking_websites[0].title_embedding().is_some());+ assert!(ranking_websites[1].title_embedding().is_none());}}diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex d8c7fee7..5d3cbb62 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -29,7 +29,7 @@ use crate::fastfield_reader::FastFieldReader;use crate::highlighted::HighlightedFragment;use crate::query::shortcircuit::ShortCircuitQuery;use crate::query::Query;-use crate::ranking::pipeline::RecallRankingWebpage;+use crate::ranking::pipeline::LocalRecallRankingWebpage;use crate::ranking::SignalComputer;use crate::schema::{fast_field, text_field, FastFieldEnum, Field, TextFieldEnum};use crate::search_ctx::Ctx;@@ -90,9 +90,13 @@ impl InvertedIndex {pointers: Vec<WebpagePointer>,mut computer: SignalComputer,fastfield_reader: &FastFieldReader,- ) -> Result<Vec<RecallRankingWebpage>> {+ ) -> Result<Vec<LocalRecallRankingWebpage>> {let mut top_websites = Vec::new();+ // the ranking webpages needs to be constructed in order+ // of ascending doc_id as they traverse the posting lists from+ // the index to calculate bm25.+let mut pointers: Vec<_> = pointers.into_iter().enumerate().collect();pointers.sort_by(|a, b| {a.1.address@@ -118,7 +122,7 @@ impl InvertedIndex {top_websites.push((orig_index,- RecallRankingWebpage::new(+ LocalRecallRankingWebpage::new(pointer,fastfield_reader.borrow_segment(&segment_reader.segment_id()),&mut computer,diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 48fed302..e7c95500 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -28,7 +28,12 @@// #![allow(clippy::module_name_repetitions)] // maybe we should remove this later// #![allow(clippy::missing_errors_doc)]-use std::path::PathBuf;+use config::GossipConfig;+use distributed::{+ cluster::Cluster,+ member::{Member, Service},+};+use std::{cmp::Reverse, path::PathBuf, sync::Arc};use thiserror::Error;pub mod entrypoint;@@ -87,6 +92,18 @@ pub mod webgraph;pub mod webpage;mod widgets;+static TOKIO_RUNTIME: once_cell::sync::Lazy<tokio::runtime::Runtime> =+ once_cell::sync::Lazy::new(|| {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ });++pub fn block_on<F: std::future::Future>(f: F) -> F::Output {+ TOKIO_RUNTIME.block_on(f)+}+#[derive(Error, Debug)]pub enum Error {#[error("Failed to parse WARC file: {0}")]@@ -149,6 +166,52 @@ pub fn gen_temp_path() -> PathBuf {}}+/// Starts a gossip cluster in the background and returns a handle to it.+/// This is useful for blocking contexts where there is no runtime to spawn the cluster on.+pub fn start_gossip_cluster_thread(config: GossipConfig, service: Option<Service>) -> Arc<Cluster> {+ let (tx, rx) = std::sync::mpsc::channel();++ std::thread::spawn(move || {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();++ rt.block_on(async {+ let cluster = match service {+ Some(service) => Cluster::join(+ Member {+ id: config.cluster_id,+ service,+ },+ config.addr,+ config.seed_nodes.unwrap_or_default(),+ )+ .await+ .unwrap(),+ None => Cluster::join_as_spectator(+ config.cluster_id,+ config.addr,+ config.seed_nodes.unwrap_or_default(),+ )+ .await+ .unwrap(),+ };++ let cluster = Arc::new(cluster);+ tx.send(cluster.clone()).unwrap();++ // need to keep tokio runtime alive+ // otherwise the spawned task in Cluster::join will be dropped+ loop {+ tokio::time::sleep(std::time::Duration::from_secs(60)).await;+ }+ });+ });++ rx.recv().unwrap()+}+#[cfg(test)]fn rand_words(num_words: usize) -> String {use rand::{distributions::Alphanumeric, Rng};@@ -251,3 +314,92 @@ macro_rules! enum_dispatch_from_discriminant {}pub(crate) use enum_dispatch_from_discriminant;++#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Debug,+ Clone,+ Hash,+)]+pub enum OneOrMany<T> {+ One(T),+ Many(Vec<T>),+}++impl<T> OneOrMany<T> {+ pub fn one(self) -> Option<T> {+ match self {+ OneOrMany::One(one) => Some(one),+ OneOrMany::Many(many) => many.into_iter().next(),+ }+ }++ pub fn many(self) -> Vec<T> {+ match self {+ OneOrMany::One(one) => vec![one],+ OneOrMany::Many(many) => many,+ }+ }+}++pub trait TopKOrderable: Ord {+ type SortKey: Ord + Copy;++ fn sort_key(&self) -> Self::SortKey;+}++impl TopKOrderable for (SortableFloat, webgraph::NodeID) {+ type SortKey = SortableFloat;++ fn sort_key(&self) -> Self::SortKey {+ self.0+ }+}++impl<T> TopKOrderable for Reverse<T>+where+ T: TopKOrderable,+{+ type SortKey = Reverse<T::SortKey>;++ fn sort_key(&self) -> Self::SortKey {+ Reverse(self.0.sort_key())+ }+}++/// Source (and explanation): [https://quickwit.io/blog/top-k-complexity]+pub fn sorted_k<T>(mut hits: impl Iterator<Item = T>, k: usize) -> Vec<T>+where+ T: TopKOrderable,+{+ if k == 0 {+ return Vec::new();+ }++ let mut top_k = Vec::with_capacity(2 * k);+ top_k.extend((&mut hits).take(k));++ let mut threshold = None;+ for hit in hits {+ if let Some(threshold) = threshold {+ if hit.sort_key() <= threshold {+ continue;+ }+ }+ top_k.push(hit);+ if top_k.len() == 2 * k {+ // The standard library does all of the heavy lifting here.+ let (_, median_el, _) = top_k.select_nth_unstable(k - 1);+ threshold = Some(median_el.sort_key());+ top_k.truncate(k);+ }+ }+ top_k.sort_unstable();+ top_k.truncate(k);+ top_k+}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex 1e780452..0a00e15c 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -306,7 +306,6 @@ fn main() -> Result<()> {output_path,} => {entrypoint::Centrality::build_harmonic(&webgraph_path, &output_path);- entrypoint::Centrality::build_similarity(&webgraph_path, &output_path);}CentralityMode::Page {webgraph_path,diff --git a/crates/core/src/models/bert.rs b/crates/core/src/models/bert.rsindex ab952904..cff96a6a 100644--- a/crates/core/src/models/bert.rs+++ b/crates/core/src/models/bert.rs@@ -5,7 +5,7 @@//!//! https://github.com/huggingface/candle/blob/main/candle-examples/examples/bert/main.rs-use candle_core::{Device, Result, Tensor};+use candle_core::{Result, Tensor};use candle_nn::{embedding, Embedding, Module, VarBuilder};use candle_transformers::models::with_tracing::{layer_norm, linear, LayerNorm, Linear};use serde::Deserialize;@@ -475,7 +475,6 @@ pub struct BertModel {embeddings: BertEmbeddings,encoder: BertEncoder,pooler: Option<BertPooler>,- pub device: Device,span: tracing::Span,}@@ -516,7 +515,6 @@ impl BertModel {embeddings,encoder,pooler,- device: vb.device().clone(),span: tracing::span!(tracing::Level::TRACE, "model"),})}diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 860b11fa..ec6e384c 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -174,6 +174,8 @@ impl Query {}pub fn signal_coefficients(&self) -> Option<SignalCoefficient> {+ // TODO: we should make sure taht `SearchQuery::signal_coefficients`+ // return the same coefficients as the ones we calculate hereif self.optics.is_empty() {return None;}@@ -183,7 +185,7 @@ impl Query {.iter().fold(SignalCoefficient::default(), |mut acc, optic| {let coeffs = SignalCoefficient::from_optic(optic);- acc.merge_into(coeffs);+ acc.merge_add(coeffs);acc}),)diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex d52266fc..68ca8a0b 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -268,10 +268,14 @@ mod tests {};use crate::{+ bangs::Bangs,gen_temp_path,index::Index,- ranking::inbound_similarity::InboundSimilarity,- searcher::{LocalSearcher, SearchQuery},+ searcher::{+ api::{ApiSearcher, Config},+ live::LiveSearcher,+ LocalSearchClient, LocalSearcher, SearchQuery,+ },webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},};@@ -612,8 +616,8 @@ mod tests {assert_eq!(res[0].url, "https://www.a.com/this/is/a/pattern");}- #[test]- fn liked_hosts() {+ #[tokio::test]+ async fn liked_hosts() {let mut index = Index::temporary().expect("Unable to open index");let mut writer = WebgraphWriter::new(@@ -729,9 +733,12 @@ mod tests {.expect("failed to insert webpage");index.commit().expect("failed to commit index");- let mut searcher = LocalSearcher::from(index);-- searcher.set_inbound_similarity(InboundSimilarity::build(&graph));+ let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(+ LocalSearchClient::from(LocalSearcher::from(index)),+ Bangs::empty(),+ Config::default(),+ )+ .with_webgraph(graph);let res = searcher.search(&SearchQuery {@@ -750,7 +757,9 @@ mod tests {}),..Default::default()})+ .await.unwrap()+ .into_websites_result().webpages;assert_eq!(res.len(), 3);diff --git a/crates/core/src/query/pattern_query/scorer.rs b/crates/core/src/query/pattern_query/scorer.rsindex 8fe0b55f..8aa13e43 100644--- a/crates/core/src/query/pattern_query/scorer.rs+++ b/crates/core/src/query/pattern_query/scorer.rs@@ -17,8 +17,7 @@use std::sync::Arc;use tantivy::{- fieldnorm::FieldNormReader, postings::SegmentPostings, query::Scorer, DocId, DocSet, Postings,- Score, TERMINATED,+ postings::SegmentPostings, query::Scorer, DocId, DocSet, Postings, Score, TERMINATED,};use crate::{@@ -179,7 +178,6 @@ impl Scorer for EmptyFieldScorer {pub struct FastSiteDomainPatternScorer {pub posting: SegmentPostings,- pub fieldnorm_reader: FieldNormReader,}impl Scorer for FastSiteDomainPatternScorer {diff --git a/crates/core/src/query/pattern_query/weight.rs b/crates/core/src/query/pattern_query/weight.rsindex 50bbb2f7..7b94bb7a 100644--- a/crates/core/src/query/pattern_query/weight.rs+++ b/crates/core/src/query/pattern_query/weight.rs@@ -16,7 +16,6 @@use optics::PatternPart;use tantivy::{- fieldnorm::FieldNormReader,query::{EmptyScorer, Explanation, Scorer},schema::IndexRecordOption,DocSet, TantivyError,@@ -42,19 +41,10 @@ pub struct FastSiteDomainPatternWeight {}impl FastSiteDomainPatternWeight {- fn fieldnorm_reader(- &self,- reader: &tantivy::SegmentReader,- ) -> tantivy::Result<FieldNormReader> {- Ok(FieldNormReader::constant(reader.max_doc(), 1))- }-fn pattern_scorer(&self,reader: &tantivy::SegmentReader,) -> tantivy::Result<Option<FastSiteDomainPatternScorer>> {- let fieldnorm_reader = self.fieldnorm_reader(reader)?;-let field_no_tokenizer = match Field::get(self.field.field_id() as usize) {Some(Field::Text(TextFieldEnum::UrlForSiteOperator(_))) => {Field::Text(text_field::SiteNoTokenizer.into())@@ -79,10 +69,7 @@ impl FastSiteDomainPatternWeight {.inverted_index(tv_field)?.read_postings(&self.term, opt)?{- Some(posting) => Ok(Some(FastSiteDomainPatternScorer {- posting,- fieldnorm_reader,- })),+ Some(posting) => Ok(Some(FastSiteDomainPatternScorer { posting })),None => Ok(None),}}diff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rsindex 900c8986..3a8468b8 100644--- a/crates/core/src/ranking/bitvec_similarity.rs+++ b/crates/core/src/ranking/bitvec_similarity.rs@@ -15,6 +15,9 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use itertools::Itertools;+use std::future::Future;++use crate::webgraph::NodeID;#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]struct VeryJankyBloomFilter {@@ -111,7 +114,29 @@ pub struct BitVec {sqrt_len: f64,}+impl Default for BitVec {+ fn default() -> Self {+ Self::new(vec![])+ }+}++pub trait Graph {+ fn batch_ingoing(&self, nodes: &[NodeID]) -> impl Future<Output = Vec<Vec<NodeID>>>;+}+impl BitVec {+ pub async fn batch_new_for<G>(nodes: &[NodeID], graph: &G) -> Vec<Self>+ where+ G: Graph,+ {+ let ingoing = graph.batch_ingoing(nodes).await;++ ingoing+ .into_iter()+ .map(|nodes| Self::new(nodes.into_iter().map(|n| n.as_u64()).collect()))+ .collect()+ }+pub fn new(mut ranks: Vec<u64>) -> Self {ranks.sort();ranks.dedup();diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 5dd2b374..ddbf63b1 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -14,59 +14,26 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{- fs::File,- io::{BufReader, BufWriter},- path::Path,- sync::Arc,-};--use dashmap::DashMap;use fnv::FnvHashMap as HashMap;-use fnv::FnvHashSet as HashSet;-use indicatif::ParallelProgressIterator;-use rayon::prelude::*;+use itertools::Itertools;-use crate::{- webgraph::{NodeID, Webgraph},- Result,-};+use crate::webgraph::NodeID;use super::bitvec_similarity;-const PRECALCULATE_TOP_N: usize = 1_000;-const TOP_CANDIDATES_PER_PRECALCULATION: usize = 1_000;--#[derive(Clone)]-pub struct Scorer {- liked: Vec<NodeScorer>,- disliked: Vec<NodeScorer>,- vectors: Arc<VecMap>,- cache: HashMap<NodeID, f64>,- normalized: bool,-}-#[derive(Clone)]struct NodeScorer {node: NodeID,inbound: bitvec_similarity::BitVec,- precalculated: Arc<PreCalculatedSimilarities>,self_score: f64,- default_if_precalculated: bool,}impl NodeScorer {- fn new(- node: NodeID,- inbound: bitvec_similarity::BitVec,- precalculated: Arc<PreCalculatedSimilarities>,- ) -> Self {+ fn new(node: NodeID, inbound: bitvec_similarity::BitVec) -> Self {Self {node,inbound,- precalculated,self_score: 1.0,- default_if_precalculated: false,}}@@ -74,264 +41,99 @@ impl NodeScorer {self.self_score = self_score;}- fn set_default_if_precalculated(&mut self, default_if_precalculated: bool) {- self.default_if_precalculated = default_if_precalculated;- }-fn sim(&self, other: &NodeID, other_inbound: &bitvec_similarity::BitVec) -> f64 {if self.node == *other {self.self_score- } else if self.default_if_precalculated {- self.precalculated- .get_or_default_if_precalculated(&self.node, other)- .or_else(|| {- self.precalculated- .get_or_default_if_precalculated(other, &self.node)- })- .unwrap_or_else(|| self.inbound.sim(other_inbound))- } else {- self.precalculated- .get(&self.node, other)- .or_else(|| self.precalculated.get(other, &self.node))- .unwrap_or_else(|| self.inbound.sim(other_inbound))- }- }-}--impl Scorer {- fn calculate_score(&mut self, node: &NodeID) -> f64 {- let s = match self.vectors.get(node) {- Some(vec) => {- (self.disliked.len() as f64)- + (self- .liked- .iter()- .map(|liked| liked.sim(node, vec))- .sum::<f64>()- - self- .disliked- .iter()- .map(|disliked| disliked.sim(node, vec))- .sum::<f64>())- }- None => 0.0,- };-- if self.normalized {- s / self.liked.len().max(1) as f64} else {- s- }- .max(0.0)- }- pub fn score(&mut self, node: &NodeID) -> f64 {- if let Some(cached) = self.cache.get(node) {- return *cached;- }-- let score = self.calculate_score(node);- self.cache.insert(*node, score);- score- }-- pub fn set_self_score(&mut self, self_score: f64) {- for scorer in self.liked.iter_mut() {- scorer.set_self_score(self_score);- }-- for scorer in self.disliked.iter_mut() {- scorer.set_self_score(self_score);- }- }-- /// Speedups calculation when we encounter a node for which we have- /// precalculated similarities. This means, the chosen node is a node- /// with many inbound links. If we encounter another node for which- /// we have no precalcalculated similarities with the chosen node, they- /// are most likely not very similar, so we can just assume a similarity- /// of 0.0.- ///- /// Needless to say, this is less accurate than calculating the similarity.- pub fn set_default_if_precalculated(&mut self, default_if_precalculated: bool) {- for scorer in self.liked.iter_mut() {- scorer.set_default_if_precalculated(default_if_precalculated);- }-- for scorer in self.disliked.iter_mut() {- scorer.set_default_if_precalculated(default_if_precalculated);- }- }-}--#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]-struct PreCalculatedSimilarities {- map: HashMap<NodeID, HashMap<NodeID, f64>>,-}--impl PreCalculatedSimilarities {- fn get_or_default_if_precalculated(&self, node: &NodeID, other: &NodeID) -> Option<f64> {- Some(self.map.get(node)?.get(other).copied().unwrap_or_default())- }-- fn get(&self, node: &NodeID, other: &NodeID) -> Option<f64> {- self.map.get(node)?.get(other).copied()- }-}--#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]-struct VecMap {- map: HashMap<NodeID, bitvec_similarity::BitVec>,-}--impl VecMap {- fn build(graph: &Webgraph) -> Self {- let mut map = HashMap::default();-- let adjacency: DashMap<NodeID, HashSet<NodeID>> = DashMap::new();-- graph.par_edges().for_each(|edge| {- adjacency.entry(edge.to).or_default().insert(edge.from);- });-- for (node_id, inbound) in adjacency {- map.insert(- node_id,- bitvec_similarity::BitVec::new(inbound.into_iter().map(|n| n.as_u64()).collect()),- );+ self.inbound.sim(other_inbound)}-- Self { map }- }-- fn get(&self, id: &NodeID) -> Option<&bitvec_similarity::BitVec> {- self.map.get(id)- }-- fn contains(&self, id: &NodeID) -> bool {- self.map.contains_key(id)}}-#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]-pub struct InboundSimilarity {- vectors: Arc<VecMap>,- precalculated: Arc<PreCalculatedSimilarities>,+#[derive(Clone)]+pub struct Scorer {+ liked: Vec<NodeScorer>,+ disliked: Vec<NodeScorer>,+ cache: HashMap<NodeID, f64>,+ normalized: bool,}-impl InboundSimilarity {- pub fn build(graph: &Webgraph) -> Self {- let vectors = VecMap::build(graph);-- tracing::info!("precalculating similarities...");-- let mut nodes = vectors- .map- .iter()- .map(|(node, ranks)| (*node, ranks.len()))- .collect::<Vec<_>>();- nodes.sort_by(|(_, a), (_, b)| b.cmp(a));- nodes.truncate(PRECALCULATE_TOP_N);-- let precalculated: HashMap<NodeID, HashMap<NodeID, f64>> = nodes- .into_par_iter()- .progress()- .map(|(node, _)| {- let node_vec = vectors.map.get(&node).unwrap();-- let candidates = graph- .raw_ingoing_edges(&node)- .into_iter()- .map(|edge| edge.from)- .flat_map(|c| graph.raw_outgoing_edges(&c).into_iter().map(|edge| edge.to))- .filter(|c| *c != node)- .collect::<HashSet<_>>();-- let mut candidates = candidates- .into_iter()- .filter_map(|c| vectors.map.get(&c).map(|vec| (c, vec)))- .map(|(c, vec)| (c, node_vec.sim(vec)))- .collect::<Vec<_>>();- candidates.sort_by(|(_, a), (_, b)| b.total_cmp(a));- candidates.truncate(TOP_CANDIDATES_PER_PRECALCULATION);-- let mut map = HashMap::default();-- for (candidate, _) in candidates {- if let Some(candidate_vec) = vectors.map.get(&candidate) {- let score = node_vec.sim(candidate_vec);- map.insert(candidate, score);- }- }-- (node, map)- })- .collect();-- let precalculated = PreCalculatedSimilarities { map: precalculated };-+impl Scorer {+ pub fn empty() -> Self {Self {- vectors: Arc::new(vectors),- precalculated: Arc::new(precalculated),+ liked: Vec::new(),+ disliked: Vec::new(),+ cache: HashMap::default(),+ normalized: false,}}- pub fn scorer(- &self,+ pub async fn new<G: bitvec_similarity::Graph>(+ graph: &G,liked_hosts: &[NodeID],disliked_hosts: &[NodeID],normalized: bool,) -> Scorer {+ let liked = bitvec_similarity::BitVec::batch_new_for(liked_hosts, graph).await;+ let disliked = bitvec_similarity::BitVec::batch_new_for(disliked_hosts, graph).await;+let liked: Vec<_> = liked_hosts.iter()- .filter_map(|id| self.vectors.get(id).cloned().map(|vec| (id, vec)))- .map(|(node, inbound)| NodeScorer::new(*node, inbound, self.precalculated.clone()))+ .zip_eq(liked)+ .map(|(node, inbound)| NodeScorer::new(*node, inbound)).collect();let disliked: Vec<_> = disliked_hosts.iter()- .filter_map(|id| self.vectors.get(id).cloned().map(|vec| (id, vec)))- .map(|(node, inbound)| NodeScorer::new(*node, inbound, self.precalculated.clone()))+ .zip_eq(disliked)+ .map(|(node, inbound)| NodeScorer::new(*node, inbound)).collect();Scorer {liked,disliked,- vectors: self.vectors.clone(),cache: HashMap::default(),normalized,}}+ fn calculate_score(&self, node: &NodeID, inbound: &bitvec_similarity::BitVec) -> f64 {+ let s = (self.disliked.len() as f64)+ + (self+ .liked+ .iter()+ .map(|liked| liked.sim(node, inbound))+ .sum::<f64>()+ - self+ .disliked+ .iter()+ .map(|disliked| disliked.sim(node, inbound))+ .sum::<f64>());- pub fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {- let mut file = BufWriter::new(- File::options()- .create(true)- .truncate(true)- .write(true)- .open(path)?,- );-- bincode::encode_into_std_write(self, &mut file, bincode::config::standard())?;-- Ok(())+ if self.normalized {+ s / self.liked.len().max(1) as f64+ } else {+ s+ }+ .max(0.0)}+ pub fn score(&mut self, node: &NodeID, inbound: &bitvec_similarity::BitVec) -> f64 {+ if let Some(cached) = self.cache.get(node) {+ return *cached;+ }- pub fn get(&self, node: &NodeID) -> Option<&bitvec_similarity::BitVec> {- self.vectors.get(node)+ let score = self.calculate_score(node, inbound);+ self.cache.insert(*node, score);+ score}- pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {- let file = File::open(path)?;- let mut reader = BufReader::new(file);-- Ok(bincode::decode_from_std_read(- &mut reader,- bincode::config::standard(),- )?)- }+ pub fn set_self_score(&mut self, self_score: f64) {+ for scorer in self.liked.iter_mut() {+ scorer.set_self_score(self_score);+ }- pub fn knows_about(&self, node_id: NodeID) -> bool {- self.vectors.contains(&node_id)+ for scorer in self.disliked.iter_mut() {+ scorer.set_self_score(self_score);+ }}}@@ -340,18 +142,31 @@ mod tests {use optics::HostRankings;use crate::{+ bangs::Bangs,gen_temp_path,index::Index,rand_words,- searcher::{LocalSearcher, SearchQuery},- webgraph::{Node, WebgraphWriter},+ searcher::{+ api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,+ },+ webgraph::{Node, Webgraph, WebgraphWriter},webpage::{Html, Webpage},};use super::*;- #[test]- fn it_favors_liked_hosts() {+ fn inbound(graph: &Webgraph, node: &NodeID) -> bitvec_similarity::BitVec {+ bitvec_similarity::BitVec::new(+ graph+ .raw_ingoing_edges(node)+ .into_iter()+ .map(|e| e.from.as_u64())+ .collect(),+ )+ }++ #[tokio::test]+ async fn it_favors_liked_hosts() {let mut wrt = WebgraphWriter::new(gen_temp_path(),crate::executor::Executor::single_thread(),@@ -370,17 +185,15 @@ mod tests {let graph = wrt.finalize();- let inbound = InboundSimilarity::build(&graph);-- let mut scorer = inbound.scorer(&[Node::from("b.com").id()], &[], false);+ let mut scorer = Scorer::new(&graph, &[Node::from("b.com").id()], &[], false).await;let e = Node::from("e.com").id();let d = Node::from("d.com").id();- assert!(scorer.score(&e) > scorer.score(&d));+ assert!(scorer.score(&e, &inbound(&graph, &e)) > scorer.score(&d, &inbound(&graph, &d)));}- #[test]- fn it_ranks_search_results() {+ #[tokio::test]+ async fn it_ranks_search_results() {let mut wrt = WebgraphWriter::new(crate::gen_temp_path(),crate::executor::Executor::single_thread(),@@ -394,8 +207,6 @@ mod tests {let graph = wrt.finalize();- let inbound = InboundSimilarity::build(&graph);-let mut index = Index::temporary().expect("Unable to open index");index@@ -450,8 +261,12 @@ mod tests {index.commit().unwrap();- let mut searcher = LocalSearcher::new(index);- searcher.set_inbound_similarity(inbound);+ let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(+ LocalSearchClient::from(LocalSearcher::new(index)),+ Bangs::empty(),+ crate::searcher::api::Config::default(),+ )+ .with_webgraph(graph);let res = searcher.search(&SearchQuery {@@ -463,7 +278,9 @@ mod tests {}),..Default::default()})+ .await.unwrap()+ .into_websites_result().webpages;assert_eq!(res.len(), 2);diff --git a/crates/core/src/ranking/initial.rs b/crates/core/src/ranking/initial.rsindex 8053df9e..9f45a44e 100644--- a/crates/core/src/ranking/initial.rs+++ b/crates/core/src/ranking/initial.rs@@ -82,7 +82,7 @@ impl ScoreSegmentTweaker<Score> for InitialSegmentScoreTweaker {.computer.compute_signals(doc).flatten()- .map(|computed| computed.score.coefficient * computed.score.value)+ .map(|computed| self.computer.coefficient(&computed.signal) * computed.score).sum();if let Some(boost) = self.computer.boosts(doc) {diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 9ee46fe2..506b42fc 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -21,7 +21,6 @@ pub mod initial;pub mod models;pub mod optics;pub mod pipeline;-pub mod query_centrality;pub mod signal;use initial::InitialScoreTweaker;@@ -117,10 +116,6 @@ impl Ranker {collector.main_collector(score_tweaker)}-- pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {- self.computer.set_query_centrality(query_centrality);- }}#[cfg(test)]@@ -777,7 +772,7 @@ mod tests {IndexingWorker::new(IndexingLocalConfig {host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),page_centrality_store_path: None,- page_webgraph_path: None,+ page_webgraph: None,topics_path: None,safety_classifier_path: None,dual_encoder: Some(IndexingDualEncoderConfig {@@ -794,6 +789,8 @@ mod tests {host_centrality_threshold: None,minimum_clean_words: None,batch_size: 10,+ autocommit_after_num_inserts:+ crate::config::defaults::Indexing::autocommit_after_num_inserts(),})}diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 23eba9bb..c4b1f08f 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -22,17 +22,19 @@ mod tests {};use crate::{+ bangs::Bangs,gen_temp_path,index::Index,- ranking::inbound_similarity::InboundSimilarity,- searcher::{LocalSearcher, SearchQuery},+ searcher::{+ api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,+ },webgraph::{Node, WebgraphWriter},webpage::{Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";- #[test]- fn host_rankings() {+ #[tokio::test]+ async fn host_rankings() {let mut index = Index::temporary().expect("Unable to open index");let mut wrt = WebgraphWriter::new(@@ -160,9 +162,12 @@ mod tests {}).expect("failed to insert webpage");index.commit().expect("failed to commit index");- let mut searcher = LocalSearcher::new(index);-- searcher.set_inbound_similarity(InboundSimilarity::build(&graph));+ let searcher: ApiSearcher<_, LiveSearcher, _> = ApiSearcher::new(+ LocalSearchClient::from(LocalSearcher::new(index)),+ Bangs::empty(),+ crate::searcher::api::Config::default(),+ )+ .with_webgraph(graph);let result = searcher.search(&SearchQuery {@@ -181,7 +186,9 @@ mod tests {}),..Default::default()})- .expect("Search failed");+ .await+ .expect("Search failed")+ .into_websites_result();assert_eq!(result.webpages.len(), 3);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -206,7 +213,9 @@ mod tests {return_ranking_signals: true,..Default::default()})- .expect("Search failed");+ .await+ .expect("Search failed")+ .into_websites_result();assert_eq!(result.webpages.len(), 2);assert_eq!(result.webpages[0].url, "https://www.third.com/");@@ -230,7 +239,9 @@ mod tests {return_ranking_signals: true,..Default::default()})- .expect("Search failed");+ .await+ .expect("Search failed")+ .into_websites_result();assert_eq!(result.webpages.len(), 3);assert_eq!(result.webpages[0].url, "https://www.first.com/");diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex c3d11e14..b0807c96 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -19,24 +19,25 @@ use std::sync::Arc;use crate::{collector::{self, BucketCollector},config::CollectorConfig,- models::dual_encoder::DualEncoder,+ enum_map::EnumMap,searcher::SearchQuery,};use super::{models::lambdamart::{self, LambdaMART},- SignalScore,+ SignalCoefficient, SignalEnum, SignalScore,};mod scorers;mod stages;pub use scorers::{ReRanker, Recall, Scorer};-pub use stages::{PrecisionRankingWebpage, RecallRankingWebpage};+pub use stages::{LocalRecallRankingWebpage, PrecisionRankingWebpage, RecallRankingWebpage};pub trait RankableWebpage: collector::Doc + Send + Sync {fn set_score(&mut self, score: f64);fn boost(&self) -> Option<f64>;+ fn signals(&self) -> &EnumMap<SignalEnum, f64>;fn boost_score(&mut self) {if let Some(boost) = self.boost() {@@ -58,6 +59,8 @@ struct RankingStage<T> {scorer: Box<dyn Scorer<T>>,stage_top_n: usize,derank_similar: bool,+ model: Option<Arc<LambdaMART>>,+ coefficients: SignalCoefficient,}impl<T: RankableWebpage> RankingStage<T> {@@ -80,6 +83,7 @@ impl<T: RankableWebpage> RankingStage<T> {BucketCollector::new(self.stage_top_n.max(top_n) + offset, collector_config);for mut website in websites {+ website.set_score(self.calculate_score(website.signals()));website.boost_score();collector.insert(website);}@@ -91,8 +95,30 @@ impl<T: RankableWebpage> RankingStage<T> {.collect()}+ fn calculate_score(&self, signals: &EnumMap<SignalEnum, f64>) -> f64 {+ match self.model.as_ref() {+ Some(model) => {+ let coeff = self.coefficients.get(&super::signal::LambdaMart.into());+ if coeff == 0.0 {+ signals+ .iter()+ .map(|(signal, score)| self.coefficients.get(&signal) * score)+ .sum()+ } else {+ coeff * model.predict(signals)+ }+ }+ None => signals+ .iter()+ .map(|(signal, score)| self.coefficients.get(&signal) * score)+ .sum(),+ }+ }+fn set_query_info(&mut self, query: &SearchQuery) {self.scorer.set_query_info(query);++ self.coefficients = query.signal_coefficients();}}@@ -103,45 +129,6 @@ pub struct RankingPipeline<T> {collector_config: CollectorConfig,}-impl RankingPipeline<crate::searcher::api::ScoredWebpagePointer> {- fn create_recall_stage(- lambdamart: Option<Arc<LambdaMART>>,- dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,- stage_top_n: usize,- ) -> Self {- let last_stage = RankingStage {- scorer: Box::new(Recall::<crate::searcher::api::ScoredWebpagePointer>::new(- lambdamart,- dual_encoder,- )),- stage_top_n,- derank_similar: true,- };-- Self {- stage: last_stage,- page: 0,- top_n: 0,- collector_config,- }- }-- pub fn recall_stage(- query: &mut SearchQuery,- lambdamart: Option<Arc<LambdaMART>>,- dual_encoder: Option<Arc<DualEncoder>>,- collector_config: CollectorConfig,- top_n_considered: usize,- ) -> Self {- let mut pipeline =- Self::create_recall_stage(lambdamart, dual_encoder, collector_config, top_n_considered);- pipeline.set_query_info(query);-- pipeline- }-}-impl<T: RankableWebpage> RankingPipeline<T> {fn set_query_info(&mut self, query: &mut SearchQuery) {self.stage.set_query_info(query);@@ -184,7 +171,6 @@ mod tests {use crate::{collector::Hashes,- enum_map::EnumMap,inverted_index::{DocAddress, WebpagePointer},prehashed::Prehashed,ranking::{self, initial::Score},@@ -192,45 +178,34 @@ mod tests {use super::*;- fn sample_websites(n: usize) -> Vec<RecallRankingWebpage> {+ fn sample_websites(n: usize) -> Vec<LocalRecallRankingWebpage> {(0..n)- .map(|i| -> RecallRankingWebpage {- let mut signals = EnumMap::new();- signals.insert(- ranking::signal::HostCentrality.into(),- SignalScore {- coefficient: 1.0,- value: 1.0 / i as f64,+ .map(|i| -> LocalRecallRankingWebpage {+ let pointer = WebpagePointer {+ score: Score { total: 0.0 },+ hashes: Hashes {+ site: Prehashed(0),+ title: Prehashed(0),+ url: Prehashed(0),+ url_without_tld: Prehashed(0),+ simhash: 0,},- );- RecallRankingWebpage {- pointer: WebpagePointer {- score: Score { total: 0.0 },- hashes: Hashes {- site: Prehashed(0),- title: Prehashed(0),- url: Prehashed(0),- url_without_tld: Prehashed(0),- simhash: 0,- },- address: DocAddress {- segment: 0,- doc_id: i as u32,- },+ address: DocAddress {+ segment: 0,+ doc_id: i as u32,},- signals,- optic_boost: None,- title_embedding: None,- keyword_embedding: None,- score: 1.0 / i as f64,- }+ };++ let mut signals = EnumMap::new();+ signals.insert(ranking::signal::HostCentrality.into(), 1.0 / i as f64);+ LocalRecallRankingWebpage::new_testing(pointer, signals, 1.0 / i as f64)}).collect()}#[test]fn simple() {- let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(&mut SearchQuery {..Default::default()},@@ -245,13 +220,13 @@ mod tests {let res: Vec<_> = pipeline.apply(sample).into_iter()- .map(|w| w.pointer.address)+ .map(|w| w.pointer().address).collect();let expected: Vec<_> = sample_websites(100).into_iter().take(20)- .map(|w| w.pointer.address)+ .map(|w| w.pointer().address).collect();assert_eq!(res, expected);@@ -260,7 +235,7 @@ mod tests {#[test]fn top_n() {let num_results = 100;- let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(&mut SearchQuery {num_results,..Default::default()@@ -277,13 +252,13 @@ mod tests {.clone().into_iter().take(num_results)- .map(|w| w.pointer.address)+ .map(|w| w.pointer().address).collect();let res = pipeline.apply(sample).into_iter()- .map(|w| w.pointer.address)+ .map(|w| w.pointer().address).collect_vec();assert_eq!(res.len(), num_results);@@ -293,7 +268,7 @@ mod tests {#[test]fn offsets() {let num_results = 20;- let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(&mut SearchQuery {page: 0,num_results,@@ -308,7 +283,7 @@ mod tests {let sample: Vec<_> = sample_websites(pipeline.collector_top_n());let mut prev: Vec<_> = pipeline.apply(sample);for p in 1..1_000 {- let pipeline = RankingPipeline::<RecallRankingWebpage>::recall_stage(+ let pipeline = RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(&mut SearchQuery {page: p,..Default::default()@@ -331,7 +306,7 @@ mod tests {assert!(!prev.iter().zip_eq(res.iter())- .any(|(p, r)| p.pointer.address.doc_id == r.pointer.address.doc_id));+ .any(|(p, r)| p.pointer().address.doc_id == r.pointer().address.doc_id));prev = res;}diff --git a/crates/core/src/ranking/pipeline/scorers/embedding.rs b/crates/core/src/ranking/pipeline/scorers/embedding.rsindex fc819c56..f68527e5 100644--- a/crates/core/src/ranking/pipeline/scorers/embedding.rs+++ b/crates/core/src/ranking/pipeline/scorers/embedding.rs@@ -22,8 +22,8 @@ use crate::{models::dual_encoder::DualEncoder,ranking::{self,- pipeline::{stages::StoredEmbeddings, RankableWebpage, RecallRankingWebpage},- SignalCoefficient, SignalEnum, SignalScore,+ pipeline::{stages::StoredEmbeddings, LocalRecallRankingWebpage, RankableWebpage},+ SignalEnum,},searcher::{api::ScoredWebpagePointer, SearchQuery},Result,@@ -72,7 +72,6 @@ impl Embedding {pub struct EmbeddingScorer<W, E: EmbeddingSignal<W>> {dual_encoder: Option<Arc<DualEncoder>>,- signal_coefficients: Option<SignalCoefficient>,query: Option<String>,_marker: std::marker::PhantomData<(E, W)>,}@@ -81,7 +80,6 @@ impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {pub fn new(dual_encoder: Option<Arc<DualEncoder>>) -> Self {Self {dual_encoder,- signal_coefficients: None,query: None,_marker: std::marker::PhantomData,}@@ -100,26 +98,17 @@ impl<W, E: EmbeddingSignal<W>> EmbeddingScorer<W, E> {})})}-- fn query_emb_and_coefficient(&self, coeff_signal: SignalEnum) -> Option<(Embedding, f64)> {- self.query_emb().and_then(|query_emb| {- self.signal_coefficients- .as_ref()- .map(|c| c.get(&coeff_signal))- .map(|coefficient| (query_emb, coefficient))- })- }}-impl RecallRankingWebpage {+impl LocalRecallRankingWebpage {fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {- self.title_embedding+ self.title_embedding().as_ref().and_then(|s| Embedding::new(s, hidden_size).ok())}fn keyword_emb(&self, hidden_size: usize) -> Option<Embedding> {- self.keyword_embedding+ self.keyword_embedding().as_ref().and_then(|s| Embedding::new(s, hidden_size).ok())}@@ -128,14 +117,14 @@ impl RecallRankingWebpage {impl ScoredWebpagePointer {fn title_emb(&self, hidden_size: usize) -> Option<Embedding> {self.as_ranking()- .title_embedding+ .title_embedding().as_ref().and_then(|s| Embedding::new(s, hidden_size).ok())}fn keyword_emb(&self, hidden_size: usize) -> Option<Embedding> {self.as_ranking()- .keyword_embedding+ .keyword_embedding().as_ref().and_then(|s| Embedding::new(s, hidden_size).ok())}@@ -147,12 +136,12 @@ impl<W: RankableWebpage, E: EmbeddingSignal<W>> Scorer<W> for EmbeddingScorer<W,return;}- if let Some((query_emb, coefficient)) = self.query_emb_and_coefficient(E::signal()) {+ if let Some(query_emb) = self.query_emb() {let hidden_size = query_emb.size();for webpage in webpages.iter_mut() {if let Some(emb) = E::embedding(webpage, hidden_size) {let sim = query_emb.dot(&emb).unwrap_or_default();- E::insert_signal(webpage, sim, coefficient);+ E::insert_signal(webpage, sim);}}}@@ -160,7 +149,6 @@ impl<W: RankableWebpage, E: EmbeddingSignal<W>> Scorer<W> for EmbeddingScorer<W,fn set_query_info(&mut self, query: &SearchQuery) {self.query = Some(query.query.clone());- self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);}}@@ -171,7 +159,7 @@ pub trait EmbeddingSignal<W>: Send + Sync {fn signal() -> SignalEnum;fn has_embedding(webpage: &W) -> bool;fn embedding(webpage: &W, hidden_size: usize) -> Option<Embedding>;- fn insert_signal(webpage: &mut W, score: f64, coefficient: f64);+ fn insert_signal(webpage: &mut W, score: f64);}impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {@@ -180,48 +168,36 @@ impl EmbeddingSignal<ScoredWebpagePointer> for TitleEmbeddings {}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {- webpage.as_ranking().title_embedding.is_some()+ webpage.as_ranking().title_embedding().is_some()}fn embedding(webpage: &ScoredWebpagePointer, hidden_size: usize) -> Option<Embedding> {webpage.title_emb(hidden_size)}- fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64, coefficient: f64) {+ fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64) {let sig = <TitleEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();- webpage.as_ranking_mut().signals.insert(- sig,- SignalScore {- coefficient,- value: score,- },- );+ webpage.as_ranking_mut().signals_mut().insert(sig, score);}}-impl EmbeddingSignal<RecallRankingWebpage> for TitleEmbeddings {+impl EmbeddingSignal<LocalRecallRankingWebpage> for TitleEmbeddings {fn signal() -> SignalEnum {ranking::signal::TitleEmbeddingSimilarity.into()}- fn has_embedding(webpage: &RecallRankingWebpage) -> bool {- webpage.title_embedding.is_some()+ fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {+ webpage.title_embedding().is_some()}- fn embedding(webpage: &RecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {+ fn embedding(webpage: &LocalRecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {webpage.title_emb(hidden_size)}- fn insert_signal(webpage: &mut RecallRankingWebpage, score: f64, coefficient: f64) {- let sig = <TitleEmbeddings as EmbeddingSignal<RecallRankingWebpage>>::signal();+ fn insert_signal(webpage: &mut LocalRecallRankingWebpage, score: f64) {+ let sig = <TitleEmbeddings as EmbeddingSignal<LocalRecallRankingWebpage>>::signal();- webpage.signals.insert(- sig,- SignalScore {- coefficient,- value: score,- },- );+ webpage.mut_signals().insert(sig, score);}}@@ -231,47 +207,35 @@ impl EmbeddingSignal<ScoredWebpagePointer> for KeywordEmbeddings {}fn has_embedding(webpage: &ScoredWebpagePointer) -> bool {- webpage.as_ranking().keyword_embedding.is_some()+ webpage.as_ranking().keyword_embedding().is_some()}fn embedding(webpage: &ScoredWebpagePointer, hidden_size: usize) -> Option<Embedding> {webpage.keyword_emb(hidden_size)}- fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64, coefficient: f64) {+ fn insert_signal(webpage: &mut ScoredWebpagePointer, score: f64) {let sig = <KeywordEmbeddings as EmbeddingSignal<ScoredWebpagePointer>>::signal();- webpage.as_ranking_mut().signals.insert(- sig,- SignalScore {- coefficient,- value: score,- },- );+ webpage.as_ranking_mut().signals_mut().insert(sig, score);}}-impl EmbeddingSignal<RecallRankingWebpage> for KeywordEmbeddings {+impl EmbeddingSignal<LocalRecallRankingWebpage> for KeywordEmbeddings {fn signal() -> SignalEnum {ranking::signal::KeywordEmbeddingSimilarity.into()}- fn has_embedding(webpage: &RecallRankingWebpage) -> bool {- webpage.keyword_embedding.is_some()+ fn has_embedding(webpage: &LocalRecallRankingWebpage) -> bool {+ webpage.keyword_embedding().is_some()}- fn embedding(webpage: &RecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {+ fn embedding(webpage: &LocalRecallRankingWebpage, hidden_size: usize) -> Option<Embedding> {webpage.keyword_emb(hidden_size)}- fn insert_signal(webpage: &mut RecallRankingWebpage, score: f64, coefficient: f64) {- let sig = <KeywordEmbeddings as EmbeddingSignal<RecallRankingWebpage>>::signal();- webpage.signals.insert(- sig,- SignalScore {- coefficient,- value: score,- },- );+ fn insert_signal(webpage: &mut LocalRecallRankingWebpage, score: f64) {+ let sig = <KeywordEmbeddings as EmbeddingSignal<LocalRecallRankingWebpage>>::signal();+ webpage.mut_signals().insert(sig, score);}}diff --git a/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rsnew file mode 100644index 00000000..19ac0852--- /dev/null+++ b/crates/core/src/ranking/pipeline/scorers/inbound_similarity.rs@@ -0,0 +1,53 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::sync::Mutex;++use crate::{+ ranking::{inbound_similarity, signal},+ searcher::api::ScoredWebpagePointer,+};++use super::Scorer;++pub struct InboundScorer {+ scorer: Mutex<inbound_similarity::Scorer>,+}++impl InboundScorer {+ pub fn new(scorer: inbound_similarity::Scorer) -> Self {+ Self {+ scorer: Mutex::new(scorer),+ }+ }+}++impl Scorer<ScoredWebpagePointer> for InboundScorer {+ fn score(&self, webpages: &mut [ScoredWebpagePointer]) {+ let mut scorer = self.scorer.lock().unwrap();++ for webpage in webpages {+ let score = scorer.score(+ webpage.as_ranking().host_id(),+ webpage.as_ranking().inbound_edges(),+ );+ webpage+ .as_ranking_mut()+ .signals_mut()+ .insert(signal::InboundSimilarity.into(), score);+ }+ }+}diff --git a/crates/core/src/ranking/pipeline/scorers/mod.rs b/crates/core/src/ranking/pipeline/scorers/mod.rsindex 98032546..624c66bd 100644--- a/crates/core/src/ranking/pipeline/scorers/mod.rs+++ b/crates/core/src/ranking/pipeline/scorers/mod.rs@@ -15,19 +15,14 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.pub mod embedding;+pub mod inbound_similarity;pub mod recall;pub mod reranker;-use std::sync::Arc;-pub use recall::Recall;pub use reranker::ReRanker;-use crate::{- enum_map::EnumMap,- ranking::{self, models::lambdamart::LambdaMART, SignalCoefficient, SignalEnum, SignalScore},- searcher::SearchQuery,-};+use crate::searcher::SearchQuery;use super::RankableWebpage;@@ -42,32 +37,6 @@ impl<T: RankableWebpage> Scorer<T> for IdentityScorer {fn score(&self, _webpages: &mut [T]) {}}-fn calculate_score(- model: &Option<Arc<LambdaMART>>,- coefficients: SignalCoefficient,- signals: &EnumMap<SignalEnum, SignalScore>,-) -> f64 {- let lambda_score = match model {- Some(model) => {- let coeff = coefficients.get(&ranking::signal::LambdaMart.into());- if coeff == 0.0 {- signals- .values()- .map(|score| score.coefficient * score.value)- .sum()- } else {- coeff * model.predict(signals)- }- }- None => signals- .values()- .map(|score| score.coefficient * score.value)- .sum(),- };-- lambda_score-}-pub struct MultiScorer<T: RankableWebpage> {scorers: Vec<Box<dyn Scorer<T>>>,}diff --git a/crates/core/src/ranking/pipeline/scorers/recall.rs b/crates/core/src/ranking/pipeline/scorers/recall.rsindex 5dbdc3ef..dbaf5f86 100644--- a/crates/core/src/ranking/pipeline/scorers/recall.rs+++ b/crates/core/src/ranking/pipeline/scorers/recall.rs@@ -19,102 +19,77 @@ use std::sync::Arc;use crate::{models::dual_encoder::DualEncoder,ranking::{- models::lambdamart::LambdaMART,- pipeline::{RankableWebpage, RecallRankingWebpage},- SignalCoefficient,+ inbound_similarity,+ pipeline::{LocalRecallRankingWebpage, RankableWebpage},+ },+ searcher::{+ api::{self},+ SearchQuery,},- searcher::{api::ScoredWebpagePointer, SearchQuery},};use super::{- calculate_score,embedding::{EmbeddingScorer, KeywordEmbeddings, TitleEmbeddings},+ inbound_similarity::InboundScorer,MultiScorer, Scorer,};pub struct Recall<T: RankableWebpage> {- sub_scorers: MultiScorer<T>,- lambdamart: Option<Arc<LambdaMART>>,- signal_coefficients: Option<SignalCoefficient>,+ scorer: MultiScorer<T>,}-impl Recall<RecallRankingWebpage> {+impl Recall<api::ScoredWebpagePointer> {pub fn new(- lambdamart: Option<Arc<LambdaMART>>,+ inbound: inbound_similarity::Scorer,dual_encoder: Option<Arc<DualEncoder>>,) -> Self {Self {- lambdamart,- signal_coefficients: None,- sub_scorers: MultiScorer::new(vec![+ scorer: MultiScorer::new(vec![Box::new(- EmbeddingScorer::<RecallRankingWebpage, TitleEmbeddings>::new(+ EmbeddingScorer::<api::ScoredWebpagePointer, TitleEmbeddings>::new(dual_encoder.clone(),),),- Box::new(- EmbeddingScorer::<RecallRankingWebpage, KeywordEmbeddings>::new(dual_encoder),- ),+ Box::new(EmbeddingScorer::<+ api::ScoredWebpagePointer,+ KeywordEmbeddings,+ >::new(dual_encoder)),+ Box::new(InboundScorer::new(inbound)),]),}}}-impl Recall<ScoredWebpagePointer> {- pub fn new(- lambdamart: Option<Arc<LambdaMART>>,- dual_encoder: Option<Arc<DualEncoder>>,- ) -> Self {+impl Scorer<api::ScoredWebpagePointer> for Recall<api::ScoredWebpagePointer> {+ fn score(&self, webpages: &mut [api::ScoredWebpagePointer]) {+ self.scorer.score(webpages);+ }+}++impl Recall<LocalRecallRankingWebpage> {+ pub fn new(dual_encoder: Option<Arc<DualEncoder>>) -> Self {Self {- lambdamart,- signal_coefficients: None,- sub_scorers: MultiScorer::new(vec![+ scorer: MultiScorer::new(vec![Box::new(- EmbeddingScorer::<ScoredWebpagePointer, TitleEmbeddings>::new(+ EmbeddingScorer::<LocalRecallRankingWebpage, TitleEmbeddings>::new(dual_encoder.clone(),),),- Box::new(- EmbeddingScorer::<ScoredWebpagePointer, KeywordEmbeddings>::new(dual_encoder),- ),+ Box::new(EmbeddingScorer::<+ LocalRecallRankingWebpage,+ KeywordEmbeddings,+ >::new(dual_encoder)),]),}}}-impl Scorer<RecallRankingWebpage> for Recall<RecallRankingWebpage> {- fn score(&self, webpages: &mut [RecallRankingWebpage]) {- self.sub_scorers.score(webpages);-- for webpage in webpages {- webpage.set_score(calculate_score(- &self.lambdamart,- self.signal_coefficients.clone().unwrap_or_default(),- &webpage.signals,- ));- }- }-- fn set_query_info(&mut self, query: &SearchQuery) {- self.sub_scorers.set_query_info(query);- self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);- }-}--impl Scorer<ScoredWebpagePointer> for Recall<ScoredWebpagePointer> {- fn score(&self, webpages: &mut [crate::searcher::api::ScoredWebpagePointer]) {- self.sub_scorers.score(webpages);-- for webpage in webpages {- webpage.set_score(calculate_score(- &self.lambdamart,- self.signal_coefficients.clone().unwrap_or_default(),- &webpage.as_ranking().signals,- ));- }+impl Scorer<LocalRecallRankingWebpage> for Recall<LocalRecallRankingWebpage> {+ fn score(&self, webpages: &mut [LocalRecallRankingWebpage]) {+ self.scorer.score(webpages);}fn set_query_info(&mut self, query: &SearchQuery) {- self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);+ self.scorer.set_query_info(query);}}diff --git a/crates/core/src/ranking/pipeline/scorers/reranker.rs b/crates/core/src/ranking/pipeline/scorers/reranker.rsindex 6ef1a56c..f5f79176 100644--- a/crates/core/src/ranking/pipeline/scorers/reranker.rs+++ b/crates/core/src/ranking/pipeline/scorers/reranker.rs@@ -17,60 +17,32 @@use std::sync::Arc;use crate::{- ranking::{- self,- models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},- Signal, SignalCoefficient, SignalEnum, SignalScore,- },+ ranking::{self, models::cross_encoder::CrossEncoder},searcher::SearchQuery,};-use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, Scorer};--use super::calculate_score;+use crate::ranking::pipeline::{PrecisionRankingWebpage, Scorer};pub struct ReRanker<M: CrossEncoder> {crossencoder: Arc<M>,- lambda_mart: Option<Arc<LambdaMART>>,query: Option<SearchQuery>,- signal_coefficients: Option<SignalCoefficient>,}impl<M: CrossEncoder> ReRanker<M> {- pub fn new(crossencoder: Arc<M>, lambda: Option<Arc<LambdaMART>>) -> Self {+ pub fn new(crossencoder: Arc<M>) -> Self {Self {crossencoder,- lambda_mart: lambda,query: None,- signal_coefficients: None,}}- fn crossencoder_snippet_coeff(&self) -> f64 {- self.signal_coefficients- .as_ref()- .map(|coeff| coeff.get(&ranking::signal::CrossEncoderSnippet.into()))- .unwrap_or_else(|| {- SignalEnum::from(ranking::signal::CrossEncoderSnippet).default_coefficient()- })- }-- fn crossencoder_title_coeff(&self) -> f64 {- self.signal_coefficients- .as_ref()- .map(|coeff| coeff.get(&ranking::signal::CrossEncoderTitle.into()))- .unwrap_or_else(|| {- SignalEnum::from(ranking::signal::CrossEncoderTitle).default_coefficient()- })- }-fn crossencoder_score_webpages(&self, webpage: &mut [PrecisionRankingWebpage]) {let mut snippets = Vec::with_capacity(webpage.len());let mut titles = Vec::with_capacity(webpage.len());for webpage in webpage.iter_mut() {- titles.push(webpage.retrieved_webpage.title.clone());- snippets.push(webpage.retrieved_webpage.snippet.unhighlighted_string());+ titles.push(webpage.retrieved_webpage().title.clone());+ snippets.push(webpage.retrieved_webpage().snippet.unhighlighted_string());}let query = &self.query.as_ref().unwrap().query;@@ -79,21 +51,15 @@ impl<M: CrossEncoder> ReRanker<M> {for ((webpage, snippet), title) in webpage.iter_mut().zip(snippet_scores).zip(title_scores){- webpage.ranking.signals.insert(- ranking::signal::CrossEncoderSnippet.into(),- SignalScore {- coefficient: self.crossencoder_snippet_coeff(),- value: snippet,- },- );-- webpage.ranking.signals.insert(- ranking::signal::CrossEncoderTitle.into(),- SignalScore {- coefficient: self.crossencoder_title_coeff(),- value: title,- },- );+ webpage+ .ranking_mut()+ .signals_mut()+ .insert(ranking::signal::CrossEncoderSnippet.into(), snippet);++ webpage+ .ranking_mut()+ .signals_mut()+ .insert(ranking::signal::CrossEncoderTitle.into(), title);}}}@@ -101,18 +67,9 @@ impl<M: CrossEncoder> ReRanker<M> {impl<M: CrossEncoder> Scorer<PrecisionRankingWebpage> for ReRanker<M> {fn score(&self, webpages: &mut [PrecisionRankingWebpage]) {self.crossencoder_score_webpages(webpages);-- for webpage in webpages.iter_mut() {- webpage.set_score(calculate_score(- &self.lambda_mart,- self.signal_coefficients.clone().unwrap_or_default(),- &webpage.ranking.signals,- ));- }}fn set_query_info(&mut self, query: &SearchQuery) {self.query = Some(query.clone());- self.signal_coefficients = query.optic.as_ref().map(SignalCoefficient::from_optic);}}diff --git a/crates/core/src/ranking/pipeline/stages/mod.rs b/crates/core/src/ranking/pipeline/stages/mod.rsindex c821db4a..f45767d3 100644--- a/crates/core/src/ranking/pipeline/stages/mod.rs+++ b/crates/core/src/ranking/pipeline/stages/mod.rs@@ -18,4 +18,4 @@ mod precision;mod recall;pub use precision::PrecisionRankingWebpage;-pub use recall::{RecallRankingWebpage, StoredEmbeddings};+pub use recall::{LocalRecallRankingWebpage, RecallRankingWebpage, StoredEmbeddings};diff --git a/crates/core/src/ranking/pipeline/stages/precision.rs b/crates/core/src/ranking/pipeline/stages/precision.rsindex 34af8f26..dd20529a 100644--- a/crates/core/src/ranking/pipeline/stages/precision.rs+++ b/crates/core/src/ranking/pipeline/stages/precision.rs@@ -19,6 +19,7 @@ use std::sync::Arc;use crate::{collector,config::CollectorConfig,+ enum_map::EnumMap,inverted_index::RetrievedWebpage,ranking::{models::{cross_encoder::CrossEncoder, lambdamart::LambdaMART},@@ -26,6 +27,7 @@ use crate::{scorers::IdentityScorer, RankableWebpage, RankingPipeline, RankingStage, ReRanker,Scorer,},+ SignalEnum,},searcher::SearchQuery,Result,@@ -35,27 +37,45 @@ use super::RecallRankingWebpage;#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct PrecisionRankingWebpage {- pub retrieved_webpage: RetrievedWebpage,- pub ranking: RecallRankingWebpage,+ retrieved_webpage: RetrievedWebpage,+ ranking: RecallRankingWebpage,+}++impl PrecisionRankingWebpage {+ pub fn retrieved_webpage(&self) -> &RetrievedWebpage {+ &self.retrieved_webpage+ }++ pub fn ranking(&self) -> &RecallRankingWebpage {+ &self.ranking+ }++ pub fn ranking_mut(&mut self) -> &mut RecallRankingWebpage {+ &mut self.ranking+ }}impl collector::Doc for PrecisionRankingWebpage {fn score(&self) -> f64 {- self.ranking.score+ self.ranking.score()}fn hashes(&self) -> collector::Hashes {- self.ranking.pointer.hashes+ self.ranking.pointer().hashes}}impl RankableWebpage for PrecisionRankingWebpage {fn set_score(&mut self, score: f64) {- self.ranking.score = score;+ self.ranking.set_score(score);}fn boost(&self) -> Option<f64> {- self.ranking.optic_boost+ self.ranking.boost()+ }++ fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ self.ranking.signals()}}@@ -80,8 +100,9 @@ impl RankingPipeline<PrecisionRankingWebpage> {top_n_considered: usize,) -> Result<Self> {let scorer = match crossencoder {- Some(cross_encoder) => Box::new(ReRanker::new(cross_encoder, lambda))- as Box<dyn Scorer<PrecisionRankingWebpage>>,+ Some(cross_encoder) => {+ Box::new(ReRanker::new(cross_encoder)) as Box<dyn Scorer<PrecisionRankingWebpage>>+ }None => Box::new(IdentityScorer) as Box<dyn Scorer<PrecisionRankingWebpage>>,};@@ -89,6 +110,8 @@ impl RankingPipeline<PrecisionRankingWebpage> {scorer,stage_top_n: top_n_considered,derank_similar: true,+ model: lambda,+ coefficients: Default::default(),};Ok(Self {diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex ed06a9ad..2c068b5f 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -24,12 +24,14 @@ use crate::{inverted_index::WebpagePointer,models::dual_encoder::DualEncoder,ranking::{+ bitvec_similarity, inbound_similarity,models::lambdamart::LambdaMART,pipeline::{RankableWebpage, RankingPipeline, RankingStage, Recall, Scorer},- SignalComputer, SignalEnum, SignalScore,+ SignalComputer, SignalEnum,},schema::fast_field,- searcher::SearchQuery,+ searcher::{api, SearchQuery},+ webgraph,};#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]@@ -40,18 +42,117 @@ impl StoredEmbeddings {&self.0}}-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct RecallRankingWebpage {- pub pointer: WebpagePointer,- pub signals: EnumMap<SignalEnum, SignalScore>,- pub optic_boost: Option<f64>,- pub title_embedding: Option<StoredEmbeddings>,- pub keyword_embedding: Option<StoredEmbeddings>,- pub score: f64,+ local: LocalRecallRankingWebpage,+ inbound_edges: bitvec_similarity::BitVec,}impl RecallRankingWebpage {+ pub fn new(local: LocalRecallRankingWebpage, inbound_edges: bitvec_similarity::BitVec) -> Self {+ RecallRankingWebpage {+ local,+ inbound_edges,+ }+ }++ pub fn pointer(&self) -> &WebpagePointer {+ self.local.pointer()+ }++ pub fn title_embedding(&self) -> Option<&StoredEmbeddings> {+ self.local.title_embedding()+ }++ pub fn keyword_embedding(&self) -> Option<&StoredEmbeddings> {+ self.local.keyword_embedding()+ }++ pub fn score(&self) -> f64 {+ self.local.score()+ }++ pub fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ self.local.signals()+ }++ pub fn signals_mut(&mut self) -> &mut EnumMap<SignalEnum, f64> {+ self.local.mut_signals()+ }++ pub fn boost(&self) -> Option<f64> {+ self.local.boost()+ }++ pub fn set_score(&mut self, score: f64) {+ self.local.set_score(score)+ }++ pub fn inbound_edges(&self) -> &bitvec_similarity::BitVec {+ &self.inbound_edges+ }++ pub fn host_id(&self) -> &webgraph::NodeID {+ self.local.host_id()+ }+}++impl collector::Doc for RecallRankingWebpage {+ fn score(&self) -> f64 {+ self.local.score()+ }++ fn hashes(&self) -> collector::Hashes {+ self.local.pointer().hashes+ }+}++impl RankableWebpage for RecallRankingWebpage {+ fn set_score(&mut self, score: f64) {+ self.local.set_score(score);+ }++ fn boost(&self) -> Option<f64> {+ self.local.boost()+ }++ fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ self.local.signals()+ }+}++#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct LocalRecallRankingWebpage {+ pointer: WebpagePointer,+ signals: EnumMap<SignalEnum, f64>,+ optic_boost: Option<f64>,+ title_embedding: Option<StoredEmbeddings>,+ keyword_embedding: Option<StoredEmbeddings>,+ score: f64,+ host_id: webgraph::NodeID,+}++impl LocalRecallRankingWebpage {+ #[cfg(test)]+ pub fn new_testing(+ pointer: WebpagePointer,+ signals: EnumMap<SignalEnum, f64>,+ score: f64,+ ) -> Self {+ LocalRecallRankingWebpage {+ pointer,+ signals,+ optic_boost: None,+ title_embedding: None,+ keyword_embedding: None,+ score,+ host_id: webgraph::NodeID::from(0u64),+ }+ }++ /// The ranking webpages needs to be constructed in order+ /// of ascending doc_id as they traverse the posting lists from+ /// the index to calculate bm25.pub fn new(pointer: WebpagePointer,fastfield_reader: &fastfield_reader::SegmentReader,@@ -67,13 +168,21 @@ impl RecallRankingWebpage {.get(fast_field::KeywordEmbeddings.into()).and_then(|v| v.into());- let mut res = RecallRankingWebpage {+ let host_id = fastfields+ .get(fast_field::HostNodeID.into())+ .unwrap()+ .as_u64()+ .unwrap()+ .into();++ let mut res = LocalRecallRankingWebpage {signals: EnumMap::new(),score: pointer.score.total,optic_boost: None,pointer: pointer.clone(),title_embedding: title_embedding.map(StoredEmbeddings),keyword_embedding: keyword_embedding.map(StoredEmbeddings),+ host_id,};for computed_signal in computer.compute_signals(pointer.address.doc_id).flatten() {@@ -87,8 +196,45 @@ impl RecallRankingWebpage {res}++ pub fn pointer(&self) -> &WebpagePointer {+ &self.pointer+ }++ pub fn title_embedding(&self) -> Option<&StoredEmbeddings> {+ self.title_embedding.as_ref()+ }++ pub fn keyword_embedding(&self) -> Option<&StoredEmbeddings> {+ self.keyword_embedding.as_ref()+ }++ pub fn score(&self) -> f64 {+ self.score+ }++ pub fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ &self.signals+ }++ pub fn mut_signals(&mut self) -> &mut EnumMap<SignalEnum, f64> {+ &mut self.signals+ }++ pub fn boost(&self) -> Option<f64> {+ self.optic_boost+ }++ pub fn set_score(&mut self, score: f64) {+ self.score = score;+ }++ pub fn host_id(&self) -> &webgraph::NodeID {+ &self.host_id+ }}-impl RankableWebpage for RecallRankingWebpage {++impl RankableWebpage for LocalRecallRankingWebpage {fn set_score(&mut self, score: f64) {self.score = score;}@@ -96,9 +242,13 @@ impl RankableWebpage for RecallRankingWebpage {fn boost(&self) -> Option<f64> {self.optic_boost}++ fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ &self.signals+ }}-impl collector::Doc for RecallRankingWebpage {+impl collector::Doc for LocalRecallRankingWebpage {fn score(&self) -> f64 {self.score}@@ -108,7 +258,7 @@ impl collector::Doc for RecallRankingWebpage {}}-impl RankingPipeline<RecallRankingWebpage> {+impl RankingPipeline<LocalRecallRankingWebpage> {fn create_recall_stage(lambdamart: Option<Arc<LambdaMART>>,dual_encoder: Option<Arc<DualEncoder>>,@@ -116,12 +266,12 @@ impl RankingPipeline<RecallRankingWebpage> {stage_top_n: usize,) -> Self {let last_stage = RankingStage {- scorer: Box::new(Recall::<RecallRankingWebpage>::new(- lambdamart,- dual_encoder,- )) as Box<dyn Scorer<RecallRankingWebpage>>,+ scorer: Box::new(Recall::<LocalRecallRankingWebpage>::new(dual_encoder))+ as Box<dyn Scorer<LocalRecallRankingWebpage>>,stage_top_n,derank_similar: true,+ model: lambdamart,+ coefficients: Default::default(),};Self {@@ -146,3 +296,51 @@ impl RankingPipeline<RecallRankingWebpage> {pipeline}}++impl RankingPipeline<api::ScoredWebpagePointer> {+ fn create_recall_stage(+ inbound: inbound_similarity::Scorer,+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,+ stage_top_n: usize,+ ) -> Self {+ let last_stage = RankingStage {+ scorer: Box::new(Recall::<api::ScoredWebpagePointer>::new(+ inbound,+ dual_encoder,+ )),+ stage_top_n,+ derank_similar: true,+ model: lambdamart,+ coefficients: Default::default(),+ };++ Self {+ stage: last_stage,+ page: 0,+ top_n: 0,+ collector_config,+ }+ }++ pub fn recall_stage(+ query: &mut SearchQuery,+ inbound: inbound_similarity::Scorer,+ lambdamart: Option<Arc<LambdaMART>>,+ dual_encoder: Option<Arc<DualEncoder>>,+ collector_config: CollectorConfig,+ top_n_considered: usize,+ ) -> Self {+ let mut pipeline = Self::create_recall_stage(+ inbound,+ lambdamart,+ dual_encoder,+ collector_config,+ top_n_considered,+ );+ pipeline.set_query_info(query);++ pipeline+ }+}diff --git a/crates/core/src/ranking/query_centrality.rs b/crates/core/src/ranking/query_centrality.rsdeleted file mode 100644index 1551b826..00000000--- a/crates/core/src/ranking/query_centrality.rs+++ /dev/null@@ -1,37 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use crate::webgraph::NodeID;--use super::inbound_similarity;--#[derive(Clone)]-pub struct Scorer {- inbound_centrality: inbound_similarity::Scorer,-}--impl Scorer {- pub fn new(inbound_centrality: inbound_similarity::Scorer) -> Self {- let mut inbound_centrality = inbound_centrality;- inbound_centrality.set_self_score(0.0);-- Self { inbound_centrality }- }-- pub fn score(&mut self, node: NodeID) -> f64 {- self.inbound_centrality.score(&node)- }-}diff --git a/crates/core/src/ranking/signal/computer/mod.rs b/crates/core/src/ranking/signal/computer/mod.rsindex 77f4b1e8..84546d60 100644--- a/crates/core/src/ranking/signal/computer/mod.rs+++ b/crates/core/src/ranking/signal/computer/mod.rs@@ -18,9 +18,7 @@ use crate::query::optic::AsSearchableRule;use crate::query::Query;use crate::schema::text_field::TextField;use crate::Result;-use crate::{- enum_map::EnumMap, fastfield_reader, schema::TextFieldEnum, webgraph::NodeID, webpage::Webpage,-};+use crate::{enum_map::EnumMap, fastfield_reader, schema::TextFieldEnum, webpage::Webpage};use std::cell::RefCell;@@ -38,9 +36,8 @@ use crate::webpage::region::RegionCount;use crate::ranking::bm25::MultiBm25Weight;use crate::ranking::models::linear::LinearRegression;-use crate::ranking::{inbound_similarity, query_centrality};-use super::{ComputedSignal, Signal, SignalCoefficient, SignalEnum, SignalScore};+use super::{ComputedSignal, Signal, SignalCoefficient, SignalEnum};mod order;pub use order::SignalComputeOrder;@@ -93,10 +90,8 @@ pub struct SignalComputer {query_data: Option<QueryData>,query_signal_coefficients: Option<SignalCoefficient>,segment_reader: Option<RefCell<SegmentReader>>,- inbound_similarity: Option<RefCell<inbound_similarity::Scorer>>,fetch_time_ms_cache: Vec<f64>,update_time_cache: Vec<f64>,- query_centrality: Option<RefCell<query_centrality::Scorer>>,region_count: Option<Arc<RegionCount>>,current_timestamp: Option<usize>,linear_regression: Option<Arc<LinearRegression>>,@@ -105,24 +100,12 @@ pub struct SignalComputer {impl Clone for SignalComputer {fn clone(&self) -> Self {- let inbound_similarity = self- .inbound_similarity- .as_ref()- .map(|scorer| RefCell::new(scorer.borrow().clone()));-- let query_centrality = self- .query_centrality- .as_ref()- .map(|scorer| RefCell::new(scorer.borrow().clone()));-Self {query_data: self.query_data.clone(),query_signal_coefficients: self.query_signal_coefficients.clone(),segment_reader: None,- inbound_similarity,fetch_time_ms_cache: self.fetch_time_ms_cache.clone(),update_time_cache: self.update_time_cache.clone(),- query_centrality,region_count: self.region_count.clone(),current_timestamp: self.current_timestamp,linear_regression: self.linear_regression.clone(),@@ -175,11 +158,9 @@ impl SignalComputer {let mut s = Self {segment_reader: None,- inbound_similarity: None,query_signal_coefficients,fetch_time_ms_cache,update_time_cache,- query_centrality: None,region_count: None,current_timestamp: None,linear_regression: None,@@ -310,17 +291,6 @@ impl SignalComputer {Ok(())}- pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {- self.query_centrality = Some(RefCell::new(query_centrality));- }-- pub fn set_inbound_similarity(&mut self, scorer: inbound_similarity::Scorer) {- let mut scorer = scorer;- scorer.set_default_if_precalculated(true);-- self.inbound_similarity = Some(RefCell::new(scorer));- }-pub fn set_region_count(&mut self, region_count: RegionCount) {self.region_count = Some(Arc::new(region_count));}@@ -333,19 +303,6 @@ impl SignalComputer {self.linear_regression = Some(linear_model);}- pub fn query_centrality(&self, host_id: NodeID) -> Option<f64> {- self.query_centrality- .as_ref()- .map(|scorer| scorer.borrow_mut().score(host_id))- }-- pub fn inbound_similarity(&self, host_id: NodeID) -> f64 {- self.inbound_similarity- .as_ref()- .map(|scorer| scorer.borrow_mut().score(&host_id))- .unwrap_or_default()- }-/// Computes the scored signals for a given document.////// Important: This function assues that the docs a scored in ascending order of docid@@ -390,15 +347,9 @@ impl SignalComputer {.filter_map(|signal| {signal.precompute(webpage, self)- .map(|value| ComputedSignal {- signal,- score: SignalScore {- coefficient: self.coefficient(&signal),- value,- },- })+ .map(|score| ComputedSignal { signal, score })})- .map(|computed| computed.score.coefficient * computed.score.value)+ .map(|computed| self.coefficient(&computed.signal) * computed.score).sum()}diff --git a/crates/core/src/ranking/signal/computer/order.rs b/crates/core/src/ranking/signal/computer/order.rsindex 64ae2d21..6923fee1 100644--- a/crates/core/src/ranking/signal/computer/order.rs+++ b/crates/core/src/ranking/signal/computer/order.rs@@ -18,7 +18,7 @@ use tantivy::DocId;use crate::{enum_map::EnumMap,- ranking::{ComputedSignal, Signal, SignalEnum, SignalScore},+ ranking::{ComputedSignal, Signal, SignalEnum},schema::{text_field::TextField, TextFieldEnum},};@@ -79,21 +79,13 @@ impl SignalComputeOrder {.chain(self.other_signals.iter()- .filter_map(|signal| {- let coefficient = signal_computer.coefficient(signal);-- if coefficient > 0.0 {- Some((signal, coefficient))- } else {- None- }- })- .map(move |(signal, coefficient)| {+ .filter(|signal| signal_computer.coefficient(signal) > 0.0)+ .map(move |signal| {signal.compute(doc, signal_computer)- .map(|value| ComputedSignal {+ .map(|score| ComputedSignal {signal: *signal,- score: SignalScore { coefficient, value },+ score,})}),)@@ -132,18 +124,14 @@ impl NGramComputeOrder {.filter_map(move |signal| {signal.compute(doc, signal_computer)- .map(|value| {- let coefficient = signal_computer.coefficient(signal);-- ComputedSignal {- signal: *signal,- score: SignalScore { coefficient, value },- }+ .map(|score| ComputedSignal {+ signal: *signal,+ score,}).map(|mut c| {- c.score.coefficient *= NGRAM_DAMPENING.powi(hits);+ c.score *= NGRAM_DAMPENING.powi(hits);- if c.score.value > 0.0 {+ if c.score > 0.0 {hits += 1;}diff --git a/crates/core/src/ranking/signal/mod.rs b/crates/core/src/ranking/signal/mod.rsindex 914bfcb5..370b4eb6 100644--- a/crates/core/src/ranking/signal/mod.rs+++ b/crates/core/src/ranking/signal/mod.rs@@ -49,11 +49,17 @@ impl FromStr for SignalEnumDiscriminants {}}-#[derive(Debug, Clone, Default)]+#[derive(Debug, Clone)]pub struct SignalCoefficient {map: EnumMap<SignalEnum, f64>,}+impl Default for SignalCoefficient {+ fn default() -> Self {+ Self::new(SignalEnum::all().map(|signal| (signal, signal.default_coefficient())))+ }+}+impl SignalCoefficient {pub fn get(&self, signal: &SignalEnum) -> f64 {self.map@@ -82,7 +88,7 @@ impl SignalCoefficient {}))}- pub fn merge_into(&mut self, coeffs: SignalCoefficient) {+ pub fn merge_add(&mut self, coeffs: SignalCoefficient) {for signal in SignalEnum::all() {if let Some(coeff) = coeffs.map.get(signal).copied() {match self.map.get_mut(signal) {@@ -94,12 +100,25 @@ impl SignalCoefficient {}}}++ pub fn merge_overwrite(&mut self, coeffs: SignalCoefficient) {+ for signal in SignalEnum::all() {+ if let Some(coeff) = coeffs.map.get(signal).copied() {+ match self.map.get_mut(signal) {+ Some(existing_coeff) => *existing_coeff = coeff,+ None => {+ self.map.insert(signal, coeff);+ }+ }+ }+ }+ }}#[derive(Debug, Clone, Copy)]pub struct ComputedSignal {pub signal: SignalEnum,- pub score: SignalScore,+ pub score: f64,}#[derive(diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex cf3b2144..2c9a6fb4 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -18,9 +18,7 @@ use tantivy::DocId;use super::{Signal, SignalComputer};use crate::{- fastfield_reader::FieldReader,schema::{self, Field, FLOAT_SCALING},- webgraph::NodeID,webpage::Webpage,};@@ -88,19 +86,6 @@ fn score_region(webpage_region: crate::webpage::Region, computer: &SignalCompute}}-fn host_id(fastfield_reader: &FieldReader<'_>) -> Option<NodeID> {- let node_id = fastfield_reader- .get(schema::fast_field::HostNodeID.into())- .and_then(|n| n.as_u64())- .unwrap();-- if node_id == u64::MAX {- None- } else {- Some(node_id.into())- }-}-#[derive(Debug,Clone,@@ -486,12 +471,8 @@ impl Signal for QueryCentrality {None}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);- let host_id = host_id(&fastfield_reader);-- host_id.and_then(|host_id| signal_computer.query_centrality(host_id))+ fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {+ unimplemented!()}}@@ -517,12 +498,8 @@ impl Signal for InboundSimilarity {None}- fn compute(&self, doc: DocId, signal_computer: &SignalComputer) -> Option<f64> {- let seg_reader = signal_computer.segment_reader().unwrap().borrow_mut();- let fastfield_reader = seg_reader.fastfield_reader().get_field_reader(doc);- let host_id = host_id(&fastfield_reader);-- host_id.map(|host_id| signal_computer.inbound_similarity(host_id))+ fn compute(&self, _doc: DocId, _signal_computer: &SignalComputer) -> Option<f64> {+ None // computed in later ranking stage}}diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rsindex 85c13e78..8e454219 100644--- a/crates/core/src/ranking/signal/prelude.rs+++ b/crates/core/src/ranking/signal/prelude.rs@@ -14,7 +14,7 @@ use crate::enum_dispatch_from_discriminant;//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>-use crate::enum_map::InsertEnumMapKey;+use crate::enum_map::{GetEnumMapKey, InsertEnumMapKey};use crate::schema::Field;use crate::{@@ -186,3 +186,12 @@ impl InsertEnumMapKey for SignalEnum {SignalEnumDiscriminants::from(self) as usize}}++impl GetEnumMapKey for SignalEnum {+ fn from_usize(value: usize) -> Option<Self> {+ SignalEnumDiscriminants::VARIANTS+ .get(value)+ .copied()+ .map(SignalEnum::from)+ }+}diff --git a/crates/core/src/search_prettifier/stack_overflow.rs b/crates/core/src/search_prettifier/stack_overflow.rsindex 7475fbf8..0d8c58c0 100644--- a/crates/core/src/search_prettifier/stack_overflow.rs+++ b/crates/core/src/search_prettifier/stack_overflow.rs@@ -19,8 +19,8 @@ use utoipa::ToSchema;use crate::{inverted_index::RetrievedWebpage,- webpage::schema_org::{self, Item, OneOrMany, Property},- Error,+ webpage::schema_org::{self, Item, Property},+ Error, OneOrMany,};use super::DisplayedSidebar;diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 079d6654..976c4ee9 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -17,31 +17,37 @@mod sidebar;mod widget;-use std::collections::HashMap;+use std::future::Future;use std::sync::Arc;use std::time::Instant;use itertools::{intersperse, Itertools};use url::Url;+use ahash::AHashMap as HashMap;+use crate::bangs::{Bang, BangHit};use crate::collector::{self, Doc};-use crate::config::{ApiConfig, CollectorConfig};+use crate::config::{ApiConfig, ApiSpellCheck, ApiThresholds, CollectorConfig, WidgetsConfig};+use crate::enum_map::EnumMap;use crate::image_store::Image;use crate::inverted_index::RetrievedWebpage;use crate::models::dual_encoder::DualEncoder;use crate::ranking::models::cross_encoder::CrossEncoderModel;use crate::ranking::pipeline::{PrecisionRankingWebpage, RankableWebpage, RecallRankingWebpage};-use crate::ranking::SignalEnum;+use crate::ranking::{+ bitvec_similarity, inbound_similarity, SignalCoefficient, SignalEnum, SignalScore,+};use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};use crate::web_spell::SpellChecker;+use crate::webgraph::remote::RemoteWebgraph;use crate::widgets::{Widget, Widgets};use crate::{bangs::Bangs,collector::BucketCollector,ranking::{models::lambdamart::LambdaMART, pipeline::RankingPipeline},};-use crate::{query, Result};+use crate::{query, webgraph, Result};use self::sidebar::SidebarManager;use self::widget::WidgetManager;@@ -78,6 +84,13 @@ impl RankableWebpage for ScoredWebpagePointer {fn boost(&self) -> Option<f64> {self.as_ranking().boost()}++ fn signals(&self) -> &EnumMap<SignalEnum, f64> {+ match self {+ ScoredWebpagePointer::Normal(p) => p.website.signals(),+ ScoredWebpagePointer::Live(p) => p.website.signals(),+ }+ }}impl collector::Doc for ScoredWebpagePointer {@@ -90,74 +103,96 @@ impl collector::Doc for ScoredWebpagePointer {}}-pub fn combine_results(- collector_config: CollectorConfig,- initial_results: Vec<distributed::InitialSearchResultShard>,- live_results: Vec<live::InitialSearchResultSplit>,- pipeline: RankingPipeline<ScoredWebpagePointer>,-) -> (Vec<ScoredWebpagePointer>, bool) {- let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);-- let mut has_more = false;- for result in initial_results {- if result.local_result.has_more {- has_more = true;- }-- for website in result.local_result.websites {- let pointer = distributed::ScoredWebpagePointer {- website,- shard: result.shard,- };-- let pointer = ScoredWebpagePointer::Normal(pointer);-- collector.insert(pointer);- }- }+pub fn add_ranking_signals(+ websites: &mut [DisplayedWebpage],+ pointers: &[ScoredWebpagePointer],+ coeffs: &SignalCoefficient,+) {+ for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {+ let mut signals = std::collections::HashMap::new();- for result in live_results {- if result.local_result.has_more {- has_more = true;+ for signal in SignalEnum::all() {+ if let Some(signal_value) = pointer.as_ranking().signals().get(signal) {+ signals.insert(+ signal.into(),+ SignalScore {+ value: *signal_value,+ coefficient: coeffs.get(&signal),+ },+ );+ }}- for website in result.local_result.websites {- let pointer = live::ScoredWebpagePointer {- website,- split_id: result.split_id.clone(),- };+ website.ranking_signals = Some(signals);+ }+}- let pointer = ScoredWebpagePointer::Live(pointer);+#[derive(Default)]+pub struct Config {+ pub thresholds: ApiThresholds,+ pub widgets: WidgetsConfig,+ pub collector: CollectorConfig,+ pub spell_check: Option<ApiSpellCheck>,+}- collector.insert(pointer);+impl From<ApiConfig> for Config {+ fn from(conf: ApiConfig) -> Self {+ Self {+ thresholds: conf.thresholds,+ widgets: conf.widgets,+ collector: conf.collector,+ spell_check: conf.spell_check,}}+}- let top_websites = collector- .into_sorted_vec(true)- .into_iter()- .take(pipeline.collector_top_n())- .collect::<Vec<_>>();+pub trait Graph {+ fn batch_raw_ingoing(+ &self,+ nodes: &[webgraph::NodeID],+ ) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>>;+}- let res = pipeline.apply(top_websites);+impl Graph for RemoteWebgraph {+ async fn batch_raw_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::Edge<()>>> {+ self.batch_raw_ingoing_edges(nodes)+ .await+ .unwrap_or_default()+ }+}- (res, has_more)+impl Graph for webgraph::Webgraph {+ async fn batch_raw_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::Edge<()>>> {+ nodes.iter().map(|n| self.raw_ingoing_edges(n)).collect()+ }}-pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[ScoredWebpagePointer]) {- for (website, pointer) in websites.iter_mut().zip(pointers.iter()) {- let mut signals = HashMap::new();- for signal in SignalEnum::all() {- if let Some(signal_value) = pointer.as_ranking().signals.get(signal) {- signals.insert(signal.into(), *signal_value);- }- }+impl<T> Graph for Arc<T>+where+ T: Graph,+{+ fn batch_raw_ingoing(+ &self,+ nodes: &[webgraph::NodeID],+ ) -> impl Future<Output = Vec<Vec<webgraph::Edge<()>>>> {+ self.as_ref().batch_raw_ingoing(nodes)+ }+}- website.ranking_signals = Some(signals);+impl<T> bitvec_similarity::Graph for T+where+ T: Graph,+{+ async fn batch_ingoing(&self, nodes: &[webgraph::NodeID]) -> Vec<Vec<webgraph::NodeID>> {+ self.batch_raw_ingoing(nodes)+ .await+ .into_iter()+ .map(|edges| edges.into_iter().map(|edge| edge.from).collect())+ .collect()}}-pub struct ApiSearcher<S, L> {+pub struct ApiSearcher<S, L, G> {distributed_searcher: Arc<S>,sidebar_manager: SidebarManager<S>,live_searcher: Option<L>,@@ -168,47 +203,68 @@ pub struct ApiSearcher<S, L> {collector_config: CollectorConfig,widget_manager: WidgetManager,spell_checker: Option<SpellChecker>,+ webgraph: Option<G>,}-impl<S, L> ApiSearcher<S, L>+impl<S, L, G> ApiSearcher<S, L, G>whereS: distributed::SearchClient,L: live::SearchClient,+ G: Graph,{- pub fn new(- dist_searcher: S,- live_searcher: Option<L>,- cross_encoder: Option<CrossEncoderModel>,- lambda_model: Option<LambdaMART>,- dual_encoder: Option<DualEncoder>,- bangs: Bangs,- config: ApiConfig,- ) -> Self {+ pub fn new<C>(dist_searcher: S, bangs: Bangs, config: C) -> Self+ where+ C: Into<Config>,+ {+ let config: Config = config.into();let dist_searcher = Arc::new(dist_searcher);let sidebar_manager =SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());- let lambda_model = lambda_model.map(Arc::new);- let dual_encoder = dual_encoder.map(Arc::new);-let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());Self {distributed_searcher: dist_searcher,sidebar_manager,- live_searcher,- cross_encoder: cross_encoder.map(Arc::new),- lambda_model,- dual_encoder,+ live_searcher: None,+ cross_encoder: None,+ lambda_model: None,+ dual_encoder: None,bangs,collector_config: config.collector,widget_manager,spell_checker: config- .spell_checker_path- .map(|c| SpellChecker::open(c, config.correction_config).unwrap()),+ .spell_check+ .map(|c| SpellChecker::open(c.path, c.correction_config).unwrap()),+ webgraph: None,}}+ pub fn with_live(mut self, live_searcher: L) -> Self {+ self.live_searcher = Some(live_searcher);+ self+ }++ pub fn with_cross_encoder(mut self, cross_encoder: CrossEncoderModel) -> Self {+ self.cross_encoder = Some(Arc::new(cross_encoder));+ self+ }++ pub fn with_dual_encoder(mut self, dual_encoder: DualEncoder) -> Self {+ self.dual_encoder = Some(Arc::new(dual_encoder));+ self+ }++ pub fn with_lambda_model(mut self, lambda_model: LambdaMART) -> Self {+ self.lambda_model = Some(Arc::new(lambda_model));+ self+ }++ pub fn with_webgraph(mut self, webgraph: G) -> Self {+ self.webgraph = Some(webgraph);+ self+ }+async fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {let parsed_terms = query::parser::parse(&query.query)?;@@ -379,6 +435,124 @@ where}}+ async fn inbound_vecs(&self, ids: &[webgraph::NodeID]) -> Vec<bitvec_similarity::BitVec> {+ match self.webgraph.as_ref() {+ Some(webgraph) => bitvec_similarity::BitVec::batch_new_for(ids, webgraph).await,+ None => vec![bitvec_similarity::BitVec::default(); ids.len()],+ }+ }++ async fn combine_results(+ &self,+ collector_config: CollectorConfig,+ initial_results: Vec<distributed::InitialSearchResultShard>,+ live_results: Vec<live::InitialSearchResultSplit>,+ pipeline: RankingPipeline<ScoredWebpagePointer>,+ ) -> (Vec<ScoredWebpagePointer>, bool) {+ let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);++ let initial_host_nodes = initial_results+ .iter()+ .flat_map(|r| r.local_result.websites.iter())+ .map(|r| *r.host_id())+ .collect::<Vec<_>>();++ let live_host_nodes = live_results+ .iter()+ .flat_map(|r| r.local_result.websites.iter())+ .map(|r| *r.host_id())+ .collect::<Vec<_>>();++ let host_nodes = initial_host_nodes+ .into_iter()+ .chain(live_host_nodes)+ .unique()+ .collect::<Vec<_>>();++ let host_nodes = self+ .inbound_vecs(&host_nodes)+ .await+ .into_iter()+ .zip_eq(host_nodes)+ .map(|(v, n)| (n, v))+ .collect::<HashMap<_, _>>();++ let mut has_more = false;+ for result in initial_results {+ if result.local_result.has_more {+ has_more = true;+ }++ for website in result.local_result.websites {+ let inbound = host_nodes+ .get(website.host_id())+ .cloned()+ .unwrap_or_default();+ let pointer = distributed::ScoredWebpagePointer {+ website: RecallRankingWebpage::new(website, inbound),+ shard: result.shard,+ };++ let pointer = ScoredWebpagePointer::Normal(pointer);++ collector.insert(pointer);+ }+ }++ for result in live_results {+ if result.local_result.has_more {+ has_more = true;+ }++ for website in result.local_result.websites {+ let inbound = host_nodes+ .get(website.host_id())+ .cloned()+ .unwrap_or_default();+ let pointer = live::ScoredWebpagePointer {+ website: RecallRankingWebpage::new(website, inbound),+ split_id: result.split_id.clone(),+ };++ let pointer = ScoredWebpagePointer::Live(pointer);++ collector.insert(pointer);+ }+ }++ let top_websites = collector+ .into_sorted_vec(true)+ .into_iter()+ .take(pipeline.collector_top_n())+ .collect::<Vec<_>>();++ let res = pipeline.apply(top_websites);++ (res, has_more)+ }++ async fn inbound_scorer(&self, query: &SearchQuery) -> inbound_similarity::Scorer {+ match self.webgraph.as_ref() {+ Some(webgraph) => {+ let host_rankings = query.host_rankings();++ let liked: Vec<_> = host_rankings+ .liked+ .iter()+ .map(|n| webgraph::Node::from(n.clone()).into_host().id())+ .collect();++ let disliked: Vec<_> = host_rankings+ .disliked+ .iter()+ .map(|n| webgraph::Node::from(n.clone()).into_host().id())+ .collect();+ inbound_similarity::Scorer::new(webgraph, &liked, &disliked, false).await+ }+ None => inbound_similarity::Scorer::empty(),+ }+ }+async fn search_websites(&self, query: &SearchQuery) -> Result<WebsitesResult> {let start = Instant::now();@@ -387,6 +561,8 @@ where}let mut search_query = query.clone();+ let inbound_scorer = self.inbound_scorer(&search_query).await;+let top_n = search_query.num_results;// This pipeline should be created before the first search is performed@@ -394,6 +570,7 @@ wherelet recall_pipeline: RankingPipeline<ScoredWebpagePointer> =RankingPipeline::<ScoredWebpagePointer>::recall_stage(&mut search_query,+ inbound_scorer,self.lambda_model.clone(),self.dual_encoder.clone(),self.collector_config.clone(),@@ -410,12 +587,14 @@ where.map(|result| result.local_result.num_websites).sum();- let (top_websites, has_more_results) = combine_results(- self.collector_config.clone(),- initial_results,- live_results.unwrap_or_default(),- recall_pipeline,- );+ let (top_websites, has_more_results) = self+ .combine_results(+ self.collector_config.clone(),+ initial_results,+ live_results.unwrap_or_default(),+ recall_pipeline,+ )+ .await;let retrieved_webpages = self.retrieve_webpages(&search_query.query, &top_websites)@@ -440,7 +619,7 @@ wherelet mut retrieved_webpages: Vec<_> = retrieved_webpages.into_iter().map(|webpage| webpage.into_retrieved_webpage())- .map(|webpage| DisplayedWebpage::new(webpage, query))+ .map(|webpage| DisplayedWebpage::new(webpage, &search_query)).collect();if retrieved_webpages.len() != top_websites.len() {@@ -448,7 +627,11 @@ where}if query.return_ranking_signals {- add_ranking_signals(&mut retrieved_webpages, &top_websites);+ add_ranking_signals(+ &mut retrieved_webpages,+ &top_websites,+ &search_query.signal_coefficients(),+ );}for (website, pointer) in retrieved_webpages.iter_mut().zip(top_websites.iter()) {diff --git a/crates/core/src/searcher/api/sidebar.rs b/crates/core/src/searcher/api/sidebar.rsindex 1f009fd3..50aa65aa 100644--- a/crates/core/src/searcher/api/sidebar.rs+++ b/crates/core/src/searcher/api/sidebar.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{config::ApiThresholds, Result};+use crate::{config::ApiThresholds, ranking::pipeline::RecallRankingWebpage, Result};use std::{cmp::Ordering, sync::Arc};use optics::Optic;@@ -64,11 +64,14 @@ where}).collect();- results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));+ results+ .sort_by(|(_, a), (_, b)| a.score().partial_cmp(&b.score()).unwrap_or(Ordering::Equal));if let Some((shard, website)) = results.pop() {- tracing::debug!(?website.score, ?self.thresholds.stackoverflow, "stackoverflow score");- if website.score > self.thresholds.stackoverflow {+ let score = website.score();+ tracing::debug!(?score, ?self.thresholds.stackoverflow, "stackoverflow score");+ if website.score() > self.thresholds.stackoverflow {+ let website = RecallRankingWebpage::new(website, Default::default());let scored_websites =vec![(0, distributed::ScoredWebpagePointer { website, shard })];let mut retrieved = selfdiff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 1858e2f4..43548392 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -29,6 +29,7 @@ use crate::{search_server::{self, SearchService},},image_store::Image,+ index::Index,inverted_index::{RetrievedWebpage, WebpagePointer},ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},Result,@@ -43,7 +44,7 @@ use std::future::Future;use thiserror::Error;use url::Url;-use super::{InitialWebsiteResult, SearchQuery};+use super::{InitialWebsiteResult, LocalSearcher, SearchQuery};#[derive(Error, Debug)]pub enum Error {@@ -57,6 +58,38 @@ pub enum Error {WebpageNotFound,}+pub trait SearchClient {+ fn search_initial(+ &self,+ query: &SearchQuery,+ ) -> impl Future<Output = Vec<InitialSearchResultShard>> + Send;++ fn retrieve_webpages(+ &self,+ top_websites: &[(usize, ScoredWebpagePointer)],+ query: &str,+ ) -> impl Future<Output = Vec<(usize, PrecisionRankingWebpage)>> + Send;++ fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;++ fn get_webpage(+ &self,+ url: &str,+ ) -> impl Future<Output = Result<Option<RetrievedWebpage>>> + Send;++ fn get_homepage_descriptions(+ &self,+ urls: &[Url],+ ) -> impl Future<Output = HashMap<Url, String>> + Send;++ fn get_entity_image(+ &self,+ image_id: &str,+ max_height: Option<u64>,+ max_width: Option<u64>,+ ) -> impl Future<Output = Result<Option<Image>>> + Send;+}+#[derive(Clone, Debug)]pub struct ScoredWebpagePointer {pub website: RecallRankingWebpage,@@ -133,7 +166,7 @@ impl DistributedSearcher {{Ok(v) => v.into_iter()- .flat_map(|(_, v)| v)+ .flat_map(|(_, v)| v.into_iter().map(|(_, v)| v)).flatten().flatten().zip_eq(idxs)@@ -160,7 +193,7 @@ impl SearchClient for DistributedSearcher {.await{for (shard_id, mut res) in res {- if let Some(Some(res)) = res.pop() {+ if let Some((_, Some(res))) = res.pop() {results.push(InitialSearchResultShard {local_result: res,shard: shard_id,@@ -184,7 +217,7 @@ impl SearchClient for DistributedSearcher {pointers.entry(pointer.shard).or_default()- .push((*i, pointer.website.pointer.clone()));+ .push((*i, pointer.website.pointer().clone()));rankings.insert(*i, pointer.website.clone());}@@ -224,7 +257,12 @@ impl SearchClient for DistributedSearcher {.await.map_err(|_| Error::SearchFailed)?;- if let Some(res) = res.into_iter().flat_map(|(_, v)| v).flatten().next() {+ if let Some(res) = res+ .into_iter()+ .flat_map(|(_, v)| v.into_iter().map(|(_, v)| v))+ .flatten()+ .next()+ {Ok(Some(res))} else {Err(Error::WebpageNotFound.into())@@ -247,7 +285,10 @@ impl SearchClient for DistributedSearcher {match res {Ok(v) => v.into_iter()- .flat_map(|(_, v)| v.into_iter().map(|crate::bincode_utils::SerdeCompat(v)| v))+ .flat_map(|(_, v)| {+ v.into_iter()+ .map(|(_, crate::bincode_utils::SerdeCompat(v))| v)+ }).flatten().collect(),_ => HashMap::new(),@@ -274,7 +315,7 @@ impl SearchClient for DistributedSearcher {.await.map_err(|_| Error::SearchFailed)?.pop()- .flatten())+ .and_then(|(_, v)| v))}async fn search_entity(&self, query: &str) -> Option<EntityMatch> {@@ -290,38 +331,81 @@ impl SearchClient for DistributedSearcher {.await.ok()?.pop()- .flatten()+ .and_then(|(_, v)| v)}}-pub trait SearchClient {- fn search_initial(- &self,- query: &SearchQuery,- ) -> impl Future<Output = Vec<InitialSearchResultShard>> + Send;+/// This should only be used for testing and benchmarks.+pub struct LocalSearchClient(LocalSearcher<Index>);+impl From<LocalSearcher<Index>> for LocalSearchClient {+ fn from(searcher: LocalSearcher<Index>) -> Self {+ Self(searcher)+ }+}- fn retrieve_webpages(+impl SearchClient for LocalSearchClient {+ async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {+ let res = self.0.search_initial(query, true).unwrap();++ vec![InitialSearchResultShard {+ local_result: res,+ shard: ShardId::new(0),+ }]+ }++ async fn retrieve_webpages(&self,top_websites: &[(usize, ScoredWebpagePointer)],query: &str,- ) -> impl Future<Output = Vec<(usize, PrecisionRankingWebpage)>> + Send;+ ) -> Vec<(usize, PrecisionRankingWebpage)> {+ let pointers = top_websites+ .iter()+ .map(|(_, p)| p.website.pointer().clone())+ .collect::<Vec<_>>();++ let res = self+ .0+ .retrieve_websites(&pointers, query)+ .unwrap()+ .into_iter()+ .zip(top_websites.iter().map(|(i, p)| (*i, p.website.clone())))+ .map(|(ret, (i, ran))| (i, PrecisionRankingWebpage::new(ret, ran)))+ .collect::<Vec<_>>();++ res+ }- fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;+ async fn get_webpage(&self, url: &str) -> Result<Option<RetrievedWebpage>> {+ Ok(self.0.get_webpage(url))+ }- fn get_webpage(+ async fn get_homepage_descriptions(&self,- url: &str,- ) -> impl Future<Output = Result<Option<RetrievedWebpage>>> + Send;+ urls: &[url::Url],+ ) -> std::collections::HashMap<url::Url, String> {+ let mut res = std::collections::HashMap::new();++ for url in urls {+ if let Some(homepage) = self.0.get_homepage(url) {+ if let Some(desc) = homepage.description() {+ res.insert(url.clone(), desc.clone());+ }+ }+ }- fn get_homepage_descriptions(- &self,- urls: &[Url],- ) -> impl Future<Output = HashMap<Url, String>> + Send;+ res+ }- fn get_entity_image(+ async fn get_entity_image(&self,- image_id: &str,- max_height: Option<u64>,- max_width: Option<u64>,- ) -> impl Future<Output = Result<Option<Image>>> + Send;+ _image_id: &str,+ _max_height: Option<u64>,+ _max_width: Option<u64>,+ ) -> Result<Option<Image>> {+ Ok(None)+ }++ async fn search_entity(&self, _query: &str) -> Option<EntityMatch> {+ None+ }}diff --git a/crates/core/src/searcher/live.rs b/crates/core/src/searcher/live.rsindex 5d25be94..f31552b0 100644--- a/crates/core/src/searcher/live.rs+++ b/crates/core/src/searcher/live.rs@@ -103,7 +103,7 @@ impl LiveSearcher {{Ok(v) => v.into_iter()- .flat_map(|(_, v)| v)+ .flat_map(|(_, v)| v.into_iter().map(|(_, v)| v)).flatten().flatten().zip_eq(idxs)@@ -130,7 +130,7 @@ impl SearchClient for LiveSearcher {.await{for (shard_id, mut res) in res {- if let Some(Some(res)) = res.pop() {+ if let Some((_, Some(res))) = res.pop() {results.push(InitialSearchResultSplit {local_result: res,split_id: shard_id,@@ -154,7 +154,7 @@ impl SearchClient for LiveSearcher {pointers.entry(pointer.split_id.clone()).or_default()- .push((*i, pointer.website.pointer.clone()));+ .push((*i, pointer.website.pointer().clone()));rankings.insert(*i, pointer.website.clone());}diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex 3e8bd9b9..99934c28 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -25,14 +25,14 @@ use crate::index::Index;use crate::inverted_index::{InvertedIndex, RetrievedWebpage};use crate::models::dual_encoder::DualEncoder;use crate::query::Query;-use crate::ranking::inbound_similarity::InboundSimilarity;use crate::ranking::models::lambdamart::LambdaMART;use crate::ranking::models::linear::LinearRegression;-use crate::ranking::pipeline::{PrecisionRankingWebpage, RankingPipeline, RecallRankingWebpage};-use crate::ranking::{self, query_centrality, Ranker, SignalComputer, SignalEnum};+use crate::ranking::pipeline::{+ LocalRecallRankingWebpage, PrecisionRankingWebpage, RankingPipeline, RecallRankingWebpage,+};+use crate::ranking::{Ranker, SignalComputer, SignalEnum, SignalScore};use crate::search_ctx::Ctx;use crate::search_prettifier::DisplayedWebpage;-use crate::webgraph::Node;use crate::{inverted_index, live_index, Result};use super::WebsitesResult;@@ -102,7 +102,6 @@ impl<'a> SearchGuard<'a> for LiveIndexSearchGuard<'a> {pub struct LocalSearcher<I: SearchableIndex> {index: I,- inbound_similarity: Option<InboundSimilarity>,linear_regression: Option<Arc<LinearRegression>>,lambda_model: Option<Arc<LambdaMART>>,dual_encoder: Option<Arc<DualEncoder>>,@@ -119,7 +118,7 @@ where}struct InvertedIndexResult {- webpages: Vec<RecallRankingWebpage>,+ webpages: Vec<LocalRecallRankingWebpage>,num_hits: Option<usize>,has_more: bool,}@@ -131,7 +130,6 @@ wherepub fn new(index: I) -> Self {LocalSearcher {index,- inbound_similarity: None,linear_regression: None,lambda_model: None,dual_encoder: None,@@ -139,10 +137,6 @@ where}}- pub fn set_inbound_similarity(&mut self, inbound: InboundSimilarity) {- self.inbound_similarity = Some(inbound);- }-pub fn set_linear_model(&mut self, model: LinearRegression) {self.linear_regression = Some(Arc::new(model));}@@ -175,13 +169,10 @@ wherefn ranker<'a, G: SearchGuard<'a>>(&'a self,query: &Query,- ctx: &Ctx,guard: &G,de_rank_similar: bool,computer: SignalComputer,) -> Result<Ranker> {- let query_centrality_coeff = computer.coefficient(&ranking::signal::QueryCentrality.into());-let mut ranker = Ranker::new(computer,guard.inverted_index().fastfield_reader(),@@ -190,27 +181,6 @@ whereranker.de_rank_similar(de_rank_similar);- if query_centrality_coeff > 0.0 {- if let Some(inbound_sim) = self.inbound_similarity.as_ref() {- ranker = ranker- .with_max_docs(1_000, guard.inverted_index().num_segments())- .with_num_results(100);-- let top_host_nodes =- guard- .search_index()- .top_nodes(query, ctx, ranker.collector(ctx.clone()))?;-- if !top_host_nodes.is_empty() {- let inbound = inbound_sim.scorer(&top_host_nodes, &[], false);-- let query_centrality = query_centrality::Scorer::new(inbound);-- ranker.set_query_centrality(query_centrality);- }- }- }-Ok(ranker.with_max_docs(self.collector_config.max_docs_considered,@@ -228,8 +198,8 @@ wherede_rank_similar: bool,) -> Result<InvertedIndexResult> {let mut query = query.clone();- let pipeline: RankingPipeline<RecallRankingWebpage> =- RankingPipeline::<RecallRankingWebpage>::recall_stage(+ let pipeline: RankingPipeline<LocalRecallRankingWebpage> =+ RankingPipeline::<LocalRecallRankingWebpage>::recall_stage(&mut query,self.lambda_model.clone(),self.dual_encoder.clone(),@@ -240,27 +210,6 @@ wherelet mut computer = SignalComputer::new(Some(&parsed_query));- if let Some(inbound_sim) = &self.inbound_similarity {- let liked_hosts: Vec<_> = parsed_query- .host_rankings()- .liked- .iter()- .map(|site| Node::from(site.clone()).into_host())- .map(|node| node.id())- .collect();-- let disliked_hosts: Vec<_> = parsed_query- .host_rankings()- .disliked- .iter()- .map(|site| Node::from(site.clone()).into_host())- .map(|node| node.id())- .collect();-- let scorer = inbound_sim.scorer(&liked_hosts, &disliked_hosts, false);- computer.set_inbound_similarity(scorer);- }-computer.set_region_count(guard.search_index()@@ -274,7 +223,7 @@ wherecomputer.set_linear_model(model.clone());}- let ranker = self.ranker(&parsed_query, ctx, guard, de_rank_similar, computer)?;+ let ranker = self.ranker(&parsed_query, guard, de_rank_similar, computer)?;let res = guard.inverted_index().search_initial(&parsed_query,@@ -368,14 +317,17 @@ wherelet pointers: Vec<_> = search_result.websites.iter()- .map(|website| website.pointer.clone())+ .map(|website| website.pointer().clone()).collect();let websites: Vec<_> = self.retrieve_websites(&pointers, &query.query)?.into_iter().zip_eq(search_result.websites)- .map(|(webpage, ranking)| PrecisionRankingWebpage::new(webpage, ranking))+ .map(|(webpage, ranking)| {+ let ranking = RecallRankingWebpage::new(ranking, Default::default());+ PrecisionRankingWebpage::new(webpage, ranking)+ }).collect();let search_len = websites.len();@@ -386,11 +338,13 @@ wherelet pointers: Vec<_> = top_websites.iter()- .map(|website| website.ranking.pointer.clone())+ .map(|website| website.ranking().pointer().clone()).collect();let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;+ let coefficients = query.signal_coefficients();+let mut webpages: Vec<_> = retrieved_sites.into_iter().map(|webpage| DisplayedWebpage::new(webpage, query))@@ -400,8 +354,14 @@ wherelet mut ranking_signals = HashMap::new();for signal in SignalEnum::all() {- if let Some(score) = ranking.ranking.signals.get(signal) {- ranking_signals.insert(signal.into(), *score);+ if let Some(score) = ranking.ranking().signals().get(signal) {+ ranking_signals.insert(+ signal.into(),+ SignalScore {+ value: *score,+ coefficient: coefficients.get(&signal),+ },+ );}}diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex d985ff27..aec003cf 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -26,8 +26,11 @@ use optics::{HostRankings, Optic};use utoipa::ToSchema;use crate::{- api::search::ReturnBody, bangs::BangHit, config::defaults,- ranking::pipeline::RecallRankingWebpage, search_prettifier::DisplayedWebpage,+ api::search::ReturnBody,+ bangs::BangHit,+ config::defaults,+ ranking::{pipeline::LocalRecallRankingWebpage, SignalCoefficient},+ search_prettifier::DisplayedWebpage,webpage::region::Region,};@@ -39,6 +42,17 @@ pub enum SearchResult {Bang(Box<BangHit>),}+#[cfg(test)]+impl SearchResult {+ /// Panics if the result is not a `WebsitesResult`.+ pub fn into_websites_result(self) -> WebsitesResult {+ match self {+ Self::Websites(result) => result,+ _ => panic!("Expected WebsitesResult"),+ }+ }+}+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,)]@@ -67,7 +81,7 @@ pub struct SearchQuery {#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct InitialWebsiteResult {pub num_websites: Option<usize>,- pub websites: Vec<RecallRankingWebpage>,+ pub websites: Vec<LocalRecallRankingWebpage>,pub has_more: bool,}@@ -95,4 +109,28 @@ impl SearchQuery {pub fn is_empty(&self) -> bool {self.query.is_empty()}++ pub fn signal_coefficients(&self) -> SignalCoefficient {+ let mut signal_coefficients = SignalCoefficient::default();++ if let Some(optic) = &self.optic {+ signal_coefficients.merge_overwrite(SignalCoefficient::from_optic(optic));+ }++ signal_coefficients+ }++ pub fn host_rankings(&self) -> HostRankings {+ let mut rankings = HostRankings::empty();++ if let Some(host_rankings) = &self.host_rankings {+ rankings.merge_into(host_rankings.clone());+ }++ if let Some(optic) = &self.optic {+ rankings.merge_into(optic.host_rankings.clone());+ }++ rankings+ }}diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex c8760d3c..2f72e254 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -14,16 +14,17 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{cmp::Reverse, collections::BinaryHeap, sync::Arc};+use std::{cmp::Reverse, sync::Arc};-use fnv::{FnvHashMap, FnvHashSet};use hashbrown::HashSet;+use itertools::Itertools;use url::Url;use crate::{- ranking::inbound_similarity::InboundSimilarity,- webgraph::{Node, NodeID, Webgraph},+ ranking::{bitvec_similarity, inbound_similarity},+ webgraph::{remote::RemoteWebgraph, Node, NodeID},webpage::url_ext::UrlExt,+ SortableFloat,};#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]@@ -32,51 +33,24 @@ pub struct ScoredNode {pub score: f64,}-struct ScoredNodeID {- node_id: NodeID,- score: f64,-}--impl PartialOrd for ScoredNodeID {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- Some(self.cmp(other))- }-}--impl PartialEq for ScoredNodeID {- fn eq(&self, other: &Self) -> bool {- self.score == other.score- }-}--impl Ord for ScoredNodeID {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.score.total_cmp(&other.score)- }-}--impl Eq for ScoredNodeID {}-pub struct SimilarHostsFinder {- webgraph: Arc<Webgraph>,- inbound_similarity: InboundSimilarity,+ webgraph: Arc<RemoteWebgraph>,max_similar_hosts: usize,}impl SimilarHostsFinder {- pub fn new(- webgraph: Arc<Webgraph>,- inbound_similarity: InboundSimilarity,- max_similar_hosts: usize,- ) -> Self {+ pub fn new(webgraph: Arc<RemoteWebgraph>, max_similar_hosts: usize) -> Self {Self {webgraph,- inbound_similarity,max_similar_hosts,}}- pub fn find_similar_hosts(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {+ async fn scorer(&self, liked: &[NodeID]) -> inbound_similarity::Scorer {+ inbound_similarity::Scorer::new(&self.webgraph, liked, &[], true).await+ }++ pub async fn find_similar_hosts(&self, nodes: &[String], limit: usize) -> Vec<ScoredNode> {const DEAD_LINKS_BUFFER: usize = 30;let orig_limit = limit.min(self.max_similar_hosts);let limit = orig_limit + nodes.len() + DEAD_LINKS_BUFFER;@@ -97,70 +71,81 @@ impl SimilarHostsFinder {let nodes = nodes.into_iter().map(|node| node.id()).collect::<Vec<_>>();- let mut scorer = self.inbound_similarity.scorer(&nodes, &[], true);-- let mut backlinks = FnvHashMap::default();+ let mut scorer = self.scorer(&nodes).await;- for node in &nodes {- for edge in self.webgraph.raw_ingoing_edges(node) {- backlinks- .entry(edge.from)- .or_insert_with(|| scorer.score(&edge.from));- }- }+ let in_edges = self+ .webgraph+ .batch_raw_ingoing_edges(&nodes)+ .await+ .unwrap_or_default();- let mut top_backlink_nodes: Vec<_> = backlinks- .into_iter()- .filter(|(_, score)| score.is_finite())+ let backlink_nodes = in_edges+ .iter()+ .flatten()+ .map(|e| e.from)+ .unique()+ .collect::<Vec<_>>();++ let outgoing_edges = self+ .webgraph+ .batch_raw_outgoing_edges(&backlink_nodes)+ .await+ .unwrap_or_default();++ let potential_nodes: Vec<_> = outgoing_edges+ .iter()+ .flatten()+ .map(|e| e.to)+ .unique()+ .filter(|n| !nodes.contains(n)).collect();- top_backlink_nodes- .sort_unstable_by(|(_, score_a), (_, score_b)| score_a.total_cmp(score_b));- top_backlink_nodes.reverse();-- let mut scored_nodes = BinaryHeap::with_capacity(limit);- let mut checked_nodes = FnvHashSet::default();-- for (backlink_node, _) in top_backlink_nodes.into_iter().take(limit) {- for edge in self.webgraph.raw_outgoing_edges(&backlink_node) {- let potential_node = edge.to;-- if checked_nodes.contains(&potential_node) {- continue;- }-- checked_nodes.insert(potential_node);-- let score = scorer.score(&potential_node);- let scored_node_id = ScoredNodeID {- node_id: potential_node,- score,- };+ let inbounds = bitvec_similarity::BitVec::batch_new_for(&potential_nodes, &self.webgraph)+ .await+ .into_iter()+ .zip_eq(potential_nodes.into_iter())+ .map(|(b, n)| (n, b));++ let scored_nodes: Vec<_> = crate::sorted_k(+ inbounds.map(|(n, b)| {+ let score = scorer.score(&n, &b);+ Reverse((SortableFloat(score), n))+ }),+ limit,+ )+ .into_iter()+ .map(|Reverse((score, n))| (n, score))+ .collect();++ let potential_nodes = scored_nodes+ .iter()+ .map(|(node_id, _)| *node_id)+ .collect::<Vec<_>>();- if scored_nodes.len() < limit {- scored_nodes.push(Reverse(scored_node_id));- } else {- let mut min_scored_node = scored_nodes.peek_mut().unwrap();+ // remove dead links (nodes without outgoing edges might be dead links)+ let known_nodes = self+ .webgraph+ .batch_raw_ingoing_edges(&potential_nodes)+ .await+ .unwrap_or_default();- if scored_node_id > min_scored_node.0 {- *min_scored_node = Reverse(scored_node_id);- }- }- }- }+ let (potential_nodes, scores): (Vec<_>, Vec<_>) = scored_nodes+ .into_iter()+ .zip_eq(known_nodes)+ .filter_map(|(s, e)| if e.is_empty() { None } else { Some(s) })+ .unzip();- let mut scored_nodes: Vec<_> = scored_nodes.into_iter().take(limit).map(|n| n.0).collect();- scored_nodes.sort_unstable();- scored_nodes.reverse();+ let nodes = self+ .webgraph+ .batch_get_node(&potential_nodes)+ .await+ .unwrap_or_default();- scored_nodes+ nodes.into_iter()- .filter(|ScoredNodeID { node_id, score: _ }| {- // remove dead links (nodes without outgoing edges might be dead links)- !self.webgraph.raw_outgoing_edges(node_id).is_empty()- })- .filter_map(|ScoredNodeID { node_id, score }| {- let node = self.webgraph.id2node(&node_id).unwrap();+ .zip_eq(scores)+ .filter_map(|(node, SortableFloat(score))| {+ let node = node.unwrap();match Url::parse(&format!("http://{}", &node.as_str())).ok().and_then(|url| url.root_domain().map(|s| s.to_string()))@@ -178,8 +163,4 @@ impl SimilarHostsFinder {.take(orig_limit).collect()}-- pub fn knows_about(&self, node: &Node) -> bool {- self.inbound_similarity.knows_about(node.id())- }}diff --git a/crates/core/src/webgraph/centrality/mod.rs b/crates/core/src/webgraph/centrality/mod.rsindex f71c15ef..bb8e088e 100644--- a/crates/core/src/webgraph/centrality/mod.rs+++ b/crates/core/src/webgraph/centrality/mod.rs@@ -31,63 +31,6 @@ pub enum TopNodes {Fraction(f64),}-trait TopKOrderable: Ord {- type SortKey: Ord + Copy;-- fn sort_key(&self) -> Self::SortKey;-}--impl TopKOrderable for (SortableFloat, NodeID) {- type SortKey = SortableFloat;-- fn sort_key(&self) -> Self::SortKey {- self.0- }-}--impl<T> TopKOrderable for Reverse<T>-where- T: TopKOrderable,-{- type SortKey = Reverse<T::SortKey>;-- fn sort_key(&self) -> Self::SortKey {- Reverse(self.0.sort_key())- }-}--/// Source (and explanation): https://quickwit.io/blog/top-k-complexity-fn top_k<T>(mut hits: impl Iterator<Item = T>, k: usize) -> Vec<T>-where- T: TopKOrderable,-{- if k == 0 {- return Vec::new();- }-- let mut top_k = Vec::with_capacity(2 * k);- top_k.extend((&mut hits).take(k));-- let mut threshold = None;- for hit in hits {- if let Some(threshold) = threshold {- if hit.sort_key() <= threshold {- continue;- }- }- top_k.push(hit);- if top_k.len() == 2 * k {- // The standard library does all of the heavy lifting here.- let (_, median_el, _) = top_k.select_nth_unstable(k - 1);- threshold = Some(median_el.sort_key());- top_k.truncate(k);- }- }- top_k.sort_unstable();- top_k.truncate(k);- top_k-}-pub fn top_nodes(host_centrality: &speedy_kv::Db<NodeID, f64>,top: TopNodes,@@ -97,7 +40,7 @@ pub fn top_nodes(TopNodes::Fraction(frac) => (host_centrality.len() as f64 * frac) as usize,};- top_k(+ crate::sorted_k(host_centrality.iter().map(|(id, centrality)| (SortableFloat(centrality), id))@@ -190,7 +133,7 @@ mod tests {(SortableFloat(9.0), NodeID::from(9 as u64)),];- let top_5 = top_k(hits.iter().copied(), 5);+ let top_5 = crate::sorted_k(hits.iter().copied(), 5);assert_eq!(top_5,vec![@@ -202,7 +145,7 @@ mod tests {]);- let top_3 = top_k(hits.iter().copied(), 3);+ let top_3 = crate::sorted_k(hits.iter().copied(), 3);assert_eq!(top_3,vec![@@ -212,7 +155,7 @@ mod tests {]);- let top_0 = top_k(hits.iter().copied(), 0);+ let top_0 = crate::sorted_k(hits.iter().copied(), 0);assert_eq!(top_0, Vec::<(SortableFloat, NodeID)>::new());}@@ -231,7 +174,7 @@ mod tests {(SortableFloat(0.0), NodeID::from(0 as u64)),];- let top_5 = top_k(hits.iter().copied().map(Reverse), 5)+ let top_5 = crate::sorted_k(hits.iter().copied().map(Reverse), 5).into_iter().map(|Reverse(x)| x).collect::<Vec<_>>();@@ -247,7 +190,7 @@ mod tests {]);- let top_3 = top_k(hits.iter().copied().map(Reverse), 3)+ let top_3 = crate::sorted_k(hits.iter().copied().map(Reverse), 3).into_iter().map(|Reverse(x)| x).collect::<Vec<_>>();diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex 0557e551..4af8d194 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -48,7 +48,7 @@ impl EdgeLabel for () {}}-#[derive(Debug, Clone, PartialEq, Eq, Hash)]+#[derive(Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode)]pub struct Edge<L>whereL: EdgeLabel,diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex c3b98137..93caeabb 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -40,6 +40,7 @@ mod compression;mod edge;mod id_node_db;mod node;+pub mod remote;mod segment;mod shortest_path;mod store;@@ -229,6 +230,15 @@ impl Webgraph {self.inner_edges(|segment| segment.ingoing_edges_with_label(node), dedup)}+ pub fn raw_outgoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {+ let dedup = |edges: &mut Vec<Edge<String>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);+ };++ self.inner_edges(|segment| segment.outgoing_edges_with_label(node), dedup)+ }+pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {let dedup = |edges: &mut Vec<Edge<String>>| {edges.sort_by_key(|e| e.to);diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsnew file mode 100644index 00000000..efc53538--- /dev/null+++ b/crates/core/src/webgraph/remote.rs@@ -0,0 +1,356 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::sync::Arc;++use itertools::Itertools;+use url::Url;++use crate::{+ config::WebgraphGranularity,+ distributed::{+ cluster::Cluster,+ member::{Service, ShardId},+ sonic::{+ self,+ replication::{+ AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient,+ },+ },+ },+ entrypoint::webgraph_server::{+ GetNode, IngoingEdges, OutgoingEdges, RawIngoingEdges, RawIngoingEdgesWithLabels,+ RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,+ },+ Result,+};++use super::{Edge, FullEdge, Node, NodeID};++pub struct RemoteWebgraph {+ granularity: WebgraphGranularity,+ cluster: Arc<Cluster>,+}++impl RemoteWebgraph {+ pub fn new(cluster: Arc<Cluster>, granularity: WebgraphGranularity) -> Self {+ Self {+ cluster,+ granularity,+ }+ }++ async fn conn(&self) -> sonic::replication::ShardedClient<WebGraphService, ShardId> {+ let shards = self+ .cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::Webgraph {+ host,+ shard,+ granularity,+ } = member.service+ {+ if granularity == self.granularity {+ Some((shard, RemoteClient::<WebGraphService>::new(host)))+ } else {+ None+ }+ } else {+ None+ }+ })+ .into_group_map();++ let shards: Vec<_> = shards+ .into_iter()+ .map(|(shard, clients)| {+ let replica = ReplicatedClient::new(clients);+ sonic::replication::Shard::new(shard, replica)+ })+ .collect();++ sonic::replication::ShardedClient::new(shards)+ }++ pub async fn knows(&self, mut host: String) -> Result<Option<Node>> {+ if let Some(suf) = host.strip_prefix("http://") {+ host = suf.to_string();+ }+ if let Some(suf) = host.strip_prefix("https://") {+ host = suf.to_string();+ }++ let url = Url::parse(&("http://".to_string() + host.as_str()))?;+ let node = Node::from(url).into_host();+ let id = node.id();+ let edges = self.raw_ingoing_edges(id).await?;++ if !edges.is_empty() {+ Ok(Some(node))+ } else {+ Ok(None)+ }+ }++ pub async fn get_node(&self, id: NodeID) -> Result<Option<Node>> {+ let res = self+ .conn()+ .await+ .send(+ &GetNode { node: id },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, res)| res.into_iter().map(|(_, v)| v))+ .find(|n| n.is_some())+ .flatten()+ .clone())+ }++ pub async fn batch_get_node(&self, ids: &[NodeID]) -> Result<Vec<Option<Node>>> {+ let reqs = ids.iter().map(|&id| GetNode { node: id }).collect_vec();++ let res = self+ .conn()+ .await+ .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ let mut nodes = vec![None; ids.len()];++ for (_, rep) in res {+ debug_assert!(rep.len() <= 1);++ for (_, rep_nodes) in rep {+ for (i, node) in rep_nodes.into_iter().enumerate() {+ if let Some(node) = node {+ nodes[i] = Some(node);+ }+ }+ }+ }++ Ok(nodes)+ }++ pub async fn ingoing_edges(&self, node: Node) -> Result<Vec<FullEdge>> {+ let res = self+ .conn()+ .await+ .send(+ &IngoingEdges { node },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, reps)| {+ debug_assert!(reps.len() <= 1);+ reps.into_iter().flat_map(|(_, rep)| rep)+ })+ .collect())+ }++ pub async fn raw_ingoing_edges(&self, id: NodeID) -> Result<Vec<Edge<()>>> {+ let res = self+ .conn()+ .await+ .send(+ &RawIngoingEdges { node: id },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, reps)| {+ debug_assert!(reps.len() <= 1);+ reps.into_iter().flat_map(|(_, rep)| rep)+ })+ .collect())+ }++ pub async fn raw_ingoing_edges_with_labels(&self, id: NodeID) -> Result<Vec<Edge<String>>> {+ let res = self+ .conn()+ .await+ .send(+ &RawIngoingEdgesWithLabels { node: id },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, reps)| {+ debug_assert!(reps.len() <= 1);+ reps.into_iter().flat_map(|(_, rep)| rep)+ })+ .collect())+ }++ pub async fn batch_raw_ingoing_edges_with_labels(+ &self,+ ids: &[NodeID],+ ) -> Result<Vec<Vec<Edge<String>>>> {+ let reqs: Vec<_> = ids+ .iter()+ .map(|id| RawIngoingEdgesWithLabels { node: *id })+ .collect();++ let res = self+ .conn()+ .await+ .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ let mut edges = vec![vec![]; ids.len()];++ for (_, res) in res {+ debug_assert!(res.len() <= 1);++ for (_, res) in res {+ for (i, rep) in res.into_iter().enumerate() {+ edges[i].extend(rep);+ }+ }+ }++ Ok(edges)+ }++ pub async fn batch_raw_ingoing_edges(&self, ids: &[NodeID]) -> Result<Vec<Vec<Edge<()>>>> {+ let reqs: Vec<_> = ids.iter().map(|id| RawIngoingEdges { node: *id }).collect();++ let res = self+ .conn()+ .await+ .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ let mut edges = vec![vec![]; ids.len()];++ for (_, res) in res {+ debug_assert!(res.len() <= 1);++ for (_, res) in res {+ for (i, rep) in res.into_iter().enumerate() {+ edges[i].extend(rep);+ }+ }+ }++ Ok(edges)+ }++ pub async fn outgoing_edges(&self, node: Node) -> Result<Vec<FullEdge>> {+ let res = self+ .conn()+ .await+ .send(+ &OutgoingEdges { node },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, reps)| {+ debug_assert!(reps.len() <= 1);+ reps.into_iter().flat_map(|(_, rep)| rep)+ })+ .collect())+ }++ pub async fn raw_outgoing_edges(&self, id: NodeID) -> Result<Vec<Edge<()>>> {+ let res = self+ .conn()+ .await+ .send(+ &RawOutgoingEdges { node: id },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, reps)| {+ debug_assert!(reps.len() <= 1);+ reps.into_iter().flat_map(|(_, rep)| rep)+ })+ .collect())+ }++ pub async fn raw_outgoing_edges_with_labels(&self, id: NodeID) -> Result<Vec<Edge<String>>> {+ let res = self+ .conn()+ .await+ .send(+ &RawOutgoingEdgesWithLabels { node: id },+ &AllShardsSelector,+ &RandomReplicaSelector,+ )+ .await?;++ Ok(res+ .into_iter()+ .flat_map(|(_, reps)| {+ debug_assert!(reps.len() <= 1);+ reps.into_iter().flat_map(|(_, rep)| rep)+ })+ .collect())+ }++ pub async fn batch_raw_outgoing_edges(&self, ids: &[NodeID]) -> Result<Vec<Vec<Edge<()>>>> {+ let reqs: Vec<_> = ids+ .iter()+ .map(|id| RawOutgoingEdges { node: *id })+ .collect();++ let res = self+ .conn()+ .await+ .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)+ .await?;++ let mut edges = vec![vec![]; ids.len()];++ for (_, res) in res {+ debug_assert!(res.len() <= 1);++ for (_, res) in res {+ for (i, rep) in res.into_iter().enumerate() {+ edges[i].extend(rep);+ }+ }+ }++ Ok(edges)+ }+}diff --git a/crates/core/src/webpage/schema_org/mod.rs b/crates/core/src/webpage/schema_org/mod.rsindex 08252376..51b0e952 100644--- a/crates/core/src/webpage/schema_org/mod.rs+++ b/crates/core/src/webpage/schema_org/mod.rs@@ -19,7 +19,7 @@ use std::collections::HashMap;use kuchiki::NodeRef;use crate::tokenizer::FlattenedJson;-use crate::Result;+use crate::{OneOrMany, Result};mod json_ld;mod microdata;@@ -196,38 +196,6 @@ impl From<RawItem> for Item {}}-#[derive(- serde::Serialize,- serde::Deserialize,- bincode::Encode,- bincode::Decode,- PartialEq,- Eq,- Debug,- Clone,- Hash,-)]-pub enum OneOrMany<T: 'static> {- One(T),- Many(Vec<T>),-}--impl<T> OneOrMany<T> {- pub fn one(self) -> Option<T> {- match self {- OneOrMany::One(one) => Some(one),- OneOrMany::Many(many) => many.into_iter().next(),- }- }-- pub fn many(self) -> Vec<T> {- match self {- OneOrMany::One(one) => vec![one],- OneOrMany::Many(many) => many,- }- }-}-#[derive(serde::Serialize,serde::Deserialize,@@ -240,7 +208,7 @@ impl<T> OneOrMany<T> {Hash,)]#[serde(untagged)]-enum RawOneOrMany<T: 'static> {+enum RawOneOrMany<T> {One(T),Many(Vec<T>),}diff --git a/crates/optics/src/lib.rs b/crates/optics/src/lib.rsindex 923837ff..1745fe94 100644--- a/crates/optics/src/lib.rs+++ b/crates/optics/src/lib.rs@@ -536,6 +536,14 @@ impl Display for HostRankings {}impl HostRankings {+ pub fn empty() -> Self {+ Self {+ liked: Vec::new(),+ disliked: Vec::new(),+ blocked: Vec::new(),+ }+ }+#[must_use]pub fn rules(&self) -> Rule {let matches: Vec<_> = selfdiff --git a/frontend/src/routes/explore/+page.svelte b/frontend/src/routes/explore/+page.svelteindex 289f416c..384cfb9b 100644--- a/frontend/src/routes/explore/+page.svelte+++ b/frontend/src/routes/explore/+page.svelte@@ -29,9 +29,9 @@$: chosenHostString = chosenHosts.join(',');$: {- api- .webgraphHostSimilar({ hosts: chosenHosts, topN: limit })- .data.then((res) => (similarHosts = res));+ api.webgraphHostSimilar({ hosts: chosenHosts, topN: limit }).data.then((res) => {+ similarHosts = res;+ });}$: {