web search done right
chitchat dynamic cluster membership
Mikkel Denker2151fcfca828
diff --git a/Cargo.lock b/Cargo.lockindex 80d3dcce..9ba03511 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -489,6 +489,22 @@ version = "1.0.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"+[[package]]+name = "chitchat"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fec1bff4fabc59f0b7408478183a796ab0a9bcb0534bb413f3cf66f44ce2aca1"+dependencies = [+ "anyhow",+ "async-trait",+ "bytes",+ "rand 0.8.5",+ "serde",+ "tokio",+ "tokio-stream",+ "tracing",+]+[[package]]name = "chrono"version = "0.4.23"@@ -808,6 +824,7 @@ dependencies = ["bitvec","byteorder","bzip2",+ "chitchat","chrono","clap 3.2.23","criterion",@@ -3878,6 +3895,7 @@ dependencies = ["futures-core","pin-project-lite","tokio",+ "tokio-util",][[package]]diff --git a/Cargo.toml b/Cargo.tomlindex 48fd28e8..7c634f79 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -69,4 +69,5 @@ tokenizers = "0.13.2"byteorder = "1.4.3"async-stream = "0.3.3"tokio-stream = "0.1.11"-scylla = "0.7.0"\ No newline at end of file+scylla = "0.7.0"+chitchat = "0.5.0"\ No newline at end of filediff --git a/configs/frontend.toml b/configs/frontend.tomlindex 54654a16..47f15ee4 100644--- a/configs/frontend.toml+++ b/configs/frontend.toml@@ -1,7 +1,10 @@+cluster_id = "dev_frontend"queries_csv_path = "data/queries_us.csv"host = "0.0.0.0:3000"prometheus_host = "0.0.0.0:3001"search_servers = [["0.0.0.0:3002"]]+gossip_addr = "0.0.0.0:3003"+gossip_seed_nodes = ["0.0.0.0:3004"]crossencoder_model_path = "data/cross_encoder"# qa_model_path = "data/qa_model"bangs_path = "data/bangs.json"diff --git a/configs/search_server.toml b/configs/search_server.tomlindex 68d86a95..74c01694 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -1,3 +1,6 @@+cluster_id = "dev_search"+gossip_addr = "0.0.0.0:3004"+shard_id = 0host = "0.0.0.0:3002"index_path = "data/index"entity_index_path = "data/entity"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex 8abd892b..2594f324 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -74,6 +74,7 @@ byteorder = { workspace = true }async-stream = { workspace = true }tokio-stream = { workspace = true }scylla = { workspace = true }+chitchat = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex ab73c816..53e85274 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -21,11 +21,15 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};use crate::{autosuggest::Autosuggest,bangs::Bangs,+ cluster::{+ member::{Member, Service},+ Cluster,+ },leaky_queue::LeakyQueue,qa_model::QaModel,query_store::{self, ImprovementEvent},ranking::models::cross_encoder::CrossEncoderModel,- searcher::{DistributedSearcher, Shard},+ searcher::DistributedSearcher,summarizer::Summarizer,FrontendConfig,};@@ -89,19 +93,11 @@ pub async fn favicon() -> impl IntoResponse {.unwrap()}-pub fn router(+pub async fn router(config: &FrontendConfig,search_counter_success: crate::metrics::Counter,search_counter_fail: crate::metrics::Counter,) -> Result<Router> {- let shards: Vec<_> = config- .search_servers- .clone()- .into_iter()- .enumerate()- .map(|(id, replicas)| Shard::new(id as u64, replicas))- .collect();-let autosuggest = Autosuggest::load_csv(&config.queries_csv_path)?;let crossencoder = CrossEncoderModel::open(&config.crossencoder_model_path)?;@@ -120,7 +116,17 @@ pub fn router(});let bangs = Bangs::from_path(&config.bangs_path);- let searcher = DistributedSearcher::new(shards, crossencoder, qa_model, bangs);++ let cluster = Cluster::join(+ Member {+ id: config.cluster_id.clone(),+ service: Service::Frontend { host: config.host },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.clone().unwrap_or_default(),+ )+ .await?;+ let searcher = DistributedSearcher::new(cluster, crossencoder, qa_model, bangs);let state = Arc::new(State {searcher,diff --git a/core/src/cluster/member.rs b/core/src/cluster/member.rsnew file mode 100644index 00000000..45ddbe8f--- /dev/null+++ b/core/src/cluster/member.rs@@ -0,0 +1,33 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use serde::{Deserialize, Serialize};++use crate::searcher::ShardId;++#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]+pub enum Service {+ Searcher { host: SocketAddr, shard: ShardId },+ Frontend { host: SocketAddr },+}++#[derive(PartialEq, Eq, Hash, Clone, Debug)]+pub struct Member {+ pub id: String,+ pub service: Service,+}diff --git a/core/src/cluster/mod.rs b/core/src/cluster/mod.rsnew file mode 100644index 00000000..e3ff824f--- /dev/null+++ b/core/src/cluster/mod.rs@@ -0,0 +1,154 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use chitchat::{+ spawn_chitchat, transport::UdpTransport, ChitchatConfig, ChitchatHandle, FailureDetectorConfig,+ NodeId,+};+use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration};+use tokio::sync::RwLock;+use tokio_stream::StreamExt;+use tracing::error;++use self::member::{Member, Service};++const CLUSTER_ID: &str = "cuely-cluster";+const GOSSIP_INTERVAL: Duration = Duration::from_secs(1);+const SERVICE_KEY: &str = "service";++pub mod member;++#[derive(Debug, thiserror::Error)]+pub enum Error {+ #[error("Anyhow")]+ Anyhow(#[from] anyhow::Error),++ #[error("serde json")]+ SerdeJson(#[from] serde_json::Error),+}++type Result<T> = std::result::Result<T, Error>;++pub struct Cluster {+ alive_nodes: Arc<RwLock<HashSet<Member>>>,+ // dropping the handle leaves the cluster+ _chitchat_handle: ChitchatHandle,+}++impl Cluster {+ pub async fn join(+ self_node: Member,+ gossip_addr: SocketAddr,+ seed_addrs: Vec<SocketAddr>,+ ) -> Result<Self> {+ let transport = UdpTransport;+ let failure_detector_config = FailureDetectorConfig {+ phi_threshold: 10.0, // TODO: choose this value based on tests+ initial_interval: GOSSIP_INTERVAL,+ ..Default::default()+ };++ let node_id = NodeId {+ id: self_node.id.clone(),+ gossip_public_address: gossip_addr,+ };++ let config = ChitchatConfig {+ node_id,+ cluster_id: CLUSTER_ID.to_string(),+ gossip_interval: GOSSIP_INTERVAL,+ listen_addr: gossip_addr,+ seed_nodes: seed_addrs+ .into_iter()+ .map(|addr| addr.to_string())+ .collect(),+ failure_detector_config,+ is_ready_predicate: None,+ };++ let chitchat_handle = spawn_chitchat(+ config,+ vec![(+ SERVICE_KEY.to_string(),+ serde_json::to_string(&self_node.service)?,+ )],+ &transport,+ )+ .await?;+ let chitchat = chitchat_handle.chitchat();++ let alive_nodes = Arc::new(RwLock::new(HashSet::new()));+ let alive_nodes_ref = alive_nodes.clone();++ tokio::spawn(async move {+ let mut node_change_receiver = chitchat.lock().await.ready_nodes_watcher();++ while let Some(members_set) = node_change_receiver.next().await {+ let alive_node_ids: HashSet<_> = alive_nodes_ref+ .read()+ .await+ .iter()+ .map(|member: &Member| member.id.clone())+ .collect();++ let member_ids: HashSet<_> =+ members_set.iter().map(|member| member.id.clone()).collect();++ if alive_node_ids != member_ids {+ let snapshot = chitchat.lock().await.state_snapshot();+ let mut new_members = Vec::new();+ for member in members_set {+ if let Some(state) = snapshot.node_states.get(&member.id) {+ if let Some(service) = state.get(SERVICE_KEY) {+ let service: Service = serde_json::from_str(service).unwrap();+ new_members.push(Member {+ service,+ id: member.id,+ });+ } else {+ error!("failed to get service");+ }+ } else {+ error!("no state found for node")+ }+ }++ let mut write = alive_nodes_ref.write().await;+ write.clear();++ for member in new_members {+ write.insert(member);+ }+ }+ }+ });++ Ok(Self {+ alive_nodes,+ _chitchat_handle: chitchat_handle,+ })+ }++ pub async fn members(&self) -> Vec<Member> {+ let lock = self.alive_nodes.read().await;+ let mut res = Vec::with_capacity(lock.len());++ for member in lock.iter() {+ res.push(member.clone());+ }++ res+ }+}diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rsindex 5a860d0c..0a278eee 100644--- a/core/src/entrypoint/frontend.rs+++ b/core/src/entrypoint/frontend.rs@@ -49,14 +49,14 @@ pub async fn run(config: FrontendConfig) -> Result<()> {}],);- let app = router(&config, search_counter_success, search_counter_fail)?;+ let app = router(&config, search_counter_success, search_counter_fail).await?;let metrics_app = metrics_router(registry);- let addr = config.host.parse()?;+ let addr = config.host;tracing::info!("frontend server listening on {}", addr);let server = axum::Server::bind(&addr).serve(app.into_make_service());- let addr = config.prometheus_host.parse()?;+ let addr = config.prometheus_host;tracing::info!("prometheus exporter listening on {}", addr);let metrics_server = axum::Server::bind(&addr).serve(metrics_app.into_make_service());diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex ba10baf4..84d69820 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -17,6 +17,10 @@use std::net::SocketAddr;use crate::{+ cluster::{+ member::{Member, Service},+ Cluster,+ },entity_index::EntityIndex,index::Index,inverted_index,@@ -26,7 +30,7 @@ use crate::{};pub async fn run(config: SearchServerConfig) -> Result<()> {- let addr: SocketAddr = config.host.parse().unwrap();+ let addr: SocketAddr = config.host;let server = sonic::Server::bind(addr).await.unwrap();tracing::info!("listening on {}", addr);@@ -48,6 +52,20 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {local_searcher.set_centrality_store(centrality_store);}+ // dropping the handle leaves the cluster+ let _cluster_handle = Cluster::join(+ Member {+ id: config.cluster_id,+ service: Service::Searcher {+ host: config.host,+ shard: config.shard_id,+ },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.unwrap_or_default(),+ )+ .await?;+loop {if let Ok(req) = server.accept::<searcher::distributed::Request>().await {match &req.body {diff --git a/core/src/lib.rs b/core/src/lib.rsindex 33229cef..a84f82b7 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -18,10 +18,11 @@#![allow(clippy::cast_precision_loss)]#![allow(clippy::missing_errors_doc)]-use searcher::distributed;+use searcher::{distributed, ShardId};use serde::{Deserialize, Serialize};use std::fs::File;use std::io::{self, BufRead};+use std::net::SocketAddr;use std::num::ParseIntError;use std::path::PathBuf;use tantivy::TantivyError;@@ -35,6 +36,7 @@ pub mod mapreduce;mod api;mod autosuggest;mod bangs;+mod cluster;mod collector;mod directory;mod entity_index;@@ -162,23 +164,30 @@ pub struct HttpConfig {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct FrontendConfig {+ pub cluster_id: String,pub queries_csv_path: String,- pub host: String,- pub prometheus_host: String,+ pub host: SocketAddr,+ pub prometheus_host: SocketAddr,pub crossencoder_model_path: String,pub qa_model_path: Option<String>,pub bangs_path: String,pub summarizer_path: String,pub search_servers: Vec<Vec<String>>,pub query_store_db_host: Option<String>,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,}#[derive(Debug, Serialize, Deserialize, Clone)]pub struct SearchServerConfig {+ pub cluster_id: String,+ pub gossip_seed_nodes: Option<Vec<SocketAddr>>,+ pub gossip_addr: SocketAddr,+ pub shard_id: ShardId,pub index_path: String,pub entity_index_path: Option<String>,pub centrality_store_path: Option<String>,- pub host: String,+ pub host: SocketAddr,}#[derive(Error, Debug)]@@ -272,6 +281,9 @@ pub enum Error {#[error("Distributed searcher")]DistributedSearcher(#[from] distributed::Error),++ #[error("Cluster")]+ Cluster(#[from] cluster::Error),}pub(crate) type Result<T> = std::result::Result<T, Error>;diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex f29b1c5f..5bb6c4cc 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -17,6 +17,7 @@use crate::{bangs::{Bang, BangHit, Bangs},ceil_char_boundary,+ cluster::{member::Service, Cluster},collector::{self, BucketCollector},exponential_backoff::ExponentialBackoff,floor_char_boundary,@@ -38,6 +39,7 @@ use crate::{use std::{cmp::Ordering,+ collections::HashMap,net::SocketAddr,ops::Range,sync::Arc,@@ -140,8 +142,8 @@ impl RemoteSearcher {}}-#[derive(Debug, Clone, PartialEq, Eq)]-struct ShardId(u64);+#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]+pub struct ShardId(u64);pub struct Shard {id: ShardId,@@ -233,7 +235,7 @@ pub enum Request {}pub struct DistributedSearcher {- shards: Vec<Shard>,+ cluster: Cluster,cross_encoder: Arc<CrossEncoderModel>,qa_model: Option<Arc<QaModel>>,bangs: Bangs,@@ -271,19 +273,39 @@ impl collector::Doc for ScoredWebsitePointer {impl DistributedSearcher {pub fn new(- shards: Vec<Shard>,+ cluster: Cluster,model: CrossEncoderModel,qa_model: Option<QaModel>,bangs: Bangs,) -> Self {Self {- shards,+ cluster,cross_encoder: Arc::new(model),qa_model: qa_model.map(Arc::new),bangs,}}+ async fn shards(&self) -> Vec<Shard> {+ let mut shards = HashMap::new();+ for member in self.cluster.members().await {+ if let Service::Searcher { host, shard } = member.service {+ shards.entry(shard).or_insert_with(Vec::new).push(host);+ }+ }++ shards+ .into_iter()+ .map(|(shard, replicas)| Shard {+ id: shard,+ replicas: replicas+ .into_iter()+ .map(|addr| RemoteSearcher { addr })+ .collect(),+ })+ .collect()+ }+fn combine_results(&self,initial_results: Vec<InitialSearchResultShard>,@@ -322,7 +344,7 @@ impl DistributedSearcher {retrieved_webpages.push(None);}- for shard in &self.shards {+ for shard in self.shards().await.iter() {let (indexes, pointers): (Vec<_>, Vec<_>) = top_websites.iter().enumerate()@@ -345,7 +367,8 @@ impl DistributedSearcher {}async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {- self.shards+ self.shards()+ .await.iter().map(|shard| shard.search(query)).collect::<FuturesUnordered<_>>()@@ -625,7 +648,8 @@ impl DistributedSearcher {}pub(crate) async fn get_webpage(&self, url: &str) -> Result<RetrievedWebpage> {- self.shards+ self.shards()+ .await.iter().map(|shard| shard.get_webpage(url)).collect::<FuturesUnordered<_>>()diff --git a/scripts/run_frontend.sh b/scripts/run_frontend.shindex 985d844f..eb3aab55 100644--- a/scripts/run_frontend.sh+++ b/scripts/run_frontend.sh@@ -1,4 +1,4 @@#!/bin/bashtrap 'kill $(jobs -p)' EXIT-cargo run $1 -- search-server configs/search_server.toml &-cargo run $1 -- frontend configs/frontend.toml+cargo run $1 -- frontend configs/frontend.toml &+cargo run $1 -- search-server configs/search_server.toml