web search done right
use specific shard_ids for search servers (backbone and live) so we can potentially simply treat the live index as extra shards and re-use all merge logic etc. for searches
Mikkel Denker3459405b458a
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rsindex 6d4340cb..aae6fdf5 100644--- a/crates/core/src/collector/top_docs.rs+++ b/crates/core/src/collector/top_docs.rs@@ -25,9 +25,8 @@ use tantivy::{};use crate::{- ampc::dht::ShardId,config::CollectorConfig,- inverted_index::{DocAddress, WebpagePointer},+ inverted_index::{DocAddress, ShardId, WebpagePointer},numericalfield_reader,prehashed::Prehashed,ranking::initial::{InitialScoreTweaker, Score},@@ -503,7 +502,7 @@ mod tests {id: doc.1,score: Score { total: doc.2 },segment: 0,- shard_id: ShardId::new(0),+ shard_id: ShardId::Backbone(0),});}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex f5fd50aa..66b2dd05 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -339,7 +339,7 @@ impl Default for SnippetConfig {pub struct SearchServerConfig {pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,- pub shard: ShardId,+ pub shard: u64,pub index_path: String,pub linear_model_path: Option<String>,pub dual_encoder_model_path: Option<String>,@@ -608,7 +608,7 @@ pub struct LiveIndexConfig {pub minimum_clean_words: Option<usize>,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,- pub shard_id: ShardId,+ pub shard_id: u64,pub index_path: String,pub linear_model_path: Option<String>,pub lambda_model_path: Option<String>,diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex 45472fa9..28fbdd9d 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -99,14 +99,14 @@ impl std::fmt::Display for LiveIndexState {pub enum Service {Searcher {host: SocketAddr,- shard: ShardId,+ shard: crate::inverted_index::ShardId,},EntitySearcher {host: SocketAddr,},LiveIndex {host: SocketAddr,- shard: ShardId,+ shard: crate::inverted_index::ShardId,state: LiveIndexState,},Api {diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsindex 936f52c8..2bd2520e 100644--- a/crates/core/src/entrypoint/live_index/search_server.rs+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -22,7 +22,6 @@ use std::{};use crate::{- ampc::dht::ShardId,config::LiveIndexConfig,distributed::{cluster::Cluster,@@ -30,7 +29,7 @@ use crate::{remote_cp,sonic::{self, service::sonic_service},},- inverted_index,+ inverted_index::{self, ShardId},live_index::{IndexManager, LiveIndex},searcher::{InitialWebsiteResult, LocalSearcher},};@@ -190,7 +189,7 @@ impl LiveIndexService {Cluster::join(Member::new(Service::LiveIndex {host: config.host,- shard: config.shard_id,+ shard: ShardId::Live(config.shard_id),state: crate::distributed::member::LiveIndexState::InSetup,}),config.gossip_addr,diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rsindex 2ff1d4a0..4d6c863f 100644--- a/crates/core/src/entrypoint/live_index/tests.rs+++ b/crates/core/src/entrypoint/live_index/tests.rs@@ -17,7 +17,7 @@use crate::{config::LiveIndexConfig,entrypoint::search_server,- inverted_index,+ inverted_index::{self, ShardId},live_index::LiveIndex,searcher::{LocalSearcher, SearchQuery},Result,@@ -27,7 +27,6 @@ use std::{net::SocketAddr, path::Path, sync::Arc};use file_store::{gen_temp_dir, temp::TempDir};use crate::{- ampc::dht::ShardId,distributed::{cluster::Cluster,member::{LiveIndexState, Service},@@ -53,7 +52,7 @@ fn config<P: AsRef<Path>>(path: P) -> LiveIndexConfig {minimum_clean_words: None,gossip_seed_nodes: None,gossip_addr: free_socket_addr(),- shard_id: ShardId::new(0),+ shard_id: 0,index_path: path.as_ref().join("index").to_str().unwrap().to_string(),linear_model_path: None,lambda_model_path: None,@@ -73,7 +72,7 @@ struct RemoteIndex {}impl RemoteIndex {- async fn start(shard: ShardId, gossip_seed: Vec<SocketAddr>) -> Result<Self> {+ async fn start(shard: u64, gossip_seed: Vec<SocketAddr>) -> Result<Self> {let dir = gen_temp_dir()?;let mut config = config(&dir);@@ -103,7 +102,7 @@ impl RemoteIndex {Ok(Self {host,- shard,+ shard: ShardId::Live(shard),gossip_addr,underlying_index: index,cluster,@@ -183,8 +182,8 @@ impl RemoteIndex {#[tokio::test]async fn test_shard_without_replica() -> Result<()> {- let shard1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;- let shard2 = RemoteIndex::start(ShardId::new(2), vec![shard1.gossip_addr]).await?;+ let shard1 = RemoteIndex::start(1, vec![]).await?;+ let shard2 = RemoteIndex::start(2, vec![shard1.gossip_addr]).await?;let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![shard1.gossip_addr]).await?;@@ -237,8 +236,8 @@ async fn test_shard_without_replica() -> Result<()> {#[tokio::test]async fn test_replica_no_fails() -> Result<()> {- let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;+ let rep1 = RemoteIndex::start(1, vec![]).await?;+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;@@ -287,7 +286,7 @@ async fn test_replica_no_fails() -> Result<()> {#[tokio::test]async fn test_replica_setup_after_inserts() -> Result<()> {- let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;+ let rep1 = RemoteIndex::start(1, vec![]).await?;let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;@@ -322,7 +321,7 @@ async fn test_replica_setup_after_inserts() -> Result<()> {rep1.commit_underlying().await;- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;rep2.await_ready(&cluster).await;rep2.commit_underlying().await;@@ -339,8 +338,8 @@ async fn test_replica_setup_after_inserts() -> Result<()> {#[tokio::test]async fn test_replica_recovery() -> Result<()> {- let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;+ let rep1 = RemoteIndex::start(1, vec![]).await?;+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;@@ -387,7 +386,7 @@ async fn test_replica_recovery() -> Result<()> {rep1.commit_underlying().await;- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;rep2.await_ready(&cluster).await;rep2.commit_underlying().await;@@ -414,7 +413,7 @@ async fn test_meta_segments() -> Result<()> {dual_encoder: None,};- let index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config.clone()).await?;+ let index = LiveIndex::new(&config.index_path, 0, indexer_config.clone()).await?;assert!(index.meta().await.segments().is_empty());index@@ -436,7 +435,7 @@ async fn test_meta_segments() -> Result<()> {assert_eq!(index.meta().await.segments().len(), 1);- let copy_index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?;+ let copy_index = LiveIndex::new(&config.index_path, 0, indexer_config.clone()).await?;assert_eq!(copy_index.meta().await.segments().len(), 1);@@ -455,8 +454,7 @@ async fn test_segment_compaction() -> Result<()> {dual_encoder: None,};- let index =- Arc::new(LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?);+ let index = Arc::new(LiveIndex::new(&config.index_path, 0, indexer_config).await?);index.insert(&[IndexableWebpage {diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 21582d72..219a78db 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -29,7 +29,7 @@ use crate::{self, GetHomepageQuery, GetSiteUrlsQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery,},index::Index,- inverted_index,+ inverted_index::{self, ShardId},models::dual_encoder::DualEncoder,ranking::models::linear::LinearRegression,searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},@@ -144,11 +144,12 @@ pub struct SearchService {impl SearchService {async fn new(config: config::SearchServerConfig) -> Result<Self> {+ let shard = ShardId::Backbone(config.shard);let mut search_index = Index::open(config.index_path)?;search_index.inverted_index.set_snippet_config(config.snippet);- search_index.inverted_index.set_shard_id(config.shard);+ search_index.set_shard_id(shard);let mut local_searcher = LocalSearcher::builder(Arc::new(search_index));@@ -165,7 +166,7 @@ impl SearchService {let cluster_handle = Cluster::join(Member::new(Service::Searcher {host: config.host,- shard: config.shard,+ shard,}),config.gossip_addr,config.gossip_seed_nodes.unwrap_or_default(),diff --git a/crates/core/src/generic_query/collector/first_doc.rs b/crates/core/src/generic_query/collector/first_doc.rsindex b4160fbe..921fcf51 100644--- a/crates/core/src/generic_query/collector/first_doc.rs+++ b/crates/core/src/generic_query/collector/first_doc.rs@@ -17,7 +17,7 @@use tantivy::{collector::SegmentCollector, DocId, SegmentOrdinal};use super::Collector;-use crate::{ampc::dht::ShardId, inverted_index::DocAddress};+use crate::inverted_index::{DocAddress, ShardId};pub struct FirstDocCollector {shard_id: Option<ShardId>,diff --git a/crates/core/src/generic_query/collector/size.rs b/crates/core/src/generic_query/collector/size.rsindex 91468278..287477bc 100644--- a/crates/core/src/generic_query/collector/size.rs+++ b/crates/core/src/generic_query/collector/size.rs@@ -16,7 +16,7 @@use rustc_hash::FxHashMap;-use crate::{ampc::dht::ShardId, generic_query::size::SizeResponse};+use crate::{generic_query::size::SizeResponse, inverted_index::ShardId};use super::Collector;diff --git a/crates/core/src/generic_query/collector/top_docs.rs b/crates/core/src/generic_query/collector/top_docs.rsindex 084aec58..4ab9afa3 100644--- a/crates/core/src/generic_query/collector/top_docs.rs+++ b/crates/core/src/generic_query/collector/top_docs.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{distributed::member::ShardId, inverted_index::DocAddress};+use crate::inverted_index::{DocAddress, ShardId};use tantivy::{collector::{SegmentCollector, TopNComputer},DocId, SegmentOrdinal,diff --git a/crates/core/src/generic_query/collector/top_key_phrases.rs b/crates/core/src/generic_query/collector/top_key_phrases.rsindex da76e24a..e9376f96 100644--- a/crates/core/src/generic_query/collector/top_key_phrases.rs+++ b/crates/core/src/generic_query/collector/top_key_phrases.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{ampc::dht::ShardId, Result, SortableFloat};+use crate::{inverted_index::ShardId, Result, SortableFloat};use std::{cmp::Reverse,collections::{BinaryHeap, HashMap},diff --git a/crates/core/src/generic_query/get_homepage.rs b/crates/core/src/generic_query/get_homepage.rsindex e3bbc021..3fd5b92d 100644--- a/crates/core/src/generic_query/get_homepage.rs+++ b/crates/core/src/generic_query/get_homepage.rs@@ -18,7 +18,7 @@ use tantivy::TantivyDocument;use url::Url;use crate::{- inverted_index::RetrievedWebpage,+ inverted_index::{RetrievedWebpage, ShardId},schema::text_field::{self, TextField},search_ctx,webpage::url_ext::UrlExt,@@ -78,7 +78,7 @@ impl GenericQuery for GetHomepageQuery {fn filter_fruit_shards(&self,- shard_id: crate::ampc::dht::ShardId,+ shard_id: ShardId,fruit: <Self::Collector as super::Collector>::Fruit,) -> <Self::Collector as super::Collector>::Fruit {match fruit {diff --git a/crates/core/src/generic_query/get_site_urls.rs b/crates/core/src/generic_query/get_site_urls.rsindex e14510d0..fe9c754f 100644--- a/crates/core/src/generic_query/get_site_urls.rs+++ b/crates/core/src/generic_query/get_site_urls.rs@@ -18,7 +18,7 @@ use tantivy::TantivyDocument;use url::Url;use crate::{- inverted_index::RetrievedWebpage,+ inverted_index::{RetrievedWebpage, ShardId},schema::text_field::{self, TextField},search_ctx,webpage::url_ext::UrlExt,@@ -74,7 +74,7 @@ impl GenericQuery for GetSiteUrlsQuery {fn filter_fruit_shards(&self,- shard_id: crate::ampc::dht::ShardId,+ shard_id: ShardId,fruit: <Self::Collector as super::Collector>::Fruit,) -> <Self::Collector as super::Collector>::Fruit {fruitdiff --git a/crates/core/src/generic_query/get_webpage.rs b/crates/core/src/generic_query/get_webpage.rsindex 19c0599b..25ebdcc1 100644--- a/crates/core/src/generic_query/get_webpage.rs+++ b/crates/core/src/generic_query/get_webpage.rs@@ -18,7 +18,7 @@ use tantivy::TantivyDocument;use url::Url;use crate::{- inverted_index::RetrievedWebpage,+ inverted_index::{RetrievedWebpage, ShardId},schema::text_field::{self, TextField},search_ctx,webpage::url_ext::UrlExt,@@ -74,7 +74,7 @@ impl GenericQuery for GetWebpageQuery {fn filter_fruit_shards(&self,- shard_id: crate::ampc::dht::ShardId,+ shard_id: ShardId,fruit: <Self::Collector as super::Collector>::Fruit,) -> <Self::Collector as super::Collector>::Fruit {match fruit {diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rsindex a5958b42..ccfb382c 100644--- a/crates/core/src/generic_query/mod.rs+++ b/crates/core/src/generic_query/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{ampc::dht::ShardId, search_ctx, Result};+use crate::{inverted_index::ShardId, search_ctx, Result};pub mod top_key_phrases;pub use top_key_phrases::TopKeyPhrasesQuery;diff --git a/crates/core/src/generic_query/size.rs b/crates/core/src/generic_query/size.rsindex 872e19ae..90988c0a 100644--- a/crates/core/src/generic_query/size.rs+++ b/crates/core/src/generic_query/size.rs@@ -16,7 +16,10 @@use rustc_hash::FxHashMap;-use crate::generic_query::{Collector, GenericQuery};+use crate::{+ generic_query::{Collector, GenericQuery},+ inverted_index::ShardId,+};use super::collector::SizeCollector;@@ -73,7 +76,7 @@ impl GenericQuery for SizeQuery {fn filter_fruit_shards(&self,- shard_id: crate::ampc::dht::ShardId,+ shard_id: ShardId,fruit: <Self::Collector as Collector>::Fruit,) -> <Self::Collector as Collector>::Fruit {let mut map = FxHashMap::default();diff --git a/crates/core/src/generic_query/top_key_phrases.rs b/crates/core/src/generic_query/top_key_phrases.rsindex efe01317..ccf9e1f6 100644--- a/crates/core/src/generic_query/top_key_phrases.rs+++ b/crates/core/src/generic_query/top_key_phrases.rs@@ -18,7 +18,7 @@ use std::collections::HashMap;use itertools::Itertools;-use crate::{ampc::dht::ShardId, inverted_index::KeyPhrase};+use crate::inverted_index::{KeyPhrase, ShardId};use super::{collector::TopKeyPhrasesCollector, GenericQuery};@@ -53,7 +53,7 @@ impl GenericQuery for TopKeyPhrasesQuery {fn filter_fruit_shards(&self,- shard_id: crate::ampc::dht::ShardId,+ shard_id: ShardId,fruit: <Self::Collector as super::Collector>::Fruit,) -> <Self::Collector as super::Collector>::Fruit {fruitdiff --git a/crates/core/src/index.rs b/crates/core/src/index.rsindex 85bf32ca..052016e2 100644--- a/crates/core/src/index.rs+++ b/crates/core/src/index.rs@@ -20,7 +20,7 @@ use std::path::{Path, PathBuf};use std::sync::Mutex;use crate::collector::MainCollector;-use crate::inverted_index::{self, InvertedIndex};+use crate::inverted_index::{self, InvertedIndex, ShardId};use crate::query::Query;use crate::search_ctx::Ctx;use crate::webgraph::NodeID;@@ -63,10 +63,19 @@ impl Index {self.inverted_index.set_auto_merge_policy();}+ pub fn set_shard_id(&mut self, shard_id: ShardId) {+ self.inverted_index.set_shard_id(shard_id);+ }++ pub fn shard_id(&self) -> Option<ShardId> {+ self.inverted_index.shard_id()+ }+#[cfg(test)]pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {let dir = crate::gen_temp_dir()?;let mut s = Self::open(&dir)?;+ s.set_shard_id(ShardId::Backbone(0));s.prepare_writer()?;diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rsindex 890d2cdb..865f334d 100644--- a/crates/core/src/inverted_index/indexing.rs+++ b/crates/core/src/inverted_index/indexing.rs@@ -132,6 +132,7 @@ impl InvertedIndex {#[must_use]pub fn merge(mut self, mut other: InvertedIndex) -> Self {+ let shard_id = self.shard_id();self.prepare_writer().expect("failed to prepare writer");other.prepare_writer().expect("failed to prepare writer");@@ -202,6 +203,9 @@ impl InvertedIndex {let mut res = Self::open(path).expect("failed to open index");res.prepare_writer().expect("failed to prepare writer");+ if let Some(shard_id) = shard_id {+ res.set_shard_id(shard_id);+ }res}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 4d6a1ad0..8fb6b8d7 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -42,7 +42,6 @@ use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;use tantivy::{IndexReader, IndexWriter};-use crate::ampc::dht::ShardId;use crate::collector::{approx_count, Hashes};use crate::config::SnippetConfig;use crate::numericalfield_reader::NumericalFieldReader;@@ -74,6 +73,34 @@ pub struct WebpagePointer {pub address: DocAddress,}+#[derive(+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+ Copy,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,+)]+pub enum ShardId {+ Live(u64),+ Backbone(u64),+}++impl std::fmt::Display for ShardId {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ Self::Live(shard) => write!(f, "Live {}", shard),+ Self::Backbone(shard) => write!(f, "Backbone {}", shard),+ }+ }+}+#[derive(Debug,serde::Serialize,@@ -204,10 +231,20 @@ impl InvertedIndex {}pub fn re_open(&mut self) -> Result<()> {+ let shard_id = self.shard_id();*self = Self::open(self.path.clone())?;++ if let Some(shard_id) = shard_id {+ self.set_shard_id(shard_id);+ }+Ok(())}+ pub fn shard_id(&self) -> Option<ShardId> {+ self.shard_id+ }+pub fn columnfield_reader(&self) -> NumericalFieldReader {self.columnfield_reader.clone()}@@ -240,7 +277,7 @@ impl InvertedIndex {pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {let dir = crate::gen_temp_dir()?;let mut s = Self::open(dir.as_ref().join("index"))?;- s.set_shard_id(ShardId::new(0));+ s.set_shard_id(ShardId::Live(0));s.prepare_writer()?;diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex c4a4fe4b..97feb46a 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -23,7 +23,6 @@ use tantivy::schema::Value;use tantivy::TantivyDocument;use url::Url;-use crate::ampc::dht::ShardId;use crate::collector::approx_count::ApproxCount;use crate::collector::{approx_count, MainCollector};@@ -98,7 +97,7 @@ impl InvertedIndex {pub fn local_search_ctx(&self) -> Ctx {let tv_searcher = self.tv_searcher();Ctx {- shard_id: self.shard_id.unwrap_or(ShardId::new(u64::MAX)),+ shard_id: self.shard_id.expect("Shard ID should be set for searches"),columnfield_reader: self.columnfield_reader.clone(),tv_searcher,}diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rsindex 13db884d..7fc005e6 100644--- a/crates/core/src/live_index/crawler/mod.rs+++ b/crates/core/src/live_index/crawler/mod.rs@@ -24,7 +24,6 @@ use std::path::PathBuf;use std::sync::Arc;use std::time::Duration;-use crate::ampc::dht::ShardId;use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};use crate::crawler::robot_client::RobotClient;use crate::distributed::cluster::Cluster;@@ -34,6 +33,7 @@ use crate::distributed::sonic::replication::{use crate::distributed::streaming_response::StreamingResponse;use crate::entrypoint::search_server;use crate::entrypoint::site_stats::FinalSiteStats;+use crate::inverted_index::ShardId;use crate::Result;use crate::{distributed::sonic::replication::ReusableShardedClient,diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex b9a04059..44443528 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -33,10 +33,9 @@ use tantivy::{use std::collections::{HashMap, HashSet};use crate::{- ampc::dht::ShardId,config::SnippetConfig,entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},- inverted_index::InvertedIndex,+ inverted_index::{InvertedIndex, ShardId},live_index::{BATCH_SIZE, TTL},Result,};@@ -124,7 +123,7 @@ impl InnerIndex {) -> Result<Self> {let mut index = crate::index::Index::open(path.as_ref())?;index.prepare_writer()?;- index.inverted_index.set_shard_id(shard_id);+ index.set_shard_id(shard_id);let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;let wal_count = write_ahead_log.iter()?.count();@@ -245,6 +244,10 @@ impl InnerIndex {fn re_open(&mut self) {self.index.inverted_index.re_open().unwrap();self.index.prepare_writer().unwrap();++ if let Some(shard_id) = self.index.shard_id() {+ self.index.set_shard_id(shard_id);+ }}fn sync_meta_with_index(&mut self) {@@ -357,12 +360,12 @@ pub struct LiveIndex {impl LiveIndex {pub async fn new<P: AsRef<Path>>(path: P,- shard_id: ShardId,+ shard_id: u64,indexer_worker_config: indexer::worker::Config,) -> Result<Self> {Ok(Self {inner: Arc::new(RwLock::new(- InnerIndex::new(path, shard_id, indexer_worker_config).await?,+ InnerIndex::new(path, ShardId::Live(shard_id), indexer_worker_config).await?,)),})}diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rsindex ff8fcee5..e6129f4a 100644--- a/crates/core/src/ranking/pipeline/mod.rs+++ b/crates/core/src/ranking/pipeline/mod.rs@@ -167,9 +167,8 @@ mod tests {use scorers::term_distance;use crate::{- ampc::dht::ShardId,collector::Hashes,- inverted_index::{DocAddress, WebpagePointer},+ inverted_index::{DocAddress, ShardId, WebpagePointer},prehashed::Prehashed,ranking::{self, bitvec_similarity::BitVec, initial::Score},searcher::api,@@ -195,7 +194,7 @@ mod tests {url_without_tld: Prehashed(0),simhash: 0,},- address: DocAddress::new(0, i as u32, ShardId::new(0)),+ address: DocAddress::new(0, i as u32, ShardId::Backbone(0)),};let mut signals = EnumMap::new();@@ -211,7 +210,7 @@ mod tests {api::ScoredWebpagePointer::Normal(crate::searcher::distributed::ScoredWebpagePointer {website: RecallRankingWebpage::new(local, BitVec::new(vec![])),- shard: ShardId::new(0),+ shard: ShardId::Backbone(0),},)})diff --git a/crates/core/src/search_ctx.rs b/crates/core/src/search_ctx.rsindex 4188fb64..2302aa63 100644--- a/crates/core/src/search_ctx.rs+++ b/crates/core/src/search_ctx.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{ampc::dht::ShardId, numericalfield_reader::NumericalFieldReader};+use crate::{inverted_index::ShardId, numericalfield_reader::NumericalFieldReader};#[derive(Clone)]pub struct Ctx {diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 380623bb..f7645cf8 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -17,7 +17,7 @@use crate::{distributed::{cluster::Cluster,- member::{LiveIndexState, Service, ShardId},+ member::{LiveIndexState, Service},sonic::{self,replication::{@@ -30,11 +30,11 @@ use crate::{entrypoint::{entity_search_server,live_index::LiveIndexService,- search_server::{self, SearchService, RetrieveReq},+ search_server::{self, RetrieveReq, SearchService},},generic_query::{self, Collector},index::Index,- inverted_index::{RetrievedWebpage, WebpagePointer},+ inverted_index::{RetrievedWebpage, ShardId, WebpagePointer},ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},Result,};@@ -586,7 +586,7 @@ impl SearchClient for LocalSearchClient {vec![InitialSearchResultShard {local_result: res,- shard: ShardId::new(0),+ shard: ShardId::Backbone(0),}]}diff --git a/crates/core/src/searcher/live.rs b/crates/core/src/searcher/live.rsindex d0670410..04f5683b 100644--- a/crates/core/src/searcher/live.rs+++ b/crates/core/src/searcher/live.rs@@ -15,7 +15,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- ampc::dht::ShardId,distributed::{cluster::Cluster,member::{LiveIndexState, Service},@@ -25,7 +24,7 @@ use crate::{},},entrypoint::search_server::{self, SearchService},- inverted_index::{RetrievedWebpage, WebpagePointer},+ inverted_index::{RetrievedWebpage, ShardId, WebpagePointer},ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},};diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex 0eefcef9..09e7189b 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -22,9 +22,10 @@ use tokio::sync::Mutex;use url::Url;use crate::{+ ampc::dht::ShardId,distributed::{cluster::Cluster,- member::{Service, ShardId},+ member::Service,sonic::{self,replication::{@@ -41,6 +42,8 @@ use crate::{use super::{query::BacklinksQuery, EdgeLimit, Node, NodeID};use crate::webgraph;+impl sonic::replication::ShardIdentifier for ShardId {}+struct WebgraphClientManager;impl sonic::replication::ReusableClientManager for WebgraphClientManager {