Skip to content

web search done right

use specific shard_ids for search servers (backbone and live) so we can potentially simply treat the live index as extra shards and re-use all merge logic etc. for searches

Mikkel Denker3459405b458a
diff --git a/crates/core/src/collector/top_docs.rs b/crates/core/src/collector/top_docs.rs
index 6d4340cb..aae6fdf5 100644
--- a/crates/core/src/collector/top_docs.rs
+++ b/crates/core/src/collector/top_docs.rs
@@ -25,9 +25,8 @@ use tantivy::{
};
use crate::{
- ampc::dht::ShardId,
config::CollectorConfig,
- inverted_index::{DocAddress, WebpagePointer},
+ inverted_index::{DocAddress, ShardId, WebpagePointer},
numericalfield_reader,
prehashed::Prehashed,
ranking::initial::{InitialScoreTweaker, Score},
@@ -503,7 +502,7 @@ mod tests {
id: doc.1,
score: Score { total: doc.2 },
segment: 0,
- shard_id: ShardId::new(0),
+ shard_id: ShardId::Backbone(0),
});
}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index f5fd50aa..66b2dd05 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -339,7 +339,7 @@ impl Default for SnippetConfig {
pub struct SearchServerConfig {
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
- pub shard: ShardId,
+ pub shard: u64,
pub index_path: String,
pub linear_model_path: Option<String>,
pub dual_encoder_model_path: Option<String>,
@@ -608,7 +608,7 @@ pub struct LiveIndexConfig {
pub minimum_clean_words: Option<usize>,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
- pub shard_id: ShardId,
+ pub shard_id: u64,
pub index_path: String,
pub linear_model_path: Option<String>,
pub lambda_model_path: Option<String>,
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index 45472fa9..28fbdd9d 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -99,14 +99,14 @@ impl std::fmt::Display for LiveIndexState {
pub enum Service {
Searcher {
host: SocketAddr,
- shard: ShardId,
+ shard: crate::inverted_index::ShardId,
},
EntitySearcher {
host: SocketAddr,
},
LiveIndex {
host: SocketAddr,
- shard: ShardId,
+ shard: crate::inverted_index::ShardId,
state: LiveIndexState,
},
Api {
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
index 936f52c8..2bd2520e 100644
--- a/crates/core/src/entrypoint/live_index/search_server.rs
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -22,7 +22,6 @@ use std::{
};
use crate::{
- ampc::dht::ShardId,
config::LiveIndexConfig,
distributed::{
cluster::Cluster,
@@ -30,7 +29,7 @@ use crate::{
remote_cp,
sonic::{self, service::sonic_service},
},
- inverted_index,
+ inverted_index::{self, ShardId},
live_index::{IndexManager, LiveIndex},
searcher::{InitialWebsiteResult, LocalSearcher},
};
@@ -190,7 +189,7 @@ impl LiveIndexService {
Cluster::join(
Member::new(Service::LiveIndex {
host: config.host,
- shard: config.shard_id,
+ shard: ShardId::Live(config.shard_id),
state: crate::distributed::member::LiveIndexState::InSetup,
}),
config.gossip_addr,
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
index 2ff1d4a0..4d6c863f 100644
--- a/crates/core/src/entrypoint/live_index/tests.rs
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -17,7 +17,7 @@
use crate::{
config::LiveIndexConfig,
entrypoint::search_server,
- inverted_index,
+ inverted_index::{self, ShardId},
live_index::LiveIndex,
searcher::{LocalSearcher, SearchQuery},
Result,
@@ -27,7 +27,6 @@ use std::{net::SocketAddr, path::Path, sync::Arc};
use file_store::{gen_temp_dir, temp::TempDir};
use crate::{
- ampc::dht::ShardId,
distributed::{
cluster::Cluster,
member::{LiveIndexState, Service},
@@ -53,7 +52,7 @@ fn config<P: AsRef<Path>>(path: P) -> LiveIndexConfig {
minimum_clean_words: None,
gossip_seed_nodes: None,
gossip_addr: free_socket_addr(),
- shard_id: ShardId::new(0),
+ shard_id: 0,
index_path: path.as_ref().join("index").to_str().unwrap().to_string(),
linear_model_path: None,
lambda_model_path: None,
@@ -73,7 +72,7 @@ struct RemoteIndex {
}
impl RemoteIndex {
- async fn start(shard: ShardId, gossip_seed: Vec<SocketAddr>) -> Result<Self> {
+ async fn start(shard: u64, gossip_seed: Vec<SocketAddr>) -> Result<Self> {
let dir = gen_temp_dir()?;
let mut config = config(&dir);
@@ -103,7 +102,7 @@ impl RemoteIndex {
Ok(Self {
host,
- shard,
+ shard: ShardId::Live(shard),
gossip_addr,
underlying_index: index,
cluster,
@@ -183,8 +182,8 @@ impl RemoteIndex {
#[tokio::test]
async fn test_shard_without_replica() -> Result<()> {
- let shard1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
- let shard2 = RemoteIndex::start(ShardId::new(2), vec![shard1.gossip_addr]).await?;
+ let shard1 = RemoteIndex::start(1, vec![]).await?;
+ let shard2 = RemoteIndex::start(2, vec![shard1.gossip_addr]).await?;
let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![shard1.gossip_addr]).await?;
@@ -237,8 +236,8 @@ async fn test_shard_without_replica() -> Result<()> {
#[tokio::test]
async fn test_replica_no_fails() -> Result<()> {
- let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+ let rep1 = RemoteIndex::start(1, vec![]).await?;
+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;
let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;
@@ -287,7 +286,7 @@ async fn test_replica_no_fails() -> Result<()> {
#[tokio::test]
async fn test_replica_setup_after_inserts() -> Result<()> {
- let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
+ let rep1 = RemoteIndex::start(1, vec![]).await?;
let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;
@@ -322,7 +321,7 @@ async fn test_replica_setup_after_inserts() -> Result<()> {
rep1.commit_underlying().await;
- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;
rep2.await_ready(&cluster).await;
rep2.commit_underlying().await;
@@ -339,8 +338,8 @@ async fn test_replica_setup_after_inserts() -> Result<()> {
#[tokio::test]
async fn test_replica_recovery() -> Result<()> {
- let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+ let rep1 = RemoteIndex::start(1, vec![]).await?;
+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;
let cluster = Cluster::join_as_spectator(free_socket_addr(), vec![rep1.gossip_addr]).await?;
@@ -387,7 +386,7 @@ async fn test_replica_recovery() -> Result<()> {
rep1.commit_underlying().await;
- let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+ let rep2 = RemoteIndex::start(1, vec![rep1.gossip_addr]).await?;
rep2.await_ready(&cluster).await;
rep2.commit_underlying().await;
@@ -414,7 +413,7 @@ async fn test_meta_segments() -> Result<()> {
dual_encoder: None,
};
- let index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config.clone()).await?;
+ let index = LiveIndex::new(&config.index_path, 0, indexer_config.clone()).await?;
assert!(index.meta().await.segments().is_empty());
index
@@ -436,7 +435,7 @@ async fn test_meta_segments() -> Result<()> {
assert_eq!(index.meta().await.segments().len(), 1);
- let copy_index = LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?;
+ let copy_index = LiveIndex::new(&config.index_path, 0, indexer_config.clone()).await?;
assert_eq!(copy_index.meta().await.segments().len(), 1);
@@ -455,8 +454,7 @@ async fn test_segment_compaction() -> Result<()> {
dual_encoder: None,
};
- let index =
- Arc::new(LiveIndex::new(&config.index_path, ShardId::new(0), indexer_config).await?);
+ let index = Arc::new(LiveIndex::new(&config.index_path, 0, indexer_config).await?);
index
.insert(&[IndexableWebpage {
diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rs
index 21582d72..219a78db 100644
--- a/crates/core/src/entrypoint/search_server.rs
+++ b/crates/core/src/entrypoint/search_server.rs
@@ -29,7 +29,7 @@ use crate::{
self, GetHomepageQuery, GetSiteUrlsQuery, GetWebpageQuery, SizeQuery, TopKeyPhrasesQuery,
},
index::Index,
- inverted_index,
+ inverted_index::{self, ShardId},
models::dual_encoder::DualEncoder,
ranking::models::linear::LinearRegression,
searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},
@@ -144,11 +144,12 @@ pub struct SearchService {
impl SearchService {
async fn new(config: config::SearchServerConfig) -> Result<Self> {
+ let shard = ShardId::Backbone(config.shard);
let mut search_index = Index::open(config.index_path)?;
search_index
.inverted_index
.set_snippet_config(config.snippet);
- search_index.inverted_index.set_shard_id(config.shard);
+ search_index.set_shard_id(shard);
let mut local_searcher = LocalSearcher::builder(Arc::new(search_index));
@@ -165,7 +166,7 @@ impl SearchService {
let cluster_handle = Cluster::join(
Member::new(Service::Searcher {
host: config.host,
- shard: config.shard,
+ shard,
}),
config.gossip_addr,
config.gossip_seed_nodes.unwrap_or_default(),
diff --git a/crates/core/src/generic_query/collector/first_doc.rs b/crates/core/src/generic_query/collector/first_doc.rs
index b4160fbe..921fcf51 100644
--- a/crates/core/src/generic_query/collector/first_doc.rs
+++ b/crates/core/src/generic_query/collector/first_doc.rs
@@ -17,7 +17,7 @@
use tantivy::{collector::SegmentCollector, DocId, SegmentOrdinal};
use super::Collector;
-use crate::{ampc::dht::ShardId, inverted_index::DocAddress};
+use crate::inverted_index::{DocAddress, ShardId};
pub struct FirstDocCollector {
shard_id: Option<ShardId>,
diff --git a/crates/core/src/generic_query/collector/size.rs b/crates/core/src/generic_query/collector/size.rs
index 91468278..287477bc 100644
--- a/crates/core/src/generic_query/collector/size.rs
+++ b/crates/core/src/generic_query/collector/size.rs
@@ -16,7 +16,7 @@
use rustc_hash::FxHashMap;
-use crate::{ampc::dht::ShardId, generic_query::size::SizeResponse};
+use crate::{generic_query::size::SizeResponse, inverted_index::ShardId};
use super::Collector;
diff --git a/crates/core/src/generic_query/collector/top_docs.rs b/crates/core/src/generic_query/collector/top_docs.rs
index 084aec58..4ab9afa3 100644
--- a/crates/core/src/generic_query/collector/top_docs.rs
+++ b/crates/core/src/generic_query/collector/top_docs.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{distributed::member::ShardId, inverted_index::DocAddress};
+use crate::inverted_index::{DocAddress, ShardId};
use tantivy::{
collector::{SegmentCollector, TopNComputer},
DocId, SegmentOrdinal,
diff --git a/crates/core/src/generic_query/collector/top_key_phrases.rs b/crates/core/src/generic_query/collector/top_key_phrases.rs
index da76e24a..e9376f96 100644
--- a/crates/core/src/generic_query/collector/top_key_phrases.rs
+++ b/crates/core/src/generic_query/collector/top_key_phrases.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{ampc::dht::ShardId, Result, SortableFloat};
+use crate::{inverted_index::ShardId, Result, SortableFloat};
use std::{
cmp::Reverse,
collections::{BinaryHeap, HashMap},
diff --git a/crates/core/src/generic_query/get_homepage.rs b/crates/core/src/generic_query/get_homepage.rs
index e3bbc021..3fd5b92d 100644
--- a/crates/core/src/generic_query/get_homepage.rs
+++ b/crates/core/src/generic_query/get_homepage.rs
@@ -18,7 +18,7 @@ use tantivy::TantivyDocument;
use url::Url;
use crate::{
- inverted_index::RetrievedWebpage,
+ inverted_index::{RetrievedWebpage, ShardId},
schema::text_field::{self, TextField},
search_ctx,
webpage::url_ext::UrlExt,
@@ -78,7 +78,7 @@ impl GenericQuery for GetHomepageQuery {
fn filter_fruit_shards(
&self,
- shard_id: crate::ampc::dht::ShardId,
+ shard_id: ShardId,
fruit: <Self::Collector as super::Collector>::Fruit,
) -> <Self::Collector as super::Collector>::Fruit {
match fruit {
diff --git a/crates/core/src/generic_query/get_site_urls.rs b/crates/core/src/generic_query/get_site_urls.rs
index e14510d0..fe9c754f 100644
--- a/crates/core/src/generic_query/get_site_urls.rs
+++ b/crates/core/src/generic_query/get_site_urls.rs
@@ -18,7 +18,7 @@ use tantivy::TantivyDocument;
use url::Url;
use crate::{
- inverted_index::RetrievedWebpage,
+ inverted_index::{RetrievedWebpage, ShardId},
schema::text_field::{self, TextField},
search_ctx,
webpage::url_ext::UrlExt,
@@ -74,7 +74,7 @@ impl GenericQuery for GetSiteUrlsQuery {
fn filter_fruit_shards(
&self,
- shard_id: crate::ampc::dht::ShardId,
+ shard_id: ShardId,
fruit: <Self::Collector as super::Collector>::Fruit,
) -> <Self::Collector as super::Collector>::Fruit {
fruit
diff --git a/crates/core/src/generic_query/get_webpage.rs b/crates/core/src/generic_query/get_webpage.rs
index 19c0599b..25ebdcc1 100644
--- a/crates/core/src/generic_query/get_webpage.rs
+++ b/crates/core/src/generic_query/get_webpage.rs
@@ -18,7 +18,7 @@ use tantivy::TantivyDocument;
use url::Url;
use crate::{
- inverted_index::RetrievedWebpage,
+ inverted_index::{RetrievedWebpage, ShardId},
schema::text_field::{self, TextField},
search_ctx,
webpage::url_ext::UrlExt,
@@ -74,7 +74,7 @@ impl GenericQuery for GetWebpageQuery {
fn filter_fruit_shards(
&self,
- shard_id: crate::ampc::dht::ShardId,
+ shard_id: ShardId,
fruit: <Self::Collector as super::Collector>::Fruit,
) -> <Self::Collector as super::Collector>::Fruit {
match fruit {
diff --git a/crates/core/src/generic_query/mod.rs b/crates/core/src/generic_query/mod.rs
index a5958b42..ccfb382c 100644
--- a/crates/core/src/generic_query/mod.rs
+++ b/crates/core/src/generic_query/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{ampc::dht::ShardId, search_ctx, Result};
+use crate::{inverted_index::ShardId, search_ctx, Result};
pub mod top_key_phrases;
pub use top_key_phrases::TopKeyPhrasesQuery;
diff --git a/crates/core/src/generic_query/size.rs b/crates/core/src/generic_query/size.rs
index 872e19ae..90988c0a 100644
--- a/crates/core/src/generic_query/size.rs
+++ b/crates/core/src/generic_query/size.rs
@@ -16,7 +16,10 @@
use rustc_hash::FxHashMap;
-use crate::generic_query::{Collector, GenericQuery};
+use crate::{
+ generic_query::{Collector, GenericQuery},
+ inverted_index::ShardId,
+};
use super::collector::SizeCollector;
@@ -73,7 +76,7 @@ impl GenericQuery for SizeQuery {
fn filter_fruit_shards(
&self,
- shard_id: crate::ampc::dht::ShardId,
+ shard_id: ShardId,
fruit: <Self::Collector as Collector>::Fruit,
) -> <Self::Collector as Collector>::Fruit {
let mut map = FxHashMap::default();
diff --git a/crates/core/src/generic_query/top_key_phrases.rs b/crates/core/src/generic_query/top_key_phrases.rs
index efe01317..ccf9e1f6 100644
--- a/crates/core/src/generic_query/top_key_phrases.rs
+++ b/crates/core/src/generic_query/top_key_phrases.rs
@@ -18,7 +18,7 @@ use std::collections::HashMap;
use itertools::Itertools;
-use crate::{ampc::dht::ShardId, inverted_index::KeyPhrase};
+use crate::inverted_index::{KeyPhrase, ShardId};
use super::{collector::TopKeyPhrasesCollector, GenericQuery};
@@ -53,7 +53,7 @@ impl GenericQuery for TopKeyPhrasesQuery {
fn filter_fruit_shards(
&self,
- shard_id: crate::ampc::dht::ShardId,
+ shard_id: ShardId,
fruit: <Self::Collector as super::Collector>::Fruit,
) -> <Self::Collector as super::Collector>::Fruit {
fruit
diff --git a/crates/core/src/index.rs b/crates/core/src/index.rs
index 85bf32ca..052016e2 100644
--- a/crates/core/src/index.rs
+++ b/crates/core/src/index.rs
@@ -20,7 +20,7 @@ use std::path::{Path, PathBuf};
use std::sync::Mutex;
use crate::collector::MainCollector;
-use crate::inverted_index::{self, InvertedIndex};
+use crate::inverted_index::{self, InvertedIndex, ShardId};
use crate::query::Query;
use crate::search_ctx::Ctx;
use crate::webgraph::NodeID;
@@ -63,10 +63,19 @@ impl Index {
self.inverted_index.set_auto_merge_policy();
}
+ pub fn set_shard_id(&mut self, shard_id: ShardId) {
+ self.inverted_index.set_shard_id(shard_id);
+ }
+
+ pub fn shard_id(&self) -> Option<ShardId> {
+ self.inverted_index.shard_id()
+ }
+
#[cfg(test)]
pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {
let dir = crate::gen_temp_dir()?;
let mut s = Self::open(&dir)?;
+ s.set_shard_id(ShardId::Backbone(0));
s.prepare_writer()?;
diff --git a/crates/core/src/inverted_index/indexing.rs b/crates/core/src/inverted_index/indexing.rs
index 890d2cdb..865f334d 100644
--- a/crates/core/src/inverted_index/indexing.rs
+++ b/crates/core/src/inverted_index/indexing.rs
@@ -132,6 +132,7 @@ impl InvertedIndex {
#[must_use]
pub fn merge(mut self, mut other: InvertedIndex) -> Self {
+ let shard_id = self.shard_id();
self.prepare_writer().expect("failed to prepare writer");
other.prepare_writer().expect("failed to prepare writer");
@@ -202,6 +203,9 @@ impl InvertedIndex {
let mut res = Self::open(path).expect("failed to open index");
res.prepare_writer().expect("failed to prepare writer");
+ if let Some(shard_id) = shard_id {
+ res.set_shard_id(shard_id);
+ }
res
}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 4d6a1ad0..8fb6b8d7 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -42,7 +42,6 @@ use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
use tantivy::{IndexReader, IndexWriter};
-use crate::ampc::dht::ShardId;
use crate::collector::{approx_count, Hashes};
use crate::config::SnippetConfig;
use crate::numericalfield_reader::NumericalFieldReader;
@@ -74,6 +73,34 @@ pub struct WebpagePointer {
pub address: DocAddress,
}
+#[derive(
+ Debug,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Clone,
+ Copy,
+ PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
+ Hash,
+)]
+pub enum ShardId {
+ Live(u64),
+ Backbone(u64),
+}
+
+impl std::fmt::Display for ShardId {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Self::Live(shard) => write!(f, "Live {}", shard),
+ Self::Backbone(shard) => write!(f, "Backbone {}", shard),
+ }
+ }
+}
+
#[derive(
Debug,
serde::Serialize,
@@ -204,10 +231,20 @@ impl InvertedIndex {
}
pub fn re_open(&mut self) -> Result<()> {
+ let shard_id = self.shard_id();
*self = Self::open(self.path.clone())?;
+
+ if let Some(shard_id) = shard_id {
+ self.set_shard_id(shard_id);
+ }
+
Ok(())
}
+ pub fn shard_id(&self) -> Option<ShardId> {
+ self.shard_id
+ }
+
pub fn columnfield_reader(&self) -> NumericalFieldReader {
self.columnfield_reader.clone()
}
@@ -240,7 +277,7 @@ impl InvertedIndex {
pub fn temporary() -> Result<(Self, file_store::temp::TempDir)> {
let dir = crate::gen_temp_dir()?;
let mut s = Self::open(dir.as_ref().join("index"))?;
- s.set_shard_id(ShardId::new(0));
+ s.set_shard_id(ShardId::Live(0));
s.prepare_writer()?;
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index c4a4fe4b..97feb46a 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -23,7 +23,6 @@ use tantivy::schema::Value;
use tantivy::TantivyDocument;
use url::Url;
-use crate::ampc::dht::ShardId;
use crate::collector::approx_count::ApproxCount;
use crate::collector::{approx_count, MainCollector};
@@ -98,7 +97,7 @@ impl InvertedIndex {
pub fn local_search_ctx(&self) -> Ctx {
let tv_searcher = self.tv_searcher();
Ctx {
- shard_id: self.shard_id.unwrap_or(ShardId::new(u64::MAX)),
+ shard_id: self.shard_id.expect("Shard ID should be set for searches"),
columnfield_reader: self.columnfield_reader.clone(),
tv_searcher,
}
diff --git a/crates/core/src/live_index/crawler/mod.rs b/crates/core/src/live_index/crawler/mod.rs
index 13db884d..7fc005e6 100644
--- a/crates/core/src/live_index/crawler/mod.rs
+++ b/crates/core/src/live_index/crawler/mod.rs
@@ -24,7 +24,6 @@ use std::path::PathBuf;
use std::sync::Arc;
use std::time::Duration;
-use crate::ampc::dht::ShardId;
use crate::config::{CheckIntervals, CrawlerConfig, LiveCrawlerConfig};
use crate::crawler::robot_client::RobotClient;
use crate::distributed::cluster::Cluster;
@@ -34,6 +33,7 @@ use crate::distributed::sonic::replication::{
use crate::distributed::streaming_response::StreamingResponse;
use crate::entrypoint::search_server;
use crate::entrypoint::site_stats::FinalSiteStats;
+use crate::inverted_index::ShardId;
use crate::Result;
use crate::{
distributed::sonic::replication::ReusableShardedClient,
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index b9a04059..44443528 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -33,10 +33,9 @@ use tantivy::{
use std::collections::{HashMap, HashSet};
use crate::{
- ampc::dht::ShardId,
config::SnippetConfig,
entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},
- inverted_index::InvertedIndex,
+ inverted_index::{InvertedIndex, ShardId},
live_index::{BATCH_SIZE, TTL},
Result,
};
@@ -124,7 +123,7 @@ impl InnerIndex {
) -> Result<Self> {
let mut index = crate::index::Index::open(path.as_ref())?;
index.prepare_writer()?;
- index.inverted_index.set_shard_id(shard_id);
+ index.set_shard_id(shard_id);
let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;
let wal_count = write_ahead_log.iter()?.count();
@@ -245,6 +244,10 @@ impl InnerIndex {
fn re_open(&mut self) {
self.index.inverted_index.re_open().unwrap();
self.index.prepare_writer().unwrap();
+
+ if let Some(shard_id) = self.index.shard_id() {
+ self.index.set_shard_id(shard_id);
+ }
}
fn sync_meta_with_index(&mut self) {
@@ -357,12 +360,12 @@ pub struct LiveIndex {
impl LiveIndex {
pub async fn new<P: AsRef<Path>>(
path: P,
- shard_id: ShardId,
+ shard_id: u64,
indexer_worker_config: indexer::worker::Config,
) -> Result<Self> {
Ok(Self {
inner: Arc::new(RwLock::new(
- InnerIndex::new(path, shard_id, indexer_worker_config).await?,
+ InnerIndex::new(path, ShardId::Live(shard_id), indexer_worker_config).await?,
)),
})
}
diff --git a/crates/core/src/ranking/pipeline/mod.rs b/crates/core/src/ranking/pipeline/mod.rs
index ff8fcee5..e6129f4a 100644
--- a/crates/core/src/ranking/pipeline/mod.rs
+++ b/crates/core/src/ranking/pipeline/mod.rs
@@ -167,9 +167,8 @@ mod tests {
use scorers::term_distance;
use crate::{
- ampc::dht::ShardId,
collector::Hashes,
- inverted_index::{DocAddress, WebpagePointer},
+ inverted_index::{DocAddress, ShardId, WebpagePointer},
prehashed::Prehashed,
ranking::{self, bitvec_similarity::BitVec, initial::Score},
searcher::api,
@@ -195,7 +194,7 @@ mod tests {
url_without_tld: Prehashed(0),
simhash: 0,
},
- address: DocAddress::new(0, i as u32, ShardId::new(0)),
+ address: DocAddress::new(0, i as u32, ShardId::Backbone(0)),
};
let mut signals = EnumMap::new();
@@ -211,7 +210,7 @@ mod tests {
api::ScoredWebpagePointer::Normal(
crate::searcher::distributed::ScoredWebpagePointer {
website: RecallRankingWebpage::new(local, BitVec::new(vec![])),
- shard: ShardId::new(0),
+ shard: ShardId::Backbone(0),
},
)
})
diff --git a/crates/core/src/search_ctx.rs b/crates/core/src/search_ctx.rs
index 4188fb64..2302aa63 100644
--- a/crates/core/src/search_ctx.rs
+++ b/crates/core/src/search_ctx.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{ampc::dht::ShardId, numericalfield_reader::NumericalFieldReader};
+use crate::{inverted_index::ShardId, numericalfield_reader::NumericalFieldReader};
#[derive(Clone)]
pub struct Ctx {
diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rs
index 380623bb..f7645cf8 100644
--- a/crates/core/src/searcher/distributed.rs
+++ b/crates/core/src/searcher/distributed.rs
@@ -17,7 +17,7 @@
use crate::{
distributed::{
cluster::Cluster,
- member::{LiveIndexState, Service, ShardId},
+ member::{LiveIndexState, Service},
sonic::{
self,
replication::{
@@ -30,11 +30,11 @@ use crate::{
entrypoint::{
entity_search_server,
live_index::LiveIndexService,
- search_server::{self, SearchService, RetrieveReq},
+ search_server::{self, RetrieveReq, SearchService},
},
generic_query::{self, Collector},
index::Index,
- inverted_index::{RetrievedWebpage, WebpagePointer},
+ inverted_index::{RetrievedWebpage, ShardId, WebpagePointer},
ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},
Result,
};
@@ -586,7 +586,7 @@ impl SearchClient for LocalSearchClient {
vec![InitialSearchResultShard {
local_result: res,
- shard: ShardId::new(0),
+ shard: ShardId::Backbone(0),
}]
}
diff --git a/crates/core/src/searcher/live.rs b/crates/core/src/searcher/live.rs
index d0670410..04f5683b 100644
--- a/crates/core/src/searcher/live.rs
+++ b/crates/core/src/searcher/live.rs
@@ -15,7 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- ampc::dht::ShardId,
distributed::{
cluster::Cluster,
member::{LiveIndexState, Service},
@@ -25,7 +24,7 @@ use crate::{
},
},
entrypoint::search_server::{self, SearchService},
- inverted_index::{RetrievedWebpage, WebpagePointer},
+ inverted_index::{RetrievedWebpage, ShardId, WebpagePointer},
ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},
};
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index 0eefcef9..09e7189b 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -22,9 +22,10 @@ use tokio::sync::Mutex;
use url::Url;
use crate::{
+ ampc::dht::ShardId,
distributed::{
cluster::Cluster,
- member::{Service, ShardId},
+ member::Service,
sonic::{
self,
replication::{
@@ -41,6 +42,8 @@ use crate::{
use super::{query::BacklinksQuery, EdgeLimit, Node, NodeID};
use crate::webgraph;
+impl sonic::replication::ShardIdentifier for ShardId {}
+
struct WebgraphClientManager;
impl sonic::replication::ReusableClientManager for WebgraphClientManager {