Skip to content

web search done right

Live index replication (#224)

Mikkel Denkerbcc0819871ee
diff --git a/crates/core/src/block_on.rs b/crates/core/src/block_on.rs
new file mode 100644
index 00000000..9d868f1a
--- /dev/null
+++ b/crates/core/src/block_on.rs
@@ -0,0 +1,30 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+static TOKIO_RUNTIME: std::sync::LazyLock<tokio::runtime::Runtime> =
+ std::sync::LazyLock::new(|| {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ });
+
+pub fn block_on<F>(f: F) -> F::Output
+where
+ F: std::future::Future,
+{
+ TOKIO_RUNTIME.block_on(f)
+}
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index a6b6c42c..44532423 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -19,7 +19,6 @@ pub mod defaults;
use super::Result;
use crate::ampc::dht;
use crate::distributed::member::ShardId;
-use crate::feed::scheduler::SplitId;
use std::fs::File;
use std::io::{self, BufRead};
@@ -473,9 +472,6 @@ pub struct CrawlPlannerConfig {
#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]
pub struct LiveIndexConfig {
- pub split_path: String,
- pub downloaded_db_path: String,
-
// crawler
pub user_agent: UserAgent,
#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]
@@ -497,7 +493,6 @@ pub struct LiveIndexConfig {
// indexer
pub host_centrality_store_path: String,
- pub page_webgraph: Option<IndexerGraphConfig>,
pub page_centrality_store_path: Option<String>,
pub safety_classifier_path: Option<String>,
pub host_centrality_threshold: Option<f64>,
@@ -507,7 +502,7 @@ pub struct LiveIndexConfig {
pub cluster_id: String,
pub gossip_seed_nodes: Option<Vec<SocketAddr>>,
pub gossip_addr: SocketAddr,
- pub split_id: SplitId,
+ pub shard_id: ShardId,
pub index_path: String,
pub linear_model_path: Option<String>,
pub lambda_model_path: Option<String>,
diff --git a/crates/core/src/distributed/cluster.rs b/crates/core/src/distributed/cluster.rs
index 5b98ab15..1cf1f1cb 100644
--- a/crates/core/src/distributed/cluster.rs
+++ b/crates/core/src/distributed/cluster.rs
@@ -15,12 +15,10 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use chitchat::{
spawn_chitchat, transport::UdpTransport, Chitchat, ChitchatConfig, ChitchatHandle,
- FailureDetectorConfig, NodeId,
+ ClusterStateSnapshot, FailureDetectorConfig, NodeId,
};
-use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration};
-use tokio::sync::{Mutex, RwLock};
-use tokio_stream::StreamExt;
-use tracing::error;
+use std::{net::SocketAddr, sync::Arc, time::Duration};
+use tokio::sync::Mutex;
use crate::distributed::member::{Member, Service};
@@ -30,65 +28,29 @@ const SERVICE_KEY: &str = "service";
type Result<T> = std::result::Result<T, anyhow::Error>;
-fn alive_nodes_updater(chitchat: Arc<Mutex<Chitchat>>) -> Arc<RwLock<HashSet<Member>>> {
- let alive_nodes = Arc::new(RwLock::new(HashSet::new()));
- let alive_nodes_ref = alive_nodes.clone();
-
- tokio::spawn(async move {
- let mut node_change_receiver = chitchat.lock().await.ready_nodes_watcher();
-
- while let Some(members_set) = node_change_receiver.next().await {
- let alive_node_ids: HashSet<_> = alive_nodes_ref
- .read()
- .await
- .iter()
- .map(|member: &Member| member.id.clone())
- .collect();
-
- let member_ids: HashSet<_> =
- members_set.iter().map(|member| member.id.clone()).collect();
-
- if alive_node_ids != member_ids {
- let snapshot = chitchat.lock().await.state_snapshot();
- let mut new_members = Vec::new();
- for member in members_set {
- if let Some(state) = snapshot.node_states.get(&member.id) {
- if let Some(service) = state.get(SERVICE_KEY) {
- let service: Service = serde_json::from_str(service).unwrap();
- new_members.push(Member {
- service,
- id: member.id,
- });
- }
- } else {
- error!("no state found for node")
- }
- }
-
- tracing::info!("new members: {:#?}", new_members);
- let mut write = alive_nodes_ref.write().await;
- write.clear();
-
- for member in new_members {
- write.insert(member);
- }
+fn snapshot_members(snapshot: ClusterStateSnapshot) -> Vec<Member> {
+ let mut res = Vec::new();
+ for (id, state) in snapshot.node_states {
+ if let Some(service) = state.get(SERVICE_KEY) {
+ if let Ok(service) = serde_json::from_str(service) {
+ res.push(Member { service, id });
}
}
- });
+ }
- alive_nodes
+ res
}
pub struct Cluster {
- alive_nodes: Arc<RwLock<HashSet<Member>>>,
self_node: Option<Member>,
+ chitchat: Arc<Mutex<Chitchat>>,
// dropping the handle leaves the cluster
_chitchat_handle: ChitchatHandle,
}
impl Cluster {
pub async fn join(
- self_node: Member,
+ mut self_node: Member,
gossip_addr: SocketAddr,
seed_addrs: Vec<SocketAddr>,
) -> Result<Self> {
@@ -103,6 +65,8 @@ impl Cluster {
id: format!("{}_{}", self_node.id, uuid),
gossip_public_address: gossip_addr,
};
+ self_node.id = node_id.id.clone();
+
let config = ChitchatConfig {
node_id,
cluster_id: CLUSTER_ID.to_string(),
@@ -169,24 +133,15 @@ impl Cluster {
let chitchat_handle = spawn_chitchat(config, key_values, &transport).await?;
let chitchat = chitchat_handle.chitchat();
- let alive_nodes = alive_nodes_updater(chitchat);
-
Ok(Self {
- alive_nodes,
self_node,
+ chitchat,
_chitchat_handle: chitchat_handle,
})
}
pub async fn members(&self) -> Vec<Member> {
- let lock = self.alive_nodes.read().await;
- let mut res = Vec::with_capacity(lock.len());
-
- for member in lock.iter() {
- res.push(member.clone());
- }
-
- res
+ snapshot_members(self.chitchat.lock().await.state_snapshot())
}
pub async fn await_member<P>(&self, pred: P) -> Member
@@ -208,4 +163,25 @@ impl Cluster {
pub fn self_node(&self) -> Option<&Member> {
self.self_node.as_ref()
}
+
+ pub async fn set_service(&self, service: Service) -> Result<()> {
+ self.chitchat
+ .lock()
+ .await
+ .self_node_state()
+ .set(SERVICE_KEY, serde_json::to_string(&service)?);
+
+ Ok(())
+ }
+
+ #[cfg(test)]
+ pub async fn remove_service(&self) -> Result<()> {
+ self.chitchat
+ .lock()
+ .await
+ .self_node_state()
+ .set(SERVICE_KEY, String::new());
+
+ Ok(())
+ }
}
diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rs
index 620cccf1..2aac7228 100644
--- a/crates/core/src/distributed/member.rs
+++ b/crates/core/src/distributed/member.rs
@@ -62,6 +62,31 @@ impl From<ShardId> for u64 {
}
}
+#[derive(
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ PartialEq,
+ Eq,
+ Hash,
+ Clone,
+ Debug,
+)]
+pub enum LiveIndexState {
+ InSetup,
+ Ready,
+}
+
+impl std::fmt::Display for LiveIndexState {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ match self {
+ LiveIndexState::InSetup => write!(f, "setup"),
+ LiveIndexState::Ready => write!(f, "ready"),
+ }
+ }
+}
+
#[derive(
serde::Serialize,
serde::Deserialize,
@@ -83,7 +108,8 @@ pub enum Service {
},
LiveIndex {
host: SocketAddr,
- split_id: crate::feed::scheduler::SplitId,
+ shard: ShardId,
+ state: LiveIndexState,
},
Api {
host: SocketAddr,
@@ -118,7 +144,9 @@ impl std::fmt::Display for Service {
match self {
Self::Searcher { host, shard } => write!(f, "Searcher {} {}", host, shard),
Self::EntitySearcher { host } => write!(f, "EntitySearcher {}", host),
- Self::LiveIndex { host, split_id } => write!(f, "LiveIndex {} {}", host, split_id),
+ Self::LiveIndex { host, shard, state } => {
+ write!(f, "LiveIndex {} {} {}", host, shard, state)
+ }
Self::Api { host } => write!(f, "Api {}", host),
Self::Webgraph {
host,
diff --git a/crates/core/src/distributed/mod.rs b/crates/core/src/distributed/mod.rs
index 9d64573f..74b51726 100644
--- a/crates/core/src/distributed/mod.rs
+++ b/crates/core/src/distributed/mod.rs
@@ -17,3 +17,4 @@ pub mod cluster;
pub mod member;
pub mod retry_strategy;
pub mod sonic;
+pub mod remote_cp;
diff --git a/crates/core/src/distributed/remote_cp.rs b/crates/core/src/distributed/remote_cp.rs
new file mode 100644
index 00000000..d0ccfbea
--- /dev/null
+++ b/crates/core/src/distributed/remote_cp.rs
@@ -0,0 +1,289 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::Result;
+use std::{
+ fs,
+ future::Future,
+ io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},
+ path::{Path, PathBuf},
+};
+
+const CHUNK_SIZE_BYTES: usize = 1024 * 1024; // 1 MB
+
+pub trait Stepper {
+ fn step(&self, req: Request) -> impl Future<Output = Response>;
+}
+
+impl<F> Stepper for F
+where
+ F: Fn(Request) -> Response,
+{
+ async fn step(&self, req: Request) -> Response {
+ self(req)
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub enum Request {
+ Overview { root: PathBuf },
+ Chunk { path: PathBuf, offset: u64 },
+}
+
+impl Request {
+ pub async fn download<P1, P2, S>(remote_path: P1, to: P2, stepper: &S)
+ where
+ P1: AsRef<Path>,
+ P2: AsRef<Path>,
+ S: Stepper,
+ {
+ let Response::Overview { paths } = stepper
+ .step(Self::Overview {
+ root: remote_path.as_ref().to_path_buf(),
+ })
+ .await
+ else {
+ panic!("unexpected response to request")
+ };
+
+ for path in paths {
+ let local = to.as_ref().join(path.clone());
+ if let Some(parent) = local.parent() {
+ if !parent.exists() {
+ fs::create_dir_all(parent).expect("failed to create directory");
+ }
+ }
+
+ Self::download_file(remote_path.as_ref().join(path.clone()), local, stepper).await;
+ }
+ }
+
+ pub async fn download_file<P1, P2, S>(remote: P1, local: P2, stepper: &S)
+ where
+ P1: AsRef<Path>,
+ P2: AsRef<Path>,
+ S: Stepper,
+ {
+ let mut offset = 0;
+ let file = std::fs::OpenOptions::new()
+ .create(true)
+ .write(true)
+ .truncate(true)
+ .open(local.as_ref())
+ .unwrap_or_else(|_| panic!("failed to create file {:?}", local.as_ref().as_os_str()));
+ let mut writer = BufWriter::new(file);
+
+ loop {
+ let Response::Chunk { bytes } = stepper
+ .step(Self::Chunk {
+ path: remote.as_ref().to_path_buf(),
+ offset,
+ })
+ .await
+ else {
+ panic!("unexpected response to request");
+ };
+
+ writer.write_all(&bytes).unwrap();
+
+ if bytes.len() < CHUNK_SIZE_BYTES {
+ break;
+ }
+
+ offset += bytes.len() as u64;
+ }
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub enum Response {
+ Overview { paths: Vec<PathBuf> },
+ Chunk { bytes: Vec<u8> },
+}
+
+impl Response {
+ pub fn handle(req: Request) -> Result<Self> {
+ match req {
+ Request::Overview { root } => Ok(Response::Overview {
+ paths: files(root.clone())
+ .into_iter()
+ .filter_map(|p| p.strip_prefix(&root).ok().map(|p| p.to_path_buf()))
+ .collect(),
+ }),
+ Request::Chunk { path, offset } => {
+ let file = fs::File::open(path)?;
+ let mut reader = BufReader::new(file);
+
+ reader.seek(SeekFrom::Start(offset)).ok();
+
+ let mut res = Vec::with_capacity(CHUNK_SIZE_BYTES);
+ let mut buf = [0; 512];
+
+ while res.len() < CHUNK_SIZE_BYTES {
+ let n = reader.read(&mut buf)?;
+
+ if n == 0 {
+ break;
+ }
+
+ res.extend_from_slice(&buf[..n]);
+ }
+
+ Ok(Response::Chunk { bytes: res })
+ }
+ }
+ }
+}
+
+fn files(root: PathBuf) -> Vec<PathBuf> {
+ let mut stack = vec![root];
+ let mut res = Vec::new();
+
+ while let Some(path) = stack.pop() {
+ if path.is_file() {
+ res.push(path);
+ } else if let Ok(dir) = fs::read_dir(path) {
+ for entry in dir.flatten() {
+ stack.push(entry.path());
+ }
+ }
+ }
+
+ res
+}
+
+#[cfg(test)]
+mod tests {
+ use std::path::Path;
+
+ use file_store::gen_temp_path;
+
+ use super::*;
+
+ fn local_step(req: Request) -> Response {
+ Response::handle(req).unwrap()
+ }
+
+ async fn download<P1: AsRef<Path>, P2: AsRef<Path>>(remote: P1, local: P2) {
+ Request::download(remote, local, &local_step).await
+ }
+
+ #[tokio::test]
+ async fn test_directory() {
+ let a = gen_temp_path();
+ std::fs::create_dir_all(a.join("test")).unwrap();
+ let contents = "this is a test";
+ std::fs::write(a.join("test").join("file.txt"), contents).unwrap();
+
+ let b = gen_temp_path();
+ download(&a, &b).await;
+
+ assert!(b.join("test").exists());
+ assert!(b.join("test").join("file.txt").exists());
+
+ let res = std::fs::read_to_string(b.join("test").join("file.txt")).unwrap();
+
+ assert_eq!(contents, res);
+ }
+
+ #[tokio::test]
+ async fn test_single_file() {
+ let a = gen_temp_path();
+ std::fs::create_dir_all(&a).unwrap();
+ let contents = "this is a test";
+ std::fs::write(a.join("file.txt"), contents).unwrap();
+
+ let b = gen_temp_path();
+ download(&a, &b).await;
+
+ assert!(b.join("file.txt").exists());
+
+ let res = std::fs::read_to_string(b.join("file.txt")).unwrap();
+
+ assert_eq!(contents, res);
+ }
+
+ #[tokio::test]
+ async fn test_overwrite() {
+ let a = gen_temp_path();
+ std::fs::create_dir_all(&a).unwrap();
+ let contents = "this is a test";
+ std::fs::write(a.join("file.txt"), contents).unwrap();
+
+ let b = gen_temp_path();
+ std::fs::create_dir_all(&b).unwrap();
+ std::fs::write(b.join("file.txt"), "this is another test").unwrap();
+ download(&a, &b).await;
+
+ assert!(b.join("file.txt").exists());
+
+ let res = std::fs::read_to_string(b.join("file.txt")).unwrap();
+
+ assert_eq!(contents, res);
+ }
+
+ #[tokio::test]
+ async fn test_keep_non_copied() {
+ let a = gen_temp_path();
+ std::fs::create_dir_all(a.join("test")).unwrap();
+ let contents = "this is a test";
+ std::fs::write(a.join("test").join("a.txt"), contents).unwrap();
+
+ let b = gen_temp_path();
+ std::fs::create_dir_all(b.join("test")).unwrap();
+ std::fs::write(b.join("test").join("b.txt"), contents).unwrap();
+
+ download(&a, &b).await;
+
+ assert!(b.join("test").exists());
+ assert!(b.join("test").join("a.txt").exists());
+ assert!(b.join("test").join("b.txt").exists());
+
+ let res = std::fs::read_to_string(b.join("test").join("a.txt")).unwrap();
+ assert_eq!(contents, res);
+
+ let res = std::fs::read_to_string(b.join("test").join("b.txt")).unwrap();
+ assert_eq!(contents, res);
+ }
+
+ #[tokio::test]
+ async fn test_file_size_edge_case() {
+ let content = "a".repeat(CHUNK_SIZE_BYTES - 1);
+ let a = gen_temp_path();
+ std::fs::create_dir_all(&a).unwrap();
+ std::fs::write(a.join("minus_1.txt"), &content).unwrap();
+ std::fs::write(a.join("edge.txt"), format!("{}a", &content)).unwrap();
+ std::fs::write(a.join("plus_1.txt"), format!("{}aa", &content)).unwrap();
+
+ let b = gen_temp_path();
+ download(&a, &b).await;
+ assert!(b.join("minus_1.txt").exists());
+ assert!(b.join("edge.txt").exists());
+ assert!(b.join("plus_1.txt").exists());
+
+ let res = std::fs::read_to_string(b.join("minus_1.txt")).unwrap();
+ assert_eq!(content, res);
+
+ let res = std::fs::read_to_string(b.join("edge.txt")).unwrap();
+ assert_eq!(format!("{}a", &content), res);
+
+ let res = std::fs::read_to_string(b.join("plus_1.txt")).unwrap();
+ assert_eq!(format!("{}aa", &content), res);
+
+ std::fs::remove_dir_all(&a).unwrap();
+ std::fs::remove_dir_all(&b).unwrap();
+ }
+}
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index 1c4025b2..e116ecf1 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -185,26 +185,29 @@ fn create_inverted_index() -> Result<()> {
Some(dual_encoder_path)
};
- let worker = indexer::IndexingWorker::new(IndexerConfig {
- host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),
- page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),
- page_webgraph: Some(IndexerGraphConfig::Local {
- path: webgraph_path.to_str().unwrap().to_string(),
- }),
- output_path: out_path.to_str().unwrap().to_string(),
- limit_warc_files: None,
- skip_warc_files: None,
- warc_source: job.source_config.clone(),
- host_centrality_threshold: None,
- safety_classifier_path: None,
- minimum_clean_words: None,
- batch_size: defaults::Indexing::batch_size(),
- autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),
- dual_encoder: dual_encoder_path.map(|p| IndexerDualEncoderConfig {
- model_path: p.to_str().unwrap().to_string(),
- page_centrality_rank_threshold: Some(100_000),
- }),
- });
+ let worker = crate::block_on(indexer::IndexingWorker::new(
+ IndexerConfig {
+ host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),
+ page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),
+ page_webgraph: Some(IndexerGraphConfig::Local {
+ path: webgraph_path.to_str().unwrap().to_string(),
+ }),
+ output_path: out_path.to_str().unwrap().to_string(),
+ limit_warc_files: None,
+ skip_warc_files: None,
+ warc_source: job.source_config.clone(),
+ host_centrality_threshold: None,
+ safety_classifier_path: None,
+ minimum_clean_words: None,
+ batch_size: defaults::Indexing::batch_size(),
+ autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),
+ dual_encoder: dual_encoder_path.map(|p| IndexerDualEncoderConfig {
+ model_path: p.to_str().unwrap().to_string(),
+ page_centrality_rank_threshold: Some(100_000),
+ }),
+ }
+ .into(),
+ ));
let index = job.process(&worker);
crate::mv(index.path(), &out_path)?;
diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rs
index 1eb507b3..f77ab685 100644
--- a/crates/core/src/entrypoint/indexer/job.rs
+++ b/crates/core/src/entrypoint/indexer/job.rs
@@ -85,7 +85,7 @@ impl Job {
batch.push(IndexableWebpage::from(record));
}
- let prepared = worker.prepare_webpages(&batch);
+ let prepared = crate::block_on(worker.prepare_webpages(&batch));
for webpage in &prepared {
if webpage.host_centrality > 0.0 {
diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rs
index 04952402..f1fd45c9 100644
--- a/crates/core/src/entrypoint/indexer/mod.rs
+++ b/crates/core/src/entrypoint/indexer/mod.rs
@@ -45,7 +45,7 @@ pub fn run(config: &config::IndexerConfig) -> Result<()> {
let job_config: WarcSource = config.warc_source.clone();
- let worker = IndexingWorker::new(config.clone());
+ let worker = crate::block_on(IndexingWorker::new(config.clone().into()));
let indexes = warc_paths
.into_par_iter()
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 72b35193..0ba35cb8 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -17,17 +17,15 @@
use chrono::Utc;
use itertools::Itertools;
use std::path::Path;
-use std::sync::Mutex;
+use std::sync::{Arc, Mutex};
use tracing::debug;
pub use super::indexable_webpage::IndexableWebpage;
pub use super::job::{Job, JobSettings};
use crate::backlink_grouper::BacklinkGrouper;
-use crate::config::{
- IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LiveIndexConfig,
- WebgraphGranularity,
-};
+use crate::config::{GossipConfig, IndexerConfig, IndexerDualEncoderConfig, WebgraphGranularity};
+use crate::distributed::cluster::Cluster;
use crate::models::dual_encoder::DualEncoder as DualEncoderModel;
use crate::webgraph::remote::RemoteWebgraph;
use crate::Result;
@@ -40,6 +38,23 @@ use crate::webpage::{safety_classifier, Html, Webpage};
const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);
+pub enum IndexerGraphConfig {
+ Local { path: String },
+ Remote { gossip: GossipConfig },
+ Existing { cluster: Arc<Cluster> },
+}
+
+impl From<crate::config::IndexerGraphConfig> for IndexerGraphConfig {
+ fn from(conf: crate::config::IndexerGraphConfig) -> Self {
+ match conf {
+ crate::config::IndexerGraphConfig::Local { path } => IndexerGraphConfig::Local { path },
+ crate::config::IndexerGraphConfig::Remote { gossip } => {
+ IndexerGraphConfig::Remote { gossip }
+ }
+ }
+ }
+}
+
pub struct Config {
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
@@ -53,25 +68,13 @@ impl From<IndexerConfig> for Config {
Self {
host_centrality_store_path: config.host_centrality_store_path,
page_centrality_store_path: config.page_centrality_store_path,
- page_webgraph: config.page_webgraph,
+ page_webgraph: config.page_webgraph.map(IndexerGraphConfig::from),
safety_classifier_path: config.safety_classifier_path,
dual_encoder: config.dual_encoder,
}
}
}
-impl From<LiveIndexConfig> for Config {
- fn from(config: LiveIndexConfig) -> Self {
- Self {
- host_centrality_store_path: config.host_centrality_store_path,
- page_centrality_store_path: config.page_centrality_store_path,
- page_webgraph: config.page_webgraph,
- safety_classifier_path: config.safety_classifier_path,
- dual_encoder: None,
- }
- }
-}
-
struct DualEncoder {
model: DualEncoderModel,
page_centrality_rank_threshold: Option<u64>,
@@ -83,16 +86,16 @@ pub(super) enum Webgraph {
}
impl Webgraph {
- fn batch_raw_ingoing_edges_with_labels(
+ async fn batch_raw_ingoing_edges_with_labels(
&self,
ids: Vec<NodeID>,
limit: EdgeLimit,
) -> Vec<Vec<Edge<String>>> {
let edges = match self {
- Self::Remote(webgraph) => {
- crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit))
- .unwrap_or_default()
- }
+ Self::Remote(webgraph) => webgraph
+ .batch_raw_ingoing_edges_with_labels(&ids, limit)
+ .await
+ .unwrap_or_default(),
Self::Local(webgraph) => {
let mut res = Vec::new();
@@ -140,7 +143,7 @@ impl Webgraph {
}
impl Webgraph {
- fn new(config: &IndexerGraphConfig) -> Self {
+ async fn new(config: &IndexerGraphConfig) -> Self {
match config {
IndexerGraphConfig::Local { path } => Self::Local(
webgraph::WebgraphBuilder::new(path)
@@ -149,10 +152,10 @@ impl Webgraph {
),
IndexerGraphConfig::Remote { gossip } => {
let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);
- Self::Remote(crate::block_on(RemoteWebgraph::new(
- cluster,
- WebgraphGranularity::Page,
- )))
+ Self::Remote(RemoteWebgraph::new(cluster, WebgraphGranularity::Page).await)
+ }
+ IndexerGraphConfig::Existing { cluster } => {
+ Self::Remote(RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await)
}
}
}
@@ -172,12 +175,7 @@ pub struct IndexingWorker {
}
impl IndexingWorker {
- pub fn new<C>(config: C) -> Self
- where
- Config: From<C>,
- {
- let config = Config::from(config);
-
+ pub async fn new(config: Config) -> Self {
let host_centrality_rank_store = speedy_kv::Db::open_or_create(
Path::new(&config.host_centrality_store_path).join("harmonic_rank"),
)
@@ -196,7 +194,10 @@ impl IndexingWorker {
page_centrality_rank_store: config.page_centrality_store_path.as_ref().map(|p| {
speedy_kv::Db::open_or_create(Path::new(&p).join("harmonic_rank")).unwrap()
}),
- page_webgraph: config.page_webgraph.as_ref().map(Webgraph::new),
+ page_webgraph: match config.page_webgraph.as_ref() {
+ Some(graph) => Some(Webgraph::new(graph).await),
+ None => None,
+ },
safety_classifier: config
.safety_classifier_path
.as_ref()
@@ -390,14 +391,16 @@ impl IndexingWorker {
}
}
- pub fn set_backlinks(&self, pages: &mut [Webpage]) {
+ pub async fn set_backlinks(&self, pages: &mut [Webpage]) {
if let Some(graph) = self.page_webgraph() {
let ids = pages
.iter()
.map(|w| Node::from(w.html.url()).id())
.collect::<Vec<_>>();
- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS);
+ let backlinks = graph
+ .batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS)
+ .await;
for (page, backlinks) in pages.iter_mut().zip_eq(backlinks) {
page.set_backlinks(backlinks.clone());
@@ -444,7 +447,7 @@ impl IndexingWorker {
}
}
- pub fn prepare_webpages(&self, batch: &[IndexableWebpage]) -> Vec<Webpage> {
+ pub async fn prepare_webpages(&self, batch: &[IndexableWebpage]) -> Vec<Webpage> {
let mut res = Vec::with_capacity(batch.len());
let mut signal_computer = SignalComputer::new(None);
@@ -473,7 +476,7 @@ impl IndexingWorker {
self.set_title_embeddings(&mut res);
self.set_keyword_embeddings(&mut res);
- self.set_backlinks(&mut res);
+ self.set_backlinks(&mut res).await;
res
}
@@ -486,28 +489,31 @@ mod tests {
use super::*;
fn setup_worker(data_path: &Path, threshold: Option<u64>) -> IndexingWorker {
- IndexingWorker::new(IndexerConfig {
- host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
- page_centrality_store_path: None,
- page_webgraph: None,
- safety_classifier_path: None,
- dual_encoder: Some(IndexerDualEncoderConfig {
- model_path: data_path.to_str().unwrap().to_string(),
- page_centrality_rank_threshold: threshold,
- }),
- output_path: crate::gen_temp_path().to_str().unwrap().to_string(),
- limit_warc_files: None,
- skip_warc_files: None,
- warc_source: WarcSource::Local(crate::config::LocalConfig {
- folder: crate::gen_temp_path().to_str().unwrap().to_string(),
- names: vec!["".to_string()],
- }),
- host_centrality_threshold: None,
- minimum_clean_words: None,
- batch_size: 10,
- autocommit_after_num_inserts:
- crate::config::defaults::Indexing::autocommit_after_num_inserts(),
- })
+ crate::block_on(IndexingWorker::new(
+ IndexerConfig {
+ host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ page_centrality_store_path: None,
+ page_webgraph: None,
+ safety_classifier_path: None,
+ dual_encoder: Some(IndexerDualEncoderConfig {
+ model_path: data_path.to_str().unwrap().to_string(),
+ page_centrality_rank_threshold: threshold,
+ }),
+ output_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ limit_warc_files: None,
+ skip_warc_files: None,
+ warc_source: WarcSource::Local(crate::config::LocalConfig {
+ folder: crate::gen_temp_path().to_str().unwrap().to_string(),
+ names: vec!["".to_string()],
+ }),
+ host_centrality_threshold: None,
+ minimum_clean_words: None,
+ batch_size: 10,
+ autocommit_after_num_inserts:
+ crate::config::defaults::Indexing::autocommit_after_num_inserts(),
+ }
+ .into(),
+ ))
}
#[test]
@@ -534,7 +540,7 @@ mod tests {
},
];
- let webpages = worker.prepare_webpages(&webpages);
+ let webpages = crate::block_on(worker.prepare_webpages(&webpages));
assert_eq!(webpages.len(), 2);
diff --git a/crates/core/src/entrypoint/live_index.rs b/crates/core/src/entrypoint/live_index.rs
deleted file mode 100644
index e9ab5145..00000000
--- a/crates/core/src/entrypoint/live_index.rs
+++ /dev/null
@@ -1,124 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::sync::Arc;
-
-use crate::{
- config::LiveIndexConfig,
- distributed::{
- cluster::Cluster,
- member::{Member, Service},
- sonic::{self, service::sonic_service},
- },
- inverted_index,
- live_index::{IndexManager, LiveIndex},
- searcher::{InitialWebsiteResult, LocalSearcher},
-};
-use anyhow::Result;
-use tracing::info;
-
-use super::{
- indexer::IndexableWebpage,
- search_server::{RetrieveWebsites, Search},
-};
-
-sonic_service!(LiveIndexService, [RetrieveWebsites, Search, IndexWebpages]);
-
-pub struct LiveIndexService {
- local_searcher: LocalSearcher<Arc<LiveIndex>>,
- index: Arc<LiveIndex>,
- // dropping the handle leaves the cluster
- #[allow(unused)]
- cluster_handle: Cluster,
-}
-
-impl LiveIndexService {
- async fn new(config: LiveIndexConfig) -> Result<Self> {
- let manager = IndexManager::new(config.clone())?;
- let local_searcher = LocalSearcher::new(manager.index());
-
- let index = manager.index();
-
- tokio::task::spawn(manager.run());
-
- let cluster_handle = Cluster::join(
- Member {
- id: config.cluster_id,
- service: Service::LiveIndex {
- host: config.host,
- split_id: config.split_id,
- },
- },
- config.gossip_addr,
- config.gossip_seed_nodes.unwrap_or_default(),
- )
- .await?;
-
- Ok(Self {
- local_searcher,
- cluster_handle,
- index,
- })
- }
-}
-
-impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {
- type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server
- .local_searcher
- .retrieve_websites(&self.websites, &self.query)
- .ok()
- }
-}
-
-impl sonic::service::Message<LiveIndexService> for Search {
- type Response = Option<InitialWebsiteResult>;
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server.local_searcher.search_initial(&self.query, true).ok()
- }
-}
-
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct IndexWebpages {
- pages: Vec<IndexableWebpage>,
-}
-
-impl sonic::service::Message<LiveIndexService> for IndexWebpages {
- type Response = ();
-
- async fn handle(self, server: &LiveIndexService) -> Self::Response {
- server.index.insert(&self.pages)
- }
-}
-
-pub async fn serve(config: LiveIndexConfig) -> Result<()> {
- let addr = config.host;
-
- let server = LiveIndexService::new(config)
- .await?
- .bind(&addr)
- .await
- .unwrap();
-
- info!("live index is ready to accept requests on {}", addr);
-
- loop {
- if let Err(e) = server.accept().await {
- tracing::error!("{:?}", e);
- }
- }
-}
diff --git a/crates/core/src/entrypoint/live_index/mod.rs b/crates/core/src/entrypoint/live_index/mod.rs
new file mode 100644
index 00000000..d782a04f
--- /dev/null
+++ b/crates/core/src/entrypoint/live_index/mod.rs
@@ -0,0 +1,436 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+#[cfg(test)]
+mod tests;
+
+use std::{
+ net::SocketAddr,
+ path::{Path, PathBuf},
+ sync::Arc,
+ time::Duration,
+};
+
+use crate::{
+ ampc::dht::ShardId,
+ config::LiveIndexConfig,
+ distributed::{
+ cluster::Cluster,
+ member::{LiveIndexState, Member, Service},
+ remote_cp,
+ sonic::{self, service::sonic_service},
+ },
+ inverted_index,
+ live_index::{IndexManager, LiveIndex},
+ searcher::{InitialWebsiteResult, LocalSearcher},
+};
+use anyhow::{Context, Result};
+use futures::stream::FuturesUnordered;
+use itertools::Itertools;
+use simple_wal::Wal;
+use tokio::sync::Mutex;
+use tokio_stream::StreamExt;
+use tracing::info;
+
+use super::{
+ indexer::{self, IndexableWebpage},
+ search_server::{RetrieveWebsites, Search},
+};
+
+const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);
+const INDEXING_RETRIES: usize = 3;
+
+sonic_service!(
+ LiveIndexService,
+ [
+ RetrieveWebsites,
+ Search,
+ IndexWebpages,
+ GetIndexPath,
+ RemoteDownload
+ ]
+);
+
+fn start_manager(index: Arc<LiveIndex>) {
+ let manager = IndexManager::new(index);
+ std::thread::spawn(|| manager.run());
+}
+
+struct FileDownloadStepper {
+ conn: Mutex<sonic::service::Connection<LiveIndexService>>,
+}
+
+impl remote_cp::Stepper for FileDownloadStepper {
+ async fn step(&self, req: remote_cp::Request) -> remote_cp::Response {
+ self.conn
+ .lock()
+ .await
+ .send(RemoteDownload { req })
+ .await
+ .unwrap()
+ }
+}
+
+async fn other_replicas(cluster: &Cluster, shard: &ShardId, id: &str) -> Vec<SocketAddr> {
+ tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+ cluster
+ .members()
+ .await
+ .into_iter()
+ .filter_map(|member| {
+ if member.id != id {
+ if let Service::LiveIndex {
+ host,
+ shard: member_shard,
+ state,
+ } = member.service
+ {
+ if &member_shard == shard && matches!(state, LiveIndexState::Ready) {
+ return Some(host);
+ }
+ }
+ }
+
+ None
+ })
+ .collect()
+}
+
+async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal) -> Result<()> {
+ let self_node = cluster
+ .self_node()
+ .expect("cluster should not be joined as spectator");
+
+ let Service::LiveIndex {
+ host,
+ shard,
+ state: _,
+ } = self_node.service.clone()
+ else {
+ panic!("self node should be a live index")
+ };
+
+ let mut others = other_replicas(&cluster, &shard, &self_node.id).await;
+
+ if let Some(other) = others.pop() {
+ let mut conn: sonic::service::Connection<LiveIndexService> =
+ sonic::service::Connection::create(other).await?;
+ index.delete_all_pages();
+ let local_path = index.path();
+ let remote_path = conn.send(GetIndexPath).await?;
+
+ remote_cp::Request::download(
+ remote_path,
+ local_path,
+ &FileDownloadStepper {
+ conn: Mutex::new(conn),
+ },
+ )
+ .await;
+ }
+
+ let mut wal = temp_wal
+ .lock()
+ .await
+ .take()
+ .expect("temp_wal should always exist before setup has been run");
+
+ for pages in wal.iter().unwrap().chunks(512).into_iter() {
+ let pages: Vec<_> = pages.into_iter().collect();
+ index.insert(&pages);
+ }
+
+ wal.clear().unwrap();
+
+ start_manager(index);
+
+ cluster
+ .set_service(Service::LiveIndex {
+ host,
+ shard,
+ state: LiveIndexState::Ready,
+ })
+ .await?;
+
+ Ok(())
+}
+
+type TempWal = Arc<Mutex<Option<Wal<IndexableWebpage>>>>;
+
+pub struct LiveIndexService {
+ local_searcher: LocalSearcher<Arc<LiveIndex>>,
+ temp_wal: TempWal,
+ index: Arc<LiveIndex>,
+ cluster_handle: Arc<Cluster>,
+}
+
+impl LiveIndexService {
+ async fn new(config: LiveIndexConfig) -> Result<Self> {
+ let cluster_handle = Arc::new(
+ Cluster::join(
+ Member {
+ id: config.cluster_id.clone(),
+ service: Service::LiveIndex {
+ host: config.host,
+ shard: config.shard_id,
+ state: crate::distributed::member::LiveIndexState::InSetup,
+ },
+ },
+ config.gossip_addr,
+ config.gossip_seed_nodes.clone().unwrap_or_default(),
+ )
+ .await?,
+ );
+ let index_path = Path::new(&config.index_path);
+
+ let index = Arc::new(
+ LiveIndex::new(
+ index_path.join("index"),
+ indexer::worker::Config {
+ host_centrality_store_path: config.host_centrality_store_path.clone(),
+ page_centrality_store_path: config.page_centrality_store_path.clone(),
+ page_webgraph: Some(indexer::worker::IndexerGraphConfig::Existing {
+ cluster: cluster_handle.clone(),
+ }),
+ safety_classifier_path: config.safety_classifier_path.clone(),
+ dual_encoder: None,
+ },
+ )
+ .await?,
+ );
+ let local_searcher = LocalSearcher::new(index.clone());
+
+ let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));
+
+ Ok(Self {
+ local_searcher,
+ cluster_handle,
+ index,
+ temp_wal,
+ })
+ }
+
+ fn background_setup(&self) {
+ let index = self.index.clone();
+ let temp_wal = self.temp_wal.clone();
+ let cluster = self.cluster_handle.clone();
+
+ tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });
+ }
+
+ async fn index_webpages_in_replicas(
+ &self,
+ webpages: &[IndexableWebpage],
+ consistency_fraction: f64,
+ ) -> Result<(), IndexingError> {
+ let self_member = self
+ .cluster_handle
+ .self_node()
+ .expect("node should be participating part of cluster");
+
+ let self_id = self_member.id.clone();
+ let Service::LiveIndex {
+ host: _,
+ shard: self_shard,
+ state: _,
+ } = self_member.service
+ else {
+ panic!("self_member should always be a live index")
+ };
+
+ let live_indexes: Vec<_> = self
+ .cluster_handle
+ .members()
+ .await
+ .into_iter()
+ .filter_map(|member| {
+ if let Service::LiveIndex { host, shard, state } = member.service {
+ if member.id != self_id && shard == self_shard {
+ Some(RemoteIndex { host, state })
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ })
+ .collect();
+
+ let ready = live_indexes
+ .iter()
+ .filter(|index| matches!(index.state, LiveIndexState::Ready))
+ .count();
+
+ let mut futures = FuturesUnordered::new();
+
+ for index in live_indexes {
+ let pages = webpages.to_vec();
+
+ futures.push(tokio::spawn(async move {
+ match index.index_webpages_without_consistency(&pages).await {
+ Ok(_) => Ok(index),
+ Err(e) => Err(e),
+ }
+ }));
+ }
+
+ let mut missing_responses =
+ (((ready as f64) * consistency_fraction).ceil() as u64).min(ready as u64);
+
+ while let Some(Ok(index)) = futures.next().await {
+ if let Ok(index) = index {
+ if matches!(index.state, LiveIndexState::Ready) {
+ missing_responses = missing_responses.saturating_sub(1);
+ }
+
+ if missing_responses == 0 {
+ break;
+ }
+ }
+ }
+
+ if missing_responses > 0 {
+ return Err(IndexingError::InsufficientReplication);
+ }
+
+ Ok(())
+ }
+}
+
+impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {
+ type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ server
+ .local_searcher
+ .retrieve_websites(&self.websites, &self.query)
+ .ok()
+ }
+}
+
+impl sonic::service::Message<LiveIndexService> for Search {
+ type Response = Option<InitialWebsiteResult>;
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ server.local_searcher.search_initial(&self.query, true).ok()
+ }
+}
+
+#[derive(Clone, Debug)]
+struct RemoteIndex {
+ host: SocketAddr,
+ state: LiveIndexState,
+}
+
+impl RemoteIndex {
+ async fn index_webpages_without_consistency(
+ &self,
+ webpages: &[IndexableWebpage],
+ ) -> Result<()> {
+ let req = IndexWebpages {
+ pages: webpages.to_vec(),
+ consistency_fraction: None,
+ };
+
+ for _ in 0..INDEXING_RETRIES {
+ let mut conn: sonic::service::Connection<LiveIndexService> =
+ sonic::service::Connection::create(self.host)
+ .await
+ .with_context(|| format!("failed to connect to {}", self.host))?;
+
+ if conn
+ .send_with_timeout(req.clone(), INDEXING_TIMEOUT)
+ .await
+ .is_ok()
+ {
+ return Ok(());
+ }
+ }
+
+ Err(anyhow::anyhow!("failed to replicate webpages"))
+ }
+}
+
+#[derive(thiserror::Error, Debug, bincode::Encode, bincode::Decode)]
+pub enum IndexingError {
+ #[error("failed to replicate to the necessary quorom")]
+ InsufficientReplication,
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct IndexWebpages {
+ pub pages: Vec<IndexableWebpage>,
+ pub consistency_fraction: Option<f64>,
+}
+
+impl sonic::service::Message<LiveIndexService> for IndexWebpages {
+ type Response = Result<(), IndexingError>;
+
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ if let Some(wal) = server.temp_wal.lock().await.as_mut() {
+ wal.batch_write(self.pages.iter()).unwrap();
+ } else {
+ server.index.insert(&self.pages);
+
+ if let Some(consistency_fraction) = self.consistency_fraction {
+ server
+ .index_webpages_in_replicas(&self.pages, consistency_fraction)
+ .await?;
+ }
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct GetIndexPath;
+
+impl sonic::service::Message<LiveIndexService> for GetIndexPath {
+ type Response = PathBuf;
+
+ async fn handle(self, server: &LiveIndexService) -> Self::Response {
+ server.index.path()
+ }
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct RemoteDownload {
+ req: remote_cp::Request,
+}
+impl sonic::service::Message<LiveIndexService> for RemoteDownload {
+ type Response = remote_cp::Response;
+
+ async fn handle(self, _: &LiveIndexService) -> Self::Response {
+ remote_cp::Response::handle(self.req).unwrap()
+ }
+}
+
+pub async fn serve(config: LiveIndexConfig) -> Result<()> {
+ let addr = config.host;
+
+ let service = LiveIndexService::new(config).await?;
+
+ service.background_setup();
+
+ let server = service.bind(&addr).await.unwrap();
+
+ info!("live index is ready to accept requests on {}", addr);
+
+ loop {
+ if let Err(e) = server.accept().await {
+ tracing::error!("{:?}", e);
+ }
+ }
+}
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
new file mode 100644
index 00000000..048f32f0
--- /dev/null
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -0,0 +1,430 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2023 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::{
+ config::LiveIndexConfig, entrypoint::search_server, inverted_index, live_index::LiveIndex,
+ Result,
+};
+use std::{net::SocketAddr, sync::Arc};
+
+use file_store::gen_temp_path;
+
+use crate::{
+ ampc::dht::ShardId,
+ distributed::{
+ cluster::Cluster,
+ member::{LiveIndexState, Service},
+ sonic,
+ },
+ entrypoint::{indexer::IndexableWebpage, live_index::IndexWebpages},
+ free_socket_addr,
+};
+
+use super::LiveIndexService;
+
+struct RemoteIndex {
+ host: SocketAddr,
+ shard: ShardId,
+ gossip_addr: SocketAddr,
+ underlying_index: Arc<LiveIndex>,
+ cluster: Arc<Cluster>,
+}
+
+impl RemoteIndex {
+ async fn start(shard: ShardId, gossip_seed: Vec<SocketAddr>) -> Result<Self> {
+ let path = gen_temp_path();
+
+ let host = free_socket_addr();
+ let gossip_addr = free_socket_addr();
+
+ let gossip_seed = if gossip_seed.is_empty() {
+ None
+ } else {
+ Some(gossip_seed)
+ };
+
+ let config = LiveIndexConfig {
+ user_agent: crate::config::UserAgent {
+ full: "TestBot".to_string(),
+ token: "TestBot".to_string(),
+ },
+ robots_txt_cache_sec: 60 * 60,
+ min_politeness_factor: 1,
+ start_politeness_factor: 3,
+ min_crawl_delay_ms: 5_000,
+ max_crawl_delay_ms: 60_000,
+ max_politeness_factor: 2048,
+ max_url_slowdown_retry: 5,
+ timeout_seconds: 30,
+ host_centrality_store_path: path
+ .as_path()
+ .join("host_centrality")
+ .to_str()
+ .unwrap()
+ .to_string(),
+ page_centrality_store_path: None,
+ safety_classifier_path: None,
+ host_centrality_threshold: None,
+ minimum_clean_words: None,
+ cluster_id: "test-cluster".to_string(),
+ gossip_seed_nodes: gossip_seed,
+ gossip_addr,
+ shard_id: shard,
+ index_path: path.as_path().join("index").to_str().unwrap().to_string(),
+ linear_model_path: None,
+ lambda_model_path: None,
+ host,
+ collector: Default::default(),
+ snippet: Default::default(),
+ };
+
+ let service = LiveIndexService::new(config).await?;
+ let cluster = service.cluster_handle.clone();
+ let index = service.index.clone();
+
+ service.background_setup();
+
+ let server = service.bind(&host).await.unwrap();
+
+ tokio::task::spawn(async move {
+ loop {
+ if let Err(e) = server.accept().await {
+ tracing::error!("{:?}", e);
+ }
+ }
+ });
+
+ Ok(Self {
+ host,
+ shard,
+ gossip_addr,
+ underlying_index: index,
+ cluster,
+ })
+ }
+
+ async fn conn(&self) -> Result<sonic::service::Connection<LiveIndexService>> {
+ Ok(sonic::service::Connection::create(self.host).await?)
+ }
+
+ async fn index_pages(
+ &self,
+ pages: Vec<IndexableWebpage>,
+ consistency_fraction: Option<f64>,
+ ) -> Result<()> {
+ self.conn()
+ .await?
+ .send(IndexWebpages {
+ pages,
+ consistency_fraction,
+ })
+ .await??;
+
+ Ok(())
+ }
+
+ async fn await_ready(&self, cluster: &Cluster) {
+ cluster
+ .await_member(|member| {
+ if let Service::LiveIndex { host, shard, state } = member.service.clone() {
+ self.shard == shard
+ && matches!(state, LiveIndexState::Ready)
+ && host == self.host
+ } else {
+ false
+ }
+ })
+ .await;
+ }
+
+ async fn search(&self, query: &str) -> Result<Vec<inverted_index::RetrievedWebpage>> {
+ let mut conn = self.conn().await?;
+
+ let websites: Vec<inverted_index::WebpagePointer> = conn
+ .send(search_server::Search {
+ query: query.to_string().into(),
+ })
+ .await?
+ .map(|res| {
+ res.websites
+ .into_iter()
+ .map(|page| page.pointer().clone())
+ .collect()
+ })
+ .unwrap_or_default();
+
+ Ok(conn
+ .send(search_server::RetrieveWebsites {
+ websites,
+ query: query.to_string(),
+ })
+ .await?
+ .unwrap())
+ }
+
+ async fn commit_underlying(&self) {
+ self.underlying_index.commit();
+ }
+
+ async fn kill(self) -> Result<()> {
+ self.cluster.remove_service().await?;
+
+ Ok(())
+ }
+}
+
+const CLUSTER_ID: &str = "test-cluster";
+
+#[tokio::test]
+async fn test_shard_without_replica() -> Result<()> {
+ let shard1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
+ let shard2 = RemoteIndex::start(ShardId::new(2), vec![shard1.gossip_addr]).await?;
+
+ let cluster = Cluster::join_as_spectator(
+ CLUSTER_ID.to_string(),
+ free_socket_addr(),
+ vec![shard1.gossip_addr],
+ )
+ .await?;
+
+ shard1.await_ready(&cluster).await;
+ shard2.await_ready(&cluster).await;
+
+ shard1
+ .index_pages(
+ vec![IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ None,
+ )
+ .await?;
+ shard2
+ .index_pages(
+ vec![IndexableWebpage {
+ url: "https://b.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ None,
+ )
+ .await?;
+
+ shard1.commit_underlying().await;
+ shard2.commit_underlying().await;
+
+ let res1 = shard1.search("test").await?;
+
+ assert_eq!(res1.len(), 1);
+ assert_eq!(res1[0].url, "https://a.com/");
+
+ let res2 = shard2.search("test").await?;
+ assert_eq!(res2.len(), 1);
+ assert_eq!(res2[0].url, "https://b.com/");
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_replica_no_fails() -> Result<()> {
+ let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
+ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+
+ let cluster = Cluster::join_as_spectator(
+ CLUSTER_ID.to_string(),
+ free_socket_addr(),
+ vec![rep1.gossip_addr],
+ )
+ .await?;
+
+ rep1.await_ready(&cluster).await;
+ rep2.await_ready(&cluster).await;
+
+ rep1.index_pages(
+ vec![IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ Some(1.0),
+ )
+ .await?;
+ rep2.index_pages(
+ vec![IndexableWebpage {
+ url: "https://b.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ Some(1.0),
+ )
+ .await?;
+
+ rep1.commit_underlying().await;
+ rep2.commit_underlying().await;
+
+ let res1 = rep1.search("test").await?;
+
+ assert_eq!(res1.len(), 2);
+
+ let res2 = rep2.search("test").await?;
+ assert_eq!(res2.len(), 2);
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_replica_setup_after_inserts() -> Result<()> {
+ let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
+
+ let cluster = Cluster::join_as_spectator(
+ CLUSTER_ID.to_string(),
+ free_socket_addr(),
+ vec![rep1.gossip_addr],
+ )
+ .await?;
+
+ rep1.await_ready(&cluster).await;
+
+ rep1.index_pages(
+ vec![IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ Some(1.0),
+ )
+ .await?;
+ rep1.index_pages(
+ vec![IndexableWebpage {
+ url: "https://b.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ Some(1.0),
+ )
+ .await?;
+
+ rep1.commit_underlying().await;
+
+ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+ rep2.await_ready(&cluster).await;
+
+ rep2.commit_underlying().await;
+
+ let res1 = rep1.search("test").await?;
+
+ assert_eq!(res1.len(), 2);
+
+ let res2 = rep2.search("test").await?;
+ assert_eq!(res2.len(), 2);
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_replica_recovery() -> Result<()> {
+ let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;
+ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+
+ let cluster = Cluster::join_as_spectator(
+ CLUSTER_ID.to_string(),
+ free_socket_addr(),
+ vec![rep1.gossip_addr],
+ )
+ .await?;
+
+ rep1.await_ready(&cluster).await;
+ rep2.await_ready(&cluster).await;
+
+ rep1.index_pages(
+ vec![IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ Some(1.0),
+ )
+ .await?;
+
+ rep2.kill().await?;
+
+ loop {
+ if let Ok(_) = rep1
+ .index_pages(
+ vec![IndexableWebpage {
+ url: "https://b.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }],
+ Some(1.0),
+ )
+ .await
+ {
+ break;
+ }
+
+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+ }
+
+ rep1.commit_underlying().await;
+
+ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;
+ rep2.await_ready(&cluster).await;
+
+ rep2.commit_underlying().await;
+
+ let res1 = rep1.search("test").await?;
+
+ assert_eq!(res1.len(), 2);
+
+ let res2 = rep2.search("test").await?;
+ assert_eq!(res2.len(), 2);
+
+ Ok(())
+}
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 8a10b5c8..0efdce1e 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -63,6 +63,13 @@ impl InvertedIndex {
let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());
if let Some(limit) = collector.top_docs().max_docs().cloned() {
+ if limit.segments == 0 {
+ return Ok(InitialSearchResult {
+ num_websites: approx_count::Count::Exact(0),
+ top_websites: vec![],
+ });
+ }
+
let docs_per_segment = (limit.total_docs / limit.segments) as u64;
query = Box::new(ShortCircuitQuery::new(query, docs_per_segment));
diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
index 730600c5..23b43519 100644
--- a/crates/core/src/lib.rs
+++ b/crates/core/src/lib.rs
@@ -47,6 +47,7 @@ pub mod autosuggest;
mod backlink_grouper;
pub mod bangs;
mod bincode_utils;
+mod block_on;
pub mod canon_index;
mod collector;
pub mod config;
@@ -93,17 +94,7 @@ pub mod webgraph;
pub mod webpage;
mod widgets;
-static TOKIO_RUNTIME: std::sync::LazyLock<tokio::runtime::Runtime> =
- std::sync::LazyLock::new(|| {
- tokio::runtime::Builder::new_current_thread()
- .enable_all()
- .build()
- .unwrap()
- });
-
-pub fn block_on<F: std::future::Future>(f: F) -> F::Output {
- TOKIO_RUNTIME.block_on(f)
-}
+pub use block_on::block_on;
#[derive(Error, Debug)]
pub enum Error {
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index 9110b16e..620df532 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -28,8 +28,8 @@ use tantivy::index::SegmentId;
use std::collections::{HashMap, HashSet};
use crate::{
- config::{LiveIndexConfig, SnippetConfig},
- entrypoint::indexer::{IndexableWebpage, IndexingWorker},
+ config::SnippetConfig,
+ entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},
live_index::{BATCH_SIZE, TTL},
searcher::SearchableIndex,
Result,
@@ -84,17 +84,19 @@ pub struct InnerIndex {
}
impl InnerIndex {
- pub fn new(config: LiveIndexConfig) -> Result<Self> {
- let path = Path::new(&config.index_path);
- let mut index = crate::index::Index::open(path.join("index"))?;
+ pub async fn new<P: AsRef<Path>>(
+ path: P,
+ indexer_worker_config: indexer::worker::Config,
+ ) -> Result<Self> {
+ let mut index = crate::index::Index::open(path.as_ref().join("index"))?;
index.prepare_writer()?;
- let write_ahead_log = Wal::open(path.join("wal"))?;
+ let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;
let wal_count = write_ahead_log.iter()?.count();
- let worker = IndexingWorker::new(config.clone());
+ let worker = IndexingWorker::new(indexer_worker_config).await;
- let meta = Meta::open_or_create(path.join("meta.json"));
+ let meta = Meta::open_or_create(path.as_ref().join("meta.json"));
Ok(Self {
index,
@@ -102,7 +104,7 @@ impl InnerIndex {
indexing_worker: worker,
has_inserts: wal_count > 0,
meta,
- path: path.to_path_buf(),
+ path: path.as_ref().to_path_buf(),
})
}
@@ -192,7 +194,9 @@ impl InnerIndex {
created: Utc::now(),
})
}
+ }
+ fn save_meta(&self) {
self.meta.save(self.path.join("meta.json"));
}
@@ -200,12 +204,23 @@ impl InnerIndex {
&self.index
}
+ pub fn delete_all_pages(&mut self) {
+ let segments = self.index.inverted_index.segment_ids();
+ self.index
+ .inverted_index
+ .delete_segments_by_id(&segments)
+ .unwrap();
+
+ self.meta = Meta::default();
+ self.save_meta()
+ }
+
pub fn insert(&mut self, pages: &[IndexableWebpage]) {
self.write_ahead_log.batch_write(pages.iter()).unwrap();
self.has_inserts = true;
}
- pub fn commit(&mut self) {
+ pub async fn commit(&mut self) {
for batch in self
.write_ahead_log
.iter()
@@ -214,7 +229,7 @@ impl InnerIndex {
.into_iter()
{
let batch: Vec<_> = batch.collect();
- for webpage in self.indexing_worker.prepare_webpages(&batch) {
+ for webpage in self.indexing_worker.prepare_webpages(&batch).await {
self.index.insert(&webpage).unwrap();
}
}
@@ -226,6 +241,10 @@ impl InnerIndex {
pub fn has_inserts(&self) -> bool {
self.has_inserts
}
+
+ pub fn path(&self) -> &Path {
+ &self.path
+ }
}
pub struct LiveIndex {
@@ -233,17 +252,24 @@ pub struct LiveIndex {
}
impl LiveIndex {
- pub fn new(config: LiveIndexConfig) -> Result<Self> {
+ pub async fn new<P: AsRef<Path>>(
+ path: P,
+ indexer_worker_config: indexer::worker::Config,
+ ) -> Result<Self> {
Ok(Self {
- inner: Arc::new(RwLock::new(InnerIndex::new(config)?)),
+ inner: Arc::new(RwLock::new(
+ InnerIndex::new(path, indexer_worker_config).await?,
+ )),
})
}
pub fn commit(&self) {
- self.inner
- .write()
- .unwrap_or_else(|e| e.into_inner())
- .commit();
+ futures::executor::block_on(
+ self.inner
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .commit(),
+ );
}
pub fn prune_segments(&self) {
@@ -285,4 +311,19 @@ impl LiveIndex {
.index
.set_snippet_config(config)
}
+
+ pub fn path(&self) -> PathBuf {
+ self.inner
+ .read()
+ .unwrap_or_else(|e| e.into_inner())
+ .path
+ .to_path_buf()
+ }
+
+ pub fn delete_all_pages(&self) {
+ self.inner
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .delete_all_pages();
+ }
}
diff --git a/crates/core/src/live_index/index_manager.rs b/crates/core/src/live_index/index_manager.rs
index dca7a6ac..1f0e902a 100644
--- a/crates/core/src/live_index/index_manager.rs
+++ b/crates/core/src/live_index/index_manager.rs
@@ -18,24 +18,20 @@ use std::sync::Arc;
use chrono::Utc;
-use crate::config::LiveIndexConfig;
-
use super::{
LiveIndex, AUTO_COMMIT_INTERVAL, COMPACT_INTERVAL, EVENT_LOOP_INTERVAL, PRUNE_INTERVAL,
};
-use crate::Result;
pub struct IndexManager {
index: Arc<LiveIndex>,
}
impl IndexManager {
- pub fn new(config: LiveIndexConfig) -> Result<Self> {
- let index = Arc::new(LiveIndex::new(config.clone())?);
- Ok(Self { index })
+ pub fn new(index: Arc<LiveIndex>) -> Self {
+ Self { index }
}
- pub async fn run(self) {
+ pub fn run(self) {
let mut last_commit = Utc::now();
let mut last_prune = Utc::now();
let mut last_compact = Utc::now();
@@ -56,11 +52,7 @@ impl IndexManager {
last_compact = Utc::now();
}
- tokio::time::sleep(EVENT_LOOP_INTERVAL).await;
+ std::thread::sleep(EVENT_LOOP_INTERVAL);
}
}
-
- pub fn index(&self) -> Arc<LiveIndex> {
- self.index.clone()
- }
}
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index 3c58526c..dc5c6653 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -756,28 +756,31 @@ mod tests {
}
fn setup_worker(data_path: &Path) -> IndexingWorker {
- IndexingWorker::new(IndexerConfig {
- host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
- page_centrality_store_path: None,
- page_webgraph: None,
- safety_classifier_path: None,
- dual_encoder: Some(IndexerDualEncoderConfig {
- model_path: data_path.to_str().unwrap().to_string(),
- page_centrality_rank_threshold: None,
- }),
- output_path: crate::gen_temp_path().to_str().unwrap().to_string(),
- limit_warc_files: None,
- skip_warc_files: None,
- warc_source: WarcSource::Local(crate::config::LocalConfig {
- folder: crate::gen_temp_path().to_str().unwrap().to_string(),
- names: vec!["".to_string()],
- }),
- host_centrality_threshold: None,
- minimum_clean_words: None,
- batch_size: 10,
- autocommit_after_num_inserts:
- crate::config::defaults::Indexing::autocommit_after_num_inserts(),
- })
+ crate::block_on(IndexingWorker::new(
+ IndexerConfig {
+ host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ page_centrality_store_path: None,
+ page_webgraph: None,
+ safety_classifier_path: None,
+ dual_encoder: Some(IndexerDualEncoderConfig {
+ model_path: data_path.to_str().unwrap().to_string(),
+ page_centrality_rank_threshold: None,
+ }),
+ output_path: crate::gen_temp_path().to_str().unwrap().to_string(),
+ limit_warc_files: None,
+ skip_warc_files: None,
+ warc_source: WarcSource::Local(crate::config::LocalConfig {
+ folder: crate::gen_temp_path().to_str().unwrap().to_string(),
+ names: vec!["".to_string()],
+ }),
+ host_centrality_threshold: None,
+ minimum_clean_words: None,
+ batch_size: 10,
+ autocommit_after_num_inserts:
+ crate::config::defaults::Indexing::autocommit_after_num_inserts(),
+ }
+ .into(),
+ ))
}
#[test]
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index d6df9aa2..5fcaa60f 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -453,7 +453,7 @@ where
async fn search_initial_from_live(
&self,
query: &SearchQuery,
- ) -> Option<Vec<live::InitialSearchResultSplit>> {
+ ) -> Option<Vec<live::InitialSearchResultShard>> {
match &self.live_searcher {
Some(searcher) => Some(searcher.search_initial(query).await),
None => None,
@@ -482,7 +482,7 @@ where
&self,
query: &SearchQuery,
initial_results: Vec<distributed::InitialSearchResultShard>,
- live_results: Vec<live::InitialSearchResultSplit>,
+ live_results: Vec<live::InitialSearchResultShard>,
) -> (Vec<ScoredWebpagePointer>, bool) {
let mut collector =
BucketCollector::new(NUM_PIPELINE_RANKING_RESULTS, self.collector_config.clone());
@@ -542,7 +542,7 @@ where
.unwrap_or_default();
let pointer = live::ScoredWebpagePointer {
website: RecallRankingWebpage::new(website, inbound),
- split_id: result.split_id.clone(),
+ shard_id: result.shard_id,
};
let pointer = ScoredWebpagePointer::Live(pointer);
diff --git a/crates/core/src/searcher/live.rs b/crates/core/src/searcher/live.rs
index 3e1eaa52..a138f4c3 100644
--- a/crates/core/src/searcher/live.rs
+++ b/crates/core/src/searcher/live.rs
@@ -15,16 +15,16 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ ampc::dht::ShardId,
distributed::{
cluster::Cluster,
- member::Service,
+ member::{LiveIndexState, Service},
sonic::replication::{
AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,
- ShardIdentifier, ShardedClient, SpecificShardSelector,
+ ShardedClient, SpecificShardSelector,
},
},
entrypoint::search_server::{self, SearchService},
- feed::scheduler::SplitId,
inverted_index::{RetrievedWebpage, WebpagePointer},
ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},
};
@@ -41,15 +41,13 @@ use super::{InitialWebsiteResult, SearchQuery};
#[derive(Clone, Debug)]
pub struct ScoredWebpagePointer {
pub website: RecallRankingWebpage,
- pub split_id: SplitId,
+ pub shard_id: ShardId,
}
-impl ShardIdentifier for SplitId {}
-
#[derive(Debug)]
-pub struct InitialSearchResultSplit {
+pub struct InitialSearchResultShard {
pub local_result: InitialWebsiteResult,
- pub split_id: SplitId,
+ pub shard_id: ShardId,
}
pub struct LiveSearcher {
@@ -61,11 +59,13 @@ impl LiveSearcher {
Self { cluster }
}
- async fn client(&self) -> ShardedClient<SearchService, SplitId> {
+ async fn client(&self) -> ShardedClient<SearchService, ShardId> {
let mut shards = HashMap::new();
for member in self.cluster.members().await {
- if let Service::LiveIndex { host, split_id } = member.service {
- shards.entry(split_id).or_insert_with(Vec::new).push(host);
+ if let Service::LiveIndex { host, shard, state } = member.service {
+ if matches!(state, LiveIndexState::Ready) {
+ shards.entry(shard).or_insert_with(Vec::new).push(host);
+ }
}
}
@@ -83,8 +83,8 @@ impl LiveSearcher {
async fn retrieve_webpages_from_shard(
&self,
- split: SplitId,
- client: &ShardedClient<SearchService, SplitId>,
+ shard: ShardId,
+ client: &ShardedClient<SearchService, ShardId>,
query: &str,
pointers: Vec<(usize, WebpagePointer)>,
) -> Vec<(usize, RetrievedWebpage)> {
@@ -96,7 +96,7 @@ impl LiveSearcher {
websites: pointers,
query: query.to_string(),
},
- &SpecificShardSelector(split),
+ &SpecificShardSelector(shard),
&RandomReplicaSelector,
)
.await
@@ -115,7 +115,7 @@ impl LiveSearcher {
}
impl SearchClient for LiveSearcher {
- async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultSplit> {
+ async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {
let client = self.client().await;
let mut results = Vec::new();
@@ -131,9 +131,9 @@ impl SearchClient for LiveSearcher {
{
for (shard_id, mut res) in res {
if let Some((_, Some(res))) = res.pop() {
- results.push(InitialSearchResultSplit {
+ results.push(InitialSearchResultShard {
local_result: res,
- split_id: shard_id,
+ shard_id,
});
}
}
@@ -152,7 +152,7 @@ impl SearchClient for LiveSearcher {
for (i, pointer) in top_websites {
pointers
- .entry(pointer.split_id.clone())
+ .entry(pointer.shard_id)
.or_default()
.push((*i, pointer.website.pointer().clone()));
@@ -185,7 +185,7 @@ pub trait SearchClient {
fn search_initial(
&self,
query: &SearchQuery,
- ) -> impl Future<Output = Vec<InitialSearchResultSplit>> + Send;
+ ) -> impl Future<Output = Vec<InitialSearchResultShard>> + Send;
fn retrieve_webpages(
&self,
top_websites: &[(usize, ScoredWebpagePointer)],
diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rs
index ad6ec8d1..7dddcad9 100644
--- a/crates/core/src/searcher/mod.rs
+++ b/crates/core/src/searcher/mod.rs
@@ -83,6 +83,16 @@ pub struct SearchQuery {
pub signal_coefficients: SignalCoefficients,
}
+#[cfg(test)]
+impl From<String> for SearchQuery {
+ fn from(query: String) -> Self {
+ Self {
+ query,
+ ..Default::default()
+ }
+ }
+}
+
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct InitialWebsiteResult {
pub num_websites: Count,