web search done right
Live index replication (#224)
Mikkel Denkerbcc0819871ee
diff --git a/crates/core/src/block_on.rs b/crates/core/src/block_on.rsnew file mode 100644index 00000000..9d868f1a--- /dev/null+++ b/crates/core/src/block_on.rs@@ -0,0 +1,30 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++static TOKIO_RUNTIME: std::sync::LazyLock<tokio::runtime::Runtime> =+ std::sync::LazyLock::new(|| {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ });++pub fn block_on<F>(f: F) -> F::Output+where+ F: std::future::Future,+{+ TOKIO_RUNTIME.block_on(f)+}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex a6b6c42c..44532423 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -19,7 +19,6 @@ pub mod defaults;use super::Result;use crate::ampc::dht;use crate::distributed::member::ShardId;-use crate::feed::scheduler::SplitId;use std::fs::File;use std::io::{self, BufRead};@@ -473,9 +472,6 @@ pub struct CrawlPlannerConfig {#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct LiveIndexConfig {- pub split_path: String,- pub downloaded_db_path: String,-// crawlerpub user_agent: UserAgent,#[serde(default = "defaults::Crawler::robots_txt_cache_sec")]@@ -497,7 +493,6 @@ pub struct LiveIndexConfig {// indexerpub host_centrality_store_path: String,- pub page_webgraph: Option<IndexerGraphConfig>,pub page_centrality_store_path: Option<String>,pub safety_classifier_path: Option<String>,pub host_centrality_threshold: Option<f64>,@@ -507,7 +502,7 @@ pub struct LiveIndexConfig {pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,pub gossip_addr: SocketAddr,- pub split_id: SplitId,+ pub shard_id: ShardId,pub index_path: String,pub linear_model_path: Option<String>,pub lambda_model_path: Option<String>,diff --git a/crates/core/src/distributed/cluster.rs b/crates/core/src/distributed/cluster.rsindex 5b98ab15..1cf1f1cb 100644--- a/crates/core/src/distributed/cluster.rs+++ b/crates/core/src/distributed/cluster.rs@@ -15,12 +15,10 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use chitchat::{spawn_chitchat, transport::UdpTransport, Chitchat, ChitchatConfig, ChitchatHandle,- FailureDetectorConfig, NodeId,+ ClusterStateSnapshot, FailureDetectorConfig, NodeId,};-use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration};-use tokio::sync::{Mutex, RwLock};-use tokio_stream::StreamExt;-use tracing::error;+use std::{net::SocketAddr, sync::Arc, time::Duration};+use tokio::sync::Mutex;use crate::distributed::member::{Member, Service};@@ -30,65 +28,29 @@ const SERVICE_KEY: &str = "service";type Result<T> = std::result::Result<T, anyhow::Error>;-fn alive_nodes_updater(chitchat: Arc<Mutex<Chitchat>>) -> Arc<RwLock<HashSet<Member>>> {- let alive_nodes = Arc::new(RwLock::new(HashSet::new()));- let alive_nodes_ref = alive_nodes.clone();-- tokio::spawn(async move {- let mut node_change_receiver = chitchat.lock().await.ready_nodes_watcher();-- while let Some(members_set) = node_change_receiver.next().await {- let alive_node_ids: HashSet<_> = alive_nodes_ref- .read()- .await- .iter()- .map(|member: &Member| member.id.clone())- .collect();-- let member_ids: HashSet<_> =- members_set.iter().map(|member| member.id.clone()).collect();-- if alive_node_ids != member_ids {- let snapshot = chitchat.lock().await.state_snapshot();- let mut new_members = Vec::new();- for member in members_set {- if let Some(state) = snapshot.node_states.get(&member.id) {- if let Some(service) = state.get(SERVICE_KEY) {- let service: Service = serde_json::from_str(service).unwrap();- new_members.push(Member {- service,- id: member.id,- });- }- } else {- error!("no state found for node")- }- }-- tracing::info!("new members: {:#?}", new_members);- let mut write = alive_nodes_ref.write().await;- write.clear();-- for member in new_members {- write.insert(member);- }+fn snapshot_members(snapshot: ClusterStateSnapshot) -> Vec<Member> {+ let mut res = Vec::new();+ for (id, state) in snapshot.node_states {+ if let Some(service) = state.get(SERVICE_KEY) {+ if let Ok(service) = serde_json::from_str(service) {+ res.push(Member { service, id });}}- });+ }- alive_nodes+ res}pub struct Cluster {- alive_nodes: Arc<RwLock<HashSet<Member>>>,self_node: Option<Member>,+ chitchat: Arc<Mutex<Chitchat>>,// dropping the handle leaves the cluster_chitchat_handle: ChitchatHandle,}impl Cluster {pub async fn join(- self_node: Member,+ mut self_node: Member,gossip_addr: SocketAddr,seed_addrs: Vec<SocketAddr>,) -> Result<Self> {@@ -103,6 +65,8 @@ impl Cluster {id: format!("{}_{}", self_node.id, uuid),gossip_public_address: gossip_addr,};+ self_node.id = node_id.id.clone();+let config = ChitchatConfig {node_id,cluster_id: CLUSTER_ID.to_string(),@@ -169,24 +133,15 @@ impl Cluster {let chitchat_handle = spawn_chitchat(config, key_values, &transport).await?;let chitchat = chitchat_handle.chitchat();- let alive_nodes = alive_nodes_updater(chitchat);-Ok(Self {- alive_nodes,self_node,+ chitchat,_chitchat_handle: chitchat_handle,})}pub async fn members(&self) -> Vec<Member> {- let lock = self.alive_nodes.read().await;- let mut res = Vec::with_capacity(lock.len());-- for member in lock.iter() {- res.push(member.clone());- }-- res+ snapshot_members(self.chitchat.lock().await.state_snapshot())}pub async fn await_member<P>(&self, pred: P) -> Member@@ -208,4 +163,25 @@ impl Cluster {pub fn self_node(&self) -> Option<&Member> {self.self_node.as_ref()}++ pub async fn set_service(&self, service: Service) -> Result<()> {+ self.chitchat+ .lock()+ .await+ .self_node_state()+ .set(SERVICE_KEY, serde_json::to_string(&service)?);++ Ok(())+ }++ #[cfg(test)]+ pub async fn remove_service(&self) -> Result<()> {+ self.chitchat+ .lock()+ .await+ .self_node_state()+ .set(SERVICE_KEY, String::new());++ Ok(())+ }}diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex 620cccf1..2aac7228 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -62,6 +62,31 @@ impl From<ShardId> for u64 {}}+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+ Clone,+ Debug,+)]+pub enum LiveIndexState {+ InSetup,+ Ready,+}++impl std::fmt::Display for LiveIndexState {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ match self {+ LiveIndexState::InSetup => write!(f, "setup"),+ LiveIndexState::Ready => write!(f, "ready"),+ }+ }+}+#[derive(serde::Serialize,serde::Deserialize,@@ -83,7 +108,8 @@ pub enum Service {},LiveIndex {host: SocketAddr,- split_id: crate::feed::scheduler::SplitId,+ shard: ShardId,+ state: LiveIndexState,},Api {host: SocketAddr,@@ -118,7 +144,9 @@ impl std::fmt::Display for Service {match self {Self::Searcher { host, shard } => write!(f, "Searcher {} {}", host, shard),Self::EntitySearcher { host } => write!(f, "EntitySearcher {}", host),- Self::LiveIndex { host, split_id } => write!(f, "LiveIndex {} {}", host, split_id),+ Self::LiveIndex { host, shard, state } => {+ write!(f, "LiveIndex {} {} {}", host, shard, state)+ }Self::Api { host } => write!(f, "Api {}", host),Self::Webgraph {host,diff --git a/crates/core/src/distributed/mod.rs b/crates/core/src/distributed/mod.rsindex 9d64573f..74b51726 100644--- a/crates/core/src/distributed/mod.rs+++ b/crates/core/src/distributed/mod.rs@@ -17,3 +17,4 @@ pub mod cluster;pub mod member;pub mod retry_strategy;pub mod sonic;+pub mod remote_cp;diff --git a/crates/core/src/distributed/remote_cp.rs b/crates/core/src/distributed/remote_cp.rsnew file mode 100644index 00000000..d0ccfbea--- /dev/null+++ b/crates/core/src/distributed/remote_cp.rs@@ -0,0 +1,289 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::Result;+use std::{+ fs,+ future::Future,+ io::{BufReader, BufWriter, Read, Seek, SeekFrom, Write},+ path::{Path, PathBuf},+};++const CHUNK_SIZE_BYTES: usize = 1024 * 1024; // 1 MB++pub trait Stepper {+ fn step(&self, req: Request) -> impl Future<Output = Response>;+}++impl<F> Stepper for F+where+ F: Fn(Request) -> Response,+{+ async fn step(&self, req: Request) -> Response {+ self(req)+ }+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub enum Request {+ Overview { root: PathBuf },+ Chunk { path: PathBuf, offset: u64 },+}++impl Request {+ pub async fn download<P1, P2, S>(remote_path: P1, to: P2, stepper: &S)+ where+ P1: AsRef<Path>,+ P2: AsRef<Path>,+ S: Stepper,+ {+ let Response::Overview { paths } = stepper+ .step(Self::Overview {+ root: remote_path.as_ref().to_path_buf(),+ })+ .await+ else {+ panic!("unexpected response to request")+ };++ for path in paths {+ let local = to.as_ref().join(path.clone());+ if let Some(parent) = local.parent() {+ if !parent.exists() {+ fs::create_dir_all(parent).expect("failed to create directory");+ }+ }++ Self::download_file(remote_path.as_ref().join(path.clone()), local, stepper).await;+ }+ }++ pub async fn download_file<P1, P2, S>(remote: P1, local: P2, stepper: &S)+ where+ P1: AsRef<Path>,+ P2: AsRef<Path>,+ S: Stepper,+ {+ let mut offset = 0;+ let file = std::fs::OpenOptions::new()+ .create(true)+ .write(true)+ .truncate(true)+ .open(local.as_ref())+ .unwrap_or_else(|_| panic!("failed to create file {:?}", local.as_ref().as_os_str()));+ let mut writer = BufWriter::new(file);++ loop {+ let Response::Chunk { bytes } = stepper+ .step(Self::Chunk {+ path: remote.as_ref().to_path_buf(),+ offset,+ })+ .await+ else {+ panic!("unexpected response to request");+ };++ writer.write_all(&bytes).unwrap();++ if bytes.len() < CHUNK_SIZE_BYTES {+ break;+ }++ offset += bytes.len() as u64;+ }+ }+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub enum Response {+ Overview { paths: Vec<PathBuf> },+ Chunk { bytes: Vec<u8> },+}++impl Response {+ pub fn handle(req: Request) -> Result<Self> {+ match req {+ Request::Overview { root } => Ok(Response::Overview {+ paths: files(root.clone())+ .into_iter()+ .filter_map(|p| p.strip_prefix(&root).ok().map(|p| p.to_path_buf()))+ .collect(),+ }),+ Request::Chunk { path, offset } => {+ let file = fs::File::open(path)?;+ let mut reader = BufReader::new(file);++ reader.seek(SeekFrom::Start(offset)).ok();++ let mut res = Vec::with_capacity(CHUNK_SIZE_BYTES);+ let mut buf = [0; 512];++ while res.len() < CHUNK_SIZE_BYTES {+ let n = reader.read(&mut buf)?;++ if n == 0 {+ break;+ }++ res.extend_from_slice(&buf[..n]);+ }++ Ok(Response::Chunk { bytes: res })+ }+ }+ }+}++fn files(root: PathBuf) -> Vec<PathBuf> {+ let mut stack = vec![root];+ let mut res = Vec::new();++ while let Some(path) = stack.pop() {+ if path.is_file() {+ res.push(path);+ } else if let Ok(dir) = fs::read_dir(path) {+ for entry in dir.flatten() {+ stack.push(entry.path());+ }+ }+ }++ res+}++#[cfg(test)]+mod tests {+ use std::path::Path;++ use file_store::gen_temp_path;++ use super::*;++ fn local_step(req: Request) -> Response {+ Response::handle(req).unwrap()+ }++ async fn download<P1: AsRef<Path>, P2: AsRef<Path>>(remote: P1, local: P2) {+ Request::download(remote, local, &local_step).await+ }++ #[tokio::test]+ async fn test_directory() {+ let a = gen_temp_path();+ std::fs::create_dir_all(a.join("test")).unwrap();+ let contents = "this is a test";+ std::fs::write(a.join("test").join("file.txt"), contents).unwrap();++ let b = gen_temp_path();+ download(&a, &b).await;++ assert!(b.join("test").exists());+ assert!(b.join("test").join("file.txt").exists());++ let res = std::fs::read_to_string(b.join("test").join("file.txt")).unwrap();++ assert_eq!(contents, res);+ }++ #[tokio::test]+ async fn test_single_file() {+ let a = gen_temp_path();+ std::fs::create_dir_all(&a).unwrap();+ let contents = "this is a test";+ std::fs::write(a.join("file.txt"), contents).unwrap();++ let b = gen_temp_path();+ download(&a, &b).await;++ assert!(b.join("file.txt").exists());++ let res = std::fs::read_to_string(b.join("file.txt")).unwrap();++ assert_eq!(contents, res);+ }++ #[tokio::test]+ async fn test_overwrite() {+ let a = gen_temp_path();+ std::fs::create_dir_all(&a).unwrap();+ let contents = "this is a test";+ std::fs::write(a.join("file.txt"), contents).unwrap();++ let b = gen_temp_path();+ std::fs::create_dir_all(&b).unwrap();+ std::fs::write(b.join("file.txt"), "this is another test").unwrap();+ download(&a, &b).await;++ assert!(b.join("file.txt").exists());++ let res = std::fs::read_to_string(b.join("file.txt")).unwrap();++ assert_eq!(contents, res);+ }++ #[tokio::test]+ async fn test_keep_non_copied() {+ let a = gen_temp_path();+ std::fs::create_dir_all(a.join("test")).unwrap();+ let contents = "this is a test";+ std::fs::write(a.join("test").join("a.txt"), contents).unwrap();++ let b = gen_temp_path();+ std::fs::create_dir_all(b.join("test")).unwrap();+ std::fs::write(b.join("test").join("b.txt"), contents).unwrap();++ download(&a, &b).await;++ assert!(b.join("test").exists());+ assert!(b.join("test").join("a.txt").exists());+ assert!(b.join("test").join("b.txt").exists());++ let res = std::fs::read_to_string(b.join("test").join("a.txt")).unwrap();+ assert_eq!(contents, res);++ let res = std::fs::read_to_string(b.join("test").join("b.txt")).unwrap();+ assert_eq!(contents, res);+ }++ #[tokio::test]+ async fn test_file_size_edge_case() {+ let content = "a".repeat(CHUNK_SIZE_BYTES - 1);+ let a = gen_temp_path();+ std::fs::create_dir_all(&a).unwrap();+ std::fs::write(a.join("minus_1.txt"), &content).unwrap();+ std::fs::write(a.join("edge.txt"), format!("{}a", &content)).unwrap();+ std::fs::write(a.join("plus_1.txt"), format!("{}aa", &content)).unwrap();++ let b = gen_temp_path();+ download(&a, &b).await;+ assert!(b.join("minus_1.txt").exists());+ assert!(b.join("edge.txt").exists());+ assert!(b.join("plus_1.txt").exists());++ let res = std::fs::read_to_string(b.join("minus_1.txt")).unwrap();+ assert_eq!(content, res);++ let res = std::fs::read_to_string(b.join("edge.txt")).unwrap();+ assert_eq!(format!("{}a", &content), res);++ let res = std::fs::read_to_string(b.join("plus_1.txt")).unwrap();+ assert_eq!(format!("{}aa", &content), res);++ std::fs::remove_dir_all(&a).unwrap();+ std::fs::remove_dir_all(&b).unwrap();+ }+}diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex 1c4025b2..e116ecf1 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -185,26 +185,29 @@ fn create_inverted_index() -> Result<()> {Some(dual_encoder_path)};- let worker = indexer::IndexingWorker::new(IndexerConfig {- host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),- page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),- page_webgraph: Some(IndexerGraphConfig::Local {- path: webgraph_path.to_str().unwrap().to_string(),- }),- output_path: out_path.to_str().unwrap().to_string(),- limit_warc_files: None,- skip_warc_files: None,- warc_source: job.source_config.clone(),- host_centrality_threshold: None,- safety_classifier_path: None,- minimum_clean_words: None,- batch_size: defaults::Indexing::batch_size(),- autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),- dual_encoder: dual_encoder_path.map(|p| IndexerDualEncoderConfig {- model_path: p.to_str().unwrap().to_string(),- page_centrality_rank_threshold: Some(100_000),- }),- });+ let worker = crate::block_on(indexer::IndexingWorker::new(+ IndexerConfig {+ host_centrality_store_path: centrality_path.to_str().unwrap().to_string(),+ page_centrality_store_path: Some(page_centrality_path.to_str().unwrap().to_string()),+ page_webgraph: Some(IndexerGraphConfig::Local {+ path: webgraph_path.to_str().unwrap().to_string(),+ }),+ output_path: out_path.to_str().unwrap().to_string(),+ limit_warc_files: None,+ skip_warc_files: None,+ warc_source: job.source_config.clone(),+ host_centrality_threshold: None,+ safety_classifier_path: None,+ minimum_clean_words: None,+ batch_size: defaults::Indexing::batch_size(),+ autocommit_after_num_inserts: defaults::Indexing::autocommit_after_num_inserts(),+ dual_encoder: dual_encoder_path.map(|p| IndexerDualEncoderConfig {+ model_path: p.to_str().unwrap().to_string(),+ page_centrality_rank_threshold: Some(100_000),+ }),+ }+ .into(),+ ));let index = job.process(&worker);crate::mv(index.path(), &out_path)?;diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rsindex 1eb507b3..f77ab685 100644--- a/crates/core/src/entrypoint/indexer/job.rs+++ b/crates/core/src/entrypoint/indexer/job.rs@@ -85,7 +85,7 @@ impl Job {batch.push(IndexableWebpage::from(record));}- let prepared = worker.prepare_webpages(&batch);+ let prepared = crate::block_on(worker.prepare_webpages(&batch));for webpage in &prepared {if webpage.host_centrality > 0.0 {diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rsindex 04952402..f1fd45c9 100644--- a/crates/core/src/entrypoint/indexer/mod.rs+++ b/crates/core/src/entrypoint/indexer/mod.rs@@ -45,7 +45,7 @@ pub fn run(config: &config::IndexerConfig) -> Result<()> {let job_config: WarcSource = config.warc_source.clone();- let worker = IndexingWorker::new(config.clone());+ let worker = crate::block_on(IndexingWorker::new(config.clone().into()));let indexes = warc_paths.into_par_iter()diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 72b35193..0ba35cb8 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -17,17 +17,15 @@use chrono::Utc;use itertools::Itertools;use std::path::Path;-use std::sync::Mutex;+use std::sync::{Arc, Mutex};use tracing::debug;pub use super::indexable_webpage::IndexableWebpage;pub use super::job::{Job, JobSettings};use crate::backlink_grouper::BacklinkGrouper;-use crate::config::{- IndexerConfig, IndexerDualEncoderConfig, IndexerGraphConfig, LiveIndexConfig,- WebgraphGranularity,-};+use crate::config::{GossipConfig, IndexerConfig, IndexerDualEncoderConfig, WebgraphGranularity};+use crate::distributed::cluster::Cluster;use crate::models::dual_encoder::DualEncoder as DualEncoderModel;use crate::webgraph::remote::RemoteWebgraph;use crate::Result;@@ -40,6 +38,23 @@ use crate::webpage::{safety_classifier, Html, Webpage};const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);+pub enum IndexerGraphConfig {+ Local { path: String },+ Remote { gossip: GossipConfig },+ Existing { cluster: Arc<Cluster> },+}++impl From<crate::config::IndexerGraphConfig> for IndexerGraphConfig {+ fn from(conf: crate::config::IndexerGraphConfig) -> Self {+ match conf {+ crate::config::IndexerGraphConfig::Local { path } => IndexerGraphConfig::Local { path },+ crate::config::IndexerGraphConfig::Remote { gossip } => {+ IndexerGraphConfig::Remote { gossip }+ }+ }+ }+}+pub struct Config {pub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,@@ -53,25 +68,13 @@ impl From<IndexerConfig> for Config {Self {host_centrality_store_path: config.host_centrality_store_path,page_centrality_store_path: config.page_centrality_store_path,- page_webgraph: config.page_webgraph,+ page_webgraph: config.page_webgraph.map(IndexerGraphConfig::from),safety_classifier_path: config.safety_classifier_path,dual_encoder: config.dual_encoder,}}}-impl From<LiveIndexConfig> for Config {- fn from(config: LiveIndexConfig) -> Self {- Self {- host_centrality_store_path: config.host_centrality_store_path,- page_centrality_store_path: config.page_centrality_store_path,- page_webgraph: config.page_webgraph,- safety_classifier_path: config.safety_classifier_path,- dual_encoder: None,- }- }-}-struct DualEncoder {model: DualEncoderModel,page_centrality_rank_threshold: Option<u64>,@@ -83,16 +86,16 @@ pub(super) enum Webgraph {}impl Webgraph {- fn batch_raw_ingoing_edges_with_labels(+ async fn batch_raw_ingoing_edges_with_labels(&self,ids: Vec<NodeID>,limit: EdgeLimit,) -> Vec<Vec<Edge<String>>> {let edges = match self {- Self::Remote(webgraph) => {- crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit))- .unwrap_or_default()- }+ Self::Remote(webgraph) => webgraph+ .batch_raw_ingoing_edges_with_labels(&ids, limit)+ .await+ .unwrap_or_default(),Self::Local(webgraph) => {let mut res = Vec::new();@@ -140,7 +143,7 @@ impl Webgraph {}impl Webgraph {- fn new(config: &IndexerGraphConfig) -> Self {+ async fn new(config: &IndexerGraphConfig) -> Self {match config {IndexerGraphConfig::Local { path } => Self::Local(webgraph::WebgraphBuilder::new(path)@@ -149,10 +152,10 @@ impl Webgraph {),IndexerGraphConfig::Remote { gossip } => {let cluster = crate::start_gossip_cluster_thread(gossip.clone(), None);- Self::Remote(crate::block_on(RemoteWebgraph::new(- cluster,- WebgraphGranularity::Page,- )))+ Self::Remote(RemoteWebgraph::new(cluster, WebgraphGranularity::Page).await)+ }+ IndexerGraphConfig::Existing { cluster } => {+ Self::Remote(RemoteWebgraph::new(cluster.clone(), WebgraphGranularity::Page).await)}}}@@ -172,12 +175,7 @@ pub struct IndexingWorker {}impl IndexingWorker {- pub fn new<C>(config: C) -> Self- where- Config: From<C>,- {- let config = Config::from(config);-+ pub async fn new(config: Config) -> Self {let host_centrality_rank_store = speedy_kv::Db::open_or_create(Path::new(&config.host_centrality_store_path).join("harmonic_rank"),)@@ -196,7 +194,10 @@ impl IndexingWorker {page_centrality_rank_store: config.page_centrality_store_path.as_ref().map(|p| {speedy_kv::Db::open_or_create(Path::new(&p).join("harmonic_rank")).unwrap()}),- page_webgraph: config.page_webgraph.as_ref().map(Webgraph::new),+ page_webgraph: match config.page_webgraph.as_ref() {+ Some(graph) => Some(Webgraph::new(graph).await),+ None => None,+ },safety_classifier: config.safety_classifier_path.as_ref()@@ -390,14 +391,16 @@ impl IndexingWorker {}}- pub fn set_backlinks(&self, pages: &mut [Webpage]) {+ pub async fn set_backlinks(&self, pages: &mut [Webpage]) {if let Some(graph) = self.page_webgraph() {let ids = pages.iter().map(|w| Node::from(w.html.url()).id()).collect::<Vec<_>>();- let backlinks = graph.batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS);+ let backlinks = graph+ .batch_raw_ingoing_edges_with_labels(ids, MAX_BACKLINKS)+ .await;for (page, backlinks) in pages.iter_mut().zip_eq(backlinks) {page.set_backlinks(backlinks.clone());@@ -444,7 +447,7 @@ impl IndexingWorker {}}- pub fn prepare_webpages(&self, batch: &[IndexableWebpage]) -> Vec<Webpage> {+ pub async fn prepare_webpages(&self, batch: &[IndexableWebpage]) -> Vec<Webpage> {let mut res = Vec::with_capacity(batch.len());let mut signal_computer = SignalComputer::new(None);@@ -473,7 +476,7 @@ impl IndexingWorker {self.set_title_embeddings(&mut res);self.set_keyword_embeddings(&mut res);- self.set_backlinks(&mut res);+ self.set_backlinks(&mut res).await;res}@@ -486,28 +489,31 @@ mod tests {use super::*;fn setup_worker(data_path: &Path, threshold: Option<u64>) -> IndexingWorker {- IndexingWorker::new(IndexerConfig {- host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),- page_centrality_store_path: None,- page_webgraph: None,- safety_classifier_path: None,- dual_encoder: Some(IndexerDualEncoderConfig {- model_path: data_path.to_str().unwrap().to_string(),- page_centrality_rank_threshold: threshold,- }),- output_path: crate::gen_temp_path().to_str().unwrap().to_string(),- limit_warc_files: None,- skip_warc_files: None,- warc_source: WarcSource::Local(crate::config::LocalConfig {- folder: crate::gen_temp_path().to_str().unwrap().to_string(),- names: vec!["".to_string()],- }),- host_centrality_threshold: None,- minimum_clean_words: None,- batch_size: 10,- autocommit_after_num_inserts:- crate::config::defaults::Indexing::autocommit_after_num_inserts(),- })+ crate::block_on(IndexingWorker::new(+ IndexerConfig {+ host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ page_centrality_store_path: None,+ page_webgraph: None,+ safety_classifier_path: None,+ dual_encoder: Some(IndexerDualEncoderConfig {+ model_path: data_path.to_str().unwrap().to_string(),+ page_centrality_rank_threshold: threshold,+ }),+ output_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ limit_warc_files: None,+ skip_warc_files: None,+ warc_source: WarcSource::Local(crate::config::LocalConfig {+ folder: crate::gen_temp_path().to_str().unwrap().to_string(),+ names: vec!["".to_string()],+ }),+ host_centrality_threshold: None,+ minimum_clean_words: None,+ batch_size: 10,+ autocommit_after_num_inserts:+ crate::config::defaults::Indexing::autocommit_after_num_inserts(),+ }+ .into(),+ ))}#[test]@@ -534,7 +540,7 @@ mod tests {},];- let webpages = worker.prepare_webpages(&webpages);+ let webpages = crate::block_on(worker.prepare_webpages(&webpages));assert_eq!(webpages.len(), 2);diff --git a/crates/core/src/entrypoint/live_index.rs b/crates/core/src/entrypoint/live_index.rsdeleted file mode 100644index e9ab5145..00000000--- a/crates/core/src/entrypoint/live_index.rs+++ /dev/null@@ -1,124 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::sync::Arc;--use crate::{- config::LiveIndexConfig,- distributed::{- cluster::Cluster,- member::{Member, Service},- sonic::{self, service::sonic_service},- },- inverted_index,- live_index::{IndexManager, LiveIndex},- searcher::{InitialWebsiteResult, LocalSearcher},-};-use anyhow::Result;-use tracing::info;--use super::{- indexer::IndexableWebpage,- search_server::{RetrieveWebsites, Search},-};--sonic_service!(LiveIndexService, [RetrieveWebsites, Search, IndexWebpages]);--pub struct LiveIndexService {- local_searcher: LocalSearcher<Arc<LiveIndex>>,- index: Arc<LiveIndex>,- // dropping the handle leaves the cluster- #[allow(unused)]- cluster_handle: Cluster,-}--impl LiveIndexService {- async fn new(config: LiveIndexConfig) -> Result<Self> {- let manager = IndexManager::new(config.clone())?;- let local_searcher = LocalSearcher::new(manager.index());-- let index = manager.index();-- tokio::task::spawn(manager.run());-- let cluster_handle = Cluster::join(- Member {- id: config.cluster_id,- service: Service::LiveIndex {- host: config.host,- split_id: config.split_id,- },- },- config.gossip_addr,- config.gossip_seed_nodes.unwrap_or_default(),- )- .await?;-- Ok(Self {- local_searcher,- cluster_handle,- index,- })- }-}--impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {- type Response = Option<Vec<inverted_index::RetrievedWebpage>>;- async fn handle(self, server: &LiveIndexService) -> Self::Response {- server- .local_searcher- .retrieve_websites(&self.websites, &self.query)- .ok()- }-}--impl sonic::service::Message<LiveIndexService> for Search {- type Response = Option<InitialWebsiteResult>;- async fn handle(self, server: &LiveIndexService) -> Self::Response {- server.local_searcher.search_initial(&self.query, true).ok()- }-}--#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct IndexWebpages {- pages: Vec<IndexableWebpage>,-}--impl sonic::service::Message<LiveIndexService> for IndexWebpages {- type Response = ();-- async fn handle(self, server: &LiveIndexService) -> Self::Response {- server.index.insert(&self.pages)- }-}--pub async fn serve(config: LiveIndexConfig) -> Result<()> {- let addr = config.host;-- let server = LiveIndexService::new(config)- .await?- .bind(&addr)- .await- .unwrap();-- info!("live index is ready to accept requests on {}", addr);-- loop {- if let Err(e) = server.accept().await {- tracing::error!("{:?}", e);- }- }-}diff --git a/crates/core/src/entrypoint/live_index/mod.rs b/crates/core/src/entrypoint/live_index/mod.rsnew file mode 100644index 00000000..d782a04f--- /dev/null+++ b/crates/core/src/entrypoint/live_index/mod.rs@@ -0,0 +1,436 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++#[cfg(test)]+mod tests;++use std::{+ net::SocketAddr,+ path::{Path, PathBuf},+ sync::Arc,+ time::Duration,+};++use crate::{+ ampc::dht::ShardId,+ config::LiveIndexConfig,+ distributed::{+ cluster::Cluster,+ member::{LiveIndexState, Member, Service},+ remote_cp,+ sonic::{self, service::sonic_service},+ },+ inverted_index,+ live_index::{IndexManager, LiveIndex},+ searcher::{InitialWebsiteResult, LocalSearcher},+};+use anyhow::{Context, Result};+use futures::stream::FuturesUnordered;+use itertools::Itertools;+use simple_wal::Wal;+use tokio::sync::Mutex;+use tokio_stream::StreamExt;+use tracing::info;++use super::{+ indexer::{self, IndexableWebpage},+ search_server::{RetrieveWebsites, Search},+};++const INDEXING_TIMEOUT: Duration = Duration::from_secs(60);+const INDEXING_RETRIES: usize = 3;++sonic_service!(+ LiveIndexService,+ [+ RetrieveWebsites,+ Search,+ IndexWebpages,+ GetIndexPath,+ RemoteDownload+ ]+);++fn start_manager(index: Arc<LiveIndex>) {+ let manager = IndexManager::new(index);+ std::thread::spawn(|| manager.run());+}++struct FileDownloadStepper {+ conn: Mutex<sonic::service::Connection<LiveIndexService>>,+}++impl remote_cp::Stepper for FileDownloadStepper {+ async fn step(&self, req: remote_cp::Request) -> remote_cp::Response {+ self.conn+ .lock()+ .await+ .send(RemoteDownload { req })+ .await+ .unwrap()+ }+}++async fn other_replicas(cluster: &Cluster, shard: &ShardId, id: &str) -> Vec<SocketAddr> {+ tokio::time::sleep(std::time::Duration::from_secs(10)).await;+ cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if member.id != id {+ if let Service::LiveIndex {+ host,+ shard: member_shard,+ state,+ } = member.service+ {+ if &member_shard == shard && matches!(state, LiveIndexState::Ready) {+ return Some(host);+ }+ }+ }++ None+ })+ .collect()+}++async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal) -> Result<()> {+ let self_node = cluster+ .self_node()+ .expect("cluster should not be joined as spectator");++ let Service::LiveIndex {+ host,+ shard,+ state: _,+ } = self_node.service.clone()+ else {+ panic!("self node should be a live index")+ };++ let mut others = other_replicas(&cluster, &shard, &self_node.id).await;++ if let Some(other) = others.pop() {+ let mut conn: sonic::service::Connection<LiveIndexService> =+ sonic::service::Connection::create(other).await?;+ index.delete_all_pages();+ let local_path = index.path();+ let remote_path = conn.send(GetIndexPath).await?;++ remote_cp::Request::download(+ remote_path,+ local_path,+ &FileDownloadStepper {+ conn: Mutex::new(conn),+ },+ )+ .await;+ }++ let mut wal = temp_wal+ .lock()+ .await+ .take()+ .expect("temp_wal should always exist before setup has been run");++ for pages in wal.iter().unwrap().chunks(512).into_iter() {+ let pages: Vec<_> = pages.into_iter().collect();+ index.insert(&pages);+ }++ wal.clear().unwrap();++ start_manager(index);++ cluster+ .set_service(Service::LiveIndex {+ host,+ shard,+ state: LiveIndexState::Ready,+ })+ .await?;++ Ok(())+}++type TempWal = Arc<Mutex<Option<Wal<IndexableWebpage>>>>;++pub struct LiveIndexService {+ local_searcher: LocalSearcher<Arc<LiveIndex>>,+ temp_wal: TempWal,+ index: Arc<LiveIndex>,+ cluster_handle: Arc<Cluster>,+}++impl LiveIndexService {+ async fn new(config: LiveIndexConfig) -> Result<Self> {+ let cluster_handle = Arc::new(+ Cluster::join(+ Member {+ id: config.cluster_id.clone(),+ service: Service::LiveIndex {+ host: config.host,+ shard: config.shard_id,+ state: crate::distributed::member::LiveIndexState::InSetup,+ },+ },+ config.gossip_addr,+ config.gossip_seed_nodes.clone().unwrap_or_default(),+ )+ .await?,+ );+ let index_path = Path::new(&config.index_path);++ let index = Arc::new(+ LiveIndex::new(+ index_path.join("index"),+ indexer::worker::Config {+ host_centrality_store_path: config.host_centrality_store_path.clone(),+ page_centrality_store_path: config.page_centrality_store_path.clone(),+ page_webgraph: Some(indexer::worker::IndexerGraphConfig::Existing {+ cluster: cluster_handle.clone(),+ }),+ safety_classifier_path: config.safety_classifier_path.clone(),+ dual_encoder: None,+ },+ )+ .await?,+ );+ let local_searcher = LocalSearcher::new(index.clone());++ let temp_wal = Arc::new(Mutex::new(Some(Wal::open(index_path.join("wal.temp"))?)));++ Ok(Self {+ local_searcher,+ cluster_handle,+ index,+ temp_wal,+ })+ }++ fn background_setup(&self) {+ let index = self.index.clone();+ let temp_wal = self.temp_wal.clone();+ let cluster = self.cluster_handle.clone();++ tokio::task::spawn(async move { setup(index, cluster, temp_wal).await.unwrap() });+ }++ async fn index_webpages_in_replicas(+ &self,+ webpages: &[IndexableWebpage],+ consistency_fraction: f64,+ ) -> Result<(), IndexingError> {+ let self_member = self+ .cluster_handle+ .self_node()+ .expect("node should be participating part of cluster");++ let self_id = self_member.id.clone();+ let Service::LiveIndex {+ host: _,+ shard: self_shard,+ state: _,+ } = self_member.service+ else {+ panic!("self_member should always be a live index")+ };++ let live_indexes: Vec<_> = self+ .cluster_handle+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::LiveIndex { host, shard, state } = member.service {+ if member.id != self_id && shard == self_shard {+ Some(RemoteIndex { host, state })+ } else {+ None+ }+ } else {+ None+ }+ })+ .collect();++ let ready = live_indexes+ .iter()+ .filter(|index| matches!(index.state, LiveIndexState::Ready))+ .count();++ let mut futures = FuturesUnordered::new();++ for index in live_indexes {+ let pages = webpages.to_vec();++ futures.push(tokio::spawn(async move {+ match index.index_webpages_without_consistency(&pages).await {+ Ok(_) => Ok(index),+ Err(e) => Err(e),+ }+ }));+ }++ let mut missing_responses =+ (((ready as f64) * consistency_fraction).ceil() as u64).min(ready as u64);++ while let Some(Ok(index)) = futures.next().await {+ if let Ok(index) = index {+ if matches!(index.state, LiveIndexState::Ready) {+ missing_responses = missing_responses.saturating_sub(1);+ }++ if missing_responses == 0 {+ break;+ }+ }+ }++ if missing_responses > 0 {+ return Err(IndexingError::InsufficientReplication);+ }++ Ok(())+ }+}++impl sonic::service::Message<LiveIndexService> for RetrieveWebsites {+ type Response = Option<Vec<inverted_index::RetrievedWebpage>>;+ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ server+ .local_searcher+ .retrieve_websites(&self.websites, &self.query)+ .ok()+ }+}++impl sonic::service::Message<LiveIndexService> for Search {+ type Response = Option<InitialWebsiteResult>;+ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ server.local_searcher.search_initial(&self.query, true).ok()+ }+}++#[derive(Clone, Debug)]+struct RemoteIndex {+ host: SocketAddr,+ state: LiveIndexState,+}++impl RemoteIndex {+ async fn index_webpages_without_consistency(+ &self,+ webpages: &[IndexableWebpage],+ ) -> Result<()> {+ let req = IndexWebpages {+ pages: webpages.to_vec(),+ consistency_fraction: None,+ };++ for _ in 0..INDEXING_RETRIES {+ let mut conn: sonic::service::Connection<LiveIndexService> =+ sonic::service::Connection::create(self.host)+ .await+ .with_context(|| format!("failed to connect to {}", self.host))?;++ if conn+ .send_with_timeout(req.clone(), INDEXING_TIMEOUT)+ .await+ .is_ok()+ {+ return Ok(());+ }+ }++ Err(anyhow::anyhow!("failed to replicate webpages"))+ }+}++#[derive(thiserror::Error, Debug, bincode::Encode, bincode::Decode)]+pub enum IndexingError {+ #[error("failed to replicate to the necessary quorom")]+ InsufficientReplication,+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct IndexWebpages {+ pub pages: Vec<IndexableWebpage>,+ pub consistency_fraction: Option<f64>,+}++impl sonic::service::Message<LiveIndexService> for IndexWebpages {+ type Response = Result<(), IndexingError>;++ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ if let Some(wal) = server.temp_wal.lock().await.as_mut() {+ wal.batch_write(self.pages.iter()).unwrap();+ } else {+ server.index.insert(&self.pages);++ if let Some(consistency_fraction) = self.consistency_fraction {+ server+ .index_webpages_in_replicas(&self.pages, consistency_fraction)+ .await?;+ }+ }++ Ok(())+ }+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct GetIndexPath;++impl sonic::service::Message<LiveIndexService> for GetIndexPath {+ type Response = PathBuf;++ async fn handle(self, server: &LiveIndexService) -> Self::Response {+ server.index.path()+ }+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct RemoteDownload {+ req: remote_cp::Request,+}+impl sonic::service::Message<LiveIndexService> for RemoteDownload {+ type Response = remote_cp::Response;++ async fn handle(self, _: &LiveIndexService) -> Self::Response {+ remote_cp::Response::handle(self.req).unwrap()+ }+}++pub async fn serve(config: LiveIndexConfig) -> Result<()> {+ let addr = config.host;++ let service = LiveIndexService::new(config).await?;++ service.background_setup();++ let server = service.bind(&addr).await.unwrap();++ info!("live index is ready to accept requests on {}", addr);++ loop {+ if let Err(e) = server.accept().await {+ tracing::error!("{:?}", e);+ }+ }+}diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rsnew file mode 100644index 00000000..048f32f0--- /dev/null+++ b/crates/core/src/entrypoint/live_index/tests.rs@@ -0,0 +1,430 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::{+ config::LiveIndexConfig, entrypoint::search_server, inverted_index, live_index::LiveIndex,+ Result,+};+use std::{net::SocketAddr, sync::Arc};++use file_store::gen_temp_path;++use crate::{+ ampc::dht::ShardId,+ distributed::{+ cluster::Cluster,+ member::{LiveIndexState, Service},+ sonic,+ },+ entrypoint::{indexer::IndexableWebpage, live_index::IndexWebpages},+ free_socket_addr,+};++use super::LiveIndexService;++struct RemoteIndex {+ host: SocketAddr,+ shard: ShardId,+ gossip_addr: SocketAddr,+ underlying_index: Arc<LiveIndex>,+ cluster: Arc<Cluster>,+}++impl RemoteIndex {+ async fn start(shard: ShardId, gossip_seed: Vec<SocketAddr>) -> Result<Self> {+ let path = gen_temp_path();++ let host = free_socket_addr();+ let gossip_addr = free_socket_addr();++ let gossip_seed = if gossip_seed.is_empty() {+ None+ } else {+ Some(gossip_seed)+ };++ let config = LiveIndexConfig {+ user_agent: crate::config::UserAgent {+ full: "TestBot".to_string(),+ token: "TestBot".to_string(),+ },+ robots_txt_cache_sec: 60 * 60,+ min_politeness_factor: 1,+ start_politeness_factor: 3,+ min_crawl_delay_ms: 5_000,+ max_crawl_delay_ms: 60_000,+ max_politeness_factor: 2048,+ max_url_slowdown_retry: 5,+ timeout_seconds: 30,+ host_centrality_store_path: path+ .as_path()+ .join("host_centrality")+ .to_str()+ .unwrap()+ .to_string(),+ page_centrality_store_path: None,+ safety_classifier_path: None,+ host_centrality_threshold: None,+ minimum_clean_words: None,+ cluster_id: "test-cluster".to_string(),+ gossip_seed_nodes: gossip_seed,+ gossip_addr,+ shard_id: shard,+ index_path: path.as_path().join("index").to_str().unwrap().to_string(),+ linear_model_path: None,+ lambda_model_path: None,+ host,+ collector: Default::default(),+ snippet: Default::default(),+ };++ let service = LiveIndexService::new(config).await?;+ let cluster = service.cluster_handle.clone();+ let index = service.index.clone();++ service.background_setup();++ let server = service.bind(&host).await.unwrap();++ tokio::task::spawn(async move {+ loop {+ if let Err(e) = server.accept().await {+ tracing::error!("{:?}", e);+ }+ }+ });++ Ok(Self {+ host,+ shard,+ gossip_addr,+ underlying_index: index,+ cluster,+ })+ }++ async fn conn(&self) -> Result<sonic::service::Connection<LiveIndexService>> {+ Ok(sonic::service::Connection::create(self.host).await?)+ }++ async fn index_pages(+ &self,+ pages: Vec<IndexableWebpage>,+ consistency_fraction: Option<f64>,+ ) -> Result<()> {+ self.conn()+ .await?+ .send(IndexWebpages {+ pages,+ consistency_fraction,+ })+ .await??;++ Ok(())+ }++ async fn await_ready(&self, cluster: &Cluster) {+ cluster+ .await_member(|member| {+ if let Service::LiveIndex { host, shard, state } = member.service.clone() {+ self.shard == shard+ && matches!(state, LiveIndexState::Ready)+ && host == self.host+ } else {+ false+ }+ })+ .await;+ }++ async fn search(&self, query: &str) -> Result<Vec<inverted_index::RetrievedWebpage>> {+ let mut conn = self.conn().await?;++ let websites: Vec<inverted_index::WebpagePointer> = conn+ .send(search_server::Search {+ query: query.to_string().into(),+ })+ .await?+ .map(|res| {+ res.websites+ .into_iter()+ .map(|page| page.pointer().clone())+ .collect()+ })+ .unwrap_or_default();++ Ok(conn+ .send(search_server::RetrieveWebsites {+ websites,+ query: query.to_string(),+ })+ .await?+ .unwrap())+ }++ async fn commit_underlying(&self) {+ self.underlying_index.commit();+ }++ async fn kill(self) -> Result<()> {+ self.cluster.remove_service().await?;++ Ok(())+ }+}++const CLUSTER_ID: &str = "test-cluster";++#[tokio::test]+async fn test_shard_without_replica() -> Result<()> {+ let shard1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;+ let shard2 = RemoteIndex::start(ShardId::new(2), vec![shard1.gossip_addr]).await?;++ let cluster = Cluster::join_as_spectator(+ CLUSTER_ID.to_string(),+ free_socket_addr(),+ vec![shard1.gossip_addr],+ )+ .await?;++ shard1.await_ready(&cluster).await;+ shard2.await_ready(&cluster).await;++ shard1+ .index_pages(+ vec![IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ None,+ )+ .await?;+ shard2+ .index_pages(+ vec![IndexableWebpage {+ url: "https://b.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ None,+ )+ .await?;++ shard1.commit_underlying().await;+ shard2.commit_underlying().await;++ let res1 = shard1.search("test").await?;++ assert_eq!(res1.len(), 1);+ assert_eq!(res1[0].url, "https://a.com/");++ let res2 = shard2.search("test").await?;+ assert_eq!(res2.len(), 1);+ assert_eq!(res2[0].url, "https://b.com/");++ Ok(())+}++#[tokio::test]+async fn test_replica_no_fails() -> Result<()> {+ let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;+ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;++ let cluster = Cluster::join_as_spectator(+ CLUSTER_ID.to_string(),+ free_socket_addr(),+ vec![rep1.gossip_addr],+ )+ .await?;++ rep1.await_ready(&cluster).await;+ rep2.await_ready(&cluster).await;++ rep1.index_pages(+ vec![IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ Some(1.0),+ )+ .await?;+ rep2.index_pages(+ vec![IndexableWebpage {+ url: "https://b.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ Some(1.0),+ )+ .await?;++ rep1.commit_underlying().await;+ rep2.commit_underlying().await;++ let res1 = rep1.search("test").await?;++ assert_eq!(res1.len(), 2);++ let res2 = rep2.search("test").await?;+ assert_eq!(res2.len(), 2);++ Ok(())+}++#[tokio::test]+async fn test_replica_setup_after_inserts() -> Result<()> {+ let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;++ let cluster = Cluster::join_as_spectator(+ CLUSTER_ID.to_string(),+ free_socket_addr(),+ vec![rep1.gossip_addr],+ )+ .await?;++ rep1.await_ready(&cluster).await;++ rep1.index_pages(+ vec![IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ Some(1.0),+ )+ .await?;+ rep1.index_pages(+ vec![IndexableWebpage {+ url: "https://b.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ Some(1.0),+ )+ .await?;++ rep1.commit_underlying().await;++ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;+ rep2.await_ready(&cluster).await;++ rep2.commit_underlying().await;++ let res1 = rep1.search("test").await?;++ assert_eq!(res1.len(), 2);++ let res2 = rep2.search("test").await?;+ assert_eq!(res2.len(), 2);++ Ok(())+}++#[tokio::test]+async fn test_replica_recovery() -> Result<()> {+ let rep1 = RemoteIndex::start(ShardId::new(1), vec![]).await?;+ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;++ let cluster = Cluster::join_as_spectator(+ CLUSTER_ID.to_string(),+ free_socket_addr(),+ vec![rep1.gossip_addr],+ )+ .await?;++ rep1.await_ready(&cluster).await;+ rep2.await_ready(&cluster).await;++ rep1.index_pages(+ vec![IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ Some(1.0),+ )+ .await?;++ rep2.kill().await?;++ loop {+ if let Ok(_) = rep1+ .index_pages(+ vec![IndexableWebpage {+ url: "https://b.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }],+ Some(1.0),+ )+ .await+ {+ break;+ }++ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+ }++ rep1.commit_underlying().await;++ let rep2 = RemoteIndex::start(ShardId::new(1), vec![rep1.gossip_addr]).await?;+ rep2.await_ready(&cluster).await;++ rep2.commit_underlying().await;++ let res1 = rep1.search("test").await?;++ assert_eq!(res1.len(), 2);++ let res2 = rep2.search("test").await?;+ assert_eq!(res2.len(), 2);++ Ok(())+}diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 8a10b5c8..0efdce1e 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -63,6 +63,13 @@ impl InvertedIndex {let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());if let Some(limit) = collector.top_docs().max_docs().cloned() {+ if limit.segments == 0 {+ return Ok(InitialSearchResult {+ num_websites: approx_count::Count::Exact(0),+ top_websites: vec![],+ });+ }+let docs_per_segment = (limit.total_docs / limit.segments) as u64;query = Box::new(ShortCircuitQuery::new(query, docs_per_segment));diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 730600c5..23b43519 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -47,6 +47,7 @@ pub mod autosuggest;mod backlink_grouper;pub mod bangs;mod bincode_utils;+mod block_on;pub mod canon_index;mod collector;pub mod config;@@ -93,17 +94,7 @@ pub mod webgraph;pub mod webpage;mod widgets;-static TOKIO_RUNTIME: std::sync::LazyLock<tokio::runtime::Runtime> =- std::sync::LazyLock::new(|| {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- });--pub fn block_on<F: std::future::Future>(f: F) -> F::Output {- TOKIO_RUNTIME.block_on(f)-}+pub use block_on::block_on;#[derive(Error, Debug)]pub enum Error {diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex 9110b16e..620df532 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -28,8 +28,8 @@ use tantivy::index::SegmentId;use std::collections::{HashMap, HashSet};use crate::{- config::{LiveIndexConfig, SnippetConfig},- entrypoint::indexer::{IndexableWebpage, IndexingWorker},+ config::SnippetConfig,+ entrypoint::indexer::{self, IndexableWebpage, IndexingWorker},live_index::{BATCH_SIZE, TTL},searcher::SearchableIndex,Result,@@ -84,17 +84,19 @@ pub struct InnerIndex {}impl InnerIndex {- pub fn new(config: LiveIndexConfig) -> Result<Self> {- let path = Path::new(&config.index_path);- let mut index = crate::index::Index::open(path.join("index"))?;+ pub async fn new<P: AsRef<Path>>(+ path: P,+ indexer_worker_config: indexer::worker::Config,+ ) -> Result<Self> {+ let mut index = crate::index::Index::open(path.as_ref().join("index"))?;index.prepare_writer()?;- let write_ahead_log = Wal::open(path.join("wal"))?;+ let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;let wal_count = write_ahead_log.iter()?.count();- let worker = IndexingWorker::new(config.clone());+ let worker = IndexingWorker::new(indexer_worker_config).await;- let meta = Meta::open_or_create(path.join("meta.json"));+ let meta = Meta::open_or_create(path.as_ref().join("meta.json"));Ok(Self {index,@@ -102,7 +104,7 @@ impl InnerIndex {indexing_worker: worker,has_inserts: wal_count > 0,meta,- path: path.to_path_buf(),+ path: path.as_ref().to_path_buf(),})}@@ -192,7 +194,9 @@ impl InnerIndex {created: Utc::now(),})}+ }+ fn save_meta(&self) {self.meta.save(self.path.join("meta.json"));}@@ -200,12 +204,23 @@ impl InnerIndex {&self.index}+ pub fn delete_all_pages(&mut self) {+ let segments = self.index.inverted_index.segment_ids();+ self.index+ .inverted_index+ .delete_segments_by_id(&segments)+ .unwrap();++ self.meta = Meta::default();+ self.save_meta()+ }+pub fn insert(&mut self, pages: &[IndexableWebpage]) {self.write_ahead_log.batch_write(pages.iter()).unwrap();self.has_inserts = true;}- pub fn commit(&mut self) {+ pub async fn commit(&mut self) {for batch in self.write_ahead_log.iter()@@ -214,7 +229,7 @@ impl InnerIndex {.into_iter(){let batch: Vec<_> = batch.collect();- for webpage in self.indexing_worker.prepare_webpages(&batch) {+ for webpage in self.indexing_worker.prepare_webpages(&batch).await {self.index.insert(&webpage).unwrap();}}@@ -226,6 +241,10 @@ impl InnerIndex {pub fn has_inserts(&self) -> bool {self.has_inserts}++ pub fn path(&self) -> &Path {+ &self.path+ }}pub struct LiveIndex {@@ -233,17 +252,24 @@ pub struct LiveIndex {}impl LiveIndex {- pub fn new(config: LiveIndexConfig) -> Result<Self> {+ pub async fn new<P: AsRef<Path>>(+ path: P,+ indexer_worker_config: indexer::worker::Config,+ ) -> Result<Self> {Ok(Self {- inner: Arc::new(RwLock::new(InnerIndex::new(config)?)),+ inner: Arc::new(RwLock::new(+ InnerIndex::new(path, indexer_worker_config).await?,+ )),})}pub fn commit(&self) {- self.inner- .write()- .unwrap_or_else(|e| e.into_inner())- .commit();+ futures::executor::block_on(+ self.inner+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .commit(),+ );}pub fn prune_segments(&self) {@@ -285,4 +311,19 @@ impl LiveIndex {.index.set_snippet_config(config)}++ pub fn path(&self) -> PathBuf {+ self.inner+ .read()+ .unwrap_or_else(|e| e.into_inner())+ .path+ .to_path_buf()+ }++ pub fn delete_all_pages(&self) {+ self.inner+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .delete_all_pages();+ }}diff --git a/crates/core/src/live_index/index_manager.rs b/crates/core/src/live_index/index_manager.rsindex dca7a6ac..1f0e902a 100644--- a/crates/core/src/live_index/index_manager.rs+++ b/crates/core/src/live_index/index_manager.rs@@ -18,24 +18,20 @@ use std::sync::Arc;use chrono::Utc;-use crate::config::LiveIndexConfig;-use super::{LiveIndex, AUTO_COMMIT_INTERVAL, COMPACT_INTERVAL, EVENT_LOOP_INTERVAL, PRUNE_INTERVAL,};-use crate::Result;pub struct IndexManager {index: Arc<LiveIndex>,}impl IndexManager {- pub fn new(config: LiveIndexConfig) -> Result<Self> {- let index = Arc::new(LiveIndex::new(config.clone())?);- Ok(Self { index })+ pub fn new(index: Arc<LiveIndex>) -> Self {+ Self { index }}- pub async fn run(self) {+ pub fn run(self) {let mut last_commit = Utc::now();let mut last_prune = Utc::now();let mut last_compact = Utc::now();@@ -56,11 +52,7 @@ impl IndexManager {last_compact = Utc::now();}- tokio::time::sleep(EVENT_LOOP_INTERVAL).await;+ std::thread::sleep(EVENT_LOOP_INTERVAL);}}-- pub fn index(&self) -> Arc<LiveIndex> {- self.index.clone()- }}diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 3c58526c..dc5c6653 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -756,28 +756,31 @@ mod tests {}fn setup_worker(data_path: &Path) -> IndexingWorker {- IndexingWorker::new(IndexerConfig {- host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),- page_centrality_store_path: None,- page_webgraph: None,- safety_classifier_path: None,- dual_encoder: Some(IndexerDualEncoderConfig {- model_path: data_path.to_str().unwrap().to_string(),- page_centrality_rank_threshold: None,- }),- output_path: crate::gen_temp_path().to_str().unwrap().to_string(),- limit_warc_files: None,- skip_warc_files: None,- warc_source: WarcSource::Local(crate::config::LocalConfig {- folder: crate::gen_temp_path().to_str().unwrap().to_string(),- names: vec!["".to_string()],- }),- host_centrality_threshold: None,- minimum_clean_words: None,- batch_size: 10,- autocommit_after_num_inserts:- crate::config::defaults::Indexing::autocommit_after_num_inserts(),- })+ crate::block_on(IndexingWorker::new(+ IndexerConfig {+ host_centrality_store_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ page_centrality_store_path: None,+ page_webgraph: None,+ safety_classifier_path: None,+ dual_encoder: Some(IndexerDualEncoderConfig {+ model_path: data_path.to_str().unwrap().to_string(),+ page_centrality_rank_threshold: None,+ }),+ output_path: crate::gen_temp_path().to_str().unwrap().to_string(),+ limit_warc_files: None,+ skip_warc_files: None,+ warc_source: WarcSource::Local(crate::config::LocalConfig {+ folder: crate::gen_temp_path().to_str().unwrap().to_string(),+ names: vec!["".to_string()],+ }),+ host_centrality_threshold: None,+ minimum_clean_words: None,+ batch_size: 10,+ autocommit_after_num_inserts:+ crate::config::defaults::Indexing::autocommit_after_num_inserts(),+ }+ .into(),+ ))}#[test]diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex d6df9aa2..5fcaa60f 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -453,7 +453,7 @@ whereasync fn search_initial_from_live(&self,query: &SearchQuery,- ) -> Option<Vec<live::InitialSearchResultSplit>> {+ ) -> Option<Vec<live::InitialSearchResultShard>> {match &self.live_searcher {Some(searcher) => Some(searcher.search_initial(query).await),None => None,@@ -482,7 +482,7 @@ where&self,query: &SearchQuery,initial_results: Vec<distributed::InitialSearchResultShard>,- live_results: Vec<live::InitialSearchResultSplit>,+ live_results: Vec<live::InitialSearchResultShard>,) -> (Vec<ScoredWebpagePointer>, bool) {let mut collector =BucketCollector::new(NUM_PIPELINE_RANKING_RESULTS, self.collector_config.clone());@@ -542,7 +542,7 @@ where.unwrap_or_default();let pointer = live::ScoredWebpagePointer {website: RecallRankingWebpage::new(website, inbound),- split_id: result.split_id.clone(),+ shard_id: result.shard_id,};let pointer = ScoredWebpagePointer::Live(pointer);diff --git a/crates/core/src/searcher/live.rs b/crates/core/src/searcher/live.rsindex 3e1eaa52..a138f4c3 100644--- a/crates/core/src/searcher/live.rs+++ b/crates/core/src/searcher/live.rs@@ -15,16 +15,16 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ ampc::dht::ShardId,distributed::{cluster::Cluster,- member::Service,+ member::{LiveIndexState, Service},sonic::replication::{AllShardsSelector, RandomReplicaSelector, RemoteClient, ReplicatedClient, Shard,- ShardIdentifier, ShardedClient, SpecificShardSelector,+ ShardedClient, SpecificShardSelector,},},entrypoint::search_server::{self, SearchService},- feed::scheduler::SplitId,inverted_index::{RetrievedWebpage, WebpagePointer},ranking::pipeline::{PrecisionRankingWebpage, RecallRankingWebpage},};@@ -41,15 +41,13 @@ use super::{InitialWebsiteResult, SearchQuery};#[derive(Clone, Debug)]pub struct ScoredWebpagePointer {pub website: RecallRankingWebpage,- pub split_id: SplitId,+ pub shard_id: ShardId,}-impl ShardIdentifier for SplitId {}-#[derive(Debug)]-pub struct InitialSearchResultSplit {+pub struct InitialSearchResultShard {pub local_result: InitialWebsiteResult,- pub split_id: SplitId,+ pub shard_id: ShardId,}pub struct LiveSearcher {@@ -61,11 +59,13 @@ impl LiveSearcher {Self { cluster }}- async fn client(&self) -> ShardedClient<SearchService, SplitId> {+ async fn client(&self) -> ShardedClient<SearchService, ShardId> {let mut shards = HashMap::new();for member in self.cluster.members().await {- if let Service::LiveIndex { host, split_id } = member.service {- shards.entry(split_id).or_insert_with(Vec::new).push(host);+ if let Service::LiveIndex { host, shard, state } = member.service {+ if matches!(state, LiveIndexState::Ready) {+ shards.entry(shard).or_insert_with(Vec::new).push(host);+ }}}@@ -83,8 +83,8 @@ impl LiveSearcher {async fn retrieve_webpages_from_shard(&self,- split: SplitId,- client: &ShardedClient<SearchService, SplitId>,+ shard: ShardId,+ client: &ShardedClient<SearchService, ShardId>,query: &str,pointers: Vec<(usize, WebpagePointer)>,) -> Vec<(usize, RetrievedWebpage)> {@@ -96,7 +96,7 @@ impl LiveSearcher {websites: pointers,query: query.to_string(),},- &SpecificShardSelector(split),+ &SpecificShardSelector(shard),&RandomReplicaSelector,).await@@ -115,7 +115,7 @@ impl LiveSearcher {}impl SearchClient for LiveSearcher {- async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultSplit> {+ async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {let client = self.client().await;let mut results = Vec::new();@@ -131,9 +131,9 @@ impl SearchClient for LiveSearcher {{for (shard_id, mut res) in res {if let Some((_, Some(res))) = res.pop() {- results.push(InitialSearchResultSplit {+ results.push(InitialSearchResultShard {local_result: res,- split_id: shard_id,+ shard_id,});}}@@ -152,7 +152,7 @@ impl SearchClient for LiveSearcher {for (i, pointer) in top_websites {pointers- .entry(pointer.split_id.clone())+ .entry(pointer.shard_id).or_default().push((*i, pointer.website.pointer().clone()));@@ -185,7 +185,7 @@ pub trait SearchClient {fn search_initial(&self,query: &SearchQuery,- ) -> impl Future<Output = Vec<InitialSearchResultSplit>> + Send;+ ) -> impl Future<Output = Vec<InitialSearchResultShard>> + Send;fn retrieve_webpages(&self,top_websites: &[(usize, ScoredWebpagePointer)],diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex ad6ec8d1..7dddcad9 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -83,6 +83,16 @@ pub struct SearchQuery {pub signal_coefficients: SignalCoefficients,}+#[cfg(test)]+impl From<String> for SearchQuery {+ fn from(query: String) -> Self {+ Self {+ query,+ ..Default::default()+ }+ }+}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct InitialWebsiteResult {pub num_websites: Count,