web search done right
Re-open live index after it has been downloaded from replica (#227)
Mikkel Denker950862be9c83
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex 7e572add..7b0a7587 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -38,6 +38,7 @@ use crate::webpage::{safety_classifier, Html, Webpage};const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);+#[derive(Clone)]pub enum IndexerGraphConfig {Local { path: String },Remote { gossip: GossipConfig },@@ -55,6 +56,7 @@ impl From<crate::config::IndexerGraphConfig> for IndexerGraphConfig {}}+#[derive(Clone)]pub struct Config {pub host_centrality_store_path: String,pub page_centrality_store_path: Option<String>,diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rsindex 0a921946..c0e598a8 100644--- a/crates/core/src/entrypoint/live_index/search_server.rs+++ b/crates/core/src/entrypoint/live_index/search_server.rs@@ -146,6 +146,7 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)},).await;+ index.re_open().unwrap();}let mut wal = temp_wal@@ -202,7 +203,7 @@ impl LiveIndexService {let index = Arc::new(LiveIndex::new(- index_path.join("index"),+ index_path,indexer::worker::Config {host_centrality_store_path: config.host_centrality_store_path.clone(),page_centrality_store_path: config.page_centrality_store_path.clone(),diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rsindex 3d376769..26c9afb2 100644--- a/crates/core/src/entrypoint/live_index/tests.rs+++ b/crates/core/src/entrypoint/live_index/tests.rs@@ -15,10 +15,14 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- config::LiveIndexConfig, entrypoint::search_server, inverted_index, live_index::LiveIndex,+ config::LiveIndexConfig,+ entrypoint::search_server,+ inverted_index,+ live_index::LiveIndex,+ searcher::{LocalSearcher, SearchQuery},Result,};-use std::{net::SocketAddr, sync::Arc};+use std::{net::SocketAddr, path::Path, sync::Arc};use file_store::gen_temp_path;@@ -35,6 +39,30 @@ use crate::{use super::LiveIndexService;+fn config<P: AsRef<Path>>(path: P) -> LiveIndexConfig {+ LiveIndexConfig {+ host_centrality_store_path: path+ .as_ref()+ .join("host_centrality")+ .to_str()+ .unwrap()+ .to_string(),+ page_centrality_store_path: None,+ safety_classifier_path: None,+ host_centrality_threshold: None,+ minimum_clean_words: None,+ gossip_seed_nodes: None,+ gossip_addr: free_socket_addr(),+ shard_id: ShardId::new(0),+ index_path: path.as_ref().join("index").to_str().unwrap().to_string(),+ linear_model_path: None,+ lambda_model_path: None,+ host: free_socket_addr(),+ collector: Default::default(),+ snippet: Default::default(),+ }+}+struct RemoteIndex {host: SocketAddr,shard: ShardId,@@ -46,37 +74,16 @@ struct RemoteIndex {impl RemoteIndex {async fn start(shard: ShardId, gossip_seed: Vec<SocketAddr>) -> Result<Self> {let path = gen_temp_path();+ let mut config = config(&path);- let host = free_socket_addr();- let gossip_addr = free_socket_addr();-- let gossip_seed = if gossip_seed.is_empty() {- None- } else {- Some(gossip_seed)- };-- let config = LiveIndexConfig {- host_centrality_store_path: path- .as_path()- .join("host_centrality")- .to_str()- .unwrap()- .to_string(),- page_centrality_store_path: None,- safety_classifier_path: None,- host_centrality_threshold: None,- minimum_clean_words: None,- gossip_seed_nodes: gossip_seed,- gossip_addr,- shard_id: shard,- index_path: path.as_path().join("index").to_str().unwrap().to_string(),- linear_model_path: None,- lambda_model_path: None,- host,- collector: Default::default(),- snippet: Default::default(),- };+ config.shard_id = shard;++ let host = config.host;+ let gossip_addr = config.gossip_addr;++ if !gossip_seed.is_empty() {+ config.gossip_seed_nodes = Some(gossip_seed);+ }let service = LiveIndexService::new(config).await?;let cluster = service.cluster_handle();@@ -393,3 +400,96 @@ async fn test_replica_recovery() -> Result<()> {Ok(())}++#[tokio::test]+async fn test_meta_segments() -> Result<()> {+ let config = config(gen_temp_path());+ let indexer_config = crate::entrypoint::indexer::worker::Config {+ host_centrality_store_path: config.host_centrality_store_path.clone(),+ page_centrality_store_path: config.page_centrality_store_path.clone(),+ page_webgraph: None,+ safety_classifier_path: None,+ dual_encoder: None,+ };++ let index = LiveIndex::new(&config.index_path, indexer_config.clone()).await?;+ assert!(index.meta().segments().is_empty());++ index.insert(&[IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }]);+ index.commit();++ assert_eq!(index.meta().segments().len(), 1);++ index.re_open()?;++ assert_eq!(index.meta().segments().len(), 1);++ let copy_index = LiveIndex::new(&config.index_path, indexer_config).await?;++ assert_eq!(copy_index.meta().segments().len(), 1);++ Ok(())+}++#[tokio::test]+async fn test_segment_compaction() -> Result<()> {+ let config = config(gen_temp_path());+ let indexer_config = crate::entrypoint::indexer::worker::Config {+ host_centrality_store_path: config.host_centrality_store_path.clone(),+ page_centrality_store_path: config.page_centrality_store_path.clone(),+ page_webgraph: None,+ safety_classifier_path: None,+ dual_encoder: None,+ };++ let index = Arc::new(LiveIndex::new(&config.index_path, indexer_config).await?);++ index.insert(&[IndexableWebpage {+ url: "https://a.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }]);++ index.commit();++ index.insert(&[IndexableWebpage {+ url: "https://b.com/".to_string(),+ body: "+ <title>test page</title>+ Example webpage+ "+ .to_string(),+ fetch_time_ms: 100,+ }]);++ index.commit();++ assert_eq!(index.meta().segments().len(), 2);++ index.compact_segments_by_date();++ assert_eq!(index.meta().segments().len(), 1);++ let searcher = LocalSearcher::from(index.clone());++ let res = searcher.search(&SearchQuery {+ query: "test".to_string(),+ ..Default::default()+ })?;++ assert_eq!(res.webpages.len(), 2);++ Ok(())+}diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex cb217b78..01da755d 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -184,6 +184,11 @@ impl InvertedIndex {})}+ pub fn re_open(&mut self) -> Result<()> {+ *self = Self::open(self.path.clone())?;+ Ok(())+ }+pub fn columnfield_reader(&self) -> NumericalFieldReader {self.columnfield_reader.clone()}@@ -205,16 +210,11 @@ impl InvertedIndex {}pub fn segment_ids(&self) -> Vec<SegmentId> {- self.tantivy_index- .searchable_segment_ids()- .unwrap_or_default()+ self.tantivy_index.searchable_segment_ids().unwrap()}pub fn num_segments(&self) -> usize {- self.tantivy_index- .searchable_segments()- .unwrap_or_default()- .len()+ self.tantivy_index.searchable_segments().unwrap().len()}pub fn num_documents(&self) -> u64 {diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rsindex 804d6521..eb8c66c9 100644--- a/crates/core/src/live_index/index.rs+++ b/crates/core/src/live_index/index.rs@@ -36,13 +36,23 @@ use crate::{};#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]-struct Segment {+pub struct Segment {id: SegmentId,created: DateTime<Utc>,}+impl Segment {+ pub fn id(&self) -> SegmentId {+ self.id+ }++ pub fn created(&self) -> DateTime<Utc> {+ self.created+ }+}+#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]-struct Meta {+pub struct Meta {segments: Vec<Segment>,}@@ -72,6 +82,10 @@ impl Meta {serde_json::to_writer_pretty(writer, self).unwrap();}++ pub fn segments(&self) -> &[Segment] {+ &self.segments+ }}pub struct InnerIndex {@@ -88,7 +102,7 @@ impl InnerIndex {path: P,indexer_worker_config: indexer::worker::Config,) -> Result<Self> {- let mut index = crate::index::Index::open(path.as_ref().join("index"))?;+ let mut index = crate::index::Index::open(path.as_ref())?;index.prepare_writer()?;let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;@@ -194,6 +208,8 @@ impl InnerIndex {created: Utc::now(),})}++ self.save_meta();}fn save_meta(&self) {@@ -234,8 +250,8 @@ impl InnerIndex {self.index.insert(&webpage).unwrap();}}- self.write_ahead_log.clear().unwrap();self.index.commit().unwrap();+ self.write_ahead_log.clear().unwrap();self.update_meta();self.has_inserts = false;}@@ -247,6 +263,10 @@ impl InnerIndex {pub fn path(&self) -> &Path {&self.path}++ pub fn meta(&self) -> &Meta {+ &self.meta+ }}pub struct LiveIndex {@@ -333,4 +353,21 @@ impl LiveIndex {.unwrap_or_else(|e| e.into_inner()).delete_all_pages();}++ pub fn re_open(&self) -> Result<()> {+ self.inner+ .write()+ .unwrap_or_else(|e| e.into_inner())+ .index+ .inverted_index+ .re_open()+ }++ pub fn meta(&self) -> Meta {+ self.inner+ .read()+ .unwrap_or_else(|e| e.into_inner())+ .meta+ .clone()+ }}diff --git a/crates/tantivy/src/directory/directory_lock.rs b/crates/tantivy/src/directory/directory_lock.rsindex 89c0a3f1..7c65e03d 100644--- a/crates/tantivy/src/directory/directory_lock.rs+++ b/crates/tantivy/src/directory/directory_lock.rs@@ -1,19 +1,10 @@use std::path::PathBuf;-use std::sync::LazyLock;-/// A directory lock.////// A lock is associated with a specific path.////// The lock will be passed to [`Directory::acquire_lock`](crate::Directory::acquire_lock).-///-/// Tantivy itself uses only two locks but client application-/// can use the directory facility to define their own locks.-/// - [`INDEX_WRITER_LOCK`]-/// - [`META_LOCK`]-///-/// Check out these locks documentation for more information.#[derive(Debug)]pub struct Lock {/// The lock needs to be associated with its own file `path`.@@ -31,30 +22,3 @@ pub struct Lock {/// the lock.pub is_blocking: bool,}--/// Only one process should be able to write tantivy's index at a time.-/// This lock file, when present, is in charge of preventing other processes to open an-/// `IndexWriter`.-///-/// If the process is killed and this file remains, it is safe to remove it manually.-///-/// Failing to acquire this lock usually means a misuse of tantivy's API,-/// (creating more than one instance of the `IndexWriter`), are a spurious-/// lock file remaining after a crash. In the latter case, removing the file after-/// checking no process running tantivy is running is safe.-pub static INDEX_WRITER_LOCK: LazyLock<Lock> = LazyLock::new(|| Lock {- filepath: PathBuf::from(".tantivy-writer.lock"),- is_blocking: false,-});-/// The meta lock file is here to protect the segment files being opened by-/// `IndexReader::reload()` from being garbage collected.-///-/// It makes it possible for another process to safely consume-/// our index in-writing. Ideally, we may have preferred `RWLock` semantics-/// here, but it is difficult to achieve on Windows.-///-/// Opening segment readers is a very fast process.-pub static META_LOCK: LazyLock<Lock> = LazyLock::new(|| Lock {- filepath: PathBuf::from(".tantivy-meta.lock"),- is_blocking: true,-});diff --git a/crates/tantivy/src/directory/managed_directory.rs b/crates/tantivy/src/directory/managed_directory.rsindex c24ec553..07e20153 100644--- a/crates/tantivy/src/directory/managed_directory.rs+++ b/crates/tantivy/src/directory/managed_directory.rs@@ -11,7 +11,7 @@ use crate::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteEruse crate::directory::footer::{Footer, FooterProxy};use crate::directory::{DirectoryLock, FileHandle, FileSlice, GarbageCollectionResult, Lock, WatchCallback,- WatchHandle, WritePtr, META_LOCK,+ WatchHandle, WritePtr,};use crate::error::DataCorruption;use crate::Directory;@@ -129,24 +129,10 @@ impl ManagedDirectory {.read().expect("Managed directory rlock poisoned in garbage collect.");- // The point of this second "file" lock is to enforce the following scenario- // 1) process B tries to load a new set of searcher.- // The list of segments is loaded- // 2) writer change meta.json (for instance after a merge or a commit)- // 3) gc kicks in.- // 4) gc removes a file that was useful for process B, before process B opened it.- match self.acquire_lock(&META_LOCK) {- Ok(_meta_lock) => {- let living_files = get_living_files();- for managed_path in &meta_informations_rlock.managed_paths {- if !living_files.contains(managed_path) {- files_to_delete.push(managed_path.clone());- }- }- }- Err(err) => {- error!("Failed to acquire lock for GC");- return Err(crate::TantivyError::from(err));+ let living_files = get_living_files();+ for managed_path in &meta_informations_rlock.managed_paths {+ if !living_files.contains(managed_path) {+ files_to_delete.push(managed_path.clone());}}}diff --git a/crates/tantivy/src/directory/mod.rs b/crates/tantivy/src/directory/mod.rsindex 316a0c92..dc886ee4 100644--- a/crates/tantivy/src/directory/mod.rs+++ b/crates/tantivy/src/directory/mod.rs@@ -24,7 +24,7 @@ pub use crate::common::{AntiCallToken, OwnedBytes, TerminatingWrite};pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};pub use self::directory::{Directory, DirectoryClone, DirectoryLock};-pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};+pub use self::directory_lock::Lock;pub use self::ram_directory::RamDirectory;pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};diff --git a/crates/tantivy/src/index/index.rs b/crates/tantivy/src/index/index.rsindex 2e55e97d..ce853dfd 100644--- a/crates/tantivy/src/index/index.rs+++ b/crates/tantivy/src/index/index.rs@@ -12,7 +12,7 @@ use crate::core::{Executor, META_FILEPATH};use crate::directory::error::OpenReadError;#[cfg(feature = "mmap")]use crate::directory::MmapDirectory;-use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_LOCK};+use crate::directory::{Directory, ManagedDirectory, RamDirectory};use crate::error::{DataCorruption, TantivyError};use crate::index::{IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory};use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN};@@ -574,27 +574,8 @@ impl Index {num_threads: usize,overall_memory_budget_in_bytes: usize,) -> crate::Result<IndexWriter<D>> {- let directory_lock = self- .directory- .acquire_lock(&INDEX_WRITER_LOCK)- .map_err(|err| {- TantivyError::LockFailure(- err,- Some(- "Failed to acquire index lock. If you are using a regular directory, this \- means there is already an `IndexWriter` working on this `Directory`, in \- this process or in a different process."- .to_string(),- ),- )- })?;let memory_arena_in_bytes_per_thread = overall_memory_budget_in_bytes / num_threads;- IndexWriter::new(- self,- num_threads,- memory_arena_in_bytes_per_thread,- directory_lock,- )+ IndexWriter::new(self, num_threads, memory_arena_in_bytes_per_thread)}/// Helper to create an index writer for tests.diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rsindex 08796af9..09fe63b0 100644--- a/crates/tantivy/src/indexer/index_writer.rs+++ b/crates/tantivy/src/indexer/index_writer.rs@@ -8,7 +8,7 @@ use smallvec::smallvec;use super::operation::{AddOperation, UserOperation};use super::segment_updater::SegmentUpdater;use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};-use crate::directory::{DirectoryLock, GarbageCollectionResult};+use crate::directory::GarbageCollectionResult;use crate::error::TantivyError;use crate::index::{Index, Segment, SegmentId, SegmentMeta};use crate::indexer::index_writer_status::IndexWriterStatus;@@ -46,10 +46,6 @@ fn error_in_index_worker_thread(context: &str) -> TantivyError {/// Each indexing thread builds its own independent [`Segment`], via/// a `SegmentWriter` object.pub struct IndexWriter<D: Document = TantivyDocument> {- // the lock is just used to bind the- // lifetime of the lock with that of the IndexWriter.- _directory_lock: Option<DirectoryLock>,-index: Index,// The memory budget per thread, after which a commit is triggered.@@ -132,7 +128,6 @@ impl<D: Document> IndexWriter<D> {index: &Index,num_threads: usize,memory_budget_in_bytes_per_thread: usize,- directory_lock: DirectoryLock,) -> crate::Result<Self> {if memory_budget_in_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {let err_msg = format!(@@ -157,8 +152,6 @@ impl<D: Document> IndexWriter<D> {let segment_updater = SegmentUpdater::create(index.clone(), stamper.clone())?;let mut index_writer = Self {- _directory_lock: Some(directory_lock),-memory_budget_in_bytes_per_thread,index: index.clone(),index_writer_status: IndexWriterStatus::from(document_receiver),@@ -371,17 +364,10 @@ impl<D: Document> IndexWriter<D> {self.segment_updater.kill();let document_receiver_res = self.operation_receiver();- // take the directory lock to create a new index_writer.- let directory_lock = self- ._directory_lock- .take()- .expect("The IndexWriter does not have any lock. This is a bug, please report.");-let new_index_writer = IndexWriter::new(&self.index,self.num_threads,self.memory_budget_in_bytes_per_thread,- directory_lock,)?;// the current `self` is dropped right away because of this call.@@ -583,8 +569,6 @@ mod tests {use super::super::operation::UserOperation;use crate::collector::{Count, TopDocs};- use crate::directory::error::LockError;- use crate::error::*;use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;use crate::indexer::NoMergePolicy;use crate::query::{QueryParser, TermQuery};@@ -634,31 +618,6 @@ mod tests {assert_eq!(batch_opstamp2, 1u64);}- #[test]- fn test_lockfile_stops_duplicates() {- let schema_builder = schema::Schema::builder();- let index = Index::create_in_ram(schema_builder.build());- let _index_writer: IndexWriter = index.writer_for_tests().unwrap();- match index.writer_for_tests::<TantivyDocument>() {- Err(TantivyError::LockFailure(LockError::LockBusy, _)) => {}- _ => panic!("Expected a `LockFailure` error"),- }- }-- #[test]- fn test_lockfile_already_exists_error_msg() {- let schema_builder = schema::Schema::builder();- let index = Index::create_in_ram(schema_builder.build());- let _index_writer: IndexWriter = index.writer_for_tests().unwrap();- match index.writer_for_tests::<TantivyDocument>() {- Err(err) => {- let err_msg = err.to_string();- assert!(err_msg.contains("already an `IndexWriter`"));- }- _ => panic!("Expected LockfileAlreadyExists error"),- }- }-#[test]fn test_set_merge_policy() {let schema_builder = schema::Schema::builder();diff --git a/crates/tantivy/src/reader/mod.rs b/crates/tantivy/src/reader/mod.rsindex 39e4c6e0..95dd7943 100644--- a/crates/tantivy/src/reader/mod.rs+++ b/crates/tantivy/src/reader/mod.rs@@ -8,7 +8,7 @@ pub use warming::Warmer;use self::warming::WarmingState;use crate::core::searcher::{SearcherGeneration, SearcherInner};-use crate::directory::{Directory, WatchCallback, WatchHandle, META_LOCK};+use crate::directory::{Directory, WatchCallback, WatchHandle};use crate::store::DOCSTORE_CACHE_CAPACITY;use crate::{Index, Inventory, Searcher, SegmentReader, TrackedObject};@@ -193,8 +193,6 @@ impl InnerIndexReader {/// This function acquires a lock to prevent GC from removing files/// as we are opening our index.fn open_segment_readers(index: &Index) -> crate::Result<Vec<SegmentReader>> {- // Prevents segment files from getting deleted while we are in the process of opening them- let _meta_lock = index.directory().acquire_lock(&META_LOCK)?;let searchable_segments = index.searchable_segments()?;let segment_readers = searchable_segments.iter()