Skip to content

web search done right

Re-open live index after it has been downloaded from replica (#227)

Mikkel Denker950862be9c83
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index 7e572add..7b0a7587 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -38,6 +38,7 @@ use crate::webpage::{safety_classifier, Html, Webpage};
const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);
+#[derive(Clone)]
pub enum IndexerGraphConfig {
Local { path: String },
Remote { gossip: GossipConfig },
@@ -55,6 +56,7 @@ impl From<crate::config::IndexerGraphConfig> for IndexerGraphConfig {
}
}
+#[derive(Clone)]
pub struct Config {
pub host_centrality_store_path: String,
pub page_centrality_store_path: Option<String>,
diff --git a/crates/core/src/entrypoint/live_index/search_server.rs b/crates/core/src/entrypoint/live_index/search_server.rs
index 0a921946..c0e598a8 100644
--- a/crates/core/src/entrypoint/live_index/search_server.rs
+++ b/crates/core/src/entrypoint/live_index/search_server.rs
@@ -146,6 +146,7 @@ async fn setup(index: Arc<LiveIndex>, cluster: Arc<Cluster>, temp_wal: TempWal)
},
)
.await;
+ index.re_open().unwrap();
}
let mut wal = temp_wal
@@ -202,7 +203,7 @@ impl LiveIndexService {
let index = Arc::new(
LiveIndex::new(
- index_path.join("index"),
+ index_path,
indexer::worker::Config {
host_centrality_store_path: config.host_centrality_store_path.clone(),
page_centrality_store_path: config.page_centrality_store_path.clone(),
diff --git a/crates/core/src/entrypoint/live_index/tests.rs b/crates/core/src/entrypoint/live_index/tests.rs
index 3d376769..26c9afb2 100644
--- a/crates/core/src/entrypoint/live_index/tests.rs
+++ b/crates/core/src/entrypoint/live_index/tests.rs
@@ -15,10 +15,14 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- config::LiveIndexConfig, entrypoint::search_server, inverted_index, live_index::LiveIndex,
+ config::LiveIndexConfig,
+ entrypoint::search_server,
+ inverted_index,
+ live_index::LiveIndex,
+ searcher::{LocalSearcher, SearchQuery},
Result,
};
-use std::{net::SocketAddr, sync::Arc};
+use std::{net::SocketAddr, path::Path, sync::Arc};
use file_store::gen_temp_path;
@@ -35,6 +39,30 @@ use crate::{
use super::LiveIndexService;
+fn config<P: AsRef<Path>>(path: P) -> LiveIndexConfig {
+ LiveIndexConfig {
+ host_centrality_store_path: path
+ .as_ref()
+ .join("host_centrality")
+ .to_str()
+ .unwrap()
+ .to_string(),
+ page_centrality_store_path: None,
+ safety_classifier_path: None,
+ host_centrality_threshold: None,
+ minimum_clean_words: None,
+ gossip_seed_nodes: None,
+ gossip_addr: free_socket_addr(),
+ shard_id: ShardId::new(0),
+ index_path: path.as_ref().join("index").to_str().unwrap().to_string(),
+ linear_model_path: None,
+ lambda_model_path: None,
+ host: free_socket_addr(),
+ collector: Default::default(),
+ snippet: Default::default(),
+ }
+}
+
struct RemoteIndex {
host: SocketAddr,
shard: ShardId,
@@ -46,37 +74,16 @@ struct RemoteIndex {
impl RemoteIndex {
async fn start(shard: ShardId, gossip_seed: Vec<SocketAddr>) -> Result<Self> {
let path = gen_temp_path();
+ let mut config = config(&path);
- let host = free_socket_addr();
- let gossip_addr = free_socket_addr();
-
- let gossip_seed = if gossip_seed.is_empty() {
- None
- } else {
- Some(gossip_seed)
- };
-
- let config = LiveIndexConfig {
- host_centrality_store_path: path
- .as_path()
- .join("host_centrality")
- .to_str()
- .unwrap()
- .to_string(),
- page_centrality_store_path: None,
- safety_classifier_path: None,
- host_centrality_threshold: None,
- minimum_clean_words: None,
- gossip_seed_nodes: gossip_seed,
- gossip_addr,
- shard_id: shard,
- index_path: path.as_path().join("index").to_str().unwrap().to_string(),
- linear_model_path: None,
- lambda_model_path: None,
- host,
- collector: Default::default(),
- snippet: Default::default(),
- };
+ config.shard_id = shard;
+
+ let host = config.host;
+ let gossip_addr = config.gossip_addr;
+
+ if !gossip_seed.is_empty() {
+ config.gossip_seed_nodes = Some(gossip_seed);
+ }
let service = LiveIndexService::new(config).await?;
let cluster = service.cluster_handle();
@@ -393,3 +400,96 @@ async fn test_replica_recovery() -> Result<()> {
Ok(())
}
+
+#[tokio::test]
+async fn test_meta_segments() -> Result<()> {
+ let config = config(gen_temp_path());
+ let indexer_config = crate::entrypoint::indexer::worker::Config {
+ host_centrality_store_path: config.host_centrality_store_path.clone(),
+ page_centrality_store_path: config.page_centrality_store_path.clone(),
+ page_webgraph: None,
+ safety_classifier_path: None,
+ dual_encoder: None,
+ };
+
+ let index = LiveIndex::new(&config.index_path, indexer_config.clone()).await?;
+ assert!(index.meta().segments().is_empty());
+
+ index.insert(&[IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }]);
+ index.commit();
+
+ assert_eq!(index.meta().segments().len(), 1);
+
+ index.re_open()?;
+
+ assert_eq!(index.meta().segments().len(), 1);
+
+ let copy_index = LiveIndex::new(&config.index_path, indexer_config).await?;
+
+ assert_eq!(copy_index.meta().segments().len(), 1);
+
+ Ok(())
+}
+
+#[tokio::test]
+async fn test_segment_compaction() -> Result<()> {
+ let config = config(gen_temp_path());
+ let indexer_config = crate::entrypoint::indexer::worker::Config {
+ host_centrality_store_path: config.host_centrality_store_path.clone(),
+ page_centrality_store_path: config.page_centrality_store_path.clone(),
+ page_webgraph: None,
+ safety_classifier_path: None,
+ dual_encoder: None,
+ };
+
+ let index = Arc::new(LiveIndex::new(&config.index_path, indexer_config).await?);
+
+ index.insert(&[IndexableWebpage {
+ url: "https://a.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }]);
+
+ index.commit();
+
+ index.insert(&[IndexableWebpage {
+ url: "https://b.com/".to_string(),
+ body: "
+ <title>test page</title>
+ Example webpage
+ "
+ .to_string(),
+ fetch_time_ms: 100,
+ }]);
+
+ index.commit();
+
+ assert_eq!(index.meta().segments().len(), 2);
+
+ index.compact_segments_by_date();
+
+ assert_eq!(index.meta().segments().len(), 1);
+
+ let searcher = LocalSearcher::from(index.clone());
+
+ let res = searcher.search(&SearchQuery {
+ query: "test".to_string(),
+ ..Default::default()
+ })?;
+
+ assert_eq!(res.webpages.len(), 2);
+
+ Ok(())
+}
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index cb217b78..01da755d 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -184,6 +184,11 @@ impl InvertedIndex {
})
}
+ pub fn re_open(&mut self) -> Result<()> {
+ *self = Self::open(self.path.clone())?;
+ Ok(())
+ }
+
pub fn columnfield_reader(&self) -> NumericalFieldReader {
self.columnfield_reader.clone()
}
@@ -205,16 +210,11 @@ impl InvertedIndex {
}
pub fn segment_ids(&self) -> Vec<SegmentId> {
- self.tantivy_index
- .searchable_segment_ids()
- .unwrap_or_default()
+ self.tantivy_index.searchable_segment_ids().unwrap()
}
pub fn num_segments(&self) -> usize {
- self.tantivy_index
- .searchable_segments()
- .unwrap_or_default()
- .len()
+ self.tantivy_index.searchable_segments().unwrap().len()
}
pub fn num_documents(&self) -> u64 {
diff --git a/crates/core/src/live_index/index.rs b/crates/core/src/live_index/index.rs
index 804d6521..eb8c66c9 100644
--- a/crates/core/src/live_index/index.rs
+++ b/crates/core/src/live_index/index.rs
@@ -36,13 +36,23 @@ use crate::{
};
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]
-struct Segment {
+pub struct Segment {
id: SegmentId,
created: DateTime<Utc>,
}
+impl Segment {
+ pub fn id(&self) -> SegmentId {
+ self.id
+ }
+
+ pub fn created(&self) -> DateTime<Utc> {
+ self.created
+ }
+}
+
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]
-struct Meta {
+pub struct Meta {
segments: Vec<Segment>,
}
@@ -72,6 +82,10 @@ impl Meta {
serde_json::to_writer_pretty(writer, self).unwrap();
}
+
+ pub fn segments(&self) -> &[Segment] {
+ &self.segments
+ }
}
pub struct InnerIndex {
@@ -88,7 +102,7 @@ impl InnerIndex {
path: P,
indexer_worker_config: indexer::worker::Config,
) -> Result<Self> {
- let mut index = crate::index::Index::open(path.as_ref().join("index"))?;
+ let mut index = crate::index::Index::open(path.as_ref())?;
index.prepare_writer()?;
let write_ahead_log = Wal::open(path.as_ref().join("wal"))?;
@@ -194,6 +208,8 @@ impl InnerIndex {
created: Utc::now(),
})
}
+
+ self.save_meta();
}
fn save_meta(&self) {
@@ -234,8 +250,8 @@ impl InnerIndex {
self.index.insert(&webpage).unwrap();
}
}
- self.write_ahead_log.clear().unwrap();
self.index.commit().unwrap();
+ self.write_ahead_log.clear().unwrap();
self.update_meta();
self.has_inserts = false;
}
@@ -247,6 +263,10 @@ impl InnerIndex {
pub fn path(&self) -> &Path {
&self.path
}
+
+ pub fn meta(&self) -> &Meta {
+ &self.meta
+ }
}
pub struct LiveIndex {
@@ -333,4 +353,21 @@ impl LiveIndex {
.unwrap_or_else(|e| e.into_inner())
.delete_all_pages();
}
+
+ pub fn re_open(&self) -> Result<()> {
+ self.inner
+ .write()
+ .unwrap_or_else(|e| e.into_inner())
+ .index
+ .inverted_index
+ .re_open()
+ }
+
+ pub fn meta(&self) -> Meta {
+ self.inner
+ .read()
+ .unwrap_or_else(|e| e.into_inner())
+ .meta
+ .clone()
+ }
}
diff --git a/crates/tantivy/src/directory/directory_lock.rs b/crates/tantivy/src/directory/directory_lock.rs
index 89c0a3f1..7c65e03d 100644
--- a/crates/tantivy/src/directory/directory_lock.rs
+++ b/crates/tantivy/src/directory/directory_lock.rs
@@ -1,19 +1,10 @@
use std::path::PathBuf;
-use std::sync::LazyLock;
-
/// A directory lock.
///
/// A lock is associated with a specific path.
///
/// The lock will be passed to [`Directory::acquire_lock`](crate::Directory::acquire_lock).
-///
-/// Tantivy itself uses only two locks but client application
-/// can use the directory facility to define their own locks.
-/// - [`INDEX_WRITER_LOCK`]
-/// - [`META_LOCK`]
-///
-/// Check out these locks documentation for more information.
#[derive(Debug)]
pub struct Lock {
/// The lock needs to be associated with its own file `path`.
@@ -31,30 +22,3 @@ pub struct Lock {
/// the lock.
pub is_blocking: bool,
}
-
-/// Only one process should be able to write tantivy's index at a time.
-/// This lock file, when present, is in charge of preventing other processes to open an
-/// `IndexWriter`.
-///
-/// If the process is killed and this file remains, it is safe to remove it manually.
-///
-/// Failing to acquire this lock usually means a misuse of tantivy's API,
-/// (creating more than one instance of the `IndexWriter`), are a spurious
-/// lock file remaining after a crash. In the latter case, removing the file after
-/// checking no process running tantivy is running is safe.
-pub static INDEX_WRITER_LOCK: LazyLock<Lock> = LazyLock::new(|| Lock {
- filepath: PathBuf::from(".tantivy-writer.lock"),
- is_blocking: false,
-});
-/// The meta lock file is here to protect the segment files being opened by
-/// `IndexReader::reload()` from being garbage collected.
-///
-/// It makes it possible for another process to safely consume
-/// our index in-writing. Ideally, we may have preferred `RWLock` semantics
-/// here, but it is difficult to achieve on Windows.
-///
-/// Opening segment readers is a very fast process.
-pub static META_LOCK: LazyLock<Lock> = LazyLock::new(|| Lock {
- filepath: PathBuf::from(".tantivy-meta.lock"),
- is_blocking: true,
-});
diff --git a/crates/tantivy/src/directory/managed_directory.rs b/crates/tantivy/src/directory/managed_directory.rs
index c24ec553..07e20153 100644
--- a/crates/tantivy/src/directory/managed_directory.rs
+++ b/crates/tantivy/src/directory/managed_directory.rs
@@ -11,7 +11,7 @@ use crate::directory::error::{DeleteError, LockError, OpenReadError, OpenWriteEr
use crate::directory::footer::{Footer, FooterProxy};
use crate::directory::{
DirectoryLock, FileHandle, FileSlice, GarbageCollectionResult, Lock, WatchCallback,
- WatchHandle, WritePtr, META_LOCK,
+ WatchHandle, WritePtr,
};
use crate::error::DataCorruption;
use crate::Directory;
@@ -129,24 +129,10 @@ impl ManagedDirectory {
.read()
.expect("Managed directory rlock poisoned in garbage collect.");
- // The point of this second "file" lock is to enforce the following scenario
- // 1) process B tries to load a new set of searcher.
- // The list of segments is loaded
- // 2) writer change meta.json (for instance after a merge or a commit)
- // 3) gc kicks in.
- // 4) gc removes a file that was useful for process B, before process B opened it.
- match self.acquire_lock(&META_LOCK) {
- Ok(_meta_lock) => {
- let living_files = get_living_files();
- for managed_path in &meta_informations_rlock.managed_paths {
- if !living_files.contains(managed_path) {
- files_to_delete.push(managed_path.clone());
- }
- }
- }
- Err(err) => {
- error!("Failed to acquire lock for GC");
- return Err(crate::TantivyError::from(err));
+ let living_files = get_living_files();
+ for managed_path in &meta_informations_rlock.managed_paths {
+ if !living_files.contains(managed_path) {
+ files_to_delete.push(managed_path.clone());
}
}
}
diff --git a/crates/tantivy/src/directory/mod.rs b/crates/tantivy/src/directory/mod.rs
index 316a0c92..dc886ee4 100644
--- a/crates/tantivy/src/directory/mod.rs
+++ b/crates/tantivy/src/directory/mod.rs
@@ -24,7 +24,7 @@ pub use crate::common::{AntiCallToken, OwnedBytes, TerminatingWrite};
pub(crate) use self::composite_file::{CompositeFile, CompositeWrite};
pub use self::directory::{Directory, DirectoryClone, DirectoryLock};
-pub use self::directory_lock::{Lock, INDEX_WRITER_LOCK, META_LOCK};
+pub use self::directory_lock::Lock;
pub use self::ram_directory::RamDirectory;
pub use self::watch_event_router::{WatchCallback, WatchCallbackList, WatchHandle};
diff --git a/crates/tantivy/src/index/index.rs b/crates/tantivy/src/index/index.rs
index 2e55e97d..ce853dfd 100644
--- a/crates/tantivy/src/index/index.rs
+++ b/crates/tantivy/src/index/index.rs
@@ -12,7 +12,7 @@ use crate::core::{Executor, META_FILEPATH};
use crate::directory::error::OpenReadError;
#[cfg(feature = "mmap")]
use crate::directory::MmapDirectory;
-use crate::directory::{Directory, ManagedDirectory, RamDirectory, INDEX_WRITER_LOCK};
+use crate::directory::{Directory, ManagedDirectory, RamDirectory};
use crate::error::{DataCorruption, TantivyError};
use crate::index::{IndexMeta, SegmentId, SegmentMeta, SegmentMetaInventory};
use crate::indexer::index_writer::{MAX_NUM_THREAD, MEMORY_BUDGET_NUM_BYTES_MIN};
@@ -574,27 +574,8 @@ impl Index {
num_threads: usize,
overall_memory_budget_in_bytes: usize,
) -> crate::Result<IndexWriter<D>> {
- let directory_lock = self
- .directory
- .acquire_lock(&INDEX_WRITER_LOCK)
- .map_err(|err| {
- TantivyError::LockFailure(
- err,
- Some(
- "Failed to acquire index lock. If you are using a regular directory, this \
- means there is already an `IndexWriter` working on this `Directory`, in \
- this process or in a different process."
- .to_string(),
- ),
- )
- })?;
let memory_arena_in_bytes_per_thread = overall_memory_budget_in_bytes / num_threads;
- IndexWriter::new(
- self,
- num_threads,
- memory_arena_in_bytes_per_thread,
- directory_lock,
- )
+ IndexWriter::new(self, num_threads, memory_arena_in_bytes_per_thread)
}
/// Helper to create an index writer for tests.
diff --git a/crates/tantivy/src/indexer/index_writer.rs b/crates/tantivy/src/indexer/index_writer.rs
index 08796af9..09fe63b0 100644
--- a/crates/tantivy/src/indexer/index_writer.rs
+++ b/crates/tantivy/src/indexer/index_writer.rs
@@ -8,7 +8,7 @@ use smallvec::smallvec;
use super::operation::{AddOperation, UserOperation};
use super::segment_updater::SegmentUpdater;
use super::{AddBatch, AddBatchReceiver, AddBatchSender, PreparedCommit};
-use crate::directory::{DirectoryLock, GarbageCollectionResult};
+use crate::directory::GarbageCollectionResult;
use crate::error::TantivyError;
use crate::index::{Index, Segment, SegmentId, SegmentMeta};
use crate::indexer::index_writer_status::IndexWriterStatus;
@@ -46,10 +46,6 @@ fn error_in_index_worker_thread(context: &str) -> TantivyError {
/// Each indexing thread builds its own independent [`Segment`], via
/// a `SegmentWriter` object.
pub struct IndexWriter<D: Document = TantivyDocument> {
- // the lock is just used to bind the
- // lifetime of the lock with that of the IndexWriter.
- _directory_lock: Option<DirectoryLock>,
-
index: Index,
// The memory budget per thread, after which a commit is triggered.
@@ -132,7 +128,6 @@ impl<D: Document> IndexWriter<D> {
index: &Index,
num_threads: usize,
memory_budget_in_bytes_per_thread: usize,
- directory_lock: DirectoryLock,
) -> crate::Result<Self> {
if memory_budget_in_bytes_per_thread < MEMORY_BUDGET_NUM_BYTES_MIN {
let err_msg = format!(
@@ -157,8 +152,6 @@ impl<D: Document> IndexWriter<D> {
let segment_updater = SegmentUpdater::create(index.clone(), stamper.clone())?;
let mut index_writer = Self {
- _directory_lock: Some(directory_lock),
-
memory_budget_in_bytes_per_thread,
index: index.clone(),
index_writer_status: IndexWriterStatus::from(document_receiver),
@@ -371,17 +364,10 @@ impl<D: Document> IndexWriter<D> {
self.segment_updater.kill();
let document_receiver_res = self.operation_receiver();
- // take the directory lock to create a new index_writer.
- let directory_lock = self
- ._directory_lock
- .take()
- .expect("The IndexWriter does not have any lock. This is a bug, please report.");
-
let new_index_writer = IndexWriter::new(
&self.index,
self.num_threads,
self.memory_budget_in_bytes_per_thread,
- directory_lock,
)?;
// the current `self` is dropped right away because of this call.
@@ -583,8 +569,6 @@ mod tests {
use super::super::operation::UserOperation;
use crate::collector::{Count, TopDocs};
- use crate::directory::error::LockError;
- use crate::error::*;
use crate::indexer::index_writer::MEMORY_BUDGET_NUM_BYTES_MIN;
use crate::indexer::NoMergePolicy;
use crate::query::{QueryParser, TermQuery};
@@ -634,31 +618,6 @@ mod tests {
assert_eq!(batch_opstamp2, 1u64);
}
- #[test]
- fn test_lockfile_stops_duplicates() {
- let schema_builder = schema::Schema::builder();
- let index = Index::create_in_ram(schema_builder.build());
- let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
- match index.writer_for_tests::<TantivyDocument>() {
- Err(TantivyError::LockFailure(LockError::LockBusy, _)) => {}
- _ => panic!("Expected a `LockFailure` error"),
- }
- }
-
- #[test]
- fn test_lockfile_already_exists_error_msg() {
- let schema_builder = schema::Schema::builder();
- let index = Index::create_in_ram(schema_builder.build());
- let _index_writer: IndexWriter = index.writer_for_tests().unwrap();
- match index.writer_for_tests::<TantivyDocument>() {
- Err(err) => {
- let err_msg = err.to_string();
- assert!(err_msg.contains("already an `IndexWriter`"));
- }
- _ => panic!("Expected LockfileAlreadyExists error"),
- }
- }
-
#[test]
fn test_set_merge_policy() {
let schema_builder = schema::Schema::builder();
diff --git a/crates/tantivy/src/reader/mod.rs b/crates/tantivy/src/reader/mod.rs
index 39e4c6e0..95dd7943 100644
--- a/crates/tantivy/src/reader/mod.rs
+++ b/crates/tantivy/src/reader/mod.rs
@@ -8,7 +8,7 @@ pub use warming::Warmer;
use self::warming::WarmingState;
use crate::core::searcher::{SearcherGeneration, SearcherInner};
-use crate::directory::{Directory, WatchCallback, WatchHandle, META_LOCK};
+use crate::directory::{Directory, WatchCallback, WatchHandle};
use crate::store::DOCSTORE_CACHE_CAPACITY;
use crate::{Index, Inventory, Searcher, SegmentReader, TrackedObject};
@@ -193,8 +193,6 @@ impl InnerIndexReader {
/// This function acquires a lock to prevent GC from removing files
/// as we are opening our index.
fn open_segment_readers(index: &Index) -> crate::Result<Vec<SegmentReader>> {
- // Prevents segment files from getting deleted while we are in the process of opening them
- let _meta_lock = index.directory().acquire_lock(&META_LOCK)?;
let searchable_segments = index.searchable_segments()?;
let segment_readers = searchable_segments
.iter()