Skip to content

web search done right

prepare index writers in functions that uses the writer where possible. this removes the annoying footgun of having a panic if we for instance forgot to prepare the writer for a new index after merge...

Mikkel Denkerf501e4a8cfdf
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index e43e8e58..8566c02e 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -426,17 +426,16 @@ impl Indexer {
threads.push(thread::spawn(move || {
let mut it = indexes.into_iter();
let mut index = Index::open(it.next().unwrap().0).unwrap();
- index.prepare_writer().unwrap();
for other in it {
let other_path = other.0;
- let mut other = Index::open(&other_path).unwrap();
- other.prepare_writer().unwrap();
+ let other = Index::open(&other_path).unwrap();
index = index.merge(other);
std::fs::remove_dir_all(other_path).unwrap();
}
+
index.inverted_index.merge_into_max_segments(1).unwrap();
index
diff --git a/core/src/index.rs b/core/src/index.rs
index a9062c69..8858bbc1 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -147,7 +147,9 @@ impl Index {
self_region_count.merge(other_region_count);
- Self::open(&self.path).expect("failed to open index")
+ let mut res = Self::open(&self.path).expect("failed to open index");
+ res.prepare_writer().expect("failed to prepare writer");
+ res
}
pub(crate) fn prepare_writer(&mut self) -> Result<()> {
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 907defca..ded79e9f 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -233,6 +233,10 @@ impl InvertedIndex {
}
pub fn prepare_writer(&mut self) -> Result<()> {
+ if self.writer.is_some() {
+ return Ok(());
+ }
+
let writer = self
.tantivy_index
.writer_with_num_threads(1, 1_000_000_000)?;
@@ -284,6 +288,7 @@ impl InvertedIndex {
}
pub fn commit(&mut self) -> Result<()> {
+ self.prepare_writer()?;
self.writer
.as_mut()
.expect("writer has not been prepared")
@@ -448,6 +453,7 @@ impl InvertedIndex {
}
pub fn merge_into_max_segments(&mut self, max_num_segments: u64) -> Result<()> {
+ self.prepare_writer()?;
let base_path = Path::new(&self.path);
let segments: Vec<_> = self
.tantivy_index
@@ -476,6 +482,9 @@ impl InvertedIndex {
}
pub fn merge(mut self, mut other: InvertedIndex) -> Self {
+ self.prepare_writer().expect("failed to prepare writer");
+ other.prepare_writer().expect("failed to prepare writer");
+
let path = self.path.clone();
{
@@ -540,7 +549,11 @@ impl InvertedIndex {
.unwrap();
}
- Self::open(path).expect("failed to open index")
+ let mut res = Self::open(path).expect("failed to open index");
+
+ res.prepare_writer().expect("failed to prepare writer");
+
+ res
}
pub fn stop(mut self) {