Skip to content

web search done right

Control the number of segments after merge. (#54)

Mikkel Denker65cf8f9f5356
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 9168d4c0..6a151e2e 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -144,7 +144,6 @@ impl Map<IndexingWorker, FrozenIndex> for Job {
std::fs::remove_file(file).ok();
}
index.commit().unwrap();
- index.inverted_index.merge_all_segments().unwrap();
info!("{} done", name);
@@ -218,10 +217,14 @@ impl Indexer {
warc_paths = Box::new(warc_paths.take(limit));
}
- let _index: Index = warc_paths
+ let mut index: Index = warc_paths
.map_reduce(&workers)
.expect("failed to build index");
+ index
+ .inverted_index
+ .merge_into_segments(config.final_num_segments.unwrap_or(20))?;
+
Ok(())
}
@@ -287,7 +290,9 @@ impl Indexer {
);
if let Some(index) = index.as_mut() {
- index.inverted_index.merge_all_segments().unwrap();
+ index
+ .inverted_index
+ .merge_into_segments(config.final_num_segments.unwrap_or(20))?;
}
Ok(())
diff --git a/src/inverted_index.rs b/src/inverted_index.rs
index 2564d39c..47f5bbf8 100644
--- a/src/inverted_index.rs
+++ b/src/inverted_index.rs
@@ -20,7 +20,7 @@ use tantivy::collector::{Collector, Count};
use tantivy::merge_policy::NoMergePolicy;
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
-use tantivy::{DocAddress, Document, IndexReader, IndexWriter};
+use tantivy::{DocAddress, Document, IndexReader, IndexWriter, SegmentMeta};
use crate::image_store::Image;
use crate::query::Query;
@@ -34,6 +34,11 @@ use std::fs;
use std::path::Path;
use std::sync::Arc;
+struct SegmentMergeCandidate {
+ num_docs: u32,
+ segments: Vec<SegmentMeta>,
+}
+
pub struct InvertedIndex {
pub path: String,
tantivy_index: tantivy::Index,
@@ -149,8 +154,19 @@ impl InvertedIndex {
})
}
- pub fn merge_all_segments(&mut self) -> Result<()> {
- let segments: Vec<_> = self
+ pub fn merge_into_segments(&mut self, num_segments: u32) -> Result<()> {
+ assert!(num_segments > 0);
+
+ let mut merge_segments = Vec::new();
+
+ for _ in 0..num_segments {
+ merge_segments.push(SegmentMergeCandidate {
+ num_docs: 0,
+ segments: Vec::new(),
+ });
+ }
+
+ let mut segments: Vec<_> = self
.tantivy_index
.load_metas()?
.segments
@@ -158,13 +174,31 @@ impl InvertedIndex {
.collect();
if segments.len() > 1 {
- let segment_ids: Vec<_> = segments.iter().map(|segment| segment.id()).collect();
- self.writer.merge(&segment_ids[..]).wait()?;
+ segments.sort_by_key(|b| std::cmp::Reverse(b.num_docs()));
- let path = Path::new(&self.path);
for segment in segments {
- for file in segment.list_files() {
- std::fs::remove_file(path.join(file)).ok();
+ let best_candidate = merge_segments
+ .iter_mut()
+ .min_by(|a, b| a.num_docs.cmp(&b.num_docs))
+ .unwrap();
+
+ best_candidate.num_docs += segment.num_docs();
+ best_candidate.segments.push(segment);
+ }
+
+ for merge in merge_segments
+ .into_iter()
+ .filter(|merge| !merge.segments.is_empty())
+ {
+ let segment_ids: Vec<_> =
+ merge.segments.iter().map(|segment| segment.id()).collect();
+ self.writer.merge(&segment_ids[..]).wait()?;
+
+ let path = Path::new(&self.path);
+ for segment in merge.segments {
+ for file in segment.list_files() {
+ std::fs::remove_file(path.join(file)).ok();
+ }
}
}
}
diff --git a/src/lib.rs b/src/lib.rs
index 58828ed8..93e8557c 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -57,6 +57,7 @@ mod webpage;
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingMasterConfig {
limit_warc_files: Option<usize>,
+ final_num_segments: Option<u32>,
warc_source: WarcSource,
workers: Vec<String>,
batch_size: Option<usize>,
@@ -67,6 +68,7 @@ pub struct IndexingMasterConfig {
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingLocalConfig {
limit_warc_files: Option<usize>,
+ final_num_segments: Option<u32>,
warc_source: WarcSource,
batch_size: Option<usize>,
webgraph_path: Option<String>,