Skip to content

web search done right

[tantivy] reduce memory usage when writing postings

Mikkel Denker94d319209b24
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 2d238fc2..c888afb3 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -182,7 +182,7 @@ impl Webgraph {
let mut handlers = Vec::new();
let graph_path = &config.graph_base_path;
- const MAX_FINALIZE_CONCURRENT: usize = 2;
+ const MAX_FINALIZE_CONCURRENT: usize = 4;
let (s, r) = crossbeam_channel::bounded(MAX_FINALIZE_CONCURRENT);
for _ in 0..MAX_FINALIZE_CONCURRENT {
diff --git a/crates/tantivy/src/indexer/merger.rs b/crates/tantivy/src/indexer/merger.rs
index 8a544610..525f7812 100644
--- a/crates/tantivy/src/indexer/merger.rs
+++ b/crates/tantivy/src/indexer/merger.rs
@@ -6,6 +6,7 @@ use crate::columnar::{
};
use itertools::Itertools;
use measure_time::debug_time;
+use rustc_hash::FxHashMap;
use crate::columnfield::ColumnFieldNotAvailableError;
use crate::directory::WritePtr;
@@ -18,9 +19,9 @@ use crate::indexer::SegmentSerializer;
use crate::postings::{InvertedIndexSerializer, Postings, SegmentPostings};
use crate::schema::{value_type_to_column_type, Field, FieldType, Schema};
use crate::store::StoreWriter;
-use crate::termdict::{TermMerger, TermOrdinal};
+use crate::termdict::TermMerger;
use crate::{
- DocAddress, DocId, IndexSettings, IndexSortByField, InvertedIndexReader, Order, SegmentOrdinal,
+ DocAddress, IndexSettings, IndexSortByField, InvertedIndexReader, Order, SegmentOrdinal,
};
/// Segment's max doc must be `< MAX_DOC_LIMIT`.
@@ -399,39 +400,20 @@ impl IndexMerger {
let mut positions_buffer: Vec<u32> = Vec::with_capacity(1_000);
let mut delta_computer = DeltaComputer::new();
- let mut max_term_ords: Vec<TermOrdinal> = Vec::new();
-
let field_readers: Vec<Arc<InvertedIndexReader>> = self
.readers
.iter()
.map(|reader| reader.inverted_index(indexed_field))
.collect::<crate::Result<Vec<_>>>()?;
- let mut field_term_streams = Vec::new();
+ let mut field_term_streams = Vec::with_capacity(field_readers.len());
for field_reader in &field_readers {
let terms = field_reader.terms();
field_term_streams.push(terms.stream()?);
- max_term_ords.push(terms.num_terms() as u64);
}
let mut merged_terms = TermMerger::new(field_term_streams);
- // map from segment doc ids to the resulting merged segment doc id.
-
- let mut merged_doc_id_map: Vec<Vec<Option<DocId>>> = self
- .readers
- .iter()
- .map(|reader| {
- let mut segment_local_map = vec![];
- segment_local_map.resize(reader.max_doc() as usize, None);
- segment_local_map
- })
- .collect();
- for (new_doc_id, old_doc_addr) in doc_id_mapping.iter_old_doc_addrs().enumerate() {
- let segment_map = &mut merged_doc_id_map[old_doc_addr.segment_ord as usize];
- segment_map[old_doc_addr.doc_id as usize] = Some(new_doc_id as DocId);
- }
-
// Note that the total number of tokens is not exact.
// It is only used as a parameter in the BM25 formula.
let total_num_tokens: u64 = estimate_total_num_tokens(&self.readers, indexed_field)?;
@@ -459,8 +441,8 @@ impl IndexMerger {
indexed. Have you modified the schema?",
);
- let mut segment_postings_containing_the_term: Vec<(usize, SegmentPostings)> = vec![];
- let mut doc_id_and_positions = vec![];
+ let mut segment_postings_containing_the_term: FxHashMap<usize, SegmentPostings> =
+ FxHashMap::default();
while merged_terms.advance() {
segment_postings_containing_the_term.clear();
@@ -476,7 +458,7 @@ impl IndexMerger {
let doc_freq = segment_postings.doc_freq();
if doc_freq > 0u32 {
total_doc_freq += doc_freq;
- segment_postings_containing_the_term.push((segment_ord, segment_postings));
+ segment_postings_containing_the_term.insert(segment_ord, segment_postings);
}
}
@@ -494,12 +476,14 @@ impl IndexMerger {
assert!(!segment_postings_containing_the_term.is_empty());
let has_term_freq = {
- let has_term_freq = !segment_postings_containing_the_term[0]
- .1
+ let has_term_freq = !segment_postings_containing_the_term
+ .values()
+ .next()
+ .unwrap()
.block_cursor
.freqs()
.is_empty();
- for (_, postings) in &segment_postings_containing_the_term[1..] {
+ for postings in segment_postings_containing_the_term.values().skip(1) {
// This may look at a strange way to test whether we have term freq or not.
// With JSON object, the schema is not sufficient to know whether a term
// has its term frequency encoded or not:
@@ -528,59 +512,45 @@ impl IndexMerger {
// We can now serialize this postings, by pushing each document to the
// postings serializer.
- for (segment_ord, mut segment_postings) in
- segment_postings_containing_the_term.drain(..)
- {
- let old_to_new_doc_id = &merged_doc_id_map[segment_ord];
-
- let mut doc = segment_postings.doc();
- while doc != TERMINATED {
- // deleted doc are skipped as they do not have a `remapped_doc_id`.
- if let Some(remapped_doc_id) = old_to_new_doc_id[doc as usize] {
- // we make sure to only write the term if
- // there is at least one document.
- let term_freq = if has_term_freq {
- segment_postings.positions(&mut positions_buffer);
- segment_postings.term_freq()
- } else {
- // The positions_buffer may contain positions from the previous term
- // Existence of positions depend on the value type in JSON fields.
- // https://github.com/quickwit-oss/tantivy/issues/2283
- positions_buffer.clear();
- 0u32
- };
-
- // if doc_id_mapping exists, the doc_ids are reordered, they are
- // not just stacked. The field serializer expects monotonically increasing
- // doc_ids, so we collect and sort them first, before writing.
- //
- // I think this is not strictly necessary, it would be possible to
- // avoid the loading into a vec via some form of kmerge, but then the merge
- // logic would deviate much more from the stacking case (unsorted index)
- if !doc_id_mapping.is_trivial() {
- doc_id_and_positions.push((
- remapped_doc_id,
- term_freq,
- positions_buffer.to_vec(),
- ));
- } else {
- let delta_positions = delta_computer.compute_delta(&positions_buffer);
- field_serializer.write_doc(remapped_doc_id, term_freq, delta_positions);
- }
+
+ // Each segment_postings is already sorted by their new doc_id's
+ // (if a new_doc_id(a) < new_doc_id(b), then old_doc_id(a) < old_doc_id(b)).
+ // We can therefore just iterate over the doc_id_mapping and write the term for each
+ // document.
+ for (remapped_doc_id, old_doc_addr) in doc_id_mapping.iter_old_doc_addrs().enumerate() {
+ if let Some(segment_postings) = segment_postings_containing_the_term
+ .get_mut(&(old_doc_addr.segment_ord as usize))
+ {
+ if segment_postings.doc() == TERMINATED {
+ continue;
}
- doc = segment_postings.advance();
- }
- }
- if !doc_id_mapping.is_trivial() {
- doc_id_and_positions.sort_unstable_by_key(|&(doc_id, _, _)| doc_id);
+ assert!(old_doc_addr.doc_id <= segment_postings.doc());
+ if old_doc_addr.doc_id < segment_postings.doc() {
+ // The doc doesn't contain the term for this posting list.
+ continue;
+ }
+ // we make sure to only write the term if
+ // there is at least one document.
+ let term_freq = if has_term_freq {
+ segment_postings.positions(&mut positions_buffer);
+ segment_postings.term_freq()
+ } else {
+ // The positions_buffer may contain positions from the previous term
+ // Existence of positions depend on the value type in JSON fields.
+ // https://github.com/quickwit-oss/tantivy/issues/2283
+ positions_buffer.clear();
+ 0u32
+ };
+
+ let delta_positions = delta_computer.compute_delta(&positions_buffer);
+ field_serializer.write_doc(remapped_doc_id as u32, term_freq, delta_positions);
- for (doc_id, term_freq, positions) in &doc_id_and_positions {
- let delta_positions = delta_computer.compute_delta(positions);
- field_serializer.write_doc(*doc_id, *term_freq, delta_positions);
+ segment_postings.advance();
}
- doc_id_and_positions.clear();
}
+ segment_postings_containing_the_term.clear();
+
// closing the term.
field_serializer.close_term()?;
}
diff --git a/crates/tantivy/src/termdict/fst_termdict/merger.rs b/crates/tantivy/src/termdict/fst_termdict/merger.rs
index a406118f..7efd2423 100644
--- a/crates/tantivy/src/termdict/fst_termdict/merger.rs
+++ b/crates/tantivy/src/termdict/fst_termdict/merger.rs
@@ -24,7 +24,7 @@ impl<'a> TermMerger<'a> {
/// Stream of merged term dictionary
pub fn new(streams: Vec<TermStreamer<'a>>) -> TermMerger<'a> {
let mut op_builder = OpBuilder::new();
- let mut dictionaries = vec![];
+ let mut dictionaries = Vec::with_capacity(streams.len());
for streamer in streams {
op_builder.push(streamer.stream);
dictionaries.push(streamer.fst_map);