web search done right
store number of position bytes as u64 instead of u32 so we can have more than 4gb of positions in a segment
Mikkel Denker297f79d46bd7
diff --git a/crates/tantivy/src/columnar/column/serialize.rs b/crates/tantivy/src/columnar/column/serialize.rsindex 226e66aa..eca3fb08 100644--- a/crates/tantivy/src/columnar/column/serialize.rs+++ b/crates/tantivy/src/columnar/column/serialize.rs@@ -98,8 +98,8 @@ pub fn open_column_u128_as_compact_u64(bytes: OwnedBytes) -> io::Result<Column<u}pub fn open_column_bytes(data: OwnedBytes) -> io::Result<BytesColumn> {- let (body, dictionary_len_bytes) = data.rsplit(4);- let dictionary_len = u32::from_le_bytes(dictionary_len_bytes.as_slice().try_into().unwrap());+ let (body, dictionary_len_bytes) = data.rsplit(8);+ let dictionary_len = u64::from_le_bytes(dictionary_len_bytes.as_slice().try_into().unwrap());let (dictionary_bytes, column_bytes) = body.split(dictionary_len as usize);let dictionary = Arc::new(Dictionary::from_bytes(dictionary_bytes)?);let term_ord_column = crate::columnar::column::open_column_u64::<u64>(column_bytes)?;diff --git a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rsindex a8e1b538..27a7c307 100644--- a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs+++ b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs@@ -9,7 +9,7 @@ use crate::columnar::column_index::SerializableColumnIndex;use crate::columnar::iterable::Iterable;use crate::columnar::{BytesColumn, MergeRowOrder, ShuffleMergeOrder};-// Serialize [Dictionary, Column, dictionary num bytes U32::LE]+// Serialize [Dictionary, Column, dictionary num bytes U64::LE]// Column: [Column Index, Column Values, column index num bytes U32::LE]pub fn merge_bytes_or_str_column(column_index: SerializableColumnIndex<'_>,@@ -21,7 +21,7 @@ pub fn merge_bytes_or_str_column(let mut output = CountingWriter::wrap(output);// TODO !!! Remove useless terms.let term_ord_mapping = serialize_merged_dict(bytes_columns, merge_row_order, &mut output)?;- let dictionary_num_bytes: u32 = output.written_bytes() as u32;+ let dictionary_num_bytes: u64 = output.written_bytes();let output = output.finish();let remapped_term_ordinals_values = RemappedTermOrdinalsValues {bytes_columns,diff --git a/crates/tantivy/src/columnar/columnar/writer/mod.rs b/crates/tantivy/src/columnar/columnar/writer/mod.rsindex 8f94a5fa..4556047d 100644--- a/crates/tantivy/src/columnar/columnar/writer/mod.rs+++ b/crates/tantivy/src/columnar/columnar/writer/mod.rs@@ -468,7 +468,7 @@ impl ColumnarWriter {}}-// Serialize [Dictionary, Column, dictionary num bytes U32::LE]+// Serialize [Dictionary, Column, dictionary num bytes U64::LE]// Column: [Column Index, Column Values, column index num bytes U32::LE]#[allow(clippy::too_many_arguments)]fn serialize_bytes_or_str_column(@@ -489,7 +489,7 @@ fn serialize_bytes_or_str_column(let mut counting_writer = CountingWriter::wrap(wrt);let term_id_mapping: TermIdMapping =dictionary_builder.serialize(arena, &mut counting_writer)?;- let dictionary_num_bytes: u32 = counting_writer.written_bytes() as u32;+ let dictionary_num_bytes: u64 = counting_writer.written_bytes();let mut wrt = counting_writer.finish();let operation_iterator = operation_it.map(|symbol: ColumnOperation<UnorderedId>| {// We map unordered ids to ordered ids.diff --git a/crates/tantivy/src/columnar/tests.rs b/crates/tantivy/src/columnar/tests.rsindex 5fb96a3e..bade724b 100644--- a/crates/tantivy/src/columnar/tests.rs+++ b/crates/tantivy/src/columnar/tests.rs@@ -26,7 +26,7 @@ fn test_dataframe_writer_str() {assert_eq!(columnar.num_columns(), 1);let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 73);+ assert_eq!(cols[0].num_bytes(), 77);}#[test]@@ -40,7 +40,7 @@ fn test_dataframe_writer_bytes() {assert_eq!(columnar.num_columns(), 1);let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 73);+ assert_eq!(cols[0].num_bytes(), 77);}#[test]diff --git a/crates/tantivy/src/postings/term_info.rs b/crates/tantivy/src/postings/term_info.rsindex 0ce98e29..210f5604 100644--- a/crates/tantivy/src/postings/term_info.rs+++ b/crates/tantivy/src/postings/term_info.rs@@ -16,16 +16,16 @@ pub struct TermInfo {}impl TermInfo {- pub(crate) fn posting_num_bytes(&self) -> u32 {+ pub(crate) fn posting_num_bytes(&self) -> u64 {let num_bytes = self.postings_range.len();- assert!(num_bytes <= u32::MAX as usize);- num_bytes as u32+ assert!(num_bytes <= u64::MAX as usize);+ num_bytes as u64}- pub(crate) fn positions_num_bytes(&self) -> u32 {+ pub(crate) fn positions_num_bytes(&self) -> u64 {let num_bytes = self.positions_range.len();- assert!(num_bytes <= u32::MAX as usize);- num_bytes as u32+ assert!(num_bytes <= u64::MAX as usize);+ num_bytes as u64}}@@ -34,7 +34,7 @@ impl FixedSize for TermInfo {/// This is large, but in practise, `TermInfo` are encoded in blocks and/// only the first `TermInfo` of a block is serialized uncompressed./// The subsequent `TermInfo` are delta encoded and bitpacked.- const SIZE_IN_BYTES: usize = 3 * u32::SIZE_IN_BYTES + 2 * u64::SIZE_IN_BYTES;+ const SIZE_IN_BYTES: usize = u32::SIZE_IN_BYTES + 4 * u64::SIZE_IN_BYTES;}impl BinarySerializable for TermInfo {@@ -50,10 +50,10 @@ impl BinarySerializable for TermInfo {fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {let doc_freq = u32::deserialize(reader)?;let postings_start_offset = u64::deserialize(reader)? as usize;- let postings_num_bytes = u32::deserialize(reader)? as usize;+ let postings_num_bytes = u64::deserialize(reader)? as usize;let postings_end_offset = postings_start_offset + postings_num_bytes;let positions_start_offset = u64::deserialize(reader)? as usize;- let positions_num_bytes = u32::deserialize(reader)? as usize;+ let positions_num_bytes = u64::deserialize(reader)? as usize;let positions_end_offset = positions_start_offset + positions_num_bytes;Ok(TermInfo {doc_freq,