Skip to content

web search done right

store number of position bytes as u64 instead of u32 so we can have more than 4gb of positions in a segment

Mikkel Denker297f79d46bd7
diff --git a/crates/tantivy/src/columnar/column/serialize.rs b/crates/tantivy/src/columnar/column/serialize.rs
index 226e66aa..eca3fb08 100644
--- a/crates/tantivy/src/columnar/column/serialize.rs
+++ b/crates/tantivy/src/columnar/column/serialize.rs
@@ -98,8 +98,8 @@ pub fn open_column_u128_as_compact_u64(bytes: OwnedBytes) -> io::Result<Column<u
}
pub fn open_column_bytes(data: OwnedBytes) -> io::Result<BytesColumn> {
- let (body, dictionary_len_bytes) = data.rsplit(4);
- let dictionary_len = u32::from_le_bytes(dictionary_len_bytes.as_slice().try_into().unwrap());
+ let (body, dictionary_len_bytes) = data.rsplit(8);
+ let dictionary_len = u64::from_le_bytes(dictionary_len_bytes.as_slice().try_into().unwrap());
let (dictionary_bytes, column_bytes) = body.split(dictionary_len as usize);
let dictionary = Arc::new(Dictionary::from_bytes(dictionary_bytes)?);
let term_ord_column = crate::columnar::column::open_column_u64::<u64>(column_bytes)?;
diff --git a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs
index a8e1b538..27a7c307 100644
--- a/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs
+++ b/crates/tantivy/src/columnar/columnar/merge/merge_dict_column.rs
@@ -9,7 +9,7 @@ use crate::columnar::column_index::SerializableColumnIndex;
use crate::columnar::iterable::Iterable;
use crate::columnar::{BytesColumn, MergeRowOrder, ShuffleMergeOrder};
-// Serialize [Dictionary, Column, dictionary num bytes U32::LE]
+// Serialize [Dictionary, Column, dictionary num bytes U64::LE]
// Column: [Column Index, Column Values, column index num bytes U32::LE]
pub fn merge_bytes_or_str_column(
column_index: SerializableColumnIndex<'_>,
@@ -21,7 +21,7 @@ pub fn merge_bytes_or_str_column(
let mut output = CountingWriter::wrap(output);
// TODO !!! Remove useless terms.
let term_ord_mapping = serialize_merged_dict(bytes_columns, merge_row_order, &mut output)?;
- let dictionary_num_bytes: u32 = output.written_bytes() as u32;
+ let dictionary_num_bytes: u64 = output.written_bytes();
let output = output.finish();
let remapped_term_ordinals_values = RemappedTermOrdinalsValues {
bytes_columns,
diff --git a/crates/tantivy/src/columnar/columnar/writer/mod.rs b/crates/tantivy/src/columnar/columnar/writer/mod.rs
index 8f94a5fa..4556047d 100644
--- a/crates/tantivy/src/columnar/columnar/writer/mod.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/mod.rs
@@ -468,7 +468,7 @@ impl ColumnarWriter {
}
}
-// Serialize [Dictionary, Column, dictionary num bytes U32::LE]
+// Serialize [Dictionary, Column, dictionary num bytes U64::LE]
// Column: [Column Index, Column Values, column index num bytes U32::LE]
#[allow(clippy::too_many_arguments)]
fn serialize_bytes_or_str_column(
@@ -489,7 +489,7 @@ fn serialize_bytes_or_str_column(
let mut counting_writer = CountingWriter::wrap(wrt);
let term_id_mapping: TermIdMapping =
dictionary_builder.serialize(arena, &mut counting_writer)?;
- let dictionary_num_bytes: u32 = counting_writer.written_bytes() as u32;
+ let dictionary_num_bytes: u64 = counting_writer.written_bytes();
let mut wrt = counting_writer.finish();
let operation_iterator = operation_it.map(|symbol: ColumnOperation<UnorderedId>| {
// We map unordered ids to ordered ids.
diff --git a/crates/tantivy/src/columnar/tests.rs b/crates/tantivy/src/columnar/tests.rs
index 5fb96a3e..bade724b 100644
--- a/crates/tantivy/src/columnar/tests.rs
+++ b/crates/tantivy/src/columnar/tests.rs
@@ -26,7 +26,7 @@ fn test_dataframe_writer_str() {
assert_eq!(columnar.num_columns(), 1);
let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].num_bytes(), 73);
+ assert_eq!(cols[0].num_bytes(), 77);
}
#[test]
@@ -40,7 +40,7 @@ fn test_dataframe_writer_bytes() {
assert_eq!(columnar.num_columns(), 1);
let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();
assert_eq!(cols.len(), 1);
- assert_eq!(cols[0].num_bytes(), 73);
+ assert_eq!(cols[0].num_bytes(), 77);
}
#[test]
diff --git a/crates/tantivy/src/postings/term_info.rs b/crates/tantivy/src/postings/term_info.rs
index 0ce98e29..210f5604 100644
--- a/crates/tantivy/src/postings/term_info.rs
+++ b/crates/tantivy/src/postings/term_info.rs
@@ -16,16 +16,16 @@ pub struct TermInfo {
}
impl TermInfo {
- pub(crate) fn posting_num_bytes(&self) -> u32 {
+ pub(crate) fn posting_num_bytes(&self) -> u64 {
let num_bytes = self.postings_range.len();
- assert!(num_bytes <= u32::MAX as usize);
- num_bytes as u32
+ assert!(num_bytes <= u64::MAX as usize);
+ num_bytes as u64
}
- pub(crate) fn positions_num_bytes(&self) -> u32 {
+ pub(crate) fn positions_num_bytes(&self) -> u64 {
let num_bytes = self.positions_range.len();
- assert!(num_bytes <= u32::MAX as usize);
- num_bytes as u32
+ assert!(num_bytes <= u64::MAX as usize);
+ num_bytes as u64
}
}
@@ -34,7 +34,7 @@ impl FixedSize for TermInfo {
/// This is large, but in practise, `TermInfo` are encoded in blocks and
/// only the first `TermInfo` of a block is serialized uncompressed.
/// The subsequent `TermInfo` are delta encoded and bitpacked.
- const SIZE_IN_BYTES: usize = 3 * u32::SIZE_IN_BYTES + 2 * u64::SIZE_IN_BYTES;
+ const SIZE_IN_BYTES: usize = u32::SIZE_IN_BYTES + 4 * u64::SIZE_IN_BYTES;
}
impl BinarySerializable for TermInfo {
@@ -50,10 +50,10 @@ impl BinarySerializable for TermInfo {
fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
let doc_freq = u32::deserialize(reader)?;
let postings_start_offset = u64::deserialize(reader)? as usize;
- let postings_num_bytes = u32::deserialize(reader)? as usize;
+ let postings_num_bytes = u64::deserialize(reader)? as usize;
let postings_end_offset = postings_start_offset + postings_num_bytes;
let positions_start_offset = u64::deserialize(reader)? as usize;
- let positions_num_bytes = u32::deserialize(reader)? as usize;
+ let positions_num_bytes = u64::deserialize(reader)? as usize;
let positions_end_offset = positions_start_offset + positions_num_bytes;
Ok(TermInfo {
doc_freq,