web search done right
[tantivy] raw column values codec
Mikkel Denker3a48c8c7b4ca
diff --git a/crates/tantivy/src/columnar/column/serialize.rs b/crates/tantivy/src/columnar/column/serialize.rsindex 211432d7..569962da 100644--- a/crates/tantivy/src/columnar/column/serialize.rs+++ b/crates/tantivy/src/columnar/column/serialize.rs@@ -19,11 +19,7 @@ pub fn serialize_column_mappable_to_u64<T: MonotonicallyMappableToU64>(output: &mut impl Write,) -> io::Result<()> {let column_index_num_bytes = serialize_column_index(column_index, output)?;- serialize_u64_based_column_values(- column_values,- &[CodecType::Bitpacked, CodecType::BlockwiseLinear],- output,- )?;+ serialize_u64_based_column_values(column_values, &[CodecType::Raw], output)?;output.write_all(&column_index_num_bytes.to_le_bytes())?;Ok(())}diff --git a/crates/tantivy/src/columnar/column_values/u64_based/mod.rs b/crates/tantivy/src/columnar/column_values/u64_based/mod.rsindex 2ed31206..cc2d44b6 100644--- a/crates/tantivy/src/columnar/column_values/u64_based/mod.rs+++ b/crates/tantivy/src/columnar/column_values/u64_based/mod.rs@@ -2,6 +2,7 @@ mod bitpacked;mod blockwise_linear;mod line;mod linear;+mod raw;mod stats_collector;use std::io;@@ -17,6 +18,7 @@ use crate::columnar::column_values::monotonic_mapping::{pub use crate::columnar::column_values::u64_based::bitpacked::BitpackedCodec;pub use crate::columnar::column_values::u64_based::blockwise_linear::BlockwiseLinearCodec;pub use crate::columnar::column_values::u64_based::linear::LinearCodec;+pub use crate::columnar::column_values::u64_based::raw::RawCodec;pub use crate::columnar::column_values::u64_based::stats_collector::StatsCollector;use crate::columnar::column_values::{monotonic_map_column, ColumnStats};use crate::columnar::iterable::Iterable;@@ -86,13 +88,16 @@ pub enum CodecType {Linear = 1u8,/// Same as [`CodecType::Linear`], but encodes in blocks of 512 elements.BlockwiseLinear = 2u8,+ /// Store the raw values without any compression.+ Raw = 3u8,}/// List of all available u64-base codecs.-pub const ALL_U64_CODEC_TYPES: [CodecType; 3] = [+pub const ALL_U64_CODEC_TYPES: [CodecType; 4] = [CodecType::Bitpacked,CodecType::Linear,CodecType::BlockwiseLinear,+ CodecType::Raw,];impl CodecType {@@ -105,6 +110,7 @@ impl CodecType {0u8 => Some(CodecType::Bitpacked),1u8 => Some(CodecType::Linear),2u8 => Some(CodecType::BlockwiseLinear),+ 3u8 => Some(CodecType::Raw),_ => None,}}@@ -117,6 +123,7 @@ impl CodecType {CodecType::Bitpacked => load_specific_codec::<BitpackedCodec, T>(bytes),CodecType::Linear => load_specific_codec::<LinearCodec, T>(bytes),CodecType::BlockwiseLinear => load_specific_codec::<BlockwiseLinearCodec, T>(bytes),+ CodecType::Raw => load_specific_codec::<RawCodec, T>(bytes),}}}@@ -139,6 +146,7 @@ impl CodecType {CodecType::Bitpacked => BitpackedCodec::boxed_estimator(),CodecType::Linear => LinearCodec::boxed_estimator(),CodecType::BlockwiseLinear => BlockwiseLinearCodec::boxed_estimator(),+ CodecType::Raw => RawCodec::boxed_estimator(),}}}diff --git a/crates/tantivy/src/columnar/column_values/u64_based/raw.rs b/crates/tantivy/src/columnar/column_values/u64_based/raw.rsnew file mode 100644index 00000000..9bea4b6f--- /dev/null+++ b/crates/tantivy/src/columnar/column_values/u64_based/raw.rs@@ -0,0 +1,130 @@+use std::io::Write;++use ownedbytes::OwnedBytes;++use crate::{+ columnar::{column_values::ColumnStats, ColumnValues},+ common::BinarySerializable,+};++use super::{ColumnCodec, ColumnCodecEstimator};++pub struct RawCodec;++impl ColumnCodec for RawCodec {+ type ColumnValues = RawReader;+ type Estimator = RawCodecEstimator;++ fn load(bytes: OwnedBytes) -> std::io::Result<Self::ColumnValues> {+ RawReader::load(bytes)+ }+}++pub struct RawReader {+ min_value: u64,+ max_value: u64,+ num_rows: u32,+ data: OwnedBytes,+}++impl RawReader {+ fn load(bytes: OwnedBytes) -> std::io::Result<Self> {+ let mut bytes = bytes;++ let num_rows = u32::deserialize(&mut bytes)?;+ let min_value = u64::deserialize(&mut bytes)?;+ let max_value = u64::deserialize(&mut bytes)?;++ let data = bytes;++ Ok(Self {+ min_value,+ max_value,+ num_rows,+ data,+ })+ }+}++impl ColumnValues for RawReader {+ fn get_val(&self, idx: u32) -> u64 {+ let idx = idx as usize;+ let mut bytes = &self.data[idx * 8..(idx + 1) * 8];+ u64::deserialize(&mut bytes).unwrap()+ }++ fn min_value(&self) -> u64 {+ self.min_value+ }++ fn max_value(&self) -> u64 {+ self.max_value+ }++ fn num_vals(&self) -> u32 {+ self.num_rows+ }+}++#[derive(Default)]+pub struct RawCodecEstimator;++impl ColumnCodecEstimator for RawCodecEstimator {+ fn collect(&mut self, _: u64) {}++ fn estimate(&self, stats: &ColumnStats) -> Option<u64> {+ Some(stats.num_rows as u64 * 8)+ }++ fn serialize(+ &self,+ stats: &ColumnStats,+ vals: &mut dyn Iterator<Item = u64>,+ wrt: &mut dyn Write,+ ) -> std::io::Result<()> {+ stats.num_rows.serialize(wrt)?;+ stats.min_value.serialize(wrt)?;+ stats.max_value.serialize(wrt)?;++ for val in vals {+ val.serialize(wrt)?;+ }++ wrt.flush()?;++ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::columnar::column_values::u64_based::tests::create_and_validate;++ #[test]+ fn test_with_codec_data_sets_simple() {+ create_and_validate::<RawCodec>(&[4, 3, 12], "name");+ }++ #[test]+ fn test_with_codec_data_sets() {+ let data_sets = crate::columnar::column_values::u64_based::tests::get_codec_test_datasets();+ for (mut data, name) in data_sets {+ create_and_validate::<RawCodec>(&data, name);+ data.reverse();+ create_and_validate::<RawCodec>(&data, name);+ }+ }++ #[test]+ fn test_column_field_rand() {+ for _ in 0..500 {+ let mut data = (0..1 + rand::random::<u8>() as usize)+ .map(|_| rand::random::<i64>() as u64 / 2)+ .collect::<Vec<_>>();+ create_and_validate::<RawCodec>(&data, "rand");+ data.reverse();+ create_and_validate::<RawCodec>(&data, "rand");+ }+ }+}diff --git a/crates/tantivy/src/columnar/column_values/u64_based/tests.rs b/crates/tantivy/src/columnar/column_values/u64_based/tests.rsindex a739fb26..6dd05fe3 100644--- a/crates/tantivy/src/columnar/column_values/u64_based/tests.rs+++ b/crates/tantivy/src/columnar/column_values/u64_based/tests.rs@@ -315,7 +315,7 @@ fn test_column_field_codec_type_to_code() {count_codec += 1;}}- assert_eq!(count_codec, 3);+ assert_eq!(count_codec, 4);}fn test_columnfield_gcd_i64_with_codec(codec_type: CodecType, num_vals: usize) -> io::Result<()> {diff --git a/crates/tantivy/src/columnar/tests.rs b/crates/tantivy/src/columnar/tests.rsindex d800a34d..4a89cbcb 100644--- a/crates/tantivy/src/columnar/tests.rs+++ b/crates/tantivy/src/columnar/tests.rs@@ -13,7 +13,7 @@ fn test_dataframe_writer_bytes() {assert_eq!(columnar.num_columns(), 1);let cols: Vec<DynamicColumnHandle> = columnar.read_columns("my_string").unwrap();assert_eq!(cols.len(), 1);- assert_eq!(cols[0].num_bytes(), 66);+ assert!(cols[0].num_bytes() > 0);}#[test]diff --git a/crates/tantivy/src/columnfield/mod.rs b/crates/tantivy/src/columnfield/mod.rsindex 8d79074f..292a4585 100644--- a/crates/tantivy/src/columnfield/mod.rs+++ b/crates/tantivy/src/columnfield/mod.rs@@ -125,7 +125,7 @@ mod tests {}let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 80);+ assert!(file.len() > 0);let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();let column = column_field_readers.u64("field").unwrap().values;assert_eq!(column.get_val(0), 13u64);@@ -172,7 +172,7 @@ mod tests {write.terminate().unwrap();}let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 108);+ assert!(file.len() > 0);let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();let col = column_field_readers.u64("field").unwrap().values;assert_eq!(col.get_val(0), 4u64);@@ -202,7 +202,7 @@ mod tests {write.terminate().unwrap();}let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 81);+ assert!(file.len() > 0);let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();let column_field_reader = column_field_readers.u64("field").unwrap().values;for doc in 0..10_000 {@@ -231,7 +231,7 @@ mod tests {write.terminate().unwrap();}let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 4476);+ assert!(file.len() > 0);{let column_field_readers = ColumnFieldReaders::open(file, SCHEMA.clone()).unwrap();let col = column_field_readers.u64("field").unwrap().values;@@ -261,7 +261,7 @@ mod tests {write.terminate().unwrap();}let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 252);+ assert!(file.len() > 0);{let column_field_readers = ColumnFieldReaders::open(file, schema).unwrap();@@ -431,7 +431,7 @@ mod tests {write.terminate().unwrap();}let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 84);+ assert!(file.len() > 0);let column_field_readers = ColumnFieldReaders::open(file, schema).unwrap();let bool_col = column_field_readers.bool("field_bool").unwrap();assert_eq!(bool_col.first(0), Some(true));@@ -465,7 +465,7 @@ mod tests {write.terminate().unwrap();}let file = directory.open_read(path).unwrap();- assert_eq!(file.len(), 96);+ assert!(file.len() > 0);let readers = ColumnFieldReaders::open(file, schema).unwrap();let bool_col = readers.bool("field_bool").unwrap();for i in 0..25 {@@ -489,6 +489,7 @@ mod tests {}#[test]+ #[ignore]pub fn test_gcd_date() {let size_prec_sec = test_gcd_date_with_codec(DateTimePrecision::Seconds);assert!((1000 * 13 / 8..100 + 1000 * 13 / 8).contains(&size_prec_sec.get_bytes())); // 13 bits per val = ceil(log_2(number of seconds in 2hours);