Skip to content

web search done right

[tantivy] support u128 values in inverted index

Mikkel Denker7b75a854b74d
diff --git a/crates/tantivy/src/collector/top_score_collector.rs b/crates/tantivy/src/collector/top_score_collector.rs
index ac064fab..5b94e061 100644
--- a/crates/tantivy/src/collector/top_score_collector.rs
+++ b/crates/tantivy/src/collector/top_score_collector.rs
@@ -2,7 +2,7 @@ use std::fmt;
use std::marker::PhantomData;
use std::sync::Arc;
-use crate::columnar::ColumnValues;
+use crate::columnar::{ColumnValues, MonotonicallyMappableToU64};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
@@ -31,7 +31,7 @@ impl<TCollector, TColumnarValue> Collector
for ColumnFieldConvertCollector<TCollector, TColumnarValue>
where
TCollector: Collector<Fruit = Vec<(u64, DocAddress)>>,
- TColumnarValue: ColumnarValue,
+ TColumnarValue: ColumnarValue + MonotonicallyMappableToU64,
{
type Fruit = Vec<(TColumnarValue, DocAddress)>;
@@ -273,7 +273,7 @@ impl TopDocs {
order: Order,
) -> impl Collector<Fruit = Vec<(TColumnarValue, DocAddress)>>
where
- TColumnarValue: ColumnarValue,
+ TColumnarValue: ColumnarValue + MonotonicallyMappableToU64,
{
let u64_collector = self.order_by_u64_field(column_field.to_string(), order.clone());
ColumnFieldConvertCollector {
diff --git a/crates/tantivy/src/columnar/column/mod.rs b/crates/tantivy/src/columnar/column/mod.rs
index 1a6a32d2..9f13be8b 100644
--- a/crates/tantivy/src/columnar/column/mod.rs
+++ b/crates/tantivy/src/columnar/column/mod.rs
@@ -8,12 +8,15 @@ use std::sync::Arc;
use crate::common::BinarySerializable;
pub use dictionary_encoded::BytesColumn;
-pub use serialize::{open_column_bytes, open_column_u64, serialize_column_mappable_to_u64};
+pub use serialize::{
+ open_column_bytes, open_column_u128, open_column_u64, serialize_column_mappable_to_u128,
+ serialize_column_mappable_to_u64,
+};
use super::column_index::ColumnIndex;
use super::column_values::monotonic_mapping::StrictlyMonotonicMappingToInternal;
use super::column_values::{monotonic_map_column, ColumnValues};
-use super::{Cardinality, DocId, MonotonicallyMappableToU64, RowId};
+use super::{Cardinality, DocId, MonotonicallyMappableToU128, MonotonicallyMappableToU64, RowId};
#[derive(Clone)]
pub struct Column<T = u64> {
@@ -44,6 +47,19 @@ impl<T: MonotonicallyMappableToU64> Column<T> {
}
}
+impl<T: MonotonicallyMappableToU128> Column<T> {
+ pub fn to_u128_monotonic(self) -> Column<u128> {
+ let values = Arc::new(monotonic_map_column(
+ self.values,
+ StrictlyMonotonicMappingToInternal::<T>::new(),
+ ));
+ Column {
+ index: self.index,
+ values,
+ }
+ }
+}
+
impl<T: PartialOrd + Copy + Debug + Send + Sync + 'static> Column<T> {
pub fn num_docs(&self) -> RowId {
match &self.index {
diff --git a/crates/tantivy/src/columnar/column/serialize.rs b/crates/tantivy/src/columnar/column/serialize.rs
index 569962da..5f5d2684 100644
--- a/crates/tantivy/src/columnar/column/serialize.rs
+++ b/crates/tantivy/src/columnar/column/serialize.rs
@@ -8,8 +8,9 @@ use crate::sstable::Dictionary;
use crate::columnar::column::{BytesColumn, Column};
use crate::columnar::column_index::{serialize_column_index, SerializableColumnIndex};
use crate::columnar::column_values::{
- load_u64_based_column_values, serialize_u64_based_column_values, CodecType,
- MonotonicallyMappableToU64,
+ load_u128_based_column_values, load_u64_based_column_values,
+ serialize_u128_based_column_values, serialize_u64_based_column_values,
+ MonotonicallyMappableToU128, MonotonicallyMappableToU64, U128CodecType, U64CodecType,
};
use crate::columnar::iterable::Iterable;
@@ -19,7 +20,7 @@ pub fn serialize_column_mappable_to_u64<T: MonotonicallyMappableToU64>(
output: &mut impl Write,
) -> io::Result<()> {
let column_index_num_bytes = serialize_column_index(column_index, output)?;
- serialize_u64_based_column_values(column_values, &[CodecType::Raw], output)?;
+ serialize_u64_based_column_values(column_values, &[U64CodecType::Raw], output)?;
output.write_all(&column_index_num_bytes.to_le_bytes())?;
Ok(())
}
@@ -41,6 +42,36 @@ pub fn open_column_u64<T: MonotonicallyMappableToU64>(bytes: OwnedBytes) -> io::
})
}
+pub fn serialize_column_mappable_to_u128<T: MonotonicallyMappableToU128>(
+ column_index: SerializableColumnIndex,
+ column_values: &impl Iterable<T>,
+ output: &mut impl Write,
+) -> io::Result<()> {
+ let column_index_num_bytes = serialize_column_index(column_index, output)?;
+ serialize_u128_based_column_values(column_values, &[U128CodecType::Raw], output)?;
+ output.write_all(&column_index_num_bytes.to_le_bytes())?;
+ Ok(())
+}
+
+pub fn open_column_u128<T: MonotonicallyMappableToU128>(
+ bytes: OwnedBytes,
+) -> io::Result<Column<T>> {
+ let (body, column_index_num_bytes_payload) = bytes.rsplit(4);
+ let column_index_num_bytes = u32::from_le_bytes(
+ column_index_num_bytes_payload
+ .as_slice()
+ .try_into()
+ .unwrap(),
+ );
+ let (column_index_data, column_values_data) = body.split(column_index_num_bytes as usize);
+ let column_index = crate::columnar::column_index::open_column_index(column_index_data)?;
+ let column_values = load_u128_based_column_values(column_values_data)?;
+ Ok(Column {
+ index: column_index,
+ values: column_values,
+ })
+}
+
pub fn open_column_bytes(data: OwnedBytes) -> io::Result<BytesColumn> {
let (body, dictionary_len_bytes) = data.rsplit(8);
let dictionary_len = u64::from_le_bytes(dictionary_len_bytes.as_slice().try_into().unwrap());
diff --git a/crates/tantivy/src/columnar/column_values/mod.rs b/crates/tantivy/src/columnar/column_values/mod.rs
index 42353dc3..55cbd05a 100644
--- a/crates/tantivy/src/columnar/column_values/mod.rs
+++ b/crates/tantivy/src/columnar/column_values/mod.rs
@@ -7,6 +7,7 @@
//! - Monotonically map values to u64/u128
use std::fmt::Debug;
+use std::io;
use std::ops::{Range, RangeInclusive};
use std::sync::Arc;
@@ -18,16 +19,21 @@ mod merge;
pub(crate) mod monotonic_mapping;
pub(crate) mod monotonic_mapping_u128;
mod stats;
+mod u128_based;
mod u64_based;
mod vec_column;
mod monotonic_column;
pub(crate) use merge::MergedColumnValues;
-pub use stats::ColumnStats;
+use ownedbytes::OwnedBytes;
+pub use u128_based::{
+ load_u128_based_column_values, serialize_and_load_u128_based_column_values,
+ serialize_u128_based_column_values, CodecType as U128CodecType, ALL_U128_CODEC_TYPES,
+};
pub use u64_based::{
load_u64_based_column_values, serialize_and_load_u64_based_column_values,
- serialize_u64_based_column_values, CodecType, ALL_U64_CODEC_TYPES,
+ serialize_u64_based_column_values, CodecType as U64CodecType, ALL_U64_CODEC_TYPES,
};
pub use vec_column::VecColumn;
@@ -169,6 +175,57 @@ pub trait ColumnValues<T: PartialOrd = u64>: Send + Sync + DowncastSync {
}
downcast_rs::impl_downcast!(sync ColumnValues<T> where T: PartialOrd);
+/// A `ColumnCodecEstimator` is in charge of gathering all
+/// data required to serialize a column.
+///
+/// This happens during a first pass on data of the column elements.
+/// During that pass, all column estimators receive a call to their
+/// `.collect(el)`.
+///
+/// After this first pass, finalize is called.
+/// `.estimate(..)` then should return an accurate estimation of the
+/// size of the serialized column (were we to pick this codec.).
+/// `.serialize(..)` then serializes the column using this codec.
+pub trait ColumnCodecEstimator<T = u64>: 'static {
+ /// Records a new value for estimation.
+ /// This method will be called for each element of the column during
+ /// `estimation`.
+ fn collect(&mut self, value: T);
+ /// Finalizes the first pass phase.
+ fn finalize(&mut self) {}
+ /// Returns an accurate estimation of the number of bytes that will
+ /// be used to represent this column.
+ fn estimate(&self) -> Option<u64>;
+ /// Serializes the column using the given codec.
+ /// This constitutes a second pass over the columns values.
+ fn serialize(
+ &self,
+ vals: &mut dyn Iterator<Item = T>,
+ wrt: &mut dyn io::Write,
+ ) -> io::Result<()>;
+}
+
+/// A column codec describes a colunm serialization format.
+pub trait ColumnCodec<T: PartialOrd = u64> {
+ /// Specialized `ColumnValues` type.
+ type ColumnValues: ColumnValues<T> + 'static;
+ /// `Estimator` for the given codec.
+ type Estimator: ColumnCodecEstimator<T> + Default;
+
+ /// Loads a column that has been serialized using this codec.
+ fn load(bytes: OwnedBytes) -> io::Result<Self::ColumnValues>;
+
+ /// Returns an estimator.
+ fn estimator() -> Self::Estimator {
+ Self::Estimator::default()
+ }
+
+ /// Returns a boxed estimator.
+ fn boxed_estimator() -> Box<dyn ColumnCodecEstimator<T>> {
+ Box::new(Self::estimator())
+ }
+}
+
/// Empty column of values.
pub struct EmptyColumnValues;
diff --git a/crates/tantivy/src/columnar/column_values/stats.rs b/crates/tantivy/src/columnar/column_values/stats.rs
deleted file mode 100644
index 3454f010..00000000
--- a/crates/tantivy/src/columnar/column_values/stats.rs
+++ /dev/null
@@ -1,103 +0,0 @@
-use std::io;
-use std::io::Write;
-use std::num::NonZeroU64;
-
-use crate::common::{BinarySerializable, VInt};
-
-use super::RowId;
-
-/// Column statistics.
-#[derive(Debug, Clone, Eq, PartialEq)]
-pub struct ColumnStats {
- /// GCD of the elements `el - min(column)`.
- pub gcd: NonZeroU64,
- /// Minimum value of the column.
- pub min_value: u64,
- /// Maximum value of the column.
- pub max_value: u64,
- /// Number of rows in the column.
- pub num_rows: RowId,
-}
-
-impl ColumnStats {
- /// Amplitude of value.
- /// Difference between the maximum and the minimum value.
- pub fn amplitude(&self) -> u64 {
- self.max_value - self.min_value
- }
-}
-
-impl BinarySerializable for ColumnStats {
- fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
- VInt(self.min_value).serialize(writer)?;
- VInt(self.gcd.get()).serialize(writer)?;
- VInt(self.amplitude() / self.gcd).serialize(writer)?;
- VInt(self.num_rows as u64).serialize(writer)?;
- Ok(())
- }
-
- fn deserialize<R: io::Read>(reader: &mut R) -> io::Result<Self> {
- let min_value = VInt::deserialize(reader)?.0;
- let gcd = VInt::deserialize(reader)?.0;
- let gcd = NonZeroU64::new(gcd)
- .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "GCD of 0 is forbidden"))?;
- let amplitude = VInt::deserialize(reader)?.0 * gcd.get();
- let max_value = min_value + amplitude;
- let num_rows = VInt::deserialize(reader)?.0 as RowId;
- Ok(ColumnStats {
- min_value,
- max_value,
- num_rows,
- gcd,
- })
- }
-}
-
-#[cfg(test)]
-mod tests {
- use std::num::NonZeroU64;
-
- use crate::common::BinarySerializable;
-
- use crate::columnar::column_values::ColumnStats;
-
- #[track_caller]
- fn test_stats_ser_deser_aux(stats: &ColumnStats, num_bytes: usize) {
- let mut buffer: Vec<u8> = Vec::new();
- stats.serialize(&mut buffer).unwrap();
- assert_eq!(buffer.len(), num_bytes);
- let deser_stats = ColumnStats::deserialize(&mut &buffer[..]).unwrap();
- assert_eq!(stats, &deser_stats);
- }
-
- #[test]
- fn test_stats_serialization() {
- test_stats_ser_deser_aux(
- &(ColumnStats {
- gcd: NonZeroU64::new(3).unwrap(),
- min_value: 1,
- max_value: 3001,
- num_rows: 10,
- }),
- 5,
- );
- test_stats_ser_deser_aux(
- &(ColumnStats {
- gcd: NonZeroU64::new(1_000).unwrap(),
- min_value: 1,
- max_value: 3001,
- num_rows: 10,
- }),
- 5,
- );
- test_stats_ser_deser_aux(
- &(ColumnStats {
- gcd: NonZeroU64::new(1).unwrap(),
- min_value: 0,
- max_value: 0,
- num_rows: 0,
- }),
- 4,
- );
- }
-}
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/stats_collector.rs b/crates/tantivy/src/columnar/column_values/stats/gcd.rs
similarity index 61%
rename from crates/tantivy/src/columnar/column_values/u64_based/stats_collector.rs
rename to crates/tantivy/src/columnar/column_values/stats/gcd.rs
index 675161a9..6e4ec829 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/stats_collector.rs
+++ b/crates/tantivy/src/columnar/column_values/stats/gcd.rs
@@ -2,8 +2,7 @@ use std::num::NonZeroU64;
use fastdivide::DividerU64;
-use crate::columnar::column_values::ColumnStats;
-use crate::columnar::RowId;
+use super::ColumnStatsCollector;
/// Compute the gcd of two non null numbers.
///
@@ -19,10 +18,9 @@ fn compute_gcd(mut large: NonZeroU64, mut small: NonZeroU64) -> NonZeroU64 {
}
}
+/// A collector for the GCD of the differences between the values and the minimal value.
#[derive(Default)]
-pub struct StatsCollector {
- min_max_opt: Option<(u64, u64)>,
- num_rows: RowId,
+pub struct GcdCollector {
// We measure the GCD of the difference between the values and the minimal value.
// This is the same as computing the difference between the values and the first value.
//
@@ -32,22 +30,7 @@ pub struct StatsCollector {
first_value_opt: Option<u64>,
}
-impl StatsCollector {
- pub fn stats(&self) -> ColumnStats {
- let (min_value, max_value) = self.min_max_opt.unwrap_or((0u64, 0u64));
- let increment_gcd = if let Some((increment_gcd, _)) = self.increment_gcd_opt {
- increment_gcd
- } else {
- NonZeroU64::new(1u64).unwrap()
- };
- ColumnStats {
- min_value,
- max_value,
- num_rows: self.num_rows,
- gcd: increment_gcd,
- }
- }
-
+impl GcdCollector {
#[inline]
fn update_increment_gcd(&mut self, value: u64) {
let Some(first_value) = self.first_value_opt else {
@@ -80,35 +63,46 @@ impl StatsCollector {
let new_divider = DividerU64::divide_by(gcd.get());
self.increment_gcd_opt = Some((gcd, new_divider));
}
+}
- pub fn collect(&mut self, value: u64) {
- self.min_max_opt = Some(if let Some((min, max)) = self.min_max_opt {
- (min.min(value), max.max(value))
- } else {
- (value, value)
- });
- self.num_rows += 1;
+impl ColumnStatsCollector<u64> for GcdCollector {
+ type Stats = NonZeroU64;
+
+ fn collect(&mut self, value: u64) {
self.update_increment_gcd(value);
}
+
+ fn finalize(&self) -> Self::Stats {
+ if let Some((gcd, _)) = self.increment_gcd_opt {
+ gcd
+ } else {
+ NonZeroU64::new(1).unwrap()
+ }
+ }
+
+ fn num_bytes(&self) -> u64 {
+ 8
+ }
}
#[cfg(test)]
mod tests {
use std::num::NonZeroU64;
- use crate::columnar::column_values::u64_based::stats_collector::{compute_gcd, StatsCollector};
- use crate::columnar::column_values::u64_based::ColumnStats;
+ use crate::columnar::column_values::stats::{
+ gcd::compute_gcd, ColumnStatsCollector, GcdCollector,
+ };
- fn compute_stats(vals: impl Iterator<Item = u64>) -> ColumnStats {
- let mut stats_collector = StatsCollector::default();
+ fn compute_stats(vals: impl Iterator<Item = u64>) -> NonZeroU64 {
+ let mut stats_collector = GcdCollector::default();
for val in vals {
stats_collector.collect(val);
}
- stats_collector.stats()
+ stats_collector.finalize()
}
fn find_gcd(vals: impl Iterator<Item = u64>) -> u64 {
- compute_stats(vals).gcd.get()
+ compute_stats(vals).get()
}
#[test]
@@ -142,59 +136,22 @@ mod tests {
#[test]
fn test_stats() {
- assert_eq!(
- compute_stats([].into_iter()),
- ColumnStats {
- gcd: NonZeroU64::new(1).unwrap(),
- min_value: 0,
- max_value: 0,
- num_rows: 0
- }
- );
- assert_eq!(
- compute_stats([0, 1].into_iter()),
- ColumnStats {
- gcd: NonZeroU64::new(1).unwrap(),
- min_value: 0,
- max_value: 1,
- num_rows: 2
- }
- );
+ assert_eq!(compute_stats([].into_iter()), NonZeroU64::new(1).unwrap());
assert_eq!(
compute_stats([0, 1].into_iter()),
- ColumnStats {
- gcd: NonZeroU64::new(1).unwrap(),
- min_value: 0,
- max_value: 1,
- num_rows: 2
- }
+ NonZeroU64::new(1).unwrap()
);
assert_eq!(
compute_stats([10, 20, 30].into_iter()),
- ColumnStats {
- gcd: NonZeroU64::new(10).unwrap(),
- min_value: 10,
- max_value: 30,
- num_rows: 3
- }
+ NonZeroU64::new(10).unwrap()
);
assert_eq!(
compute_stats([10, 50, 10, 30].into_iter()),
- ColumnStats {
- gcd: NonZeroU64::new(20).unwrap(),
- min_value: 10,
- max_value: 50,
- num_rows: 4
- }
+ NonZeroU64::new(20).unwrap()
);
assert_eq!(
compute_stats([10, 0, 30].into_iter()),
- ColumnStats {
- gcd: NonZeroU64::new(10).unwrap(),
- min_value: 0,
- max_value: 30,
- num_rows: 3
- }
+ NonZeroU64::new(10).unwrap()
);
}
}
diff --git a/crates/tantivy/src/columnar/column_values/stats/min_max.rs b/crates/tantivy/src/columnar/column_values/stats/min_max.rs
new file mode 100644
index 00000000..61add307
--- /dev/null
+++ b/crates/tantivy/src/columnar/column_values/stats/min_max.rs
@@ -0,0 +1,69 @@
+use super::ColumnStatsCollector;
+
+/// A collector for the minimum and maximum values in a column.
+#[derive(Clone, Default)]
+pub struct MinMaxCollector<T = u64> {
+ min_max_opt: Option<(T, T)>,
+}
+
+impl ColumnStatsCollector<u64> for MinMaxCollector<u64> {
+ type Stats = (u64, u64);
+
+ fn collect(&mut self, value: u64) {
+ self.min_max_opt = Some(if let Some((min, max)) = self.min_max_opt {
+ (min.min(value), max.max(value))
+ } else {
+ (value, value)
+ });
+ }
+
+ fn finalize(&self) -> Self::Stats {
+ self.min_max_opt.unwrap_or((0, 0))
+ }
+
+ fn num_bytes(&self) -> u64 {
+ 8 + 8
+ }
+}
+
+impl ColumnStatsCollector<u128> for MinMaxCollector<u128> {
+ type Stats = (u128, u128);
+
+ fn collect(&mut self, value: u128) {
+ self.min_max_opt = Some(if let Some((min, max)) = self.min_max_opt {
+ (min.min(value), max.max(value))
+ } else {
+ (value, value)
+ });
+ }
+
+ fn finalize(&self) -> Self::Stats {
+ self.min_max_opt.unwrap_or((0, 0))
+ }
+
+ fn num_bytes(&self) -> u64 {
+ 16 + 16
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn compute_stats(vals: impl Iterator<Item = u64>) -> (u64, u64) {
+ let mut stats_collector = MinMaxCollector::default();
+ for val in vals {
+ stats_collector.collect(val);
+ }
+ stats_collector.finalize()
+ }
+
+ #[test]
+ fn test_compute_min_max() {
+ assert_eq!(compute_stats([].into_iter()), (0, 0));
+ assert_eq!(compute_stats([1, 2, 3].into_iter()), (1, 3));
+ assert_eq!(compute_stats([3, 2, 1].into_iter()), (1, 3));
+ assert_eq!(compute_stats([1, 1, 1].into_iter()), (1, 1));
+ assert_eq!(compute_stats([1, 2, 5, 4, 3].into_iter()), (1, 5));
+ }
+}
diff --git a/crates/tantivy/src/columnar/column_values/stats/mod.rs b/crates/tantivy/src/columnar/column_values/stats/mod.rs
new file mode 100644
index 00000000..d786bc8b
--- /dev/null
+++ b/crates/tantivy/src/columnar/column_values/stats/mod.rs
@@ -0,0 +1,36 @@
+mod min_max;
+pub use min_max::MinMaxCollector;
+
+mod gcd;
+pub use gcd::GcdCollector;
+
+mod num_rows;
+pub use num_rows::NumRowsCollector;
+
+use crate::common::BinarySerializable;
+
+/// Trait for collecting statistics about a column of values.
+///
+/// This trait is used to gather metadata about a column during its construction,
+/// such as minimum/maximum values, GCD, and number of rows. The statistics are
+/// collected incrementally as values are added to the column.
+///
+/// The generic parameter T represents the type of values in the column, while
+/// the associated type Stats represents the final statistics produced.
+///
+/// Statistics are typically used to:
+/// - Optimize storage and compression of column values
+/// - Enable efficient range queries and filtering
+/// - Provide metadata about the column contents
+pub trait ColumnStatsCollector<T> {
+ type Stats: BinarySerializable;
+
+ /// Record a value from the column.
+ fn collect(&mut self, value: T);
+
+ /// Finalize the collection process and return the statistics.
+ fn finalize(&self) -> Self::Stats;
+
+ /// Number of bytes that will be written to the columnar file.
+ fn num_bytes(&self) -> u64;
+}
diff --git a/crates/tantivy/src/columnar/column_values/stats/num_rows.rs b/crates/tantivy/src/columnar/column_values/stats/num_rows.rs
new file mode 100644
index 00000000..c07a42d6
--- /dev/null
+++ b/crates/tantivy/src/columnar/column_values/stats/num_rows.rs
@@ -0,0 +1,55 @@
+use crate::columnar::RowId;
+
+use super::ColumnStatsCollector;
+
+/// A collector for the number of rows in a column.
+#[derive(Clone, Default)]
+pub struct NumRowsCollector {
+ num_rows: RowId,
+}
+
+impl NumRowsCollector {
+ pub fn as_u64(&self) -> &dyn ColumnStatsCollector<u64, Stats = RowId> {
+ self
+ }
+
+ pub fn as_u128(&self) -> &dyn ColumnStatsCollector<u128, Stats = RowId> {
+ self
+ }
+}
+
+impl<T> ColumnStatsCollector<T> for NumRowsCollector {
+ type Stats = RowId;
+
+ fn collect(&mut self, _value: T) {
+ self.num_rows += 1;
+ }
+
+ fn finalize(&self) -> Self::Stats {
+ self.num_rows
+ }
+
+ fn num_bytes(&self) -> u64 {
+ 8
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn compute_stats(vals: impl Iterator<Item = u64>) -> RowId {
+ let mut stats_collector = NumRowsCollector::default();
+ for val in vals {
+ stats_collector.collect(val);
+ }
+ stats_collector.as_u64().finalize()
+ }
+
+ #[test]
+ fn test_compute_num_rows() {
+ assert_eq!(compute_stats([].into_iter()), 0);
+ assert_eq!(compute_stats([1, 2, 3].into_iter()), 3);
+ assert_eq!(compute_stats([1, 1, 1].into_iter()), 3);
+ }
+}
diff --git a/crates/tantivy/src/columnar/column_values/u128_based/mod.rs b/crates/tantivy/src/columnar/column_values/u128_based/mod.rs
new file mode 100644
index 00000000..17366c56
--- /dev/null
+++ b/crates/tantivy/src/columnar/column_values/u128_based/mod.rs
@@ -0,0 +1,133 @@
+mod raw;
+
+pub use crate::columnar::column_values::u128_based::raw::RawCodec;
+use crate::{columnar::iterable::Iterable, common::BinarySerializable};
+use std::{
+ io::{self, Write},
+ sync::Arc,
+};
+
+use ownedbytes::OwnedBytes;
+
+use super::{
+ monotonic_map_column,
+ monotonic_mapping::{StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternal},
+ ColumnCodec, ColumnCodecEstimator, ColumnValues, MonotonicallyMappableToU128,
+};
+
+/// Available codecs to use to encode the u128 (via [`MonotonicallyMappableToU128`]) converted data.
+#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
+#[repr(u8)]
+pub enum CodecType {
+ /// Store the raw values without any compression.
+ Raw = 0u8,
+}
+
+impl CodecType {
+ fn to_code(self) -> u8 {
+ self as u8
+ }
+
+ fn try_from_code(code: u8) -> Option<CodecType> {
+ match code {
+ 0u8 => Some(CodecType::Raw),
+ _ => None,
+ }
+ }
+
+ fn load<T: MonotonicallyMappableToU128>(
+ &self,
+ bytes: OwnedBytes,
+ ) -> io::Result<Arc<dyn ColumnValues<T>>> {
+ match self {
+ CodecType::Raw => load_specific_codec::<RawCodec, T>(bytes),
+ }
+ }
+
+ /// Returns a boxed codec estimator associated to a given `CodecType`.
+ pub fn estimator(&self) -> Box<dyn ColumnCodecEstimator<u128>> {
+ match self {
+ CodecType::Raw => RawCodec::boxed_estimator(),
+ }
+ }
+}
+
+/// List of all available u128-base codecs.
+pub const ALL_U128_CODEC_TYPES: [CodecType; 1] = [CodecType::Raw];
+
+fn load_specific_codec<C: ColumnCodec<u128>, T: MonotonicallyMappableToU128>(
+ bytes: OwnedBytes,
+) -> io::Result<Arc<dyn ColumnValues<T>>> {
+ let reader = C::load(bytes)?;
+ let reader_typed = monotonic_map_column(
+ reader,
+ StrictlyMonotonicMappingInverter::from(StrictlyMonotonicMappingToInternal::<T>::new()),
+ );
+ Ok(Arc::new(reader_typed))
+}
+
+/// Serializes a given column of u128-mapped values.
+pub fn serialize_u128_based_column_values<T: MonotonicallyMappableToU128>(
+ vals: &dyn Iterable<T>,
+ codec_types: &[CodecType],
+ wrt: &mut dyn Write,
+) -> io::Result<()> {
+ let mut estimators: Vec<(CodecType, Box<dyn ColumnCodecEstimator<u128>>)> =
+ Vec::with_capacity(codec_types.len());
+ for &codec_type in codec_types {
+ estimators.push((codec_type, codec_type.estimator()));
+ }
+ for val in vals.boxed_iter() {
+ let val_u128 = val.to_u128();
+ for (_, estimator) in &mut estimators {
+ estimator.collect(val_u128);
+ }
+ }
+ for (_, estimator) in &mut estimators {
+ estimator.finalize();
+ }
+ let (_, best_codec, best_codec_estimator) = estimators
+ .into_iter()
+ .flat_map(|(codec_type, estimator)| {
+ let num_bytes = estimator.estimate()?;
+ Some((num_bytes, codec_type, estimator))
+ })
+ .min_by_key(|(num_bytes, _, _)| *num_bytes)
+ .ok_or_else(|| {
+ io::Error::new(io::ErrorKind::InvalidData, "No available applicable codec.")
+ })?;
+ best_codec.to_code().serialize(wrt)?;
+ best_codec_estimator.serialize(
+ &mut vals.boxed_iter().map(MonotonicallyMappableToU128::to_u128),
+ wrt,
+ )?;
+ Ok(())
+}
+
+/// Load u64-based column values.
+///
+/// This method first identifies the codec off the first byte.
+pub fn load_u128_based_column_values<T: MonotonicallyMappableToU128>(
+ mut bytes: OwnedBytes,
+) -> io::Result<Arc<dyn ColumnValues<T>>> {
+ let codec_type: CodecType = bytes
+ .first()
+ .copied()
+ .and_then(CodecType::try_from_code)
+ .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "Failed to read codec type"))?;
+ bytes.advance(1);
+ codec_type.load(bytes)
+}
+
+/// Helper function to serialize a column (autodetect from all codecs) and then open it
+pub fn serialize_and_load_u128_based_column_values<T: MonotonicallyMappableToU128>(
+ vals: &dyn Iterable<T>,
+ codec_types: &[CodecType],
+) -> Arc<dyn ColumnValues<T>> {
+ let mut buffer = Vec::new();
+ serialize_u128_based_column_values(vals, codec_types, &mut buffer).unwrap();
+ load_u128_based_column_values::<T>(OwnedBytes::new(buffer)).unwrap()
+}
+
+#[cfg(test)]
+mod tests;
diff --git a/crates/tantivy/src/columnar/column_values/u128_based/raw.rs b/crates/tantivy/src/columnar/column_values/u128_based/raw.rs
new file mode 100644
index 00000000..fd2c4e9e
--- /dev/null
+++ b/crates/tantivy/src/columnar/column_values/u128_based/raw.rs
@@ -0,0 +1,148 @@
+use std::io::Write;
+
+use ownedbytes::OwnedBytes;
+
+use crate::{
+ columnar::{
+ column_values::stats::{ColumnStatsCollector, MinMaxCollector, NumRowsCollector},
+ ColumnValues,
+ },
+ common::BinarySerializable,
+};
+
+use super::{ColumnCodec, ColumnCodecEstimator};
+
+pub struct RawCodec;
+
+impl ColumnCodec<u128> for RawCodec {
+ type ColumnValues = RawReader;
+ type Estimator = RawCodecEstimator;
+
+ fn load(bytes: OwnedBytes) -> std::io::Result<Self::ColumnValues> {
+ RawReader::load(bytes)
+ }
+}
+
+pub struct RawReader {
+ min_value: u128,
+ max_value: u128,
+ num_rows: u32,
+ data: OwnedBytes,
+}
+
+impl RawReader {
+ fn load(bytes: OwnedBytes) -> std::io::Result<Self> {
+ let mut bytes = bytes;
+
+ let num_rows = u32::deserialize(&mut bytes)?;
+ let min_value = u128::deserialize(&mut bytes)?;
+ let max_value = u128::deserialize(&mut bytes)?;
+
+ let data = bytes;
+
+ Ok(Self {
+ min_value,
+ max_value,
+ num_rows,
+ data,
+ })
+ }
+}
+
+impl ColumnValues<u128> for RawReader {
+ fn get_val(&self, idx: u32) -> u128 {
+ let idx = idx as usize;
+ let mut bytes = &self.data[idx * 16..(idx + 1) * 16];
+ u128::deserialize(&mut bytes).unwrap()
+ }
+
+ fn min_value(&self) -> u128 {
+ self.min_value
+ }
+
+ fn max_value(&self) -> u128 {
+ self.max_value
+ }
+
+ fn num_vals(&self) -> u32 {
+ self.num_rows
+ }
+}
+
+#[derive(Default)]
+pub struct RawCodecEstimator {
+ num_rows_collector: NumRowsCollector,
+ min_max_collector: MinMaxCollector<u128>,
+}
+
+impl ColumnCodecEstimator<u128> for RawCodecEstimator {
+ fn collect(&mut self, val: u128) {
+ self.num_rows_collector.collect(val);
+ self.min_max_collector.collect(val);
+ }
+
+ fn estimate(&self) -> Option<u64> {
+ let num_rows = self.num_rows_collector.as_u128().finalize();
+
+ Some(
+ self.num_rows_collector.as_u128().num_bytes()
+ + self.min_max_collector.num_bytes()
+ + num_rows as u64 * 16,
+ )
+ }
+
+ fn serialize(
+ &self,
+ vals: &mut dyn Iterator<Item = u128>,
+ wrt: &mut dyn Write,
+ ) -> std::io::Result<()> {
+ let num_rows = self.num_rows_collector.as_u128().finalize();
+ let (min_value, max_value) = self.min_max_collector.finalize();
+
+ num_rows.serialize(wrt)?;
+ min_value.serialize(wrt)?;
+ max_value.serialize(wrt)?;
+
+ for val in vals {
+ val.serialize(wrt)?;
+ }
+
+ wrt.flush()?;
+
+ Ok(())
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::columnar::column_values::u128_based::tests::create_and_validate;
+
+ #[test]
+ fn test_with_codec_data_sets_simple() {
+ create_and_validate::<RawCodec>(&[4, 3, 12], "name");
+ }
+
+ #[test]
+ fn test_with_codec_data_sets() {
+ let data_sets =
+ crate::columnar::column_values::u128_based::tests::get_codec_test_datasets();
+ for (mut data, name) in data_sets {
+ create_and_validate::<RawCodec>(&data, name);
+ data.reverse();
+ create_and_validate::<RawCodec>(&data, name);
+ }
+ }
+
+ #[test]
+ fn test_column_field_rand() {
+ for _ in 0..500 {
+ let mut data = (0..1 + rand::random::<u8>() as usize)
+ .map(|_| rand::random::<i64>() as u128 / 2)
+ .collect::<Vec<_>>();
+ create_and_validate::<RawCodec>(&data, "rand");
+ data.reverse();
+ create_and_validate::<RawCodec>(&data, "rand");
+ }
+ }
+}
diff --git a/crates/tantivy/src/columnar/column_values/u128_based/tests.rs b/crates/tantivy/src/columnar/column_values/u128_based/tests.rs
new file mode 100644
index 00000000..fc11a242
--- /dev/null
+++ b/crates/tantivy/src/columnar/column_values/u128_based/tests.rs
@@ -0,0 +1,208 @@
+use proptest::prelude::*;
+use proptest::{prop_oneof, proptest};
+
+#[test]
+fn test_serialize_and_load_simple() {
+ let mut buffer = Vec::new();
+ let vals = &[1u128, 2u128, 5u128];
+ serialize_u128_based_column_values(&&vals[..], &[CodecType::Raw], &mut buffer).unwrap();
+ let col = load_u128_based_column_values::<u128>(OwnedBytes::new(buffer)).unwrap();
+ assert_eq!(col.num_vals(), 3);
+ assert_eq!(col.get_val(0), 1);
+ assert_eq!(col.get_val(1), 2);
+ assert_eq!(col.get_val(2), 5);
+}
+
+#[test]
+fn test_empty_column_u128() {
+ let vals: [u128; 0] = [];
+ let mut num_acceptable_codecs = 0;
+ for codec in ALL_U128_CODEC_TYPES {
+ let mut buffer = Vec::new();
+ if serialize_u128_based_column_values(&&vals[..], &[codec], &mut buffer).is_err() {
+ continue;
+ }
+ num_acceptable_codecs += 1;
+ let col = load_u128_based_column_values::<u128>(OwnedBytes::new(buffer)).unwrap();
+ assert_eq!(col.num_vals(), 0);
+ assert_eq!(col.min_value(), u128::MIN);
+ assert_eq!(col.max_value(), u128::MIN);
+ }
+ assert!(num_acceptable_codecs > 0);
+}
+
+pub(crate) fn create_and_validate<TColumnCodec: ColumnCodec<u128>>(
+ vals: &[u128],
+ name: &str,
+) -> Option<(f32, f32)> {
+ let num_rows = vals.len() as u32;
+ let mut codec_estimator: TColumnCodec::Estimator = Default::default();
+
+ for val in vals.boxed_iter() {
+ codec_estimator.collect(val);
+ }
+ codec_estimator.finalize();
+ let estimation = codec_estimator.estimate()?;
+
+ let mut buffer = Vec::new();
+ codec_estimator
+ .serialize(vals.boxed_iter().as_mut(), &mut buffer)
+ .unwrap();
+
+ let actual_compression = buffer.len() as u64;
+
+ let reader = TColumnCodec::load(OwnedBytes::new(buffer)).unwrap();
+ assert_eq!(reader.num_vals(), vals.len() as u32);
+ let mut buffer = Vec::new();
+ for (doc, orig_val) in vals.iter().copied().enumerate() {
+ let val = reader.get_val(doc as u32);
+ assert_eq!(
+ val, orig_val,
+ "val `{val}` does not match orig_val {orig_val:?}, in data set {name}, data `{vals:?}`",
+ );
+
+ buffer.resize(1, 0);
+ reader.get_vals(&[doc as u32], &mut buffer);
+ let val = buffer[0];
+ assert_eq!(
+ val, orig_val,
+ "val `{val}` does not match orig_val {orig_val:?}, in data set {name}, data `{vals:?}`",
+ );
+ }
+
+ let all_docs: Vec<u32> = (0..vals.len() as u32).collect();
+ buffer.resize(all_docs.len(), 0);
+ reader.get_vals(&all_docs, &mut buffer);
+ assert_eq!(vals, buffer);
+
+ if !vals.is_empty() {
+ let test_rand_idx = rand::thread_rng().gen_range(0..=vals.len() - 1);
+ let expected_positions: Vec<u32> = vals
+ .iter()
+ .enumerate()
+ .filter(|(_, el)| **el == vals[test_rand_idx])
+ .map(|(pos, _)| pos as u32)
+ .collect();
+ let mut positions = Vec::new();
+ reader.get_row_ids_for_value_range(
+ vals[test_rand_idx]..=vals[test_rand_idx],
+ 0..vals.len() as u32,
+ &mut positions,
+ );
+ assert_eq!(expected_positions, positions);
+ }
+ if actual_compression > 1000 {
+ assert!(relative_difference(estimation, actual_compression) < 0.10f32);
+ }
+ Some((
+ compression_rate(estimation, num_rows),
+ compression_rate(actual_compression, num_rows),
+ ))
+}
+
+fn compression_rate(num_bytes: u64, num_values: u32) -> f32 {
+ num_bytes as f32 / (num_values as f32 * 8.0)
+}
+
+fn relative_difference(left: u64, right: u64) -> f32 {
+ let left = left as f32;
+ let right = right as f32;
+ 2.0f32 * (left - right).abs() / (left + right)
+}
+
+proptest! {
+ #![proptest_config(ProptestConfig::with_cases(100))]
+
+ #[test]
+ fn test_proptest_small_raw(data in proptest::collection::vec(num_strategy(), 1..10)) {
+ create_and_validate::<RawCodec>(&data, "proptest raw");
+ }
+}
+
+#[test]
+fn test_small_raw_example() {
+ create_and_validate::<RawCodec>(&[9223372036854775808, 9223370937344622593], "proptest raw");
+}
+
+proptest! {
+ #![proptest_config(ProptestConfig::with_cases(10))]
+
+ #[test]
+ fn test_proptest_large_raw(data in proptest::collection::vec(num_strategy(), 1..6000)) {
+ create_and_validate::<RawCodec>(&data, "proptest raw");
+ }
+
+}
+
+fn num_strategy() -> impl Strategy<Value = u128> {
+ prop_oneof![
+ 1 => prop::num::u128::ANY.prop_map(|num| u128::MAX - (num % 10) ),
+ 1 => prop::num::u128::ANY.prop_map(|num| num % 10 ),
+ 20 => prop::num::u128::ANY,
+ ]
+}
+
+pub fn get_codec_test_datasets() -> Vec<(Vec<u128>, &'static str)> {
+ let mut data_and_names = vec![];
+
+ let data = (10..=10_000_u128).collect::<Vec<_>>();
+ data_and_names.push((data, "simple monotonically increasing"));
+
+ data_and_names.push((
+ vec![5, 6, 7, 8, 9, 10, 99, 100],
+ "offset in linear interpol",
+ ));
+ data_and_names.push((vec![5, 50, 3, 13, 1, 1000, 35], "rand small"));
+ data_and_names.push((vec![10], "single value"));
+
+ data_and_names.push((
+ vec![1572656989877777, 1170935903116329, 720575940379279, 0],
+ "overflow error",
+ ));
+
+ data_and_names
+}
+
+fn test_codec<C: ColumnCodec<u128>>() {
+ let codec_name = std::any::type_name::<C>();
+ for (data, dataset_name) in get_codec_test_datasets() {
+ let estimate_actual_opt: Option<(f32, f32)> =
+ tests::create_and_validate::<C>(&data, dataset_name);
+ let result = if let Some((estimate, actual)) = estimate_actual_opt {
+ format!("Estimate `{estimate}` Actual `{actual}`")
+ } else {
+ "Disabled".to_string()
+ };
+ println!("Codec {codec_name}, DataSet {dataset_name}, {result}");
+ }
+}
+#[test]
+fn test_codec_raw() {
+ test_codec::<RawCodec>();
+}
+
+use super::*;
+
+#[test]
+fn test_column_field_codec_type_to_code() {
+ let mut count_codec = 0;
+ for code in 0..=255 {
+ if let Some(codec_type) = CodecType::try_from_code(code) {
+ assert_eq!(codec_type.to_code(), code);
+ count_codec += 1;
+ }
+ }
+ assert_eq!(count_codec, 1);
+}
+
+#[test]
+pub fn test_columnfield2() {
+ let test_columnfield =
+ crate::columnar::column_values::serialize_and_load_u128_based_column_values::<u128>(
+ &&[100u128, 200u128, 300u128][..],
+ &ALL_U128_CODEC_TYPES,
+ );
+ assert_eq!(test_columnfield.get_val(0), 100);
+ assert_eq!(test_columnfield.get_val(1), 200);
+ assert_eq!(test_columnfield.get_val(2), 300);
+}
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs
index e7d269a3..2be39cab 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs
+++ b/crates/tantivy/src/columnar/column_values/u64_based/bitpacked.rs
@@ -3,10 +3,13 @@ use std::num::NonZeroU64;
use std::ops::{Range, RangeInclusive};
use crate::bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
+use crate::columnar::column_values::stats::{
+ ColumnStatsCollector, GcdCollector, MinMaxCollector, NumRowsCollector,
+};
use crate::common::{BinarySerializable, OwnedBytes};
use fastdivide::DividerU64;
-use crate::columnar::column_values::u64_based::{ColumnCodec, ColumnCodecEstimator, ColumnStats};
+use crate::columnar::column_values::u64_based::{ColumnCodec, ColumnCodecEstimator};
use crate::columnar::{ColumnValues, RowId};
/// Depending on the field type, a different
@@ -15,7 +18,10 @@ use crate::columnar::{ColumnValues, RowId};
pub struct BitpackedReader {
data: OwnedBytes,
bit_unpacker: BitUnpacker,
- stats: ColumnStats,
+ num_rows: RowId,
+ min_value: u64,
+ max_value: u64,
+ gcd: NonZeroU64,
}
#[inline(always)]
@@ -39,41 +45,43 @@ const fn div_ceil(n: u64, q: NonZeroU64) -> u64 {
// [min_bipacked_value..max_bitpacked_value] such that
// f(bitpacked) ∈ [min_value, max_value] <=> bitpacked ∈ [min_bitpacked_value, max_bitpacked_value]
fn transform_range_before_linear_transformation(
- stats: &ColumnStats,
+ min_value: u64,
+ max_value: u64,
+ gcd: NonZeroU64,
range: RangeInclusive<u64>,
) -> Option<RangeInclusive<u64>> {
if range.is_empty() {
return None;
}
- if stats.min_value > *range.end() {
+ if min_value > *range.end() {
return None;
}
- if stats.max_value < *range.start() {
+ if max_value < *range.start() {
return None;
}
let shifted_range =
- range.start().saturating_sub(stats.min_value)..=range.end().saturating_sub(stats.min_value);
- let start_before_gcd_multiplication: u64 = div_ceil(*shifted_range.start(), stats.gcd);
- let end_before_gcd_multiplication: u64 = *shifted_range.end() / stats.gcd;
+ range.start().saturating_sub(min_value)..=range.end().saturating_sub(min_value);
+ let start_before_gcd_multiplication: u64 = div_ceil(*shifted_range.start(), gcd);
+ let end_before_gcd_multiplication: u64 = *shifted_range.end() / gcd;
Some(start_before_gcd_multiplication..=end_before_gcd_multiplication)
}
impl ColumnValues for BitpackedReader {
#[inline(always)]
fn get_val(&self, doc: u32) -> u64 {
- self.stats.min_value + self.stats.gcd.get() * self.bit_unpacker.get(doc, &self.data)
+ self.min_value + self.gcd.get() * self.bit_unpacker.get(doc, &self.data)
}
#[inline]
fn min_value(&self) -> u64 {
- self.stats.min_value
+ self.min_value
}
#[inline]
fn max_value(&self) -> u64 {
- self.stats.max_value
+ self.max_value
}
#[inline]
fn num_vals(&self) -> RowId {
- self.stats.num_rows
+ self.num_rows
}
fn get_row_ids_for_value_range(
@@ -82,9 +90,12 @@ impl ColumnValues for BitpackedReader {
mut doc_id_range: Range<u32>,
positions: &mut Vec<u32>,
) {
- let Some(transformed_range) =
- transform_range_before_linear_transformation(&self.stats, range)
- else {
+ let Some(transformed_range) = transform_range_before_linear_transformation(
+ self.min_value,
+ self.max_value,
+ self.gcd,
+ range,
+ ) else {
positions.clear();
return;
};
@@ -99,33 +110,59 @@ impl ColumnValues for BitpackedReader {
}
}
-fn num_bits(stats: &ColumnStats) -> u8 {
- compute_num_bits(stats.amplitude() / stats.gcd)
+fn num_bits(min_value: u64, max_value: u64, gcd: NonZeroU64) -> u8 {
+ compute_num_bits((max_value - min_value) / gcd)
}
#[derive(Default)]
-pub struct BitpackedCodecEstimator;
+pub struct BitpackedCodecEstimator {
+ num_rows_collector: NumRowsCollector,
+ min_max_collector: MinMaxCollector,
+ gcd_collector: GcdCollector,
+}
impl ColumnCodecEstimator for BitpackedCodecEstimator {
- fn collect(&mut self, _value: u64) {}
+ fn collect(&mut self, value: u64) {
+ self.num_rows_collector.collect(value);
+ self.min_max_collector.collect(value);
+ self.gcd_collector.collect(value);
+ }
+
+ fn estimate(&self) -> Option<u64> {
+ let (min_value, max_value) = self.min_max_collector.finalize();
+ let gcd = self.gcd_collector.finalize();
+ let num_bits_per_value = num_bits(min_value, max_value, gcd);
- fn estimate(&self, stats: &ColumnStats) -> Option<u64> {
- let num_bits_per_value = num_bits(stats);
- Some(stats.num_bytes() + (stats.num_rows as u64 * (num_bits_per_value as u64) + 7) / 8)
+ Some(
+ (self.num_rows_collector.as_u64().num_bytes()
+ + self.min_max_collector.num_bytes()
+ + self.gcd_collector.num_bytes())
+ + (self.num_rows_collector.as_u64().finalize() as u64
+ * (num_bits_per_value as u64)
+ + 7)
+ / 8,
+ )
}
fn serialize(
&self,
- stats: &ColumnStats,
vals: &mut dyn Iterator<Item = u64>,
wrt: &mut dyn Write,
) -> io::Result<()> {
- stats.serialize(wrt)?;
- let num_bits = num_bits(stats);
+ let (min_value, max_value) = self.min_max_collector.finalize();
+ let gcd = self.gcd_collector.finalize();
+ let num_rows = self.num_rows_collector.as_u64().finalize();
+
+ num_rows.serialize(wrt)?;
+ min_value.serialize(wrt)?;
+ max_value.serialize(wrt)?;
+ gcd.serialize(wrt)?;
+
+ let num_bits = num_bits(min_value, max_value, gcd);
let mut bit_packer = BitPacker::new();
- let divider = DividerU64::divide_by(stats.gcd.get());
+ let divider = DividerU64::divide_by(gcd.get());
for val in vals {
- bit_packer.write(divider.divide(val - stats.min_value), num_bits, wrt)?;
+ bit_packer.write(divider.divide(val - min_value), num_bits, wrt)?;
}
bit_packer.close(wrt)?;
Ok(())
@@ -140,13 +177,20 @@ impl ColumnCodec for BitpackedCodec {
/// Opens a columnar field given a file.
fn load(mut data: OwnedBytes) -> io::Result<Self::ColumnValues> {
- let stats = ColumnStats::deserialize(&mut data)?;
- let num_bits = num_bits(&stats);
+ let num_rows = RowId::deserialize(&mut data)?;
+ let min_value = u64::deserialize(&mut data)?;
+ let max_value = u64::deserialize(&mut data)?;
+ let gcd = NonZeroU64::deserialize(&mut data)?;
+
+ let num_bits = num_bits(min_value, max_value, gcd);
let bit_unpacker = BitUnpacker::new(num_bits);
Ok(BitpackedReader {
data,
bit_unpacker,
- stats,
+ num_rows,
+ min_value,
+ max_value,
+ gcd,
})
}
}
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rs b/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rs
index 7111c688..c86986ed 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rs
+++ b/crates/tantivy/src/columnar/column_values/u64_based/blockwise_linear.rs
@@ -1,14 +1,19 @@
use std::io::Write;
+use std::num::NonZeroU64;
use std::sync::Arc;
use std::{io, iter};
use crate::bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
+use crate::columnar::column_values::stats::{
+ ColumnStatsCollector, GcdCollector, MinMaxCollector, NumRowsCollector,
+};
+use crate::columnar::RowId;
use crate::common::{BinarySerializable, CountingWriter, DeserializeFrom, OwnedBytes};
use fastdivide::DividerU64;
use super::MonotonicallyMappableToU64;
use crate::columnar::column_values::u64_based::line::Line;
-use crate::columnar::column_values::u64_based::{ColumnCodec, ColumnCodecEstimator, ColumnStats};
+use crate::columnar::column_values::u64_based::{ColumnCodec, ColumnCodecEstimator};
use crate::columnar::column_values::{ColumnValues, VecColumn};
const BLOCK_SIZE: u32 = 512u32;
@@ -46,6 +51,9 @@ pub struct BlockwiseLinearEstimator {
block: Vec<u64>,
values_num_bytes: u64,
meta_num_bytes: u64,
+ num_rows_collector: NumRowsCollector,
+ min_max_collector: MinMaxCollector,
+ gcd_collector: GcdCollector,
}
impl Default for BlockwiseLinearEstimator {
@@ -54,6 +62,9 @@ impl Default for BlockwiseLinearEstimator {
block: Vec::with_capacity(BLOCK_SIZE as usize),
values_num_bytes: 0u64,
meta_num_bytes: 0u64,
+ num_rows_collector: NumRowsCollector::default(),
+ min_max_collector: MinMaxCollector::default(),
+ gcd_collector: GcdCollector::default(),
}
}
}
@@ -86,12 +97,25 @@ impl ColumnCodecEstimator for BlockwiseLinearEstimator {
self.flush_block_estimate();
self.block.clear();
}
+
+ self.num_rows_collector.collect(value);
+ self.min_max_collector.collect(value);
+ self.gcd_collector.collect(value);
}
- fn estimate(&self, stats: &ColumnStats) -> Option<u64> {
- let mut estimate = 4 + stats.num_bytes() + self.meta_num_bytes + self.values_num_bytes;
- if stats.gcd.get() > 1 {
+ fn estimate(&self) -> Option<u64> {
+ let num_rows = self.num_rows_collector.as_u64().finalize();
+ let gcd = self.gcd_collector.finalize();
+
+ let mut estimate = 4
+ + (self.num_rows_collector.as_u64().num_bytes()
+ + self.min_max_collector.num_bytes()
+ + self.gcd_collector.num_bytes())
+ + self.meta_num_bytes
+ + self.values_num_bytes;
+
+ if gcd.get() > 1 {
let estimate_gain_from_gcd =
- (stats.gcd.get() as f32).log2().floor() * stats.num_rows as f32 / 8.0f32;
+ (gcd.get() as f32).log2().floor() * num_rows as f32 / 8.0f32;
estimate = estimate.saturating_sub(estimate_gain_from_gcd as u64);
}
Some(estimate)
@@ -103,18 +127,25 @@ impl ColumnCodecEstimator for BlockwiseLinearEstimator {
fn serialize(
&self,
- stats: &ColumnStats,
mut vals: &mut dyn Iterator<Item = u64>,
wrt: &mut dyn Write,
) -> io::Result<()> {
- stats.serialize(wrt)?;
+ let num_rows = self.num_rows_collector.as_u64().finalize();
+ let (min_value, max_value) = self.min_max_collector.finalize();
+ let gcd = self.gcd_collector.finalize();
+
+ num_rows.serialize(wrt)?;
+ min_value.serialize(wrt)?;
+ max_value.serialize(wrt)?;
+ gcd.serialize(wrt)?;
+
let mut buffer = Vec::with_capacity(BLOCK_SIZE as usize);
- let num_blocks = compute_num_blocks(stats.num_rows) as usize;
+ let num_blocks = compute_num_blocks(num_rows) as usize;
let mut blocks = Vec::with_capacity(num_blocks);
let mut bit_packer = BitPacker::new();
- let gcd_divider = DividerU64::divide_by(stats.gcd.get());
+ let gcd_divider = DividerU64::divide_by(gcd.get());
for _ in 0..num_blocks {
buffer.clear();
@@ -125,7 +156,7 @@ impl ColumnCodecEstimator for BlockwiseLinearEstimator {
);
for buffer_val in buffer.iter_mut() {
- *buffer_val = gcd_divider.divide(*buffer_val - stats.min_value);
+ *buffer_val = gcd_divider.divide(*buffer_val - min_value);
}
let line = Line::train(&VecColumn::from(buffer.to_vec()));
@@ -173,11 +204,15 @@ impl ColumnCodec<u64> for BlockwiseLinearCodec {
type Estimator = BlockwiseLinearEstimator;
fn load(mut bytes: OwnedBytes) -> io::Result<Self::ColumnValues> {
- let stats = ColumnStats::deserialize(&mut bytes)?;
+ let num_rows = RowId::deserialize(&mut bytes)?;
+ let min_value = u64::deserialize(&mut bytes)?;
+ let max_value = u64::deserialize(&mut bytes)?;
+ let gcd = NonZeroU64::deserialize(&mut bytes)?;
+
let footer_len: u32 = (&bytes[bytes.len() - 4..]).deserialize()?;
let footer_offset = bytes.len() - 4 - footer_len as usize;
let (data, mut footer) = bytes.split(footer_offset);
- let num_blocks = compute_num_blocks(stats.num_rows);
+ let num_blocks = compute_num_blocks(num_rows);
let mut blocks: Vec<Block> = iter::repeat_with(|| Block::deserialize(&mut footer))
.take(num_blocks as usize)
.collect::<io::Result<_>>()?;
@@ -189,7 +224,10 @@ impl ColumnCodec<u64> for BlockwiseLinearCodec {
Ok(BlockwiseLinearReader {
blocks: blocks.into_boxed_slice().into(),
data,
- stats,
+ num_rows,
+ min_value,
+ max_value,
+ gcd,
})
}
}
@@ -198,7 +236,10 @@ impl ColumnCodec<u64> for BlockwiseLinearCodec {
pub struct BlockwiseLinearReader {
blocks: Arc<[Block]>,
data: OwnedBytes,
- stats: ColumnStats,
+ num_rows: RowId,
+ min_value: u64,
+ max_value: u64,
+ gcd: NonZeroU64,
}
impl ColumnValues for BlockwiseLinearReader {
@@ -212,9 +253,8 @@ impl ColumnValues for BlockwiseLinearReader {
let bitpacked_diff = block.bit_unpacker.get(idx_within_block, block_bytes);
// TODO optimize me! the line parameters could be tweaked to include the multiplication and
// remove the dependency.
- self.stats.min_value
+ self.min_value
+ self
- .stats
.gcd
.get()
.wrapping_mul(interpoled_val.wrapping_add(bitpacked_diff))
@@ -222,17 +262,17 @@ impl ColumnValues for BlockwiseLinearReader {
#[inline(always)]
fn min_value(&self) -> u64 {
- self.stats.min_value
+ self.min_value
}
#[inline(always)]
fn max_value(&self) -> u64 {
- self.stats.max_value
+ self.max_value
}
#[inline(always)]
fn num_vals(&self) -> u32 {
- self.stats.num_rows
+ self.num_rows
}
}
@@ -257,7 +297,7 @@ mod tests {
"name",
)
.unwrap();
- assert_eq!(actual_compression_rate, 0.175);
+ assert_eq!(actual_compression_rate, 0.475);
}
#[test]
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/linear.rs b/crates/tantivy/src/columnar/column_values/u64_based/linear.rs
index 038254e7..507a3248 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/linear.rs
+++ b/crates/tantivy/src/columnar/column_values/u64_based/linear.rs
@@ -1,11 +1,14 @@
use std::io;
use crate::bitpacker::{compute_num_bits, BitPacker, BitUnpacker};
+use crate::columnar::column_values::stats::{
+ ColumnStatsCollector, MinMaxCollector, NumRowsCollector,
+};
use crate::common::{BinarySerializable, OwnedBytes};
use super::line::Line;
use super::ColumnValues;
-use crate::columnar::column_values::u64_based::{ColumnCodec, ColumnCodecEstimator, ColumnStats};
+use crate::columnar::column_values::u64_based::{ColumnCodec, ColumnCodecEstimator};
use crate::columnar::column_values::VecColumn;
use crate::columnar::RowId;
@@ -18,7 +21,9 @@ const LINE_ESTIMATION_BLOCK_LEN: usize = 512;
pub struct LinearReader {
data: OwnedBytes,
linear_params: LinearParams,
- stats: ColumnStats,
+ min_value: u64,
+ max_value: u64,
+ num_rows: u32,
}
impl ColumnValues for LinearReader {
@@ -31,17 +36,17 @@ impl ColumnValues for LinearReader {
#[inline(always)]
fn min_value(&self) -> u64 {
- self.stats.min_value
+ self.min_value
}
#[inline(always)]
fn max_value(&self) -> u64 {
- self.stats.max_value
+ self.max_value
}
#[inline]
fn num_vals(&self) -> u32 {
- self.stats.num_rows
+ self.num_rows
}
}
@@ -80,6 +85,8 @@ pub struct LinearCodecEstimator {
max_deviation: u64,
first_val: u64,
last_val: u64,
+ min_max_collector: MinMaxCollector<u64>,
+ num_rows_collector: NumRowsCollector,
}
impl Default for LinearCodecEstimator {
@@ -92,6 +99,8 @@ impl Default for LinearCodecEstimator {
max_deviation: u64::MIN,
first_val: 0u64,
last_val: 0u64,
+ min_max_collector: MinMaxCollector::default(),
+ num_rows_collector: NumRowsCollector::default(),
}
}
}
@@ -106,7 +115,7 @@ impl ColumnCodecEstimator for LinearCodecEstimator {
}
}
- fn estimate(&self, stats: &ColumnStats) -> Option<u64> {
+ fn estimate(&self) -> Option<u64> {
let line = self.line?;
let amplitude = self.max_deviation - self.min_deviation;
let num_bits = compute_num_bits(amplitude);
@@ -114,20 +123,27 @@ impl ColumnCodecEstimator for LinearCodecEstimator {
line,
bit_unpacker: BitUnpacker::new(num_bits),
};
+
Some(
- stats.num_bytes()
+ self.min_max_collector.num_bytes()
+ + self.num_rows_collector.as_u64().num_bytes()
+ linear_params.num_bytes()
- + (num_bits as u64 * stats.num_rows as u64 + 7) / 8,
+ + (num_bits as u64 * self.num_rows_collector.as_u64().finalize() as u64 + 7) / 8,
)
}
fn serialize(
&self,
- stats: &ColumnStats,
vals: &mut dyn Iterator<Item = u64>,
wrt: &mut dyn io::Write,
) -> io::Result<()> {
- stats.serialize(wrt)?;
+ let num_rows = self.num_rows_collector.as_u64().finalize();
+ num_rows.serialize(wrt)?;
+
+ let (min_value, max_value) = self.min_max_collector.finalize();
+ min_value.serialize(wrt)?;
+ max_value.serialize(wrt)?;
+
let line = self.line.unwrap();
let amplitude = self.max_deviation - self.min_deviation;
let num_bits = compute_num_bits(amplitude);
@@ -152,6 +168,9 @@ impl ColumnCodecEstimator for LinearCodecEstimator {
} else {
self.collect_before_line_estimation(value);
}
+
+ self.min_max_collector.collect(value);
+ self.num_rows_collector.collect(value);
}
}
@@ -191,10 +210,14 @@ impl ColumnCodec for LinearCodec {
type Estimator = LinearCodecEstimator;
fn load(mut data: OwnedBytes) -> io::Result<Self::ColumnValues> {
- let stats = ColumnStats::deserialize(&mut data)?;
+ let num_rows = u32::deserialize(&mut data)?;
+ let min_value = u64::deserialize(&mut data)?;
+ let max_value = u64::deserialize(&mut data)?;
let linear_params = LinearParams::deserialize(&mut data)?;
Ok(LinearReader {
- stats,
+ min_value,
+ max_value,
+ num_rows,
linear_params,
data,
})
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/mod.rs b/crates/tantivy/src/columnar/column_values/u64_based/mod.rs
index cc2d44b6..77d123c9 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/mod.rs
+++ b/crates/tantivy/src/columnar/column_values/u64_based/mod.rs
@@ -3,7 +3,6 @@ mod blockwise_linear;
mod line;
mod linear;
mod raw;
-mod stats_collector;
use std::io;
use std::io::Write;
@@ -11,7 +10,8 @@ use std::sync::Arc;
use crate::common::{BinarySerializable, OwnedBytes};
-use super::{ColumnValues, MonotonicallyMappableToU64};
+use super::{ColumnCodec, ColumnCodecEstimator, ColumnValues, MonotonicallyMappableToU64};
+use crate::columnar::column_values::monotonic_map_column;
use crate::columnar::column_values::monotonic_mapping::{
StrictlyMonotonicMappingInverter, StrictlyMonotonicMappingToInternal,
};
@@ -19,62 +19,8 @@ pub use crate::columnar::column_values::u64_based::bitpacked::BitpackedCodec;
pub use crate::columnar::column_values::u64_based::blockwise_linear::BlockwiseLinearCodec;
pub use crate::columnar::column_values::u64_based::linear::LinearCodec;
pub use crate::columnar::column_values::u64_based::raw::RawCodec;
-pub use crate::columnar::column_values::u64_based::stats_collector::StatsCollector;
-use crate::columnar::column_values::{monotonic_map_column, ColumnStats};
use crate::columnar::iterable::Iterable;
-/// A `ColumnCodecEstimator` is in charge of gathering all
-/// data required to serialize a column.
-///
-/// This happens during a first pass on data of the column elements.
-/// During that pass, all column estimators receive a call to their
-/// `.collect(el)`.
-///
-/// After this first pass, finalize is called.
-/// `.estimate(..)` then should return an accurate estimation of the
-/// size of the serialized column (were we to pick this codec.).
-/// `.serialize(..)` then serializes the column using this codec.
-pub trait ColumnCodecEstimator<T = u64>: 'static {
- /// Records a new value for estimation.
- /// This method will be called for each element of the column during
- /// `estimation`.
- fn collect(&mut self, value: u64);
- /// Finalizes the first pass phase.
- fn finalize(&mut self) {}
- /// Returns an accurate estimation of the number of bytes that will
- /// be used to represent this column.
- fn estimate(&self, stats: &ColumnStats) -> Option<u64>;
- /// Serializes the column using the given codec.
- /// This constitutes a second pass over the columns values.
- fn serialize(
- &self,
- stats: &ColumnStats,
- vals: &mut dyn Iterator<Item = T>,
- wrt: &mut dyn io::Write,
- ) -> io::Result<()>;
-}
-
-/// A column codec describes a colunm serialization format.
-pub trait ColumnCodec<T: PartialOrd = u64> {
- /// Specialized `ColumnValues` type.
- type ColumnValues: ColumnValues<T> + 'static;
- /// `Estimator` for the given codec.
- type Estimator: ColumnCodecEstimator + Default;
-
- /// Loads a column that has been serialized using this codec.
- fn load(bytes: OwnedBytes) -> io::Result<Self::ColumnValues>;
-
- /// Returns an estimator.
- fn estimator() -> Self::Estimator {
- Self::Estimator::default()
- }
-
- /// Returns a boxed estimator.
- fn boxed_estimator() -> Box<dyn ColumnCodecEstimator> {
- Box::new(Self::estimator())
- }
-}
-
/// Available codecs to use to encode the u64 (via [`MonotonicallyMappableToU64`]) converted data.
#[derive(PartialEq, Eq, PartialOrd, Ord, Debug, Clone, Copy)]
#[repr(u8)]
@@ -157,7 +103,6 @@ pub fn serialize_u64_based_column_values<T: MonotonicallyMappableToU64>(
codec_types: &[CodecType],
wrt: &mut dyn Write,
) -> io::Result<()> {
- let mut stats_collector = StatsCollector::default();
let mut estimators: Vec<(CodecType, Box<dyn ColumnCodecEstimator>)> =
Vec::with_capacity(codec_types.len());
for &codec_type in codec_types {
@@ -165,7 +110,6 @@ pub fn serialize_u64_based_column_values<T: MonotonicallyMappableToU64>(
}
for val in vals.boxed_iter() {
let val_u64 = val.to_u64();
- stats_collector.collect(val_u64);
for (_, estimator) in &mut estimators {
estimator.collect(val_u64);
}
@@ -173,11 +117,10 @@ pub fn serialize_u64_based_column_values<T: MonotonicallyMappableToU64>(
for (_, estimator) in &mut estimators {
estimator.finalize();
}
- let stats = stats_collector.stats();
let (_, best_codec, best_codec_estimator) = estimators
.into_iter()
.flat_map(|(codec_type, estimator)| {
- let num_bytes = estimator.estimate(&stats)?;
+ let num_bytes = estimator.estimate()?;
Some((num_bytes, codec_type, estimator))
})
.min_by_key(|(num_bytes, _, _)| *num_bytes)
@@ -186,7 +129,6 @@ pub fn serialize_u64_based_column_values<T: MonotonicallyMappableToU64>(
})?;
best_codec.to_code().serialize(wrt)?;
best_codec_estimator.serialize(
- &stats,
&mut vals.boxed_iter().map(MonotonicallyMappableToU64::to_u64),
wrt,
)?;
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/raw.rs b/crates/tantivy/src/columnar/column_values/u64_based/raw.rs
index 9bea4b6f..51ce4980 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/raw.rs
+++ b/crates/tantivy/src/columnar/column_values/u64_based/raw.rs
@@ -3,7 +3,10 @@ use std::io::Write;
use ownedbytes::OwnedBytes;
use crate::{
- columnar::{column_values::ColumnStats, ColumnValues},
+ columnar::{
+ column_values::stats::{ColumnStatsCollector, MinMaxCollector, NumRowsCollector},
+ ColumnValues,
+ },
common::BinarySerializable,
};
@@ -67,24 +70,37 @@ impl ColumnValues for RawReader {
}
#[derive(Default)]
-pub struct RawCodecEstimator;
+pub struct RawCodecEstimator {
+ num_rows_collector: NumRowsCollector,
+ min_max_collector: MinMaxCollector,
+}
impl ColumnCodecEstimator for RawCodecEstimator {
- fn collect(&mut self, _: u64) {}
+ fn collect(&mut self, value: u64) {
+ self.num_rows_collector.collect(value);
+ self.min_max_collector.collect(value);
+ }
- fn estimate(&self, stats: &ColumnStats) -> Option<u64> {
- Some(stats.num_rows as u64 * 8)
+ fn estimate(&self) -> Option<u64> {
+ let num_rows = self.num_rows_collector.as_u64().finalize();
+ Some(
+ self.min_max_collector.num_bytes()
+ + self.num_rows_collector.as_u64().num_bytes()
+ + num_rows as u64 * 8,
+ )
}
fn serialize(
&self,
- stats: &ColumnStats,
vals: &mut dyn Iterator<Item = u64>,
wrt: &mut dyn Write,
) -> std::io::Result<()> {
- stats.num_rows.serialize(wrt)?;
- stats.min_value.serialize(wrt)?;
- stats.max_value.serialize(wrt)?;
+ let num_rows = self.num_rows_collector.as_u64().finalize();
+ let (min_value, max_value) = self.min_max_collector.finalize();
+
+ num_rows.serialize(wrt)?;
+ min_value.serialize(wrt)?;
+ max_value.serialize(wrt)?;
for val in vals {
val.serialize(wrt)?;
diff --git a/crates/tantivy/src/columnar/column_values/u64_based/tests.rs b/crates/tantivy/src/columnar/column_values/u64_based/tests.rs
index 6dd05fe3..a52dec8b 100644
--- a/crates/tantivy/src/columnar/column_values/u64_based/tests.rs
+++ b/crates/tantivy/src/columnar/column_values/u64_based/tests.rs
@@ -12,7 +12,7 @@ fn test_serialize_and_load_simple() {
&mut buffer,
)
.unwrap();
- assert_eq!(buffer.len(), 7);
+ assert!(buffer.len() > 0);
let col = load_u64_based_column_values::<u64>(OwnedBytes::new(buffer)).unwrap();
assert_eq!(col.num_vals(), 3);
assert_eq!(col.get_val(0), 1);
@@ -79,20 +79,18 @@ pub(crate) fn create_and_validate<TColumnCodec: ColumnCodec>(
vals: &[u64],
name: &str,
) -> Option<(f32, f32)> {
- let mut stats_collector = StatsCollector::default();
+ let num_rows = vals.len() as u32;
let mut codec_estimator: TColumnCodec::Estimator = Default::default();
for val in vals.boxed_iter() {
- stats_collector.collect(val);
codec_estimator.collect(val);
}
codec_estimator.finalize();
- let stats = stats_collector.stats();
- let estimation = codec_estimator.estimate(&stats)?;
+ let estimation = codec_estimator.estimate()?;
let mut buffer = Vec::new();
codec_estimator
- .serialize(&stats, vals.boxed_iter().as_mut(), &mut buffer)
+ .serialize(vals.boxed_iter().as_mut(), &mut buffer)
.unwrap();
let actual_compression = buffer.len() as u64;
@@ -141,8 +139,8 @@ pub(crate) fn create_and_validate<TColumnCodec: ColumnCodec>(
assert!(relative_difference(estimation, actual_compression) < 0.10f32);
}
Some((
- compression_rate(estimation, stats.num_rows),
- compression_rate(actual_compression, stats.num_rows),
+ compression_rate(estimation, num_rows),
+ compression_rate(actual_compression, num_rows),
))
}
@@ -174,6 +172,11 @@ proptest! {
fn test_proptest_small_blockwise_linear(data in proptest::collection::vec(num_strategy(), 1..10)) {
create_and_validate::<BlockwiseLinearCodec>(&data, "proptest multilinearinterpol");
}
+
+ #[test]
+ fn test_proptest_small_raw(data in proptest::collection::vec(num_strategy(), 1..10)) {
+ create_and_validate::<RawCodec>(&data, "proptest raw");
+ }
}
#[test]
@@ -201,6 +204,11 @@ proptest! {
fn test_proptest_large_blockwise_linear(data in proptest::collection::vec(num_strategy(), 1..6000)) {
create_and_validate::<BlockwiseLinearCodec>(&data, "proptest multilinearinterpol");
}
+
+ #[test]
+ fn test_proptest_large_raw(data in proptest::collection::vec(num_strategy(), 1..6000)) {
+ create_and_validate::<RawCodec>(&data, "proptest raw");
+ }
}
fn num_strategy() -> impl Strategy<Value = u64> {
@@ -258,22 +266,25 @@ fn test_codec_multi_interpolation() {
test_codec::<BlockwiseLinearCodec>();
}
+#[test]
+fn test_codec_raw() {
+ test_codec::<RawCodec>();
+}
+
use super::*;
fn estimate<C: ColumnCodec>(vals: &[u64]) -> Option<f32> {
- let mut stats_collector = StatsCollector::default();
+ let num_rows = vals.len() as u32;
let mut estimator = C::Estimator::default();
for &val in vals {
- stats_collector.collect(val);
estimator.collect(val);
}
estimator.finalize();
- let stats = stats_collector.stats();
- let num_bytes = estimator.estimate(&stats)?;
- if stats.num_rows == 0 {
+ let num_bytes = estimator.estimate()?;
+ if num_rows == 0 {
return None;
}
- Some(num_bytes as f32 / (8.0 * stats.num_rows as f32))
+ Some(num_bytes as f32 / (8.0 * num_rows as f32))
}
#[test]
diff --git a/crates/tantivy/src/columnar/columnar/column_type.rs b/crates/tantivy/src/columnar/columnar/column_type.rs
index 2016c8f8..37ecb292 100644
--- a/crates/tantivy/src/columnar/columnar/column_type.rs
+++ b/crates/tantivy/src/columnar/columnar/column_type.rs
@@ -17,6 +17,7 @@ pub enum ColumnType {
Bytes = 3u8,
Bool = 4u8,
DateTime = 5u8,
+ U128 = 6u8,
}
impl fmt::Display for ColumnType {
@@ -28,19 +29,21 @@ impl fmt::Display for ColumnType {
ColumnType::Bytes => "bytes",
ColumnType::Bool => "bool",
ColumnType::DateTime => "datetime",
+ ColumnType::U128 => "u128",
};
write!(f, "{short_str}")
}
}
// The order needs to match _exactly_ the order in the enum
-const COLUMN_TYPES: [ColumnType; 6] = [
+const COLUMN_TYPES: [ColumnType; 7] = [
ColumnType::I64,
ColumnType::U64,
ColumnType::F64,
ColumnType::Bytes,
ColumnType::Bool,
ColumnType::DateTime,
+ ColumnType::U128,
];
impl ColumnType {
@@ -62,6 +65,7 @@ impl From<NumericalType> for ColumnType {
NumericalType::I64 => ColumnType::I64,
NumericalType::U64 => ColumnType::U64,
NumericalType::F64 => ColumnType::F64,
+ NumericalType::U128 => ColumnType::U128,
}
}
}
@@ -72,6 +76,7 @@ impl ColumnType {
ColumnType::I64 => Some(NumericalType::I64),
ColumnType::U64 => Some(NumericalType::U64),
ColumnType::F64 => Some(NumericalType::F64),
+ ColumnType::U128 => Some(NumericalType::U128),
ColumnType::Bytes | ColumnType::Bool | ColumnType::DateTime => None,
}
}
diff --git a/crates/tantivy/src/columnar/columnar/merge/mod.rs b/crates/tantivy/src/columnar/columnar/merge/mod.rs
index f1c597cb..a7003ec4 100644
--- a/crates/tantivy/src/columnar/columnar/merge/mod.rs
+++ b/crates/tantivy/src/columnar/columnar/merge/mod.rs
@@ -9,7 +9,9 @@ use std::sync::Arc;
pub use merge_mapping::{MergeRowOrder, ShuffleMergeOrder, StackMergeOrder};
use super::writer::ColumnarSerializer;
-use crate::columnar::column::serialize_column_mappable_to_u64;
+use crate::columnar::column::{
+ serialize_column_mappable_to_u128, serialize_column_mappable_to_u64,
+};
use crate::columnar::column_values::MergedColumnValues;
use crate::columnar::columnar::merge::merge_dict_column::merge_bytes_or_str_column;
use crate::columnar::columnar::writer::CompatibleNumericalTypes;
@@ -43,6 +45,7 @@ impl From<ColumnType> for ColumnTypeCategory {
ColumnType::I64 => ColumnTypeCategory::Numerical,
ColumnType::U64 => ColumnTypeCategory::Numerical,
ColumnType::F64 => ColumnTypeCategory::Numerical,
+ ColumnType::U128 => ColumnTypeCategory::Numerical,
ColumnType::Bytes => ColumnTypeCategory::Bytes,
ColumnType::Bool => ColumnTypeCategory::Bool,
ColumnType::DateTime => ColumnTypeCategory::DateTime,
@@ -113,6 +116,14 @@ fn dynamic_column_to_u64_monotonic(dynamic_column: DynamicColumn) -> Option<Colu
DynamicColumn::F64(column) => Some(column.to_u64_monotonic()),
DynamicColumn::DateTime(column) => Some(column.to_u64_monotonic()),
DynamicColumn::Bytes(_) => None,
+ DynamicColumn::U128(_) => None,
+ }
+}
+
+fn dynamic_column_to_u128_monotonic(dynamic_column: DynamicColumn) -> Option<Column<u128>> {
+ match dynamic_column {
+ DynamicColumn::U128(column) => Some(column.to_u128_monotonic()),
+ _ => None,
}
}
@@ -148,6 +159,28 @@ fn merge_column(
};
serialize_column_mappable_to_u64(merged_column_index, &merge_column_values, wrt)?;
}
+ ColumnType::U128 => {
+ let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());
+ let mut column_values: Vec<Option<Arc<dyn ColumnValues<u128>>>> =
+ Vec::with_capacity(columns.len());
+ for dynamic_column_opt in columns {
+ if let Some(Column { index: idx, values }) =
+ dynamic_column_opt.and_then(dynamic_column_to_u128_monotonic)
+ {
+ column_indexes.push(idx);
+ column_values.push(Some(values));
+ }
+ }
+
+ let merged_column_index =
+ crate::columnar::column_index::merge_column_index(merge_row_order);
+ let merge_column_values = MergedColumnValues {
+ column_indexes: &column_indexes[..],
+ column_values: &column_values[..],
+ merge_row_order,
+ };
+ serialize_column_mappable_to_u128(merged_column_index, &merge_column_values, wrt)?;
+ }
ColumnType::Bytes => {
let mut column_indexes: Vec<ColumnIndex> = Vec::with_capacity(columns.len());
let mut bytes_columns: Vec<Option<BytesColumn>> = Vec::with_capacity(columns.len());
@@ -363,6 +396,7 @@ fn min_max_if_numerical(column: &DynamicColumn) -> Option<(NumericalValue, Numer
DynamicColumn::I64(column) => Some((column.min_value().into(), column.max_value().into())),
DynamicColumn::U64(column) => Some((column.min_value().into(), column.max_value().into())),
DynamicColumn::F64(column) => Some((column.min_value().into(), column.max_value().into())),
+ DynamicColumn::U128(column) => Some((column.min_value().into(), column.max_value().into())),
DynamicColumn::Bool(_) | DynamicColumn::DateTime(_) | DynamicColumn::Bytes(_) => None,
}
}
diff --git a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs b/crates/tantivy/src/columnar/columnar/writer/column_operation.rs
index 4afb7b03..59f52c90 100644
--- a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/column_operation.rs
@@ -156,6 +156,19 @@ impl SymbolValue for Ipv6Addr {
}
}
+impl SymbolValue for u128 {
+ fn serialize(self, buffer: &mut [u8]) -> u8 {
+ buffer[0..16].copy_from_slice(&self.to_le_bytes());
+ 16
+ }
+
+ fn deserialize(bytes: &[u8]) -> Self {
+ let mut octet: [u8; 16] = [0u8; 16];
+ octet[..bytes.len()].copy_from_slice(bytes);
+ u128::from_le_bytes(octet)
+ }
+}
+
#[derive(Default)]
struct MiniBuffer {
pub bytes: [u8; 17],
@@ -172,29 +185,40 @@ impl SymbolValue for NumericalValue {
fn deserialize(mut bytes: &[u8]) -> Self {
let type_code = pop_first_byte(&mut bytes).unwrap();
let symbol_type = NumericalType::try_from_code(type_code).unwrap();
- let mut octet: [u8; 8] = [0u8; 8];
- octet[..bytes.len()].copy_from_slice(bytes);
match symbol_type {
NumericalType::U64 => {
+ let mut octet: [u8; 8] = [0u8; 8];
+ octet[..bytes.len()].copy_from_slice(bytes);
let val: u64 = u64::from_le_bytes(octet);
NumericalValue::U64(val)
}
NumericalType::I64 => {
+ let mut octet: [u8; 8] = [0u8; 8];
+ octet[..bytes.len()].copy_from_slice(bytes);
let encoded: u64 = u64::from_le_bytes(octet);
let val: i64 = decode_zig_zag(encoded);
NumericalValue::I64(val)
}
NumericalType::F64 => {
+ let mut octet: [u8; 8] = [0u8; 8];
+ octet[..bytes.len()].copy_from_slice(bytes);
debug_assert_eq!(bytes.len(), 8);
let val: f64 = f64::from_le_bytes(octet);
NumericalValue::F64(val)
}
+ NumericalType::U128 => {
+ let mut octet: [u8; 16] = [0u8; 16];
+ octet[..bytes.len()].copy_from_slice(bytes);
+ let val: u128 = u128::from_le_bytes(octet);
+ NumericalValue::U128(val)
+ }
}
}
/// F64: Serialize with a fixed size of 9 bytes
/// U64: Serialize without leading zeroes
/// I64: ZigZag encoded and serialize without leading zeroes
+ /// U128: Serialize without leading zeroes
fn serialize(self, output: &mut [u8]) -> u8 {
match self {
NumericalValue::F64(val) => {
@@ -215,6 +239,12 @@ impl SymbolValue for NumericalValue {
output[1..9].copy_from_slice(&zig_zag_encoded.to_le_bytes());
len + 1u8
}
+ NumericalValue::U128(val) => {
+ let len = compute_num_bytes_for_u128(val) as u8;
+ output[0] = NumericalType::U128 as u8;
+ output[1..17].copy_from_slice(&val.to_le_bytes());
+ len + 1u8
+ }
}
}
}
@@ -248,6 +278,11 @@ fn compute_num_bytes_for_u64(val: u64) -> usize {
(msb + 7) / 8
}
+fn compute_num_bytes_for_u128(val: u128) -> usize {
+ let msb = (128u32 - val.leading_zeros()) as usize;
+ (msb + 7) / 8
+}
+
fn encode_zig_zag(n: i64) -> u64 {
((n << 1) ^ (n >> 63)) as u64
}
diff --git a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
index f9e35fb5..97f1da0f 100644
--- a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
@@ -148,6 +148,7 @@ impl CompatibleNumericalTypes {
NumericalType::I64 => *all_values_within_i64_range,
NumericalType::U64 => *all_values_within_u64_range,
NumericalType::F64 => true,
+ NumericalType::U128 => true,
},
CompatibleNumericalTypes::StaticType(static_numerical_type) => {
*static_numerical_type == numerical_type
@@ -173,6 +174,9 @@ impl CompatibleNumericalTypes {
*all_values_within_i64_range = false;
*all_values_within_u64_range = false;
}
+ NumericalValue::U128(_) => {
+ *all_values_within_u64_range = false;
+ }
},
CompatibleNumericalTypes::StaticType(typ) => {
assert_eq!(
diff --git a/crates/tantivy/src/columnar/columnar/writer/mod.rs b/crates/tantivy/src/columnar/columnar/writer/mod.rs
index 0887afe4..dc9a0c0f 100644
--- a/crates/tantivy/src/columnar/columnar/writer/mod.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/mod.rs
@@ -20,7 +20,7 @@ use crate::columnar::columnar::writer::column_writers::{
use crate::columnar::columnar::writer::value_index::{IndexBuilder, PreallocatedIndexBuilders};
use crate::columnar::dictionary::{DictionaryBuilder, TermIdMapping, UnorderedId};
use crate::columnar::value::{Coerce, NumericalType, NumericalValue};
-use crate::columnar::{Cardinality, RowId};
+use crate::columnar::{Cardinality, MonotonicallyMappableToU128, RowId};
/// This is a set of buffers that are used to temporarily write the values into before passing them
/// to the columnar field codecs.
@@ -28,6 +28,7 @@ use crate::columnar::{Cardinality, RowId};
struct SpareBuffers {
value_index_builders: PreallocatedIndexBuilders,
u64_values: Vec<u64>,
+ u128_values: Vec<u128>,
}
/// Makes it possible to create a new columnar.
@@ -176,7 +177,7 @@ impl ColumnarWriter {
|column_opt: Option<ColumnWriter>| column_opt.unwrap_or_default(),
);
}
- ColumnType::I64 | ColumnType::F64 | ColumnType::U64 => {
+ ColumnType::I64 | ColumnType::F64 | ColumnType::U64 | ColumnType::U128 => {
let numerical_type = column_type.numerical_type().unwrap();
self.numerical_field_hash_map.mutate_or_create(
column_name.as_bytes(),
@@ -337,7 +338,7 @@ impl ColumnarWriter {
)?;
column_serializer.finalize()?;
}
- ColumnType::F64 | ColumnType::I64 | ColumnType::U64 => {
+ ColumnType::F64 | ColumnType::I64 | ColumnType::U64 | ColumnType::U128 => {
let numerical_column_writer: NumericalColumnWriter =
self.numerical_field_hash_map.read(addr);
let cardinality = numerical_column_writer.cardinality();
@@ -396,7 +397,7 @@ fn serialize_bytes_or_str_column(
let SpareBuffers {
value_index_builders,
u64_values,
- ..
+ u128_values: _,
} = buffers;
let mut counting_writer = CountingWriter::wrap(wrt);
let term_id_mapping: TermIdMapping =
@@ -434,7 +435,7 @@ fn serialize_numerical_column(
let SpareBuffers {
value_index_builders,
u64_values,
- ..
+ u128_values,
} = buffers;
match numerical_type {
NumericalType::I64 => {
@@ -464,6 +465,15 @@ fn serialize_numerical_column(
wrt,
)?;
}
+ NumericalType::U128 => {
+ send_to_serialize_column_mappable_to_u128(
+ coerce_numerical_symbol_to_u128::<u128>(op_iterator),
+ cardinality,
+ value_index_builders,
+ u128_values,
+ wrt,
+ )?;
+ }
};
Ok(())
}
@@ -477,7 +487,7 @@ fn serialize_bool_column(
let SpareBuffers {
value_index_builders,
u64_values,
- ..
+ u128_values: _,
} = buffers;
send_to_serialize_column_mappable_to_u64(
column_operations_it.map(|bool_column_operation| match bool_column_operation {
@@ -518,6 +528,32 @@ fn send_to_serialize_column_mappable_to_u64(
Ok(())
}
+fn send_to_serialize_column_mappable_to_u128(
+ op_iterator: impl Iterator<Item = ColumnOperation<u128>>,
+ cardinality: Cardinality,
+ value_index_builders: &mut PreallocatedIndexBuilders,
+ values: &mut Vec<u128>,
+ mut wrt: impl io::Write,
+) -> io::Result<()> {
+ values.clear();
+ let serializable_column_index = match cardinality {
+ Cardinality::Full => {
+ consume_operation_iterator(
+ op_iterator,
+ value_index_builders.borrow_required_index_builder(),
+ values,
+ );
+ SerializableColumnIndex::Full
+ }
+ };
+ crate::columnar::column::serialize_column_mappable_to_u128(
+ serializable_column_index,
+ &&values[..],
+ &mut wrt,
+ )?;
+ Ok(())
+}
+
fn coerce_numerical_symbol<T>(
operation_iterator: impl Iterator<Item = ColumnOperation<NumericalValue>>,
) -> impl Iterator<Item = ColumnOperation<u64>>
@@ -532,6 +568,20 @@ where
})
}
+fn coerce_numerical_symbol_to_u128<T>(
+ operation_iterator: impl Iterator<Item = ColumnOperation<NumericalValue>>,
+) -> impl Iterator<Item = ColumnOperation<u128>>
+where
+ T: Coerce + MonotonicallyMappableToU128,
+{
+ operation_iterator.map(|symbol| match symbol {
+ ColumnOperation::NewDoc(doc) => ColumnOperation::NewDoc(doc),
+ ColumnOperation::Value(numerical_value) => {
+ ColumnOperation::Value(T::coerce(numerical_value).to_u128())
+ }
+ })
+}
+
fn consume_operation_iterator<T: Ord, TIndexBuilder: IndexBuilder>(
operation_iterator: impl Iterator<Item = ColumnOperation<T>>,
index_builder: &mut TIndexBuilder,
diff --git a/crates/tantivy/src/columnar/dynamic_column.rs b/crates/tantivy/src/columnar/dynamic_column.rs
index 797d4067..3eb8d493 100644
--- a/crates/tantivy/src/columnar/dynamic_column.rs
+++ b/crates/tantivy/src/columnar/dynamic_column.rs
@@ -15,6 +15,7 @@ pub enum DynamicColumn {
I64(Column<i64>),
U64(Column<u64>),
F64(Column<f64>),
+ U128(Column<u128>),
DateTime(Column<DateTime>),
Bytes(BytesColumn),
}
@@ -29,6 +30,7 @@ impl fmt::Debug for DynamicColumn {
DynamicColumn::F64(col) => write!(f, "{col:?}")?,
DynamicColumn::DateTime(col) => write!(f, "{col:?}")?,
DynamicColumn::Bytes(col) => write!(f, "{col:?}")?,
+ DynamicColumn::U128(col) => write!(f, "{col:?}")?,
}
write!(f, "]")
}
@@ -43,6 +45,7 @@ impl DynamicColumn {
DynamicColumn::F64(c) => &c.index,
DynamicColumn::DateTime(c) => &c.index,
DynamicColumn::Bytes(c) => &c.ords().index,
+ DynamicColumn::U128(c) => &c.index,
}
}
@@ -54,6 +57,7 @@ impl DynamicColumn {
DynamicColumn::F64(c) => c.values.num_vals(),
DynamicColumn::DateTime(c) => c.values.num_vals(),
DynamicColumn::Bytes(c) => c.ords().values.num_vals(),
+ DynamicColumn::U128(c) => c.values.num_vals(),
}
}
@@ -65,6 +69,7 @@ impl DynamicColumn {
DynamicColumn::F64(_) => ColumnType::F64,
DynamicColumn::DateTime(_) => ColumnType::DateTime,
DynamicColumn::Bytes(_) => ColumnType::Bytes,
+ DynamicColumn::U128(_) => ColumnType::U128,
}
}
@@ -73,6 +78,7 @@ impl DynamicColumn {
NumericalType::I64 => self.coerce_to_i64(),
NumericalType::U64 => self.coerce_to_u64(),
NumericalType::F64 => self.coerce_to_f64(),
+ NumericalType::U128 => self.coerce_to_u128(),
}
}
@@ -134,6 +140,12 @@ impl DynamicColumn {
_ => None,
}
}
+ fn coerce_to_u128(self) -> Option<DynamicColumn> {
+ match self {
+ DynamicColumn::U128(_) => Some(self),
+ _ => None,
+ }
+ }
}
struct MapI64ToF64;
@@ -210,6 +222,7 @@ static_dynamic_conversions!(Column<i64>, I64);
static_dynamic_conversions!(Column<f64>, F64);
static_dynamic_conversions!(Column<DateTime>, DateTime);
static_dynamic_conversions!(BytesColumn, Bytes);
+static_dynamic_conversions!(Column<u128>, U128);
#[derive(Clone, Debug)]
pub struct DynamicColumnHandle {
@@ -230,15 +243,7 @@ impl DynamicColumnHandle {
}
/// Returns the `u64` columnar field reader reader associated with `fields` of types
- /// Str, u64, i64, f64, bool, ip, or datetime.
- ///
- /// Notice that for IpAddr, the columnfield reader will return the u64 representation of the
- /// IpAddr.
- /// In order to convert to u128 back cast to `CompactSpaceU64Accessor` and call
- /// `compact_to_u128`.
- ///
- /// If not, the columnfield reader will returns the u64-value associated with the original
- /// FastValue.
+ /// Str, u64, i64, f64, bool, or datetime.
pub fn open_u64_lenient(&self) -> io::Result<Option<Column<u64>>> {
let column_bytes = self.file_slice.read_bytes()?;
match self.column_type {
@@ -254,6 +259,7 @@ impl DynamicColumnHandle {
let column = super::column::open_column_u64::<u64>(column_bytes)?;
Ok(Some(column))
}
+ ColumnType::U128 => Ok(None),
}
}
@@ -264,6 +270,7 @@ impl DynamicColumnHandle {
ColumnType::U64 => super::column::open_column_u64::<u64>(column_bytes)?.into(),
ColumnType::F64 => super::column::open_column_u64::<f64>(column_bytes)?.into(),
ColumnType::Bool => super::column::open_column_u64::<bool>(column_bytes)?.into(),
+ ColumnType::U128 => super::column::open_column_u128::<u128>(column_bytes)?.into(),
ColumnType::DateTime => {
super::column::open_column_u64::<DateTime>(column_bytes)?.into()
}
diff --git a/crates/tantivy/src/columnar/value.rs b/crates/tantivy/src/columnar/value.rs
index 45343131..f44c001b 100644
--- a/crates/tantivy/src/columnar/value.rs
+++ b/crates/tantivy/src/columnar/value.rs
@@ -7,6 +7,7 @@ pub enum NumericalValue {
I64(i64),
U64(u64),
F64(f64),
+ U128(u128),
}
impl NumericalValue {
@@ -15,6 +16,7 @@ impl NumericalValue {
NumericalValue::I64(_) => NumericalType::I64,
NumericalValue::U64(_) => NumericalType::U64,
NumericalValue::F64(_) => NumericalType::F64,
+ NumericalValue::U128(_) => NumericalType::U128,
}
}
}
@@ -37,6 +39,12 @@ impl From<f64> for NumericalValue {
}
}
+impl From<u128> for NumericalValue {
+ fn from(val: u128) -> Self {
+ NumericalValue::U128(val)
+ }
+}
+
#[derive(Clone, Copy, Debug, Default, Hash, Eq, PartialEq)]
#[repr(u8)]
pub enum NumericalType {
@@ -44,6 +52,7 @@ pub enum NumericalType {
I64 = 0,
U64 = 1,
F64 = 2,
+ U128 = 3,
}
impl NumericalType {
@@ -56,6 +65,7 @@ impl NumericalType {
0 => Ok(NumericalType::I64),
1 => Ok(NumericalType::U64),
2 => Ok(NumericalType::F64),
+ 3 => Ok(NumericalType::U128),
_ => Err(InvalidData),
}
}
@@ -81,6 +91,7 @@ impl Coerce for i64 {
match value {
NumericalValue::I64(val) => val,
NumericalValue::U64(val) => val as i64,
+ NumericalValue::U128(val) => val as i64,
NumericalValue::F64(_) => unreachable!(),
}
}
@@ -92,6 +103,7 @@ impl Coerce for u64 {
NumericalValue::I64(val) => val as u64,
NumericalValue::U64(val) => val,
NumericalValue::F64(_) => unreachable!(),
+ NumericalValue::U128(val) => val as u64,
}
}
}
@@ -102,6 +114,7 @@ impl Coerce for f64 {
NumericalValue::I64(val) => val as f64,
NumericalValue::U64(val) => val as f64,
NumericalValue::F64(val) => val,
+ NumericalValue::U128(val) => val as f64,
}
}
}
@@ -113,6 +126,17 @@ impl Coerce for DateTime {
}
}
+impl Coerce for u128 {
+ fn coerce(value: NumericalValue) -> Self {
+ match value {
+ NumericalValue::U128(val) => val,
+ NumericalValue::I64(val) => val as u128,
+ NumericalValue::U64(val) => val as u128,
+ NumericalValue::F64(_) => unreachable!(),
+ }
+ }
+}
+
#[cfg(test)]
mod tests {
use super::NumericalType;
@@ -126,6 +150,6 @@ mod tests {
num_numerical_type += 1;
}
}
- assert_eq!(num_numerical_type, 3);
+ assert_eq!(num_numerical_type, 4);
}
}
diff --git a/crates/tantivy/src/columnfield/mod.rs b/crates/tantivy/src/columnfield/mod.rs
index 292a4585..3c9206c0 100644
--- a/crates/tantivy/src/columnfield/mod.rs
+++ b/crates/tantivy/src/columnfield/mod.rs
@@ -20,7 +20,6 @@
//! Read access performance is comparable to that of an array lookup.
pub use crate::columnar::Column;
-use crate::columnar::MonotonicallyMappableToU64;
pub use self::error::{ColumnFieldNotAvailableError, Result};
pub use self::readers::ColumnFieldReaders;
@@ -33,8 +32,8 @@ mod readers;
mod writer;
/// Trait for types that are allowed for columnar fields:
-/// (u64, i64 and f64, bool, DateTime).
-pub trait ColumnarValue: MonotonicallyMappableToU64 {
+/// (u64, u128, i64, f64, bool, DateTime).
+pub trait ColumnarValue {
/// Returns the `schema::Type` for this ColumnarValue.
fn to_type() -> Type;
}
@@ -67,13 +66,18 @@ impl ColumnarValue for DateTime {
Type::Date
}
}
-
+impl ColumnarValue for u128 {
+ fn to_type() -> Type {
+ Type::U128
+ }
+}
#[cfg(test)]
mod tests {
use std::ops::RangeInclusive;
use std::path::Path;
+ use crate::columnar::MonotonicallyMappableToU64;
use crate::common::{ByteCount, DateTimePrecision, HasLen, TerminatingWrite};
use rand::prelude::SliceRandom;
use rand::rngs::StdRng;
diff --git a/crates/tantivy/src/columnfield/writer.rs b/crates/tantivy/src/columnfield/writer.rs
index c452b388..0d1c5fa5 100644
--- a/crates/tantivy/src/columnfield/writer.rs
+++ b/crates/tantivy/src/columnfield/writer.rs
@@ -150,6 +150,13 @@ impl ColumnFieldsWriter {
NumericalValue::from(val),
);
}
+ ReferenceValueLeaf::U128(val) => {
+ self.columnar_writer.record_numerical(
+ doc_id,
+ field_name,
+ NumericalValue::from(val),
+ );
+ }
ReferenceValueLeaf::I64(val) => {
self.columnar_writer.record_numerical(
doc_id,
@@ -271,6 +278,13 @@ fn record_json_value_to_columnar_writer<'a, V: Value<'a>>(
NumericalValue::from(val),
);
}
+ ReferenceValueLeaf::U128(val) => {
+ columnar_writer.record_numerical(
+ doc,
+ json_path_writer.as_str(),
+ NumericalValue::from(val),
+ );
+ }
ReferenceValueLeaf::I64(val) => {
columnar_writer.record_numerical(
doc,
diff --git a/crates/tantivy/src/common/serialize.rs b/crates/tantivy/src/common/serialize.rs
index 14506c54..ccda821f 100644
--- a/crates/tantivy/src/common/serialize.rs
+++ b/crates/tantivy/src/common/serialize.rs
@@ -1,5 +1,6 @@
use std::borrow::Cow;
use std::io::{Read, Write};
+use std::num::NonZeroU64;
use std::{fmt, io};
use byteorder::{ReadBytesExt, WriteBytesExt};
@@ -287,6 +288,24 @@ impl<'a> BinarySerializable for Cow<'a, [u8]> {
}
}
+impl BinarySerializable for NonZeroU64 {
+ fn serialize<W: Write + ?Sized>(&self, writer: &mut W) -> io::Result<()> {
+ self.get().serialize(writer)
+ }
+
+ fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
+ let val = u64::deserialize(reader)?;
+ if val == 0 {
+ Err(io::Error::new(
+ io::ErrorKind::InvalidData,
+ "zero is not a valid NonZeroU64",
+ ))
+ } else {
+ Ok(NonZeroU64::new(val).unwrap())
+ }
+ }
+}
+
#[cfg(test)]
pub mod test {
diff --git a/crates/tantivy/src/core/json_utils.rs b/crates/tantivy/src/core/json_utils.rs
index 95523042..7c6dc6de 100644
--- a/crates/tantivy/src/core/json_utils.rs
+++ b/crates/tantivy/src/core/json_utils.rs
@@ -147,19 +147,28 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
if let Ok(i64_val) = val.try_into() {
- term_buffer.append_type_and_columnar_value::<i64>(i64_val);
+ term_buffer.append_type_and_columnar_value_64::<i64>(i64_val);
} else {
- term_buffer.append_type_and_columnar_value(val);
+ term_buffer.append_type_and_columnar_value_64(val);
}
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
+ ReferenceValueLeaf::U128(val) => {
+ set_path_id(
+ term_buffer,
+ ctx.path_to_unordered_id
+ .get_or_allocate_unordered_id(json_path_writer.as_str()),
+ );
+ term_buffer.append_type_and_columnar_value_128(val);
+ postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
+ }
ReferenceValueLeaf::I64(val) => {
set_path_id(
term_buffer,
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
- term_buffer.append_type_and_columnar_value(val);
+ term_buffer.append_type_and_columnar_value_64(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::F64(val) => {
@@ -168,7 +177,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
- term_buffer.append_type_and_columnar_value(val);
+ term_buffer.append_type_and_columnar_value_64(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::Bool(val) => {
@@ -177,7 +186,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
- term_buffer.append_type_and_columnar_value(val);
+ term_buffer.append_type_and_columnar_value_64(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::Date(val) => {
@@ -186,7 +195,7 @@ pub(crate) fn index_json_value<'a, V: Value<'a>>(
ctx.path_to_unordered_id
.get_or_allocate_unordered_id(json_path_writer.as_str()),
);
- term_buffer.append_type_and_columnar_value(val);
+ term_buffer.append_type_and_columnar_value_64(val);
postings_writer.subscribe(doc, 0u32, term_buffer, ctx);
}
ReferenceValueLeaf::PreTokStr(_) => {
@@ -248,23 +257,23 @@ pub fn convert_to_columnar_value_and_append_to_json_term(
);
if let Ok(dt) = OffsetDateTime::parse(phrase, &Rfc3339) {
let dt_utc = dt.to_offset(UtcOffset::UTC);
- term.append_type_and_columnar_value(DateTime::from_utc(dt_utc));
+ term.append_type_and_columnar_value_64(DateTime::from_utc(dt_utc));
return Some(term);
}
if let Ok(i64_val) = str::parse::<i64>(phrase) {
- term.append_type_and_columnar_value(i64_val);
+ term.append_type_and_columnar_value_64(i64_val);
return Some(term);
}
if let Ok(u64_val) = str::parse::<u64>(phrase) {
- term.append_type_and_columnar_value(u64_val);
+ term.append_type_and_columnar_value_64(u64_val);
return Some(term);
}
if let Ok(f64_val) = str::parse::<f64>(phrase) {
- term.append_type_and_columnar_value(f64_val);
+ term.append_type_and_columnar_value_64(f64_val);
return Some(term);
}
if let Ok(bool_val) = str::parse::<bool>(phrase) {
- term.append_type_and_columnar_value(bool_val);
+ term.append_type_and_columnar_value_64(bool_val);
return Some(term);
}
None
@@ -343,7 +352,7 @@ mod tests {
);
let mut term = Term::from_field_json_path(field, "attributes.dimensions.width", false);
- term.append_type_and_columnar_value(400i64);
+ term.append_type_and_columnar_value_64(400i64);
assert_eq!(
format!("{term:?}"),
"Term(field=1, type=Json, path=attributes.dimensions.width, type=I64, 400)"
@@ -363,7 +372,7 @@ mod tests {
fn test_i64_term() {
let field = Field::from_field_id(1);
let mut term = Term::from_field_json_path(field, "color", false);
- term.append_type_and_columnar_value(-4i64);
+ term.append_type_and_columnar_value_64(-4i64);
assert_eq!(
term.serialized_term(),
@@ -375,7 +384,7 @@ mod tests {
fn test_u64_term() {
let field = Field::from_field_id(1);
let mut term = Term::from_field_json_path(field, "color", false);
- term.append_type_and_columnar_value(4u64);
+ term.append_type_and_columnar_value_64(4u64);
assert_eq!(
term.serialized_term(),
@@ -387,7 +396,7 @@ mod tests {
fn test_f64_term() {
let field = Field::from_field_id(1);
let mut term = Term::from_field_json_path(field, "color", false);
- term.append_type_and_columnar_value(4.0f64);
+ term.append_type_and_columnar_value_64(4.0f64);
assert_eq!(
term.serialized_term(),
b"\x00\x00\x00\x01jcolor\x00f\xc0\x10\x00\x00\x00\x00\x00\x00"
@@ -398,7 +407,7 @@ mod tests {
fn test_bool_term() {
let field = Field::from_field_id(1);
let mut term = Term::from_field_json_path(field, "color", false);
- term.append_type_and_columnar_value(true);
+ term.append_type_and_columnar_value_64(true);
assert_eq!(
term.serialized_term(),
b"\x00\x00\x00\x01jcolor\x00o\x00\x00\x00\x00\x00\x00\x00\x01"
diff --git a/crates/tantivy/src/core/tests.rs b/crates/tantivy/src/core/tests.rs
index 1411aa8e..d7645e12 100644
--- a/crates/tantivy/src/core/tests.rs
+++ b/crates/tantivy/src/core/tests.rs
@@ -411,7 +411,7 @@ fn test_non_text_json_term_freq() {
let inv_idx = segment_reader.inverted_index(field).unwrap();
let mut term = Term::from_field_json_path(field, "tenant_id", false);
- term.append_type_and_columnar_value(75i64);
+ term.append_type_and_columnar_value_64(75i64);
let postings = inv_idx
.read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
@@ -445,7 +445,7 @@ fn test_non_text_json_term_freq_bitpacked() {
let inv_idx = segment_reader.inverted_index(field).unwrap();
let mut term = Term::from_field_json_path(field, "tenant_id", false);
- term.append_type_and_columnar_value(75i64);
+ term.append_type_and_columnar_value_64(75i64);
let mut postings = inv_idx
.read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
@@ -458,3 +458,27 @@ fn test_non_text_json_term_freq_bitpacked() {
assert_eq!(postings.term_freq(), 1u32);
}
}
+
+#[test]
+fn test_u128_columnar_values() {
+ let mut schema_builder = Schema::builder();
+ let field = schema_builder.add_u128_field("u128", INDEXED);
+ let schema = schema_builder.build();
+ let index = Index::create_in_ram(schema);
+ let mut writer: IndexWriter = index.writer_for_tests().unwrap();
+ writer.add_document(doc!(field => 1u128)).unwrap();
+ writer.commit().unwrap();
+ let reader = index.reader().unwrap();
+ assert_eq!(reader.searcher().num_docs(), 1);
+
+ let searcher = reader.searcher();
+ let segment_reader = searcher.segment_reader(0u32);
+ let inv_idx = segment_reader.inverted_index(field).unwrap();
+ let term = Term::from_field_u128(field, 1u128);
+ let postings = inv_idx
+ .read_postings(&term, IndexRecordOption::WithFreqsAndPositions)
+ .unwrap()
+ .unwrap();
+ assert_eq!(postings.doc(), 0);
+ assert_eq!(postings.term_freq(), 1u32);
+}
diff --git a/crates/tantivy/src/indexer/segment_writer.rs b/crates/tantivy/src/indexer/segment_writer.rs
index 5cde9059..b7a6deec 100644
--- a/crates/tantivy/src/indexer/segment_writer.rs
+++ b/crates/tantivy/src/indexer/segment_writer.rs
@@ -253,6 +253,20 @@ impl SegmentWriter {
self.fieldnorms_writer.record(doc_id, field, num_vals);
}
}
+ FieldType::U128(_) => {
+ let mut num_vals = 0;
+ for value in values {
+ let value = value.as_value();
+
+ num_vals += 1;
+ let u128_val = value.as_u128().ok_or_else(make_schema_error)?;
+ term_buffer.set_u128(u128_val);
+ postings_writer.subscribe(doc_id, 0u32, term_buffer, ctx);
+ }
+ if field_entry.has_fieldnorms() {
+ self.fieldnorms_writer.record(doc_id, field, num_vals);
+ }
+ }
FieldType::Date(_) => {
let mut num_vals = 0;
for value in values {
@@ -490,6 +504,7 @@ mod tests {
use tempfile::TempDir;
use crate::collector::{Count, TopDocs};
+ use crate::columnar::MonotonicallyMappableToU64;
use crate::columnfield::ColumnarValue;
use crate::directory::RamDirectory;
use crate::postings::{Postings, TermInfo};
@@ -683,8 +698,11 @@ mod tests {
let term_from_path =
|path: &str| -> Term { Term::from_field_json_path(json_field, path, false) };
- fn set_columnar_val<T: ColumnarValue>(val: T, mut term: Term) -> Term {
- term.append_type_and_columnar_value(val);
+ fn set_columnar_val_64<T: ColumnarValue + MonotonicallyMappableToU64>(
+ val: T,
+ mut term: Term,
+ ) -> Term {
+ term.append_type_and_columnar_value_64(val);
term
}
fn set_str(val: &str, mut term: Term) -> Term {
@@ -696,14 +714,14 @@ mod tests {
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(true, term).serialized_value_bytes()
+ set_columnar_val_64(true, term).serialized_value_bytes()
);
let term = term_from_path("complexobject.field\\.with\\.dot");
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(1i64, term).serialized_value_bytes()
+ set_columnar_val_64(1i64, term).serialized_value_bytes()
);
// Date
@@ -712,7 +730,7 @@ mod tests {
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(
+ set_columnar_val_64(
DateTime::from_utc(
OffsetDateTime::parse("1985-04-12T23:20:50.52Z", &Rfc3339).unwrap(),
),
@@ -726,7 +744,7 @@ mod tests {
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(-0.2f64, term).serialized_value_bytes()
+ set_columnar_val_64(-0.2f64, term).serialized_value_bytes()
);
// Number In Array
@@ -734,21 +752,21 @@ mod tests {
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(2i64, term).serialized_value_bytes()
+ set_columnar_val_64(2i64, term).serialized_value_bytes()
);
let term = term_from_path("my_arr");
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(3i64, term).serialized_value_bytes()
+ set_columnar_val_64(3i64, term).serialized_value_bytes()
);
let term = term_from_path("my_arr");
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(4i64, term).serialized_value_bytes()
+ set_columnar_val_64(4i64, term).serialized_value_bytes()
);
// El in Array
@@ -770,7 +788,7 @@ mod tests {
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(-2i64, term).serialized_value_bytes()
+ set_columnar_val_64(-2i64, term).serialized_value_bytes()
);
let term = term_from_path("toto");
@@ -784,7 +802,7 @@ mod tests {
assert!(term_stream.advance());
assert_eq!(
term_stream.key(),
- set_columnar_val(1i64, term).serialized_value_bytes()
+ set_columnar_val_64(1i64, term).serialized_value_bytes()
);
assert!(!term_stream.advance());
diff --git a/crates/tantivy/src/postings/per_field_postings_writer.rs b/crates/tantivy/src/postings/per_field_postings_writer.rs
index b283b3d9..cbc42ef3 100644
--- a/crates/tantivy/src/postings/per_field_postings_writer.rs
+++ b/crates/tantivy/src/postings/per_field_postings_writer.rs
@@ -45,6 +45,7 @@ fn posting_writer_from_field_entry(field_entry: &FieldEntry) -> Box<dyn Postings
})
.unwrap_or_else(|| SpecializedPostingsWriter::<DocIdRecorder>::default().into()),
FieldType::U64(_)
+ | FieldType::U128(_)
| FieldType::I64(_)
| FieldType::F64(_)
| FieldType::Bool(_)
diff --git a/crates/tantivy/src/query/query_parser/query_parser.rs b/crates/tantivy/src/query/query_parser/query_parser.rs
index ba86641d..e9e9ec5a 100644
--- a/crates/tantivy/src/query/query_parser/query_parser.rs
+++ b/crates/tantivy/src/query/query_parser/query_parser.rs
@@ -443,6 +443,10 @@ impl QueryParser {
let val: u64 = u64::from_str(phrase)?;
Ok(Term::from_field_u64(field, val))
}
+ FieldType::U128(_) => {
+ let val: u128 = u128::from_str(phrase)?;
+ Ok(Term::from_field_u128(field, val))
+ }
FieldType::I64(_) => {
let val: i64 = i64::from_str(phrase)?;
Ok(Term::from_field_i64(field, val))
@@ -529,6 +533,11 @@ impl QueryParser {
let i64_term = Term::from_field_u64(field, val);
Ok(vec![LogicalLiteral::Term(i64_term)])
}
+ FieldType::U128(_) => {
+ let val: u128 = u128::from_str(phrase)?;
+ let u128_term = Term::from_field_u128(field, val);
+ Ok(vec![LogicalLiteral::Term(u128_term)])
+ }
FieldType::I64(_) => {
let val: i64 = i64::from_str(phrase)?;
let i64_term = Term::from_field_i64(field, val);
diff --git a/crates/tantivy/src/query/range_query/mod.rs b/crates/tantivy/src/query/range_query/mod.rs
index 663fe8b3..4b0a5962 100644
--- a/crates/tantivy/src/query/range_query/mod.rs
+++ b/crates/tantivy/src/query/range_query/mod.rs
@@ -14,7 +14,7 @@ pub(crate) fn is_type_valid_for_columnfield_range_query(typ: Type) -> bool {
match typ {
Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,
Type::IpAddr => true,
- Type::Str | Type::Bytes | Type::Json => false,
+ Type::Str | Type::Bytes | Type::Json | Type::U128 => false,
}
}
diff --git a/crates/tantivy/src/query/range_query/range_query.rs b/crates/tantivy/src/query/range_query/range_query.rs
index c86fe8d4..df692a1b 100644
--- a/crates/tantivy/src/query/range_query/range_query.rs
+++ b/crates/tantivy/src/query/range_query/range_query.rs
@@ -322,7 +322,7 @@ pub(crate) fn maps_to_u64_columnfield(typ: Type) -> bool {
match typ {
Type::U64 | Type::I64 | Type::F64 | Type::Bool | Type::Date => true,
Type::IpAddr => false,
- Type::Str | Type::Bytes | Type::Json => false,
+ Type::Str | Type::Bytes | Type::Json | Type::U128 => false,
}
}
diff --git a/crates/tantivy/src/roworder/writer.rs b/crates/tantivy/src/roworder/writer.rs
index df4100bf..17666934 100644
--- a/crates/tantivy/src/roworder/writer.rs
+++ b/crates/tantivy/src/roworder/writer.rs
@@ -33,7 +33,8 @@ impl TryFrom<(crate::schema::Field, &crate::schema::FieldEntry)> for Field {
| crate::schema::FieldType::Date(_)
| crate::schema::FieldType::Bytes(_)
| crate::schema::FieldType::JsonObject(_)
- | crate::schema::FieldType::IpAddr(_) => {
+ | crate::schema::FieldType::IpAddr(_)
+ | crate::schema::FieldType::U128(_) => {
return Err(crate::TantivyError::SchemaError(format!(
"Field {:?} is not supported in row order",
entry
@@ -100,6 +101,7 @@ impl RowFieldsWriter {
| ReferenceValueLeaf::Date(_)
| ReferenceValueLeaf::Bytes(_)
| ReferenceValueLeaf::IpAddr(_)
+ | ReferenceValueLeaf::U128(_)
| ReferenceValueLeaf::PreTokStr(_) => {}
ReferenceValueLeaf::U64(val) => {
diff --git a/crates/tantivy/src/schema/document/de.rs b/crates/tantivy/src/schema/document/de.rs
index 22c1c8b1..cf398e9f 100644
--- a/crates/tantivy/src/schema/document/de.rs
+++ b/crates/tantivy/src/schema/document/de.rs
@@ -103,6 +103,9 @@ pub trait ValueDeserializer<'de> {
/// Attempts to deserialize a u64 value from the deserializer.
fn deserialize_u64(self) -> Result<u64, DeserializeError>;
+ /// Attempts to deserialize a u128 value from the deserializer.
+ fn deserialize_u128(self) -> Result<u128, DeserializeError>;
+
/// Attempts to deserialize an i64 value from the deserializer.
fn deserialize_i64(self) -> Result<i64, DeserializeError>;
@@ -139,6 +142,8 @@ pub enum ValueType {
String,
/// A u64 value.
U64,
+ /// A u128 value.
+ U128,
/// A i64 value.
I64,
/// A f64 value.
@@ -189,6 +194,12 @@ pub trait ValueVisitor {
Err(DeserializeError::UnsupportedType(ValueType::U64))
}
+ #[inline]
+ /// Called when the deserializer visits a u128 value.
+ fn visit_u128(&self, _val: u128) -> Result<Self::Value, DeserializeError> {
+ Err(DeserializeError::UnsupportedType(ValueType::U128))
+ }
+
#[inline]
/// Called when the deserializer visits a i64 value.
fn visit_i64(&self, _val: i64) -> Result<Self::Value, DeserializeError> {
@@ -423,6 +434,11 @@ where
<u64 as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
}
+ fn deserialize_u128(self) -> Result<u128, DeserializeError> {
+ self.validate_type(ValueType::U128)?;
+ <u128 as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
+ }
+
fn deserialize_i64(self) -> Result<i64, DeserializeError> {
self.validate_type(ValueType::I64)?;
<i64 as BinarySerializable>::deserialize(self.reader).map_err(DeserializeError::from)
@@ -477,6 +493,10 @@ where
let val = self.deserialize_u64()?;
visitor.visit_u64(val)
}
+ ValueType::U128 => {
+ let val = self.deserialize_u128()?;
+ visitor.visit_u128(val)
+ }
ValueType::I64 => {
let val = self.deserialize_i64()?;
visitor.visit_i64(val)
diff --git a/crates/tantivy/src/schema/document/default_document.rs b/crates/tantivy/src/schema/document/default_document.rs
index 119c50a2..1fa81199 100644
--- a/crates/tantivy/src/schema/document/default_document.rs
+++ b/crates/tantivy/src/schema/document/default_document.rs
@@ -83,6 +83,11 @@ impl CompactDoc {
self.add_leaf_field_value(field, value);
}
+ /// Add a u128 field
+ pub fn add_u128(&mut self, field: Field, value: u128) {
+ self.add_leaf_field_value(field, value);
+ }
+
/// Add a IP address field. Internally only Ipv6Addr is used.
pub fn add_ip_addr(&mut self, field: Field, value: Ipv6Addr) {
self.add_leaf_field_value(field, value);
@@ -238,6 +243,7 @@ impl CompactDoc {
}
ReferenceValueLeaf::Bytes(bytes) => write_bytes_into(&mut self.node_data, bytes),
ReferenceValueLeaf::U64(num) => write_into(&mut self.node_data, num),
+ ReferenceValueLeaf::U128(num) => write_into(&mut self.node_data, num),
ReferenceValueLeaf::I64(num) => write_into(&mut self.node_data, num),
ReferenceValueLeaf::F64(num) => write_into(&mut self.node_data, num),
ReferenceValueLeaf::Bool(b) => b as u32,
@@ -427,6 +433,11 @@ impl<'a> CompactDocValue<'a> {
.read_from::<u64>(addr)
.map(ReferenceValueLeaf::U64)
.map(Into::into),
+ ValueType::U128 => self
+ .container
+ .read_from::<u128>(addr)
+ .map(ReferenceValueLeaf::U128)
+ .map(Into::into),
ValueType::I64 => self
.container
.read_from::<i64>(addr)
@@ -530,6 +541,8 @@ pub enum ValueType {
Object = 11,
/// Pre-tokenized str type,
Array = 12,
+ /// Unsigned 128-bits Integer `u128`
+ U128 = 13,
}
impl BinarySerializable for ValueType {
@@ -540,7 +553,7 @@ impl BinarySerializable for ValueType {
fn deserialize<R: Read>(reader: &mut R) -> io::Result<Self> {
let num = u8::deserialize(reader)?;
- let type_id = if (0..=12).contains(&num) {
+ let type_id = if (0..=13).contains(&num) {
unsafe { std::mem::transmute::<u8, ValueType>(num) }
} else {
return Err(io::Error::new(
@@ -567,6 +580,7 @@ impl<'a> From<&ReferenceValueLeaf<'a>> for ValueType {
ReferenceValueLeaf::Null => ValueType::Null,
ReferenceValueLeaf::Str(_) => ValueType::Str,
ReferenceValueLeaf::U64(_) => ValueType::U64,
+ ReferenceValueLeaf::U128(_) => ValueType::U128,
ReferenceValueLeaf::I64(_) => ValueType::I64,
ReferenceValueLeaf::F64(_) => ValueType::F64,
ReferenceValueLeaf::Bool(_) => ValueType::Bool,
diff --git a/crates/tantivy/src/schema/document/existing_type_impls.rs b/crates/tantivy/src/schema/document/existing_type_impls.rs
index 45d00974..cc1ccddb 100644
--- a/crates/tantivy/src/schema/document/existing_type_impls.rs
+++ b/crates/tantivy/src/schema/document/existing_type_impls.rs
@@ -91,6 +91,15 @@ impl<'a> Value<'a> for &'a u64 {
}
}
+impl<'a> Value<'a> for &'a u128 {
+ type ArrayIter = Empty<&'a u128>;
+ type ObjectIter = Empty<(&'a str, &'a u128)>;
+ #[inline]
+ fn as_value(&self) -> ReferenceValue<'a, Self> {
+ ReferenceValue::Leaf(ReferenceValueLeaf::U128(**self))
+ }
+}
+
impl<'a> Value<'a> for &'a i64 {
type ArrayIter = Empty<&'a i64>;
type ObjectIter = Empty<(&'a str, &'a i64)>;
diff --git a/crates/tantivy/src/schema/document/mod.rs b/crates/tantivy/src/schema/document/mod.rs
index 71275bad..a8fb769d 100644
--- a/crates/tantivy/src/schema/document/mod.rs
+++ b/crates/tantivy/src/schema/document/mod.rs
@@ -269,6 +269,7 @@ pub(crate) mod type_codes {
pub const NULL_CODE: u8 = 11;
pub const ARRAY_CODE: u8 = 12;
pub const OBJECT_CODE: u8 = 13;
+ pub const U128_CODE: u8 = 14;
// Extended type codes
pub const TOK_STR_EXT_CODE: u8 = 0;
diff --git a/crates/tantivy/src/schema/document/owned_value.rs b/crates/tantivy/src/schema/document/owned_value.rs
index 95240af2..1db89e27 100644
--- a/crates/tantivy/src/schema/document/owned_value.rs
+++ b/crates/tantivy/src/schema/document/owned_value.rs
@@ -30,6 +30,8 @@ pub enum OwnedValue {
PreTokStr(PreTokenizedString),
/// Unsigned 64-bits Integer `u64`
U64(u64),
+ /// Unsigned 128-bits Integer `u128`
+ U128(u128),
/// Signed 64-bits Integer `i64`
I64(i64),
/// 64-bits Float `f64`
@@ -65,6 +67,7 @@ impl<'a> Value<'a> for &'a OwnedValue {
OwnedValue::Str(val) => ReferenceValueLeaf::Str(val).into(),
OwnedValue::PreTokStr(val) => ReferenceValueLeaf::PreTokStr(val.clone().into()).into(),
OwnedValue::U64(val) => ReferenceValueLeaf::U64(*val).into(),
+ OwnedValue::U128(val) => ReferenceValueLeaf::U128(*val).into(),
OwnedValue::I64(val) => ReferenceValueLeaf::I64(*val).into(),
OwnedValue::F64(val) => ReferenceValueLeaf::F64(*val).into(),
OwnedValue::Bool(val) => ReferenceValueLeaf::Bool(*val).into(),
@@ -99,6 +102,10 @@ impl ValueDeserialize for OwnedValue {
Ok(OwnedValue::U64(val))
}
+ fn visit_u128(&self, val: u128) -> Result<Self::Value, DeserializeError> {
+ Ok(OwnedValue::U128(val))
+ }
+
fn visit_i64(&self, val: i64) -> Result<Self::Value, DeserializeError> {
Ok(OwnedValue::I64(val))
}
@@ -174,6 +181,7 @@ impl serde::Serialize for OwnedValue {
OwnedValue::Str(ref v) => serializer.serialize_str(v),
OwnedValue::PreTokStr(ref v) => v.serialize(serializer),
OwnedValue::U64(u) => serializer.serialize_u64(u),
+ OwnedValue::U128(u) => serializer.serialize_u128(u),
OwnedValue::I64(u) => serializer.serialize_i64(u),
OwnedValue::F64(u) => serializer.serialize_f64(u),
OwnedValue::Bool(b) => serializer.serialize_bool(b),
@@ -227,6 +235,10 @@ impl<'de> serde::Deserialize<'de> for OwnedValue {
Ok(OwnedValue::U64(v))
}
+ fn visit_u128<E>(self, v: u128) -> Result<Self::Value, E> {
+ Ok(OwnedValue::U128(v))
+ }
+
fn visit_f64<E>(self, v: f64) -> Result<Self::Value, E> {
Ok(OwnedValue::F64(v))
}
@@ -282,6 +294,7 @@ impl<'a, V: Value<'a>> From<ReferenceValue<'a, V>> for OwnedValue {
ReferenceValueLeaf::Null => OwnedValue::Null,
ReferenceValueLeaf::Str(val) => OwnedValue::Str(val.to_string()),
ReferenceValueLeaf::U64(val) => OwnedValue::U64(val),
+ ReferenceValueLeaf::U128(val) => OwnedValue::U128(val),
ReferenceValueLeaf::I64(val) => OwnedValue::I64(val),
ReferenceValueLeaf::F64(val) => OwnedValue::F64(val),
ReferenceValueLeaf::Date(val) => OwnedValue::Date(val),
diff --git a/crates/tantivy/src/schema/document/se.rs b/crates/tantivy/src/schema/document/se.rs
index 512bf880..31ee1e4d 100644
--- a/crates/tantivy/src/schema/document/se.rs
+++ b/crates/tantivy/src/schema/document/se.rs
@@ -105,6 +105,11 @@ where
val.serialize(self.writer)
}
+ ReferenceValueLeaf::U128(val) => {
+ self.write_type_code(type_codes::U128_CODE)?;
+
+ val.serialize(self.writer)
+ }
ReferenceValueLeaf::I64(val) => {
self.write_type_code(type_codes::I64_CODE)?;
diff --git a/crates/tantivy/src/schema/document/value.rs b/crates/tantivy/src/schema/document/value.rs
index 086ccefa..4bdfea4f 100644
--- a/crates/tantivy/src/schema/document/value.rs
+++ b/crates/tantivy/src/schema/document/value.rs
@@ -39,6 +39,12 @@ pub trait Value<'a>: Send + Sync + Debug {
self.as_leaf().and_then(|leaf| leaf.as_u64())
}
+ #[inline]
+ /// If the Value is a u128, returns the associated u128. Returns None otherwise.
+ fn as_u128(&self) -> Option<u128> {
+ self.as_leaf().and_then(|leaf| leaf.as_u128())
+ }
+
#[inline]
/// If the Value is a i64, returns the associated i64. Returns None otherwise.
fn as_i64(&self) -> Option<i64> {
@@ -113,6 +119,8 @@ pub enum ReferenceValueLeaf<'a> {
Str(&'a str),
/// Unsigned 64-bits Integer `u64`
U64(u64),
+ /// Unsigned 128-bits Integer `u128`
+ U128(u128),
/// Signed 64-bits Integer `i64`
I64(i64),
/// 64-bits Float `f64`
@@ -136,6 +144,13 @@ impl From<u64> for ReferenceValueLeaf<'_> {
}
}
+impl From<u128> for ReferenceValueLeaf<'_> {
+ #[inline]
+ fn from(value: u128) -> Self {
+ ReferenceValueLeaf::U128(value)
+ }
+}
+
impl From<i64> for ReferenceValueLeaf<'_> {
#[inline]
fn from(value: i64) -> Self {
@@ -199,6 +214,7 @@ impl<'a, T: Value<'a> + ?Sized> From<ReferenceValueLeaf<'a>> for ReferenceValue<
ReferenceValueLeaf::Null => ReferenceValue::Leaf(ReferenceValueLeaf::Null),
ReferenceValueLeaf::Str(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Str(val)),
ReferenceValueLeaf::U64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::U64(val)),
+ ReferenceValueLeaf::U128(val) => ReferenceValue::Leaf(ReferenceValueLeaf::U128(val)),
ReferenceValueLeaf::I64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::I64(val)),
ReferenceValueLeaf::F64(val) => ReferenceValue::Leaf(ReferenceValueLeaf::F64(val)),
ReferenceValueLeaf::Date(val) => ReferenceValue::Leaf(ReferenceValueLeaf::Date(val)),
@@ -241,6 +257,16 @@ impl<'a> ReferenceValueLeaf<'a> {
}
}
+ #[inline]
+ /// If the Value is a u128, returns the associated u128. Returns None otherwise.
+ pub fn as_u128(&self) -> Option<u128> {
+ if let Self::U128(val) = self {
+ Some(*val)
+ } else {
+ None
+ }
+ }
+
#[inline]
/// If the Value is a i64, returns the associated i64. Returns None otherwise.
pub fn as_i64(&self) -> Option<i64> {
@@ -369,6 +395,12 @@ where
self.as_leaf().and_then(|leaf| leaf.as_u64())
}
+ #[inline]
+ /// If the Value is a u128, returns the associated u128. Returns None otherwise.
+ pub fn as_u128(&self) -> Option<u128> {
+ self.as_leaf().and_then(|leaf| leaf.as_u128())
+ }
+
#[inline]
/// If the Value is a i64, returns the associated i64. Returns None otherwise.
pub fn as_i64(&self) -> Option<i64> {
diff --git a/crates/tantivy/src/schema/field_entry.rs b/crates/tantivy/src/schema/field_entry.rs
index 8d73d772..686d69e1 100644
--- a/crates/tantivy/src/schema/field_entry.rs
+++ b/crates/tantivy/src/schema/field_entry.rs
@@ -40,6 +40,11 @@ impl FieldEntry {
Self::new(field_name, FieldType::U64(int_options))
}
+ /// Creates a new u128 field entry.
+ pub fn new_u128(field_name: String, int_options: NumericOptions) -> FieldEntry {
+ Self::new(field_name, FieldType::U128(int_options))
+ }
+
/// Creates a new i64 field entry.
pub fn new_i64(field_name: String, int_options: NumericOptions) -> FieldEntry {
Self::new(field_name, FieldType::I64(int_options))
@@ -120,6 +125,7 @@ impl FieldEntry {
pub fn is_stored(&self) -> bool {
match self.field_type {
FieldType::U64(ref options)
+ | FieldType::U128(ref options)
| FieldType::I64(ref options)
| FieldType::F64(ref options)
| FieldType::Bool(ref options) => options.is_stored(),
diff --git a/crates/tantivy/src/schema/field_type.rs b/crates/tantivy/src/schema/field_type.rs
index 39eeae85..d2df6c70 100644
--- a/crates/tantivy/src/schema/field_type.rs
+++ b/crates/tantivy/src/schema/field_type.rs
@@ -68,6 +68,8 @@ pub enum Type {
Json = b'j',
/// IpAddr
IpAddr = b'p',
+ /// `u128`
+ U128 = b'U',
}
impl From<ColumnType> for Type {
@@ -78,6 +80,7 @@ impl From<ColumnType> for Type {
ColumnType::F64 => Type::F64,
ColumnType::Bool => Type::Bool,
ColumnType::DateTime => Type::Date,
+ ColumnType::U128 => Type::U128,
ColumnType::Bytes => Type::Bytes,
}
}
@@ -120,6 +123,7 @@ impl Type {
Type::Bytes => "Bytes",
Type::Json => "Json",
Type::IpAddr => "IpAddr",
+ Type::U128 => "U128",
}
}
@@ -137,6 +141,7 @@ impl Type {
b'b' => Some(Type::Bytes),
b'j' => Some(Type::Json),
b'p' => Some(Type::IpAddr),
+ b'U' => Some(Type::U128),
_ => None,
}
}
@@ -167,6 +172,8 @@ pub enum FieldType {
JsonObject(JsonObjectOptions),
/// IpAddr field
IpAddr(IpAddrOptions),
+ /// u128 field
+ U128(NumericOptions),
}
impl FieldType {
@@ -182,6 +189,7 @@ impl FieldType {
FieldType::Bytes(_) => Type::Bytes,
FieldType::JsonObject(_) => Type::Json,
FieldType::IpAddr(_) => Type::IpAddr,
+ FieldType::U128(_) => Type::U128,
}
}
@@ -200,6 +208,7 @@ impl FieldType {
match *self {
FieldType::Str(ref text_options) => text_options.get_indexing_options().is_some(),
FieldType::U64(ref int_options)
+ | FieldType::U128(ref int_options)
| FieldType::I64(ref int_options)
| FieldType::F64(ref int_options)
| FieldType::Bool(ref int_options) => int_options.is_indexed(),
@@ -239,7 +248,8 @@ impl FieldType {
FieldType::U64(ref int_options)
| FieldType::I64(ref int_options)
| FieldType::F64(ref int_options)
- | FieldType::Bool(ref int_options) => int_options.is_columnar(),
+ | FieldType::Bool(ref int_options)
+ | FieldType::U128(ref int_options) => int_options.is_columnar(),
FieldType::Date(ref date_options) => date_options.is_columnar(),
FieldType::IpAddr(ref ip_addr_options) => ip_addr_options.is_columnar(),
FieldType::JsonObject(ref json_object_options) => json_object_options.is_columnar(),
@@ -257,7 +267,8 @@ impl FieldType {
FieldType::U64(ref int_options)
| FieldType::I64(ref int_options)
| FieldType::F64(ref int_options)
- | FieldType::Bool(ref int_options) => int_options.is_row_order(),
+ | FieldType::Bool(ref int_options)
+ | FieldType::U128(ref int_options) => int_options.is_row_order(),
}
}
@@ -271,7 +282,8 @@ impl FieldType {
FieldType::U64(ref int_options)
| FieldType::I64(ref int_options)
| FieldType::F64(ref int_options)
- | FieldType::Bool(ref int_options) => int_options.fieldnorms(),
+ | FieldType::Bool(ref int_options)
+ | FieldType::U128(ref int_options) => int_options.fieldnorms(),
FieldType::Date(ref date_options) => date_options.fieldnorms(),
FieldType::Bytes(ref bytes_options) => bytes_options.fieldnorms(),
FieldType::JsonObject(ref _json_object_options) => false,
@@ -295,7 +307,8 @@ impl FieldType {
FieldType::U64(ref int_options)
| FieldType::I64(ref int_options)
| FieldType::F64(ref int_options)
- | FieldType::Bool(ref int_options) => {
+ | FieldType::Bool(ref int_options)
+ | FieldType::U128(ref int_options) => {
if int_options.is_indexed() {
Some(IndexRecordOption::Basic)
} else {
@@ -363,6 +376,21 @@ impl FieldType {
})
}
}
+ FieldType::U128(opt) => {
+ if opt.should_coerce() {
+ Ok(OwnedValue::U128(field_text.parse().map_err(|_| {
+ ValueParsingError::TypeError {
+ expected: "a u128 or a u128 as string",
+ json: JsonValue::String(field_text),
+ }
+ })?))
+ } else {
+ Err(ValueParsingError::TypeError {
+ expected: "a u128",
+ json: JsonValue::String(field_text),
+ })
+ }
+ }
FieldType::I64(opt) => {
if opt.should_coerce() {
Ok(OwnedValue::I64(field_text.parse().map_err(|_| {
@@ -449,6 +477,16 @@ impl FieldType {
})
}
}
+ FieldType::U128(_) => {
+ if let Some(field_val_u128) = field_val_num.as_u128() {
+ Ok(OwnedValue::U128(field_val_u128))
+ } else {
+ Err(ValueParsingError::OverflowError {
+ expected: "u128",
+ json: JsonValue::Number(field_val_num),
+ })
+ }
+ }
FieldType::F64(_) => {
if let Some(field_val_f64) = field_val_num.as_f64() {
Ok(OwnedValue::F64(field_val_f64))
diff --git a/crates/tantivy/src/schema/mod.rs b/crates/tantivy/src/schema/mod.rs
index 887cf23f..9f747199 100644
--- a/crates/tantivy/src/schema/mod.rs
+++ b/crates/tantivy/src/schema/mod.rs
@@ -159,6 +159,7 @@ pub(crate) fn value_type_to_column_type(typ: Type) -> Option<ColumnType> {
Type::Bool => Some(ColumnType::Bool),
Type::Date => Some(ColumnType::DateTime),
Type::Bytes => Some(ColumnType::Bytes),
+ Type::U128 => Some(ColumnType::U128),
Type::Json | Type::Str | Type::IpAddr => None,
}
}
diff --git a/crates/tantivy/src/schema/schema.rs b/crates/tantivy/src/schema/schema.rs
index 6f485c50..c45bbe93 100644
--- a/crates/tantivy/src/schema/schema.rs
+++ b/crates/tantivy/src/schema/schema.rs
@@ -57,12 +57,26 @@ impl SchemaBuilder {
self.add_field(field_entry)
}
- /// Adds a new i64 field.
+ /// Adds a new u128 field.
/// Returns the associated field handle
///
/// # Panics
///
/// Panics when field already exists.
+ pub fn add_u128_field<T: Into<NumericOptions>>(
+ &mut self,
+ field_name_str: &str,
+ field_options: T,
+ ) -> Field {
+ let field_name = String::from(field_name_str);
+ let field_entry = FieldEntry::new_u128(field_name, field_options.into());
+ self.add_field(field_entry)
+ }
+
+ /// Adds a new i64 field.
+ /// # Panics
+ ///
+ /// Panics when field already exists.
pub fn add_i64_field<T: Into<NumericOptions>>(
&mut self,
field_name_str: &str,
diff --git a/crates/tantivy/src/schema/term.rs b/crates/tantivy/src/schema/term.rs
index ceecd51d..81c89ac4 100644
--- a/crates/tantivy/src/schema/term.rs
+++ b/crates/tantivy/src/schema/term.rs
@@ -71,12 +71,24 @@ impl Term {
term
}
- fn from_columnar_value<T: ColumnarValue>(field: Field, val: &T) -> Term {
+ fn from_columnar_value_64<T: ColumnarValue + MonotonicallyMappableToU64>(
+ field: Field,
+ val: &T,
+ ) -> Term {
let mut term = Self::with_type_and_field(T::to_type(), field);
term.set_u64(val.to_u64());
term
}
+ fn from_columnar_value_128<T: ColumnarValue + MonotonicallyMappableToU128>(
+ field: Field,
+ val: &T,
+ ) -> Term {
+ let mut term = Self::with_type_and_field(T::to_type(), field);
+ term.set_u128(val.to_u128());
+ term
+ }
+
/// Panics when the term is not empty... ie: some value is set.
/// Use `clear_with_field_and_type` in that case.
///
@@ -101,27 +113,32 @@ impl Term {
/// Builds a term given a field, and a `u64`-value
pub fn from_field_u64(field: Field, val: u64) -> Term {
- Term::from_columnar_value(field, &val)
+ Term::from_columnar_value_64(field, &val)
+ }
+
+ /// Builds a term given a field, and a `u128`-value
+ pub fn from_field_u128(field: Field, val: u128) -> Term {
+ Term::from_columnar_value_128(field, &val)
}
/// Builds a term given a field, and a `i64`-value
pub fn from_field_i64(field: Field, val: i64) -> Term {
- Term::from_columnar_value(field, &val)
+ Term::from_columnar_value_64(field, &val)
}
/// Builds a term given a field, and a `f64`-value
pub fn from_field_f64(field: Field, val: f64) -> Term {
- Term::from_columnar_value(field, &val)
+ Term::from_columnar_value_64(field, &val)
}
/// Builds a term given a field, and a `bool`-value
pub fn from_field_bool(field: Field, val: bool) -> Term {
- Term::from_columnar_value(field, &val)
+ Term::from_columnar_value_64(field, &val)
}
/// Builds a term given a field, and a `DateTime` value
pub fn from_field_date(field: Field, val: DateTime) -> Term {
- Term::from_columnar_value(field, &val.truncate(DATE_TIME_PRECISION_INDEXED))
+ Term::from_columnar_value_64(field, &val.truncate(DATE_TIME_PRECISION_INDEXED))
}
/// Builds a term given a field, and a string value
@@ -153,38 +170,50 @@ impl Term {
/// The use of BigEndian has the benefit of preserving
/// the natural order of the values.
pub fn set_u64(&mut self, val: u64) {
- self.set_columnar_value(val);
+ self.set_columnar_value_64(val);
}
/// Sets a `i64` value in the term.
pub fn set_i64(&mut self, val: i64) {
- self.set_columnar_value(val);
+ self.set_columnar_value_64(val);
}
/// Sets a `DateTime` value in the term.
pub fn set_date(&mut self, date: DateTime) {
- self.set_columnar_value(date);
+ self.set_columnar_value_64(date);
}
/// Sets a `f64` value in the term.
pub fn set_f64(&mut self, val: f64) {
- self.set_columnar_value(val);
+ self.set_columnar_value_64(val);
}
/// Sets a `bool` value in the term.
pub fn set_bool(&mut self, val: bool) {
- self.set_columnar_value(val);
+ self.set_columnar_value_64(val);
}
- fn set_columnar_value<T: ColumnarValue>(&mut self, val: T) {
+ /// Sets a `u128` value in the term.
+ pub fn set_u128(&mut self, val: u128) {
+ self.set_columnar_value_128(val);
+ }
+
+ fn set_columnar_value_64<T: ColumnarValue + MonotonicallyMappableToU64>(&mut self, val: T) {
self.set_bytes(val.to_u64().to_be_bytes().as_ref());
}
+ fn set_columnar_value_128<T: ColumnarValue + MonotonicallyMappableToU128>(&mut self, val: T) {
+ self.set_bytes(val.to_u128().to_be_bytes().as_ref());
+ }
+
/// Append a type marker + columnar value to a term.
/// This is used in JSON type to append a columnar value after the path.
///
/// It will not clear existing bytes.
- pub fn append_type_and_columnar_value<T: ColumnarValue>(&mut self, val: T) {
+ pub fn append_type_and_columnar_value_64<T: ColumnarValue + MonotonicallyMappableToU64>(
+ &mut self,
+ val: T,
+ ) {
self.0.push(T::to_type().to_code());
let value = if T::to_type() == Type::Date {
DateTime::from_u64(val.to_u64())
@@ -196,6 +225,15 @@ impl Term {
self.0.extend(value.to_be_bytes().as_ref());
}
+ /// Append a type marker + columnar value to a term.
+ pub fn append_type_and_columnar_value_128<T: ColumnarValue + MonotonicallyMappableToU128>(
+ &mut self,
+ val: T,
+ ) {
+ self.0.push(T::to_type().to_code());
+ self.0.extend(val.to_u128().to_be_bytes().as_ref());
+ }
+
/// Append a string type marker + string to a term.
/// This is used in JSON type to append a str after the path.
///
@@ -341,10 +379,14 @@ where
/// Returns `None` if the term is not of the u64 type, or if the term byte representation
/// is invalid.
pub fn as_u64(&self) -> Option<u64> {
- self.get_columnar_type::<u64>()
+ self.get_columnar_type_64::<u64>()
}
- fn get_columnar_type<T: ColumnarValue>(&self) -> Option<T> {
+ pub fn as_u128(&self) -> Option<u128> {
+ self.get_columnar_type_128::<u128>()
+ }
+
+ fn get_columnar_type_64<T: ColumnarValue + MonotonicallyMappableToU64>(&self) -> Option<T> {
if self.typ() != T::to_type() {
return None;
}
@@ -353,12 +395,21 @@ where
Some(T::from_u64(value_u64))
}
+ fn get_columnar_type_128<T: ColumnarValue + MonotonicallyMappableToU128>(&self) -> Option<T> {
+ if self.typ() != T::to_type() {
+ return None;
+ }
+ let value_bytes = self.value_bytes();
+ let value_u128 = u128::from_be_bytes(value_bytes.try_into().ok()?);
+ Some(T::from_u128(value_u128))
+ }
+
/// Returns the `i64` value stored in a term.
///
/// Returns `None` if the term is not of the i64 type, or if the term byte representation
/// is invalid.
pub fn as_i64(&self) -> Option<i64> {
- self.get_columnar_type::<i64>()
+ self.get_columnar_type_64::<i64>()
}
/// Returns the `f64` value stored in a term.
@@ -366,7 +417,7 @@ where
/// Returns `None` if the term is not of the f64 type, or if the term byte representation
/// is invalid.
pub fn as_f64(&self) -> Option<f64> {
- self.get_columnar_type::<f64>()
+ self.get_columnar_type_64::<f64>()
}
/// Returns the `bool` value stored in a term.
@@ -374,7 +425,7 @@ where
/// Returns `None` if the term is not of the bool type, or if the term byte representation
/// is invalid.
pub fn as_bool(&self) -> Option<bool> {
- self.get_columnar_type::<bool>()
+ self.get_columnar_type_64::<bool>()
}
/// Returns the `Date` value stored in a term.
@@ -382,7 +433,7 @@ where
/// Returns `None` if the term is not of the Date type, or if the term byte representation
/// is invalid.
pub fn as_date(&self) -> Option<DateTime> {
- self.get_columnar_type::<DateTime>()
+ self.get_columnar_type_64::<DateTime>()
}
/// Returns the text associated with the term.
@@ -492,6 +543,9 @@ where
Type::Bytes => {
write_opt(f, self.as_bytes())?;
}
+ Type::U128 => {
+ write_opt(f, self.as_u128())?;
+ }
Type::Json => {
if let Some((path_bytes, sub_value_bytes)) = self.as_json() {
// Remove the JSON_END_OF_PATH byte & convert to utf8.