Skip to content

web search done right

[tantivy] support u128 column values

Mikkel Denkerb81d84762c88
diff --git a/crates/tantivy/src/columnar/columnar/column_type.rs b/crates/tantivy/src/columnar/columnar/column_type.rs
index 37ecb292..5be854fc 100644
--- a/crates/tantivy/src/columnar/columnar/column_type.rs
+++ b/crates/tantivy/src/columnar/columnar/column_type.rs
@@ -98,6 +98,16 @@ impl HasAssociatedColumnType for u64 {
}
}
+impl HasAssociatedColumnType for u128 {
+ fn column_type() -> ColumnType {
+ ColumnType::U128
+ }
+
+ fn default_value() -> Self {
+ 0u128
+ }
+}
+
impl HasAssociatedColumnType for i64 {
fn column_type() -> ColumnType {
ColumnType::I64
diff --git a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs b/crates/tantivy/src/columnar/columnar/writer/column_operation.rs
index 59f52c90..81869c0b 100644
--- a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/column_operation.rs
@@ -171,7 +171,7 @@ impl SymbolValue for u128 {
#[derive(Default)]
struct MiniBuffer {
- pub bytes: [u8; 17],
+ pub bytes: [u8; 18],
pub len: u8,
}
diff --git a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
index 97f1da0f..06922ca3 100644
--- a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
+++ b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs
@@ -125,6 +125,7 @@ pub(crate) enum CompatibleNumericalTypes {
Dynamic {
all_values_within_i64_range: bool,
all_values_within_u64_range: bool,
+ all_values_within_u128_range: bool,
},
StaticType(NumericalType),
}
@@ -134,6 +135,7 @@ impl Default for CompatibleNumericalTypes {
CompatibleNumericalTypes::Dynamic {
all_values_within_i64_range: true,
all_values_within_u64_range: true,
+ all_values_within_u128_range: true,
}
}
}
@@ -144,11 +146,12 @@ impl CompatibleNumericalTypes {
CompatibleNumericalTypes::Dynamic {
all_values_within_i64_range,
all_values_within_u64_range,
+ all_values_within_u128_range,
} => match numerical_type {
NumericalType::I64 => *all_values_within_i64_range,
NumericalType::U64 => *all_values_within_u64_range,
NumericalType::F64 => true,
- NumericalType::U128 => true,
+ NumericalType::U128 => *all_values_within_u128_range,
},
CompatibleNumericalTypes::StaticType(static_numerical_type) => {
*static_numerical_type == numerical_type
@@ -161,10 +164,13 @@ impl CompatibleNumericalTypes {
CompatibleNumericalTypes::Dynamic {
all_values_within_i64_range,
all_values_within_u64_range,
+ all_values_within_u128_range,
} => match numerical_value {
NumericalValue::I64(val_i64) => {
let value_within_u64_range = val_i64 >= 0i64;
+ let value_within_u128_range = val_i64 >= 0i64;
*all_values_within_u64_range &= value_within_u64_range;
+ *all_values_within_u128_range &= value_within_u128_range;
}
NumericalValue::U64(val_u64) => {
let value_within_i64_range = val_u64 < i64::MAX as u64;
@@ -173,9 +179,14 @@ impl CompatibleNumericalTypes {
NumericalValue::F64(_) => {
*all_values_within_i64_range = false;
*all_values_within_u64_range = false;
+ *all_values_within_u128_range = false;
}
- NumericalValue::U128(_) => {
- *all_values_within_u64_range = false;
+ NumericalValue::U128(val_u128) => {
+ let value_within_u64_range = val_u128 < u64::MAX as u128;
+ *all_values_within_u64_range &= value_within_u64_range;
+
+ let value_within_i64_range = val_u128 < i64::MAX as u128;
+ *all_values_within_i64_range &= value_within_i64_range;
}
},
CompatibleNumericalTypes::StaticType(typ) => {
@@ -190,7 +201,7 @@ impl CompatibleNumericalTypes {
}
pub fn to_numerical_type(self) -> NumericalType {
- for numerical_type in [NumericalType::I64, NumericalType::U64] {
+ for numerical_type in [NumericalType::I64, NumericalType::U64, NumericalType::U128] {
if self.is_type_accepted(numerical_type) {
return numerical_type;
}
@@ -346,7 +357,12 @@ mod tests {
#[test]
fn test_compatible_numerical_types_static() {
- for typ in [NumericalType::I64, NumericalType::I64, NumericalType::F64] {
+ for typ in [
+ NumericalType::I64,
+ NumericalType::U64,
+ NumericalType::F64,
+ NumericalType::U128,
+ ] {
let compatible_numerical_types = CompatibleNumericalTypes::StaticType(typ);
assert_eq!(compatible_numerical_types.to_numerical_type(), typ);
}
diff --git a/crates/tantivy/src/columnfield/readers.rs b/crates/tantivy/src/columnfield/readers.rs
index 119f6b6c..66b1d06c 100644
--- a/crates/tantivy/src/columnfield/readers.rs
+++ b/crates/tantivy/src/columnfield/readers.rs
@@ -315,6 +315,13 @@ impl ColumnFieldReaders {
pub fn bool(&self, field_name: &str) -> crate::Result<Column<bool>> {
self.column(field_name)
}
+
+ /// Returns the `u128` columnar field reader reader associated with `field`.
+ ///
+ /// If `field` is not a u128 columnar field, this method returns an Error.
+ pub fn u128(&self, field_name: &str) -> crate::Result<Column<u128>> {
+ self.column(field_name)
+ }
}
#[cfg(test)]
diff --git a/crates/tantivy/src/core/tests.rs b/crates/tantivy/src/core/tests.rs
index d7645e12..0158282f 100644
--- a/crates/tantivy/src/core/tests.rs
+++ b/crates/tantivy/src/core/tests.rs
@@ -4,7 +4,7 @@ use crate::index::SegmentId;
use crate::indexer::{LogMergePolicy, NoMergePolicy};
use crate::postings::Postings;
use crate::query::TermQuery;
-use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, TEXT};
+use crate::schema::{Field, IndexRecordOption, Schema, COLUMN, INDEXED, TEXT};
use crate::tokenizer::TokenizerManager;
use crate::{
Directory, DocSet, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,
@@ -462,7 +462,7 @@ fn test_non_text_json_term_freq_bitpacked() {
#[test]
fn test_u128_columnar_values() {
let mut schema_builder = Schema::builder();
- let field = schema_builder.add_u128_field("u128", INDEXED);
+ let field = schema_builder.add_u128_field("u128", INDEXED | COLUMN);
let schema = schema_builder.build();
let index = Index::create_in_ram(schema);
let mut writer: IndexWriter = index.writer_for_tests().unwrap();
@@ -481,4 +481,6 @@ fn test_u128_columnar_values() {
.unwrap();
assert_eq!(postings.doc(), 0);
assert_eq!(postings.term_freq(), 1u32);
+ let column = segment_reader.column_fields().u128("u128").unwrap();
+ assert_eq!(column.first(0).unwrap(), 1u128);
}