web search done right
[tantivy] support u128 column values
Mikkel Denkerb81d84762c88
diff --git a/crates/tantivy/src/columnar/columnar/column_type.rs b/crates/tantivy/src/columnar/columnar/column_type.rsindex 37ecb292..5be854fc 100644--- a/crates/tantivy/src/columnar/columnar/column_type.rs+++ b/crates/tantivy/src/columnar/columnar/column_type.rs@@ -98,6 +98,16 @@ impl HasAssociatedColumnType for u64 {}}+impl HasAssociatedColumnType for u128 {+ fn column_type() -> ColumnType {+ ColumnType::U128+ }++ fn default_value() -> Self {+ 0u128+ }+}+impl HasAssociatedColumnType for i64 {fn column_type() -> ColumnType {ColumnType::I64diff --git a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs b/crates/tantivy/src/columnar/columnar/writer/column_operation.rsindex 59f52c90..81869c0b 100644--- a/crates/tantivy/src/columnar/columnar/writer/column_operation.rs+++ b/crates/tantivy/src/columnar/columnar/writer/column_operation.rs@@ -171,7 +171,7 @@ impl SymbolValue for u128 {#[derive(Default)]struct MiniBuffer {- pub bytes: [u8; 17],+ pub bytes: [u8; 18],pub len: u8,}diff --git a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs b/crates/tantivy/src/columnar/columnar/writer/column_writers.rsindex 97f1da0f..06922ca3 100644--- a/crates/tantivy/src/columnar/columnar/writer/column_writers.rs+++ b/crates/tantivy/src/columnar/columnar/writer/column_writers.rs@@ -125,6 +125,7 @@ pub(crate) enum CompatibleNumericalTypes {Dynamic {all_values_within_i64_range: bool,all_values_within_u64_range: bool,+ all_values_within_u128_range: bool,},StaticType(NumericalType),}@@ -134,6 +135,7 @@ impl Default for CompatibleNumericalTypes {CompatibleNumericalTypes::Dynamic {all_values_within_i64_range: true,all_values_within_u64_range: true,+ all_values_within_u128_range: true,}}}@@ -144,11 +146,12 @@ impl CompatibleNumericalTypes {CompatibleNumericalTypes::Dynamic {all_values_within_i64_range,all_values_within_u64_range,+ all_values_within_u128_range,} => match numerical_type {NumericalType::I64 => *all_values_within_i64_range,NumericalType::U64 => *all_values_within_u64_range,NumericalType::F64 => true,- NumericalType::U128 => true,+ NumericalType::U128 => *all_values_within_u128_range,},CompatibleNumericalTypes::StaticType(static_numerical_type) => {*static_numerical_type == numerical_type@@ -161,10 +164,13 @@ impl CompatibleNumericalTypes {CompatibleNumericalTypes::Dynamic {all_values_within_i64_range,all_values_within_u64_range,+ all_values_within_u128_range,} => match numerical_value {NumericalValue::I64(val_i64) => {let value_within_u64_range = val_i64 >= 0i64;+ let value_within_u128_range = val_i64 >= 0i64;*all_values_within_u64_range &= value_within_u64_range;+ *all_values_within_u128_range &= value_within_u128_range;}NumericalValue::U64(val_u64) => {let value_within_i64_range = val_u64 < i64::MAX as u64;@@ -173,9 +179,14 @@ impl CompatibleNumericalTypes {NumericalValue::F64(_) => {*all_values_within_i64_range = false;*all_values_within_u64_range = false;+ *all_values_within_u128_range = false;}- NumericalValue::U128(_) => {- *all_values_within_u64_range = false;+ NumericalValue::U128(val_u128) => {+ let value_within_u64_range = val_u128 < u64::MAX as u128;+ *all_values_within_u64_range &= value_within_u64_range;++ let value_within_i64_range = val_u128 < i64::MAX as u128;+ *all_values_within_i64_range &= value_within_i64_range;}},CompatibleNumericalTypes::StaticType(typ) => {@@ -190,7 +201,7 @@ impl CompatibleNumericalTypes {}pub fn to_numerical_type(self) -> NumericalType {- for numerical_type in [NumericalType::I64, NumericalType::U64] {+ for numerical_type in [NumericalType::I64, NumericalType::U64, NumericalType::U128] {if self.is_type_accepted(numerical_type) {return numerical_type;}@@ -346,7 +357,12 @@ mod tests {#[test]fn test_compatible_numerical_types_static() {- for typ in [NumericalType::I64, NumericalType::I64, NumericalType::F64] {+ for typ in [+ NumericalType::I64,+ NumericalType::U64,+ NumericalType::F64,+ NumericalType::U128,+ ] {let compatible_numerical_types = CompatibleNumericalTypes::StaticType(typ);assert_eq!(compatible_numerical_types.to_numerical_type(), typ);}diff --git a/crates/tantivy/src/columnfield/readers.rs b/crates/tantivy/src/columnfield/readers.rsindex 119f6b6c..66b1d06c 100644--- a/crates/tantivy/src/columnfield/readers.rs+++ b/crates/tantivy/src/columnfield/readers.rs@@ -315,6 +315,13 @@ impl ColumnFieldReaders {pub fn bool(&self, field_name: &str) -> crate::Result<Column<bool>> {self.column(field_name)}++ /// Returns the `u128` columnar field reader reader associated with `field`.+ ///+ /// If `field` is not a u128 columnar field, this method returns an Error.+ pub fn u128(&self, field_name: &str) -> crate::Result<Column<u128>> {+ self.column(field_name)+ }}#[cfg(test)]diff --git a/crates/tantivy/src/core/tests.rs b/crates/tantivy/src/core/tests.rsindex d7645e12..0158282f 100644--- a/crates/tantivy/src/core/tests.rs+++ b/crates/tantivy/src/core/tests.rs@@ -4,7 +4,7 @@ use crate::index::SegmentId;use crate::indexer::{LogMergePolicy, NoMergePolicy};use crate::postings::Postings;use crate::query::TermQuery;-use crate::schema::{Field, IndexRecordOption, Schema, INDEXED, TEXT};+use crate::schema::{Field, IndexRecordOption, Schema, COLUMN, INDEXED, TEXT};use crate::tokenizer::TokenizerManager;use crate::{Directory, DocSet, Index, IndexBuilder, IndexReader, IndexSettings, IndexWriter, ReloadPolicy,@@ -462,7 +462,7 @@ fn test_non_text_json_term_freq_bitpacked() {#[test]fn test_u128_columnar_values() {let mut schema_builder = Schema::builder();- let field = schema_builder.add_u128_field("u128", INDEXED);+ let field = schema_builder.add_u128_field("u128", INDEXED | COLUMN);let schema = schema_builder.build();let index = Index::create_in_ram(schema);let mut writer: IndexWriter = index.writer_for_tests().unwrap();@@ -481,4 +481,6 @@ fn test_u128_columnar_values() {.unwrap();assert_eq!(postings.doc(), 0);assert_eq!(postings.term_freq(), 1u32);+ let column = segment_reader.column_fields().u128("u128").unwrap();+ assert_eq!(column.first(0).unwrap(), 1u128);}