web search done right
[tantivy] support u128 in row order
Mikkel Denkerefbe1db91827
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rsindex 2927a674..32c75d9c 100644--- a/crates/bloom/src/lib.rs+++ b/crates/bloom/src/lib.rs@@ -92,11 +92,19 @@ impl U64BloomFilter {self.bit_vec.set(h % num_bits, true);}+ pub fn insert_u128(&mut self, item: u128) {+ self.insert(item as u64) // TODO: properly support u128+ }+pub fn contains(&self, item: u64) -> bool {let h = Self::hash(item);self.bit_vec[h % self.bit_vec.len()]}+ pub fn contains_u128(&self, item: u128) -> bool {+ self.contains(item as u64) // TODO: properly support u128+ }+pub fn estimate_card(&self) -> u64 {let num_ones = self.bit_vec.count_ones() as u64;diff --git a/crates/core/benches/bitvec_similarity.rs b/crates/core/benches/bitvec_similarity.rsindex 16a8de5e..93ff3c73 100644--- a/crates/core/benches/bitvec_similarity.rs+++ b/crates/core/benches/bitvec_similarity.rs@@ -8,7 +8,7 @@ fn random_bitvec(max_len: usize, max_id: usize) -> BitVec {let mut ranks = Vec::with_capacity(max_len);for _ in 0..max_len {- ranks.push(rng.gen_range(0..max_id) as u64);+ ranks.push(rng.gen_range(0..max_id) as u128);}ranks.sort_unstable();diff --git a/crates/core/examples/bitvec_similarity.rs b/crates/core/examples/bitvec_similarity.rsindex a5d86ad7..756d86d7 100644--- a/crates/core/examples/bitvec_similarity.rs+++ b/crates/core/examples/bitvec_similarity.rs@@ -7,7 +7,7 @@ fn random_bitvec(max_len: usize, max_id: usize) -> BitVec {let mut ranks = Vec::with_capacity(max_len);for _ in 0..max_len {- ranks.push(rng.gen_range(0..max_id) as u64);+ ranks.push(rng.gen_range(0..max_id) as u128);}ranks.sort_unstable();diff --git a/crates/core/src/ampc/dht/key.rs b/crates/core/src/ampc/dht/key.rsindex 4f1acaa3..3fdcf5a0 100644--- a/crates/core/src/ampc/dht/key.rs+++ b/crates/core/src/ampc/dht/key.rs@@ -29,7 +29,7 @@ impl KeyTrait for String {impl KeyTrait for NodeID {fn as_bytes(&self) -> Vec<u8> {- self.as_u64().to_le_bytes().to_vec()+ self.as_u128().to_le_bytes().to_vec()}}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rsindex b39f3934..8058402d 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs@@ -66,7 +66,7 @@ impl CentralityMapper {.iter().map(|node| {let mut hll = HyperLogLog::default();- hll.add(node.as_u64());+ hll.add_u128(node.as_u128());(*node, hll)}).collect(),@@ -77,7 +77,7 @@ impl CentralityMapper {.iter().map(|node| {let mut hll = HyperLogLog::default();- hll.add(node.as_u64());+ hll.add_u128(node.as_u128());(*node, hll)}).collect(),@@ -96,7 +96,7 @@ impl CentralityMapper {.iter().map(|edge| {let mut counter = old_counters.get(&edge.from).cloned().unwrap_or_default();- counter.add(edge.from.as_u64());+ counter.add_u128(edge.from.as_u128());(edge.to, counter)}).collect();@@ -119,7 +119,7 @@ impl CentralityMapper {for (node, upsert_res) in changes {if let UpsertAction::Merged = upsert_res {- new_changed_nodes.insert(node.as_u64());+ new_changed_nodes.insert_u128(node.as_u128());}}}@@ -224,7 +224,7 @@ impl CentralityMapper {let mut changed_nodes = worker.changed_nodes().lock().unwrap();for node in worker.graph().host_nodes() {- changed_nodes.insert(node.as_u64());+ changed_nodes.insert_u128(node.as_u128());batch.push(node);if batch.len() >= batch_size {let update_batch = batch.clone();@@ -270,7 +270,7 @@ impl CentralityMapper {.graph().host_edges().filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))- .filter(|e| changed_nodes.contains(e.from.as_u64()))+ .filter(|e| changed_nodes.contains_u128(e.from.as_u128())){batch.push(edge);if batch.len() >= batch_size {@@ -315,7 +315,7 @@ impl CentralityMapper {.graph().host_nodes().into_iter()- .filter(|n| changed_nodes.contains(n.as_u64()))+ .filter(|n| changed_nodes.contains_u128(n.as_u128())){batch.push(node);if batch.len() >= batch_size {diff --git a/crates/core/src/entrypoint/site_stats.rs b/crates/core/src/entrypoint/site_stats.rsindex 730600a6..d47de179 100644--- a/crates/core/src/entrypoint/site_stats.rs+++ b/crates/core/src/entrypoint/site_stats.rs@@ -136,13 +136,13 @@ impl Site {}#[derive(Clone, Debug, Hash, PartialEq, Eq)]-struct SiteId([u8; 8]);+struct SiteId([u8; 16]);impl SiteId {fn from_url(url: &str) -> Result<Self> {let url = Url::robust_parse(url)?;let node_id = Node::from(&url).into_host().id();- Ok(Self(node_id.as_u64().to_be_bytes()))+ Ok(Self(node_id.as_u128().to_be_bytes()))}}@@ -274,7 +274,7 @@ pub fn run(config: SiteStatsConfig) -> Result<()> {crate::webgraph::centrality::TopNodes::Top(config.top_sites),).into_iter()- .map(|(node, _)| SiteId(node.as_u64().to_be_bytes()))+ .map(|(node, _)| SiteId(node.as_u128().to_be_bytes())).collect();let jobs: Vec<_> = configdiff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rsindex 5a60d43f..571b7a77 100644--- a/crates/core/src/hyperloglog.rs+++ b/crates/core/src/hyperloglog.rs@@ -4385,6 +4385,10 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {self.registers[j] = self.registers[j].max(p as u8);}+ pub fn add_u128(&mut self, item: u128) {+ self.add(item as u64) // TODO: properly support u128+ }+pub fn clear(&mut self) {self.registers.iter_mut().for_each(|r| *r = 0);}diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rsindex 97feb46a..298073d6 100644--- a/crates/core/src/inverted_index/search.rs+++ b/crates/core/src/inverted_index/search.rs@@ -182,9 +182,9 @@ impl InvertedIndex {).unwrap();- let id = doc.get_first(field).unwrap().as_u64().unwrap();+ let id = doc.get_first(field).unwrap().as_u128().unwrap();- if id == u64::MAX {+ if id == u128::MAX {Ok(None)} else {Ok(Some(id.into()))diff --git a/crates/core/src/numericalfield_reader.rs b/crates/core/src/numericalfield_reader.rsindex ace5e941..072b9959 100644--- a/crates/core/src/numericalfield_reader.rs+++ b/crates/core/src/numericalfield_reader.rs@@ -51,6 +51,7 @@ impl NumericalFieldReader {let columnfield_readers = reader.column_fields();let mut u64s = EnumMap::new();+ let mut u128s = EnumMap::new();let mut bytes = EnumMap::new();let mut bools = EnumMap::new();let mut f64s = EnumMap::new();@@ -62,6 +63,11 @@ impl NumericalFieldReader {u64s.insert(field, reader.values);}}+ DataType::U128 => {+ if let Ok(reader) = columnfield_readers.u128(field.name()) {+ u128s.insert(field, reader.values);+ }+ }DataType::F64 => {if let Ok(reader) = columnfield_readers.f64(field.name()) {f64s.insert(field, reader.values);@@ -91,6 +97,7 @@ impl NumericalFieldReader {row_reader: reader.row_fields().clone(),columnar_readers: ColumnarReaders {u64s,+ u128s,bytes,bools,f64s,@@ -110,6 +117,7 @@ impl NumericalFieldReader {#[derive(Clone)]struct ColumnarReaders {u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,+ u128s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u128>>>,f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,bools: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<bool>>>,bytes: EnumMap<NumericalFieldEnum, tantivy::columnar::BytesColumn>,@@ -117,6 +125,7 @@ struct ColumnarReaders {pub enum Value {U64(u64),+ U128(u128),F64(f64),Bytes(Vec<u8>),Bool(bool),@@ -130,6 +139,13 @@ impl Value {}}+ pub fn as_u128(&self) -> Option<u128> {+ match self {+ Value::U128(val) => Some(*val),+ _ => None,+ }+ }+pub fn as_f64(&self) -> Option<f64> {match self {Value::F64(val) => Some(*val),@@ -158,6 +174,12 @@ impl From<u64> for Value {}}+impl From<u128> for Value {+ fn from(val: u128) -> Self {+ Value::U128(val)+ }+}+impl From<f64> for Value {fn from(val: f64) -> Self {Value::F64(val)@@ -219,6 +241,7 @@ impl<'a> FieldReader<'a> {DataType::U64 => row.get_u64(field_id).map(Value::U64),DataType::F64 => row.get_f64(field_id).map(Value::F64),DataType::Bool => row.get_bool(field_id).map(Value::Bool),+ DataType::U128 => row.get_u128(field_id).map(Value::U128),DataType::Bytes => unimplemented!("bytes fields cannot be row oriented"),})} else if field.orientation().contains(Orientation::COLUMNAR) {@@ -230,7 +253,13 @@ impl<'a> FieldReader<'a> {.get_val(self.doc).into(),),-+ DataType::U128 => Some(+ self.columnar_readers+ .u128s+ .get(field)?+ .get_val(self.doc)+ .into(),+ ),DataType::F64 => Some(self.columnar_readers.f64sdiff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rsindex 7bec574c..596ae84d 100644--- a/crates/core/src/ranking/bitvec_similarity.rs+++ b/crates/core/src/ranking/bitvec_similarity.rs@@ -40,6 +40,10 @@ impl VeryJankyBloomFilter {)}+ fn insert_u128(&mut self, item: u128) {+ self.insert(item as u64); // TODO: fix+ }+fn insert(&mut self, item: u64) {let (a, b) = self.hash(&item);@@ -70,11 +74,11 @@ impl VeryJankyBloomFilter {#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]struct Posting {- ranks: Vec<u64>,+ ranks: Vec<u128>,}impl Posting {- fn new(ranks: Vec<u64>) -> Self {+ fn new(ranks: Vec<u128>) -> Self {Self { ranks }}@@ -133,11 +137,11 @@ impl BitVec {ingoing.into_iter()- .map(|nodes| Self::new(nodes.into_iter().map(|n| n.as_u64()).collect()))+ .map(|nodes| Self::new(nodes.into_iter().map(|n| n.as_u128()).collect())).collect()}- pub fn new(mut ranks: Vec<u64>) -> Self {+ pub fn new(mut ranks: Vec<u128>) -> Self {ranks.sort();ranks.dedup();ranks.shrink_to_fit();@@ -146,7 +150,7 @@ impl BitVec {let mut bloom = VeryJankyBloomFilter::new(16);for rank in &ranks {- bloom.insert(*rank);+ bloom.insert_u128(*rank);}let posting = Posting::new(ranks);@@ -194,11 +198,11 @@ mod tests {use super::*;use std::iter::repeat;- fn into_ranks(a: &[bool]) -> Vec<u64> {+ fn into_ranks(a: &[bool]) -> Vec<u128> {a.iter().enumerate().filter(|(_, b)| **b)- .map(|(i, _)| i as u64)+ .map(|(i, _)| i as u128).collect()}diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 25d83520..7b5d9ff4 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -160,7 +160,7 @@ mod tests {graph.raw_ingoing_edges(node, EdgeLimit::Unlimited).into_iter()- .map(|e| e.from.as_u64())+ .map(|e| e.from.as_u128()).collect(),)}diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex 72531bb1..688af169 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -179,7 +179,7 @@ impl LocalRecallRankingWebpage {let host_id = columnfields.get(numerical_field::HostNodeID.into()).unwrap()- .as_u64()+ .as_u128().unwrap().into();diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rsindex aa6842e0..5a2eb1bd 100644--- a/crates/core/src/schema/mod.rs+++ b/crates/core/src/schema/mod.rs@@ -106,6 +106,7 @@ pub fn create_schema() -> tantivy::schema::Schema {let fast = field.as_numerical().expect("Expected fast field");match fast.data_type() {DataType::U64 => builder.add_u64_field(field.name(), options),+ DataType::U128 => builder.add_u128_field(field.name(), options),DataType::F64 => builder.add_f64_field(field.name(), options),DataType::Bool => builder.add_bool_field(field.name(), options),DataType::Bytes => {diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rsindex 09c407a6..3692c1dc 100644--- a/crates/core/src/schema/numerical_field.rs+++ b/crates/core/src/schema/numerical_field.rs@@ -80,7 +80,7 @@ pub trait NumericalField:fn indexing_option(&self) -> IndexingOption {let orientation = self.orientation();match self.data_type() {- DataType::U64 | DataType::Bool | DataType::F64 => {+ DataType::U64 | DataType::Bool | DataType::F64 | DataType::U128 => {let mut opt = NumericOptions::default();if orientation.contains(Orientation::COLUMNAR) {@@ -240,6 +240,7 @@ impl NumericalFieldEnum {pub enum DataType {U64,+ U128,F64,Bool,Bytes,@@ -1061,16 +1062,20 @@ impl NumericalField for HostNodeID {) -> Result<()> {match &webpage.node_id {Some(node_id) => {- doc.add_u64(self.tantivy_field(index.schema_ref()), node_id.as_u64());+ doc.add_u128(self.tantivy_field(index.schema_ref()), node_id.as_u128());}None => {- doc.add_u64(self.tantivy_field(index.schema_ref()), u64::MAX);+ doc.add_u128(self.tantivy_field(index.schema_ref()), u128::MAX);}}Ok(())}+ fn data_type(&self) -> DataType {+ DataType::U128+ }+fn orientation(&self) -> Orientation {Orientation::ROW}diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex 8bb5721a..ce693ddc 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -59,7 +59,7 @@ fn initialize(for node in graph.host_nodes() {let mut counter = HyperLogLog::default();- counter.add(node.as_u64());+ counter.add_u128(node.as_u128());counters.old.insert(node, counter);centralities.insert(node, KahanSum::default());@@ -98,7 +98,7 @@ fn update_changed_counters(.any(|(to, from)| *from > *to){counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.as_u64());+ new_changed_nodes.insert_u128(edge.to.as_u128());new_exact_changed_nodes.insert(edge.to);@@ -130,7 +130,7 @@ fn update_all_counters(.host_edges().filter(|e| !e.rel_flags.intersects(*SKIPPED_REL)).for_each(|edge| {- if changed_nodes.contains(edge.from.as_u64()) {+ if changed_nodes.contains_u128(edge.from.as_u128()) {if let (Some(counter_to), Some(counter_from)) =(counters.new.get_mut(&edge.to), counters.old.get(&edge.from)){@@ -141,7 +141,7 @@ fn update_all_counters(.any(|(to, from)| *from > *to){counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.as_u64());+ new_changed_nodes.insert_u128(edge.to.as_u128());if let Some(exact_changed_nodes) = &mut exact_changed_nodes {exact_changed_nodes.insert(edge.to);@@ -221,7 +221,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {let mut changed_nodes = U64BloomFilter::new(num_nodes, 0.05);for node in graph.host_nodes() {- changed_nodes.insert(node.as_u64());+ changed_nodes.insert_u128(node.as_u128());}info!("Found {} nodes in the graph", num_nodes);diff --git a/crates/core/src/webgraph/document.rs b/crates/core/src/webgraph/document.rsindex 873b4cae..f44c6eb2 100644--- a/crates/core/src/webgraph/document.rs+++ b/crates/core/src/webgraph/document.rs@@ -252,6 +252,7 @@ impl<'a> Iterator for FieldsIter<'a> {pub enum ReferenceValue<'a> {Str(&'a str),U64(u64),+ U128(u128),F64(f64),}@@ -267,6 +268,9 @@ impl<'a> tantivy::schema::Value<'a> for ReferenceValue<'a> {ReferenceValue::U64(u) => tantivy::schema::document::ReferenceValue::Leaf(tantivy::schema::document::ReferenceValueLeaf::U64(*u),),+ ReferenceValue::U128(u) => tantivy::schema::document::ReferenceValue::Leaf(+ tantivy::schema::document::ReferenceValueLeaf::U128(*u),+ ),ReferenceValue::F64(f) => tantivy::schema::document::ReferenceValue::Leaf(tantivy::schema::document::ReferenceValueLeaf::F64(*f),),diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex f590c097..46bc2b4d 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -34,31 +34,29 @@ use crate::{intmap, webpage::url_ext::UrlExt};Ord,Hash,)]-pub struct NodeID(u64);+pub struct NodeID(u128);impl NodeID {+ #[cfg(test)]pub fn as_u64(self) -> u64 {- self.0- }-- pub fn from_be_bytes(bytes: [u8; 8]) -> Self {- NodeID(u64::from_be_bytes(bytes))+ self.0 as u64}- pub fn to_be_bytes(self) -> [u8; 8] {- self.0.to_be_bytes()+ pub fn as_u128(self) -> u128 {+ self.0}}impl From<u128> for NodeID {fn from(val: u128) -> Self {- NodeID(val as u64)+ NodeID(val)}}+#[cfg(test)]impl From<u64> for NodeID {fn from(val: u64) -> Self {- NodeID(val)+ NodeID(val as u128)}}@@ -95,7 +93,7 @@ impl intmap::Key for NodeID {}fn modulus_usize(self, rhs: usize) -> usize {- (self.0 % (rhs as u64)) as usize+ (self.0 % (rhs as u128)) as usize}}diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rsindex 0b22772a..04c164d5 100644--- a/crates/core/src/webgraph/query/backlink.rs+++ b/crates/core/src/webgraph/query/backlink.rs@@ -56,7 +56,7 @@ pub fn fetch_small_edges<F: Field>(.segment_reader(doc.segment_ord).segment_id(),);- field_column = Some(segment_column_fields.u64(node_id_field).unwrap());+ field_column = Some(segment_column_fields.u128(node_id_field).unwrap());rel_flags_column = Some(segment_column_fields.u64(RelFlags).unwrap());score_column = Some(segment_column_fields.f64(SortScore).unwrap());}diff --git a/crates/core/src/webgraph/query/collector/fast_count.rs b/crates/core/src/webgraph/query/collector/fast_count.rsindex 3a6ec69d..f5bf188a 100644--- a/crates/core/src/webgraph/query/collector/fast_count.rs+++ b/crates/core/src/webgraph/query/collector/fast_count.rs@@ -21,13 +21,13 @@ use crate::ampc::dht::ShardId;use super::Collector;pub enum FastCountValue {- U64(u64),+ U128(u128),}impl FastCountValue {fn as_term(&self, field: tantivy::schema::Field) -> tantivy::schema::Term {match self {- FastCountValue::U64(value) => tantivy::schema::Term::from_field_u64(field, *value),+ FastCountValue::U128(value) => tantivy::schema::Term::from_field_u128(field, *value),}}}diff --git a/crates/core/src/webgraph/query/collector/group_exact.rs b/crates/core/src/webgraph/query/collector/group_exact.rsindex 9a5fcd2f..c6a02536 100644--- a/crates/core/src/webgraph/query/collector/group_exact.rs+++ b/crates/core/src/webgraph/query/collector/group_exact.rs@@ -57,7 +57,7 @@ impl GroupExactCollector {}impl Collector for GroupExactCollector {- type Fruit = FxHashMap<u64, FxHashSet<u64>>;+ type Fruit = FxHashMap<u128, FxHashSet<u128>>;type Child = GroupExactSegmentCollector;fn for_segment(@@ -71,12 +71,12 @@ impl Collector for GroupExactCollector {let group = warmed_column_fields.segment(&segment.segment_id())- .u64(self.group_field)+ .u128(self.group_field).ok_or(anyhow!("Group field missing from index"))?;let value = warmed_column_fields.segment(&segment.segment_id())- .u64(self.value_field)+ .u128(self.value_field).ok_or(anyhow!("Value field missing from index"))?;let filter = self@@ -96,7 +96,7 @@ impl Collector for GroupExactCollector {&self,segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,) -> crate::Result<Self::Fruit> {- let mut groups: FxHashMap<u64, FxHashSet<u64>> = FxHashMap::default();+ let mut groups: FxHashMap<u128, FxHashSet<u128>> = FxHashMap::default();for fruit in segment_fruits {for (group, set) in fruit {@@ -109,14 +109,14 @@ impl Collector for GroupExactCollector {}pub struct GroupExactSegmentCollector {- group: Column<u64>,- value: Column<u64>,- groups: FxHashMap<u64, FxHashSet<u64>>,+ group: Column<u128>,+ value: Column<u128>,+ groups: FxHashMap<u128, FxHashSet<u128>>,filter: Option<Box<dyn SegmentColumnFieldFilter>>,}impl tantivy::collector::SegmentCollector for GroupExactSegmentCollector {- type Fruit = FxHashMap<u64, FxHashSet<u64>>;+ type Fruit = FxHashMap<u128, FxHashSet<u128>>;fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {if let Some(filter) = &self.filter {diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rsindex e5f24e48..08e37792 100644--- a/crates/core/src/webgraph/query/collector/group_sketch.rs+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs@@ -60,7 +60,7 @@ impl GroupSketchCollector {}impl Collector for GroupSketchCollector {- type Fruit = FxHashMap<u64, HyperLogLog<4096>>;+ type Fruit = FxHashMap<u128, HyperLogLog<4096>>;type Child = GroupSketchSegmentCollector;fn for_segment(@@ -74,12 +74,12 @@ impl Collector for GroupSketchCollector {let group = warmed_column_fields.segment(&segment.segment_id())- .u64(self.group_field)+ .u128(self.group_field).ok_or(anyhow!("Group field missing from index"))?;let value = warmed_column_fields.segment(&segment.segment_id())- .u64(self.value_field)+ .u128(self.value_field).ok_or(anyhow!("Value field missing from index"))?;let filter = self@@ -99,7 +99,7 @@ impl Collector for GroupSketchCollector {&self,segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,) -> crate::Result<Self::Fruit> {- let mut groups: FxHashMap<u64, HyperLogLog<4096>> = FxHashMap::default();+ let mut groups: FxHashMap<u128, HyperLogLog<4096>> = FxHashMap::default();for fruit in segment_fruits {for (group, hll) in fruit {@@ -112,14 +112,14 @@ impl Collector for GroupSketchCollector {}pub struct GroupSketchSegmentCollector {- group: Column<u64>,- value: Column<u64>,- groups: FxHashMap<u64, HyperLogLog<4096>>,+ group: Column<u128>,+ value: Column<u128>,+ groups: FxHashMap<u128, HyperLogLog<4096>>,filter: Option<Box<dyn SegmentColumnFieldFilter>>,}impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {- type Fruit = FxHashMap<u64, HyperLogLog<4096>>;+ type Fruit = FxHashMap<u128, HyperLogLog<4096>>;fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {if let Some(filter) = &self.filter {@@ -131,7 +131,7 @@ impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {let group = self.group.first(doc).unwrap();let value = self.value.first(doc).unwrap();- self.groups.entry(group).or_default().add(value);+ self.groups.entry(group).or_default().add_u128(value);}fn harvest(self) -> Self::Fruit {diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rsindex f9babee7..d22ed9a4 100644--- a/crates/core/src/webgraph/query/collector/top_docs.rs+++ b/crates/core/src/webgraph/query/collector/top_docs.rs@@ -96,7 +96,7 @@ impl Deduplicator for NoDeduplicator {)]pub struct DocAddressWithHost {pub address: DocAddress,- pub host: u64,+ pub host: u128,}impl DeduplicatorDoc for DocAddressWithHost {@@ -322,7 +322,7 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDoscorer,host_column: column_fields.host_field- .map(|host_field| segment_column_fields.u64_by_enum(host_field).unwrap()),+ .map(|host_field| segment_column_fields.u128_by_enum(host_field).unwrap()),filter: self.filter.as_ref()@@ -416,7 +416,7 @@ pub struct TopDocsSegmentCollector<S: DocumentScorer, D: Deduplicator> {computer: Computer<D>,segment_ord: SegmentOrdinal,scorer: S,- host_column: Option<Column<u64>>,+ host_column: Option<Column<u128>>,filter: Option<Box<dyn SegmentColumnFieldFilter>>,_deduplicator: PhantomData<D>,}diff --git a/crates/core/src/webgraph/query/degree.rs b/crates/core/src/webgraph/query/degree.rsindex ec8947e0..8a021c9b 100644--- a/crates/core/src/webgraph/query/degree.rs+++ b/crates/core/src/webgraph/query/degree.rs@@ -48,7 +48,7 @@ impl Query for InDegreeQuery {fn collector(&self, searcher: &Searcher) -> Self::Collector {FastCountCollector::new(ToId.name().to_string(),- FastCountValue::U64(self.node.as_u64()),+ FastCountValue::U128(self.node.as_u128()),).with_shard_id(searcher.shard())}@@ -56,7 +56,7 @@ impl Query for InDegreeQuery {fn remote_collector(&self) -> Self::Collector {FastCountCollector::new(ToId.name().to_string(),- FastCountValue::U64(self.node.as_u64()),+ FastCountValue::U128(self.node.as_u128()),)}diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rsindex 2eedb609..4cfa9515 100644--- a/crates/core/src/webgraph/query/group_by.rs+++ b/crates/core/src/webgraph/query/group_by.rs@@ -93,8 +93,8 @@ impl HostGroupSketchQuery {impl Query for HostGroupSketchQuery {type Collector = GroupSketchCollector;type TantivyQuery = Box<dyn tantivy::query::Query>;- type IntermediateOutput = FxHashMap<u64, HyperLogLog<4096>>;- type Output = FxHashMap<u64, HyperLogLog<4096>>;+ type IntermediateOutput = FxHashMap<u128, HyperLogLog<4096>>;+ type Output = FxHashMap<u128, HyperLogLog<4096>>;fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {let mut raw: Self::TantivyQuery = match self.node {@@ -211,8 +211,8 @@ impl HostGroupQuery {impl Query for HostGroupQuery {type Collector = GroupExactCollector;type TantivyQuery = Box<dyn tantivy::query::Query>;- type IntermediateOutput = FxHashMap<u64, FxHashSet<u64>>;- type Output = FxHashMap<u64, FxHashSet<u64>>;+ type IntermediateOutput = FxHashMap<u128, FxHashSet<u128>>;+ type Output = FxHashMap<u128, FxHashSet<u128>>;fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {let mut raw: Self::TantivyQuery = match self.node {@@ -287,7 +287,7 @@ mod tests {let result = graph.search(&query).unwrap();assert_eq!(result.len(), 1);- assert_eq!(result.get(&id.as_u64()).unwrap().size(), 3);+ assert_eq!(result.get(&id.as_u128()).unwrap().size(), 3);}#[test]@@ -299,6 +299,6 @@ mod tests {let result = graph.search(&query).unwrap();assert_eq!(result.len(), 1);- assert_eq!(result.get(&id.as_u64()).unwrap().len(), 3);+ assert_eq!(result.get(&id.as_u128()).unwrap().len(), 3);}}diff --git a/crates/core/src/webgraph/query/raw/id2node.rs b/crates/core/src/webgraph/query/raw/id2node.rsindex a21b0885..c4ebc955 100644--- a/crates/core/src/webgraph/query/raw/id2node.rs+++ b/crates/core/src/webgraph/query/raw/id2node.rs@@ -58,7 +58,7 @@ impl Weight for Id2NodeWeight {let field = reader.schema().get_field(field.name())?;let inverted_index = reader.inverted_index(field)?;- let term = Term::from_field_u64(field, self.node.as_u64());+ let term = Term::from_field_u128(field, self.node.as_u128());if let Some(postings) =inverted_index.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?diff --git a/crates/core/src/webgraph/query/raw/links.rs b/crates/core/src/webgraph/query/raw/links.rsindex 6bff9554..eebddd53 100644--- a/crates/core/src/webgraph/query/raw/links.rs+++ b/crates/core/src/webgraph/query/raw/links.rs@@ -88,14 +88,14 @@ impl tantivy::query::Weight for LinksWeight {) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {let schema = reader.schema();let field = schema.get_field(self.field.name())?;- let term = tantivy::Term::from_field_u64(field, self.node.as_u64());+ let term = tantivy::Term::from_field_u128(field, self.node.as_u128());match LinksScorer::new(reader,term,self.deduplication_field,&self.warmed_column_fields,- self.node.as_u64(),+ self.node.as_u128(),self.skip_self_links,) {Ok(Some(scorer)) => Ok(Box::new(scorer)),@@ -114,9 +114,9 @@ impl tantivy::query::Weight for LinksWeight {struct LinksScorer {postings: SegmentPostings,- dedup_column: Option<Column<u64>>,- last_dedup_val: Option<u64>,- self_dedup_val: u64,+ dedup_column: Option<Column<u128>>,+ last_dedup_val: Option<u128>,+ self_dedup_val: u128,skip_self_links: bool,}@@ -126,13 +126,13 @@ impl LinksScorer {term: tantivy::Term,deduplication_field: Option<FieldEnum>,warmed_column_fields: &WarmedColumnFields,- self_dedup_val: u64,+ self_dedup_val: u128,skip_self_links: bool,) -> tantivy::Result<Option<Self>> {let dedup_column = deduplication_field.map(|f| {warmed_column_fields.segment(&reader.segment_id())- .u64(f)+ .u128(f).unwrap()});@@ -175,7 +175,7 @@ impl LinksScorer {}impl LinksScorer {- fn dedup_val(&self, doc: tantivy::DocId) -> Option<u64> {+ fn dedup_val(&self, doc: tantivy::DocId) -> Option<u128> {if doc == tantivy::TERMINATED {return None;}diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rsindex d09ae463..87078974 100644--- a/crates/core/src/webgraph/schema.rs+++ b/crates/core/src/webgraph/schema.rs@@ -136,7 +136,7 @@ impl Field for FromId {}fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {- ReferenceValue::U64(edge.from.id().as_u64())+ ReferenceValue::U128(edge.from.id().as_u128())}fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {@@ -144,7 +144,7 @@ impl Field for FromId {}fn field_type(&self) -> FieldType {- FieldType::U64(+ FieldType::U128(NumericOptions::default().set_indexed().set_stored()@@ -161,7 +161,7 @@ impl Field for ToId {}fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {- ReferenceValue::U64(edge.to.id().as_u64())+ ReferenceValue::U128(edge.to.id().as_u128())}fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {@@ -169,7 +169,7 @@ impl Field for ToId {}fn field_type(&self) -> FieldType {- FieldType::U64(+ FieldType::U128(NumericOptions::default().set_indexed().set_stored()@@ -186,7 +186,7 @@ impl Field for FromHostId {}fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {- ReferenceValue::U64(edge.from.clone().into_host().id().as_u64())+ ReferenceValue::U128(edge.from.clone().into_host().id().as_u128())}fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {@@ -194,7 +194,7 @@ impl Field for FromHostId {}fn field_type(&self) -> FieldType {- FieldType::U64(+ FieldType::U128(NumericOptions::default().set_indexed().set_stored()@@ -211,7 +211,7 @@ impl Field for ToHostId {}fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {- ReferenceValue::U64(edge.to.clone().into_host().id().as_u64())+ ReferenceValue::U128(edge.to.clone().into_host().id().as_u128())}fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {@@ -219,7 +219,7 @@ impl Field for ToHostId {}fn field_type(&self) -> FieldType {- FieldType::U64(+ FieldType::U128(NumericOptions::default().set_indexed().set_stored()diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 5c390010..1e171abc 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -356,9 +356,9 @@ impl EdgeStore {}pub struct SmallSegmentEdgesIter {- from_id: Column,- to_id: Column,- rel_flags: Column,+ from_id: Column<u128>,+ to_id: Column<u128>,+ rel_flags: Column<u64>,doc_range: Range<DocId>,current_doc: DocId,}@@ -382,8 +382,8 @@ impl SmallSegmentEdgesIter {}Self {- from_id: columns.u64(from_id).unwrap(),- to_id: columns.u64(to_id).unwrap(),+ from_id: columns.u128(from_id).unwrap(),+ to_id: columns.u128(to_id).unwrap(),rel_flags: columns.u64(schema::RelFlags).unwrap(),current_doc: doc_range.start,doc_range,diff --git a/crates/core/src/webgraph/warmed_column_fields.rs b/crates/core/src/webgraph/warmed_column_fields.rsindex 5da45820..8bcc572e 100644--- a/crates/core/src/webgraph/warmed_column_fields.rs+++ b/crates/core/src/webgraph/warmed_column_fields.rs@@ -24,6 +24,7 @@ use crate::{enum_map::EnumMap, Result};#[derive(Debug, Clone)]pub struct SegmentColumnFields {u64_fields: EnumMap<FieldEnumDiscriminants, Column<u64>>,+ u128_fields: EnumMap<FieldEnumDiscriminants, Column<u128>>,f64_fields: EnumMap<FieldEnumDiscriminants, Column<f64>>,}@@ -38,6 +39,16 @@ impl SegmentColumnFields {.cloned()}+ pub fn u128<F: Field>(&self, field: F) -> Option<Column<u128>> {+ self.u128_by_enum(field.into())+ }++ pub fn u128_by_enum(&self, field: FieldEnum) -> Option<Column<u128>> {+ self.u128_fields+ .get(FieldEnumDiscriminants::from(field))+ .cloned()+ }+pub fn f64<F: Field>(&self, field: F) -> Option<Column<f64>> {self.f64_by_enum(field.into())}@@ -59,6 +70,7 @@ impl WarmedColumnFields {let mut segments = HashMap::new();for segment in tantivy_searcher.segment_readers() {let mut u64_fields = EnumMap::new();+ let mut u128_fields = EnumMap::new();let mut f64_fields = EnumMap::new();for field in FieldEnum::iter() {match field.field_type() {@@ -68,6 +80,12 @@ impl WarmedColumnFields {let column = segment.column_fields().u64(field.name())?;u64_fields.insert(field.into(), column);}+ tantivy::schema::FieldType::U128(numeric_options)+ if numeric_options.is_columnar() =>+ {+ let column = segment.column_fields().u128(field.name())?;+ u128_fields.insert(field.into(), column);+ }tantivy::schema::FieldType::F64(numeric_options)if numeric_options.is_columnar() =>{@@ -81,6 +99,7 @@ impl WarmedColumnFields {segment.segment_id(),SegmentColumnFields {u64_fields,+ u128_fields,f64_fields,},);diff --git a/crates/tantivy/src/roworder/mod.rs b/crates/tantivy/src/roworder/mod.rsindex 479d052c..5f335268 100644--- a/crates/tantivy/src/roworder/mod.rs+++ b/crates/tantivy/src/roworder/mod.rs@@ -151,6 +151,7 @@ impl Header {#[strum_discriminants(name(RowValueType))]pub enum RowValue {U64(u64),+ U128(u128),I64(i64),F64(f64),Bool(bool),@@ -160,6 +161,7 @@ impl RowValueType {const fn size(&self) -> usize {match self {RowValueType::U64 => 8,+ RowValueType::U128 => 16,RowValueType::F64 => 8,RowValueType::I64 => 8,RowValueType::Bool => 1,@@ -172,6 +174,7 @@ impl RowValueType {RowValueType::F64 => 1,RowValueType::I64 => 2,RowValueType::Bool => 3,+ RowValueType::U128 => 4,}}@@ -181,6 +184,7 @@ impl RowValueType {1 => RowValueType::F64,2 => RowValueType::I64,3 => RowValueType::Bool,+ 4 => RowValueType::U128,_ => panic!("Invalid id"),}}@@ -201,6 +205,9 @@ impl RowValue {RowValue::Bool(v) => {buf[0] = *v as u8;}+ RowValue::U128(v) => {+ buf.copy_from_slice(&v.to_le_bytes());+ }}}@@ -223,6 +230,12 @@ impl RowValue {}RowValueType::Bool => RowValue::Bool(buf[0] != 0),++ RowValueType::U128 => {+ let mut bytes = [0u8; 16];+ bytes.copy_from_slice(&buf[..typ.size()]);+ RowValue::U128(u128::from_le_bytes(bytes))+ }}}@@ -253,6 +266,13 @@ impl RowValue {_ => None,}}++ pub fn as_u128(&self) -> Option<u128> {+ match self {+ RowValue::U128(v) => Some(*v),+ _ => None,+ }+ }}pub struct Row {@@ -305,6 +325,10 @@ impl Row {self.get(field).and_then(|v| v.as_u64())}+ pub fn get_u128<F: AsFieldId>(&self, field: &F) -> Option<u128> {+ self.get(field).and_then(|v| v.as_u128())+ }+pub fn get_i64<F: AsFieldId>(&self, field: &F) -> Option<i64> {self.get(field).and_then(|v| v.as_i64())}diff --git a/crates/tantivy/src/roworder/writer.rs b/crates/tantivy/src/roworder/writer.rsindex 17666934..8727eb62 100644--- a/crates/tantivy/src/roworder/writer.rs+++ b/crates/tantivy/src/roworder/writer.rs@@ -29,12 +29,12 @@ impl TryFrom<(crate::schema::Field, &crate::schema::FieldEntry)> for Field {crate::schema::FieldType::I64(_) => RowValueType::I64,crate::schema::FieldType::F64(_) => RowValueType::F64,crate::schema::FieldType::Bool(_) => RowValueType::Bool,+ crate::schema::FieldType::U128(_) => RowValueType::U128,crate::schema::FieldType::Str(_)| crate::schema::FieldType::Date(_)| crate::schema::FieldType::Bytes(_)| crate::schema::FieldType::JsonObject(_)- | crate::schema::FieldType::IpAddr(_)- | crate::schema::FieldType::U128(_) => {+ | crate::schema::FieldType::IpAddr(_) => {return Err(crate::TantivyError::SchemaError(format!("Field {:?} is not supported in row order",entry@@ -96,17 +96,31 @@ impl RowFieldsWriter {match value_access.as_value() {ReferenceValue::Leaf(leaf) => match leaf {- ReferenceValueLeaf::Null- | ReferenceValueLeaf::Str(_)- | ReferenceValueLeaf::Date(_)- | ReferenceValueLeaf::Bytes(_)- | ReferenceValueLeaf::IpAddr(_)- | ReferenceValueLeaf::U128(_)- | ReferenceValueLeaf::PreTokStr(_) => {}+ ReferenceValueLeaf::Null => {+ unimplemented!("Null values are not supported in row order")+ }+ ReferenceValueLeaf::Date(_) => {+ unimplemented!("Date values are not supported in row order")+ }+ ReferenceValueLeaf::Bytes(_) => {+ unimplemented!("Bytes values are not supported in row order")+ }+ ReferenceValueLeaf::IpAddr(_) => {+ unimplemented!("IpAddr values are not supported in row order")+ }+ ReferenceValueLeaf::PreTokStr(_) => {+ unimplemented!("PreTokStr values are not supported in row order")+ }+ ReferenceValueLeaf::Str(_) => {+ unimplemented!("String values are not supported in row order")+ }ReferenceValueLeaf::U64(val) => {buf.insert(field.field_id(), RowValue::U64(val));}+ ReferenceValueLeaf::U128(val) => {+ buf.insert(field.field_id(), RowValue::U128(val));+ }ReferenceValueLeaf::I64(val) => {buf.insert(field.field_id(), RowValue::I64(val));}diff --git a/crates/tantivy/src/schema/document/de.rs b/crates/tantivy/src/schema/document/de.rsindex cf398e9f..6608c273 100644--- a/crates/tantivy/src/schema/document/de.rs+++ b/crates/tantivy/src/schema/document/de.rs@@ -366,6 +366,7 @@ wherelet value_type = match type_code {type_codes::TEXT_CODE => ValueType::String,type_codes::U64_CODE => ValueType::U64,+ type_codes::U128_CODE => ValueType::U128,type_codes::I64_CODE => ValueType::I64,type_codes::F64_CODE => ValueType::F64,type_codes::BOOL_CODE => ValueType::Bool,