Skip to content

web search done right

[tantivy] support u128 in row order

Mikkel Denkerefbe1db91827
diff --git a/crates/bloom/src/lib.rs b/crates/bloom/src/lib.rs
index 2927a674..32c75d9c 100644
--- a/crates/bloom/src/lib.rs
+++ b/crates/bloom/src/lib.rs
@@ -92,11 +92,19 @@ impl U64BloomFilter {
self.bit_vec.set(h % num_bits, true);
}
+ pub fn insert_u128(&mut self, item: u128) {
+ self.insert(item as u64) // TODO: properly support u128
+ }
+
pub fn contains(&self, item: u64) -> bool {
let h = Self::hash(item);
self.bit_vec[h % self.bit_vec.len()]
}
+ pub fn contains_u128(&self, item: u128) -> bool {
+ self.contains(item as u64) // TODO: properly support u128
+ }
+
pub fn estimate_card(&self) -> u64 {
let num_ones = self.bit_vec.count_ones() as u64;
diff --git a/crates/core/benches/bitvec_similarity.rs b/crates/core/benches/bitvec_similarity.rs
index 16a8de5e..93ff3c73 100644
--- a/crates/core/benches/bitvec_similarity.rs
+++ b/crates/core/benches/bitvec_similarity.rs
@@ -8,7 +8,7 @@ fn random_bitvec(max_len: usize, max_id: usize) -> BitVec {
let mut ranks = Vec::with_capacity(max_len);
for _ in 0..max_len {
- ranks.push(rng.gen_range(0..max_id) as u64);
+ ranks.push(rng.gen_range(0..max_id) as u128);
}
ranks.sort_unstable();
diff --git a/crates/core/examples/bitvec_similarity.rs b/crates/core/examples/bitvec_similarity.rs
index a5d86ad7..756d86d7 100644
--- a/crates/core/examples/bitvec_similarity.rs
+++ b/crates/core/examples/bitvec_similarity.rs
@@ -7,7 +7,7 @@ fn random_bitvec(max_len: usize, max_id: usize) -> BitVec {
let mut ranks = Vec::with_capacity(max_len);
for _ in 0..max_len {
- ranks.push(rng.gen_range(0..max_id) as u64);
+ ranks.push(rng.gen_range(0..max_id) as u128);
}
ranks.sort_unstable();
diff --git a/crates/core/src/ampc/dht/key.rs b/crates/core/src/ampc/dht/key.rs
index 4f1acaa3..3fdcf5a0 100644
--- a/crates/core/src/ampc/dht/key.rs
+++ b/crates/core/src/ampc/dht/key.rs
@@ -29,7 +29,7 @@ impl KeyTrait for String {
impl KeyTrait for NodeID {
fn as_bytes(&self) -> Vec<u8> {
- self.as_u64().to_le_bytes().to_vec()
+ self.as_u128().to_le_bytes().to_vec()
}
}
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
index b39f3934..8058402d 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
@@ -66,7 +66,7 @@ impl CentralityMapper {
.iter()
.map(|node| {
let mut hll = HyperLogLog::default();
- hll.add(node.as_u64());
+ hll.add_u128(node.as_u128());
(*node, hll)
})
.collect(),
@@ -77,7 +77,7 @@ impl CentralityMapper {
.iter()
.map(|node| {
let mut hll = HyperLogLog::default();
- hll.add(node.as_u64());
+ hll.add_u128(node.as_u128());
(*node, hll)
})
.collect(),
@@ -96,7 +96,7 @@ impl CentralityMapper {
.iter()
.map(|edge| {
let mut counter = old_counters.get(&edge.from).cloned().unwrap_or_default();
- counter.add(edge.from.as_u64());
+ counter.add_u128(edge.from.as_u128());
(edge.to, counter)
})
.collect();
@@ -119,7 +119,7 @@ impl CentralityMapper {
for (node, upsert_res) in changes {
if let UpsertAction::Merged = upsert_res {
- new_changed_nodes.insert(node.as_u64());
+ new_changed_nodes.insert_u128(node.as_u128());
}
}
}
@@ -224,7 +224,7 @@ impl CentralityMapper {
let mut changed_nodes = worker.changed_nodes().lock().unwrap();
for node in worker.graph().host_nodes() {
- changed_nodes.insert(node.as_u64());
+ changed_nodes.insert_u128(node.as_u128());
batch.push(node);
if batch.len() >= batch_size {
let update_batch = batch.clone();
@@ -270,7 +270,7 @@ impl CentralityMapper {
.graph()
.host_edges()
.filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))
- .filter(|e| changed_nodes.contains(e.from.as_u64()))
+ .filter(|e| changed_nodes.contains_u128(e.from.as_u128()))
{
batch.push(edge);
if batch.len() >= batch_size {
@@ -315,7 +315,7 @@ impl CentralityMapper {
.graph()
.host_nodes()
.into_iter()
- .filter(|n| changed_nodes.contains(n.as_u64()))
+ .filter(|n| changed_nodes.contains_u128(n.as_u128()))
{
batch.push(node);
if batch.len() >= batch_size {
diff --git a/crates/core/src/entrypoint/site_stats.rs b/crates/core/src/entrypoint/site_stats.rs
index 730600a6..d47de179 100644
--- a/crates/core/src/entrypoint/site_stats.rs
+++ b/crates/core/src/entrypoint/site_stats.rs
@@ -136,13 +136,13 @@ impl Site {
}
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-struct SiteId([u8; 8]);
+struct SiteId([u8; 16]);
impl SiteId {
fn from_url(url: &str) -> Result<Self> {
let url = Url::robust_parse(url)?;
let node_id = Node::from(&url).into_host().id();
- Ok(Self(node_id.as_u64().to_be_bytes()))
+ Ok(Self(node_id.as_u128().to_be_bytes()))
}
}
@@ -274,7 +274,7 @@ pub fn run(config: SiteStatsConfig) -> Result<()> {
crate::webgraph::centrality::TopNodes::Top(config.top_sites),
)
.into_iter()
- .map(|(node, _)| SiteId(node.as_u64().to_be_bytes()))
+ .map(|(node, _)| SiteId(node.as_u128().to_be_bytes()))
.collect();
let jobs: Vec<_> = config
diff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rs
index 5a60d43f..571b7a77 100644
--- a/crates/core/src/hyperloglog.rs
+++ b/crates/core/src/hyperloglog.rs
@@ -4385,6 +4385,10 @@ impl<H: HyperLogLogHasher, const N: usize> HyperLogLog<N, H> {
self.registers[j] = self.registers[j].max(p as u8);
}
+ pub fn add_u128(&mut self, item: u128) {
+ self.add(item as u64) // TODO: properly support u128
+ }
+
pub fn clear(&mut self) {
self.registers.iter_mut().for_each(|r| *r = 0);
}
diff --git a/crates/core/src/inverted_index/search.rs b/crates/core/src/inverted_index/search.rs
index 97feb46a..298073d6 100644
--- a/crates/core/src/inverted_index/search.rs
+++ b/crates/core/src/inverted_index/search.rs
@@ -182,9 +182,9 @@ impl InvertedIndex {
)
.unwrap();
- let id = doc.get_first(field).unwrap().as_u64().unwrap();
+ let id = doc.get_first(field).unwrap().as_u128().unwrap();
- if id == u64::MAX {
+ if id == u128::MAX {
Ok(None)
} else {
Ok(Some(id.into()))
diff --git a/crates/core/src/numericalfield_reader.rs b/crates/core/src/numericalfield_reader.rs
index ace5e941..072b9959 100644
--- a/crates/core/src/numericalfield_reader.rs
+++ b/crates/core/src/numericalfield_reader.rs
@@ -51,6 +51,7 @@ impl NumericalFieldReader {
let columnfield_readers = reader.column_fields();
let mut u64s = EnumMap::new();
+ let mut u128s = EnumMap::new();
let mut bytes = EnumMap::new();
let mut bools = EnumMap::new();
let mut f64s = EnumMap::new();
@@ -62,6 +63,11 @@ impl NumericalFieldReader {
u64s.insert(field, reader.values);
}
}
+ DataType::U128 => {
+ if let Ok(reader) = columnfield_readers.u128(field.name()) {
+ u128s.insert(field, reader.values);
+ }
+ }
DataType::F64 => {
if let Ok(reader) = columnfield_readers.f64(field.name()) {
f64s.insert(field, reader.values);
@@ -91,6 +97,7 @@ impl NumericalFieldReader {
row_reader: reader.row_fields().clone(),
columnar_readers: ColumnarReaders {
u64s,
+ u128s,
bytes,
bools,
f64s,
@@ -110,6 +117,7 @@ impl NumericalFieldReader {
#[derive(Clone)]
struct ColumnarReaders {
u64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u64>>>,
+ u128s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<u128>>>,
f64s: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<f64>>>,
bools: EnumMap<NumericalFieldEnum, Arc<dyn ColumnValues<bool>>>,
bytes: EnumMap<NumericalFieldEnum, tantivy::columnar::BytesColumn>,
@@ -117,6 +125,7 @@ struct ColumnarReaders {
pub enum Value {
U64(u64),
+ U128(u128),
F64(f64),
Bytes(Vec<u8>),
Bool(bool),
@@ -130,6 +139,13 @@ impl Value {
}
}
+ pub fn as_u128(&self) -> Option<u128> {
+ match self {
+ Value::U128(val) => Some(*val),
+ _ => None,
+ }
+ }
+
pub fn as_f64(&self) -> Option<f64> {
match self {
Value::F64(val) => Some(*val),
@@ -158,6 +174,12 @@ impl From<u64> for Value {
}
}
+impl From<u128> for Value {
+ fn from(val: u128) -> Self {
+ Value::U128(val)
+ }
+}
+
impl From<f64> for Value {
fn from(val: f64) -> Self {
Value::F64(val)
@@ -219,6 +241,7 @@ impl<'a> FieldReader<'a> {
DataType::U64 => row.get_u64(field_id).map(Value::U64),
DataType::F64 => row.get_f64(field_id).map(Value::F64),
DataType::Bool => row.get_bool(field_id).map(Value::Bool),
+ DataType::U128 => row.get_u128(field_id).map(Value::U128),
DataType::Bytes => unimplemented!("bytes fields cannot be row oriented"),
})
} else if field.orientation().contains(Orientation::COLUMNAR) {
@@ -230,7 +253,13 @@ impl<'a> FieldReader<'a> {
.get_val(self.doc)
.into(),
),
-
+ DataType::U128 => Some(
+ self.columnar_readers
+ .u128s
+ .get(field)?
+ .get_val(self.doc)
+ .into(),
+ ),
DataType::F64 => Some(
self.columnar_readers
.f64s
diff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rs
index 7bec574c..596ae84d 100644
--- a/crates/core/src/ranking/bitvec_similarity.rs
+++ b/crates/core/src/ranking/bitvec_similarity.rs
@@ -40,6 +40,10 @@ impl VeryJankyBloomFilter {
)
}
+ fn insert_u128(&mut self, item: u128) {
+ self.insert(item as u64); // TODO: fix
+ }
+
fn insert(&mut self, item: u64) {
let (a, b) = self.hash(&item);
@@ -70,11 +74,11 @@ impl VeryJankyBloomFilter {
#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]
struct Posting {
- ranks: Vec<u64>,
+ ranks: Vec<u128>,
}
impl Posting {
- fn new(ranks: Vec<u64>) -> Self {
+ fn new(ranks: Vec<u128>) -> Self {
Self { ranks }
}
@@ -133,11 +137,11 @@ impl BitVec {
ingoing
.into_iter()
- .map(|nodes| Self::new(nodes.into_iter().map(|n| n.as_u64()).collect()))
+ .map(|nodes| Self::new(nodes.into_iter().map(|n| n.as_u128()).collect()))
.collect()
}
- pub fn new(mut ranks: Vec<u64>) -> Self {
+ pub fn new(mut ranks: Vec<u128>) -> Self {
ranks.sort();
ranks.dedup();
ranks.shrink_to_fit();
@@ -146,7 +150,7 @@ impl BitVec {
let mut bloom = VeryJankyBloomFilter::new(16);
for rank in &ranks {
- bloom.insert(*rank);
+ bloom.insert_u128(*rank);
}
let posting = Posting::new(ranks);
@@ -194,11 +198,11 @@ mod tests {
use super::*;
use std::iter::repeat;
- fn into_ranks(a: &[bool]) -> Vec<u64> {
+ fn into_ranks(a: &[bool]) -> Vec<u128> {
a.iter()
.enumerate()
.filter(|(_, b)| **b)
- .map(|(i, _)| i as u64)
+ .map(|(i, _)| i as u128)
.collect()
}
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rs
index 25d83520..7b5d9ff4 100644
--- a/crates/core/src/ranking/inbound_similarity.rs
+++ b/crates/core/src/ranking/inbound_similarity.rs
@@ -160,7 +160,7 @@ mod tests {
graph
.raw_ingoing_edges(node, EdgeLimit::Unlimited)
.into_iter()
- .map(|e| e.from.as_u64())
+ .map(|e| e.from.as_u128())
.collect(),
)
}
diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rs
index 72531bb1..688af169 100644
--- a/crates/core/src/ranking/pipeline/stages/recall.rs
+++ b/crates/core/src/ranking/pipeline/stages/recall.rs
@@ -179,7 +179,7 @@ impl LocalRecallRankingWebpage {
let host_id = columnfields
.get(numerical_field::HostNodeID.into())
.unwrap()
- .as_u64()
+ .as_u128()
.unwrap()
.into();
diff --git a/crates/core/src/schema/mod.rs b/crates/core/src/schema/mod.rs
index aa6842e0..5a2eb1bd 100644
--- a/crates/core/src/schema/mod.rs
+++ b/crates/core/src/schema/mod.rs
@@ -106,6 +106,7 @@ pub fn create_schema() -> tantivy::schema::Schema {
let fast = field.as_numerical().expect("Expected fast field");
match fast.data_type() {
DataType::U64 => builder.add_u64_field(field.name(), options),
+ DataType::U128 => builder.add_u128_field(field.name(), options),
DataType::F64 => builder.add_f64_field(field.name(), options),
DataType::Bool => builder.add_bool_field(field.name(), options),
DataType::Bytes => {
diff --git a/crates/core/src/schema/numerical_field.rs b/crates/core/src/schema/numerical_field.rs
index 09c407a6..3692c1dc 100644
--- a/crates/core/src/schema/numerical_field.rs
+++ b/crates/core/src/schema/numerical_field.rs
@@ -80,7 +80,7 @@ pub trait NumericalField:
fn indexing_option(&self) -> IndexingOption {
let orientation = self.orientation();
match self.data_type() {
- DataType::U64 | DataType::Bool | DataType::F64 => {
+ DataType::U64 | DataType::Bool | DataType::F64 | DataType::U128 => {
let mut opt = NumericOptions::default();
if orientation.contains(Orientation::COLUMNAR) {
@@ -240,6 +240,7 @@ impl NumericalFieldEnum {
pub enum DataType {
U64,
+ U128,
F64,
Bool,
Bytes,
@@ -1061,16 +1062,20 @@ impl NumericalField for HostNodeID {
) -> Result<()> {
match &webpage.node_id {
Some(node_id) => {
- doc.add_u64(self.tantivy_field(index.schema_ref()), node_id.as_u64());
+ doc.add_u128(self.tantivy_field(index.schema_ref()), node_id.as_u128());
}
None => {
- doc.add_u64(self.tantivy_field(index.schema_ref()), u64::MAX);
+ doc.add_u128(self.tantivy_field(index.schema_ref()), u128::MAX);
}
}
Ok(())
}
+ fn data_type(&self) -> DataType {
+ DataType::U128
+ }
+
fn orientation(&self) -> Orientation {
Orientation::ROW
}
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index 8bb5721a..ce693ddc 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -59,7 +59,7 @@ fn initialize(
for node in graph.host_nodes() {
let mut counter = HyperLogLog::default();
- counter.add(node.as_u64());
+ counter.add_u128(node.as_u128());
counters.old.insert(node, counter);
centralities.insert(node, KahanSum::default());
@@ -98,7 +98,7 @@ fn update_changed_counters(
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.as_u64());
+ new_changed_nodes.insert_u128(edge.to.as_u128());
new_exact_changed_nodes.insert(edge.to);
@@ -130,7 +130,7 @@ fn update_all_counters(
.host_edges()
.filter(|e| !e.rel_flags.intersects(*SKIPPED_REL))
.for_each(|edge| {
- if changed_nodes.contains(edge.from.as_u64()) {
+ if changed_nodes.contains_u128(edge.from.as_u128()) {
if let (Some(counter_to), Some(counter_from)) =
(counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
{
@@ -141,7 +141,7 @@ fn update_all_counters(
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.as_u64());
+ new_changed_nodes.insert_u128(edge.to.as_u128());
if let Some(exact_changed_nodes) = &mut exact_changed_nodes {
exact_changed_nodes.insert(edge.to);
@@ -221,7 +221,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {
let mut changed_nodes = U64BloomFilter::new(num_nodes, 0.05);
for node in graph.host_nodes() {
- changed_nodes.insert(node.as_u64());
+ changed_nodes.insert_u128(node.as_u128());
}
info!("Found {} nodes in the graph", num_nodes);
diff --git a/crates/core/src/webgraph/document.rs b/crates/core/src/webgraph/document.rs
index 873b4cae..f44c6eb2 100644
--- a/crates/core/src/webgraph/document.rs
+++ b/crates/core/src/webgraph/document.rs
@@ -252,6 +252,7 @@ impl<'a> Iterator for FieldsIter<'a> {
pub enum ReferenceValue<'a> {
Str(&'a str),
U64(u64),
+ U128(u128),
F64(f64),
}
@@ -267,6 +268,9 @@ impl<'a> tantivy::schema::Value<'a> for ReferenceValue<'a> {
ReferenceValue::U64(u) => tantivy::schema::document::ReferenceValue::Leaf(
tantivy::schema::document::ReferenceValueLeaf::U64(*u),
),
+ ReferenceValue::U128(u) => tantivy::schema::document::ReferenceValue::Leaf(
+ tantivy::schema::document::ReferenceValueLeaf::U128(*u),
+ ),
ReferenceValue::F64(f) => tantivy::schema::document::ReferenceValue::Leaf(
tantivy::schema::document::ReferenceValueLeaf::F64(*f),
),
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index f590c097..46bc2b4d 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -34,31 +34,29 @@ use crate::{intmap, webpage::url_ext::UrlExt};
Ord,
Hash,
)]
-pub struct NodeID(u64);
+pub struct NodeID(u128);
impl NodeID {
+ #[cfg(test)]
pub fn as_u64(self) -> u64 {
- self.0
- }
-
- pub fn from_be_bytes(bytes: [u8; 8]) -> Self {
- NodeID(u64::from_be_bytes(bytes))
+ self.0 as u64
}
- pub fn to_be_bytes(self) -> [u8; 8] {
- self.0.to_be_bytes()
+ pub fn as_u128(self) -> u128 {
+ self.0
}
}
impl From<u128> for NodeID {
fn from(val: u128) -> Self {
- NodeID(val as u64)
+ NodeID(val)
}
}
+#[cfg(test)]
impl From<u64> for NodeID {
fn from(val: u64) -> Self {
- NodeID(val)
+ NodeID(val as u128)
}
}
@@ -95,7 +93,7 @@ impl intmap::Key for NodeID {
}
fn modulus_usize(self, rhs: usize) -> usize {
- (self.0 % (rhs as u64)) as usize
+ (self.0 % (rhs as u128)) as usize
}
}
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index 0b22772a..04c164d5 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -56,7 +56,7 @@ pub fn fetch_small_edges<F: Field>(
.segment_reader(doc.segment_ord)
.segment_id(),
);
- field_column = Some(segment_column_fields.u64(node_id_field).unwrap());
+ field_column = Some(segment_column_fields.u128(node_id_field).unwrap());
rel_flags_column = Some(segment_column_fields.u64(RelFlags).unwrap());
score_column = Some(segment_column_fields.f64(SortScore).unwrap());
}
diff --git a/crates/core/src/webgraph/query/collector/fast_count.rs b/crates/core/src/webgraph/query/collector/fast_count.rs
index 3a6ec69d..f5bf188a 100644
--- a/crates/core/src/webgraph/query/collector/fast_count.rs
+++ b/crates/core/src/webgraph/query/collector/fast_count.rs
@@ -21,13 +21,13 @@ use crate::ampc::dht::ShardId;
use super::Collector;
pub enum FastCountValue {
- U64(u64),
+ U128(u128),
}
impl FastCountValue {
fn as_term(&self, field: tantivy::schema::Field) -> tantivy::schema::Term {
match self {
- FastCountValue::U64(value) => tantivy::schema::Term::from_field_u64(field, *value),
+ FastCountValue::U128(value) => tantivy::schema::Term::from_field_u128(field, *value),
}
}
}
diff --git a/crates/core/src/webgraph/query/collector/group_exact.rs b/crates/core/src/webgraph/query/collector/group_exact.rs
index 9a5fcd2f..c6a02536 100644
--- a/crates/core/src/webgraph/query/collector/group_exact.rs
+++ b/crates/core/src/webgraph/query/collector/group_exact.rs
@@ -57,7 +57,7 @@ impl GroupExactCollector {
}
impl Collector for GroupExactCollector {
- type Fruit = FxHashMap<u64, FxHashSet<u64>>;
+ type Fruit = FxHashMap<u128, FxHashSet<u128>>;
type Child = GroupExactSegmentCollector;
fn for_segment(
@@ -71,12 +71,12 @@ impl Collector for GroupExactCollector {
let group = warmed_column_fields
.segment(&segment.segment_id())
- .u64(self.group_field)
+ .u128(self.group_field)
.ok_or(anyhow!("Group field missing from index"))?;
let value = warmed_column_fields
.segment(&segment.segment_id())
- .u64(self.value_field)
+ .u128(self.value_field)
.ok_or(anyhow!("Value field missing from index"))?;
let filter = self
@@ -96,7 +96,7 @@ impl Collector for GroupExactCollector {
&self,
segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
) -> crate::Result<Self::Fruit> {
- let mut groups: FxHashMap<u64, FxHashSet<u64>> = FxHashMap::default();
+ let mut groups: FxHashMap<u128, FxHashSet<u128>> = FxHashMap::default();
for fruit in segment_fruits {
for (group, set) in fruit {
@@ -109,14 +109,14 @@ impl Collector for GroupExactCollector {
}
pub struct GroupExactSegmentCollector {
- group: Column<u64>,
- value: Column<u64>,
- groups: FxHashMap<u64, FxHashSet<u64>>,
+ group: Column<u128>,
+ value: Column<u128>,
+ groups: FxHashMap<u128, FxHashSet<u128>>,
filter: Option<Box<dyn SegmentColumnFieldFilter>>,
}
impl tantivy::collector::SegmentCollector for GroupExactSegmentCollector {
- type Fruit = FxHashMap<u64, FxHashSet<u64>>;
+ type Fruit = FxHashMap<u128, FxHashSet<u128>>;
fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {
if let Some(filter) = &self.filter {
diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rs
index e5f24e48..08e37792 100644
--- a/crates/core/src/webgraph/query/collector/group_sketch.rs
+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs
@@ -60,7 +60,7 @@ impl GroupSketchCollector {
}
impl Collector for GroupSketchCollector {
- type Fruit = FxHashMap<u64, HyperLogLog<4096>>;
+ type Fruit = FxHashMap<u128, HyperLogLog<4096>>;
type Child = GroupSketchSegmentCollector;
fn for_segment(
@@ -74,12 +74,12 @@ impl Collector for GroupSketchCollector {
let group = warmed_column_fields
.segment(&segment.segment_id())
- .u64(self.group_field)
+ .u128(self.group_field)
.ok_or(anyhow!("Group field missing from index"))?;
let value = warmed_column_fields
.segment(&segment.segment_id())
- .u64(self.value_field)
+ .u128(self.value_field)
.ok_or(anyhow!("Value field missing from index"))?;
let filter = self
@@ -99,7 +99,7 @@ impl Collector for GroupSketchCollector {
&self,
segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
) -> crate::Result<Self::Fruit> {
- let mut groups: FxHashMap<u64, HyperLogLog<4096>> = FxHashMap::default();
+ let mut groups: FxHashMap<u128, HyperLogLog<4096>> = FxHashMap::default();
for fruit in segment_fruits {
for (group, hll) in fruit {
@@ -112,14 +112,14 @@ impl Collector for GroupSketchCollector {
}
pub struct GroupSketchSegmentCollector {
- group: Column<u64>,
- value: Column<u64>,
- groups: FxHashMap<u64, HyperLogLog<4096>>,
+ group: Column<u128>,
+ value: Column<u128>,
+ groups: FxHashMap<u128, HyperLogLog<4096>>,
filter: Option<Box<dyn SegmentColumnFieldFilter>>,
}
impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {
- type Fruit = FxHashMap<u64, HyperLogLog<4096>>;
+ type Fruit = FxHashMap<u128, HyperLogLog<4096>>;
fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {
if let Some(filter) = &self.filter {
@@ -131,7 +131,7 @@ impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {
let group = self.group.first(doc).unwrap();
let value = self.value.first(doc).unwrap();
- self.groups.entry(group).or_default().add(value);
+ self.groups.entry(group).or_default().add_u128(value);
}
fn harvest(self) -> Self::Fruit {
diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rs
index f9babee7..d22ed9a4 100644
--- a/crates/core/src/webgraph/query/collector/top_docs.rs
+++ b/crates/core/src/webgraph/query/collector/top_docs.rs
@@ -96,7 +96,7 @@ impl Deduplicator for NoDeduplicator {
)]
pub struct DocAddressWithHost {
pub address: DocAddress,
- pub host: u64,
+ pub host: u128,
}
impl DeduplicatorDoc for DocAddressWithHost {
@@ -322,7 +322,7 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDo
scorer,
host_column: column_fields
.host_field
- .map(|host_field| segment_column_fields.u64_by_enum(host_field).unwrap()),
+ .map(|host_field| segment_column_fields.u128_by_enum(host_field).unwrap()),
filter: self
.filter
.as_ref()
@@ -416,7 +416,7 @@ pub struct TopDocsSegmentCollector<S: DocumentScorer, D: Deduplicator> {
computer: Computer<D>,
segment_ord: SegmentOrdinal,
scorer: S,
- host_column: Option<Column<u64>>,
+ host_column: Option<Column<u128>>,
filter: Option<Box<dyn SegmentColumnFieldFilter>>,
_deduplicator: PhantomData<D>,
}
diff --git a/crates/core/src/webgraph/query/degree.rs b/crates/core/src/webgraph/query/degree.rs
index ec8947e0..8a021c9b 100644
--- a/crates/core/src/webgraph/query/degree.rs
+++ b/crates/core/src/webgraph/query/degree.rs
@@ -48,7 +48,7 @@ impl Query for InDegreeQuery {
fn collector(&self, searcher: &Searcher) -> Self::Collector {
FastCountCollector::new(
ToId.name().to_string(),
- FastCountValue::U64(self.node.as_u64()),
+ FastCountValue::U128(self.node.as_u128()),
)
.with_shard_id(searcher.shard())
}
@@ -56,7 +56,7 @@ impl Query for InDegreeQuery {
fn remote_collector(&self) -> Self::Collector {
FastCountCollector::new(
ToId.name().to_string(),
- FastCountValue::U64(self.node.as_u64()),
+ FastCountValue::U128(self.node.as_u128()),
)
}
diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rs
index 2eedb609..4cfa9515 100644
--- a/crates/core/src/webgraph/query/group_by.rs
+++ b/crates/core/src/webgraph/query/group_by.rs
@@ -93,8 +93,8 @@ impl HostGroupSketchQuery {
impl Query for HostGroupSketchQuery {
type Collector = GroupSketchCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = FxHashMap<u64, HyperLogLog<4096>>;
- type Output = FxHashMap<u64, HyperLogLog<4096>>;
+ type IntermediateOutput = FxHashMap<u128, HyperLogLog<4096>>;
+ type Output = FxHashMap<u128, HyperLogLog<4096>>;
fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
let mut raw: Self::TantivyQuery = match self.node {
@@ -211,8 +211,8 @@ impl HostGroupQuery {
impl Query for HostGroupQuery {
type Collector = GroupExactCollector;
type TantivyQuery = Box<dyn tantivy::query::Query>;
- type IntermediateOutput = FxHashMap<u64, FxHashSet<u64>>;
- type Output = FxHashMap<u64, FxHashSet<u64>>;
+ type IntermediateOutput = FxHashMap<u128, FxHashSet<u128>>;
+ type Output = FxHashMap<u128, FxHashSet<u128>>;
fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
let mut raw: Self::TantivyQuery = match self.node {
@@ -287,7 +287,7 @@ mod tests {
let result = graph.search(&query).unwrap();
assert_eq!(result.len(), 1);
- assert_eq!(result.get(&id.as_u64()).unwrap().size(), 3);
+ assert_eq!(result.get(&id.as_u128()).unwrap().size(), 3);
}
#[test]
@@ -299,6 +299,6 @@ mod tests {
let result = graph.search(&query).unwrap();
assert_eq!(result.len(), 1);
- assert_eq!(result.get(&id.as_u64()).unwrap().len(), 3);
+ assert_eq!(result.get(&id.as_u128()).unwrap().len(), 3);
}
}
diff --git a/crates/core/src/webgraph/query/raw/id2node.rs b/crates/core/src/webgraph/query/raw/id2node.rs
index a21b0885..c4ebc955 100644
--- a/crates/core/src/webgraph/query/raw/id2node.rs
+++ b/crates/core/src/webgraph/query/raw/id2node.rs
@@ -58,7 +58,7 @@ impl Weight for Id2NodeWeight {
let field = reader.schema().get_field(field.name())?;
let inverted_index = reader.inverted_index(field)?;
- let term = Term::from_field_u64(field, self.node.as_u64());
+ let term = Term::from_field_u128(field, self.node.as_u128());
if let Some(postings) =
inverted_index.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?
diff --git a/crates/core/src/webgraph/query/raw/links.rs b/crates/core/src/webgraph/query/raw/links.rs
index 6bff9554..eebddd53 100644
--- a/crates/core/src/webgraph/query/raw/links.rs
+++ b/crates/core/src/webgraph/query/raw/links.rs
@@ -88,14 +88,14 @@ impl tantivy::query::Weight for LinksWeight {
) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
let schema = reader.schema();
let field = schema.get_field(self.field.name())?;
- let term = tantivy::Term::from_field_u64(field, self.node.as_u64());
+ let term = tantivy::Term::from_field_u128(field, self.node.as_u128());
match LinksScorer::new(
reader,
term,
self.deduplication_field,
&self.warmed_column_fields,
- self.node.as_u64(),
+ self.node.as_u128(),
self.skip_self_links,
) {
Ok(Some(scorer)) => Ok(Box::new(scorer)),
@@ -114,9 +114,9 @@ impl tantivy::query::Weight for LinksWeight {
struct LinksScorer {
postings: SegmentPostings,
- dedup_column: Option<Column<u64>>,
- last_dedup_val: Option<u64>,
- self_dedup_val: u64,
+ dedup_column: Option<Column<u128>>,
+ last_dedup_val: Option<u128>,
+ self_dedup_val: u128,
skip_self_links: bool,
}
@@ -126,13 +126,13 @@ impl LinksScorer {
term: tantivy::Term,
deduplication_field: Option<FieldEnum>,
warmed_column_fields: &WarmedColumnFields,
- self_dedup_val: u64,
+ self_dedup_val: u128,
skip_self_links: bool,
) -> tantivy::Result<Option<Self>> {
let dedup_column = deduplication_field.map(|f| {
warmed_column_fields
.segment(&reader.segment_id())
- .u64(f)
+ .u128(f)
.unwrap()
});
@@ -175,7 +175,7 @@ impl LinksScorer {
}
impl LinksScorer {
- fn dedup_val(&self, doc: tantivy::DocId) -> Option<u64> {
+ fn dedup_val(&self, doc: tantivy::DocId) -> Option<u128> {
if doc == tantivy::TERMINATED {
return None;
}
diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rs
index d09ae463..87078974 100644
--- a/crates/core/src/webgraph/schema.rs
+++ b/crates/core/src/webgraph/schema.rs
@@ -136,7 +136,7 @@ impl Field for FromId {
}
fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
- ReferenceValue::U64(edge.from.id().as_u64())
+ ReferenceValue::U128(edge.from.id().as_u128())
}
fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
@@ -144,7 +144,7 @@ impl Field for FromId {
}
fn field_type(&self) -> FieldType {
- FieldType::U64(
+ FieldType::U128(
NumericOptions::default()
.set_indexed()
.set_stored()
@@ -161,7 +161,7 @@ impl Field for ToId {
}
fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
- ReferenceValue::U64(edge.to.id().as_u64())
+ ReferenceValue::U128(edge.to.id().as_u128())
}
fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
@@ -169,7 +169,7 @@ impl Field for ToId {
}
fn field_type(&self) -> FieldType {
- FieldType::U64(
+ FieldType::U128(
NumericOptions::default()
.set_indexed()
.set_stored()
@@ -186,7 +186,7 @@ impl Field for FromHostId {
}
fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
- ReferenceValue::U64(edge.from.clone().into_host().id().as_u64())
+ ReferenceValue::U128(edge.from.clone().into_host().id().as_u128())
}
fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
@@ -194,7 +194,7 @@ impl Field for FromHostId {
}
fn field_type(&self) -> FieldType {
- FieldType::U64(
+ FieldType::U128(
NumericOptions::default()
.set_indexed()
.set_stored()
@@ -211,7 +211,7 @@ impl Field for ToHostId {
}
fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {
- ReferenceValue::U64(edge.to.clone().into_host().id().as_u64())
+ ReferenceValue::U128(edge.to.clone().into_host().id().as_u128())
}
fn set_value(&self, _: &mut Edge, _: OwnedValue) -> Result<()> {
@@ -219,7 +219,7 @@ impl Field for ToHostId {
}
fn field_type(&self) -> FieldType {
- FieldType::U64(
+ FieldType::U128(
NumericOptions::default()
.set_indexed()
.set_stored()
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 5c390010..1e171abc 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -356,9 +356,9 @@ impl EdgeStore {
}
pub struct SmallSegmentEdgesIter {
- from_id: Column,
- to_id: Column,
- rel_flags: Column,
+ from_id: Column<u128>,
+ to_id: Column<u128>,
+ rel_flags: Column<u64>,
doc_range: Range<DocId>,
current_doc: DocId,
}
@@ -382,8 +382,8 @@ impl SmallSegmentEdgesIter {
}
Self {
- from_id: columns.u64(from_id).unwrap(),
- to_id: columns.u64(to_id).unwrap(),
+ from_id: columns.u128(from_id).unwrap(),
+ to_id: columns.u128(to_id).unwrap(),
rel_flags: columns.u64(schema::RelFlags).unwrap(),
current_doc: doc_range.start,
doc_range,
diff --git a/crates/core/src/webgraph/warmed_column_fields.rs b/crates/core/src/webgraph/warmed_column_fields.rs
index 5da45820..8bcc572e 100644
--- a/crates/core/src/webgraph/warmed_column_fields.rs
+++ b/crates/core/src/webgraph/warmed_column_fields.rs
@@ -24,6 +24,7 @@ use crate::{enum_map::EnumMap, Result};
#[derive(Debug, Clone)]
pub struct SegmentColumnFields {
u64_fields: EnumMap<FieldEnumDiscriminants, Column<u64>>,
+ u128_fields: EnumMap<FieldEnumDiscriminants, Column<u128>>,
f64_fields: EnumMap<FieldEnumDiscriminants, Column<f64>>,
}
@@ -38,6 +39,16 @@ impl SegmentColumnFields {
.cloned()
}
+ pub fn u128<F: Field>(&self, field: F) -> Option<Column<u128>> {
+ self.u128_by_enum(field.into())
+ }
+
+ pub fn u128_by_enum(&self, field: FieldEnum) -> Option<Column<u128>> {
+ self.u128_fields
+ .get(FieldEnumDiscriminants::from(field))
+ .cloned()
+ }
+
pub fn f64<F: Field>(&self, field: F) -> Option<Column<f64>> {
self.f64_by_enum(field.into())
}
@@ -59,6 +70,7 @@ impl WarmedColumnFields {
let mut segments = HashMap::new();
for segment in tantivy_searcher.segment_readers() {
let mut u64_fields = EnumMap::new();
+ let mut u128_fields = EnumMap::new();
let mut f64_fields = EnumMap::new();
for field in FieldEnum::iter() {
match field.field_type() {
@@ -68,6 +80,12 @@ impl WarmedColumnFields {
let column = segment.column_fields().u64(field.name())?;
u64_fields.insert(field.into(), column);
}
+ tantivy::schema::FieldType::U128(numeric_options)
+ if numeric_options.is_columnar() =>
+ {
+ let column = segment.column_fields().u128(field.name())?;
+ u128_fields.insert(field.into(), column);
+ }
tantivy::schema::FieldType::F64(numeric_options)
if numeric_options.is_columnar() =>
{
@@ -81,6 +99,7 @@ impl WarmedColumnFields {
segment.segment_id(),
SegmentColumnFields {
u64_fields,
+ u128_fields,
f64_fields,
},
);
diff --git a/crates/tantivy/src/roworder/mod.rs b/crates/tantivy/src/roworder/mod.rs
index 479d052c..5f335268 100644
--- a/crates/tantivy/src/roworder/mod.rs
+++ b/crates/tantivy/src/roworder/mod.rs
@@ -151,6 +151,7 @@ impl Header {
#[strum_discriminants(name(RowValueType))]
pub enum RowValue {
U64(u64),
+ U128(u128),
I64(i64),
F64(f64),
Bool(bool),
@@ -160,6 +161,7 @@ impl RowValueType {
const fn size(&self) -> usize {
match self {
RowValueType::U64 => 8,
+ RowValueType::U128 => 16,
RowValueType::F64 => 8,
RowValueType::I64 => 8,
RowValueType::Bool => 1,
@@ -172,6 +174,7 @@ impl RowValueType {
RowValueType::F64 => 1,
RowValueType::I64 => 2,
RowValueType::Bool => 3,
+ RowValueType::U128 => 4,
}
}
@@ -181,6 +184,7 @@ impl RowValueType {
1 => RowValueType::F64,
2 => RowValueType::I64,
3 => RowValueType::Bool,
+ 4 => RowValueType::U128,
_ => panic!("Invalid id"),
}
}
@@ -201,6 +205,9 @@ impl RowValue {
RowValue::Bool(v) => {
buf[0] = *v as u8;
}
+ RowValue::U128(v) => {
+ buf.copy_from_slice(&v.to_le_bytes());
+ }
}
}
@@ -223,6 +230,12 @@ impl RowValue {
}
RowValueType::Bool => RowValue::Bool(buf[0] != 0),
+
+ RowValueType::U128 => {
+ let mut bytes = [0u8; 16];
+ bytes.copy_from_slice(&buf[..typ.size()]);
+ RowValue::U128(u128::from_le_bytes(bytes))
+ }
}
}
@@ -253,6 +266,13 @@ impl RowValue {
_ => None,
}
}
+
+ pub fn as_u128(&self) -> Option<u128> {
+ match self {
+ RowValue::U128(v) => Some(*v),
+ _ => None,
+ }
+ }
}
pub struct Row {
@@ -305,6 +325,10 @@ impl Row {
self.get(field).and_then(|v| v.as_u64())
}
+ pub fn get_u128<F: AsFieldId>(&self, field: &F) -> Option<u128> {
+ self.get(field).and_then(|v| v.as_u128())
+ }
+
pub fn get_i64<F: AsFieldId>(&self, field: &F) -> Option<i64> {
self.get(field).and_then(|v| v.as_i64())
}
diff --git a/crates/tantivy/src/roworder/writer.rs b/crates/tantivy/src/roworder/writer.rs
index 17666934..8727eb62 100644
--- a/crates/tantivy/src/roworder/writer.rs
+++ b/crates/tantivy/src/roworder/writer.rs
@@ -29,12 +29,12 @@ impl TryFrom<(crate::schema::Field, &crate::schema::FieldEntry)> for Field {
crate::schema::FieldType::I64(_) => RowValueType::I64,
crate::schema::FieldType::F64(_) => RowValueType::F64,
crate::schema::FieldType::Bool(_) => RowValueType::Bool,
+ crate::schema::FieldType::U128(_) => RowValueType::U128,
crate::schema::FieldType::Str(_)
| crate::schema::FieldType::Date(_)
| crate::schema::FieldType::Bytes(_)
| crate::schema::FieldType::JsonObject(_)
- | crate::schema::FieldType::IpAddr(_)
- | crate::schema::FieldType::U128(_) => {
+ | crate::schema::FieldType::IpAddr(_) => {
return Err(crate::TantivyError::SchemaError(format!(
"Field {:?} is not supported in row order",
entry
@@ -96,17 +96,31 @@ impl RowFieldsWriter {
match value_access.as_value() {
ReferenceValue::Leaf(leaf) => match leaf {
- ReferenceValueLeaf::Null
- | ReferenceValueLeaf::Str(_)
- | ReferenceValueLeaf::Date(_)
- | ReferenceValueLeaf::Bytes(_)
- | ReferenceValueLeaf::IpAddr(_)
- | ReferenceValueLeaf::U128(_)
- | ReferenceValueLeaf::PreTokStr(_) => {}
+ ReferenceValueLeaf::Null => {
+ unimplemented!("Null values are not supported in row order")
+ }
+ ReferenceValueLeaf::Date(_) => {
+ unimplemented!("Date values are not supported in row order")
+ }
+ ReferenceValueLeaf::Bytes(_) => {
+ unimplemented!("Bytes values are not supported in row order")
+ }
+ ReferenceValueLeaf::IpAddr(_) => {
+ unimplemented!("IpAddr values are not supported in row order")
+ }
+ ReferenceValueLeaf::PreTokStr(_) => {
+ unimplemented!("PreTokStr values are not supported in row order")
+ }
+ ReferenceValueLeaf::Str(_) => {
+ unimplemented!("String values are not supported in row order")
+ }
ReferenceValueLeaf::U64(val) => {
buf.insert(field.field_id(), RowValue::U64(val));
}
+ ReferenceValueLeaf::U128(val) => {
+ buf.insert(field.field_id(), RowValue::U128(val));
+ }
ReferenceValueLeaf::I64(val) => {
buf.insert(field.field_id(), RowValue::I64(val));
}
diff --git a/crates/tantivy/src/schema/document/de.rs b/crates/tantivy/src/schema/document/de.rs
index cf398e9f..6608c273 100644
--- a/crates/tantivy/src/schema/document/de.rs
+++ b/crates/tantivy/src/schema/document/de.rs
@@ -366,6 +366,7 @@ where
let value_type = match type_code {
type_codes::TEXT_CODE => ValueType::String,
type_codes::U64_CODE => ValueType::U64,
+ type_codes::U128_CODE => ValueType::U128,
type_codes::I64_CODE => ValueType::I64,
type_codes::F64_CODE => ValueType::F64,
type_codes::BOOL_CODE => ValueType::Bool,