web search done right
Use md5 digest as id for webgraph nodes. There is a small collision risk where multiple nodes will get the same id in the graph. This risk is extremely small since the digest is 128 bits and the benefits we get from the significantly reduced complexity with everything surrounding the webgraph hugely outweighs this risk. It's now much faster to get edges for a node in the graph, since we don't need to map segment_node_ids to node_ids. Segment merges is also way less complex.
Mikkel Denker3c2c0bf7582b
diff --git a/core/examples/print_inbound_links.rs b/core/examples/print_inbound_links.rsindex 06b7ab74..dccdddd2 100644--- a/core/examples/print_inbound_links.rs+++ b/core/examples/print_inbound_links.rs@@ -27,8 +27,7 @@ pub fn main() {println!("{host}:");let node = Node::from(host).into_host();- let node_id = graph.node2id(&node).unwrap();- let inbound_vec = inbound.get(&node_id).unwrap();+ let inbound_vec = inbound.get(&node.id()).unwrap();println!("{:?}", inbound_vec);for edge in graph.ingoing_edges(node) {diff --git a/core/src/collector.rs b/core/src/collector.rsindex 009ef174..1f91246b 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -24,9 +24,9 @@ use tantivy::{};use crate::{- fastfield_reader,+ combine_u64s, fastfield_reader,inverted_index::{DocAddress, WebsitePointer},- prehashed::{combine_u64s, Prehashed},+ prehashed::Prehashed,ranking::initial::Score,schema::FastField,simhash,diff --git a/core/src/intmap.rs b/core/src/intmap.rsindex 8fe2566e..878d04cd 100644--- a/core/src/intmap.rs+++ b/core/src/intmap.rs@@ -16,15 +16,42 @@use serde::{Deserialize, Serialize};-type Key = u64;+pub trait Key: PartialEq + Copy + std::fmt::Debug {+ const BIG_PRIME: Self;++ fn wrapping_mul(self, rhs: Self) -> Self;+ fn bit_and(self, rhs: Self) -> Self;+ fn from_usize(val: usize) -> Self;+ fn as_usize(self) -> usize;+}++impl Key for u64 {+ const BIG_PRIME: Self = 11400714819323198549;++ fn wrapping_mul(self, rhs: Self) -> Self {+ self.wrapping_mul(rhs)+ }++ fn bit_and(self, rhs: Self) -> Self {+ self & rhs+ }++ fn from_usize(val: usize) -> Self {+ val as Self+ }++ fn as_usize(self) -> usize {+ self as usize+ }+}#[derive(Serialize, Deserialize, Debug)]-pub struct IntMap<V> {- bins: Vec<Vec<(Key, V)>>,+pub struct IntMap<K: Key, V> {+ bins: Vec<Vec<(K, V)>>,len: usize,}-impl<V: Clone> Clone for IntMap<V> {+impl<K: Key, V: Clone> Clone for IntMap<K, V> {fn clone(&self) -> Self {Self {bins: self.bins.clone(),@@ -33,7 +60,7 @@ impl<V: Clone> Clone for IntMap<V> {}}-impl<V> IntMap<V> {+impl<K: Key, V> IntMap<K, V> {pub fn new() -> Self {Self::with_capacity(2)}@@ -48,13 +75,14 @@ impl<V> IntMap<V> {Self { bins, len: 0 }}- fn bin_idx(&self, key: &Key) -> usize {- let mask = (self.bins.len() - 1) as Key;- let key = key.wrapping_mul(11400714819323198549 as Key);- (key & mask) as usize+ fn bin_idx(&self, key: &K) -> usize {+ let mask = K::from_usize(self.bins.len() - 1);+ let key = key.wrapping_mul(K::BIG_PRIME);++ key.bit_and(mask).as_usize()}- pub fn insert(&mut self, key: Key, value: V) {+ pub fn insert(&mut self, key: K, value: V) {if self.len >= (self.bins.len() as f64 * 1.2) as usize {self.grow();}@@ -87,7 +115,7 @@ impl<V> IntMap<V> {}}- pub fn get(&self, key: &Key) -> Option<&V> {+ pub fn get(&self, key: &K) -> Option<&V> {let bin = self.bin_idx(key);self.bins[bin].iter()@@ -95,7 +123,7 @@ impl<V> IntMap<V> {.map(|(_, val)| val)}- pub fn get_mut(&mut self, key: &Key) -> Option<&mut V> {+ pub fn get_mut(&mut self, key: &K) -> Option<&mut V> {let bin = self.bin_idx(key);self.bins[bin].iter_mut()@@ -107,25 +135,25 @@ impl<V> IntMap<V> {self.len}- pub fn into_iter(self) -> impl Iterator<Item = (Key, V)> {+ pub fn into_iter(self) -> impl Iterator<Item = (K, V)> {self.bins.into_iter().flat_map(|bin| bin.into_iter())}- pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut (Key, V)> {+ pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut (K, V)> {self.bins.iter_mut().flat_map(|bin| bin.iter_mut())}- pub fn iter(&self) -> impl Iterator<Item = &(Key, V)> {+ pub fn iter(&self) -> impl Iterator<Item = &(K, V)> {self.bins.iter().flat_map(|bin| bin.iter())}- pub fn contains_key(&self, key: &Key) -> bool {+ pub fn contains_key(&self, key: &K) -> bool {self.get(key).is_some()}}-impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {- fn from_iter<T: IntoIterator<Item = (u64, V)>>(iter: T) -> Self {+impl<K: Key, V> std::iter::FromIterator<(K, V)> for IntMap<K, V> {+ fn from_iter<T: IntoIterator<Item = (K, V)>>(iter: T) -> Self {let iter = iter.into_iter();let (_, upper) = iter.size_hint();@@ -144,27 +172,33 @@ impl<V> std::iter::FromIterator<(u64, V)> for IntMap<V> {}}-impl<V> Default for IntMap<V> {+impl<K: Key, V> Default for IntMap<K, V> {fn default() -> Self {Self::new()}}-#[derive(Default, Debug)]-pub struct IntSet {- map: IntMap<()>,+#[derive(Debug)]+pub struct IntSet<K: Key = u64> {+ map: IntMap<K, ()>,+}++impl<K: Key> Default for IntSet<K> {+ fn default() -> Self {+ Self::new()+ }}-impl IntSet {+impl<K: Key> IntSet<K> {pub fn new() -> Self {Self { map: IntMap::new() }}- pub fn insert(&mut self, item: Key) {+ pub fn insert(&mut self, item: K) {self.map.insert(item, ());}- pub fn into_iter(self) -> impl Iterator<Item = Key> {+ pub fn into_iter(self) -> impl Iterator<Item = K> {self.map.into_iter().map(|(key, _)| key)}@@ -172,13 +206,13 @@ impl IntSet {self.map.len()}- pub fn contains(&self, item: &Key) -> bool {+ pub fn contains(&self, item: &K) -> bool {self.map.contains_key(item)}}-impl std::iter::FromIterator<u64> for IntSet {- fn from_iter<T: IntoIterator<Item = u64>>(iter: T) -> Self {+impl<K: Key> std::iter::FromIterator<K> for IntSet<K> {+ fn from_iter<T: IntoIterator<Item = K>>(iter: T) -> Self {let mut set = Self::new();for num in iter {diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 9ec10c24..017ef8e1 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -42,12 +42,12 @@ use crate::ranking::pipeline::RankingWebsite;use crate::ranking::SignalAggregator;use crate::schema::{FastField, Field, TextField, ALL_FIELDS};use crate::search_ctx::Ctx;-use crate::snippet;use crate::tokenizer::{BigramTokenizer, Identity, TrigramTokenizer};use crate::webgraph::NodeID;use crate::webpage::region::Region;use crate::webpage::{schema_org, Url, Webpage};use crate::Result;+use crate::{combine_u64s, snippet};use crate::{schema::create_schema, tokenizer::Tokenizer};use std::collections::HashSet;use std::fs;@@ -271,15 +271,23 @@ impl InvertedIndex {pub fn website_host_node(&self, website: &WebsitePointer) -> Result<Option<NodeID>> {let searcher = self.reader.searcher();let doc = searcher.doc(website.address.into())?;- let field = self++ let field1 = self+ .schema()+ .get_field(Field::Fast(FastField::HostNodeID1).name())+ .unwrap();+ let field2 = self.schema()- .get_field(Field::Fast(FastField::HostNodeID).name())+ .get_field(Field::Fast(FastField::HostNodeID2).name()).unwrap();- let id = doc.get_first(field).unwrap().as_u64().unwrap();- if id == u64::MAX {+ let id1 = doc.get_first(field1).unwrap().as_u64().unwrap();+ let id2 = doc.get_first(field2).unwrap().as_u64().unwrap();++ if id1 == u64::MAX && id2 == u64::MAX {Ok(None)} else {+ let id = combine_u64s([id1, id2]);Ok(Some(id.into()))}}diff --git a/core/src/lib.rs b/core/src/lib.rsindex 2bc9ec04..5cb425e4 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -483,3 +483,23 @@ fn floor_char_boundary(str: &str, index: usize) -> usize {res}++pub fn split_u128(num: u128) -> [u64; 2] {+ [(num >> 64) as u64, num as u64]+}++pub fn combine_u64s(nums: [u64; 2]) -> u128 {+ ((nums[0] as u128) << 64) | (nums[1] as u128)+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn split_combine_u128() {+ for num in 0..10000_u128 {+ assert_eq!(combine_u64s(split_u128(num)), num);+ }+ }+}diff --git a/core/src/prehashed.rs b/core/src/prehashed.rsindex e7a11c70..b4b133e4 100644--- a/core/src/prehashed.rs+++ b/core/src/prehashed.rs@@ -71,23 +71,3 @@ pub fn hash<T: AsRef<[u8]>>(val: T) -> Prehashed {let digest = md5::compute(val);Prehashed(u128::from_be_bytes(*digest))}--pub fn split_u128(num: u128) -> [u64; 2] {- [(num >> 64) as u64, num as u64]-}--pub fn combine_u64s(nums: [u64; 2]) -> u128 {- ((nums[0] as u128) << 64) | (nums[1] as u128)-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn split_combine_u128() {- for num in 0..10000_u128 {- assert_eq!(combine_u64s(split_u128(num)), num);- }- }-}diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex ee057897..8871fc38 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -655,7 +655,7 @@ mod tests {pre_computed_score: 0.0,dmoz_description: None,- node_id: Some(graph.node2id(&Node::from("www.a.com").into_host()).unwrap()),+ node_id: Some(Node::from("www.a.com").into_host().id()),}).expect("failed to insert webpage");index@@ -683,7 +683,7 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,dmoz_description: None,- node_id: Some(graph.node2id(&Node::from("www.b.com").into_host()).unwrap()),+ node_id: Some(Node::from("www.b.com").into_host().id()),}).expect("failed to insert webpage");index@@ -711,7 +711,7 @@ mod tests {pre_computed_score: 0.0,fetch_time_ms: 500,dmoz_description: None,- node_id: Some(graph.node2id(&Node::from("www.c.com").into_host()).unwrap()),+ node_id: Some(Node::from("www.c.com").into_host().id()),}).expect("failed to insert webpage");diff --git a/core/src/ranking/bitvec_similarity.rs b/core/src/ranking/bitvec_similarity.rsindex 493bf5ea..6918d40e 100644--- a/core/src/ranking/bitvec_similarity.rs+++ b/core/src/ranking/bitvec_similarity.rs@@ -19,7 +19,7 @@ use serde::{Deserialize, Serialize};#[derive(Serialize, Deserialize, Debug, Clone)]struct VeryJankyBloomFilter {- data: Vec<u64>,+ data: Vec<u128>,}impl VeryJankyBloomFilter {@@ -29,14 +29,15 @@ impl VeryJankyBloomFilter {}}- fn hash(&self, item: &usize) -> (usize, usize) {+ fn hash(&self, item: &u128) -> (usize, u128) {(- item.wrapping_mul(11400714819323198549) % self.data.len(),- item.wrapping_mul(14029467366897019727) % 64,+ (item.wrapping_mul(180811355057196146399512320062616506473) % self.data.len() as u128)+ as usize,+ item.wrapping_mul(258349730118558892798889897730042154643) % 128,)}- fn insert(&mut self, item: usize) {+ fn insert(&mut self, item: u128) {let (a, b) = self.hash(&item);self.data[a] |= 1 << b;}@@ -52,13 +53,13 @@ impl VeryJankyBloomFilter {#[derive(Serialize, Deserialize, Debug, Clone)]struct Posting {- ranks: Vec<usize>,- skip_pointers: Vec<usize>,+ ranks: Vec<u128>,+ skip_pointers: Vec<u128>,skip_size: usize,}impl Posting {- fn new(ranks: Vec<usize>) -> Self {+ fn new(ranks: Vec<u128>) -> Self {let skip_size = (ranks.len() as f64).sqrt() as usize;let skip_pointers = ranks@@ -134,7 +135,7 @@ pub struct BitVec {}impl BitVec {- pub fn new(mut ranks: Vec<usize>) -> Self {+ pub fn new(mut ranks: Vec<u128>) -> Self {ranks.sort();ranks.dedup();ranks.shrink_to_fit();@@ -180,11 +181,11 @@ mod tests {use itertools::Itertools;use std::iter::repeat;- fn into_ranks(a: &[bool]) -> Vec<usize> {+ fn into_ranks(a: &[bool]) -> Vec<u128> {a.iter().enumerate().filter(|(_, b)| **b)- .map(|(i, _)| i)+ .map(|(i, _)| i as u128).collect()}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex b41051c9..67f679bc 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -38,7 +38,7 @@ const DEFAULT_NUM_TOP_HARMONIC_CENTRALITY_FOR_NODES: usize = 1_000_000;pub struct Scorer {liked: Vec<NodeScorer>,disliked: Vec<NodeScorer>,- vectors: Arc<IntMap<bitvec_similarity::BitVec>>,+ vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,cache: DashMap<NodeID, f64>,}@@ -73,7 +73,7 @@ impl NodeScorer {impl Scorer {fn calculate_score(&self, node: &NodeID) -> f64 {- match self.vectors.get(&node.0) {+ match self.vectors.get(node) {Some(vec) => ((self.disliked.len() as f64)+ (self.liked@@ -139,7 +139,7 @@ impl Eq for ScoredNode {}#[derive(Serialize, Deserialize, Default)]pub struct InboundSimilarity {- vectors: Arc<IntMap<bitvec_similarity::BitVec>>,+ vectors: Arc<IntMap<NodeID, bitvec_similarity::BitVec>>,}impl InboundSimilarity {@@ -177,20 +177,20 @@ impl InboundSimilarity {}}- let top_nodes: IntSet = top_nodes.into_iter().map(|n| n.node.0).collect();+ let top_nodes: IntSet<NodeID> = top_nodes.into_iter().map(|n| n.node).collect();for node_id in graph.nodes() {let mut ranks = Vec::new();for edge in graph.raw_ingoing_edges(&node_id) {- if !top_nodes.contains(&edge.from.0) {+ if !top_nodes.contains(&edge.from) {continue;}- ranks.push(edge.from.0 as usize);+ ranks.push(edge.from.bit_128());}- vectors.insert(node_id.0, bitvec_similarity::BitVec::new(ranks));+ vectors.insert(node_id, bitvec_similarity::BitVec::new(ranks));}Self {@@ -201,13 +201,13 @@ impl InboundSimilarity {pub fn scorer(&self, liked_sites: &[NodeID], disliked_sites: &[NodeID]) -> Scorer {let liked: Vec<_> = liked_sites.iter()- .filter_map(|id| self.vectors.get(&id.0).cloned().map(|vec| (id, vec)))+ .filter_map(|id| self.vectors.get(id).cloned().map(|vec| (id, vec))).map(|(node, inbound)| NodeScorer::new(*node, inbound)).collect();let disliked: Vec<_> = disliked_sites.iter()- .filter_map(|id| self.vectors.get(&id.0).cloned().map(|vec| (id, vec)))+ .filter_map(|id| self.vectors.get(id).cloned().map(|vec| (id, vec))).map(|(node, inbound)| NodeScorer::new(*node, inbound)).collect();@@ -234,7 +234,7 @@ impl InboundSimilarity {}pub fn get(&self, node: &NodeID) -> Option<&bitvec_similarity::BitVec> {- self.vectors.get(&node.0)+ self.vectors.get(node)}pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {@@ -248,7 +248,7 @@ impl InboundSimilarity {}pub fn knows_about(&self, node_id: NodeID) -> bool {- self.vectors.contains_key(&node_id.0)+ self.vectors.contains_key(&node_id)}}@@ -296,9 +296,9 @@ mod tests {let inbound =InboundSimilarity::build_with_threshold(&graph, &harmonic_centrality_store, 1000);- let scorer = inbound.scorer(&[graph.node2id(&Node::from("b.com")).unwrap()], &[]);- let e = graph.node2id(&Node::from("e.com")).unwrap();- let d = graph.node2id(&Node::from("d.com")).unwrap();+ let scorer = inbound.scorer(&[Node::from("b.com").id()], &[]);+ let e = Node::from("e.com").id();+ let d = Node::from("d.com").id();assert!(scorer.score(&e) > scorer.score(&d));}@@ -354,7 +354,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- node_id: Some(graph.node2id(&Node::from("e.com")).unwrap()),+ node_id: Some(Node::from("e.com").id()),dmoz_description: None,}).expect("failed to insert webpage");@@ -382,7 +382,7 @@ mod tests {page_centrality: 0.0,pre_computed_score: 0.0,- node_id: Some(graph.node2id(&Node::from("d.com")).unwrap()),+ node_id: Some(Node::from("d.com").id()),dmoz_description: None,}).expect("failed to insert webpage");diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 967d8b4a..c107bfc9 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -102,11 +102,7 @@ mod tests {page_centrality: 0.0,dmoz_description: None,- node_id: Some(- graph- .node2id(&Node::from("https://www.first.com").into_host())- .unwrap(),- ),+ node_id: Some(Node::from("https://www.first.com").into_host().id()),}).expect("failed to insert webpage");index@@ -134,11 +130,7 @@ mod tests {pre_computed_score: 0.0,dmoz_description: None,- node_id: Some(- graph- .node2id(&Node::from("https://www.second.com").into_host())- .unwrap(),- ),+ node_id: Some(Node::from("https://www.second.com").into_host().id()),}).expect("failed to insert webpage");index@@ -166,11 +158,7 @@ mod tests {pre_computed_score: 0.0,dmoz_description: None,- node_id: Some(- graph- .node2id(&Node::from("https://www.third.com").into_host())- .unwrap(),- ),+ node_id: Some(Node::from("https://www.third.com").into_host().id()),}).expect("failed to insert webpage");index.commit().expect("failed to commit index");diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rsindex 9137f773..94b15541 100644--- a/core/src/ranking/signal.rs+++ b/core/src/ranking/signal.rs@@ -17,7 +17,7 @@use crate::fastfield_reader::FieldValue;use crate::query::optic::AsSearchableRule;use crate::query::Query;-use crate::Result;+use crate::{combine_u64s, Result};use crate::{enum_map::EnumMap,fastfield_reader,@@ -293,17 +293,26 @@ impl Signal {fn host_id(&self, aggregator: &SignalAggregator, doc: DocId) -> Option<NodeID> {aggregator.segment_reader.as_ref().and_then(|reader| {- let node_id: Option<u64> = reader+ let node_id1: Option<u64> = reader.fastfield_reader- .get_field_reader(&FastField::HostNodeID)+ .get_field_reader(&FastField::HostNodeID1).get(&doc).into();- let node_id = node_id.unwrap();+ let node_id1 = node_id1.unwrap();- if node_id == u64::MAX {+ let node_id2: Option<u64> = reader+ .fastfield_reader+ .get_field_reader(&FastField::HostNodeID2)+ .get(&doc)+ .into();+ let node_id2 = node_id2.unwrap();++ if node_id1 == u64::MAX && node_id2 == u64::MAX {None} else {- Some(NodeID::from(node_id))+ let id = combine_u64s([node_id1, node_id2]);++ Some(NodeID::from(id))}})}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 4bf72b86..766f4cb5 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -182,7 +182,8 @@ pub enum FastField {DomainHash1,DomainHash2,PreComputedScore,- HostNodeID,+ HostNodeID1,+ HostNodeID2,SimHash,NumFlattenedSchemaTokens,NumPathAndQuerySlashes,@@ -217,7 +218,8 @@ impl FastField {FastField::DomainHash1 => "domain_hash1",FastField::DomainHash2 => "domain_hash2",FastField::PreComputedScore => "pre_computed_score",- FastField::HostNodeID => "host_node_id",+ FastField::HostNodeID1 => "host_node_id1",+ FastField::HostNodeID2 => "host_node_id2",FastField::SimHash => "sim_hash",FastField::NumPathAndQuerySlashes => "num_path_and_query_slashes",FastField::NumPathAndQueryDigits => "num_path_and_query_digits",@@ -237,7 +239,7 @@ pub enum Field {Text(TextField),}-pub static ALL_FIELDS: [Field; 53] = [+pub static ALL_FIELDS: [Field; 54] = [Field::Text(TextField::Title),Field::Text(TextField::CleanBody),Field::Text(TextField::StemmedTitle),@@ -288,7 +290,8 @@ pub static ALL_FIELDS: [Field; 53] = [Field::Fast(FastField::DomainHash1),Field::Fast(FastField::DomainHash2),Field::Fast(FastField::PreComputedScore),- Field::Fast(FastField::HostNodeID),+ Field::Fast(FastField::HostNodeID1),+ Field::Fast(FastField::HostNodeID2),Field::Fast(FastField::SimHash),Field::Fast(FastField::NumPathAndQuerySlashes),Field::Fast(FastField::NumPathAndQueryDigits),@@ -487,7 +490,13 @@ impl Field {.set_indexed().set_stored(),),- Field::Fast(FastField::HostNodeID) => IndexingOption::Integer(+ Field::Fast(FastField::HostNodeID1) => IndexingOption::Integer(+ NumericOptions::default()+ .set_fast(Cardinality::SingleValue)+ .set_indexed()+ .set_stored(),+ ),+ Field::Fast(FastField::HostNodeID2) => IndexingOption::Integer(NumericOptions::default().set_fast(Cardinality::SingleValue).set_indexed()@@ -605,7 +614,8 @@ impl FastField {FastField::DomainHash1 => DataType::U64,FastField::DomainHash2 => DataType::U64,FastField::PreComputedScore => DataType::F64,- FastField::HostNodeID => DataType::U64,+ FastField::HostNodeID1 => DataType::U64,+ FastField::HostNodeID2 => DataType::U64,FastField::SimHash => DataType::U64,FastField::NumPathAndQuerySlashes => DataType::U64,FastField::NumPathAndQueryDigits => DataType::U64,diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 60e28203..4d3c7377 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -74,20 +74,20 @@ impl SimilarSitesFinder {let nodes: Vec<_> = nodes.iter().map(|url| Node::from(url.to_string()).into_host())- .filter_map(|node| self.webgraph.node2id(&node))+ .map(|node| node.id()).collect();let scorer = self.inbound_similarity.scorer(&nodes, &[]);- let mut backlink_count: IntMap<usize> = IntMap::new();+ let mut backlink_count: IntMap<NodeID, usize> = IntMap::new();for node in &nodes {for edge in self.webgraph.raw_ingoing_edges(node) {- if !backlink_count.contains_key(&edge.from.0) {- backlink_count.insert(edge.from.0, 0);+ if !backlink_count.contains_key(&edge.from) {+ backlink_count.insert(edge.from, 0);}- *backlink_count.get_mut(&edge.from.0).unwrap() += 1;+ *backlink_count.get_mut(&edge.from).unwrap() += 1;}}@@ -101,15 +101,14 @@ impl SimilarSitesFinder {let mut potential_nodes = IntSet::new();for (backlink_node, _) in top_backlink_nodes {- for edge in self.webgraph.raw_outgoing_edges(&NodeID(backlink_node)) {- potential_nodes.insert(edge.to.0);+ for edge in self.webgraph.raw_outgoing_edges(&backlink_node) {+ potential_nodes.insert(edge.to);}}let mut scored_nodes = BinaryHeap::with_capacity(limit);for potential_node in potential_nodes.into_iter() {- let potential_node = NodeID(potential_node);let score = scorer.score(&potential_node);let scored_node_id = ScoredNodeID {node_id: potential_node,@@ -141,9 +140,6 @@ impl SimilarSitesFinder {}pub fn knows_about(&self, node: &Node) -> bool {- match self.webgraph.node2id(node) {- Some(node_id) => self.inbound_similarity.knows_about(node_id),- None => false,- }+ self.inbound_similarity.knows_about(node.id())}}diff --git a/core/src/webgraph/centrality/betweenness.rs b/core/src/webgraph/centrality/betweenness.rsindex 3bed8968..4de8fc4f 100644--- a/core/src/webgraph/centrality/betweenness.rs+++ b/core/src/webgraph/centrality/betweenness.rs@@ -56,14 +56,14 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)centrality.entry(s).or_default();let mut stack = Vec::new();- let mut predecessors: IntMap<Vec<u64>> = IntMap::new();+ let mut predecessors: IntMap<NodeID, Vec<NodeID>> = IntMap::new();let mut sigma = IntMap::new();- sigma.insert(s.0, 1);+ sigma.insert(s, 1);let mut distances = IntMap::new();- distances.insert(s.0, 0);+ distances.insert(s, 0);let mut q = VecDeque::new();q.push_back(s);@@ -73,25 +73,25 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)for edge in graph.raw_outgoing_edges(&v) {let w = edge.to;- if !distances.contains_key(&w.0) {- let dist_v = distances.get(&v.0).unwrap();+ if !distances.contains_key(&w) {+ let dist_v = distances.get(&v).unwrap();q.push_back(w);- distances.insert(w.0, dist_v + 1);+ distances.insert(w, dist_v + 1);}- if *distances.get(&w.0).unwrap() == distances.get(&v.0).unwrap() + 1 {- let sigma_v = *sigma.get(&v.0).unwrap_or(&0);+ if *distances.get(&w).unwrap() == distances.get(&v).unwrap() + 1 {+ let sigma_v = *sigma.get(&v).unwrap_or(&0);- if !sigma.contains_key(&w.0) {- sigma.insert(w.0, 0);+ if !sigma.contains_key(&w) {+ sigma.insert(w, 0);}- *sigma.get_mut(&w.0).unwrap() += sigma_v;+ *sigma.get_mut(&w).unwrap() += sigma_v;- if !predecessors.contains_key(&w.0) {- predecessors.insert(w.0, Vec::new());+ if !predecessors.contains_key(&w) {+ predecessors.insert(w, Vec::new());}- predecessors.get_mut(&w.0).unwrap().push(v.0);+ predecessors.get_mut(&w).unwrap().push(v);}}}@@ -100,20 +100,20 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)let mut delta = IntMap::new();while let Some(w) = stack.pop() {- if let Some(pred) = predecessors.get(&w.0) {+ if let Some(pred) = predecessors.get(&w) {for v in pred {let dv = delta.get(v).copied().unwrap_or(0.0);delta.insert(*v,- dv + (*sigma.get(v).unwrap() as f64 / *sigma.get(&w.0).unwrap() as f64)- * (1.0 + delta.get(&w.0).unwrap_or(&0.0)),+ dv + (*sigma.get(v).unwrap() as f64 / *sigma.get(&w).unwrap() as f64)+ * (1.0 + delta.get(&w).unwrap_or(&0.0)),);}}if w != s {- *centrality.entry(w).or_insert(0.0) += *delta.get(&w.0).unwrap_or(&0.0);+ *centrality.entry(w).or_insert(0.0) += *delta.get(&w).unwrap_or(&0.0);}}}diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex a5b9a40d..fdd851bb 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -84,28 +84,28 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {info!("Found {} nodes in the graph", nodes.len());let norm_factor = (nodes.len() - 1) as f64;- let mut counters: IntMap<HyperLogLog<HYPERLOGLOG_COUNTERS>> = nodes+ let mut counters: IntMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>> = nodes.iter().map(|node| {let mut counter = HyperLogLog::default();- counter.add(node.0);+ counter.add(node.bit_64());- (node.0, counter)+ (*node, counter)}).collect();let mut exact_counting = false;let mut has_changes = true;let mut t = 0;- let mut centralities: IntMap<KahanSum> = nodes+ let mut centralities: IntMap<NodeID, KahanSum> = nodes.iter()- .map(|node| (node.0, KahanSum::default()))+ .map(|node| (*node, KahanSum::default())).collect();- let mut exact_changed_nodes = IntSet::default();+ let mut exact_changed_nodes: IntSet<NodeID> = IntSet::default();let mut changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);for node in &nodes {- changed_nodes.insert(node.0);+ changed_nodes.insert(node.bit_64());}loop {@@ -113,7 +113,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {break;}- let mut new_counters: IntMap<_> = counters.clone();+ let mut new_counters: IntMap<_, _> = counters.clone();has_changes = false;let mut new_changed_nodes = JankyBloomFilter::new(nodes.len() as u64, 0.05);@@ -122,9 +122,9 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {let mut new_exact_changed_nodes = IntSet::default();for changed_node in exact_changed_nodes.into_iter() {- for edge in graph.raw_outgoing_edges(&NodeID(changed_node)) {+ for edge in graph.raw_outgoing_edges(&changed_node) {if let (Some(counter_to), Some(counter_from)) =- (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))+ (new_counters.get_mut(&edge.to), counters.get(&edge.from)){if counter_to.registers()@@ -133,9 +133,9 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {.any(|(to, from)| *from > *to){counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.0);+ new_changed_nodes.insert(edge.to.bit_64());- new_exact_changed_nodes.insert(edge.to.0);+ new_exact_changed_nodes.insert(edge.to);has_changes = true;}@@ -147,12 +147,12 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {} else {exact_changed_nodes = IntSet::default();for edge in graph.edges() {- if !changed_nodes.contains(&edge.from.0) {+ if !changed_nodes.contains(&edge.from.bit_64()) {continue;}if let (Some(counter_to), Some(counter_from)) =- (new_counters.get_mut(&edge.to.0), counters.get(&edge.from.0))+ (new_counters.get_mut(&edge.to), counters.get(&edge.from)){if counter_to.registers()@@ -161,10 +161,10 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {.any(|(to, from)| *from > *to){counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.0);+ new_changed_nodes.insert(edge.to.bit_64());if exact_counting {- exact_changed_nodes.insert(edge.to.0);+ exact_changed_nodes.insert(edge.to);}has_changes = true;@@ -201,7 +201,7 @@ fn calculate_centrality(graph: &Webgraph) -> BTreeMap<NodeID, f64> {.into_iter().map(|(node_id, sum)| (node_id, f64::from(sum))).filter(|(_, centrality)| *centrality > 0.0)- .map(|(node_id, centrality)| (NodeID(node_id), centrality / norm_factor))+ .map(|(node_id, centrality)| (node_id, centrality / norm_factor)).collect()}@@ -331,12 +331,8 @@ mod tests {let centrality = HarmonicCentrality::calculate(&graph);assert!(- centrality- .get(&graph.node2id(&Node::from("B.com")).unwrap())- .unwrap()- > centrality- .get(&graph.node2id(&Node::from("A.com")).unwrap())- .unwrap_or(0.0)+ centrality.get(&Node::from("B.com").id()).unwrap()+ > centrality.get(&Node::from("A.com").id()).unwrap_or(0.0));}@@ -346,25 +342,14 @@ mod tests {let centrality = HarmonicCentrality::calculate(&graph);assert!(- centrality- .get(&graph.node2id(&Node::from("C")).unwrap())- .unwrap()- > centrality- .get(&graph.node2id(&Node::from("A")).unwrap())- .unwrap()+ centrality.get(&Node::from("C").id()).unwrap()+ > centrality.get(&Node::from("A").id()).unwrap());assert!(- centrality- .get(&graph.node2id(&Node::from("A")).unwrap())- .unwrap()- > centrality- .get(&graph.node2id(&Node::from("B")).unwrap())- .unwrap()- );- assert_eq!(- centrality.get(&graph.node2id(&Node::from("D")).unwrap()),- None+ centrality.get(&Node::from("A").id()).unwrap()+ > centrality.get(&Node::from("B").id()).unwrap());+ assert_eq!(centrality.get(&Node::from("D").id()), None);}#[test]diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex ea1e9789..7c840401 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -15,10 +15,9 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.mod segment;-use serde::de::DeserializeOwned;use serde::{Deserialize, Serialize};use std::collections::{BTreeMap, BinaryHeap, HashSet};-use std::fs::{File, OpenOptions};+use std::fs::File;use std::io::{BufReader, BufWriter, Read, Write};use std::path::Path;use std::sync::Arc;@@ -26,31 +25,62 @@ use std::{cmp, fs};use crate::directory::{self, DirEntry};use crate::executor::Executor;+use crate::intmap;use crate::webpage::Url;pub mod centrality;mod store;-use self::segment::{LiveSegment, SegmentNodeID, StoredSegment};+use self::segment::{LiveSegment, StoredSegment};use self::store::Store;#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]-pub struct NodeID(pub u64);+pub struct NodeID(u128);-impl From<u64> for NodeID {- fn from(val: u64) -> Self {- Self(val)+impl NodeID {+ pub fn bit_64(self) -> u64 {+ self.0 as u64+ }++ pub fn bit_128(self) -> u128 {+ self.0+ }+}++impl From<u128> for NodeID {+ fn from(val: u128) -> Self {+ NodeID(val)+ }+}++impl intmap::Key for NodeID {+ const BIG_PRIME: Self = NodeID(11400714819323198549);++ fn wrapping_mul(self, rhs: Self) -> Self {+ NodeID(self.0.wrapping_mul(rhs.0))+ }++ fn bit_and(self, rhs: Self) -> Self {+ NodeID(self.0 & rhs.0)+ }++ fn from_usize(val: usize) -> Self {+ NodeID(val as u128)+ }++ fn as_usize(self) -> usize {+ self.0 as usize}}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub(crate) struct FullStoredEdge {- other: SegmentNodeID,+ other: NodeID,label: String,}#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]pub(crate) struct SmallStoredEdge {- other: SegmentNodeID,+ other: NodeID,}#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]@@ -68,6 +98,11 @@ impl Node {name: host.to_string(),}}++ pub fn id(&self) -> NodeID {+ let digest = md5::compute(self.name.as_bytes());+ NodeID(u128::from_be_bytes(*digest))+ }}impl From<String> for Node {@@ -157,22 +192,12 @@ pub trait ShortestPaths {fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8>;}-fn dijkstra<F1, F2>(- source: Node,- node_edges: F1,- edge_node: F2,- graph: &Webgraph,-) -> BTreeMap<NodeID, u8>+fn dijkstra<F1, F2>(source: Node, node_edges: F1, edge_node: F2) -> BTreeMap<NodeID, u8>whereF1: Fn(NodeID) -> Vec<Edge>,F2: Fn(&Edge) -> NodeID,{- let source_id = graph.node2id(&source);- if source_id.is_none() {- return BTreeMap::new();- }-- let source_id = source_id.unwrap();+ let source_id = source.id();let mut distances: BTreeMap<NodeID, u8> = BTreeMap::default();let mut queue = BinaryHeap::new();@@ -207,7 +232,7 @@ impl ShortestPaths for Webgraph {fn distances(&self, source: Node) -> BTreeMap<Node, u8> {self.raw_distances(source).into_iter()- .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))+ .filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist))).collect()}@@ -216,7 +241,6 @@ impl ShortestPaths for Webgraph {source,|node| self.raw_outgoing_edges(&node),|edge| edge.to,- self,)}@@ -225,14 +249,13 @@ impl ShortestPaths for Webgraph {source,|node| self.raw_ingoing_edges(&node),|edge| edge.from,- self,)}fn reversed_distances(&self, source: Node) -> BTreeMap<Node, u8> {self.raw_reversed_distances(source).into_iter()- .map(|(id, dist)| (self.id2node(&id).expect("unknown node"), dist))+ .filter_map(|(id, dist)| self.id2node(&id).map(|node| (node, dist))).collect()}}@@ -250,43 +273,11 @@ struct SegmentMergeCandidate {merges: Vec<StoredSegment>,}-fn open_bin<T, P>(path: P) -> T-where- P: AsRef<Path>,- T: DeserializeOwned + Default,-{- let f = OpenOptions::new()- .read(true)- .write(true)- .create(true)- .open(path)- .unwrap();-- let reader = BufReader::new(f);-- bincode::deserialize_from(reader).unwrap_or_default()-}--fn save_bin<T, P>(val: &T, path: P)-where- P: AsRef<Path>,- T: Serialize,-{- let f = OpenOptions::new()- .read(true)- .write(true)- .create(true)- .open(path)- .unwrap();-- bincode::serialize_into(f, val).unwrap();-}pub struct Webgraph {pub path: String,live_segment: LiveSegment,segments: Vec<StoredSegment>,executor: Arc<Executor>,- node2id: Store<Node, NodeID>,id2node: Store<NodeID, Node>,meta: Meta,}@@ -341,30 +332,19 @@ impl Webgraph {live_segment: LiveSegment::default(),segments,executor: Arc::new(Executor::multi_thread("webgraph").unwrap()),- node2id: Store::open(path.as_ref().join("node2id")),id2node: Store::open(path.as_ref().join("id2node")),meta,}}- fn id_and_increment(&mut self) -> NodeID {- let id = self.meta.next_node_id.into();- self.meta.next_node_id += 1;- id- }-fn id_or_assign(&mut self, node: &Node) -> NodeID {- match self.node2id(node) {- Some(id) => id,- None => {- let id = self.id_and_increment();-- self.node2id.put(node, &id);- self.id2node.put(&id, node);+ let id = node.id();- id- }+ if self.id2node(&id).is_none() {+ self.id2node.put(&id, node);}++ id}pub fn insert(&mut self, from: Node, to: Node, label: String) {@@ -374,24 +354,10 @@ impl Webgraph {pub fn merge(&mut self, mut other: Webgraph) {other.commit();- let mut mapping = Vec::new();-- for (node, other_id) in other.node2id.iter() {- match self.node2id(&node) {- Some(this_id) => mapping.push((other_id, this_id)),- None => {- let new_id = self.id_or_assign(&node);- mapping.push((other_id, new_id));- }- }- }- self.executor- .map(- |segment| segment.update_id_mapping(mapping.clone()),- other.segments.iter_mut(),- )- .expect("failed to merge webgraphs");+ for (other_id, node) in other.id2node.iter() {+ self.id2node.put(&other_id, &node);+ }for segment in other.segments {let id = segment.id();@@ -416,7 +382,6 @@ impl Webgraph {}self.save_metadata();- self.node2id.flush();self.id2node.flush();if self.segments.len() > 2 * num_cpus::get() {@@ -425,18 +390,14 @@ impl Webgraph {}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {- if let Some(node_id) = self.node2id(&node) {- self.inner_ingoing_edges(&node_id, true)- .into_iter()- .map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap(),- to: self.id2node(&edge.to).unwrap(),- label: edge.label.loaded().unwrap(),- })- .collect()- } else {- Vec::new()- }+ self.inner_ingoing_edges(&node.id(), true)+ .into_iter()+ .map(|edge| FullEdge {+ from: self.id2node(&edge.from).unwrap(),+ to: self.id2node(&edge.to).unwrap(),+ label: edge.label.loaded().unwrap(),+ })+ .collect()}pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge> {@@ -472,22 +433,14 @@ impl Webgraph {}pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {- if let Some(node_id) = self.node2id(&node) {- self.inner_outgoing_edges(&node_id, true)- .into_iter()- .map(|edge| FullEdge {- from: self.id2node(&edge.from).unwrap(),- to: self.id2node(&edge.to).unwrap(),- label: edge.label.loaded().unwrap(),- })- .collect()- } else {- Vec::new()- }- }-- pub fn node2id(&self, node: &Node) -> Option<NodeID> {- self.node2id.get(node)+ self.inner_outgoing_edges(&node.id(), true)+ .into_iter()+ .map(|edge| FullEdge {+ from: self.id2node(&edge.from).unwrap(),+ to: self.id2node(&edge.to).unwrap(),+ label: edge.label.loaded().unwrap(),+ })+ .collect()}pub fn id2node(&self, id: &NodeID) -> Option<Node> {@@ -495,11 +448,11 @@ impl Webgraph {}pub fn nodes(&self) -> impl Iterator<Item = NodeID> + '_ {- self.node2id.values()+ self.id2node.keys()}pub fn node_ids(&self) -> impl Iterator<Item = (Node, NodeID)> + '_ {- self.node2id.iter()+ self.id2node.iter().map(|(id, node)| (node, id))}pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {@@ -512,7 +465,7 @@ impl Webgraph {}self.segments- .sort_by_key(|segment| std::cmp::Reverse(segment.num_nodes()));+ .sort_by_key(|segment| std::cmp::Reverse(segment.estimate_num_nodes()));let mut candidates = Vec::with_capacity(num_segments);diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 454c2f69..24da4ac1 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -14,68 +14,28 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use lru::LruCache;-use rkyv::Archive;use std::{collections::{BTreeMap, HashSet},- fmt::Debug,path::Path,- sync::Mutex,};-use super::{- open_bin, save_bin, store::Store, Edge, FullStoredEdge, Loaded, NodeID, SmallStoredEdge,-};+use super::{store::Store, Edge, FullStoredEdge, Loaded, NodeID, SmallStoredEdge};const FULL_ADJACENCY_STORE: &str = "full_adjacency";const FULL_REVERSED_ADJACENCY_STORE: &str = "full_reversed_adjacency";const SMALL_ADJACENCY_STORE: &str = "small_adjacency";const SMALL_REVERSED_ADJACENCY_STORE: &str = "small_reversed_adjacency";-const ID_MAPPING_STORE: &str = "id_mapping";-const REV_ID_MAPPING_STORE: &str = "rev_id_mapping";-const META_STORE: &str = "meta.bin";--#[derive(- Debug,- Clone,- Copy,- serde::Serialize,- serde::Deserialize,- Archive,- rkyv::Serialize,- rkyv::Deserialize,- PartialEq,- Eq,- PartialOrd,- Ord,- Hash,-)]-#[archive_attr(derive(Eq, Hash, PartialEq, Debug))]-pub struct SegmentNodeID(u64);--impl From<u64> for SegmentNodeID {- fn from(val: u64) -> Self {- Self(val)- }-}struct MergePacket {- next_segment_node_id: SegmentNodeID,- new_id_mapping: Store<NodeID, SegmentNodeID>,- new_rev_id_mapping: Store<SegmentNodeID, NodeID>,- new_full_adjacency: Store<SegmentNodeID, HashSet<FullStoredEdge>>,- new_small_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,+ new_full_adjacency: Store<NodeID, HashSet<FullStoredEdge>>,+ new_small_adjacency: Store<NodeID, HashSet<SmallStoredEdge>>,}pub struct StoredSegment {- full_adjacency: Store<SegmentNodeID, HashSet<FullStoredEdge>>,- full_reversed_adjacency: Store<SegmentNodeID, HashSet<FullStoredEdge>>,- small_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,- small_reversed_adjacency: Store<SegmentNodeID, HashSet<SmallStoredEdge>>,- id_mapping: Store<NodeID, SegmentNodeID>,- rev_id_mapping: Store<SegmentNodeID, NodeID>,- rev_id_mapping_cache: Mutex<LruCache<SegmentNodeID, NodeID>>,- meta: Meta,+ full_adjacency: Store<NodeID, HashSet<FullStoredEdge>>,+ full_reversed_adjacency: Store<NodeID, HashSet<FullStoredEdge>>,+ small_adjacency: Store<NodeID, HashSet<SmallStoredEdge>>,+ small_reversed_adjacency: Store<NodeID, HashSet<SmallStoredEdge>>,id: String,folder_path: String,}@@ -91,9 +51,6 @@ impl StoredSegment {small_reversed_adjacency: Store::open(folder_path.as_ref().join(SMALL_REVERSED_ADJACENCY_STORE),),- id_mapping: Store::open(folder_path.as_ref().join(ID_MAPPING_STORE)),- rev_id_mapping: Store::open(folder_path.as_ref().join(REV_ID_MAPPING_STORE)),- meta: open_bin(folder_path.as_ref().join(META_STORE)),folder_path: folder_path.as_ref().as_os_str()@@ -101,101 +58,82 @@ impl StoredSegment {.unwrap().to_string(),id,- rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),}}- pub fn num_nodes(&self) -> usize {- self.meta.num_nodes as usize- }-- fn id_mapping(&self, node: &NodeID) -> Option<SegmentNodeID> {- self.id_mapping.get(node)- }-- fn rev_id_mapping(&self, node: &SegmentNodeID) -> Option<NodeID> {- // check cache- let mut guard = self.rev_id_mapping_cache.lock().unwrap();- if let Some(node_id) = guard.get(node) {- return Some(*node_id);- }-- let node_id = self.rev_id_mapping.get(node);-- if let Some(node_id) = node_id {- guard.put(*node, node_id);- }-- node_id+ pub fn estimate_num_nodes(&self) -> usize {+ self.small_adjacency.estimate_len()}pub fn outgoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {- self.id_mapping(node)- .and_then(|segment_id| {- if load_label {- self.full_adjacency.get(&segment_id).map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: *node,- to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),- label: Loaded::Some(edge.label),- })- .collect()- })- } else {- self.small_adjacency.get(&segment_id).map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: *node,- to: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),- label: Loaded::NotYet,- })- .collect()- })- }- })- .unwrap_or_default()+ if load_label {+ self.full_adjacency+ .get(node)+ .map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: *node,+ to: edge.other,+ label: Loaded::Some(edge.label),+ })+ .collect()+ })+ .unwrap_or_default()+ } else {+ self.small_adjacency+ .get(node)+ .map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: *node,+ to: edge.other,+ label: Loaded::NotYet,+ })+ .collect()+ })+ .unwrap_or_default()+ }}pub fn ingoing_edges(&self, node: &NodeID, load_label: bool) -> Vec<Edge> {- self.id_mapping(node)- .and_then(|segment_id| {- if load_label {- self.full_reversed_adjacency.get(&segment_id).map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),- to: *node,- label: Loaded::Some(edge.label),- })- .collect()- })- } else {- self.small_reversed_adjacency.get(&segment_id).map(|edges| {- edges- .into_iter()- .map(move |edge| Edge {- from: self.rev_id_mapping(&SegmentNodeID(edge.other.0)).unwrap(),- to: *node,- label: Loaded::NotYet,- })- .collect()- })- }- })- .unwrap_or_default()+ if load_label {+ self.full_reversed_adjacency+ .get(node)+ .map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: edge.other,+ to: *node,+ label: Loaded::Some(edge.label),+ })+ .collect()+ })+ .unwrap_or_default()+ } else {+ self.small_reversed_adjacency+ .get(node)+ .map(|edges| {+ edges+ .into_iter()+ .map(move |edge| Edge {+ from: edge.other,+ to: *node,+ label: Loaded::NotYet,+ })+ .collect()+ })+ .unwrap_or_default()+ }}fn flush(&mut self) {self.full_adjacency.flush();self.full_reversed_adjacency.flush();- self.id_mapping.flush();- self.rev_id_mapping.flush();-- save_bin(&self.meta, self.path().join(META_STORE));+ self.small_adjacency.flush();+ self.small_reversed_adjacency.flush();}pub fn id(&self) -> String {@@ -207,19 +145,13 @@ impl StoredSegment {}pub fn edges(&self) -> impl Iterator<Item = Edge> + '_ {- self.full_adjacency- .iter()- .flat_map(move |(node_id, edges)| {- let from = self.rev_id_mapping(&node_id).unwrap();-- edges.into_iter().map(move |stored_edge| Edge {- from,- to: self- .rev_id_mapping(&SegmentNodeID(stored_edge.other.0))- .unwrap(),- label: Loaded::NotYet,- })+ self.small_adjacency.iter().flat_map(move |(from, edges)| {+ edges.into_iter().map(move |stored_edge| Edge {+ from,+ to: stored_edge.other,+ label: Loaded::NotYet,})+ })}fn merge_adjacency<F1>(edges_fn: F1, packet: &mut MergePacket, segments: &[Self])@@ -230,82 +162,27 @@ impl StoredSegment {return;}- let mut i = 0;+ for segment in segments {+ for node_id in segment+ .small_adjacency+ .keys()+ .chain(segment.small_reversed_adjacency.keys())+ {+ let mut existing = packet.new_full_adjacency.get(&node_id).unwrap_or_default();- while i < segments.len() {- let segment = &segments[i];-- for node_id in segment.id_mapping.keys() {- let mut node_seen_before = false;- let mut j = 0;- while j < i {- if segments[j].id_mapping.contains_key(&node_id) {- node_seen_before = true;- break;- }- j += 1;+ if let Some(edges) = edges_fn(segment, &node_id) {+ existing.extend(edges);}- if !node_seen_before {- if !packet.new_id_mapping.contains_key(&node_id) {- packet- .new_id_mapping- .put(&node_id, &packet.next_segment_node_id);- packet- .new_rev_id_mapping- .put(&packet.next_segment_node_id, &node_id);- packet.next_segment_node_id.0 += 1;- }-- let mut edges = HashSet::new();-- let mut j = i;-- while j < segments.len() {- let segment = &segments[j];- if let Some(edges_j) = edges_fn(segment, &node_id) {- for edge in edges_j {- let node = segment.rev_id_mapping(&edge.other).unwrap();-- if !packet.new_id_mapping.contains_key(&node) {- packet- .new_id_mapping- .put(&node, &packet.next_segment_node_id);- packet- .new_rev_id_mapping- .put(&packet.next_segment_node_id, &node);- packet.next_segment_node_id.0 += 1;- }-- let new_segment_node_id = packet.new_id_mapping.get(&node).unwrap();-- let new_edge = FullStoredEdge {- other: new_segment_node_id,- label: edge.label,- };-- edges.insert(new_edge);- }- }-- j += 1;- }-- if !edges.is_empty() {- let segment_node_id = packet.new_id_mapping.get(&node_id).unwrap();- let small_edges: HashSet<SmallStoredEdge> = edges- .iter()- .map(|edge| SmallStoredEdge { other: edge.other })- .collect();-- packet.new_full_adjacency.put(&segment_node_id, &edges);- packet- .new_small_adjacency- .put(&segment_node_id, &small_edges);- }- }+ packet.new_full_adjacency.put(&node_id, &existing);+ packet.new_small_adjacency.put(+ &node_id,+ &existing+ .into_iter()+ .map(|edge| SmallStoredEdge { other: edge.other })+ .collect::<HashSet<_>>(),+ );}- i += 1;}}@@ -319,28 +196,13 @@ impl StoredSegment {let new_full_adjacency = Store::open(new_path.join(FULL_ADJACENCY_STORE));let new_small_adjacency = Store::open(new_path.join(SMALL_ADJACENCY_STORE));- let next_segment_node_id = SegmentNodeID(0);-- let new_id_mapping = Store::open(new_path.join(ID_MAPPING_STORE));- let new_rev_id_mapping = Store::open(new_path.join(REV_ID_MAPPING_STORE));-let mut packet = MergePacket {- next_segment_node_id,- new_id_mapping,- new_rev_id_mapping,new_full_adjacency,new_small_adjacency,};Self::merge_adjacency(- |segment: &StoredSegment, node_id: &NodeID| {- segment.id_mapping.get(node_id).map(|segment_node_id| {- segment- .full_adjacency- .get(&segment_node_id)- .unwrap_or_default()- })- },+ |segment: &StoredSegment, node_id: &NodeID| segment.full_adjacency.get(node_id),&mut packet,&segments,);@@ -352,21 +214,13 @@ impl StoredSegment {Store::open(new_path.join(SMALL_REVERSED_ADJACENCY_STORE));let mut packet = MergePacket {- next_segment_node_id: packet.next_segment_node_id,- new_id_mapping: packet.new_id_mapping,- new_rev_id_mapping: packet.new_rev_id_mapping,new_full_adjacency: new_full_reversed_adjacency,new_small_adjacency: new_small_reversed_adjacency,};Self::merge_adjacency(|segment: &StoredSegment, node_id: &NodeID| {- segment.id_mapping.get(node_id).map(|segment_node_id| {- segment- .full_reversed_adjacency- .get(&segment_node_id)- .unwrap_or_default()- })+ segment.full_reversed_adjacency.get(node_id)},&mut packet,&segments,@@ -377,81 +231,24 @@ impl StoredSegment {small_adjacency: new_small_adjacency,full_reversed_adjacency: packet.new_full_adjacency,small_reversed_adjacency: packet.new_small_adjacency,- id_mapping: packet.new_id_mapping,- rev_id_mapping: packet.new_rev_id_mapping,- meta: Meta {- num_nodes: packet.next_segment_node_id.0,- },id: new_segment_id,folder_path: new_path.to_str().unwrap().to_string(),- rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),};res.flush();res}-- pub fn update_id_mapping(&mut self, mapping: Vec<(NodeID, NodeID)>) {- let mut new_mappings = Vec::with_capacity(mapping.len());-- for (old_id, new_id) in mapping {- if let Some(segment_id) = self.id_mapping(&old_id) {- self.id_mapping.remove(&old_id);- self.rev_id_mapping.remove(&segment_id);-- new_mappings.push((segment_id, new_id));- }- }-- self.id_mapping.batch_put(- new_mappings- .iter()- .map(|(segment_id, new_id)| (new_id, segment_id)),- );- self.rev_id_mapping.batch_put(- new_mappings- .iter()- .map(|(segment_id, new_id)| (segment_id, new_id)),- );-- self.flush();- }-}--#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, Default)]-struct Meta {- num_nodes: u64,}#[derive(Default)]pub struct LiveSegment {- adjacency: BTreeMap<SegmentNodeID, HashSet<FullStoredEdge>>,- reversed_adjacency: BTreeMap<SegmentNodeID, HashSet<FullStoredEdge>>,- id_mapping: BTreeMap<NodeID, SegmentNodeID>,- rev_id_mapping: BTreeMap<SegmentNodeID, NodeID>,- next_id: u64,+ adjacency: BTreeMap<NodeID, HashSet<FullStoredEdge>>,+ reversed_adjacency: BTreeMap<NodeID, HashSet<FullStoredEdge>>,}impl LiveSegment {- fn get_or_create_id(&mut self, id: NodeID) -> SegmentNodeID {- if let Some(segment_id) = self.id_mapping.get(&id) {- *segment_id- } else {- let segment_id = SegmentNodeID(self.next_id);- self.next_id += 1;-- self.id_mapping.insert(id, segment_id);- self.rev_id_mapping.insert(segment_id, id);-- segment_id- }- }-pub fn insert(&mut self, from: NodeID, to: NodeID, label: String) {- let from = self.get_or_create_id(from);- let to = self.get_or_create_id(to);-self.adjacency.entry(from).or_default()@@ -470,7 +267,7 @@ impl LiveSegment {let segment_id = uuid::Uuid::new_v4().to_string();let path = folder_path.as_ref().join(&segment_id);- let small_adjacency: BTreeMap<SegmentNodeID, HashSet<SmallStoredEdge>> = self+ let small_adjacency: BTreeMap<NodeID, HashSet<SmallStoredEdge>> = self.adjacency.clone().into_iter()@@ -484,7 +281,7 @@ impl LiveSegment {}).collect();- let small_rev_adjacency: BTreeMap<SegmentNodeID, HashSet<SmallStoredEdge>> = self+ let small_rev_adjacency: BTreeMap<NodeID, HashSet<SmallStoredEdge>> = self.reversed_adjacency.clone().into_iter()@@ -510,25 +307,13 @@ impl LiveSegment {let small_rev_adjacency_store = Store::open(path.join(SMALL_REVERSED_ADJACENCY_STORE));small_rev_adjacency_store.batch_put(small_rev_adjacency.iter());- let id_mapping = Store::open(path.join(ID_MAPPING_STORE));- id_mapping.batch_put(self.id_mapping.iter());-- let rev_id_mapping = Store::open(path.join(REV_ID_MAPPING_STORE));- rev_id_mapping.batch_put(self.rev_id_mapping.iter());-let mut stored_segment = StoredSegment {small_adjacency: small_adjacency_store,small_reversed_adjacency: small_rev_adjacency_store,full_adjacency,full_reversed_adjacency,- id_mapping,- rev_id_mapping,- meta: Meta {- num_nodes: self.next_id,- },folder_path: path.as_os_str().to_str().unwrap().to_string(),id: segment_id,- rev_id_mapping_cache: Mutex::new(LruCache::new(1_000_000.try_into().unwrap())),};stored_segment.flush();@@ -537,7 +322,7 @@ impl LiveSegment {}pub fn is_empty(&self) -> bool {- self.id_mapping.is_empty()+ self.adjacency.is_empty()}}diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 103dd094..97c85e44 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -89,12 +89,6 @@ whereself.db.write(batch).unwrap();}- pub fn contains_key(&self, key: &K) -> bool {- let bytes = bincode::serialize(key).unwrap();-- self.db.get(bytes).unwrap().is_some()- }-pub fn keys(&self) -> impl Iterator<Item = K> + '_ {let mut read_opts = rocksdb::ReadOptions::default();@@ -108,19 +102,6 @@ where})}- pub fn values(&self) -> impl Iterator<Item = V> + '_ {- let mut read_opts = rocksdb::ReadOptions::default();-- read_opts.set_readahead_size(4_194_304); // 4 MB-- self.db- .iterator_opt(rocksdb::IteratorMode::Start, read_opts)- .map(|res| {- let (_, val) = res.unwrap();- bincode::deserialize(&val).unwrap()- })- }-pub fn iter(&self) -> impl Iterator<Item = (K, V)> + '_ {let mut read_opts = rocksdb::ReadOptions::default();@@ -141,9 +122,11 @@ whereself.db.flush().unwrap();}- pub fn remove(&self, key: &K) {- let bytes = bincode::serialize(key).unwrap();- self.db.delete(bytes).unwrap();+ pub fn estimate_len(&self) -> usize {+ self.db+ .property_int_value("rocksdb.estimate-num-keys")+ .unwrap()+ .unwrap_or(0) as usize}}diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 89978985..4c37f154 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -15,9 +15,9 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{enum_map::EnumSet,- prehashed::{hash, split_u128},+ prehashed::hash,schema::{FastField, TextField},- simhash, tokenizer,+ simhash, split_u128, tokenizer,webgraph::NodeID,Error, Result,};@@ -218,12 +218,37 @@ impl Webpage {self.pre_computed_score,);- doc.add_u64(- schema- .get_field(Field::Fast(FastField::HostNodeID).name())- .expect("Failed to get node_id field"),- self.node_id.map(|n| n.0).unwrap_or(u64::MAX),- );+ match &self.node_id {+ Some(node_id) => {+ let [node_id1, node_id2] = split_u128(node_id.bit_128());+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::HostNodeID1).name())+ .expect("Failed to get node_id field 1"),+ node_id1,+ );+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::HostNodeID2).name())+ .expect("Failed to get node_id field 2"),+ node_id2,+ );+ }+ None => {+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::HostNodeID1).name())+ .expect("Failed to get node_id field 1"),+ u64::MAX,+ );+ doc.add_u64(+ schema+ .get_field(Field::Fast(FastField::HostNodeID2).name())+ .expect("Failed to get node_id field 2"),+ u64::MAX,+ );+ }+ }doc.add_text(schema@@ -1041,7 +1066,8 @@ impl Html {| Field::Fast(FastField::FetchTimeMs)| Field::Fast(FastField::PreComputedScore)| Field::Fast(FastField::Region)- | Field::Fast(FastField::HostNodeID)+ | Field::Fast(FastField::HostNodeID1)+ | Field::Fast(FastField::HostNodeID2)| Field::Text(TextField::DmozDescription) => {}}}