web search done right
remove 'linksfrom' operator again
Mikkel Denker88df429dad78
diff --git a/crates/core/src/backlink_grouper.rs b/crates/core/src/backlink_grouper.rsindex d8c6b634..7ae68130 100644--- a/crates/core/src/backlink_grouper.rs+++ b/crates/core/src/backlink_grouper.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::{log_group::HarmonicRankGroup, webgraph::FullEdge};+use crate::{log_group::HarmonicRankGroup, webgraph::Edge};/// Number of groups to divide the backlinks into./// If this is changed, also change the grouped backlink fields in the schema.@@ -23,7 +23,7 @@ const NUM_GROUPS: u64 = 10;#[derive(Debug)]pub struct Group {group: u64,- backlinks: Vec<FullEdge>,+ backlinks: Vec<Edge<String>>,}impl Group {@@ -34,7 +34,7 @@ impl Group {}}- fn insert(&mut self, backlink: FullEdge) {+ fn insert(&mut self, backlink: Edge<String>) {self.backlinks.push(backlink);}@@ -42,7 +42,7 @@ impl Group {self.group}- pub fn backlinks(&self) -> &[FullEdge] {+ pub fn backlinks(&self) -> &[Edge<String>] {&self.backlinks}}@@ -66,7 +66,7 @@ impl GroupedBacklinks {self.groups.get(group as usize)}- fn add(&mut self, group: u64, backlink: FullEdge) {+ fn add(&mut self, group: u64, backlink: Edge<String>) {if let Some(group) = self.groups.get_mut(group as usize) {group.insert(backlink)}@@ -93,7 +93,9 @@ impl BacklinkGrouper {}}- pub fn add(&mut self, backlink: FullEdge, rank: u64) {+ pub fn add(&mut self, backlink: Edge<String>) {+ let rank = backlink.from.host_rank();+let group = self.grouper.group(rank);self.groups.add(group, backlink);}@@ -105,93 +107,73 @@ impl BacklinkGrouper {#[cfg(test)]mod tests {- use crate::webgraph::Node;+ use crate::webgraph::NodeDatum;use super::*;#[test]fn test_grouped_backlinks() {let mut grouper = BacklinkGrouper::new(10);- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 0,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 1,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 2,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 3,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 4,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 5,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 6,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 7,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 8,- );- grouper.add(- FullEdge {- from: Node::from("https://a.com"),- to: Node::from("https://b.com"),- label: String::new(),- },- 9,- );+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 0u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 1u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 2u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 3u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 4u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 5u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 6u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 7u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 8u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });+ grouper.add(Edge {+ from: NodeDatum::new(0u64, 9u64),+ to: NodeDatum::new(1u64, 0u64),+ label: String::new(),+ rel: Default::default(),+ });{let groups = grouper.groups();diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex c930dd1a..911c2fac 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -77,13 +77,13 @@ impl Workers {{let d = dist + 1;- let current_dist = distances.entry(outgoing).or_insert(u8::MAX);+ let current_dist = distances.entry(outgoing.node()).or_insert(u8::MAX);if d < *current_dist {*current_dist = d;if d < max_dist {- queue.push(cmp::Reverse((d, outgoing)));+ queue.push(cmp::Reverse((d, outgoing.node())));}}}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rsindex 4c9765fb..a5efc4b4 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs@@ -51,7 +51,7 @@ impl CentralityMapper {batch: &[webgraph::Edge<()>],dht: &DhtConn<CentralityTables>,) -> BTreeMap<webgraph::NodeID, HyperLogLog<64>> {- let nodes: Vec<_> = batch.iter().map(|edge| edge.from).collect();+ let nodes: Vec<_> = batch.iter().map(|edge| edge.from.node()).collect();if nodes.is_empty() {return BTreeMap::new();@@ -95,9 +95,12 @@ impl CentralityMapper {let updates: Vec<_> = batch.iter().map(|edge| {- let mut counter = old_counters.get(&edge.from).cloned().unwrap_or_default();- counter.add(edge.from.as_u64());- (edge.to, counter)+ let mut counter = old_counters+ .get(&edge.from.node())+ .cloned()+ .unwrap_or_default();+ counter.add(edge.from.node().as_u64());+ (edge.to.node(), counter)}).collect();@@ -270,7 +273,7 @@ impl CentralityMapper {.graph().edges().filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))- .filter(|e| changed_nodes.contains(e.from.as_u64()))+ .filter(|e| changed_nodes.contains(e.from.node().as_u64())){batch.push(edge);if batch.len() >= batch_size {diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rsindex c4c4fa66..72b35193 100644--- a/crates/core/src/entrypoint/indexer/worker.rs+++ b/crates/core/src/entrypoint/indexer/worker.rs@@ -35,7 +35,7 @@ use crate::Result;use crate::index::Index;use crate::rake::RakeModel;use crate::ranking::SignalComputer;-use crate::webgraph::{self, EdgeLimit, FullEdge, Node, NodeID};+use crate::webgraph::{self, Edge, EdgeLimit, Node, NodeID};use crate::webpage::{safety_classifier, Html, Webpage};const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);@@ -87,16 +87,17 @@ impl Webgraph {&self,ids: Vec<NodeID>,limit: EdgeLimit,- ) -> Vec<Vec<FullEdge>> {+ ) -> Vec<Vec<Edge<String>>> {let edges = match self {Self::Remote(webgraph) => {- crate::block_on(webgraph.batch_ingoing_edges_by_id(&ids, limit)).unwrap_or_default()+ crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit))+ .unwrap_or_default()}Self::Local(webgraph) => {let mut res = Vec::new();for id in ids {- res.push(webgraph.ingoing_edges_by_id(&id, limit));+ res.push(webgraph.raw_ingoing_edges_with_labels(&id, limit));}res@@ -405,18 +406,7 @@ impl IndexingWorker {BacklinkGrouper::new(self.host_centrality_rank_store.len() as u64);for backlink in backlinks {- let node = backlink.from.clone().into_host().id();-- if Some(node) == page.node_id {- continue;- }-- let rank = self- .host_centrality_rank_store- .get(&node)- .unwrap()- .unwrap_or(u64::MAX);- grouper.add(backlink, rank);+ grouper.add(backlink);}page.set_grouped_backlinks(grouper.groups())diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 5d26c26e..42ef0156 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -53,7 +53,6 @@ sonic_service!([GetNode,IngoingEdges,- IngoingEdgesById,OutgoingEdges,RawIngoingEdges,RawOutgoingEdges,@@ -90,20 +89,6 @@ impl Message<WebGraphService> for IngoingEdges {}}-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct IngoingEdgesById {- pub node: NodeID,- pub limit: EdgeLimit,-}--impl Message<WebGraphService> for IngoingEdgesById {- type Response = Vec<FullEdge>;-- async fn handle(self, server: &WebGraphService) -> Self::Response {- server.graph.ingoing_edges_by_id(&self.node, self.limit)- }-}-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct OutgoingEdges {pub node: Node,diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 863ed9ac..cf03025f 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -236,7 +236,7 @@ mod tests {ranking::{Ranker, SignalComputer},search_ctx::Ctx,searcher::SearchQuery,- webgraph::{FullEdge, Node},+ webgraph::{Edge, NodeDatum},webpage::{schema_org, Html, Webpage},OneOrMany,};@@ -511,10 +511,11 @@ mod tests {..Default::default()};- webpage.set_backlinks(vec![FullEdge {- from: Node::from("https://www.a.com"),- to: Node::from("https://www.b.com"),+ webpage.set_backlinks(vec![Edge {+ from: NodeDatum::new(0u64, 0),+ to: NodeDatum::new(1u64, 0),label: "B site is great".to_string(),+ rel: Default::default(),}]);index.insert(&webpage).expect("failed to insert webpage");diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rsindex 868deac3..ebd592ba 100644--- a/crates/core/src/query/mod.rs+++ b/crates/core/src/query/mod.rs@@ -206,13 +206,7 @@ impl tantivy::query::Query for Query {#[cfg(test)]mod tests {- use crate::{- index::Index,- rand_words,- searcher::LocalSearcher,- webgraph::{FullEdge, Node},- webpage::Webpage,- };+ use crate::{index::Index, rand_words, searcher::LocalSearcher, webpage::Webpage};use proptest::prelude::*;use super::*;@@ -641,86 +635,6 @@ mod tests {assert_eq!(result.webpages[0].url, "https://www.first.com/");}- #[test]- fn link_from_query() {- let mut index = Index::temporary().expect("Unable to open index");-- let mut webpage = Webpage::test_parse(- r#"- <html>- <head>- <title>Test website</title>- </head>- <body>- This is a test website- </body>- </html>- "#,- "https://www.first.com",- )- .unwrap();-- webpage.set_backlinks(vec![FullEdge {- from: Node::from("https://www.second.com/example/abc"),- to: Node::from("https://www.first.com"),- label: String::new(),- }]);-- index.insert(&webpage).expect("failed to insert webpage");- let mut webpage = Webpage::test_parse(- r#"- <html>- <head>- <title>Test test</title>- </head>- <body>- This test page does not contain the forbidden word- <a href="https://www.first.com">First</a>- </body>- </html>- "#,- "https://www.second.com/example/abc",- )- .unwrap();- webpage.set_backlinks(vec![]);-- index.insert(&webpage).expect("failed to insert webpage");- index.commit().expect("failed to commit index");- let searcher = LocalSearcher::from(index);-- let query = SearchQuery {- query: "test linkfrom:second.com".to_string(),- ..Default::default()- };- let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.webpages.len(), 1);- assert_eq!(result.webpages[0].url, "https://www.first.com/");-- let query = SearchQuery {- query: "test linkfrom:www.second.com".to_string(),- ..Default::default()- };- let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.webpages.len(), 1);- assert_eq!(result.webpages[0].url, "https://www.first.com/");-- let query = SearchQuery {- query: "test linkfrom:second.com/example".to_string(),- ..Default::default()- };- let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.webpages.len(), 1);- assert_eq!(result.webpages[0].url, "https://www.first.com/");-- let query = SearchQuery {- query: "test linksfrom:second.com/example/abc".to_string(),- ..Default::default()- };- let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.webpages.len(), 1);- assert_eq!(result.webpages[0].url, "https://www.first.com/");- }-#[test]fn title_query() {let mut index = Index::temporary().expect("Unable to open index");diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rsindex 5590fefe..c454bedb 100644--- a/crates/core/src/query/parser/mod.rs+++ b/crates/core/src/query/parser/mod.rs@@ -130,17 +130,6 @@ fn links_to_field(input: &str) -> nom::IResult<&str, Term> {Ok((input, Term::LinkTo(output.to_string())))}-fn links_from_field(input: &str) -> nom::IResult<&str, Term> {- // parse 'linksfrom:' and then a simple term- let (input, _) = nom::branch::alt((- nom::bytes::complete::tag("linksfrom:"),- nom::bytes::complete::tag("linkfrom:"),- ))(input)?;- let (input, output) = simple_str(input)?;-- Ok((input, Term::LinkFrom(output.to_string())))-}-fn title_field(input: &str) -> nom::IResult<&str, Term> {// parse 'title:' and then a simple termlet (input, _) = nom::bytes::complete::tag("intitle:")(input)?;@@ -169,7 +158,6 @@ fn field_selector(input: &str) -> nom::IResult<&str, Term> {nom::branch::alt((site_field,links_to_field,- links_from_field,title_field,body_field,url_field,diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rsindex 01631829..7eb26d31 100644--- a/crates/core/src/query/parser/term.rs+++ b/crates/core/src/query/parser/term.rs@@ -99,7 +99,6 @@ pub enum Term {SimpleOrPhrase(SimpleOrPhrase),Site(String),LinkTo(String),- LinkFrom(String),Title(SimpleOrPhrase),Body(SimpleOrPhrase),Url(SimpleOrPhrase),@@ -114,7 +113,6 @@ impl std::fmt::Display for Term {Term::Not(term) => write!(f, "-{}", term),Term::Site(site) => write!(f, "site:{}", site),Term::LinkTo(site) => write!(f, "linkto:{}", site),- Term::LinkFrom(site) => write!(f, "linkfrom:{}", site),Term::Title(title) => write!(f, "intitle:{}", title),Term::Body(body) => write!(f, "inbody:{}", body),Term::Url(url) => write!(f, "inurl:{}", url),@@ -136,7 +134,6 @@ impl Term {Term::SimpleOrPhrase(s) => Term::SimpleOrPhrase(s.truncate()),Term::Site(s) => Term::Site(s),Term::LinkTo(s) => Term::LinkTo(s),- Term::LinkFrom(s) => Term::LinkFrom(s),Term::Title(s) => Term::Title(s.truncate()),Term::Body(s) => Term::Body(s.truncate()),Term::Url(s) => Term::Url(s.truncate()),diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rsindex 214945fb..2840e221 100644--- a/crates/core/src/query/plan/node.rs+++ b/crates/core/src/query/plan/node.rs@@ -135,10 +135,6 @@ impl Node {text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),field: text_field::Links.into(),}),- ParserTerm::LinkFrom(s) => Node::Term(Term {- text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),- field: text_field::Backlinks.into(),- }),ParserTerm::Title(t) => Node::Term(Term {text: t,field: text_field::Title.into(),diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex b818783f..dd31e635 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -160,7 +160,7 @@ mod tests {graph.raw_ingoing_edges(node, EdgeLimit::Unlimited).into_iter()- .map(|e| e.from.as_u64())+ .map(|e| e.from.node().as_u64()).collect(),)}diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rsindex 26dc6c2b..fcc31d64 100644--- a/crates/core/src/ranking/mod.rs+++ b/crates/core/src/ranking/mod.rs@@ -130,7 +130,7 @@ mod tests {index::Index,models::dual_encoder::DualEncoder,searcher::{LocalSearcher, SearchQuery},- webgraph::{FullEdge, Node},+ webgraph::{Edge, NodeDatum},webpage::{Html, Webpage},};@@ -433,10 +433,11 @@ mod tests {..Default::default()};- webpage.set_backlinks(vec![FullEdge {- from: Node::from("https://www.backlink.com"),- to: Node::from("https://www.first.com"),+ webpage.set_backlinks(vec![Edge {+ from: NodeDatum::new(0u64, 0),+ to: NodeDatum::new(1u64, 1),label: "test this is the best test site".to_string(),+ rel: Default::default(),}]);index.insert(&webpage).expect("failed to insert webpage");diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rsindex b540ba51..3a808018 100644--- a/crates/core/src/schema/text_field.rs+++ b/crates/core/src/schema/text_field.rs@@ -199,7 +199,6 @@ pub enum TextFieldEnum {Keywords,KeyPhrases,Links,- Backlinks,BacklinkLabelsGroup0,BacklinkLabelsGroup1,BacklinkLabelsGroup2,@@ -248,7 +247,6 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,Keywords,KeyPhrases,Links,- Backlinks,BacklinkLabelsGroup0,BacklinkLabelsGroup1,BacklinkLabelsGroup2,@@ -1586,51 +1584,6 @@ impl TextField for Links {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]-pub struct Backlinks;-impl TextField for Backlinks {- fn name(&self) -> &str {- "backlinks"- }-- fn has_pos(&self) -> bool {- true- }-- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {- FieldTokenizer::Url(UrlTokenizer)- }-- fn add_webpage_tantivy(- &self,- webpage: &crate::webpage::Webpage,- doc: &mut TantivyDocument,- index: &crate::inverted_index::InvertedIndex,- ) -> Result<()> {- doc.add_text(- self.tantivy_field(index.schema_ref())- .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),- webpage- .backlinks()- .iter()- .map(|l| l.from.as_str().to_string())- .join("\n"),- );-- Ok(())- }-- fn add_html_tantivy(- &self,- _: &Html,- _: &mut FnCache,- _: &mut TantivyDocument,- _: &crate::inverted_index::InvertedIndex,- ) -> Result<()> {- Ok(())- }-}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]pub struct Keywords;impl TextField for Keywords {diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex a0c37032..1ee3d739 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -201,7 +201,7 @@ whereself.batch_raw_ingoing(nodes, EdgeLimit::Limit(128)).await.into_iter()- .map(|edges| edges.into_iter().map(|edge| edge.from).collect())+ .map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect()).collect()}}diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex 31b91617..f436aa48 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -82,7 +82,7 @@ impl SimilarHostsFinder {let backlink_nodes = in_edges.iter().flatten()- .map(|e| e.from)+ .map(|e| e.from.node()).unique().collect::<Vec<_>>();@@ -95,7 +95,7 @@ impl SimilarHostsFinder {let potential_nodes: Vec<_> = outgoing_edges.iter().flatten()- .map(|e| e.to)+ .map(|e| e.to.node()).unique().filter(|n| !nodes.contains(n)).collect();diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rsindex 4dbecb19..b4a69d29 100644--- a/crates/core/src/webgraph/centrality/betweenness.rs+++ b/crates/core/src/webgraph/centrality/betweenness.rs@@ -71,7 +71,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)while let Some(v) = q.pop_front() {stack.push(v);for edge in graph.raw_outgoing_edges(&v, EdgeLimit::Unlimited) {- let w = edge.to;+ let w = edge.to.node();if !distances.contains_key(&w) {let dist_v = distances.get(&v).unwrap();diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rsindex d5505de6..b87e5f46 100644--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs+++ b/crates/core/src/webgraph/centrality/derived_harmonic.rs@@ -83,7 +83,7 @@ impl DerivedCentrality {let has_outgoing = BloomMap::new(8, num_nodes as u64, 0.01);page_graph.par_edges().for_each(|edge| {- has_outgoing.insert(&edge.from);+ has_outgoing.insert(&edge.from.node());});let has_outgoing = has_outgoing.finalize();@@ -103,7 +103,7 @@ impl DerivedCentrality {let mut ingoing: Vec<_> = page_graph.raw_ingoing_edges(&id, EdgeLimit::Limit(128)).into_iter()- .filter_map(|e| page_graph.id2node(&e.from))+ .filter_map(|e| page_graph.id2node(&e.from.node())).map(|n| n.into_host()).collect();ingoing.sort();diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex 8f82726a..d435a27b 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -85,9 +85,10 @@ fn update_changed_counters(.into_iter().filter(|e| !e.rel_flags().intersects(*SKIPPED_REL)){- if let (Some(counter_to), Some(counter_from)) =- (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))- {+ if let (Some(counter_to), Some(counter_from)) = (+ counters.new.get_mut(&edge.to.node()),+ counters.old.get(&edge.from.node()),+ ) {if counter_to.registers().iter()@@ -95,9 +96,9 @@ fn update_changed_counters(.any(|(to, from)| *from > *to){counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.as_u64());+ new_changed_nodes.insert(edge.to.node().as_u64());- new_exact_changed_nodes.insert(edge.to);+ new_exact_changed_nodes.insert(edge.to.node());has_changes.store(true, Ordering::Relaxed);}@@ -127,10 +128,11 @@ fn update_all_counters(.edges().filter(|e| !e.rel_flags().intersects(*SKIPPED_REL)).for_each(|edge| {- if changed_nodes.contains(edge.from.as_u64()) {- if let (Some(counter_to), Some(counter_from)) =- (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))- {+ if changed_nodes.contains(edge.from.node().as_u64()) {+ if let (Some(counter_to), Some(counter_from)) = (+ counters.new.get_mut(&edge.to.node()),+ counters.old.get(&edge.from.node()),+ ) {if counter_to.registers().iter()@@ -138,10 +140,10 @@ fn update_all_counters(.any(|(to, from)| *from > *to){counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.as_u64());+ new_changed_nodes.insert(edge.to.node().as_u64());if let Some(exact_changed_nodes) = &mut exact_changed_nodes {- exact_changed_nodes.insert(edge.to);+ exact_changed_nodes.insert(edge.to.node());}has_changes.store(true, Ordering::Relaxed);diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex 4cfcf67f..8c1e7fd5 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -18,7 +18,7 @@ use utoipa::ToSchema;use crate::webpage::html::links::RelFlags;-use super::{merge::NodeDatum, FullNodeID, Node, NodeID};+use super::{FullNodeID, Node, NodeDatum, NodeID};pub const MAX_LABEL_LENGTH: usize = 1024;@@ -55,8 +55,8 @@ pub struct Edge<L>whereL: EdgeLabel,{- pub from: NodeID,- pub to: NodeID,+ pub from: NodeDatum,+ pub to: NodeDatum,pub rel: RelFlags,pub label: L,}@@ -88,8 +88,8 @@ where{fn from(edge: InsertableEdge<L>) -> Self {Edge {- from: edge.from.id,- to: edge.to.id,+ from: NodeDatum::new(edge.from.id, u64::MAX),+ to: NodeDatum::new(edge.to.id, u64::MAX),rel: edge.rel,label: edge.label,}@@ -132,8 +132,8 @@ where{fn from(edge: SegmentEdge<L>) -> Self {Edge {- from: edge.from.node(),- to: edge.to.node(),+ from: edge.from,+ to: edge.to,rel: edge.rel,label: edge.label,}@@ -147,8 +147,8 @@ where{fn from(edge: Edge<L>) -> Self {SegmentEdge {- from: NodeDatum::new(edge.from, 0),- to: NodeDatum::new(edge.to, 0),+ from: edge.from,+ to: edge.to,rel: edge.rel,label: edge.label,}@@ -162,8 +162,8 @@ where{fn from(edge: InsertableEdge<L>) -> Self {SegmentEdge {- from: NodeDatum::new(edge.from.id, 0),- to: NodeDatum::new(edge.to.id, 0),+ from: NodeDatum::new(edge.from.id, u64::MAX),+ to: NodeDatum::new(edge.to.id, u64::MAX),rel: edge.rel,label: edge.label,}@@ -210,4 +210,9 @@ impl<L> StoredEdge<L> {pub fn rel(&self) -> RelFlags {self.rel}++ #[inline]+ pub fn other_host_rank(&self) -> u64 {+ self.other.host_rank()+ }}diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rsindex 6dad73e5..9eeffb55 100644--- a/crates/core/src/webgraph/merge.rs+++ b/crates/core/src/webgraph/merge.rs@@ -179,7 +179,7 @@ impl<'a, L> Iterator for EdgeMerger<'a, L> {if let Some(edge) = &res {while let Some(mut peek) = self.iters.peek_mut() {- if peek.0.peek().map(|x| x.other.id) == Some(edge.other.id) {+ if peek.0.peek().map(|x| x.other.node()) == Some(edge.other.node()) {peek.0.next().unwrap();} else {break;@@ -193,59 +193,6 @@ impl<'a, L> Iterator for EdgeMerger<'a, L> {}}-#[derive(Debug, Clone)]-pub struct NodeDatum {- id: NodeID,- sort_key: u64,-}--impl PartialOrd for NodeDatum {- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {- Some(self.cmp(other))- }-}--impl Ord for NodeDatum {- fn cmp(&self, other: &Self) -> std::cmp::Ordering {- self.sort_key- .cmp(&other.sort_key)- .then_with(|| self.id.cmp(&other.id))- }-}--impl PartialEq for NodeDatum {- fn eq(&self, other: &Self) -> bool {- self.id == other.id && self.sort_key == other.sort_key- }-}--impl Eq for NodeDatum {}--impl Hash for NodeDatum {- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {- self.id.hash(state);- self.sort_key.hash(state);- }-}--impl NodeDatum {- pub fn new(node: NodeID, sort_key: u64) -> Self {- Self { id: node, sort_key }- }-}--impl NodeDatum {- #[inline]- pub fn node(&self) -> NodeID {- self.id- }-- #[inline]- pub fn sort_key(&self) -> u64 {- self.sort_key- }-}-impl<L> Ord for StoredEdge<L> {fn cmp(&self, other: &Self) -> std::cmp::Ordering {self.other.cmp(&other.other)@@ -274,7 +221,7 @@ impl<L> Hash for StoredEdge<L> {#[cfg(test)]mod tests {- use crate::webpage::html::links::RelFlags;+ use crate::{webgraph::NodeDatum, webpage::html::links::RelFlags};use super::*;@@ -354,46 +301,46 @@ mod tests {#[test]fn test_datum_merge() {let a = vec![- StoredEdge::new(NodeDatum::new(1u64.into(), 1), RelFlags::default()),- StoredEdge::new(NodeDatum::new(2u64.into(), 4), RelFlags::default()),- StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(1u64, 1), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(2u64, 4), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),];let b = vec![- StoredEdge::new(NodeDatum::new(4u64.into(), 2), RelFlags::default()),- StoredEdge::new(NodeDatum::new(5u64.into(), 3), RelFlags::default()),- StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(4u64, 2), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(5u64, 3), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),];let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);- assert_eq!(merger.next().unwrap().other.sort_key(), 1);- assert_eq!(merger.next().unwrap().other.sort_key(), 2);- assert_eq!(merger.next().unwrap().other.sort_key(), 3);- assert_eq!(merger.next().unwrap().other.sort_key(), 4);- assert_eq!(merger.next().unwrap().other.sort_key(), 5);+ assert_eq!(merger.next().unwrap().other.host_rank(), 1);+ assert_eq!(merger.next().unwrap().other.host_rank(), 2);+ assert_eq!(merger.next().unwrap().other.host_rank(), 3);+ assert_eq!(merger.next().unwrap().other.host_rank(), 4);+ assert_eq!(merger.next().unwrap().other.host_rank(), 5);assert!(merger.next().is_none());}#[test]fn test_datum_merge_unequal_len() {let a = vec![- StoredEdge::new(NodeDatum::new(1u64.into(), 1), RelFlags::default()),- StoredEdge::new(NodeDatum::new(2u64.into(), 4), RelFlags::default()),- StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(1u64, 1), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(2u64, 4), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),];let b = vec![StoredEdge::new(- NodeDatum::new(4u64.into(), 2),+ NodeDatum::new(4u64, 2),RelFlags::default(),)];let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);- assert_eq!(merger.next().unwrap().other.sort_key(), 1);- assert_eq!(merger.next().unwrap().other.sort_key(), 2);- assert_eq!(merger.next().unwrap().other.sort_key(), 4);- assert_eq!(merger.next().unwrap().other.sort_key(), 5);+ assert_eq!(merger.next().unwrap().other.host_rank(), 1);+ assert_eq!(merger.next().unwrap().other.host_rank(), 2);+ assert_eq!(merger.next().unwrap().other.host_rank(), 4);+ assert_eq!(merger.next().unwrap().other.host_rank(), 5);assert!(merger.next().is_none());}}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 8898f519..838aabcc 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -220,7 +220,7 @@ impl Webgraph {|segment| segment.ingoing_edges_with_label(node_id, &limit),dedup,);- edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));+ edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));limit.apply(edges.into_iter())@@ -257,13 +257,13 @@ impl Webgraph {};let mut edges = self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup);- edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));+ edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));limit.apply(edges.into_iter()).map(|e| Edge {- from: e.from.node(),- to: e.to.node(),+ from: e.from,+ to: e.to,label: e.label,rel: e.rel,})@@ -284,13 +284,13 @@ impl Webgraph {|segment| segment.ingoing_edges_with_label(node, &limit),dedup,);- edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));+ edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));limit.apply(edges.into_iter()).map(|e| Edge {- from: e.from.node(),- to: e.to.node(),+ from: e.from,+ to: e.to,label: e.label,rel: e.rel,})@@ -312,13 +312,13 @@ impl Webgraph {dedup,);- edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));+ edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));limit.apply(edges.into_iter()).map(|e| Edge {- from: e.from.node(),- to: e.to.node(),+ from: e.from,+ to: e.to,label: e.label,rel: e.rel,})@@ -335,7 +335,7 @@ impl Webgraph {|segment| segment.outgoing_edges_with_label(&node.id(), &limit),dedup,);- edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));+ edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));limit.apply(edges.into_iter())@@ -354,13 +354,13 @@ impl Webgraph {};let mut edges = self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup);- edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));+ edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));limit.apply(edges.into_iter()).map(|e| Edge {- from: e.from.node(),- to: e.to.node(),+ from: e.from,+ to: e.to,label: e.label,rel: e.rel,})@@ -398,7 +398,7 @@ impl Webgraph {let mut rng = rand::thread_rng();let mut nodes = self.edges()- .map(|e| e.from)+ .map(|e| e.from.node()).unique().choose_multiple(&mut rng, num);nodes.shuffle(&mut rng);diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex 45b63127..9f02a28d 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -192,3 +192,62 @@ pub fn normalize_url(url: &Url) -> String {normalized}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct NodeDatum {+ id: NodeID,+ host_rank: u64,+}++impl PartialOrd for NodeDatum {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++impl Ord for NodeDatum {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.host_rank+ .cmp(&other.host_rank)+ .then_with(|| self.id.cmp(&other.id))+ }+}++impl PartialEq for NodeDatum {+ fn eq(&self, other: &Self) -> bool {+ self.id == other.id && self.host_rank == other.host_rank+ }+}++impl Eq for NodeDatum {}++impl std::hash::Hash for NodeDatum {+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {+ self.id.hash(state);+ self.host_rank.hash(state);+ }+}++impl NodeDatum {+ pub fn new<N>(node: N, host_rank: u64) -> Self+ where+ N: Into<NodeID>,+ {+ Self {+ id: node.into(),+ host_rank,+ }+ }+}++impl NodeDatum {+ #[inline]+ pub fn node(&self) -> NodeID {+ self.id+ }++ #[inline]+ pub fn host_rank(&self) -> u64 {+ self.host_rank+ }+}diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rsindex 2ecee109..ecc7b2eb 100644--- a/crates/core/src/webgraph/remote.rs+++ b/crates/core/src/webgraph/remote.rs@@ -33,7 +33,7 @@ use crate::{},},entrypoint::webgraph_server::{- GetNode, IngoingEdges, IngoingEdgesById, OutgoingEdges, PagesByHosts, RawIngoingEdges,+ GetNode, IngoingEdges, OutgoingEdges, PagesByHosts, RawIngoingEdges,RawIngoingEdgesWithLabels, RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,},Result,@@ -300,37 +300,6 @@ impl RemoteWebgraph {Ok(edges)}- pub async fn batch_ingoing_edges_by_id(- &self,- ids: &[NodeID],- limit: EdgeLimit,- ) -> Result<Vec<Vec<FullEdge>>> {- let reqs: Vec<_> = ids- .iter()- .map(|id| IngoingEdgesById { node: *id, limit })- .collect();-- let res = self- .conn()- .await- .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)- .await?;-- let mut edges = vec![vec![]; ids.len()];-- for (_, res) in res {- debug_assert!(res.len() <= 1);-- for (_, res) in res {- for (i, rep) in res.into_iter().enumerate() {- edges[i].extend(rep);- }- }- }-- Ok(edges)- }-pub async fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {let res = self.conn()diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex 34d6c4a9..b7799fbb 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -209,7 +209,7 @@ impl Segment {#[cfg(test)]mod test {use crate::{- webgraph::{Edge, FullNodeID},+ webgraph::{Edge, FullNodeID, NodeDatum},webpage::html::links::RelFlags,};@@ -285,15 +285,15 @@ mod test {out,vec![Edge {- from: a.id,- to: b.id,+ from: NodeDatum::new(a.id, u64::MAX),+ to: NodeDatum::new(b.id, u64::MAX),label: (),rel: RelFlags::default(),}.into(),Edge {- from: a.id,- to: c.id,+ from: NodeDatum::new(a.id, u64::MAX),+ to: NodeDatum::new(c.id, u64::MAX),label: (),rel: RelFlags::default(),}@@ -306,8 +306,8 @@ mod test {assert_eq!(out,vec![Edge {- from: b.id,- to: c.id,+ from: NodeDatum::new(b.id, u64::MAX),+ to: NodeDatum::new(c.id, u64::MAX),label: (),rel: RelFlags::default(),}@@ -319,8 +319,8 @@ mod test {assert_eq!(out,vec![Edge {- from: c.id,- to: a.id,+ from: NodeDatum::new(c.id, u64::MAX),+ to: NodeDatum::new(a.id, u64::MAX),label: (),rel: RelFlags::default(),}@@ -331,8 +331,8 @@ mod test {assert_eq!(out,vec![Edge {- from: c.id,- to: a.id,+ from: NodeDatum::new(c.id, u64::MAX),+ to: NodeDatum::new(a.id, u64::MAX),label: (),rel: RelFlags::default(),}@@ -343,8 +343,8 @@ mod test {assert_eq!(out,vec![Edge {- from: a.id,- to: b.id,+ from: NodeDatum::new(a.id, u64::MAX),+ to: NodeDatum::new(b.id, u64::MAX),label: (),rel: RelFlags::default(),}@@ -357,15 +357,15 @@ mod test {out,vec![Edge {- from: a.id,- to: c.id,+ from: NodeDatum::new(a.id, u64::MAX),+ to: NodeDatum::new(c.id, u64::MAX),label: (),rel: RelFlags::default(),}.into(),Edge {- from: b.id,- to: c.id,+ from: NodeDatum::new(b.id, u64::MAX),+ to: NodeDatum::new(c.id, u64::MAX),label: (),rel: RelFlags::default(),}diff --git a/crates/core/src/webgraph/shortest_path.rs b/crates/core/src/webgraph/shortest_path.rsindex 1915c54d..16071736 100644--- a/crates/core/src/webgraph/shortest_path.rs+++ b/crates/core/src/webgraph/shortest_path.rs@@ -90,7 +90,7 @@ impl ShortestPaths for Webgraph {dijkstra_multi(&[source],|node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),- |edge| edge.to,+ |edge| edge.to.node(),Some(max_dist),)}@@ -99,7 +99,7 @@ impl ShortestPaths for Webgraph {dijkstra_multi(&[source],|node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),- |edge| edge.to,+ |edge| edge.to.node(),None,)}@@ -108,7 +108,7 @@ impl ShortestPaths for Webgraph {dijkstra_multi(&[source],|node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),- |edge| edge.from,+ |edge| edge.from.node(),None,)}diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 95ec3af5..bb31caa0 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -33,8 +33,8 @@ use fst::Automaton;use itertools::Itertools;use super::{- merge::{MergeNode, MergeSegmentOrd, NodeDatum},- Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge, StoredEdge,+ merge::{MergeNode, MergeSegmentOrd},+ Compression, EdgeLimit, FullNodeID, NodeDatum, NodeID, SegmentEdge, StoredEdge,};#[derive(@@ -107,12 +107,12 @@ impl HostDb {#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct EdgeRange {range: std::ops::Range<u64>,- sort_key: u64,+ host_rank: u64,}impl EdgeRange {- pub fn new(range: std::ops::Range<u64>, sort_key: u64) -> Self {- Self { range, sort_key }+ pub fn new(range: std::ops::Range<u64>, host_rank: u64) -> Self {+ Self { range, host_rank }}}@@ -219,14 +219,14 @@ impl ConstSerializable for NodeDatum {fn serialize(&self, buf: &mut [u8]) {self.node().serialize(&mut buf[..NodeID::BYTES]);- self.sort_key().serialize(&mut buf[NodeID::BYTES..]);+ self.host_rank().serialize(&mut buf[NodeID::BYTES..]);}fn deserialize(buf: &[u8]) -> Self {let node = NodeID::deserialize(&buf[..NodeID::BYTES]);- let sort_key = u64::deserialize(&buf[NodeID::BYTES..]);+ let host_rank = u64::deserialize(&buf[NodeID::BYTES..]);- Self::new(node, sort_key)+ Self::new(node, host_rank)}}@@ -236,15 +236,15 @@ impl ConstSerializable for EdgeRange {fn serialize(&self, buf: &mut [u8]) {const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;self.range.serialize(&mut buf[..RANGE_BYTES]);- self.sort_key.serialize(&mut buf[RANGE_BYTES..]);+ self.host_rank.serialize(&mut buf[RANGE_BYTES..]);}fn deserialize(buf: &[u8]) -> Self {const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;let range: Range<u64> = Range::deserialize(&buf[..RANGE_BYTES]);- let sort_key = u64::deserialize(&buf[RANGE_BYTES..]);+ let host_rank = u64::deserialize(&buf[RANGE_BYTES..]);- Self { range, sort_key }+ Self { range, host_rank }}}@@ -401,7 +401,7 @@ impl EdgeStore {let edges = Self::merge_postings_for_node(&buf, stores);// write postings- let node_sort_key = buf[0].range().sort_key;+ let node_host_rank = buf[0].range().host_rank;let node_id = buf[0].id();let mut first_label_offset = None;let mut last_label_offset = None;@@ -444,7 +444,7 @@ impl EdgeStore {start: first_node_offset.unwrap().start,end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,},- node_sort_key,+ node_host_rank,);let node_range_bytes = node_range.serialize_to_vec();@@ -528,13 +528,13 @@ impl EdgeStore {if self.reversed {SegmentEdge {from: edge.other,- to: NodeDatum::new(*node, node_range.sort_key),+ to: NodeDatum::new(*node, node_range.host_rank),rel: edge.rel,label,}} else {SegmentEdge {- from: NodeDatum::new(*node, node_range.sort_key),+ from: NodeDatum::new(*node, node_range.host_rank),to: edge.other,rel: edge.rel,label,@@ -564,13 +564,13 @@ impl EdgeStore {if self.reversed {SegmentEdge {from: edge.other,- to: NodeDatum::new(*node, edge_range.sort_key),+ to: NodeDatum::new(*node, edge_range.host_rank),rel: edge.rel,label: (),}} else {SegmentEdge {- from: NodeDatum::new(*node, edge_range.sort_key),+ from: NodeDatum::new(*node, edge_range.host_rank),to: edge.other,rel: edge.rel,label: (),@@ -603,13 +603,13 @@ impl EdgeStore {if self.reversed {SegmentEdge {from: edge.other,- to: NodeDatum::new(node, edge_range.sort_key),+ to: NodeDatum::new(node, edge_range.host_rank),rel: edge.rel,label: (),}} else {SegmentEdge {- from: NodeDatum::new(node, edge_range.sort_key),+ from: NodeDatum::new(node, edge_range.host_rank),to: edge.other,rel: edge.rel,label: (),@@ -657,7 +657,7 @@ mod tests {let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64), &EdgeLimit::Unlimited);assert_eq!(edges.len(), 1);- assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));+ assert_eq!(&edges[0].to.node(), &SegmentEdge::from(e.clone()).to.node());let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);@@ -801,8 +801,17 @@ mod tests {assert_eq!(edges.len(), 3);- assert_eq!(Edge::from(edges[0].clone()), Edge::from(e2.clone()));- assert_eq!(Edge::from(edges[1].clone()), Edge::from(e3.clone()));- assert_eq!(Edge::from(edges[2].clone()), Edge::from(e1.clone()));+ assert_eq!(+ Edge::from(edges[0].clone()).from.node(),+ Edge::from(e2.clone()).from.node()+ );+ assert_eq!(+ Edge::from(edges[1].clone()).from.node(),+ Edge::from(e3.clone()).from.node()+ );+ assert_eq!(+ Edge::from(edges[2].clone()).from.node(),+ Edge::from(e1.clone()).from.node()+ );}}diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex cb8bdd6b..b6588bcc 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -39,9 +39,8 @@ use file_store::{};use super::{- merge::NodeDatum,store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},- Compression, EdgeLabel, InsertableEdge, NodeID, StoredEdge,+ Compression, EdgeLabel, InsertableEdge, NodeDatum, NodeID, StoredEdge,};#[derive(bincode::Encode, bincode::Decode)]@@ -315,13 +314,13 @@ impl FinalEdgeStoreWriter {(edge.to.id, edge.to.host)};- let sort_key = self+ let host_rank = self.host_centrality_rank_store.as_ref().map(|store| store.get(&host).unwrap().unwrap_or(u64::MAX))- .unwrap_or(0);+ .unwrap_or(u64::MAX);- let datum = NodeDatum::new(node, sort_key);+ let datum = NodeDatum::new(node, host_rank);let rel = edge.rel;stored_edges.push(StoredEdge::new(datum, rel));}@@ -367,7 +366,7 @@ impl FinalEdgeStoreWriter {.host_centrality_rank_store.as_ref().map(|store| store.get(&node.host).unwrap().unwrap_or(u64::MAX))- .unwrap_or(0);+ .unwrap_or(u64::MAX);let node_range: EdgeRange = EdgeRange::new(Range {diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex acf807c6..707afa89 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -18,7 +18,7 @@ use crate::{backlink_grouper::GroupedBacklinks,inverted_index::InvertedIndex,schema::{numerical_field::NumericalField, text_field::TextField, Field},- webgraph::{FullEdge, NodeID},+ webgraph::{Edge, NodeID},Result,};use candle_core::Tensor;@@ -45,13 +45,17 @@ pub struct Webpage {pub html: Html,#[cfg(test)]- pub backlinks: Vec<FullEdge>,+ pub backlinks: Vec<Edge<String>>,#[cfg(not(test))]- backlinks: Vec<FullEdge>,+ backlinks: Vec<Edge<String>>,+ #[cfg(test)]pub grouped_backlinks: GroupedBacklinks,+ #[cfg(not(test))]+ grouped_backlinks: GroupedBacklinks,+pub host_centrality: f64,pub host_centrality_rank: u64,pub page_centrality: f64,@@ -141,11 +145,11 @@ impl Webpage {})}- pub fn backlinks(&self) -> &[FullEdge] {+ pub fn backlinks(&self) -> &[Edge<String>] {&self.backlinks}- pub fn set_backlinks(&mut self, backlinks: Vec<FullEdge>) {+ pub fn set_backlinks(&mut self, backlinks: Vec<Edge<String>>) {self.backlinks = backlinks;}