Skip to content

web search done right

remove 'linksfrom' operator again

Mikkel Denker88df429dad78
diff --git a/crates/core/src/backlink_grouper.rs b/crates/core/src/backlink_grouper.rs
index d8c6b634..7ae68130 100644
--- a/crates/core/src/backlink_grouper.rs
+++ b/crates/core/src/backlink_grouper.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{log_group::HarmonicRankGroup, webgraph::FullEdge};
+use crate::{log_group::HarmonicRankGroup, webgraph::Edge};
/// Number of groups to divide the backlinks into.
/// If this is changed, also change the grouped backlink fields in the schema.
@@ -23,7 +23,7 @@ const NUM_GROUPS: u64 = 10;
#[derive(Debug)]
pub struct Group {
group: u64,
- backlinks: Vec<FullEdge>,
+ backlinks: Vec<Edge<String>>,
}
impl Group {
@@ -34,7 +34,7 @@ impl Group {
}
}
- fn insert(&mut self, backlink: FullEdge) {
+ fn insert(&mut self, backlink: Edge<String>) {
self.backlinks.push(backlink);
}
@@ -42,7 +42,7 @@ impl Group {
self.group
}
- pub fn backlinks(&self) -> &[FullEdge] {
+ pub fn backlinks(&self) -> &[Edge<String>] {
&self.backlinks
}
}
@@ -66,7 +66,7 @@ impl GroupedBacklinks {
self.groups.get(group as usize)
}
- fn add(&mut self, group: u64, backlink: FullEdge) {
+ fn add(&mut self, group: u64, backlink: Edge<String>) {
if let Some(group) = self.groups.get_mut(group as usize) {
group.insert(backlink)
}
@@ -93,7 +93,9 @@ impl BacklinkGrouper {
}
}
- pub fn add(&mut self, backlink: FullEdge, rank: u64) {
+ pub fn add(&mut self, backlink: Edge<String>) {
+ let rank = backlink.from.host_rank();
+
let group = self.grouper.group(rank);
self.groups.add(group, backlink);
}
@@ -105,93 +107,73 @@ impl BacklinkGrouper {
#[cfg(test)]
mod tests {
- use crate::webgraph::Node;
+ use crate::webgraph::NodeDatum;
use super::*;
#[test]
fn test_grouped_backlinks() {
let mut grouper = BacklinkGrouper::new(10);
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 0,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 1,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 2,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 3,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 4,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 5,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 6,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 7,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 8,
- );
- grouper.add(
- FullEdge {
- from: Node::from("https://a.com"),
- to: Node::from("https://b.com"),
- label: String::new(),
- },
- 9,
- );
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 0u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 1u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 2u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 3u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 4u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 5u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 6u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 7u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 8u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
+ grouper.add(Edge {
+ from: NodeDatum::new(0u64, 9u64),
+ to: NodeDatum::new(1u64, 0u64),
+ label: String::new(),
+ rel: Default::default(),
+ });
{
let groups = grouper.groups();
diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
index c930dd1a..911c2fac 100644
--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs
@@ -77,13 +77,13 @@ impl Workers {
{
let d = dist + 1;
- let current_dist = distances.entry(outgoing).or_insert(u8::MAX);
+ let current_dist = distances.entry(outgoing.node()).or_insert(u8::MAX);
if d < *current_dist {
*current_dist = d;
if d < max_dist {
- queue.push(cmp::Reverse((d, outgoing)));
+ queue.push(cmp::Reverse((d, outgoing.node())));
}
}
}
diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
index 4c9765fb..a5efc4b4 100644
--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs
@@ -51,7 +51,7 @@ impl CentralityMapper {
batch: &[webgraph::Edge<()>],
dht: &DhtConn<CentralityTables>,
) -> BTreeMap<webgraph::NodeID, HyperLogLog<64>> {
- let nodes: Vec<_> = batch.iter().map(|edge| edge.from).collect();
+ let nodes: Vec<_> = batch.iter().map(|edge| edge.from.node()).collect();
if nodes.is_empty() {
return BTreeMap::new();
@@ -95,9 +95,12 @@ impl CentralityMapper {
let updates: Vec<_> = batch
.iter()
.map(|edge| {
- let mut counter = old_counters.get(&edge.from).cloned().unwrap_or_default();
- counter.add(edge.from.as_u64());
- (edge.to, counter)
+ let mut counter = old_counters
+ .get(&edge.from.node())
+ .cloned()
+ .unwrap_or_default();
+ counter.add(edge.from.node().as_u64());
+ (edge.to.node(), counter)
})
.collect();
@@ -270,7 +273,7 @@ impl CentralityMapper {
.graph()
.edges()
.filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))
- .filter(|e| changed_nodes.contains(e.from.as_u64()))
+ .filter(|e| changed_nodes.contains(e.from.node().as_u64()))
{
batch.push(edge);
if batch.len() >= batch_size {
diff --git a/crates/core/src/entrypoint/indexer/worker.rs b/crates/core/src/entrypoint/indexer/worker.rs
index c4c4fa66..72b35193 100644
--- a/crates/core/src/entrypoint/indexer/worker.rs
+++ b/crates/core/src/entrypoint/indexer/worker.rs
@@ -35,7 +35,7 @@ use crate::Result;
use crate::index::Index;
use crate::rake::RakeModel;
use crate::ranking::SignalComputer;
-use crate::webgraph::{self, EdgeLimit, FullEdge, Node, NodeID};
+use crate::webgraph::{self, Edge, EdgeLimit, Node, NodeID};
use crate::webpage::{safety_classifier, Html, Webpage};
const MAX_BACKLINKS: EdgeLimit = EdgeLimit::Limit(1024);
@@ -87,16 +87,17 @@ impl Webgraph {
&self,
ids: Vec<NodeID>,
limit: EdgeLimit,
- ) -> Vec<Vec<FullEdge>> {
+ ) -> Vec<Vec<Edge<String>>> {
let edges = match self {
Self::Remote(webgraph) => {
- crate::block_on(webgraph.batch_ingoing_edges_by_id(&ids, limit)).unwrap_or_default()
+ crate::block_on(webgraph.batch_raw_ingoing_edges_with_labels(&ids, limit))
+ .unwrap_or_default()
}
Self::Local(webgraph) => {
let mut res = Vec::new();
for id in ids {
- res.push(webgraph.ingoing_edges_by_id(&id, limit));
+ res.push(webgraph.raw_ingoing_edges_with_labels(&id, limit));
}
res
@@ -405,18 +406,7 @@ impl IndexingWorker {
BacklinkGrouper::new(self.host_centrality_rank_store.len() as u64);
for backlink in backlinks {
- let node = backlink.from.clone().into_host().id();
-
- if Some(node) == page.node_id {
- continue;
- }
-
- let rank = self
- .host_centrality_rank_store
- .get(&node)
- .unwrap()
- .unwrap_or(u64::MAX);
- grouper.add(backlink, rank);
+ grouper.add(backlink);
}
page.set_grouped_backlinks(grouper.groups())
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index 5d26c26e..42ef0156 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -53,7 +53,6 @@ sonic_service!(
[
GetNode,
IngoingEdges,
- IngoingEdgesById,
OutgoingEdges,
RawIngoingEdges,
RawOutgoingEdges,
@@ -90,20 +89,6 @@ impl Message<WebGraphService> for IngoingEdges {
}
}
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
-pub struct IngoingEdgesById {
- pub node: NodeID,
- pub limit: EdgeLimit,
-}
-
-impl Message<WebGraphService> for IngoingEdgesById {
- type Response = Vec<FullEdge>;
-
- async fn handle(self, server: &WebGraphService) -> Self::Response {
- server.graph.ingoing_edges_by_id(&self.node, self.limit)
- }
-}
-
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct OutgoingEdges {
pub node: Node,
diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rs
index 863ed9ac..cf03025f 100644
--- a/crates/core/src/inverted_index/mod.rs
+++ b/crates/core/src/inverted_index/mod.rs
@@ -236,7 +236,7 @@ mod tests {
ranking::{Ranker, SignalComputer},
search_ctx::Ctx,
searcher::SearchQuery,
- webgraph::{FullEdge, Node},
+ webgraph::{Edge, NodeDatum},
webpage::{schema_org, Html, Webpage},
OneOrMany,
};
@@ -511,10 +511,11 @@ mod tests {
..Default::default()
};
- webpage.set_backlinks(vec![FullEdge {
- from: Node::from("https://www.a.com"),
- to: Node::from("https://www.b.com"),
+ webpage.set_backlinks(vec![Edge {
+ from: NodeDatum::new(0u64, 0),
+ to: NodeDatum::new(1u64, 0),
label: "B site is great".to_string(),
+ rel: Default::default(),
}]);
index.insert(&webpage).expect("failed to insert webpage");
diff --git a/crates/core/src/query/mod.rs b/crates/core/src/query/mod.rs
index 868deac3..ebd592ba 100644
--- a/crates/core/src/query/mod.rs
+++ b/crates/core/src/query/mod.rs
@@ -206,13 +206,7 @@ impl tantivy::query::Query for Query {
#[cfg(test)]
mod tests {
- use crate::{
- index::Index,
- rand_words,
- searcher::LocalSearcher,
- webgraph::{FullEdge, Node},
- webpage::Webpage,
- };
+ use crate::{index::Index, rand_words, searcher::LocalSearcher, webpage::Webpage};
use proptest::prelude::*;
use super::*;
@@ -641,86 +635,6 @@ mod tests {
assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
- #[test]
- fn link_from_query() {
- let mut index = Index::temporary().expect("Unable to open index");
-
- let mut webpage = Webpage::test_parse(
- r#"
- <html>
- <head>
- <title>Test website</title>
- </head>
- <body>
- This is a test website
- </body>
- </html>
- "#,
- "https://www.first.com",
- )
- .unwrap();
-
- webpage.set_backlinks(vec![FullEdge {
- from: Node::from("https://www.second.com/example/abc"),
- to: Node::from("https://www.first.com"),
- label: String::new(),
- }]);
-
- index.insert(&webpage).expect("failed to insert webpage");
- let mut webpage = Webpage::test_parse(
- r#"
- <html>
- <head>
- <title>Test test</title>
- </head>
- <body>
- This test page does not contain the forbidden word
- <a href="https://www.first.com">First</a>
- </body>
- </html>
- "#,
- "https://www.second.com/example/abc",
- )
- .unwrap();
- webpage.set_backlinks(vec![]);
-
- index.insert(&webpage).expect("failed to insert webpage");
- index.commit().expect("failed to commit index");
- let searcher = LocalSearcher::from(index);
-
- let query = SearchQuery {
- query: "test linkfrom:second.com".to_string(),
- ..Default::default()
- };
- let result = searcher.search(&query).expect("Search failed");
- assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com/");
-
- let query = SearchQuery {
- query: "test linkfrom:www.second.com".to_string(),
- ..Default::default()
- };
- let result = searcher.search(&query).expect("Search failed");
- assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com/");
-
- let query = SearchQuery {
- query: "test linkfrom:second.com/example".to_string(),
- ..Default::default()
- };
- let result = searcher.search(&query).expect("Search failed");
- assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com/");
-
- let query = SearchQuery {
- query: "test linksfrom:second.com/example/abc".to_string(),
- ..Default::default()
- };
- let result = searcher.search(&query).expect("Search failed");
- assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com/");
- }
-
#[test]
fn title_query() {
let mut index = Index::temporary().expect("Unable to open index");
diff --git a/crates/core/src/query/parser/mod.rs b/crates/core/src/query/parser/mod.rs
index 5590fefe..c454bedb 100644
--- a/crates/core/src/query/parser/mod.rs
+++ b/crates/core/src/query/parser/mod.rs
@@ -130,17 +130,6 @@ fn links_to_field(input: &str) -> nom::IResult<&str, Term> {
Ok((input, Term::LinkTo(output.to_string())))
}
-fn links_from_field(input: &str) -> nom::IResult<&str, Term> {
- // parse 'linksfrom:' and then a simple term
- let (input, _) = nom::branch::alt((
- nom::bytes::complete::tag("linksfrom:"),
- nom::bytes::complete::tag("linkfrom:"),
- ))(input)?;
- let (input, output) = simple_str(input)?;
-
- Ok((input, Term::LinkFrom(output.to_string())))
-}
-
fn title_field(input: &str) -> nom::IResult<&str, Term> {
// parse 'title:' and then a simple term
let (input, _) = nom::bytes::complete::tag("intitle:")(input)?;
@@ -169,7 +158,6 @@ fn field_selector(input: &str) -> nom::IResult<&str, Term> {
nom::branch::alt((
site_field,
links_to_field,
- links_from_field,
title_field,
body_field,
url_field,
diff --git a/crates/core/src/query/parser/term.rs b/crates/core/src/query/parser/term.rs
index 01631829..7eb26d31 100644
--- a/crates/core/src/query/parser/term.rs
+++ b/crates/core/src/query/parser/term.rs
@@ -99,7 +99,6 @@ pub enum Term {
SimpleOrPhrase(SimpleOrPhrase),
Site(String),
LinkTo(String),
- LinkFrom(String),
Title(SimpleOrPhrase),
Body(SimpleOrPhrase),
Url(SimpleOrPhrase),
@@ -114,7 +113,6 @@ impl std::fmt::Display for Term {
Term::Not(term) => write!(f, "-{}", term),
Term::Site(site) => write!(f, "site:{}", site),
Term::LinkTo(site) => write!(f, "linkto:{}", site),
- Term::LinkFrom(site) => write!(f, "linkfrom:{}", site),
Term::Title(title) => write!(f, "intitle:{}", title),
Term::Body(body) => write!(f, "inbody:{}", body),
Term::Url(url) => write!(f, "inurl:{}", url),
@@ -136,7 +134,6 @@ impl Term {
Term::SimpleOrPhrase(s) => Term::SimpleOrPhrase(s.truncate()),
Term::Site(s) => Term::Site(s),
Term::LinkTo(s) => Term::LinkTo(s),
- Term::LinkFrom(s) => Term::LinkFrom(s),
Term::Title(s) => Term::Title(s.truncate()),
Term::Body(s) => Term::Body(s.truncate()),
Term::Url(s) => Term::Url(s.truncate()),
diff --git a/crates/core/src/query/plan/node.rs b/crates/core/src/query/plan/node.rs
index 214945fb..2840e221 100644
--- a/crates/core/src/query/plan/node.rs
+++ b/crates/core/src/query/plan/node.rs
@@ -135,10 +135,6 @@ impl Node {
text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),
field: text_field::Links.into(),
}),
- ParserTerm::LinkFrom(s) => Node::Term(Term {
- text: SimpleOrPhrase::Simple(SimpleTerm::from(s)),
- field: text_field::Backlinks.into(),
- }),
ParserTerm::Title(t) => Node::Term(Term {
text: t,
field: text_field::Title.into(),
diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rs
index b818783f..dd31e635 100644
--- a/crates/core/src/ranking/inbound_similarity.rs
+++ b/crates/core/src/ranking/inbound_similarity.rs
@@ -160,7 +160,7 @@ mod tests {
graph
.raw_ingoing_edges(node, EdgeLimit::Unlimited)
.into_iter()
- .map(|e| e.from.as_u64())
+ .map(|e| e.from.node().as_u64())
.collect(),
)
}
diff --git a/crates/core/src/ranking/mod.rs b/crates/core/src/ranking/mod.rs
index 26dc6c2b..fcc31d64 100644
--- a/crates/core/src/ranking/mod.rs
+++ b/crates/core/src/ranking/mod.rs
@@ -130,7 +130,7 @@ mod tests {
index::Index,
models::dual_encoder::DualEncoder,
searcher::{LocalSearcher, SearchQuery},
- webgraph::{FullEdge, Node},
+ webgraph::{Edge, NodeDatum},
webpage::{Html, Webpage},
};
@@ -433,10 +433,11 @@ mod tests {
..Default::default()
};
- webpage.set_backlinks(vec![FullEdge {
- from: Node::from("https://www.backlink.com"),
- to: Node::from("https://www.first.com"),
+ webpage.set_backlinks(vec![Edge {
+ from: NodeDatum::new(0u64, 0),
+ to: NodeDatum::new(1u64, 1),
label: "test this is the best test site".to_string(),
+ rel: Default::default(),
}]);
index.insert(&webpage).expect("failed to insert webpage");
diff --git a/crates/core/src/schema/text_field.rs b/crates/core/src/schema/text_field.rs
index b540ba51..3a808018 100644
--- a/crates/core/src/schema/text_field.rs
+++ b/crates/core/src/schema/text_field.rs
@@ -199,7 +199,6 @@ pub enum TextFieldEnum {
Keywords,
KeyPhrases,
Links,
- Backlinks,
BacklinkLabelsGroup0,
BacklinkLabelsGroup1,
BacklinkLabelsGroup2,
@@ -248,7 +247,6 @@ enum_dispatch_from_discriminant!(TextFieldEnumDiscriminants => TextFieldEnum,
Keywords,
KeyPhrases,
Links,
- Backlinks,
BacklinkLabelsGroup0,
BacklinkLabelsGroup1,
BacklinkLabelsGroup2,
@@ -1586,51 +1584,6 @@ impl TextField for Links {
}
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Backlinks;
-impl TextField for Backlinks {
- fn name(&self) -> &str {
- "backlinks"
- }
-
- fn has_pos(&self) -> bool {
- true
- }
-
- fn tokenizer(&self, _: Option<&whatlang::Lang>) -> FieldTokenizer {
- FieldTokenizer::Url(UrlTokenizer)
- }
-
- fn add_webpage_tantivy(
- &self,
- webpage: &crate::webpage::Webpage,
- doc: &mut TantivyDocument,
- index: &crate::inverted_index::InvertedIndex,
- ) -> Result<()> {
- doc.add_text(
- self.tantivy_field(index.schema_ref())
- .unwrap_or_else(|| panic!("could not find field '{}' in index", self.name())),
- webpage
- .backlinks()
- .iter()
- .map(|l| l.from.as_str().to_string())
- .join("\n"),
- );
-
- Ok(())
- }
-
- fn add_html_tantivy(
- &self,
- _: &Html,
- _: &mut FnCache,
- _: &mut TantivyDocument,
- _: &crate::inverted_index::InvertedIndex,
- ) -> Result<()> {
- Ok(())
- }
-}
-
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct Keywords;
impl TextField for Keywords {
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index a0c37032..1ee3d739 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -201,7 +201,7 @@ where
self.batch_raw_ingoing(nodes, EdgeLimit::Limit(128))
.await
.into_iter()
- .map(|edges| edges.into_iter().map(|edge| edge.from).collect())
+ .map(|edges| edges.into_iter().map(|edge| edge.from.node()).collect())
.collect()
}
}
diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rs
index 31b91617..f436aa48 100644
--- a/crates/core/src/similar_hosts.rs
+++ b/crates/core/src/similar_hosts.rs
@@ -82,7 +82,7 @@ impl SimilarHostsFinder {
let backlink_nodes = in_edges
.iter()
.flatten()
- .map(|e| e.from)
+ .map(|e| e.from.node())
.unique()
.collect::<Vec<_>>();
@@ -95,7 +95,7 @@ impl SimilarHostsFinder {
let potential_nodes: Vec<_> = outgoing_edges
.iter()
.flatten()
- .map(|e| e.to)
+ .map(|e| e.to.node())
.unique()
.filter(|n| !nodes.contains(n))
.collect();
diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rs
index 4dbecb19..b4a69d29 100644
--- a/crates/core/src/webgraph/centrality/betweenness.rs
+++ b/crates/core/src/webgraph/centrality/betweenness.rs
@@ -71,7 +71,7 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32)
while let Some(v) = q.pop_front() {
stack.push(v);
for edge in graph.raw_outgoing_edges(&v, EdgeLimit::Unlimited) {
- let w = edge.to;
+ let w = edge.to.node();
if !distances.contains_key(&w) {
let dist_v = distances.get(&v).unwrap();
diff --git a/crates/core/src/webgraph/centrality/derived_harmonic.rs b/crates/core/src/webgraph/centrality/derived_harmonic.rs
index d5505de6..b87e5f46 100644
--- a/crates/core/src/webgraph/centrality/derived_harmonic.rs
+++ b/crates/core/src/webgraph/centrality/derived_harmonic.rs
@@ -83,7 +83,7 @@ impl DerivedCentrality {
let has_outgoing = BloomMap::new(8, num_nodes as u64, 0.01);
page_graph.par_edges().for_each(|edge| {
- has_outgoing.insert(&edge.from);
+ has_outgoing.insert(&edge.from.node());
});
let has_outgoing = has_outgoing.finalize();
@@ -103,7 +103,7 @@ impl DerivedCentrality {
let mut ingoing: Vec<_> = page_graph
.raw_ingoing_edges(&id, EdgeLimit::Limit(128))
.into_iter()
- .filter_map(|e| page_graph.id2node(&e.from))
+ .filter_map(|e| page_graph.id2node(&e.from.node()))
.map(|n| n.into_host())
.collect();
ingoing.sort();
diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rs
index 8f82726a..d435a27b 100644
--- a/crates/core/src/webgraph/centrality/harmonic.rs
+++ b/crates/core/src/webgraph/centrality/harmonic.rs
@@ -85,9 +85,10 @@ fn update_changed_counters(
.into_iter()
.filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))
{
- if let (Some(counter_to), Some(counter_from)) =
- (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
- {
+ if let (Some(counter_to), Some(counter_from)) = (
+ counters.new.get_mut(&edge.to.node()),
+ counters.old.get(&edge.from.node()),
+ ) {
if counter_to
.registers()
.iter()
@@ -95,9 +96,9 @@ fn update_changed_counters(
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.as_u64());
+ new_changed_nodes.insert(edge.to.node().as_u64());
- new_exact_changed_nodes.insert(edge.to);
+ new_exact_changed_nodes.insert(edge.to.node());
has_changes.store(true, Ordering::Relaxed);
}
@@ -127,10 +128,11 @@ fn update_all_counters(
.edges()
.filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))
.for_each(|edge| {
- if changed_nodes.contains(edge.from.as_u64()) {
- if let (Some(counter_to), Some(counter_from)) =
- (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))
- {
+ if changed_nodes.contains(edge.from.node().as_u64()) {
+ if let (Some(counter_to), Some(counter_from)) = (
+ counters.new.get_mut(&edge.to.node()),
+ counters.old.get(&edge.from.node()),
+ ) {
if counter_to
.registers()
.iter()
@@ -138,10 +140,10 @@ fn update_all_counters(
.any(|(to, from)| *from > *to)
{
counter_to.merge(counter_from);
- new_changed_nodes.insert(edge.to.as_u64());
+ new_changed_nodes.insert(edge.to.node().as_u64());
if let Some(exact_changed_nodes) = &mut exact_changed_nodes {
- exact_changed_nodes.insert(edge.to);
+ exact_changed_nodes.insert(edge.to.node());
}
has_changes.store(true, Ordering::Relaxed);
diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rs
index 4cfcf67f..8c1e7fd5 100644
--- a/crates/core/src/webgraph/edge.rs
+++ b/crates/core/src/webgraph/edge.rs
@@ -18,7 +18,7 @@ use utoipa::ToSchema;
use crate::webpage::html::links::RelFlags;
-use super::{merge::NodeDatum, FullNodeID, Node, NodeID};
+use super::{FullNodeID, Node, NodeDatum, NodeID};
pub const MAX_LABEL_LENGTH: usize = 1024;
@@ -55,8 +55,8 @@ pub struct Edge<L>
where
L: EdgeLabel,
{
- pub from: NodeID,
- pub to: NodeID,
+ pub from: NodeDatum,
+ pub to: NodeDatum,
pub rel: RelFlags,
pub label: L,
}
@@ -88,8 +88,8 @@ where
{
fn from(edge: InsertableEdge<L>) -> Self {
Edge {
- from: edge.from.id,
- to: edge.to.id,
+ from: NodeDatum::new(edge.from.id, u64::MAX),
+ to: NodeDatum::new(edge.to.id, u64::MAX),
rel: edge.rel,
label: edge.label,
}
@@ -132,8 +132,8 @@ where
{
fn from(edge: SegmentEdge<L>) -> Self {
Edge {
- from: edge.from.node(),
- to: edge.to.node(),
+ from: edge.from,
+ to: edge.to,
rel: edge.rel,
label: edge.label,
}
@@ -147,8 +147,8 @@ where
{
fn from(edge: Edge<L>) -> Self {
SegmentEdge {
- from: NodeDatum::new(edge.from, 0),
- to: NodeDatum::new(edge.to, 0),
+ from: edge.from,
+ to: edge.to,
rel: edge.rel,
label: edge.label,
}
@@ -162,8 +162,8 @@ where
{
fn from(edge: InsertableEdge<L>) -> Self {
SegmentEdge {
- from: NodeDatum::new(edge.from.id, 0),
- to: NodeDatum::new(edge.to.id, 0),
+ from: NodeDatum::new(edge.from.id, u64::MAX),
+ to: NodeDatum::new(edge.to.id, u64::MAX),
rel: edge.rel,
label: edge.label,
}
@@ -210,4 +210,9 @@ impl<L> StoredEdge<L> {
pub fn rel(&self) -> RelFlags {
self.rel
}
+
+ #[inline]
+ pub fn other_host_rank(&self) -> u64 {
+ self.other.host_rank()
+ }
}
diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rs
index 6dad73e5..9eeffb55 100644
--- a/crates/core/src/webgraph/merge.rs
+++ b/crates/core/src/webgraph/merge.rs
@@ -179,7 +179,7 @@ impl<'a, L> Iterator for EdgeMerger<'a, L> {
if let Some(edge) = &res {
while let Some(mut peek) = self.iters.peek_mut() {
- if peek.0.peek().map(|x| x.other.id) == Some(edge.other.id) {
+ if peek.0.peek().map(|x| x.other.node()) == Some(edge.other.node()) {
peek.0.next().unwrap();
} else {
break;
@@ -193,59 +193,6 @@ impl<'a, L> Iterator for EdgeMerger<'a, L> {
}
}
-#[derive(Debug, Clone)]
-pub struct NodeDatum {
- id: NodeID,
- sort_key: u64,
-}
-
-impl PartialOrd for NodeDatum {
- fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
- Some(self.cmp(other))
- }
-}
-
-impl Ord for NodeDatum {
- fn cmp(&self, other: &Self) -> std::cmp::Ordering {
- self.sort_key
- .cmp(&other.sort_key)
- .then_with(|| self.id.cmp(&other.id))
- }
-}
-
-impl PartialEq for NodeDatum {
- fn eq(&self, other: &Self) -> bool {
- self.id == other.id && self.sort_key == other.sort_key
- }
-}
-
-impl Eq for NodeDatum {}
-
-impl Hash for NodeDatum {
- fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
- self.id.hash(state);
- self.sort_key.hash(state);
- }
-}
-
-impl NodeDatum {
- pub fn new(node: NodeID, sort_key: u64) -> Self {
- Self { id: node, sort_key }
- }
-}
-
-impl NodeDatum {
- #[inline]
- pub fn node(&self) -> NodeID {
- self.id
- }
-
- #[inline]
- pub fn sort_key(&self) -> u64 {
- self.sort_key
- }
-}
-
impl<L> Ord for StoredEdge<L> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.other.cmp(&other.other)
@@ -274,7 +221,7 @@ impl<L> Hash for StoredEdge<L> {
#[cfg(test)]
mod tests {
- use crate::webpage::html::links::RelFlags;
+ use crate::{webgraph::NodeDatum, webpage::html::links::RelFlags};
use super::*;
@@ -354,46 +301,46 @@ mod tests {
#[test]
fn test_datum_merge() {
let a = vec![
- StoredEdge::new(NodeDatum::new(1u64.into(), 1), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(2u64.into(), 4), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(1u64, 1), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(2u64, 4), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),
];
let b = vec![
- StoredEdge::new(NodeDatum::new(4u64.into(), 2), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(5u64.into(), 3), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(4u64, 2), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(5u64, 3), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),
];
let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);
- assert_eq!(merger.next().unwrap().other.sort_key(), 1);
- assert_eq!(merger.next().unwrap().other.sort_key(), 2);
- assert_eq!(merger.next().unwrap().other.sort_key(), 3);
- assert_eq!(merger.next().unwrap().other.sort_key(), 4);
- assert_eq!(merger.next().unwrap().other.sort_key(), 5);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 1);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 2);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 3);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 4);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 5);
assert!(merger.next().is_none());
}
#[test]
fn test_datum_merge_unequal_len() {
let a = vec![
- StoredEdge::new(NodeDatum::new(1u64.into(), 1), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(2u64.into(), 4), RelFlags::default()),
- StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(1u64, 1), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(2u64, 4), RelFlags::default()),
+ StoredEdge::new(NodeDatum::new(3u64, 5), RelFlags::default()),
];
let b = vec![StoredEdge::new(
- NodeDatum::new(4u64.into(), 2),
+ NodeDatum::new(4u64, 2),
RelFlags::default(),
)];
let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);
- assert_eq!(merger.next().unwrap().other.sort_key(), 1);
- assert_eq!(merger.next().unwrap().other.sort_key(), 2);
- assert_eq!(merger.next().unwrap().other.sort_key(), 4);
- assert_eq!(merger.next().unwrap().other.sort_key(), 5);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 1);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 2);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 4);
+ assert_eq!(merger.next().unwrap().other.host_rank(), 5);
assert!(merger.next().is_none());
}
}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 8898f519..838aabcc 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -220,7 +220,7 @@ impl Webgraph {
|segment| segment.ingoing_edges_with_label(node_id, &limit),
dedup,
);
- edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));
+ edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));
limit
.apply(edges.into_iter())
@@ -257,13 +257,13 @@ impl Webgraph {
};
let mut edges = self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup);
- edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));
+ edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));
limit
.apply(edges.into_iter())
.map(|e| Edge {
- from: e.from.node(),
- to: e.to.node(),
+ from: e.from,
+ to: e.to,
label: e.label,
rel: e.rel,
})
@@ -284,13 +284,13 @@ impl Webgraph {
|segment| segment.ingoing_edges_with_label(node, &limit),
dedup,
);
- edges.sort_by(|a, b| a.from.sort_key().cmp(&b.from.sort_key()));
+ edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));
limit
.apply(edges.into_iter())
.map(|e| Edge {
- from: e.from.node(),
- to: e.to.node(),
+ from: e.from,
+ to: e.to,
label: e.label,
rel: e.rel,
})
@@ -312,13 +312,13 @@ impl Webgraph {
dedup,
);
- edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));
+ edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));
limit
.apply(edges.into_iter())
.map(|e| Edge {
- from: e.from.node(),
- to: e.to.node(),
+ from: e.from,
+ to: e.to,
label: e.label,
rel: e.rel,
})
@@ -335,7 +335,7 @@ impl Webgraph {
|segment| segment.outgoing_edges_with_label(&node.id(), &limit),
dedup,
);
- edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));
+ edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));
limit
.apply(edges.into_iter())
@@ -354,13 +354,13 @@ impl Webgraph {
};
let mut edges = self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup);
- edges.sort_by(|a, b| a.to.sort_key().cmp(&b.to.sort_key()));
+ edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));
limit
.apply(edges.into_iter())
.map(|e| Edge {
- from: e.from.node(),
- to: e.to.node(),
+ from: e.from,
+ to: e.to,
label: e.label,
rel: e.rel,
})
@@ -398,7 +398,7 @@ impl Webgraph {
let mut rng = rand::thread_rng();
let mut nodes = self
.edges()
- .map(|e| e.from)
+ .map(|e| e.from.node())
.unique()
.choose_multiple(&mut rng, num);
nodes.shuffle(&mut rng);
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index 45b63127..9f02a28d 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -192,3 +192,62 @@ pub fn normalize_url(url: &Url) -> String {
normalized
}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct NodeDatum {
+ id: NodeID,
+ host_rank: u64,
+}
+
+impl PartialOrd for NodeDatum {
+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+ Some(self.cmp(other))
+ }
+}
+
+impl Ord for NodeDatum {
+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+ self.host_rank
+ .cmp(&other.host_rank)
+ .then_with(|| self.id.cmp(&other.id))
+ }
+}
+
+impl PartialEq for NodeDatum {
+ fn eq(&self, other: &Self) -> bool {
+ self.id == other.id && self.host_rank == other.host_rank
+ }
+}
+
+impl Eq for NodeDatum {}
+
+impl std::hash::Hash for NodeDatum {
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+ self.id.hash(state);
+ self.host_rank.hash(state);
+ }
+}
+
+impl NodeDatum {
+ pub fn new<N>(node: N, host_rank: u64) -> Self
+ where
+ N: Into<NodeID>,
+ {
+ Self {
+ id: node.into(),
+ host_rank,
+ }
+ }
+}
+
+impl NodeDatum {
+ #[inline]
+ pub fn node(&self) -> NodeID {
+ self.id
+ }
+
+ #[inline]
+ pub fn host_rank(&self) -> u64 {
+ self.host_rank
+ }
+}
diff --git a/crates/core/src/webgraph/remote.rs b/crates/core/src/webgraph/remote.rs
index 2ecee109..ecc7b2eb 100644
--- a/crates/core/src/webgraph/remote.rs
+++ b/crates/core/src/webgraph/remote.rs
@@ -33,7 +33,7 @@ use crate::{
},
},
entrypoint::webgraph_server::{
- GetNode, IngoingEdges, IngoingEdgesById, OutgoingEdges, PagesByHosts, RawIngoingEdges,
+ GetNode, IngoingEdges, OutgoingEdges, PagesByHosts, RawIngoingEdges,
RawIngoingEdgesWithLabels, RawOutgoingEdges, RawOutgoingEdgesWithLabels, WebGraphService,
},
Result,
@@ -300,37 +300,6 @@ impl RemoteWebgraph {
Ok(edges)
}
- pub async fn batch_ingoing_edges_by_id(
- &self,
- ids: &[NodeID],
- limit: EdgeLimit,
- ) -> Result<Vec<Vec<FullEdge>>> {
- let reqs: Vec<_> = ids
- .iter()
- .map(|id| IngoingEdgesById { node: *id, limit })
- .collect();
-
- let res = self
- .conn()
- .await
- .batch_send(&reqs, &AllShardsSelector, &RandomReplicaSelector)
- .await?;
-
- let mut edges = vec![vec![]; ids.len()];
-
- for (_, res) in res {
- debug_assert!(res.len() <= 1);
-
- for (_, res) in res {
- for (i, rep) in res.into_iter().enumerate() {
- edges[i].extend(rep);
- }
- }
- }
-
- Ok(edges)
- }
-
pub async fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Result<Vec<FullEdge>> {
let res = self
.conn()
diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rs
index 34d6c4a9..b7799fbb 100644
--- a/crates/core/src/webgraph/segment.rs
+++ b/crates/core/src/webgraph/segment.rs
@@ -209,7 +209,7 @@ impl Segment {
#[cfg(test)]
mod test {
use crate::{
- webgraph::{Edge, FullNodeID},
+ webgraph::{Edge, FullNodeID, NodeDatum},
webpage::html::links::RelFlags,
};
@@ -285,15 +285,15 @@ mod test {
out,
vec![
Edge {
- from: a.id,
- to: b.id,
+ from: NodeDatum::new(a.id, u64::MAX),
+ to: NodeDatum::new(b.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
.into(),
Edge {
- from: a.id,
- to: c.id,
+ from: NodeDatum::new(a.id, u64::MAX),
+ to: NodeDatum::new(c.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
@@ -306,8 +306,8 @@ mod test {
assert_eq!(
out,
vec![Edge {
- from: b.id,
- to: c.id,
+ from: NodeDatum::new(b.id, u64::MAX),
+ to: NodeDatum::new(c.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
@@ -319,8 +319,8 @@ mod test {
assert_eq!(
out,
vec![Edge {
- from: c.id,
- to: a.id,
+ from: NodeDatum::new(c.id, u64::MAX),
+ to: NodeDatum::new(a.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
@@ -331,8 +331,8 @@ mod test {
assert_eq!(
out,
vec![Edge {
- from: c.id,
- to: a.id,
+ from: NodeDatum::new(c.id, u64::MAX),
+ to: NodeDatum::new(a.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
@@ -343,8 +343,8 @@ mod test {
assert_eq!(
out,
vec![Edge {
- from: a.id,
- to: b.id,
+ from: NodeDatum::new(a.id, u64::MAX),
+ to: NodeDatum::new(b.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
@@ -357,15 +357,15 @@ mod test {
out,
vec![
Edge {
- from: a.id,
- to: c.id,
+ from: NodeDatum::new(a.id, u64::MAX),
+ to: NodeDatum::new(c.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
.into(),
Edge {
- from: b.id,
- to: c.id,
+ from: NodeDatum::new(b.id, u64::MAX),
+ to: NodeDatum::new(c.id, u64::MAX),
label: (),
rel: RelFlags::default(),
}
diff --git a/crates/core/src/webgraph/shortest_path.rs b/crates/core/src/webgraph/shortest_path.rs
index 1915c54d..16071736 100644
--- a/crates/core/src/webgraph/shortest_path.rs
+++ b/crates/core/src/webgraph/shortest_path.rs
@@ -90,7 +90,7 @@ impl ShortestPaths for Webgraph {
dijkstra_multi(
&[source],
|node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),
- |edge| edge.to,
+ |edge| edge.to.node(),
Some(max_dist),
)
}
@@ -99,7 +99,7 @@ impl ShortestPaths for Webgraph {
dijkstra_multi(
&[source],
|node| self.raw_outgoing_edges(&node, super::EdgeLimit::Unlimited),
- |edge| edge.to,
+ |edge| edge.to.node(),
None,
)
}
@@ -108,7 +108,7 @@ impl ShortestPaths for Webgraph {
dijkstra_multi(
&[source],
|node| self.raw_ingoing_edges(&node, super::EdgeLimit::Unlimited),
- |edge| edge.from,
+ |edge| edge.from.node(),
None,
)
}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index 95ec3af5..bb31caa0 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -33,8 +33,8 @@ use fst::Automaton;
use itertools::Itertools;
use super::{
- merge::{MergeNode, MergeSegmentOrd, NodeDatum},
- Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge, StoredEdge,
+ merge::{MergeNode, MergeSegmentOrd},
+ Compression, EdgeLimit, FullNodeID, NodeDatum, NodeID, SegmentEdge, StoredEdge,
};
#[derive(
@@ -107,12 +107,12 @@ impl HostDb {
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct EdgeRange {
range: std::ops::Range<u64>,
- sort_key: u64,
+ host_rank: u64,
}
impl EdgeRange {
- pub fn new(range: std::ops::Range<u64>, sort_key: u64) -> Self {
- Self { range, sort_key }
+ pub fn new(range: std::ops::Range<u64>, host_rank: u64) -> Self {
+ Self { range, host_rank }
}
}
@@ -219,14 +219,14 @@ impl ConstSerializable for NodeDatum {
fn serialize(&self, buf: &mut [u8]) {
self.node().serialize(&mut buf[..NodeID::BYTES]);
- self.sort_key().serialize(&mut buf[NodeID::BYTES..]);
+ self.host_rank().serialize(&mut buf[NodeID::BYTES..]);
}
fn deserialize(buf: &[u8]) -> Self {
let node = NodeID::deserialize(&buf[..NodeID::BYTES]);
- let sort_key = u64::deserialize(&buf[NodeID::BYTES..]);
+ let host_rank = u64::deserialize(&buf[NodeID::BYTES..]);
- Self::new(node, sort_key)
+ Self::new(node, host_rank)
}
}
@@ -236,15 +236,15 @@ impl ConstSerializable for EdgeRange {
fn serialize(&self, buf: &mut [u8]) {
const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;
self.range.serialize(&mut buf[..RANGE_BYTES]);
- self.sort_key.serialize(&mut buf[RANGE_BYTES..]);
+ self.host_rank.serialize(&mut buf[RANGE_BYTES..]);
}
fn deserialize(buf: &[u8]) -> Self {
const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;
let range: Range<u64> = Range::deserialize(&buf[..RANGE_BYTES]);
- let sort_key = u64::deserialize(&buf[RANGE_BYTES..]);
+ let host_rank = u64::deserialize(&buf[RANGE_BYTES..]);
- Self { range, sort_key }
+ Self { range, host_rank }
}
}
@@ -401,7 +401,7 @@ impl EdgeStore {
let edges = Self::merge_postings_for_node(&buf, stores);
// write postings
- let node_sort_key = buf[0].range().sort_key;
+ let node_host_rank = buf[0].range().host_rank;
let node_id = buf[0].id();
let mut first_label_offset = None;
let mut last_label_offset = None;
@@ -444,7 +444,7 @@ impl EdgeStore {
start: first_node_offset.unwrap().start,
end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,
},
- node_sort_key,
+ node_host_rank,
);
let node_range_bytes = node_range.serialize_to_vec();
@@ -528,13 +528,13 @@ impl EdgeStore {
if self.reversed {
SegmentEdge {
from: edge.other,
- to: NodeDatum::new(*node, node_range.sort_key),
+ to: NodeDatum::new(*node, node_range.host_rank),
rel: edge.rel,
label,
}
} else {
SegmentEdge {
- from: NodeDatum::new(*node, node_range.sort_key),
+ from: NodeDatum::new(*node, node_range.host_rank),
to: edge.other,
rel: edge.rel,
label,
@@ -564,13 +564,13 @@ impl EdgeStore {
if self.reversed {
SegmentEdge {
from: edge.other,
- to: NodeDatum::new(*node, edge_range.sort_key),
+ to: NodeDatum::new(*node, edge_range.host_rank),
rel: edge.rel,
label: (),
}
} else {
SegmentEdge {
- from: NodeDatum::new(*node, edge_range.sort_key),
+ from: NodeDatum::new(*node, edge_range.host_rank),
to: edge.other,
rel: edge.rel,
label: (),
@@ -603,13 +603,13 @@ impl EdgeStore {
if self.reversed {
SegmentEdge {
from: edge.other,
- to: NodeDatum::new(node, edge_range.sort_key),
+ to: NodeDatum::new(node, edge_range.host_rank),
rel: edge.rel,
label: (),
}
} else {
SegmentEdge {
- from: NodeDatum::new(node, edge_range.sort_key),
+ from: NodeDatum::new(node, edge_range.host_rank),
to: edge.other,
rel: edge.rel,
label: (),
@@ -657,7 +657,7 @@ mod tests {
let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64), &EdgeLimit::Unlimited);
assert_eq!(edges.len(), 1);
- assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));
+ assert_eq!(&edges[0].to.node(), &SegmentEdge::from(e.clone()).to.node());
let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);
@@ -801,8 +801,17 @@ mod tests {
assert_eq!(edges.len(), 3);
- assert_eq!(Edge::from(edges[0].clone()), Edge::from(e2.clone()));
- assert_eq!(Edge::from(edges[1].clone()), Edge::from(e3.clone()));
- assert_eq!(Edge::from(edges[2].clone()), Edge::from(e1.clone()));
+ assert_eq!(
+ Edge::from(edges[0].clone()).from.node(),
+ Edge::from(e2.clone()).from.node()
+ );
+ assert_eq!(
+ Edge::from(edges[1].clone()).from.node(),
+ Edge::from(e3.clone()).from.node()
+ );
+ assert_eq!(
+ Edge::from(edges[2].clone()).from.node(),
+ Edge::from(e1.clone()).from.node()
+ );
}
}
diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rs
index cb8bdd6b..b6588bcc 100644
--- a/crates/core/src/webgraph/store_writer.rs
+++ b/crates/core/src/webgraph/store_writer.rs
@@ -39,9 +39,8 @@ use file_store::{
};
use super::{
- merge::NodeDatum,
store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},
- Compression, EdgeLabel, InsertableEdge, NodeID, StoredEdge,
+ Compression, EdgeLabel, InsertableEdge, NodeDatum, NodeID, StoredEdge,
};
#[derive(bincode::Encode, bincode::Decode)]
@@ -315,13 +314,13 @@ impl FinalEdgeStoreWriter {
(edge.to.id, edge.to.host)
};
- let sort_key = self
+ let host_rank = self
.host_centrality_rank_store
.as_ref()
.map(|store| store.get(&host).unwrap().unwrap_or(u64::MAX))
- .unwrap_or(0);
+ .unwrap_or(u64::MAX);
- let datum = NodeDatum::new(node, sort_key);
+ let datum = NodeDatum::new(node, host_rank);
let rel = edge.rel;
stored_edges.push(StoredEdge::new(datum, rel));
}
@@ -367,7 +366,7 @@ impl FinalEdgeStoreWriter {
.host_centrality_rank_store
.as_ref()
.map(|store| store.get(&node.host).unwrap().unwrap_or(u64::MAX))
- .unwrap_or(0);
+ .unwrap_or(u64::MAX);
let node_range: EdgeRange = EdgeRange::new(
Range {
diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rs
index acf807c6..707afa89 100644
--- a/crates/core/src/webpage/mod.rs
+++ b/crates/core/src/webpage/mod.rs
@@ -18,7 +18,7 @@ use crate::{
backlink_grouper::GroupedBacklinks,
inverted_index::InvertedIndex,
schema::{numerical_field::NumericalField, text_field::TextField, Field},
- webgraph::{FullEdge, NodeID},
+ webgraph::{Edge, NodeID},
Result,
};
use candle_core::Tensor;
@@ -45,13 +45,17 @@ pub struct Webpage {
pub html: Html,
#[cfg(test)]
- pub backlinks: Vec<FullEdge>,
+ pub backlinks: Vec<Edge<String>>,
#[cfg(not(test))]
- backlinks: Vec<FullEdge>,
+ backlinks: Vec<Edge<String>>,
+ #[cfg(test)]
pub grouped_backlinks: GroupedBacklinks,
+ #[cfg(not(test))]
+ grouped_backlinks: GroupedBacklinks,
+
pub host_centrality: f64,
pub host_centrality_rank: u64,
pub page_centrality: f64,
@@ -141,11 +145,11 @@ impl Webpage {
})
}
- pub fn backlinks(&self) -> &[FullEdge] {
+ pub fn backlinks(&self) -> &[Edge<String>] {
&self.backlinks
}
- pub fn set_backlinks(&mut self, backlinks: Vec<FullEdge>) {
+ pub fn set_backlinks(&mut self, backlinks: Vec<Edge<String>>) {
self.backlinks = backlinks;
}