web search done right
[webgraph] add from_centrality, to_centrality etc. fields to edges
Mikkel Denker13f06e41ba23
diff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rsindex 7292bb9b..0a6b6c9f 100644--- a/crates/core/examples/distributed_harmonic.rs+++ b/crates/core/examples/distributed_harmonic.rs@@ -104,7 +104,7 @@ fn build_graphs_if_not_exist(warc_path: &str, graph_path: &str) -> anyhow::Resulto: destination,rel_flags: link.rel,label: link.text,- sort_score: 0.0,+ ..Edge::empty()})?;} else {b.insert(Edge {@@ -112,7 +112,7 @@ fn build_graphs_if_not_exist(warc_path: &str, graph_path: &str) -> anyhow::Resulto: destination,rel_flags: link.rel,label: link.text,- sort_score: 0.0,+ ..Edge::empty()})?;}}diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 53287abf..05f4955f 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -100,6 +100,7 @@ pub struct WebgraphConstructConfig {pub graph_base_path: String,pub shard: ShardId,pub host_centrality_store_path: String,+ pub host_rank_store_path: String,pub warc_source: WarcSource,pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rsindex 008ddedb..45dd419c 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs@@ -83,10 +83,7 @@ mod tests {use tracing_test::traced_test;use webgraph::{Edge, Webgraph};- use crate::{- free_socket_addr, webgraph::centrality::harmonic::HarmonicCentrality,- webpage::html::links::RelFlags,- };+ use crate::{free_socket_addr, webgraph::centrality::harmonic::HarmonicCentrality};use super::*;@@ -106,14 +103,8 @@ mod tests {let edges = crate::webgraph::tests::test_edges();- for (i, (from, to, label)) in edges.into_iter().enumerate() {- let e = Edge {- from: from.clone(),- to: to.clone(),- label: label.clone(),- rel_flags: RelFlags::default(),- sort_score: 0.0,- };+ for (i, (from, to)) in edges.into_iter().enumerate() {+ let e = Edge::new_test(from.clone(), to.clone());combined.insert(e.clone()).unwrap();if i % 2 == 0 {diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex 1df056a5..6b07e7c2 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -117,6 +117,7 @@ fn create_webgraph() -> Result<()> {let mut worker = webgraph::WebgraphWorker {graph: crate::webgraph::Webgraph::open(&out_path, 0u64.into()).unwrap(),host_centrality_store: None,+ host_rank_store: None,canonical_index: None,};diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex c888afb3..cd052c65 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -72,6 +72,7 @@ fn canonical_or_self(index: &CanonicalIndex, url: Url) -> Url {pub struct WebgraphWorker {pub host_centrality_store: Option<Arc<speedy_kv::Db<NodeID, f64>>>,+ pub host_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,pub graph: webgraph::Webgraph,pub canonical_index: Option<Arc<CanonicalIndex>>,}@@ -109,6 +110,18 @@ impl WebgraphWorker {.as_ref().and_then(|store| store.get(&source.clone().into_host().id()).unwrap()).unwrap_or(0.0);+ let source_rank = self+ .host_rank_store+ .as_ref()+ .and_then(|store| store.get(&source.clone().into_host().id()).unwrap())+ .unwrap_or(u64::MAX);++ let num_outgoing_hosts_from_page = webpage+ .anchor_links()+ .into_iter()+ .filter_map(|l| l.destination.host_str().map(|h| h.to_string()))+ .unique()+ .count() as u64;for mut link in webpage.anchor_links().into_iter() {let mut destination = link.destination.clone();@@ -126,6 +139,11 @@ impl WebgraphWorker {.as_ref().and_then(|store| store.get(&destination.clone().into_host().id()).unwrap()).unwrap_or(0.0);+ let destination_rank = self+ .host_rank_store+ .as_ref()+ .and_then(|store| store.get(&destination.clone().into_host().id()).unwrap())+ .unwrap_or(u64::MAX);trace!("inserting link {:?}", link);self.graph@@ -135,6 +153,11 @@ impl WebgraphWorker {rel_flags: link.rel,label: link.text,sort_score: source_centrality + destination_centrality,+ from_centrality: source_centrality,+ to_centrality: destination_centrality,+ from_rank: source_rank,+ to_rank: destination_rank,+ num_outgoing_hosts_from_page,}).unwrap();}@@ -177,6 +200,9 @@ impl Webgraph {&config.host_centrality_store_path,)?);+ let host_rank_store =+ Arc::new(speedy_kv::Db::open_or_create(&config.host_rank_store_path)?);+let num_workers = usize::from(std::thread::available_parallelism()?);let mut handlers = Vec::new();@@ -196,6 +222,7 @@ impl Webgraph {let mut worker = WebgraphWorker {graph: webgraph::Webgraph::open(graph_path, config.shard)?,host_centrality_store: Some(host_centrality_store.clone()),+ host_rank_store: Some(host_rank_store.clone()),canonical_index: canonical_index.clone(),};diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 81129ca9..304d6fe1 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -262,7 +262,7 @@ mod tests {LocalSearchClient, LocalSearcher, SearchQuery,},webgraph::{Edge, Node, Webgraph},- webpage::{html::links::RelFlags, Html, Webpage},+ webpage::{Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -610,49 +610,34 @@ mod tests {let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();graph- .insert(Edge {- from: Node::from("https://www.e.com").into_host(),- to: Node::from("https://www.a.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.e.com").into_host(),+ Node::from("https://www.a.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.a.com").into_host(),- to: Node::from("https://www.e.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.a.com").into_host(),+ Node::from("https://www.e.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.c.com").into_host(),- to: Node::from("https://www.c.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.c.com").into_host(),+ Node::from("https://www.c.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.b.com").into_host(),- to: Node::from("https://www.e.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.b.com").into_host(),+ Node::from("https://www.e.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.e.com").into_host(),- to: Node::from("https://www.b.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.e.com").into_host(),+ Node::from("https://www.b.com").into_host(),+ )).unwrap();graph.commit().unwrap();diff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rsindex 596ae84d..741b5727 100644--- a/crates/core/src/ranking/bitvec_similarity.rs+++ b/crates/core/src/ranking/bitvec_similarity.rs@@ -309,8 +309,7 @@ mod tests {from: a.clone(),to: b.clone(),rel_flags: RelFlags::NOFOLLOW,- label: String::new(),- sort_score: 0.0,+ ..Edge::empty()}).unwrap();@@ -318,9 +317,7 @@ mod tests {.insert(Edge {from: a.clone(),to: c.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,+ ..Edge::empty()}).unwrap();diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 7b5d9ff4..24399af6 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -150,7 +150,7 @@ mod tests {rand_words,searcher::{api::ApiSearcher, LocalSearchClient, LocalSearcher, SearchQuery},webgraph::{Edge, EdgeLimit, Node, Webgraph},- webpage::{html::links::RelFlags, Html, Webpage},+ webpage::{Html, Webpage},};use super::*;@@ -171,86 +171,59 @@ mod tests {let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();graph- .insert(Edge {- from: Node::from("a.com").into_host(),- to: Node::from("b.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("a.com").into_host(),+ Node::from("b.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("c.com").into_host(),- to: Node::from("d.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("c.com").into_host(),+ Node::from("d.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("a.com").into_host(),- to: Node::from("e.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("a.com").into_host(),+ Node::from("e.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("z.com").into_host(),- to: Node::from("a.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("z.com").into_host(),+ Node::from("a.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("z.com").into_host(),- to: Node::from("b.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("z.com").into_host(),+ Node::from("b.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("z.com").into_host(),- to: Node::from("c.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("z.com").into_host(),+ Node::from("c.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("z.com").into_host(),- to: Node::from("d.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("z.com").into_host(),+ Node::from("d.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("z.com").into_host(),- to: Node::from("d.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("z.com").into_host(),+ Node::from("d.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("z.com").into_host(),- to: Node::from("e.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("z.com").into_host(),+ Node::from("e.com").into_host(),+ )).unwrap();graph.commit().unwrap();@@ -269,40 +242,28 @@ mod tests {let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();graph- .insert(Edge {- from: Node::from("b.com").into_host(),- to: Node::from("a.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("b.com").into_host(),+ Node::from("a.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("c.com").into_host(),- to: Node::from("d.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("c.com").into_host(),+ Node::from("d.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("b.com").into_host(),- to: Node::from("e.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("b.com").into_host(),+ Node::from("e.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("c.com").into_host(),- to: Node::from("b.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("c.com").into_host(),+ Node::from("b.com").into_host(),+ )).unwrap();graph.commit().unwrap();diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 8025a073..2fd26fed 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -26,7 +26,7 @@ mod tests {index::Index,searcher::{api::ApiSearcher, LocalSearchClient, LocalSearcher, SearchQuery},webgraph::{Edge, Node, Webgraph},- webpage::{html::links::RelFlags, Html, Webpage},+ webpage::{Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -39,76 +39,52 @@ mod tests {let mut graph = Webgraph::open(&dir, 0u64.into()).unwrap();graph- .insert(Edge {- from: Node::from("https://www.first.com").into_host(),- to: Node::from("https://www.nan.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.first.com").into_host(),+ Node::from("https://www.nan.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.nan.com").into_host(),- to: Node::from("https://www.first.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.nan.com").into_host(),+ Node::from("https://www.first.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.third.com").into_host(),- to: Node::from("https://www.third.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.third.com").into_host(),+ Node::from("https://www.third.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.nan.com").into_host(),- to: Node::from("https://www.second.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.nan.com").into_host(),+ Node::from("https://www.second.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.second.com").into_host(),- to: Node::from("https://www.nan.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.second.com").into_host(),+ Node::from("https://www.nan.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.second.com").into_host(),- to: Node::from("https://www.third.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.second.com").into_host(),+ Node::from("https://www.third.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.extra.com").into_host(),- to: Node::from("https://www.first.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.extra.com").into_host(),+ Node::from("https://www.first.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://www.second.com").into_host(),- to: Node::from("https://www.extra.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.second.com").into_host(),+ Node::from("https://www.extra.com").into_host(),+ )).unwrap();graph.commit().unwrap();diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rsindex cc4fe910..d74eeccd 100644--- a/crates/core/src/webgraph/centrality/betweenness.rs+++ b/crates/core/src/webgraph/centrality/betweenness.rs@@ -177,10 +177,7 @@ mod tests {use file_store::temp::TempDir;use maplit::hashmap;- use crate::{- webgraph::{Edge, Webgraph},- webpage::html::links::RelFlags,- };+ use crate::webgraph::{Edge, Webgraph};use super::*;@@ -192,13 +189,10 @@ mod tests {for i in 0..n - 1 {graph- .insert(Edge {- from: Node::from(i.to_string()),- to: Node::from((i + 1).to_string()),- label: String::new(),- rel_flags: RelFlags::default(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from(i.to_string()),+ Node::from((i + 1).to_string()),+ )).unwrap();}graph.commit().unwrap();diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex ce693ddc..8547a8e9 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -320,7 +320,7 @@ mod tests {webpage::html::links::RelFlags,};- fn test_edges() -> Vec<(Node, Node, String)> {+ fn test_edges() -> Vec<(Node, Node)> {// ┌────┐// │ │// ┌───A◄─┐ │@@ -332,11 +332,11 @@ mod tests {// │// Dvec![- (Node::from("A"), Node::from("B"), String::new()),- (Node::from("B"), Node::from("C"), String::new()),- (Node::from("A"), Node::from("C"), String::new()),- (Node::from("C"), Node::from("A"), String::new()),- (Node::from("D"), Node::from("C"), String::new()),+ (Node::from("A"), Node::from("B")),+ (Node::from("B"), Node::from("C")),+ (Node::from("A"), Node::from("C")),+ (Node::from("C"), Node::from("A")),+ (Node::from("D"), Node::from("C")),]}@@ -346,16 +346,8 @@ mod tests {.open().unwrap();- for (from, to, label) in test_edges() {- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ for (from, to) in test_edges() {+ graph.insert(Edge::new_test(from, to)).unwrap();}graph.commit().unwrap();@@ -371,130 +363,88 @@ mod tests {.unwrap();graph- .insert(Edge {- from: Node::from("A.com/1").into_host(),- to: Node::from("A.com/2").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/1").into_host(),+ Node::from("A.com/2").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/1").into_host(),- to: Node::from("A.com/3").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/1").into_host(),+ Node::from("A.com/3").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/1").into_host(),- to: Node::from("A.com/4").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/1").into_host(),+ Node::from("A.com/4").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/2").into_host(),- to: Node::from("A.com/1").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/2").into_host(),+ Node::from("A.com/1").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/2").into_host(),- to: Node::from("A.com/3").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/2").into_host(),+ Node::from("A.com/3").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/2").into_host(),- to: Node::from("A.com/4").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/2").into_host(),+ Node::from("A.com/4").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/3").into_host(),- to: Node::from("A.com/1").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/3").into_host(),+ Node::from("A.com/1").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/3").into_host(),- to: Node::from("A.com/2").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/3").into_host(),+ Node::from("A.com/2").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/3").into_host(),- to: Node::from("A.com/4").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/3").into_host(),+ Node::from("A.com/4").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/4").into_host(),- to: Node::from("A.com/1").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/4").into_host(),+ Node::from("A.com/1").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/4").into_host(),- to: Node::from("A.com/2").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/4").into_host(),+ Node::from("A.com/2").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("A.com/4").into_host(),- to: Node::from("A.com/3").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("A.com/4").into_host(),+ Node::from("A.com/3").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("C.com").into_host(),- to: Node::from("B.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("C.com").into_host(),+ Node::from("B.com").into_host(),+ )).unwrap();graph- .insert(Edge {- from: Node::from("D.com").into_host(),- to: Node::from("B.com").into_host(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("D.com").into_host(),+ Node::from("B.com").into_host(),+ )).unwrap();graph.commit().unwrap();@@ -533,86 +483,40 @@ mod tests {.open().unwrap();- for (from, to, label) in test_edges() {- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ for (from, to) in test_edges() {+ graph.insert(Edge::new_test(from, to)).unwrap();}graph- .insert(Edge {- from: Node::from("A"),- to: Node::from("B"),- rel_flags: RelFlags::default(),- label: "1".to_string(),- sort_score: 0.0,- })+ .insert(Edge::new_test(Node::from("A"), Node::from("B")))+ .unwrap();+ graph.commit().unwrap();+ graph+ .insert(Edge::new_test(Node::from("A"), Node::from("B"))).unwrap();graph.commit().unwrap();graph- .insert(Edge {- from: Node::from("A"),- to: Node::from("B"),- rel_flags: RelFlags::default(),- label: "2".to_string(),- sort_score: 0.0,- })+ .insert(Edge::new_test(Node::from("A"), Node::from("B"))).unwrap();graph.commit().unwrap();graph- .insert(Edge {- from: Node::from("A"),- to: Node::from("B"),- rel_flags: RelFlags::default(),- label: "3".to_string(),- sort_score: 0.0,- })+ .insert(Edge::new_test(Node::from("A"), Node::from("B"))).unwrap();graph.commit().unwrap();graph- .insert(Edge {- from: Node::from("A"),- to: Node::from("B"),- rel_flags: RelFlags::default(),- label: "4".to_string(),- sort_score: 0.0,- })+ .insert(Edge::new_test(Node::from("A"), Node::from("B"))).unwrap();graph.commit().unwrap();graph- .insert(Edge {- from: Node::from("A"),- to: Node::from("B"),- rel_flags: RelFlags::default(),- label: "5".to_string(),- sort_score: 0.0,- })+ .insert(Edge::new_test(Node::from("A"), Node::from("B"))).unwrap();graph.commit().unwrap();graph- .insert(Edge {- from: Node::from("A"),- to: Node::from("B"),- rel_flags: RelFlags::default(),- label: "6".to_string(),- sort_score: 0.0,- })+ .insert(Edge::new_test(Node::from("A"), Node::from("B"))).unwrap();graph.commit().unwrap();graph- .insert(Edge {- from: Node::from("A"),- to: Node::from("B"),- rel_flags: RelFlags::default(),- label: "7".to_string(),- sort_score: 0.0,- })+ .insert(Edge::new_test(Node::from("A"), Node::from("B"))).unwrap();graph.commit().unwrap();@@ -630,14 +534,13 @@ mod tests {.open().unwrap();- for (from, to, label) in test_edges() {+ for (from, to) in test_edges() {graph.insert(Edge {from,to,rel_flags: RelFlags::TAG,- label,- sort_score: 0.0,+ ..Edge::empty()}).unwrap();}@@ -656,14 +559,13 @@ mod tests {.open().unwrap();- for (from, to, label) in test_edges() {+ for (from, to) in test_edges() {graph.insert(Edge {from,to,rel_flags: RelFlags::SAME_ICANN_DOMAIN,- label,- sort_score: 0.0,+ ..Edge::empty()}).unwrap();}diff --git a/crates/core/src/webgraph/document.rs b/crates/core/src/webgraph/document.rsindex f44c6eb2..890ccf88 100644--- a/crates/core/src/webgraph/document.rs+++ b/crates/core/src/webgraph/document.rs@@ -172,6 +172,11 @@ pub struct Edge {pub rel_flags: RelFlags,pub label: String,pub sort_score: f64,+ pub from_centrality: f64,+ pub to_centrality: f64,+ pub from_rank: u64,+ pub to_rank: u64,+ pub num_outgoing_hosts_from_page: u64,}impl Edge {@@ -182,6 +187,20 @@ impl Edge {rel_flags: RelFlags::default(),label: String::default(),sort_score: 0.0,+ from_centrality: 0.0,+ to_centrality: 0.0,+ from_rank: 0,+ to_rank: 0,+ num_outgoing_hosts_from_page: 0,+ }+ }++ #[cfg(test)]+ pub fn new_test(from: Node, to: Node) -> Self {+ Self {+ from,+ to,+ ..Self::empty()}}}diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rsindex 94aabbef..0c151e87 100644--- a/crates/core/src/webgraph/query/between.rs+++ b/crates/core/src/webgraph/query/between.rs@@ -153,7 +153,7 @@ impl Query for FullLinksBetweenQuery {#[cfg(test)]mod tests {- use crate::{webgraph::Webgraph, webpage::RelFlags};+ use crate::webgraph::Webgraph;use super::*;@@ -166,13 +166,7 @@ mod tests {let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();graph- .insert(Edge {- from: from.clone(),- to: to.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(from.clone(), to.clone())).unwrap();graph.commit().unwrap();diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rsindex d22ed9a4..b64d51d4 100644--- a/crates/core/src/webgraph/query/collector/top_docs.rs+++ b/crates/core/src/webgraph/query/collector/top_docs.rs@@ -449,12 +449,9 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> SegmentCollector#[cfg(test)]mod tests {- use crate::{- webgraph::{- query::{BacklinksQuery, HostBacklinksQuery},- Edge, EdgeLimit, Node, Webgraph,- },- webpage::RelFlags,+ use crate::webgraph::{+ query::{BacklinksQuery, HostBacklinksQuery},+ Edge, EdgeLimit, Node, Webgraph,};#[test]@@ -463,13 +460,10 @@ mod tests {let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();graph- .insert(Edge {- from: Node::from("https://A.com/1"),- to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/1"),+ Node::from("https://B.com/1"),+ )).unwrap();graph.commit().unwrap();@@ -488,22 +482,16 @@ mod tests {let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();graph- .insert(Edge {- from: Node::from("https://A.com/1"),- to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/1"),+ Node::from("https://B.com/1"),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://A.com/2"),- to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/2"),+ Node::from("https://B.com/1"),+ )).unwrap();graph.commit().unwrap();@@ -529,24 +517,16 @@ mod tests {let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();graph- .insert(Edge {- from: Node::from("https://A.com/1"),- to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/1"),+ Node::from("https://B.com/1"),+ )).unwrap();- graph.commit().unwrap();-graph- .insert(Edge {- from: Node::from("https://A.com/2"),- to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/2"),+ Node::from("https://B.com/1"),+ )).unwrap();graph.commit().unwrap();@@ -569,27 +549,24 @@ mod tests {.insert(Edge {from: Node::from("https://A.com/1"),to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),sort_score: 0.1,+ ..Edge::empty()}).unwrap();graph.insert(Edge {from: Node::from("https://A.com/2"),to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),sort_score: 0.1,+ ..Edge::empty()}).unwrap();graph.insert(Edge {from: Node::from("https://C.com/1"),to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),sort_score: 0.0,+ ..Edge::empty()}).unwrap();diff --git a/crates/core/src/webgraph/query/filter/and.rs b/crates/core/src/webgraph/query/filter/and.rsindex 5aa5621a..3091ef35 100644--- a/crates/core/src/webgraph/query/filter/and.rs+++ b/crates/core/src/webgraph/query/filter/and.rs@@ -124,21 +124,18 @@ impl super::InvertedIndexFilter for AndInvertedIndexFilter {mod tests {use file_store::temp::TempDir;- use crate::{- webgraph::{- query::{ForwardlinksQuery, TextFilter},- schema::ToUrl,- Edge, Node, Webgraph,- },- webpage::RelFlags,+ use crate::webgraph::{+ query::{ForwardlinksQuery, TextFilter},+ schema::ToUrl,+ Edge, Node, Webgraph,};- pub fn test_edges() -> Vec<(Node, Node, String)> {+ pub fn test_edges() -> Vec<(Node, Node)> {vec![- (Node::from("a.com"), Node::from("b.com/123"), String::new()),- (Node::from("a.com"), Node::from("b.dk/123"), String::new()),- (Node::from("a.com"), Node::from("b.com/321"), String::new()),- (Node::from("a.com"), Node::from("c.com"), String::new()),+ (Node::from("a.com"), Node::from("b.com/123")),+ (Node::from("a.com"), Node::from("b.dk/123")),+ (Node::from("a.com"), Node::from("b.com/321")),+ (Node::from("a.com"), Node::from("c.com")),]}@@ -146,16 +143,8 @@ mod tests {let temp_dir = crate::gen_temp_dir().unwrap();let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();- for (from, to, label) in test_edges() {- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ for (from, to) in test_edges() {+ graph.insert(Edge::new_test(from, to)).unwrap();}graph.commit().unwrap();diff --git a/crates/core/src/webgraph/query/filter/not.rs b/crates/core/src/webgraph/query/filter/not.rsindex a0916809..4fc0fb9f 100644--- a/crates/core/src/webgraph/query/filter/not.rs+++ b/crates/core/src/webgraph/query/filter/not.rs@@ -93,24 +93,21 @@ impl super::InvertedIndexFilter for NotInvertedIndexFilter {mod tests {use file_store::temp::TempDir;- use crate::{- webgraph::{- query::{FullForwardlinksQuery, OrFilter, TextFilter},- schema::ToUrl,- Edge, Node, Webgraph,- },- webpage::RelFlags,+ use crate::webgraph::{+ query::{FullForwardlinksQuery, OrFilter, TextFilter},+ schema::ToUrl,+ Edge, Node, Webgraph,};use super::*;- pub fn test_edges() -> Vec<(Node, Node, String)> {+ pub fn test_edges() -> Vec<(Node, Node)> {vec![- (Node::from("a.com"), Node::from("b.com/123"), String::new()),- (Node::from("a.com"), Node::from("b.dk/123"), String::new()),- (Node::from("a.com"), Node::from("b.se/123"), String::new()),- (Node::from("a.com"), Node::from("b.com/321"), String::new()),- (Node::from("a.com"), Node::from("c.com"), String::new()),+ (Node::from("a.com"), Node::from("b.com/123")),+ (Node::from("a.com"), Node::from("b.dk/123")),+ (Node::from("a.com"), Node::from("b.se/123")),+ (Node::from("a.com"), Node::from("b.com/321")),+ (Node::from("a.com"), Node::from("c.com")),]}@@ -118,16 +115,8 @@ mod tests {let temp_dir = crate::gen_temp_dir().unwrap();let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();- for (from, to, label) in test_edges() {- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ for (from, to) in test_edges() {+ graph.insert(Edge::new_test(from, to)).unwrap();}graph.commit().unwrap();diff --git a/crates/core/src/webgraph/query/filter/or.rs b/crates/core/src/webgraph/query/filter/or.rsindex 89ce7106..1ab866b9 100644--- a/crates/core/src/webgraph/query/filter/or.rs+++ b/crates/core/src/webgraph/query/filter/or.rs@@ -137,24 +137,21 @@ impl super::InvertedIndexFilter for OrInvertedIndexFilter {mod tests {use file_store::temp::TempDir;- use crate::{- webgraph::{- query::{FullForwardlinksQuery, TextFilter},- schema::ToUrl,- Edge, Node, Webgraph,- },- webpage::RelFlags,+ use crate::webgraph::{+ query::{FullForwardlinksQuery, TextFilter},+ schema::ToUrl,+ Edge, Node, Webgraph,};use super::*;- pub fn test_edges() -> Vec<(Node, Node, String)> {+ pub fn test_edges() -> Vec<(Node, Node)> {vec![- (Node::from("a.com"), Node::from("b.com/123"), String::new()),- (Node::from("a.com"), Node::from("b.dk/123"), String::new()),- (Node::from("a.com"), Node::from("b.se/123"), String::new()),- (Node::from("a.com"), Node::from("b.com/321"), String::new()),- (Node::from("a.com"), Node::from("c.com"), String::new()),+ (Node::from("a.com"), Node::from("b.com/123")),+ (Node::from("a.com"), Node::from("b.dk/123")),+ (Node::from("a.com"), Node::from("b.se/123")),+ (Node::from("a.com"), Node::from("b.com/321")),+ (Node::from("a.com"), Node::from("c.com")),]}@@ -162,16 +159,8 @@ mod tests {let temp_dir = crate::gen_temp_dir().unwrap();let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();- for (from, to, label) in test_edges() {- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ for (from, to) in test_edges() {+ graph.insert(Edge::new_test(from, to)).unwrap();}graph.commit().unwrap();diff --git a/crates/core/src/webgraph/query/filter/rel_flag.rs b/crates/core/src/webgraph/query/filter/rel_flag.rsindex 8aa48601..5a747160 100644--- a/crates/core/src/webgraph/query/filter/rel_flag.rs+++ b/crates/core/src/webgraph/query/filter/rel_flag.rs@@ -113,8 +113,7 @@ mod tests {from,to,rel_flags,- label: String::new(),- sort_score: 0.0,+ ..Edge::empty()}).unwrap();}diff --git a/crates/core/src/webgraph/query/filter/text.rs b/crates/core/src/webgraph/query/filter/text.rsindex 1285329d..5c5b55cd 100644--- a/crates/core/src/webgraph/query/filter/text.rs+++ b/crates/core/src/webgraph/query/filter/text.rs@@ -76,24 +76,21 @@ impl super::InvertedIndexFilter for TextInvertedIndexFilter {mod tests {use file_store::temp::TempDir;- use crate::{- webgraph::{- query::{BacklinksQuery, ForwardlinksQuery, FullBacklinksQuery, FullForwardlinksQuery},- schema::{FromUrl, ToUrl},- Edge, Node, Webgraph,- },- webpage::RelFlags,+ use crate::webgraph::{+ query::{BacklinksQuery, ForwardlinksQuery, FullBacklinksQuery, FullForwardlinksQuery},+ schema::{FromUrl, ToUrl},+ Edge, Node, Webgraph,};use super::*;- pub fn test_edges() -> Vec<(Node, Node, String)> {+ pub fn test_edges() -> Vec<(Node, Node)> {vec![- (Node::from("a.com"), Node::from("b.com"), String::new()),- (Node::from("a.com"), Node::from("b.dk"), String::new()),- (Node::from("b.com"), Node::from("b.dk"), String::new()),- (Node::from("c.dk"), Node::from("b.dk"), String::new()),- (Node::from("c.com"), Node::from("a.com"), String::new()),+ (Node::from("a.com"), Node::from("b.com")),+ (Node::from("a.com"), Node::from("b.dk")),+ (Node::from("b.com"), Node::from("b.dk")),+ (Node::from("c.dk"), Node::from("b.dk")),+ (Node::from("c.com"), Node::from("a.com")),]}@@ -101,16 +98,8 @@ mod tests {let temp_dir = crate::gen_temp_dir().unwrap();let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();- for (from, to, label) in test_edges() {- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ for (from, to) in test_edges() {+ graph.insert(Edge::new_test(from, to)).unwrap();}graph.commit().unwrap();diff --git a/crates/core/src/webgraph/query/raw/links.rs b/crates/core/src/webgraph/query/raw/links.rsindex eebddd53..e4048c4c 100644--- a/crates/core/src/webgraph/query/raw/links.rs+++ b/crates/core/src/webgraph/query/raw/links.rs@@ -238,10 +238,7 @@ impl tantivy::DocSet for LinksScorer {#[cfg(test)]mod tests {- use crate::{- webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph},- webpage::RelFlags,- };+ use crate::webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph};#[test]fn test_simple() {@@ -253,23 +250,11 @@ mod tests {let node_c = Node::from("C");graph- .insert(Edge {- from: node_a.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(node_a.clone(), node_b.clone())).unwrap();graph- .insert(Edge {- from: node_c.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(node_c.clone(), node_b.clone())).unwrap();graph.commit().unwrap();@@ -290,23 +275,11 @@ mod tests {let node_b = Node::from("B");graph- .insert(Edge {- from: node_a.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(node_a.clone(), node_b.clone())).unwrap();graph- .insert(Edge {- from: node_b.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(node_b.clone(), node_b.clone())).unwrap();graph.commit().unwrap();@@ -323,33 +296,24 @@ mod tests {let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();graph- .insert(Edge {- from: Node::from("https://A.com/1"),- to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/1"),+ Node::from("https://B.com/1"),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://A.com/2"),- to: Node::from("https://B.com/2"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/2"),+ Node::from("https://B.com/2"),+ )).unwrap();graph- .insert(Edge {- from: Node::from("https://A.com/3"),- to: Node::from("https://B.com/3"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://A.com/3"),+ Node::from("https://B.com/3"),+ )).unwrap();graph.commit().unwrap();diff --git a/crates/core/src/webgraph/schema.rs b/crates/core/src/webgraph/schema.rsindex 87078974..655db907 100644--- a/crates/core/src/webgraph/schema.rs+++ b/crates/core/src/webgraph/schema.rs@@ -321,6 +321,160 @@ impl Field for SortScore {}}+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]+pub struct FromCentrality;+impl Field for FromCentrality {+ fn name(&self) -> &'static str {+ "from_centrality"+ }++ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {+ ReferenceValue::F64(edge.from_centrality)+ }++ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {+ edge.from_centrality = value+ .as_ref()+ .as_f64()+ .ok_or(anyhow::anyhow!("Invalid centrality"))?;++ Ok(())+ }++ fn field_type(&self) -> tantivy::schema::FieldType {+ FieldType::F64(+ NumericOptions::default()+ .set_indexed()+ .set_stored()+ .set_columnar(),+ )+ }+}++#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]+pub struct ToCentrality;++impl Field for ToCentrality {+ fn name(&self) -> &'static str {+ "to_centrality"+ }++ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {+ ReferenceValue::F64(edge.to_centrality)+ }++ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {+ edge.to_centrality = value+ .as_ref()+ .as_f64()+ .ok_or(anyhow::anyhow!("Invalid centrality"))?;++ Ok(())+ }++ fn field_type(&self) -> tantivy::schema::FieldType {+ FieldType::F64(+ NumericOptions::default()+ .set_indexed()+ .set_stored()+ .set_columnar(),+ )+ }+}++#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]+pub struct FromRank;++impl Field for FromRank {+ fn name(&self) -> &'static str {+ "from_rank"+ }++ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {+ ReferenceValue::U64(edge.from_rank)+ }++ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {+ edge.from_rank = value+ .as_ref()+ .as_u64()+ .ok_or(anyhow::anyhow!("Invalid rank"))?;++ Ok(())+ }++ fn field_type(&self) -> tantivy::schema::FieldType {+ FieldType::U64(+ NumericOptions::default()+ .set_indexed()+ .set_stored()+ .set_columnar(),+ )+ }+}++#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]+pub struct ToRank;++impl Field for ToRank {+ fn name(&self) -> &'static str {+ "to_rank"+ }++ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {+ ReferenceValue::U64(edge.to_rank)+ }++ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {+ edge.to_rank = value+ .as_ref()+ .as_u64()+ .ok_or(anyhow::anyhow!("Invalid rank"))?;++ Ok(())+ }++ fn field_type(&self) -> tantivy::schema::FieldType {+ FieldType::U64(+ NumericOptions::default()+ .set_indexed()+ .set_stored()+ .set_columnar(),+ )+ }+}++#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]+pub struct NumOutgoingHostsFromPage;++impl Field for NumOutgoingHostsFromPage {+ fn name(&self) -> &'static str {+ "num_outgoing_hosts_from_page"+ }++ fn document_value<'a>(&self, edge: &'a Edge) -> ReferenceValue<'a> {+ ReferenceValue::U64(edge.num_outgoing_hosts_from_page)+ }++ fn set_value(&self, edge: &mut Edge, value: OwnedValue) -> Result<()> {+ edge.num_outgoing_hosts_from_page = value+ .as_ref()+ .as_u64()+ .ok_or(anyhow::anyhow!("Invalid number of outgoing hosts"))?;++ Ok(())+ }++ fn field_type(&self) -> tantivy::schema::FieldType {+ FieldType::U64(+ NumericOptions::default()+ .set_indexed()+ .set_stored()+ .set_columnar(),+ )+ }+}+#[enum_dispatch(Field)]#[derive(Clone, Copy, Debug, EnumDiscriminants, bincode::Encode, bincode::Decode)]#[strum_discriminants(derive(VariantArray))]@@ -334,6 +488,11 @@ pub enum FieldEnum {RelFlags,Label,SortScore,+ FromCentrality,+ ToCentrality,+ FromRank,+ ToRank,+ NumOutgoingHostsFromPage,}impl FieldEnum {@@ -356,6 +515,11 @@ enum_dispatch_from_discriminant!(FieldEnumDiscriminants => FieldEnum,RelFlags,Label,SortScore,+ FromCentrality,+ ToCentrality,+ FromRank,+ ToRank,+ NumOutgoingHostsFromPage,]);impl crate::enum_map::InsertEnumMapKey for FieldEnumDiscriminants {diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 1e171abc..13b21c29 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -425,13 +425,10 @@ mod tests {let temp_dir = crate::gen_temp_dir().unwrap();let mut store = EdgeStore::open(&temp_dir, ShardId::new(0)).unwrap();- let e = Edge {- from: Node::from("https://www.first.com").into_host(),- to: Node::from("https://www.second.com").into_host(),- label: "test".to_string(),- rel_flags: RelFlags::default(),- sort_score: 0.0,- };+ let e = Edge::new_test(+ Node::from("https://www.first.com").into_host(),+ Node::from("https://www.second.com").into_host(),+ );let from_node_id = e.from.id();let to_node_id = e.to.id();@@ -477,6 +474,11 @@ mod tests {label: "test".to_string(),rel_flags: RelFlags::default(),sort_score: a_centrality + b_centrality,+ from_centrality: b_centrality,+ to_centrality: a_centrality,+ from_rank: 2,+ to_rank: 1,+ ..Edge::empty()};let e2 = Edge {@@ -485,6 +487,11 @@ mod tests {label: "2".to_string(),rel_flags: RelFlags::default(),sort_score: a_centrality + c_centrality,+ from_centrality: c_centrality,+ to_centrality: a_centrality,+ from_rank: 3,+ to_rank: 1,+ ..Edge::empty()};let e3 = Edge {@@ -493,6 +500,11 @@ mod tests {label: "3".to_string(),rel_flags: RelFlags::default(),sort_score: a_centrality + d_centrality,+ from_centrality: d_centrality,+ to_centrality: a_centrality,+ from_rank: 4,+ to_rank: 1,+ ..Edge::empty()};store.insert(e1.clone()).unwrap();@@ -517,35 +529,26 @@ mod tests {let mut store = EdgeStore::open(&temp_dir, ShardId::new(0)).unwrap();store- .insert(Edge {- from: Node::from("https://www.first.com").into_host(),- to: Node::from("https://www.second.com").into_host(),- label: String::new(),- rel_flags: RelFlags::default(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.first.com").into_host(),+ Node::from("https://www.second.com").into_host(),+ )).unwrap();store.commit().unwrap();store- .insert(Edge {- from: Node::from("https://www.second.com").into_host(),- to: Node::from("https://www.first.com").into_host(),- label: String::new(),- rel_flags: RelFlags::default(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.second.com").into_host(),+ Node::from("https://www.first.com").into_host(),+ )).unwrap();store.commit().unwrap();store- .insert(Edge {- from: Node::from("https://www.third.com").into_host(),- to: Node::from("https://www.first.com").into_host(),- label: String::new(),- rel_flags: RelFlags::default(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::from("https://www.third.com").into_host(),+ Node::from("https://www.first.com").into_host(),+ )).unwrap();store.commit().unwrap();diff --git a/crates/core/src/webgraph/tests.rs b/crates/core/src/webgraph/tests.rsindex 22e1a4cd..e8f6a6a3 100644--- a/crates/core/src/webgraph/tests.rs+++ b/crates/core/src/webgraph/tests.rs@@ -55,13 +55,13 @@ use crate::webpage::html::links::RelFlags;use file_store::temp::TempDir;use proptest::prelude::*;-pub fn test_edges() -> Vec<(Node, Node, String)> {+pub fn test_edges() -> Vec<(Node, Node)> {vec![- (Node::from("A"), Node::from("B"), String::new()),- (Node::from("B"), Node::from("C"), String::new()),- (Node::from("A"), Node::from("C"), String::new()),- (Node::from("C"), Node::from("A"), String::new()),- (Node::from("D"), Node::from("C"), String::new()),+ (Node::from("A"), Node::from("B")),+ (Node::from("B"), Node::from("C")),+ (Node::from("A"), Node::from("C")),+ (Node::from("C"), Node::from("A")),+ (Node::from("D"), Node::from("C")),]}@@ -80,16 +80,8 @@ pub fn test_graph() -> (Webgraph, TempDir) {let temp_dir = crate::gen_temp_dir().unwrap();let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();- for (from, to, label) in test_edges() {- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ for (from, to) in test_edges() {+ graph.insert(Edge::new_test(from, to)).unwrap();}graph.commit().unwrap();@@ -146,28 +138,20 @@ fn reversed_distance_calculation() {fn merge_path() {let mut graphs = Vec::new();let temp_dir = crate::gen_temp_dir().unwrap();- for (i, (from, to, label)) in (0..).zip([- (Node::from("A"), Node::from("B"), String::new()),- (Node::from("B"), Node::from("C"), String::new()),- (Node::from("C"), Node::from("D"), String::new()),- (Node::from("D"), Node::from("E"), String::new()),- (Node::from("E"), Node::from("F"), String::new()),- (Node::from("F"), Node::from("G"), String::new()),- (Node::from("G"), Node::from("H"), String::new()),+ for (i, (from, to)) in (0..).zip([+ (Node::from("A"), Node::from("B")),+ (Node::from("B"), Node::from("C")),+ (Node::from("C"), Node::from("D")),+ (Node::from("D"), Node::from("E")),+ (Node::from("E"), Node::from("F")),+ (Node::from("F"), Node::from("G")),+ (Node::from("G"), Node::from("H")),]) {let mut graph =Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into()).open().unwrap();- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ graph.insert(Edge::new_test(from, to)).unwrap();graph.commit().unwrap();graphs.push(graph);}@@ -199,20 +183,12 @@ fn merge_path() {fn merge_simple() {let mut graphs = Vec::new();let temp_dir = crate::gen_temp_dir().unwrap();- for (i, (from, to, label)) in (0..).zip(test_edges()) {+ for (i, (from, to)) in (0..).zip(test_edges()) {let mut graph =Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into()).open().unwrap();- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ graph.insert(Edge::new_test(from, to)).unwrap();graph.commit().unwrap();graphs.push(graph);}@@ -275,25 +251,17 @@ fn merge_simple() {fn merge_cycle() {let mut graphs = Vec::new();let temp_dir = crate::gen_temp_dir().unwrap();- for (i, (from, to, label)) in (0..).zip([- (Node::from("A"), Node::from("B"), String::new()),- (Node::from("B"), Node::from("A"), String::new()),- (Node::from("B"), Node::from("C"), String::new()),- (Node::from("C"), Node::from("A"), String::new()),+ for (i, (from, to)) in (0..).zip([+ (Node::from("A"), Node::from("B")),+ (Node::from("B"), Node::from("A")),+ (Node::from("B"), Node::from("C")),+ (Node::from("C"), Node::from("A")),]) {let mut graph =Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into()).open().unwrap();- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ graph.insert(Edge::new_test(from, to)).unwrap();graph.commit().unwrap();graphs.push(graph);}@@ -344,25 +312,17 @@ fn merge_cycle() {fn merge_star() {let mut graphs = Vec::new();let temp_dir = crate::gen_temp_dir().unwrap();- for (i, (from, to, label)) in (0..).zip([- (Node::from("A"), Node::from("B"), String::new()),- (Node::from("A"), Node::from("C"), String::new()),- (Node::from("A"), Node::from("D"), String::new()),- (Node::from("A"), Node::from("E"), String::new()),+ for (i, (from, to)) in (0..).zip([+ (Node::from("A"), Node::from("B")),+ (Node::from("A"), Node::from("C")),+ (Node::from("A"), Node::from("D")),+ (Node::from("A"), Node::from("E")),]) {let mut graph =Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into()).open().unwrap();- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ graph.insert(Edge::new_test(from, to)).unwrap();graph.commit().unwrap();graphs.push(graph);}@@ -408,25 +368,17 @@ fn merge_star() {fn merge_reverse_star() {let mut graphs = Vec::new();let temp_dir = crate::gen_temp_dir().unwrap();- for (i, (from, to, label)) in (0..).zip([- (Node::from("B"), Node::from("A"), String::new()),- (Node::from("C"), Node::from("A"), String::new()),- (Node::from("D"), Node::from("A"), String::new()),- (Node::from("E"), Node::from("A"), String::new()),+ for (i, (from, to)) in (0..).zip([+ (Node::from("B"), Node::from("A")),+ (Node::from("C"), Node::from("A")),+ (Node::from("D"), Node::from("A")),+ (Node::from("E"), Node::from("A")),]) {let mut graph =Webgraph::builder(&temp_dir.as_ref().join(format!("test_{}", i)), 0u64.into()).open().unwrap();- graph- .insert(Edge {- from,- to,- rel_flags: RelFlags::default(),- label,- sort_score: 0.0,- })- .unwrap();+ graph.insert(Edge::new_test(from, to)).unwrap();graph.commit().unwrap();graphs.push(graph);}@@ -484,13 +436,10 @@ proptest! {.open().unwrap();for (from, to) in nodes.clone() {- graph.insert(Edge {- from: Node::new_for_test(from.as_str()),- to: Node::new_for_test(to.as_str()),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- }).unwrap();+ graph.insert(Edge::new_test(+ Node::new_for_test(from.as_str()),+ Node::new_for_test(to.as_str()),+ )).unwrap();if rand::random::<usize>() % 10 == 0 {graph.commit().unwrap();@@ -544,13 +493,10 @@ fn proptest_case(nodes: &[(&str, &str)]) {for (i, (from, to)) in nodes.iter().enumerate() {graph- .insert(Edge {- from: Node::new_for_test(from),- to: Node::new_for_test(to),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })+ .insert(Edge::new_test(+ Node::new_for_test(from),+ Node::new_for_test(to),+ )).unwrap();if i % 2 == 0 {@@ -642,9 +588,8 @@ fn cap_label_length() {.insert(Edge {from: Node::from("A"),to: Node::from("B"),- rel_flags: RelFlags::default(),label: "a".repeat(MAX_LABEL_LENGTH + 1),- sort_score: 0.0,+ ..Edge::empty()}).unwrap();@@ -675,9 +620,9 @@ fn test_edge_limits() {);let mut graphs = Vec::new();- for (from, to, label) in &[- (Node::from("A"), Node::from("B"), String::new()),- (Node::from("A"), Node::from("C"), String::new()),+ for (from, to) in &[+ (Node::from("A"), Node::from("B")),+ (Node::from("A"), Node::from("C")),] {let mut graph = Webgraph::builder(&temp_dir.as_ref().join(uuid::Uuid::new_v4().to_string()),@@ -686,13 +631,7 @@ fn test_edge_limits() {.open().unwrap();graph- .insert(Edge {- from: from.clone(),- to: to.clone(),- rel_flags: RelFlags::default(),- label: label.clone(),- sort_score: 0.0,- })+ .insert(Edge::new_test(from.clone(), to.clone())).unwrap();graph.commit().unwrap();graphs.push(graph);@@ -759,8 +698,7 @@ fn test_rel_flags() {from: Node::from("A"),to: Node::from("B"),rel_flags: RelFlags::IS_IN_FOOTER | RelFlags::TAG,- label: String::new(),- sort_score: 0.0,+ ..Edge::empty()}).unwrap();