web search done right
store 'rel' attribute for each edge in the webgraph
Mikkel Denkera9eb8acd801e
diff --git a/Cargo.lock b/Cargo.lockindex b99d44f7..d0b91b50 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -549,9 +549,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"[[package]]name = "bitflags"-version = "2.4.2"+version = "2.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf"+checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"[[package]]name = "bitpacking"@@ -2742,7 +2742,7 @@ version = "0.0.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"dependencies = [- "bitflags 2.4.2",+ "bitflags 2.5.0","libc","redox_syscall",]@@ -3323,7 +3323,7 @@ version = "0.10.64"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f"dependencies = [- "bitflags 2.4.2",+ "bitflags 2.5.0","cfg-if","foreign-types","libc",@@ -3779,7 +3779,7 @@ checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf"dependencies = ["bit-set","bit-vec",- "bitflags 2.4.2",+ "bitflags 2.5.0","lazy_static","num-traits","rand",@@ -4428,7 +4428,7 @@ version = "0.38.31"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949"dependencies = [- "bitflags 2.4.2",+ "bitflags 2.5.0","errno","libc","linux-raw-sys",@@ -4674,7 +4674,7 @@ version = "0.25.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"dependencies = [- "bitflags 2.4.2",+ "bitflags 2.5.0","cssparser","derive_more","fxhash",@@ -4976,6 +4976,7 @@ dependencies = ["axum-macros","base64 0.21.7","bincode",+ "bitflags 2.5.0","bitvec","bloom","bytecount",@@ -5204,7 +5205,7 @@ version = "0.5.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"dependencies = [- "bitflags 2.4.2",+ "bitflags 2.5.0","byteorder 1.5.0","enum-as-inner","libc",@@ -5744,7 +5745,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"dependencies = ["async-compression",- "bitflags 2.4.2",+ "bitflags 2.5.0","bytes","futures-core","http 1.1.0",diff --git a/Cargo.toml b/Cargo.tomlindex e2e23212..006f40db 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -33,6 +33,7 @@ bincode = {git = "https://github.com/bincode-org/bincode", rev = "8f65c60", feat"serde",]}bindgen = "0.69.2"+bitflags = "2.5.0"bitvec = {version = "1.0.1", features = ["serde"]}bytecount = "0.6.7"bytemuck = {version = "1.13.1", features = ["derive"]}diff --git a/assets/licenses.html b/assets/licenses.htmlindex 41b31b71..091097df 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -8320,7 +8320,7 @@ limitations under the License.</pre><li><a href=" https://github.com/marshallpierce/rust-base64 ">base64 0.21.7</a></li><li><a href=" https://github.com/marshallpierce/rust-base64 ">base64 0.22.0</a></li><li><a href=" https://github.com/bitflags/bitflags ">bitflags 1.3.2</a></li>- <li><a href=" https://github.com/bitflags/bitflags ">bitflags 2.4.2</a></li>+ <li><a href=" https://github.com/bitflags/bitflags ">bitflags 2.5.0</a></li><li><a href=" https://github.com/tuffy/bitstream-io ">bitstream-io 2.3.0</a></li><li><a href=" https://github.com/fitzgen/bumpalo ">bumpalo 3.15.3</a></li><li><a href=" https://github.com/alexcrichton/bzip2-rs ">bzip2-sys 0.1.11+1.0.8</a></li>diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 66a63460..336dea3e 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -31,6 +31,7 @@ axum-macros.workspace = trueaxum.workspace = truebase64.workspace = truebincode.workspace = true+bitflags.workspace = truebitvec.workspace = truebloom = {path = "../bloom"}bytecount.workspace = truediff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rsindex e6eb7306..a2960914 100644--- a/crates/core/examples/distributed_harmonic.rs+++ b/crates/core/examples/distributed_harmonic.rs@@ -99,9 +99,9 @@ fn build_graphs_if_not_exist(warc_path: &str, graph_path: &str) -> anyhow::Resulif dest_domain.is_some() && source_domain.is_some() && dest_domain != source_domain {if i <= num_records / 2 {- a.insert(source, destination, link.text);+ a.insert(source, destination, link.text, link.rel);} else {- b.insert(source, destination, link.text);+ b.insert(source, destination, link.text, link.rel);}}}diff --git a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rsindex ce3fe0c3..fddba1d7 100644--- a/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/approximated_harmonic_centrality/mapper.rs@@ -20,12 +20,15 @@ use std::{cmp, collections::BTreeMap};use super::{worker::ApproxCentralityWorker, ApproxCentralityJob, Mapper};use crate::{- ampc::dht::upsert, entrypoint::ampc::approximated_harmonic_centrality::DhtTable,- kahan_sum::KahanSum, webgraph,+ ampc::dht::upsert,+ entrypoint::ampc::approximated_harmonic_centrality::DhtTable,+ kahan_sum::KahanSum,+ webgraph::{self, centrality::harmonic::SKIPPED_REL},};use rayon::prelude::*;const BATCH_SIZE: usize = 1024;+const MAX_OUTGOING_EDGES: usize = 128;#[derive(Debug, Clone, bincode::Decode, bincode::Encode)]pub enum ApproxCentralityMapper {@@ -57,8 +60,9 @@ impl Workers {for outgoing in self.worker.graph()- .raw_outgoing_edges(&node, webgraph::EdgeLimit::Unlimited)+ .raw_outgoing_edges(&node, webgraph::EdgeLimit::Limit(MAX_OUTGOING_EDGES)).into_iter()+ .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL)).map(|e| e.to){let d = dist + 1;diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rsindex bd0c0852..4c9765fb 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs@@ -28,7 +28,7 @@ use crate::{DhtConn,},hyperloglog::HyperLogLog,- webgraph,+ webgraph::{self, centrality::harmonic::SKIPPED_REL},};use super::{CentralityJob, CentralityTables, CentralityWorker};@@ -269,6 +269,7 @@ impl CentralityMapper {for edge in worker.graph().edges()+ .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL)).filter(|e| changed_nodes.contains(e.from.as_u64())){batch.push(edge);diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rsindex 90b06feb..51b6c1cc 100644--- a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs@@ -86,6 +86,7 @@ mod tests {executor::Executor,free_socket_addr,webgraph::{centrality::harmonic::HarmonicCentrality, Compression, WebgraphWriter},+ webpage::html::links::RelFlags,};use super::*;@@ -115,12 +116,12 @@ mod tests {let edges = crate::webgraph::tests::test_edges();for (i, (from, to, label)) in edges.into_iter().enumerate() {- combined.insert(from.clone(), to.clone(), label.clone());+ combined.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());if i % 2 == 0 {- a.insert(from, to, label);+ a.insert(from, to, label, RelFlags::default());} else {- b.insert(from, to, label);+ b.insert(from, to, label, RelFlags::default());}}@@ -155,8 +156,6 @@ mod tests {let b = RemoteCentralityWorker::new(2.into(), worker_addr).unwrap();- // assert_eq!(a.num_nodes() + b.num_nodes(), num_nodes as u64);-let (dht_shard, dht_addr) = crate::entrypoint::ampc::dht::tests::setup();let res = coordinator::build(&[(dht_shard, dht_addr)], vec![a, b]).run(diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 54f0467d..da177773 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -146,8 +146,12 @@ impl WebgraphWorker {let mut destination = Node::from(destination);trace!("inserting link {:?}", link);- self.page_graph- .insert(source.clone(), destination.clone(), link.text.clone());+ self.page_graph.insert(+ source.clone(),+ destination.clone(),+ link.text.clone(),+ link.rel,+ );let dest_domain = link.destination.root_domain();let source_domain = link.source.root_domain();@@ -158,7 +162,8 @@ impl WebgraphWorker {source = source.into_host();destination = destination.into_host();- self.host_graph.insert(source, destination, link.text);+ self.host_graph+ .insert(source, destination, link.text, link.rel);}}}diff --git a/crates/core/src/query/optic.rs b/crates/core/src/query/optic.rsindex 1fd60de8..c3f1e647 100644--- a/crates/core/src/query/optic.rs+++ b/crates/core/src/query/optic.rs@@ -286,7 +286,7 @@ mod tests {LocalSearchClient, LocalSearcher, SearchQuery,},webgraph::{Node, WebgraphWriter},- webpage::{Html, Webpage},+ webpage::{html::links::RelFlags, Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -640,28 +640,33 @@ mod tests {Node::from("https://www.e.com").into_host(),Node::from("https://www.a.com").into_host(),String::new(),+ RelFlags::default());writer.insert(Node::from("https://www.a.com").into_host(),Node::from("https://www.e.com").into_host(),String::new(),+ RelFlags::default());writer.insert(Node::from("https://www.c.com").into_host(),Node::from("https://www.c.com").into_host(),String::new(),+ RelFlags::default());writer.insert(Node::from("https://www.b.com").into_host(),Node::from("https://www.e.com").into_host(),String::new(),+ RelFlags::default());writer.insert(Node::from("https://www.e.com").into_host(),Node::from("https://www.b.com").into_host(),String::new(),+ RelFlags::default());let graph = writer.finalize();diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 87b80d27..108057a1 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -150,7 +150,7 @@ mod tests {api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,},webgraph::{EdgeLimit, Node, Webgraph, WebgraphWriter},- webpage::{Html, Webpage},+ webpage::{html::links::RelFlags, Html, Webpage},};use super::*;@@ -174,15 +174,55 @@ mod tests {None,);- wrt.insert(Node::from("a.com"), Node::from("b.com"), String::new());- wrt.insert(Node::from("c.com"), Node::from("d.com"), String::new());- wrt.insert(Node::from("a.com"), Node::from("e.com"), String::new());+ wrt.insert(+ Node::from("a.com"),+ Node::from("b.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("c.com"),+ Node::from("d.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("a.com"),+ Node::from("e.com"),+ String::new(),+ RelFlags::default(),+ );- wrt.insert(Node::from("z.com"), Node::from("a.com"), String::new());- wrt.insert(Node::from("z.com"), Node::from("b.com"), String::new());- wrt.insert(Node::from("z.com"), Node::from("c.com"), String::new());- wrt.insert(Node::from("z.com"), Node::from("d.com"), String::new());- wrt.insert(Node::from("z.com"), Node::from("e.com"), String::new());+ wrt.insert(+ Node::from("z.com"),+ Node::from("a.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("z.com"),+ Node::from("b.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("z.com"),+ Node::from("c.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("z.com"),+ Node::from("d.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("z.com"),+ Node::from("e.com"),+ String::new(),+ RelFlags::default(),+ );let graph = wrt.finalize();@@ -202,10 +242,30 @@ mod tests {None,);- wrt.insert(Node::from("b.com"), Node::from("a.com"), String::new());- wrt.insert(Node::from("c.com"), Node::from("d.com"), String::new());- wrt.insert(Node::from("b.com"), Node::from("e.com"), String::new());- wrt.insert(Node::from("c.com"), Node::from("b.com"), String::new());+ wrt.insert(+ Node::from("b.com"),+ Node::from("a.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("c.com"),+ Node::from("d.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("b.com"),+ Node::from("e.com"),+ String::new(),+ RelFlags::default(),+ );+ wrt.insert(+ Node::from("c.com"),+ Node::from("b.com"),+ String::new(),+ RelFlags::default(),+ );let graph = wrt.finalize();diff --git a/crates/core/src/ranking/optics.rs b/crates/core/src/ranking/optics.rsindex 78573b79..d8e4baaf 100644--- a/crates/core/src/ranking/optics.rs+++ b/crates/core/src/ranking/optics.rs@@ -26,7 +26,7 @@ mod tests {api::ApiSearcher, live::LiveSearcher, LocalSearchClient, LocalSearcher, SearchQuery,},webgraph::{Node, WebgraphWriter},- webpage::{Html, Webpage},+ webpage::{html::links::RelFlags, Html, Webpage},};const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";@@ -46,41 +46,49 @@ mod tests {Node::from("https://www.first.com").into_host(),Node::from("https://www.nan.com").into_host(),String::new(),+ RelFlags::default(),);wrt.insert(Node::from("https://www.nan.com").into_host(),Node::from("https://www.first.com").into_host(),String::new(),+ RelFlags::default(),);wrt.insert(Node::from("https://www.third.com").into_host(),Node::from("https://www.third.com").into_host(),String::new(),+ RelFlags::default(),);wrt.insert(Node::from("https://www.nan.com").into_host(),Node::from("https://www.second.com").into_host(),String::new(),+ RelFlags::default(),);wrt.insert(Node::from("https://www.second.com").into_host(),Node::from("https://www.nan.com").into_host(),String::new(),+ RelFlags::default(),);wrt.insert(Node::from("https://www.second.com").into_host(),Node::from("https://www.third.com").into_host(),String::new(),+ RelFlags::default(),);wrt.insert(Node::from("https://www.extra.com").into_host(),Node::from("https://www.first.com").into_host(),String::new(),+ RelFlags::default(),);wrt.insert(Node::from("https://www.second.com").into_host(),Node::from("https://www.extra.com").into_host(),String::new(),+ RelFlags::default(),);let graph = wrt.finalize();diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rsindex 44a2f060..4dbecb19 100644--- a/crates/core/src/webgraph/centrality/betweenness.rs+++ b/crates/core/src/webgraph/centrality/betweenness.rs@@ -165,7 +165,7 @@ impl Betweenness {mod tests {use maplit::hashmap;- use crate::webgraph::WebgraphWriter;+ use crate::{webgraph::WebgraphWriter, webpage::html::links::RelFlags};use super::*;@@ -182,6 +182,7 @@ mod tests {Node::from(i.to_string()),Node::from((i + 1).to_string()),String::new(),+ RelFlags::default(),);}diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex 637bf89d..b6678d6d 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -28,10 +28,23 @@ use crate::{hyperloglog::HyperLogLog,kahan_sum::KahanSum,webgraph::{EdgeLimit, NodeID, Webgraph},+ webpage::html::links::RelFlags,};const HYPERLOGLOG_COUNTERS: usize = 64;+pub static SKIPPED_REL: once_cell::sync::Lazy<RelFlags> = once_cell::sync::Lazy::new(|| {+ RelFlags::TAG+ | RelFlags::NOFOLLOW+ | RelFlags::IS_IN_FOOTER+ | RelFlags::IS_IN_NAVIGATION+ | RelFlags::PRIVACY_POLICY+ | RelFlags::TERMS_OF_SERVICE+ | RelFlags::SEARCH+ | RelFlags::LINK_TAG+ | RelFlags::SCRIPT_TAG+});+type Counter = BTreeMap<NodeID, HyperLogLog<HYPERLOGLOG_COUNTERS>>;fn initialize(@@ -66,7 +79,11 @@ fn update_changed_counters(let has_changes = AtomicBool::new(false);exact_changed_nodes.iter().for_each(|changed_node| {- for edge in graph.raw_outgoing_edges(changed_node, EdgeLimit::Unlimited) {+ for edge in graph+ .raw_outgoing_edges(changed_node, EdgeLimit::Unlimited)+ .into_iter()+ .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))+ {if let (Some(counter_to), Some(counter_from)) =(counters.new.get_mut(&edge.to), counters.old.get(&edge.from)){@@ -105,29 +122,32 @@ fn update_all_counters(**exact_changed_nodes = BTreeSet::default();}- graph.edges().for_each(|edge| {- if changed_nodes.contains(edge.from.as_u64()) {- if let (Some(counter_to), Some(counter_from)) =- (counters.new.get_mut(&edge.to), counters.old.get(&edge.from))- {- if counter_to- .registers()- .iter()- .zip(counter_from.registers().iter())- .any(|(to, from)| *from > *to)+ graph+ .edges()+ .filter(|e| !e.rel_flags().intersects(*SKIPPED_REL))+ .for_each(|edge| {+ if changed_nodes.contains(edge.from.as_u64()) {+ if let (Some(counter_to), Some(counter_from)) =+ (counters.new.get_mut(&edge.to), counters.old.get(&edge.from)){- counter_to.merge(counter_from);- new_changed_nodes.insert(edge.to.as_u64());-- if let Some(exact_changed_nodes) = &mut exact_changed_nodes {- exact_changed_nodes.insert(edge.to);+ if counter_to+ .registers()+ .iter()+ .zip(counter_from.registers().iter())+ .any(|(to, from)| *from > *to)+ {+ counter_to.merge(counter_from);+ new_changed_nodes.insert(edge.to.as_u64());++ if let Some(exact_changed_nodes) = &mut exact_changed_nodes {+ exact_changed_nodes.insert(edge.to);+ }++ has_changes.store(true, Ordering::Relaxed);}-- has_changes.store(true, Ordering::Relaxed);}}- }- });+ });has_changes.load(Ordering::Relaxed)}@@ -289,7 +309,10 @@ impl HarmonicCentrality {#[cfg(test)]mod tests {use super::*;- use crate::webgraph::{Node, WebgraphWriter};+ use crate::{+ webgraph::{Node, WebgraphWriter},+ webpage::html::links::RelFlags,+ };fn test_edges() -> Vec<(Node, Node, String)> {// ┌────┐@@ -320,7 +343,7 @@ mod tests {);for (from, to, label) in test_edges() {- writer.insert(from, to, label);+ writer.insert(from, to, label, RelFlags::default());}writer.finalize()@@ -339,71 +362,85 @@ mod tests {Node::from("A.com/1").into_host(),Node::from("A.com/2").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/1").into_host(),Node::from("A.com/3").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/1").into_host(),Node::from("A.com/4").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/2").into_host(),Node::from("A.com/1").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/2").into_host(),Node::from("A.com/3").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/2").into_host(),Node::from("A.com/4").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/3").into_host(),Node::from("A.com/1").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/3").into_host(),Node::from("A.com/2").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/3").into_host(),Node::from("A.com/4").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/4").into_host(),Node::from("A.com/1").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/4").into_host(),Node::from("A.com/2").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("A.com/4").into_host(),Node::from("A.com/3").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("C.com").into_host(),Node::from("B.com").into_host(),String::new(),+ RelFlags::default(),);writer.insert(Node::from("D.com").into_host(),Node::from("B.com").into_host(),String::new(),+ RelFlags::default(),);let graph = writer.finalize();@@ -445,22 +482,57 @@ mod tests {);for (from, to, label) in test_edges() {- other.insert(from, to, label);+ other.insert(from, to, label, RelFlags::default());}- other.insert(Node::from("A"), Node::from("B"), "1".to_string());+ other.insert(+ Node::from("A"),+ Node::from("B"),+ "1".to_string(),+ RelFlags::default(),+ );other.commit();- other.insert(Node::from("A"), Node::from("B"), "2".to_string());+ other.insert(+ Node::from("A"),+ Node::from("B"),+ "2".to_string(),+ RelFlags::default(),+ );other.commit();- other.insert(Node::from("A"), Node::from("B"), "3".to_string());+ other.insert(+ Node::from("A"),+ Node::from("B"),+ "3".to_string(),+ RelFlags::default(),+ );other.commit();- other.insert(Node::from("A"), Node::from("B"), "4".to_string());+ other.insert(+ Node::from("A"),+ Node::from("B"),+ "4".to_string(),+ RelFlags::default(),+ );other.commit();- other.insert(Node::from("A"), Node::from("B"), "5".to_string());+ other.insert(+ Node::from("A"),+ Node::from("B"),+ "5".to_string(),+ RelFlags::default(),+ );other.commit();- other.insert(Node::from("A"), Node::from("B"), "6".to_string());+ other.insert(+ Node::from("A"),+ Node::from("B"),+ "6".to_string(),+ RelFlags::default(),+ );other.commit();- other.insert(Node::from("A"), Node::from("B"), "7".to_string());+ other.insert(+ Node::from("A"),+ Node::from("B"),+ "7".to_string(),+ RelFlags::default(),+ );other.commit();let mut graph = other.finalize();@@ -470,4 +542,24 @@ mod tests {assert_eq!(centrality.0, centrality_extra.0);}++ #[test]+ fn test_rel_flag_ignored() {+ let mut graph = WebgraphWriter::new(+ crate::gen_temp_path(),+ crate::executor::Executor::single_thread(),+ crate::webgraph::Compression::default(),+ None,+ );++ for (from, to, label) in test_edges() {+ graph.insert(from, to, label, RelFlags::TAG);+ }++ let graph = graph.finalize();++ let centrality = HarmonicCentrality::calculate(&graph);++ assert!(centrality.0.values().all(|&v| v == 0.0));+ }}diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex b477585f..f1ef91af 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -16,6 +16,8 @@use utoipa::ToSchema;+use crate::webpage::html::links::RelFlags;+use super::{merge::NodeDatum, FullNodeID, Node, NodeID};pub const MAX_LABEL_LENGTH: usize = 1024;@@ -55,9 +57,19 @@ where{pub from: NodeID,pub to: NodeID,+ pub rel: RelFlags,pub label: L,}+impl<L> Edge<L>+where+ L: EdgeLabel,+{+ pub fn rel_flags(&self) -> RelFlags {+ self.rel+ }+}+#[derive(Debug, Clone, PartialEq, Eq, Hash, bincode::Encode, bincode::Decode)]pub struct InsertableEdge<L>where@@ -65,6 +77,7 @@ where{pub from: FullNodeID,pub to: FullNodeID,+ pub rel: RelFlags,pub label: L,}@@ -77,6 +90,7 @@ whereEdge {from: edge.from.id,to: edge.to.id,+ rel: edge.rel,label: edge.label,}}@@ -106,8 +120,9 @@ pub struct SegmentEdge<L>whereL: EdgeLabel,{- pub from: NodeDatum<()>,- pub to: NodeDatum<()>,+ pub from: NodeDatum,+ pub to: NodeDatum,+ pub rel: RelFlags,pub label: L,}@@ -119,6 +134,7 @@ whereEdge {from: edge.from.node(),to: edge.to.node(),+ rel: edge.rel,label: edge.label,}}@@ -133,6 +149,7 @@ whereSegmentEdge {from: NodeDatum::new(edge.from, 0),to: NodeDatum::new(edge.to, 0),+ rel: edge.rel,label: edge.label,}}@@ -147,7 +164,49 @@ whereSegmentEdge {from: NodeDatum::new(edge.from.id, 0),to: NodeDatum::new(edge.to.id, 0),+ rel: edge.rel,label: edge.label,}}}++pub struct StoredEdge<L = ()> {+ pub other: NodeDatum,+ pub rel: RelFlags,+ pub label: L,+}++impl StoredEdge<()> {+ pub fn new(other: NodeDatum, rel: RelFlags) -> Self {+ StoredEdge {+ other,+ rel,+ label: (),+ }+ }+}++impl<L> StoredEdge<L> {+ pub fn with_label<L2>(self, label: L2) -> StoredEdge<L2> {+ StoredEdge {+ other: self.other,+ rel: self.rel,+ label,+ }+ }++ #[inline]+ pub fn label(&self) -> &L {+ &self.label+ }++ #[inline]+ pub fn other(&self) -> NodeID {+ self.other.node()+ }++ #[inline]+ pub fn rel(&self) -> RelFlags {+ self.rel+ }+}diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rsindex 8d06b3e5..ef1cc91e 100644--- a/crates/core/src/webgraph/merge.rs+++ b/crates/core/src/webgraph/merge.rs@@ -16,17 +16,17 @@use std::{cmp::Reverse, collections::BinaryHeap};-use super::{store::NodeRange, NodeID};+use super::{store::EdgeRange, NodeID, StoredEdge};pub struct MergeNode<O = ()> {node: NodeID,- range: NodeRange,+ range: EdgeRange,labels: std::ops::Range<u64>,ord: O,}impl MergeNode {- pub fn new(node: NodeID, range: NodeRange, labels: std::ops::Range<u64>) -> Self {+ pub fn new(node: NodeID, range: EdgeRange, labels: std::ops::Range<u64>) -> Self {Self {node,range,@@ -50,7 +50,7 @@ impl<O> MergeNode<O> {self.node}- pub fn range(&self) -> &NodeRange {+ pub fn range(&self) -> &EdgeRange {&self.range}@@ -152,12 +152,12 @@ impl<'a> MergeIter<'a> {}/// Merge multiple iterators of NodeDatum into a single iterator based on the sort key-pub struct NodeDatumMerger<'a, L = String> {- iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = NodeDatum<L>> + 'a>>>,+pub struct EdgeMerger<'a, L = String> {+ iters: MinHeap<file_store::Peekable<Box<dyn Iterator<Item = StoredEdge<L>> + 'a>>>,}-impl<'a, L> NodeDatumMerger<'a, L> {- pub fn new(iters: Vec<impl Iterator<Item = NodeDatum<L>> + 'a>) -> Self {+impl<'a, L> EdgeMerger<'a, L> {+ pub fn new(iters: Vec<impl Iterator<Item = StoredEdge<L>> + 'a>) -> Self {let iters = iters.into_iter().map(|iter| {@@ -170,15 +170,15 @@ impl<'a, L> NodeDatumMerger<'a, L> {}}-impl<'a, L> Iterator for NodeDatumMerger<'a, L> {- type Item = NodeDatum<L>;+impl<'a, L> Iterator for EdgeMerger<'a, L> {+ type Item = StoredEdge<L>;fn next(&mut self) -> Option<Self::Item> {let res = self.iters.peek_mut().and_then(|mut item| item.0.next());- if let Some(datum) = &res {+ if let Some(edge) = &res {while let Some(mut peek) = self.iters.peek_mut() {- if peek.0.peek().map(|x| x.sort_key) == Some(datum.sort_key) {+ if peek.0.peek().map(|x| x.other.sort_key) == Some(edge.other.sort_key) {peek.0.next().unwrap();} else {break;@@ -191,19 +191,18 @@ impl<'a, L> Iterator for NodeDatumMerger<'a, L> {}#[derive(Debug, Clone)]-pub struct NodeDatum<L = ()> {+pub struct NodeDatum {id: NodeID,sort_key: u64,- label: L,}-impl<L> PartialOrd for NodeDatum<L> {+impl PartialOrd for NodeDatum {fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {Some(self.cmp(other))}}-impl<L> Ord for NodeDatum<L> {+impl Ord for NodeDatum {fn cmp(&self, other: &Self) -> std::cmp::Ordering {self.sort_key.cmp(&other.sort_key)@@ -211,25 +210,21 @@ impl<L> Ord for NodeDatum<L> {}}-impl<L> PartialEq for NodeDatum<L> {+impl PartialEq for NodeDatum {fn eq(&self, other: &Self) -> bool {self.id == other.id && self.sort_key == other.sort_key}}-impl<L> Eq for NodeDatum<L> {}+impl Eq for NodeDatum {}impl NodeDatum {pub fn new(node: NodeID, sort_key: u64) -> Self {- Self {- id: node,- sort_key,- label: (),- }+ Self { id: node, sort_key }}}-impl<L> NodeDatum<L> {+impl NodeDatum {#[inline]pub fn node(&self) -> NodeID {self.id@@ -239,37 +234,46 @@ impl<L> NodeDatum<L> {pub fn sort_key(&self) -> u64 {self.sort_key}+}- pub fn with_label<L2>(self, label: L2) -> NodeDatum<L2> {- NodeDatum {- id: self.id,- sort_key: self.sort_key,- label,- }+impl<L> Ord for StoredEdge<L> {+ fn cmp(&self, other: &Self) -> std::cmp::Ordering {+ self.other.cmp(&other.other)}+}- #[inline]- pub fn label(&self) -> &L {- &self.label+impl<L> PartialOrd for StoredEdge<L> {+ fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {+ Some(self.cmp(other))+ }+}++impl<L> PartialEq for StoredEdge<L> {+ fn eq(&self, other: &Self) -> bool {+ self.other == other.other}}+impl<L> Eq for StoredEdge<L> {}+#[cfg(test)]mod tests {+ use crate::webpage::html::links::RelFlags;+use super::*;#[test]fn test_merge_nodes() {let a = vec![- MergeNode::new(1u64.into(), NodeRange::new(0..10, 1), 0..10),- MergeNode::new(4u64.into(), NodeRange::new(0..10, 2), 0..10),- MergeNode::new(5u64.into(), NodeRange::new(0..10, 3), 0..10),+ MergeNode::new(1u64.into(), EdgeRange::new(0..10, 1), 0..10),+ MergeNode::new(4u64.into(), EdgeRange::new(0..10, 2), 0..10),+ MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),];let b = vec![- MergeNode::new(2u64.into(), NodeRange::new(0..10, 4), 0..10),- MergeNode::new(3u64.into(), NodeRange::new(0..10, 5), 0..10),- MergeNode::new(5u64.into(), NodeRange::new(0..10, 3), 0..10),+ MergeNode::new(2u64.into(), EdgeRange::new(0..10, 4), 0..10),+ MergeNode::new(3u64.into(), EdgeRange::new(0..10, 5), 0..10),+ MergeNode::new(5u64.into(), EdgeRange::new(0..10, 3), 0..10),];let mut merger = MergeIter::new(vec![a.into_iter(), b.into_iter()]);@@ -302,24 +306,24 @@ mod tests {#[test]fn test_datum_merge() {let a = vec![- NodeDatum::new(1u64.into(), 1),- NodeDatum::new(2u64.into(), 4),- NodeDatum::new(3u64.into(), 5),+ StoredEdge::new(NodeDatum::new(1u64.into(), 1), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(2u64.into(), 4), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),];let b = vec![- NodeDatum::new(4u64.into(), 2),- NodeDatum::new(5u64.into(), 3),- NodeDatum::new(3u64.into(), 5),+ StoredEdge::new(NodeDatum::new(4u64.into(), 2), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(5u64.into(), 3), RelFlags::default()),+ StoredEdge::new(NodeDatum::new(3u64.into(), 5), RelFlags::default()),];- let mut merger = NodeDatumMerger::new(vec![a.into_iter(), b.into_iter()]);+ let mut merger = EdgeMerger::new(vec![a.into_iter(), b.into_iter()]);- assert_eq!(merger.next().unwrap().sort_key(), 1);- assert_eq!(merger.next().unwrap().sort_key(), 2);- assert_eq!(merger.next().unwrap().sort_key(), 3);- assert_eq!(merger.next().unwrap().sort_key(), 4);- assert_eq!(merger.next().unwrap().sort_key(), 5);+ assert_eq!(merger.next().unwrap().other.sort_key(), 1);+ assert_eq!(merger.next().unwrap().other.sort_key(), 2);+ assert_eq!(merger.next().unwrap().other.sort_key(), 3);+ assert_eq!(merger.next().unwrap().other.sort_key(), 4);+ assert_eq!(merger.next().unwrap().other.sort_key(), 5);assert!(merger.next().is_none());}}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 67af0386..912685d8 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -257,6 +257,7 @@ impl Webgraph {from: e.from.node(),to: e.to.node(),label: e.label,+ rel: e.rel,}).collect()}@@ -283,6 +284,7 @@ impl Webgraph {from: e.from.node(),to: e.to.node(),label: e.label,+ rel: e.rel,}).collect()}@@ -310,6 +312,7 @@ impl Webgraph {from: e.from.node(),to: e.to.node(),label: e.label,+ rel: e.rel,}).collect()}@@ -351,6 +354,7 @@ impl Webgraph {from: e.from.node(),to: e.to.node(),label: e.label,+ rel: e.rel,}).collect()}@@ -423,6 +427,8 @@ impl Webgraph {#[cfg(test)]pub mod tests {+ use crate::webpage::html::links::RelFlags;+use super::*;pub fn test_edges() -> Vec<(Node, Node, String)> {@@ -455,7 +461,7 @@ pub mod tests {);for (from, to, label) in test_edges() {- graph.insert(from, to, label);+ graph.insert(from, to, label, RelFlags::default());}graph.commit();@@ -516,7 +522,7 @@ pub mod tests {Compression::default(),None,);- wrt.insert(from.clone(), to.clone(), label.clone());+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());graphs.push(wrt.finalize());}@@ -549,7 +555,7 @@ pub mod tests {Compression::default(),None,);- wrt.insert(from.clone(), to.clone(), label.clone());+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());graphs.push(wrt.finalize());}@@ -604,6 +610,7 @@ pub mod tests {Node::from("A"),Node::from("B"),"a".repeat(MAX_LABEL_LENGTH + 1),+ RelFlags::default(),);let graph = writer.finalize();@@ -644,7 +651,7 @@ pub mod tests {Compression::default(),None,);- wrt.insert(from.clone(), to.clone(), label.clone());+ wrt.insert(from.clone(), to.clone(), label.clone(), RelFlags::default());graphs.push(wrt.finalize());}@@ -693,4 +700,28 @@ pub mod tests {let n = Node::from("http://www.example.com/abc#123");assert_eq!(n.as_str(), "example.com/abc");}++ #[test]+ fn test_rel_flags() {+ let mut writer = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ None,+ );++ writer.insert(+ Node::from("A"),+ Node::from("B"),+ String::new(),+ RelFlags::IS_IN_FOOTER | RelFlags::TAG,+ );++ let graph = writer.finalize();++ assert_eq!(+ graph.raw_outgoing_edges(&Node::from("A").id(), EdgeLimit::Unlimited)[0].rel,+ RelFlags::IS_IN_FOOTER | RelFlags::TAG,+ );+ }}diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex d4bd308a..a3497c76 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -198,7 +198,10 @@ impl Segment {#[cfg(test)]mod test {- use crate::webgraph::{Edge, FullNodeID};+ use crate::{+ webgraph::{Edge, FullNodeID},+ webpage::html::links::RelFlags,+ };use super::*;@@ -238,21 +241,25 @@ mod test {from: a.clone(),to: b.clone(),label: String::new(),+ rel: RelFlags::default(),});edges.push(InsertableEdge {from: b.clone(),to: c.clone(),label: String::new(),+ rel: RelFlags::default(),});edges.push(InsertableEdge {from: c.clone(),to: a.clone(),label: String::new(),+ rel: RelFlags::default(),});edges.push(InsertableEdge {from: a.clone(),to: c.clone(),label: String::new(),+ rel: RelFlags::default(),});for edge in &edges {@@ -270,13 +277,15 @@ mod test {Edge {from: a.id,to: b.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),Edge {from: a.id,to: c.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),]@@ -289,7 +298,8 @@ mod test {vec![Edge {from: b.id,to: c.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),]);@@ -301,7 +311,8 @@ mod test {vec![Edge {from: c.id,to: a.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),]);@@ -312,7 +323,8 @@ mod test {vec![Edge {from: c.id,to: a.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),]);@@ -323,7 +335,8 @@ mod test {vec![Edge {from: a.id,to: b.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),]);@@ -336,13 +349,15 @@ mod test {Edge {from: a.id,to: c.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),Edge {from: b.id,to: c.id,- label: ()+ label: (),+ rel: RelFlags::default(),}.into(),]diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex 948f1e80..8f3148da 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -17,7 +17,8 @@use std::{fs::File, ops::Range, path::Path};use crate::{- webgraph::merge::{MergeIter, NodeDatumMerger},+ webgraph::merge::{EdgeMerger, MergeIter},+ webpage::html::links::RelFlags,Result,};use anyhow::bail;@@ -33,7 +34,7 @@ use itertools::Itertools;use super::{merge::{MergeNode, MergeSegmentOrd, NodeDatum},- Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge,+ Compression, EdgeLimit, FullNodeID, NodeID, SegmentEdge, StoredEdge,};#[derive(@@ -104,12 +105,12 @@ impl HostDb {}#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]-pub struct NodeRange {+pub struct EdgeRange {range: std::ops::Range<u64>,sort_key: u64,}-impl NodeRange {+impl EdgeRange {pub fn new(range: std::ops::Range<u64>, sort_key: u64) -> Self {Self { range, sort_key }}@@ -121,37 +122,37 @@ fn usize_range(range: std::ops::Range<u64>) -> std::ops::Range<usize> {}pub struct RangesDb {- nodes: speedy_kv::Db<NodeID, NodeRange>,+ edges: speedy_kv::Db<NodeID, EdgeRange>,labels: speedy_kv::Db<NodeID, std::ops::Range<u64>>,}impl RangesDb {pub fn open<P: AsRef<Path>>(path: P) -> Self {- let nodes = speedy_kv::Db::open_or_create(path.as_ref().join("nodes")).unwrap();+ let edges = speedy_kv::Db::open_or_create(path.as_ref().join("edges")).unwrap();let labels = speedy_kv::Db::open_or_create(path.as_ref().join("labels")).unwrap();- Self { nodes, labels }+ Self { edges, labels }}pub fn optimize_read(&mut self) {- self.nodes.merge_all_segments().unwrap();+ self.edges.merge_all_segments().unwrap();self.labels.merge_all_segments().unwrap();}fn uncommitted_node_inserts(&self) -> usize {- self.nodes.uncommitted_inserts()+ self.edges.uncommitted_inserts()}pub fn commit(&mut self) {- self.nodes.commit().unwrap();+ self.edges.commit().unwrap();self.labels.commit().unwrap();}pub fn nodes_get_raw<'a>(&'a self,key: &'a [u8],- ) -> Option<speedy_kv::SerializedRef<'a, NodeRange>> {- self.nodes.get_raw(key)+ ) -> Option<speedy_kv::SerializedRef<'a, EdgeRange>> {+ self.edges.get_raw(key)}pub fn labels_get_raw<'a>(@@ -162,7 +163,7 @@ impl RangesDb {}pub fn insert_raw_node(&mut self, node: Vec<u8>, range: Vec<u8>) {- self.nodes.insert_raw(node, range);+ self.edges.insert_raw(node, range);}pub fn insert_raw_label(&mut self, node: Vec<u8>, range: Vec<u8>) {@@ -170,12 +171,12 @@ impl RangesDb {}fn merge_nodes(&self) -> impl Iterator<Item = MergeNode> + '_ {- self.nodes.iter_raw().zip_eq(self.labels.iter_raw()).map(+ self.edges.iter_raw().zip_eq(self.labels.iter_raw()).map(move |((key_node, val), (key_label, labels))| {debug_assert_eq!(key_node, key_label);let node = NodeID::deserialize(key_node.as_bytes());- let range = NodeRange::deserialize(val.as_bytes());+ let range = EdgeRange::deserialize(val.as_bytes());let labels = {let range = Range::deserialize(labels.as_bytes());@@ -201,6 +202,18 @@ impl ConstSerializable for NodeID {}}+impl ConstSerializable for RelFlags {+ const BYTES: usize = std::mem::size_of::<RelFlags>();++ fn serialize(&self, buf: &mut [u8]) {+ self.as_u32().serialize(buf);+ }++ fn deserialize(buf: &[u8]) -> Self {+ u32::deserialize(buf).into()+ }+}+impl ConstSerializable for NodeDatum {const BYTES: usize = std::mem::size_of::<NodeDatum>();@@ -217,8 +230,8 @@ impl ConstSerializable for NodeDatum {}}-impl ConstSerializable for NodeRange {- const BYTES: usize = std::mem::size_of::<NodeRange>();+impl ConstSerializable for EdgeRange {+ const BYTES: usize = std::mem::size_of::<EdgeRange>();fn serialize(&self, buf: &mut [u8]) {const RANGE_BYTES: usize = std::mem::size_of::<u64>() * 2;@@ -235,6 +248,22 @@ impl ConstSerializable for NodeRange {}}+impl ConstSerializable for StoredEdge {+ const BYTES: usize = std::mem::size_of::<StoredEdge>();++ fn serialize(&self, buf: &mut [u8]) {+ self.other.serialize(&mut buf[..NodeDatum::BYTES]);+ self.rel.serialize(&mut buf[NodeDatum::BYTES..]);+ }++ fn deserialize(buf: &[u8]) -> Self {+ let other = NodeDatum::deserialize(&buf[..NodeDatum::BYTES]);+ let rel = RelFlags::deserialize(&buf[NodeDatum::BYTES..]);++ Self::new(other, rel)+ }+}+pub const NUM_LABELS_PER_BLOCK: usize = 128;#[derive(bincode::Encode, bincode::Decode)]@@ -278,7 +307,7 @@ pub struct EdgeStore {hosts: HostDb,edge_labels: IterableStoreReader<CompressedLabelBlock>,- edge_nodes: ConstIterableStoreReader<NodeDatum>,+ edges: ConstIterableStoreReader<StoredEdge>,}impl EdgeStore {@@ -287,13 +316,13 @@ impl EdgeStore {let edge_labels = IterableStoreReader::open(path.as_ref().join("labels")).unwrap();- let edge_nodes = ConstIterableStoreReader::open(path.as_ref().join("nodes")).unwrap();+ let edges = ConstIterableStoreReader::open(path.as_ref().join("edges")).unwrap();Self {ranges,hosts: HostDb::open(path.as_ref().join("hosts")),edge_labels,- edge_nodes,+ edges,reversed,}}@@ -306,14 +335,12 @@ impl EdgeStore {fn merge_postings_for_node<'a>(buf: &[MergeNode<MergeSegmentOrd>],stores: &'a [EdgeStore],- ) -> NodeDatumMerger<'a> {+ ) -> EdgeMerger<'a> {let mut edges = Vec::new();for node in buf {let store = &stores[node.ord().as_usize()];- let edge_nodes = store- .edge_nodes- .slice(usize_range(node.range().range.clone()));+ let edge_nodes = store.edges.slice(usize_range(node.range().range.clone()));let edge_labels = store.edge_labels.slice(usize_range(node.labels()))@@ -323,11 +350,11 @@ impl EdgeStore {edges.push(edge_nodes.zip_eq(edge_labels)- .map(|(node, label)| node.with_label(label)),+ .map(|(edge, label)| edge.with_label(label)),);}- NodeDatumMerger::new(edges)+ EdgeMerger::new(edges)}fn merge_postings<P: AsRef<Path>>(@@ -338,23 +365,24 @@ impl EdgeStore {let reversed = stores[0].reversed;let mut ranges = RangesDb::open(folder.as_ref().join("ranges"));- let edge_labels_file = File::options()+ let labels_file = File::options().read(true).create(true).truncate(false).write(true).open(folder.as_ref().join("labels")).unwrap();- let mut edge_labels = IterableStoreWriter::new(edge_labels_file);+ let mut labels_store = IterableStoreWriter::new(labels_file);- let edge_nodes_file = File::options()+ let edges_file = File::options().read(true).create(true).truncate(false).write(true)- .open(folder.as_ref().join("nodes"))+ .open(folder.as_ref().join("edges")).unwrap();- let mut edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);+ let mut edges_store: ConstIterableStoreWriter<StoredEdge, File> =+ ConstIterableStoreWriter::new(edges_file);let mut merge_iter = MergeIter::new(stores@@ -381,13 +409,13 @@ impl EdgeStore {let mut last_node_offset = None;for chunk in edges.chunks(NUM_LABELS_PER_BLOCK).into_iter() {- let (labels, nodes): (Vec<_>, Vec<_>) = chunk+ let (labels, edges): (Vec<_>, Vec<_>) = chunk.map(|edge| (edge.label().clone(), edge.with_label(()))).unzip();let label_block = LabelBlock::new(labels).compress(label_compression);- let label_offset = edge_labels.write(&label_block).unwrap();+ let label_offset = labels_store.write(&label_block).unwrap();if first_label_offset.is_none() {first_label_offset = Some(label_offset);@@ -395,8 +423,8 @@ impl EdgeStore {last_label_offset = Some(label_offset);- for node in nodes {- let offset = edge_nodes.write(&node).unwrap();+ for edge in edges {+ let offset = edges_store.write(&edge).unwrap();if first_node_offset.is_none() {first_node_offset = Some(offset);@@ -411,7 +439,7 @@ impl EdgeStore {end: last_label_offset.unwrap().start + last_label_offset.unwrap().num_bytes,};- let node_range: NodeRange = NodeRange::new(+ let node_range: EdgeRange = EdgeRange::new(Range {start: first_node_offset.unwrap().start,end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,@@ -429,15 +457,15 @@ impl EdgeStore {if ranges.uncommitted_node_inserts() > 100_000_000 {ranges.commit();- edge_nodes.flush().unwrap();- edge_labels.flush().unwrap();+ edges_store.flush().unwrap();+ labels_store.flush().unwrap();}}if ranges.uncommitted_node_inserts() > 0 {ranges.commit();- edge_nodes.flush().unwrap();- edge_labels.flush().unwrap();+ edges_store.flush().unwrap();+ labels_store.flush().unwrap();}Ok(Self::open(folder, reversed))@@ -476,37 +504,39 @@ impl EdgeStore {let node_bytes = node.as_u64().to_le_bytes();match (- self.ranges.nodes.get_raw(&node_bytes),+ self.ranges.edges.get_raw(&node_bytes),self.ranges.labels.get_raw(&node_bytes),) {(Some(node_range_bytes), Some(edge_range_bytes)) => {- let node_range = NodeRange::deserialize(node_range_bytes.as_bytes());+ let node_range = EdgeRange::deserialize(node_range_bytes.as_bytes());let edge_range: Range<u64> = Range::deserialize(edge_range_bytes.as_bytes());- let edge_labels = self+ let labels = self.edge_labels.slice(usize_range(edge_range)).map(|r| r.decompress()).flat_map(|block| block.labels.into_iter());- let edge_labels = limit.apply(edge_labels);+ let labels = limit.apply(labels);- let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));- let edge_nodes = limit.apply(edge_nodes);+ let edges = self.edges.slice(usize_range(node_range.range));+ let edges = limit.apply(edges);- edge_labels- .zip_eq(edge_nodes)- .map(|(label, other)| {+ labels+ .zip_eq(edges)+ .map(|(label, edge)| {if self.reversed {SegmentEdge {- from: other,+ from: edge.other,to: NodeDatum::new(*node, node_range.sort_key),+ rel: edge.rel,label,}} else {SegmentEdge {from: NodeDatum::new(*node, node_range.sort_key),- to: other,+ to: edge.other,+ rel: edge.rel,label,}}@@ -520,27 +550,29 @@ impl EdgeStore {pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {let node_bytes = node.as_u64().to_le_bytes();- match self.ranges.nodes.get_raw(&node_bytes) {+ match self.ranges.edges.get_raw(&node_bytes) {Some(node_range_bytes) => {- let node_range = NodeRange::deserialize(node_range_bytes.as_bytes());+ let edge_range = EdgeRange::deserialize(node_range_bytes.as_bytes());- let edge_nodes = self.edge_nodes.slice(usize_range(node_range.range));+ let edges = self.edges.slice(usize_range(edge_range.range));- let edge_nodes = limit.apply(edge_nodes);+ let edges = limit.apply(edges);- edge_nodes+ edges.into_iter()- .map(|other| {+ .map(|edge| {if self.reversed {SegmentEdge {- from: other,- to: NodeDatum::new(*node, node_range.sort_key),+ from: edge.other,+ to: NodeDatum::new(*node, edge_range.sort_key),+ rel: edge.rel,label: (),}} else {SegmentEdge {- from: NodeDatum::new(*node, node_range.sort_key),- to: other,+ from: NodeDatum::new(*node, edge_range.sort_key),+ to: edge.other,+ rel: edge.rel,label: (),}}@@ -556,28 +588,30 @@ impl EdgeStore {}pub fn iter_without_label(&self) -> impl Iterator<Item = SegmentEdge<()>> + '_ + Send + Sync {- self.ranges.nodes.iter_raw().flat_map(move |(key, val)| {+ self.ranges.edges.iter_raw().flat_map(move |(key, val)| {let node = u64::from_le_bytes((key.as_bytes()).try_into().unwrap());let node = NodeID::from(node);- let node_range = NodeRange::deserialize(val.as_bytes());+ let edge_range = EdgeRange::deserialize(val.as_bytes());- let edge_nodes = self- .edge_nodes- .slice(usize_range(node_range.range))+ let edges = self+ .edges+ .slice(usize_range(edge_range.range)).collect::<Vec<_>>();- edge_nodes.into_iter().map(move |other| {+ edges.into_iter().map(move |edge| {if self.reversed {SegmentEdge {- from: other,- to: NodeDatum::new(node, node_range.sort_key),+ from: edge.other,+ to: NodeDatum::new(node, edge_range.sort_key),+ rel: edge.rel,label: (),}} else {SegmentEdge {- from: NodeDatum::new(node, node_range.sort_key),- to: other,+ from: NodeDatum::new(node, edge_range.sort_key),+ to: edge.other,+ rel: edge.rel,label: (),}}@@ -613,6 +647,7 @@ mod tests {host: NodeID::from(0_u64),},label: "test".to_string(),+ rel: RelFlags::default(),};kv.put(e.clone());@@ -652,6 +687,7 @@ mod tests {host: NodeID::from(0_u64),},label: "test".to_string(),+ rel: RelFlags::default(),};kv.put(e.clone());@@ -686,6 +722,7 @@ mod tests {host: NodeID::from(0_u64),},label: "test".to_string(),+ rel: RelFlags::default(),};kv.put(e.clone());@@ -725,6 +762,7 @@ mod tests {host: NodeID::from(0_u64),},label: "1".to_string(),+ rel: RelFlags::default(),};let e2 = InsertableEdge {@@ -737,6 +775,7 @@ mod tests {host: NodeID::from(0_u64),},label: "2".to_string(),+ rel: RelFlags::default(),};let e3 = InsertableEdge {@@ -749,6 +788,7 @@ mod tests {host: NodeID::from(0_u64),},label: "3".to_string(),+ rel: RelFlags::default(),};kv.put(e1.clone());diff --git a/crates/core/src/webgraph/store_writer.rs b/crates/core/src/webgraph/store_writer.rsindex c84e7210..116381a5 100644--- a/crates/core/src/webgraph/store_writer.rs+++ b/crates/core/src/webgraph/store_writer.rs@@ -27,7 +27,7 @@ use std::{use itertools::Itertools;use crate::{- webgraph::store::{NodeRange, NUM_LABELS_PER_BLOCK},+ webgraph::store::{EdgeRange, NUM_LABELS_PER_BLOCK},Result,};use file_store::{@@ -41,7 +41,7 @@ use file_store::{use super::{merge::NodeDatum,store::{CompressedLabelBlock, EdgeStore, HostDb, LabelBlock, RangesDb},- Compression, EdgeLabel, InsertableEdge, NodeID,+ Compression, EdgeLabel, InsertableEdge, NodeID, StoredEdge,};#[derive(bincode::Encode, bincode::Decode)]@@ -223,7 +223,7 @@ struct FinalEdgeStoreWriter {hosts: HostDb,edge_labels: IterableStoreWriter<CompressedLabelBlock, File>,- edge_nodes: ConstIterableStoreWriter<NodeDatum, File>,+ edge_nodes: ConstIterableStoreWriter<StoredEdge, File>,host_centrality_rank_store: Option<Arc<speedy_kv::Db<NodeID, u64>>>,@@ -256,7 +256,7 @@ impl FinalEdgeStoreWriter {.create(true).truncate(false).write(true)- .open(path.as_ref().join("nodes"))+ .open(path.as_ref().join("edges")).unwrap();let edge_nodes = ConstIterableStoreWriter::new(edge_nodes_file);@@ -304,7 +304,7 @@ impl FinalEdgeStoreWriter {debug_assert!(self.ranges.labels_get_raw(&node_bytes).is_none());let mut edge_labels = Vec::new();- let mut edge_nodes: Vec<NodeDatum> = Vec::new();+ let mut edge_nodes: Vec<StoredEdge> = Vec::new();for edge in edges {edge_labels.push(edge.label.clone());@@ -322,7 +322,8 @@ impl FinalEdgeStoreWriter {.unwrap_or(0);let datum = NodeDatum::new(node, sort_key);- edge_nodes.push(datum);+ let rel = edge.rel;+ edge_nodes.push(StoredEdge::new(datum, rel));}let edge_labels: Vec<_> = edge_labels@@ -368,7 +369,7 @@ impl FinalEdgeStoreWriter {.map(|store| store.get(&node.host).unwrap().unwrap_or(u64::MAX)).unwrap_or(0);- let node_range: NodeRange = NodeRange::new(+ let node_range: EdgeRange = EdgeRange::new(Range {start: first_node_offset.unwrap().start,end: last_node_offset.unwrap().start + last_node_offset.unwrap().num_bytes,diff --git a/crates/core/src/webgraph/writer.rs b/crates/core/src/webgraph/writer.rsindex 943ca6c5..09028264 100644--- a/crates/core/src/webgraph/writer.rs+++ b/crates/core/src/webgraph/writer.rs@@ -16,7 +16,7 @@use std::{fs, path::Path, sync::Arc};-use crate::executor::Executor;+use crate::{executor::Executor, webpage::html::links::RelFlags};use super::{id_node_db::Id2NodeDb, segment::SegmentWriter, Compression, FullNodeID, InsertableEdge, Meta,@@ -85,7 +85,7 @@ impl WebgraphWriter {id}- pub fn insert(&mut self, from: Node, to: Node, label: String) {+ pub fn insert(&mut self, from: Node, to: Node, label: String, rel: RelFlags) {if from == to {return;}@@ -99,6 +99,7 @@ impl WebgraphWriter {from: from_id,to: to_id,label: label.chars().take(MAX_LABEL_LENGTH).collect(),+ rel,};self.segment.insert(edge);diff --git a/crates/core/src/webpage/html/links.rs b/crates/core/src/webpage/html/links.rsindex 44ed75e1..d4b31d7c 100644--- a/crates/core/src/webpage/html/links.rs+++ b/crates/core/src/webpage/html/links.rs@@ -14,7 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use kuchiki::iter::NodeEdge;+use bitflags::bitflags;+use kuchiki::{iter::NodeEdge, Attributes};use url::Url;use crate::webpage::{url_ext::UrlExt, Link};@@ -36,6 +37,133 @@ pub struct ImageLink {pub description: Option<String>,}+/// Flags for the `rel` attribute of the `a` element.+/// See https://html.spec.whatwg.org/multipage/links.html#linkTypes+#[derive(Default, Debug, Clone, Copy, bincode::Encode, bincode::Decode, PartialEq, Eq, Hash)]+pub struct RelFlags(u32);++impl RelFlags {+ pub fn as_u32(&self) -> u32 {+ self.0+ }++ fn from_html(url: &Url, attributes: &Attributes, location: &Location) -> Self {+ let mut res = RelFlags::empty();++ if let Some(rel) = attributes.get("rel") {+ for rel in rel.split_whitespace() {+ match rel {+ "alternate" => res |= RelFlags::ALTERNATE,+ "author" => res |= RelFlags::AUTHOR,+ "canonical" => res |= RelFlags::CANONICAL,+ "help" => res |= RelFlags::HELP,+ "icon" => res |= RelFlags::ICON,+ "license" => res |= RelFlags::LICENSE,+ "me" => res |= RelFlags::ME,+ "next" => res |= RelFlags::NEXT,+ "nofollow" => res |= RelFlags::NOFOLLOW,+ "prev" => res |= RelFlags::PREV,+ "privacy-policy" => res |= RelFlags::PRIVACY_POLICY,+ "search" => res |= RelFlags::SEARCH,+ "stylesheet" => res |= RelFlags::STYLESHEET,+ "tag" => res |= RelFlags::TAG,+ "terms-of-service" => res |= RelFlags::TERMS_OF_SERVICE,+ "sponsored" => res |= RelFlags::SPONSORED,+ _ => {}+ }+ }+ }++ if let Some(mut path) = url.path_segments() {+ if path.any(|segment| {+ matches!(+ segment,+ "tags" | "tag" | "tagged" | "topic" | "topics" | "category" | "categories"+ )+ }) {+ res |= RelFlags::TAG;+ }+ }++ res |= location.as_rel();++ res+ }+}++impl From<u32> for RelFlags {+ fn from(value: u32) -> Self {+ Self(value)+ }+}++bitflags! {+ impl RelFlags: u32 {+ const ALTERNATE = 1 << 0;+ const AUTHOR = 1 << 1;+ const CANONICAL = 1 << 2;+ const HELP = 1 << 3;+ const ICON = 1 << 4;+ const LICENSE = 1 << 5;+ const ME = 1 << 6;+ const NEXT = 1 << 7;+ const NOFOLLOW = 1 << 8;+ const PREV = 1 << 9;+ const PRIVACY_POLICY = 1 << 10;+ const SEARCH = 1 << 11;+ const STYLESHEET = 1 << 12;+ const TAG = 1 << 13;+ const TERMS_OF_SERVICE = 1 << 14;+ // Custom flags+ const SPONSORED = 1 << 15;+ const IS_IN_FOOTER = 1 << 16;+ const IS_IN_NAVIGATION = 1 << 17;+ const LINK_TAG = 1 << 18;+ const SCRIPT_TAG = 1 << 19;+ const META_TAG = 1 << 20;+ }+}++struct Location(u8);++impl Location {+ pub fn as_rel(&self) -> RelFlags {+ let mut res = RelFlags::empty();++ if self.contains(Location::FOOTER) {+ res |= RelFlags::IS_IN_FOOTER;+ }++ if self.contains(Location::NAVIGATION) {+ res |= RelFlags::IS_IN_NAVIGATION;+ }++ if self.contains(Location::LINK) {+ res |= RelFlags::LINK_TAG;+ }++ if self.contains(Location::SCRIPT) {+ res |= RelFlags::SCRIPT_TAG;+ }++ if self.contains(Location::META) {+ res |= RelFlags::META_TAG;+ }++ res+ }+}++bitflags! {+ impl Location: u8 {+ const FOOTER = 1 << 0;+ const NAVIGATION = 1 << 1;+ const LINK = 1 << 2;+ const SCRIPT = 1 << 3;+ const META = 1 << 4;+ }+}+impl Html {pub fn favicon(&self) -> Option<FaviconLink> {for node in self.root.select("link").unwrap() {@@ -124,6 +252,7 @@ impl Html {let mut links = Vec::new();let mut open_links = Vec::new();+ let mut location = Location::empty();for edge in self.root.traverse() {match edge {@@ -131,6 +260,10 @@ impl Html {if let Some(element) = node.as_element() {if &element.name.local == "a" {open_links.push((String::new(), element.attributes.clone()));+ } else if &element.name.local == "footer" {+ location |= Location::FOOTER;+ } else if &element.name.local == "nav" {+ location |= Location::NAVIGATION;}}}@@ -148,12 +281,21 @@ impl Html {{links.push(Link {source: self.url().clone(),- destination: dest,text: text.trim().to_string(),+ rel: RelFlags::from_html(+ &dest,+ &attributes.borrow(),+ &location,+ ),+ destination: dest,});}}}+ } else if &element.name.local == "footer" {+ location.remove(Location::FOOTER);+ } else if &element.name.local == "nav" {+ location.remove(Location::NAVIGATION);}}@@ -187,6 +329,7 @@ impl Html {if let Ok(dest) = Url::parse(dest).or_else(|_| self.url().join(dest)) {links.push(Link {source: self.url().clone(),+ rel: RelFlags::from_html(&dest, &attributes.borrow(), &location),destination: dest,text: text.trim().to_string(),});@@ -200,12 +343,19 @@ impl Html {fn links_tag(&self) -> Vec<Link> {let mut links = Vec::new();+ let location = Location::LINK;+for node in self.root.select("link").unwrap() {if let Some(element) = node.as_node().as_element() {if let Some(href) = element.attributes.borrow().get("href") {if let Ok(href) = Url::parse(href).or_else(|_| self.url().join(href)) {links.push(Link {source: self.url().clone(),+ rel: RelFlags::from_html(+ &href,+ &element.attributes.borrow(),+ &location,+ ),destination: href,text: String::new(),});@@ -218,6 +368,8 @@ impl Html {}fn metadata_links(&self) -> Vec<Link> {+ let location = Location::META;+self.metadata().into_iter().filter_map(|metadata| {@@ -243,6 +395,7 @@ impl Html {source: self.url().clone(),destination,text: String::new(),+ rel: location.as_rel(),});}}@@ -267,6 +420,7 @@ impl Html {source: self.url().clone(),destination,text: String::new(),+ rel: location.as_rel(),});}}@@ -293,6 +447,7 @@ impl Html {source: self.url().clone(),destination: script_url,text: String::new(),+ rel: Location::SCRIPT.as_rel(),})} else {None@@ -460,4 +615,92 @@ mod tests {}));}++ #[test]+ fn test_rel() {+ let raw = r#"+ <html>+ <head>+ <title>Best website</title>+ </head>+ <body>+ <a href="https://example.com/tags/example" rel="tag">Example</a>+ <a href="https://example.com/tags/example" rel="tag nofollow">Example</a>+ <a href="https://example.com/tags/example" rel="tag sponsored">Example</a>+ <a href="https://example.com/authors/example" rel="author">Example</a>++ <footer>+ <a href="https://example.com/terms-of-service" rel="terms-of-service">Terms of service</a>+ <a href="https://example.com/privacy-policy" rel="privacy-policy">Privacy policy</a>+ </footer>+ </body>+ </html>+ "#;++ let webpage = Html::parse(raw, "https://www.example.com/whatever").unwrap();++ let links = webpage.all_links();++ assert_eq!(links.len(), 6);++ assert_eq!(+ links[0],+ Link {+ source: Url::parse("https://www.example.com/whatever").unwrap(),+ destination: Url::parse("https://example.com/tags/example").unwrap(),+ text: "Example".to_string(),+ rel: RelFlags::TAG+ }+ );++ assert_eq!(+ links[1],+ Link {+ source: Url::parse("https://www.example.com/whatever").unwrap(),+ destination: Url::parse("https://example.com/tags/example").unwrap(),+ text: "Example".to_string(),+ rel: RelFlags::TAG | RelFlags::NOFOLLOW+ }+ );++ assert_eq!(+ links[2],+ Link {+ source: Url::parse("https://www.example.com/whatever").unwrap(),+ destination: Url::parse("https://example.com/tags/example").unwrap(),+ text: "Example".to_string(),+ rel: RelFlags::TAG | RelFlags::SPONSORED+ }+ );++ assert_eq!(+ links[3],+ Link {+ source: Url::parse("https://www.example.com/whatever").unwrap(),+ destination: Url::parse("https://example.com/authors/example").unwrap(),+ text: "Example".to_string(),+ rel: RelFlags::AUTHOR+ }+ );++ assert_eq!(+ links[4],+ Link {+ source: Url::parse("https://www.example.com/whatever").unwrap(),+ destination: Url::parse("https://example.com/terms-of-service").unwrap(),+ text: "Terms of service".to_string(),+ rel: RelFlags::TERMS_OF_SERVICE | RelFlags::IS_IN_FOOTER+ }+ );++ assert_eq!(+ links[5],+ Link {+ source: Url::parse("https://www.example.com/whatever").unwrap(),+ destination: Url::parse("https://example.com/privacy-policy").unwrap(),+ text: "Privacy policy".to_string(),+ rel: RelFlags::PRIVACY_POLICY | RelFlags::IS_IN_FOOTER+ }+ );+ }}diff --git a/crates/core/src/webpage/html/mod.rs b/crates/core/src/webpage/html/mod.rsindex 2dc56a52..926fe5e0 100644--- a/crates/core/src/webpage/html/mod.rs+++ b/crates/core/src/webpage/html/mod.rs@@ -32,7 +32,7 @@ pub use fn_cache::FnCache;mod fn_cache;mod into_tantivy;-mod links;+pub mod links;mod microformats;mod parse_text;mod robots_meta;@@ -476,6 +476,8 @@ mod tests {use std::collections::HashMap;+ use tests::links::RelFlags;+use crate::{schema::create_schema, webpage::Link};use super::*;@@ -509,7 +511,8 @@ mod tests {vec![Link {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com").unwrap(),- text: "Link to example".to_string()+ text: "Link to example".to_string(),+ rel: RelFlags::empty(),}]);assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));@@ -952,7 +955,8 @@ mod tests {vec![Link {source: Url::parse("https://www.example.com/whatever").unwrap(),destination: Url::parse("https://example.com").unwrap(),- text: "Link to example".to_string()+ text: "Link to example".to_string(),+ rel: RelFlags::empty(),},]);}diff --git a/crates/core/src/webpage/mod.rs b/crates/core/src/webpage/mod.rsindex 63b32ffe..a84f2d56 100644--- a/crates/core/src/webpage/mod.rs+++ b/crates/core/src/webpage/mod.rs@@ -33,6 +33,7 @@ pub mod region;pub mod safety_classifier;pub mod schema_org;pub mod url_ext;+use self::html::links::RelFlags;pub use self::html::Html;pub use region::Region;@@ -151,6 +152,7 @@ struct Script {pub struct Link {pub source: Url,pub destination: Url,+ pub rel: RelFlags,pub text: String,}diff --git a/crates/file-store/src/const_serializable.rs b/crates/file-store/src/const_serializable.rsindex 2fe980de..81a5f64b 100644--- a/crates/file-store/src/const_serializable.rs+++ b/crates/file-store/src/const_serializable.rs@@ -59,12 +59,12 @@ macro_rules! impl_const_serializable_num {const BYTES: usize = $n;fn serialize(&self, buf: &mut [u8]) {- buf.copy_from_slice(&self.to_le_bytes());+ buf[..Self::BYTES].copy_from_slice(&self.to_le_bytes());}fn deserialize(buf: &[u8]) -> Self {- let mut bytes = [0; $n];- bytes.copy_from_slice(&buf[..$n]);+ let mut bytes = [0; Self::BYTES];+ bytes.copy_from_slice(&buf[..Self::BYTES]);<$t>::from_le_bytes(bytes)}}