web search done right
ability to get nodes in page-level graph by host
Mikkel Denker2d2f147ba637
diff --git a/core/src/api/user_count.rs b/core/src/api/user_count.rsindex 6635b2e6..96cdc3f7 100644--- a/core/src/api/user_count.rs+++ b/core/src/api/user_count.rs@@ -97,7 +97,7 @@ impl<F: Frequency> InnerUserCount<F> {let mut user_id = [0u8; 8];user_id.copy_from_slice(&hash[..8]);- self.counter.add(u64::from_le_bytes(user_id));+ self.counter.add(u64::from_be_bytes(user_id));self.metric.store(self.counter.size() as u64);Ok(())diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rsindex 02665c51..6337a3b7 100644--- a/core/src/crawler/router.rs+++ b/core/src/crawler/router.rs@@ -107,7 +107,7 @@ impl Router {fn coordinator_index(&self, domain: &Domain) -> usize {let hash = md5::compute(domain.0.as_bytes());- let hash = u128::from_le_bytes(hash.0) as usize;+ let hash = u128::from_be_bytes(hash.0) as usize;hash % self.coordinators.len()}diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 7a43d3da..3be628ab 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -23,7 +23,6 @@ use std::sync::Arc;use std::{cmp, fs};use rayon::prelude::*;-use serde::{Deserialize, Serialize};use url::Url;use utoipa::ToSchema;@@ -36,7 +35,9 @@ use self::segment::{Segment, SegmentWriter};pub const MAX_LABEL_LENGTH: usize = 1024;-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]+#[derive(+ Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,+)]pub struct NodeID(u128);impl NodeID {@@ -55,6 +56,21 @@ impl From<u128> for NodeID {}}+#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct FullNodeID {+ pub prefix: NodeID,+ pub id: NodeID,+}++impl From<Node> for FullNodeID {+ fn from(value: Node) -> Self {+ let id = value.id();+ let prefix = value.into_host().id();++ FullNodeID { prefix, id }+ }+}+impl intmap::Key for NodeID {const BIG_PRIME: Self = NodeID(335579573203413586826293107669396558523);@@ -113,6 +129,29 @@ wherepub label: L,}+#[derive(Debug, Clone, PartialEq, Eq, Hash)]+pub struct InnerEdge<L>+where+ L: EdgeLabel,+{+ pub from: FullNodeID,+ pub to: FullNodeID,+ pub label: L,+}++impl<L> From<InnerEdge<L>> for Edge<L>+where+ L: EdgeLabel,+{+ fn from(edge: InnerEdge<L>) -> Self {+ Edge {+ from: edge.from.id,+ to: edge.to.id,+ label: edge.label,+ }+ }+}+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash, ToSchema)]#[serde(rename_all = "camelCase")]pub struct FullEdge {@@ -121,7 +160,18 @@ pub struct FullEdge {pub label: String,}-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct Node {pub name: String,@@ -360,7 +410,7 @@ impl ShortestPaths for Webgraph {type SegmentID = String;-#[derive(Serialize, Deserialize, Default)]+#[derive(serde::Serialize, serde::Deserialize, Default)]struct Meta {comitted_segments: Vec<SegmentID>,}@@ -479,7 +529,7 @@ impl Id2NodeDb {pub struct WebgraphWriter {pub path: String,segment: SegmentWriter,- insert_batch: Vec<Edge<String>>,+ insert_batch: Vec<InnerEdge<String>>,id2node: Id2NodeDb,executor: Executor,meta: Meta,@@ -524,20 +574,23 @@ impl WebgraphWriter {self.id2node.get(id)}- fn id_or_assign(&mut self, node: &Node) -> NodeID {- let id = node.id();+ fn id_or_assign(&mut self, node: Node) -> FullNodeID {+ let id = FullNodeID::from(node.clone());- if self.id2node(&id).is_none() {- self.id2node.put(&id, node);+ if self.id2node(&id.id).is_none() {+ self.id2node.put(&id.id, &node);}id}pub fn insert(&mut self, from: Node, to: Node, label: String) {- let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));+ let (from_id, to_id) = (+ self.id_or_assign(from.clone()),+ self.id_or_assign(to.clone()),+ );- let edge = Edge {+ let edge = InnerEdge {from: from_id,to: to_id,label: label.chars().take(MAX_LABEL_LENGTH).collect(),@@ -657,6 +710,15 @@ impl Webgraph {.collect()}+ pub fn raw_ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {+ let dedup = |edges: &mut Vec<Edge<()>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);+ };++ self.inner_edges(|segment| segment.ingoing_edges_by_host(host_node), dedup)+ }+pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {let dedup = |edges: &mut Vec<Edge<()>>| {edges.sort_by_key(|e| e.from);@@ -927,4 +989,42 @@ mod test {"a".repeat(MAX_LABEL_LENGTH));}++ #[test]+ fn edges_by_host() {+ let mut writer = WebgraphWriter::new(+ crate::gen_temp_path(),+ Executor::single_thread(),+ Compression::default(),+ );++ writer.insert(+ Node::from("http://a.com/first"),+ Node::from("http://b.com/first"),+ String::new(),+ );+ writer.insert(+ Node::from("http://c.com/first"),+ Node::from("http://b.com/second"),+ String::new(),+ );++ let graph = writer.finalize();++ let mut res = graph+ .raw_ingoing_edges_by_host(&Node::from("b.com").id())+ .into_iter()+ .map(|e| e.to)+ .map(|id| graph.id2node(&id).unwrap())+ .collect::<Vec<_>>();+ res.sort();++ assert_eq!(+ res,+ vec![+ Node::from("http://b.com/first"),+ Node::from("http://b.com/second")+ ]+ );+ }}diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 26a8ebee..715d8671 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -18,7 +18,7 @@ use std::path::{Path, PathBuf};use super::{store::{EdgeStore, EdgeStoreWriter},- Compression, Edge, NodeID,+ Compression, Edge, InnerEdge, NodeID,};const ADJACENCY_STORE: &str = "adjacency";@@ -73,7 +73,7 @@ impl SegmentWriter {self.full_reversed_adjacency.flush();}- pub fn insert(&mut self, edges: &[Edge<String>]) {+ pub fn insert(&mut self, edges: &[InnerEdge<String>]) {self.full_adjacency.put(edges.iter());self.full_reversed_adjacency.put(edges.iter());}@@ -128,6 +128,14 @@ impl Segment {self.full_reversed_adjacency.get_without_label(node)}+ pub fn ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {+ self.full_reversed_adjacency+ .nodes_by_prefix(host_node)+ .into_iter()+ .flat_map(|node| self.ingoing_edges(&node))+ .collect()+ }+pub fn id(&self) -> String {self.id.clone()}@@ -143,6 +151,8 @@ impl Segment {#[cfg(test)]mod test {+ use crate::webgraph::FullNodeID;+use super::*;#[test]@@ -162,35 +172,44 @@ mod test {let mut edges = Vec::new();- let a = NodeID(0);- let b = NodeID(1);- let c = NodeID(2);-- edges.push(Edge {- from: a,- to: b,+ let a = FullNodeID {+ id: NodeID(0),+ prefix: NodeID(0),+ };+ let b = FullNodeID {+ id: NodeID(1),+ prefix: NodeID(0),+ };+ let c = FullNodeID {+ id: NodeID(2),+ prefix: NodeID(0),+ };++ edges.push(InnerEdge {+ from: a.clone(),+ to: b.clone(),label: String::new(),});- edges.push(Edge {- from: b,- to: c,+ edges.push(InnerEdge {+ from: b.clone(),+ to: c.clone(),label: String::new(),});- edges.push(Edge {- from: c,- to: a,+ edges.push(InnerEdge {+ from: c.clone(),+ to: a.clone(),label: String::new(),});- edges.push(Edge {- from: a,- to: c,+ edges.push(InnerEdge {+ from: a.clone(),+ to: c.clone(),label: String::new(),});writer.insert(&edges);let segment = writer.finalize();- let mut out: Vec<_> = segment.outgoing_edges(&a);+ let mut out: Vec<_> = segment.outgoing_edges(&a.id);out.sort_by(|a, b| a.to.cmp(&b.to));@@ -198,73 +217,73 @@ mod test {out,vec![Edge {- from: a,- to: b,+ from: a.id,+ to: b.id,label: ()},Edge {- from: a,- to: c,+ from: a.id,+ to: c.id,label: ()},]);- let mut out: Vec<_> = segment.outgoing_edges(&b);+ let mut out: Vec<_> = segment.outgoing_edges(&b.id);out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,vec![Edge {- from: b,- to: c,+ from: b.id,+ to: c.id,label: ()},]);- let mut out: Vec<_> = segment.outgoing_edges(&c);+ let mut out: Vec<_> = segment.outgoing_edges(&c.id);out.sort_by(|a, b| a.to.cmp(&b.to));assert_eq!(out,vec![Edge {- from: c,- to: a,+ from: c.id,+ to: a.id,label: ()},]);- let out: Vec<_> = segment.ingoing_edges(&a);+ let out: Vec<_> = segment.ingoing_edges(&a.id);assert_eq!(out,vec![Edge {- from: c,- to: a,+ from: c.id,+ to: a.id,label: ()},]);- let out: Vec<_> = segment.ingoing_edges(&b);+ let out: Vec<_> = segment.ingoing_edges(&b.id);assert_eq!(out,vec![Edge {- from: a,- to: b,+ from: a.id,+ to: b.id,label: ()},]);- let mut out: Vec<_> = segment.ingoing_edges(&c);+ let mut out: Vec<_> = segment.ingoing_edges(&c.id);out.sort_by(|a, b| a.from.cmp(&b.from));assert_eq!(out,vec![Edge {- from: a,- to: c,+ from: a.id,+ to: c.id,label: ()},Edge {- from: b,- to: c,+ from: b.id,+ to: c.id,label: ()},]diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex dc8fb350..bd9711ca 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -20,7 +20,14 @@ use itertools::Itertools;use memmap2::Mmap;use rocksdb::BlockBasedOptions;-use super::{Compression, Edge, EdgeLabel, NodeID};+use super::{Compression, Edge, EdgeLabel, FullNodeID, InnerEdge, NodeID};++#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+pub struct SerializedEdge {+ from: FullNodeID,+ to: FullNodeID,+ label: Vec<u8>,+}pub const MAX_BATCH_SIZE: usize = 100_000;@@ -71,24 +78,35 @@ impl EdgeStoreWriter {}}- pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a Edge<L>>) {+ pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a InnerEdge<L>>) {let mut batch = rocksdb::WriteBatch::default();let mut opts = rocksdb::WriteOptions::default();opts.disable_wal(true);for edge in edges {- let (prefix, suffix) = if self.reversed {- (edge.to, edge.from)- } else {- (edge.from, edge.to)- };+ let value_bytes = L::to_bytes(&edge.label).unwrap();- let prefix_bytes = prefix.bit_128().to_be_bytes();- let suffix_bytes = suffix.bit_128().to_be_bytes();+ let value_bytes = bincode::serialize(&SerializedEdge {+ from: edge.from.clone(),+ to: edge.to.clone(),+ label: value_bytes.clone(),+ })+ .unwrap();- let key_bytes = [prefix_bytes, suffix_bytes].concat();- let value_bytes = edge.label.to_bytes().unwrap();+ let key_bytes = if self.reversed {+ [+ edge.to.id.bit_128().to_be_bytes(),+ edge.from.id.bit_128().to_be_bytes(),+ ]+ .concat()+ } else {+ [+ edge.from.id.bit_128().to_be_bytes(),+ edge.to.id.bit_128().to_be_bytes(),+ ]+ .concat()+ };batch.put(key_bytes, value_bytes);@@ -101,35 +119,21 @@ impl EdgeStoreWriter {self.db.write_opt(batch, &opts).unwrap();}- pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = Edge<L>> + '_ + Send + Sync {+ pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = InnerEdge<L>> + '_ + Send + Sync {let read_opts = rocksdb::ReadOptions::default();self.db.iterator_opt(rocksdb::IteratorMode::Start, read_opts).map(|res| {- let (key, val) = res.unwrap();-- let (from, to) = if self.reversed {- let from = u128::from_be_bytes(key[16..32].try_into().unwrap());- let from = NodeID(from);+ let (_, val) = res.unwrap();- let to = u128::from_be_bytes(key[0..16].try_into().unwrap());- let to = NodeID(to);+ let val: SerializedEdge = bincode::deserialize(&val).unwrap();- (from, to)- } else {- let from = u128::from_be_bytes(key[0..16].try_into().unwrap());- let from = NodeID(from);-- let to = u128::from_be_bytes(key[16..32].try_into().unwrap());- let to = NodeID(to);-- (from, to)- };-- let label = L::from_bytes(&val).unwrap();-- Edge { from, to, label }+ InnerEdge {+ from: val.from,+ to: val.to,+ label: L::from_bytes(&val.label).unwrap(),+ }})}@@ -154,9 +158,95 @@ impl EdgeStoreWriter {}}+struct PrefixDb {+ db: rocksdb::DB,+}++impl PrefixDb {+ fn open<P: AsRef<Path>>(path: P) -> Self {+ let mut options = rocksdb::Options::default();+ options.create_if_missing(true);++ options.set_max_background_jobs(8);+ options.increase_parallelism(8);+ options.set_max_subcompactions(8);++ options.set_allow_mmap_reads(true);+ options.set_allow_mmap_writes(true);++ options.set_level_zero_slowdown_writes_trigger(-1);+ options.set_level_zero_stop_writes_trigger(-1);++ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)+ options.set_level_compaction_dynamic_level_bytes(true);+ options.set_bytes_per_sync(1048576);+ let mut block_options = BlockBasedOptions::default();+ block_options.set_block_size(16 * 1024);+ block_options.set_format_version(5);+ block_options.set_cache_index_and_filter_blocks(true);+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);++ options.set_block_based_table_factory(&block_options);+ options.set_compression_type(rocksdb::DBCompressionType::Lz4);++ let db = rocksdb::DB::open(&options, path).unwrap();++ Self { db }+ }++ fn insert(&self, node: &FullNodeID) {+ let mut opts = rocksdb::WriteOptions::default();+ opts.disable_wal(true);++ let key = [+ node.prefix.bit_128().to_be_bytes(),+ node.id.bit_128().to_be_bytes(),+ ]+ .concat();+ let value = [];++ self.db.put_opt(key, value, &opts).unwrap();+ }++ fn get(&self, prefix: &NodeID) -> Vec<NodeID> {+ let start = [+ prefix.bit_128().to_be_bytes().to_vec(),+ [0].repeat(u128::BITS as usize / 8).to_vec(),+ ]+ .concat();++ let mut res = Vec::new();++ let iter = self.db.iterator(rocksdb::IteratorMode::From(+ &start,+ rocksdb::Direction::Forward,+ ));++ for item in iter {+ let (key, _) = item.unwrap();++ let p = u128::from_be_bytes(key[..(u128::BITS as usize / 8)].try_into().unwrap());+ let node = u128::from_be_bytes(key[(u128::BITS as usize / 8)..].try_into().unwrap());++ if p != prefix.bit_128() {+ break;+ }++ res.push(NodeID::from(node));+ }++ res+ }++ fn flush(&self) {+ self.db.flush().unwrap();+ }+}+pub struct EdgeStore {reversed: bool,- ranges: rocksdb::DB, // column[nodes] = nodeid -> (start, end); column[labels] = nodeid -> (start, end)+ ranges: rocksdb::DB, // column[nodes] = full_nodeid -> (start, end); column[labels] = nodeid -> (start, end)+ prefixes: PrefixDb,_cache: rocksdb::Cache,edge_labels_file: File,@@ -240,6 +330,7 @@ impl EdgeStore {Self {reversed,ranges,+ prefixes: PrefixDb::open(path.as_ref().join("prefixes")),_cache: cache,edge_labels,edge_labels_len,@@ -255,17 +346,19 @@ impl EdgeStore {/// The edges *must* have been de-duplicated by their from/to node./// I.e. if the store is not reversed, there should only ever be a single/// put for each from node, and vice versa.- fn put(&mut self, edges: &[Edge<String>]) {+ fn put(&mut self, edges: &[InnerEdge<String>]) {if edges.is_empty() {return;}let node = if self.reversed {- edges[0].to+ edges[0].to.clone()} else {- edges[0].from+ edges[0].from.clone()};- let node_bytes = node.bit_128().to_be_bytes();++ self.prefixes.insert(&node);+ let node_bytes = node.id.bit_128().to_be_bytes();let node_cf = self.ranges.cf_handle("nodes").unwrap();let label_cf = self.ranges.cf_handle("labels").unwrap();@@ -278,7 +371,11 @@ impl EdgeStore {for edge in edges {edge_labels.push(edge.label.clone());- edge_nodes.push(if self.reversed { edge.from } else { edge.to });+ edge_nodes.push(if self.reversed {+ edge.from.id+ } else {+ edge.to.id+ });}let edge_labels_bytes = bincode::serialize(&edge_labels).unwrap();@@ -324,7 +421,7 @@ impl EdgeStore {path: P,compression: Compression,reversed: bool,- edges: impl Iterator<Item = Edge<String>>,+ edges: impl Iterator<Item = InnerEdge<String>>,) -> Self {let mut s = Self::open(path, reversed, compression);@@ -333,13 +430,14 @@ impl EdgeStore {let mut last_node = None;for edge in edges {if let Some(last_node) = last_node {- if (reversed && edge.to != last_node) || (!reversed && edge.from != last_node) {+ if (reversed && edge.to.id != last_node) || (!reversed && edge.from.id != last_node)+ {s.put(&batch);batch.clear();}}- last_node = Some(if reversed { edge.to } else { edge.from });+ last_node = Some(if reversed { edge.to.id } else { edge.from.id });batch.push(edge);}@@ -353,6 +451,8 @@ impl EdgeStore {}fn flush(&mut self) {+ self.prefixes.flush();+self.ranges.flush().unwrap();self.ranges.flush_cf(self.ranges.cf_handle("nodes").unwrap())@@ -453,6 +553,10 @@ impl EdgeStore {}}+ pub fn nodes_by_prefix(&self, prefix: &NodeID) -> Vec<NodeID> {+ self.prefixes.get(prefix)+ }+pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {let node_cf = self.ranges.cf_handle("nodes").unwrap();@@ -490,6 +594,8 @@ impl EdgeStore {#[cfg(test)]mod tests {+ use crate::webgraph::FullNodeID;+use super::*;#[test]@@ -500,9 +606,15 @@ mod tests {false,);- let e = Edge {- from: NodeID(0),- to: NodeID(1),+ let e = InnerEdge {+ from: FullNodeID {+ id: NodeID(0),+ prefix: NodeID(0),+ },+ to: FullNodeID {+ id: NodeID(1),+ prefix: NodeID(0),+ },label: "test".to_string(),};@@ -513,7 +625,7 @@ mod tests {let edges: Vec<_> = store.get_with_label(&NodeID(0));assert_eq!(edges.len(), 1);- assert_eq!(&edges[0], &e);+ assert_eq!(&edges[0], &Edge::from(e.clone()));let edges: Vec<_> = store.get_with_label(&NodeID(1));@@ -528,9 +640,15 @@ mod tests {true,);- let e = Edge {- from: NodeID(0),- to: NodeID(1),+ let e = InnerEdge {+ from: FullNodeID {+ id: NodeID(0),+ prefix: NodeID(0),+ },+ to: FullNodeID {+ id: NodeID(1),+ prefix: NodeID(0),+ },label: "test".to_string(),};@@ -543,6 +661,6 @@ mod tests {let edges: Vec<_> = store.get_with_label(&NodeID(1));assert_eq!(edges.len(), 1);- assert_eq!(&edges[0], &e);+ assert_eq!(&edges[0], &Edge::from(e.clone()));}}