Skip to content

web search done right

ability to get nodes in page-level graph by host

Mikkel Denker2d2f147ba637
diff --git a/core/src/api/user_count.rs b/core/src/api/user_count.rs
index 6635b2e6..96cdc3f7 100644
--- a/core/src/api/user_count.rs
+++ b/core/src/api/user_count.rs
@@ -97,7 +97,7 @@ impl<F: Frequency> InnerUserCount<F> {
let mut user_id = [0u8; 8];
user_id.copy_from_slice(&hash[..8]);
- self.counter.add(u64::from_le_bytes(user_id));
+ self.counter.add(u64::from_be_bytes(user_id));
self.metric.store(self.counter.size() as u64);
Ok(())
diff --git a/core/src/crawler/router.rs b/core/src/crawler/router.rs
index 02665c51..6337a3b7 100644
--- a/core/src/crawler/router.rs
+++ b/core/src/crawler/router.rs
@@ -107,7 +107,7 @@ impl Router {
fn coordinator_index(&self, domain: &Domain) -> usize {
let hash = md5::compute(domain.0.as_bytes());
- let hash = u128::from_le_bytes(hash.0) as usize;
+ let hash = u128::from_be_bytes(hash.0) as usize;
hash % self.coordinators.len()
}
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 7a43d3da..3be628ab 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -23,7 +23,6 @@ use std::sync::Arc;
use std::{cmp, fs};
use rayon::prelude::*;
-use serde::{Deserialize, Serialize};
use url::Url;
use utoipa::ToSchema;
@@ -36,7 +35,9 @@ use self::segment::{Segment, SegmentWriter};
pub const MAX_LABEL_LENGTH: usize = 1024;
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[derive(
+ Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,
+)]
pub struct NodeID(u128);
impl NodeID {
@@ -55,6 +56,21 @@ impl From<u128> for NodeID {
}
}
+#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
+pub struct FullNodeID {
+ pub prefix: NodeID,
+ pub id: NodeID,
+}
+
+impl From<Node> for FullNodeID {
+ fn from(value: Node) -> Self {
+ let id = value.id();
+ let prefix = value.into_host().id();
+
+ FullNodeID { prefix, id }
+ }
+}
+
impl intmap::Key for NodeID {
const BIG_PRIME: Self = NodeID(335579573203413586826293107669396558523);
@@ -113,6 +129,29 @@ where
pub label: L,
}
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct InnerEdge<L>
+where
+ L: EdgeLabel,
+{
+ pub from: FullNodeID,
+ pub to: FullNodeID,
+ pub label: L,
+}
+
+impl<L> From<InnerEdge<L>> for Edge<L>
+where
+ L: EdgeLabel,
+{
+ fn from(edge: InnerEdge<L>) -> Self {
+ Edge {
+ from: edge.from.id,
+ to: edge.to.id,
+ label: edge.label,
+ }
+ }
+}
+
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct FullEdge {
@@ -121,7 +160,18 @@ pub struct FullEdge {
pub label: String,
}
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, ToSchema)]
+#[derive(
+ Debug,
+ Clone,
+ PartialEq,
+ Eq,
+ PartialOrd,
+ Ord,
+ Hash,
+ serde::Serialize,
+ serde::Deserialize,
+ ToSchema,
+)]
#[serde(rename_all = "camelCase")]
pub struct Node {
pub name: String,
@@ -360,7 +410,7 @@ impl ShortestPaths for Webgraph {
type SegmentID = String;
-#[derive(Serialize, Deserialize, Default)]
+#[derive(serde::Serialize, serde::Deserialize, Default)]
struct Meta {
comitted_segments: Vec<SegmentID>,
}
@@ -479,7 +529,7 @@ impl Id2NodeDb {
pub struct WebgraphWriter {
pub path: String,
segment: SegmentWriter,
- insert_batch: Vec<Edge<String>>,
+ insert_batch: Vec<InnerEdge<String>>,
id2node: Id2NodeDb,
executor: Executor,
meta: Meta,
@@ -524,20 +574,23 @@ impl WebgraphWriter {
self.id2node.get(id)
}
- fn id_or_assign(&mut self, node: &Node) -> NodeID {
- let id = node.id();
+ fn id_or_assign(&mut self, node: Node) -> FullNodeID {
+ let id = FullNodeID::from(node.clone());
- if self.id2node(&id).is_none() {
- self.id2node.put(&id, node);
+ if self.id2node(&id.id).is_none() {
+ self.id2node.put(&id.id, &node);
}
id
}
pub fn insert(&mut self, from: Node, to: Node, label: String) {
- let (from_id, to_id) = (self.id_or_assign(&from), self.id_or_assign(&to));
+ let (from_id, to_id) = (
+ self.id_or_assign(from.clone()),
+ self.id_or_assign(to.clone()),
+ );
- let edge = Edge {
+ let edge = InnerEdge {
from: from_id,
to: to_id,
label: label.chars().take(MAX_LABEL_LENGTH).collect(),
@@ -657,6 +710,15 @@ impl Webgraph {
.collect()
}
+ pub fn raw_ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {
+ let dedup = |edges: &mut Vec<Edge<()>>| {
+ edges.sort_by_key(|e| e.from);
+ edges.dedup_by_key(|e| e.from);
+ };
+
+ self.inner_edges(|segment| segment.ingoing_edges_by_host(host_node), dedup)
+ }
+
pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
let dedup = |edges: &mut Vec<Edge<()>>| {
edges.sort_by_key(|e| e.from);
@@ -927,4 +989,42 @@ mod test {
"a".repeat(MAX_LABEL_LENGTH)
);
}
+
+ #[test]
+ fn edges_by_host() {
+ let mut writer = WebgraphWriter::new(
+ crate::gen_temp_path(),
+ Executor::single_thread(),
+ Compression::default(),
+ );
+
+ writer.insert(
+ Node::from("http://a.com/first"),
+ Node::from("http://b.com/first"),
+ String::new(),
+ );
+ writer.insert(
+ Node::from("http://c.com/first"),
+ Node::from("http://b.com/second"),
+ String::new(),
+ );
+
+ let graph = writer.finalize();
+
+ let mut res = graph
+ .raw_ingoing_edges_by_host(&Node::from("b.com").id())
+ .into_iter()
+ .map(|e| e.to)
+ .map(|id| graph.id2node(&id).unwrap())
+ .collect::<Vec<_>>();
+ res.sort();
+
+ assert_eq!(
+ res,
+ vec![
+ Node::from("http://b.com/first"),
+ Node::from("http://b.com/second")
+ ]
+ );
+ }
}
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 26a8ebee..715d8671 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -18,7 +18,7 @@ use std::path::{Path, PathBuf};
use super::{
store::{EdgeStore, EdgeStoreWriter},
- Compression, Edge, NodeID,
+ Compression, Edge, InnerEdge, NodeID,
};
const ADJACENCY_STORE: &str = "adjacency";
@@ -73,7 +73,7 @@ impl SegmentWriter {
self.full_reversed_adjacency.flush();
}
- pub fn insert(&mut self, edges: &[Edge<String>]) {
+ pub fn insert(&mut self, edges: &[InnerEdge<String>]) {
self.full_adjacency.put(edges.iter());
self.full_reversed_adjacency.put(edges.iter());
}
@@ -128,6 +128,14 @@ impl Segment {
self.full_reversed_adjacency.get_without_label(node)
}
+ pub fn ingoing_edges_by_host(&self, host_node: &NodeID) -> Vec<Edge<()>> {
+ self.full_reversed_adjacency
+ .nodes_by_prefix(host_node)
+ .into_iter()
+ .flat_map(|node| self.ingoing_edges(&node))
+ .collect()
+ }
+
pub fn id(&self) -> String {
self.id.clone()
}
@@ -143,6 +151,8 @@ impl Segment {
#[cfg(test)]
mod test {
+ use crate::webgraph::FullNodeID;
+
use super::*;
#[test]
@@ -162,35 +172,44 @@ mod test {
let mut edges = Vec::new();
- let a = NodeID(0);
- let b = NodeID(1);
- let c = NodeID(2);
-
- edges.push(Edge {
- from: a,
- to: b,
+ let a = FullNodeID {
+ id: NodeID(0),
+ prefix: NodeID(0),
+ };
+ let b = FullNodeID {
+ id: NodeID(1),
+ prefix: NodeID(0),
+ };
+ let c = FullNodeID {
+ id: NodeID(2),
+ prefix: NodeID(0),
+ };
+
+ edges.push(InnerEdge {
+ from: a.clone(),
+ to: b.clone(),
label: String::new(),
});
- edges.push(Edge {
- from: b,
- to: c,
+ edges.push(InnerEdge {
+ from: b.clone(),
+ to: c.clone(),
label: String::new(),
});
- edges.push(Edge {
- from: c,
- to: a,
+ edges.push(InnerEdge {
+ from: c.clone(),
+ to: a.clone(),
label: String::new(),
});
- edges.push(Edge {
- from: a,
- to: c,
+ edges.push(InnerEdge {
+ from: a.clone(),
+ to: c.clone(),
label: String::new(),
});
writer.insert(&edges);
let segment = writer.finalize();
- let mut out: Vec<_> = segment.outgoing_edges(&a);
+ let mut out: Vec<_> = segment.outgoing_edges(&a.id);
out.sort_by(|a, b| a.to.cmp(&b.to));
@@ -198,73 +217,73 @@ mod test {
out,
vec![
Edge {
- from: a,
- to: b,
+ from: a.id,
+ to: b.id,
label: ()
},
Edge {
- from: a,
- to: c,
+ from: a.id,
+ to: c.id,
label: ()
},
]
);
- let mut out: Vec<_> = segment.outgoing_edges(&b);
+ let mut out: Vec<_> = segment.outgoing_edges(&b.id);
out.sort_by(|a, b| a.to.cmp(&b.to));
assert_eq!(
out,
vec![Edge {
- from: b,
- to: c,
+ from: b.id,
+ to: c.id,
label: ()
},]
);
- let mut out: Vec<_> = segment.outgoing_edges(&c);
+ let mut out: Vec<_> = segment.outgoing_edges(&c.id);
out.sort_by(|a, b| a.to.cmp(&b.to));
assert_eq!(
out,
vec![Edge {
- from: c,
- to: a,
+ from: c.id,
+ to: a.id,
label: ()
},]
);
- let out: Vec<_> = segment.ingoing_edges(&a);
+ let out: Vec<_> = segment.ingoing_edges(&a.id);
assert_eq!(
out,
vec![Edge {
- from: c,
- to: a,
+ from: c.id,
+ to: a.id,
label: ()
},]
);
- let out: Vec<_> = segment.ingoing_edges(&b);
+ let out: Vec<_> = segment.ingoing_edges(&b.id);
assert_eq!(
out,
vec![Edge {
- from: a,
- to: b,
+ from: a.id,
+ to: b.id,
label: ()
},]
);
- let mut out: Vec<_> = segment.ingoing_edges(&c);
+ let mut out: Vec<_> = segment.ingoing_edges(&c.id);
out.sort_by(|a, b| a.from.cmp(&b.from));
assert_eq!(
out,
vec![
Edge {
- from: a,
- to: c,
+ from: a.id,
+ to: c.id,
label: ()
},
Edge {
- from: b,
- to: c,
+ from: b.id,
+ to: c.id,
label: ()
},
]
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index dc8fb350..bd9711ca 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -20,7 +20,14 @@ use itertools::Itertools;
use memmap2::Mmap;
use rocksdb::BlockBasedOptions;
-use super::{Compression, Edge, EdgeLabel, NodeID};
+use super::{Compression, Edge, EdgeLabel, FullNodeID, InnerEdge, NodeID};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
+pub struct SerializedEdge {
+ from: FullNodeID,
+ to: FullNodeID,
+ label: Vec<u8>,
+}
pub const MAX_BATCH_SIZE: usize = 100_000;
@@ -71,24 +78,35 @@ impl EdgeStoreWriter {
}
}
- pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a Edge<L>>) {
+ pub fn put<'a, L: EdgeLabel + 'a>(&'a self, edges: impl Iterator<Item = &'a InnerEdge<L>>) {
let mut batch = rocksdb::WriteBatch::default();
let mut opts = rocksdb::WriteOptions::default();
opts.disable_wal(true);
for edge in edges {
- let (prefix, suffix) = if self.reversed {
- (edge.to, edge.from)
- } else {
- (edge.from, edge.to)
- };
+ let value_bytes = L::to_bytes(&edge.label).unwrap();
- let prefix_bytes = prefix.bit_128().to_be_bytes();
- let suffix_bytes = suffix.bit_128().to_be_bytes();
+ let value_bytes = bincode::serialize(&SerializedEdge {
+ from: edge.from.clone(),
+ to: edge.to.clone(),
+ label: value_bytes.clone(),
+ })
+ .unwrap();
- let key_bytes = [prefix_bytes, suffix_bytes].concat();
- let value_bytes = edge.label.to_bytes().unwrap();
+ let key_bytes = if self.reversed {
+ [
+ edge.to.id.bit_128().to_be_bytes(),
+ edge.from.id.bit_128().to_be_bytes(),
+ ]
+ .concat()
+ } else {
+ [
+ edge.from.id.bit_128().to_be_bytes(),
+ edge.to.id.bit_128().to_be_bytes(),
+ ]
+ .concat()
+ };
batch.put(key_bytes, value_bytes);
@@ -101,35 +119,21 @@ impl EdgeStoreWriter {
self.db.write_opt(batch, &opts).unwrap();
}
- pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = Edge<L>> + '_ + Send + Sync {
+ pub fn iter<L: EdgeLabel>(&self) -> impl Iterator<Item = InnerEdge<L>> + '_ + Send + Sync {
let read_opts = rocksdb::ReadOptions::default();
self.db
.iterator_opt(rocksdb::IteratorMode::Start, read_opts)
.map(|res| {
- let (key, val) = res.unwrap();
-
- let (from, to) = if self.reversed {
- let from = u128::from_be_bytes(key[16..32].try_into().unwrap());
- let from = NodeID(from);
+ let (_, val) = res.unwrap();
- let to = u128::from_be_bytes(key[0..16].try_into().unwrap());
- let to = NodeID(to);
+ let val: SerializedEdge = bincode::deserialize(&val).unwrap();
- (from, to)
- } else {
- let from = u128::from_be_bytes(key[0..16].try_into().unwrap());
- let from = NodeID(from);
-
- let to = u128::from_be_bytes(key[16..32].try_into().unwrap());
- let to = NodeID(to);
-
- (from, to)
- };
-
- let label = L::from_bytes(&val).unwrap();
-
- Edge { from, to, label }
+ InnerEdge {
+ from: val.from,
+ to: val.to,
+ label: L::from_bytes(&val.label).unwrap(),
+ }
})
}
@@ -154,9 +158,95 @@ impl EdgeStoreWriter {
}
}
+struct PrefixDb {
+ db: rocksdb::DB,
+}
+
+impl PrefixDb {
+ fn open<P: AsRef<Path>>(path: P) -> Self {
+ let mut options = rocksdb::Options::default();
+ options.create_if_missing(true);
+
+ options.set_max_background_jobs(8);
+ options.increase_parallelism(8);
+ options.set_max_subcompactions(8);
+
+ options.set_allow_mmap_reads(true);
+ options.set_allow_mmap_writes(true);
+
+ options.set_level_zero_slowdown_writes_trigger(-1);
+ options.set_level_zero_stop_writes_trigger(-1);
+
+ // some recommended settings (https://github.com/facebook/rocksdb/wiki/Setup-Options-and-Basic-Tuning)
+ options.set_level_compaction_dynamic_level_bytes(true);
+ options.set_bytes_per_sync(1048576);
+ let mut block_options = BlockBasedOptions::default();
+ block_options.set_block_size(16 * 1024);
+ block_options.set_format_version(5);
+ block_options.set_cache_index_and_filter_blocks(true);
+ block_options.set_pin_l0_filter_and_index_blocks_in_cache(true);
+
+ options.set_block_based_table_factory(&block_options);
+ options.set_compression_type(rocksdb::DBCompressionType::Lz4);
+
+ let db = rocksdb::DB::open(&options, path).unwrap();
+
+ Self { db }
+ }
+
+ fn insert(&self, node: &FullNodeID) {
+ let mut opts = rocksdb::WriteOptions::default();
+ opts.disable_wal(true);
+
+ let key = [
+ node.prefix.bit_128().to_be_bytes(),
+ node.id.bit_128().to_be_bytes(),
+ ]
+ .concat();
+ let value = [];
+
+ self.db.put_opt(key, value, &opts).unwrap();
+ }
+
+ fn get(&self, prefix: &NodeID) -> Vec<NodeID> {
+ let start = [
+ prefix.bit_128().to_be_bytes().to_vec(),
+ [0].repeat(u128::BITS as usize / 8).to_vec(),
+ ]
+ .concat();
+
+ let mut res = Vec::new();
+
+ let iter = self.db.iterator(rocksdb::IteratorMode::From(
+ &start,
+ rocksdb::Direction::Forward,
+ ));
+
+ for item in iter {
+ let (key, _) = item.unwrap();
+
+ let p = u128::from_be_bytes(key[..(u128::BITS as usize / 8)].try_into().unwrap());
+ let node = u128::from_be_bytes(key[(u128::BITS as usize / 8)..].try_into().unwrap());
+
+ if p != prefix.bit_128() {
+ break;
+ }
+
+ res.push(NodeID::from(node));
+ }
+
+ res
+ }
+
+ fn flush(&self) {
+ self.db.flush().unwrap();
+ }
+}
+
pub struct EdgeStore {
reversed: bool,
- ranges: rocksdb::DB, // column[nodes] = nodeid -> (start, end); column[labels] = nodeid -> (start, end)
+ ranges: rocksdb::DB, // column[nodes] = full_nodeid -> (start, end); column[labels] = nodeid -> (start, end)
+ prefixes: PrefixDb,
_cache: rocksdb::Cache,
edge_labels_file: File,
@@ -240,6 +330,7 @@ impl EdgeStore {
Self {
reversed,
ranges,
+ prefixes: PrefixDb::open(path.as_ref().join("prefixes")),
_cache: cache,
edge_labels,
edge_labels_len,
@@ -255,17 +346,19 @@ impl EdgeStore {
/// The edges *must* have been de-duplicated by their from/to node.
/// I.e. if the store is not reversed, there should only ever be a single
/// put for each from node, and vice versa.
- fn put(&mut self, edges: &[Edge<String>]) {
+ fn put(&mut self, edges: &[InnerEdge<String>]) {
if edges.is_empty() {
return;
}
let node = if self.reversed {
- edges[0].to
+ edges[0].to.clone()
} else {
- edges[0].from
+ edges[0].from.clone()
};
- let node_bytes = node.bit_128().to_be_bytes();
+
+ self.prefixes.insert(&node);
+ let node_bytes = node.id.bit_128().to_be_bytes();
let node_cf = self.ranges.cf_handle("nodes").unwrap();
let label_cf = self.ranges.cf_handle("labels").unwrap();
@@ -278,7 +371,11 @@ impl EdgeStore {
for edge in edges {
edge_labels.push(edge.label.clone());
- edge_nodes.push(if self.reversed { edge.from } else { edge.to });
+ edge_nodes.push(if self.reversed {
+ edge.from.id
+ } else {
+ edge.to.id
+ });
}
let edge_labels_bytes = bincode::serialize(&edge_labels).unwrap();
@@ -324,7 +421,7 @@ impl EdgeStore {
path: P,
compression: Compression,
reversed: bool,
- edges: impl Iterator<Item = Edge<String>>,
+ edges: impl Iterator<Item = InnerEdge<String>>,
) -> Self {
let mut s = Self::open(path, reversed, compression);
@@ -333,13 +430,14 @@ impl EdgeStore {
let mut last_node = None;
for edge in edges {
if let Some(last_node) = last_node {
- if (reversed && edge.to != last_node) || (!reversed && edge.from != last_node) {
+ if (reversed && edge.to.id != last_node) || (!reversed && edge.from.id != last_node)
+ {
s.put(&batch);
batch.clear();
}
}
- last_node = Some(if reversed { edge.to } else { edge.from });
+ last_node = Some(if reversed { edge.to.id } else { edge.from.id });
batch.push(edge);
}
@@ -353,6 +451,8 @@ impl EdgeStore {
}
fn flush(&mut self) {
+ self.prefixes.flush();
+
self.ranges.flush().unwrap();
self.ranges
.flush_cf(self.ranges.cf_handle("nodes").unwrap())
@@ -453,6 +553,10 @@ impl EdgeStore {
}
}
+ pub fn nodes_by_prefix(&self, prefix: &NodeID) -> Vec<NodeID> {
+ self.prefixes.get(prefix)
+ }
+
pub fn iter_without_label(&self) -> impl Iterator<Item = Edge<()>> + '_ + Send + Sync {
let node_cf = self.ranges.cf_handle("nodes").unwrap();
@@ -490,6 +594,8 @@ impl EdgeStore {
#[cfg(test)]
mod tests {
+ use crate::webgraph::FullNodeID;
+
use super::*;
#[test]
@@ -500,9 +606,15 @@ mod tests {
false,
);
- let e = Edge {
- from: NodeID(0),
- to: NodeID(1),
+ let e = InnerEdge {
+ from: FullNodeID {
+ id: NodeID(0),
+ prefix: NodeID(0),
+ },
+ to: FullNodeID {
+ id: NodeID(1),
+ prefix: NodeID(0),
+ },
label: "test".to_string(),
};
@@ -513,7 +625,7 @@ mod tests {
let edges: Vec<_> = store.get_with_label(&NodeID(0));
assert_eq!(edges.len(), 1);
- assert_eq!(&edges[0], &e);
+ assert_eq!(&edges[0], &Edge::from(e.clone()));
let edges: Vec<_> = store.get_with_label(&NodeID(1));
@@ -528,9 +640,15 @@ mod tests {
true,
);
- let e = Edge {
- from: NodeID(0),
- to: NodeID(1),
+ let e = InnerEdge {
+ from: FullNodeID {
+ id: NodeID(0),
+ prefix: NodeID(0),
+ },
+ to: FullNodeID {
+ id: NodeID(1),
+ prefix: NodeID(0),
+ },
label: "test".to_string(),
};
@@ -543,6 +661,6 @@ mod tests {
let edges: Vec<_> = store.get_with_label(&NodeID(1));
assert_eq!(edges.len(), 1);
- assert_eq!(&edges[0], &e);
+ assert_eq!(&edges[0], &Edge::from(e.clone()));
}
}