Skip to content

web search done right

ability to turn off deduplication for page webgraph inserts

Mikkel Denker30e9f12ddc60
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rs
index 1ecee632..66933a52 100644
--- a/core/src/entrypoint/configure.rs
+++ b/core/src/entrypoint/configure.rs
@@ -84,8 +84,8 @@ fn create_webgraph() -> Result<()> {
let mut worker = webgraph::WebgraphWorker {
redirect: None,
- host_graph: webgraph::open_graph(&out_path_host),
- page_graph: webgraph::open_graph(&out_path_page),
+ host_graph: webgraph::open_host_graph(&out_path_host),
+ page_graph: webgraph::open_page_graph(&out_path_page),
};
worker.process_job(&job);
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 1456e05d..b6cf6d24 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -67,10 +67,16 @@ pub struct Job {
pub warc_paths: Vec<String>,
}
-pub fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
+pub fn open_host_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
WebgraphBuilder::new(path).open()
}
+pub fn open_page_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
+ WebgraphBuilder::new(path)
+ .deduplication(webgraph::Deduplication::OnlyQuery)
+ .open()
+}
+
pub struct WebgraphWorker {
pub host_graph: webgraph::Webgraph,
pub page_graph: webgraph::Webgraph,
@@ -191,8 +197,8 @@ impl Webgraph {
let mut worker = WebgraphWorker {
redirect: redirect.clone(),
- host_graph: open_graph(host_path),
- page_graph: open_graph(page_path),
+ host_graph: open_host_graph(host_path),
+ page_graph: open_page_graph(page_path),
};
let jobs = jobs.clone();
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 7fa72b00..2cea3b83 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -231,9 +231,21 @@ pub fn normalize_url(url: &Url) -> String {
normalized
}
+#[derive(Default, Clone, Copy)]
+pub enum Deduplication {
+ /// Do not deduplicate the inserted edges in the segment. Only deduplicate
+ /// the edges when querying all the segments.
+ OnlyQuery,
+ /// Deduplicate the inserted edges in the segmnt by first checking if the
+ /// edge is already present in the segment.
+ #[default]
+ QueryAndInserts,
+}
+
pub struct WebgraphBuilder {
path: Box<Path>,
executor: Executor,
+ deduplication: Deduplication,
}
impl WebgraphBuilder {
@@ -249,16 +261,22 @@ impl WebgraphBuilder {
Self {
path: path.as_ref().into(),
executor: Executor::multi_thread("webgraph").unwrap(),
+ deduplication: Deduplication::default(),
}
}
+ pub fn deduplication(mut self, deduplication: Deduplication) -> Self {
+ self.deduplication = deduplication;
+ self
+ }
+
pub fn single_threaded(mut self) -> Self {
self.executor = Executor::single_thread();
self
}
pub fn open(self) -> Webgraph {
- Webgraph::open(self.path, self.executor)
+ Webgraph::open(self.path, self.executor, self.deduplication)
}
}
@@ -437,6 +455,7 @@ pub struct Webgraph {
insert_batch: Vec<Edge<String>>,
id2node: Id2NodeDb,
meta: Meta,
+ deduplication: Deduplication,
}
impl Webgraph {
@@ -471,16 +490,18 @@ impl Webgraph {
writer.write_all(json.as_bytes()).unwrap();
}
- fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {
+ fn open<P: AsRef<Path>>(path: P, executor: Executor, deduplication: Deduplication) -> Self {
fs::create_dir_all(&path).unwrap();
let mut meta = Self::meta(&path);
fs::create_dir_all(path.as_ref().join("segments")).unwrap();
+
let mut segments = Vec::new();
for segment in &meta.comitted_segments {
segments.push(StoredSegment::open(
path.as_ref().join("segments"),
segment.clone(),
+ deduplication,
));
}
@@ -488,6 +509,7 @@ impl Webgraph {
segments.push(StoredSegment::open(
path.as_ref().join("segments"),
uuid::Uuid::new_v4().to_string(),
+ deduplication,
));
meta.comitted_segments.push(segments[0].id());
@@ -495,6 +517,7 @@ impl Webgraph {
Self {
path: path.as_ref().as_os_str().to_str().unwrap().to_string(),
+ deduplication,
segments,
executor: Arc::new(executor),
id2node: Id2NodeDb::open(path.as_ref().join("id2node")),
@@ -541,7 +564,8 @@ impl Webgraph {
self.meta.comitted_segments.push(segment.id());
drop(segment);
- self.segments.push(StoredSegment::open(new_path, id));
+ self.segments
+ .push(StoredSegment::open(new_path, id, self.deduplication));
}
self.commit();
@@ -564,12 +588,18 @@ impl Webgraph {
}
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
+ let dedup = match self.deduplication {
+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {
+ |edges: &mut Vec<Edge<String>>| {
+ edges.sort_by_key(|e| e.from);
+ edges.dedup_by_key(|e| e.from);
+ }
+ }
+ };
+
self.inner_edges(
|segment| segment.ingoing_edges_with_label(&node.id()).collect_vec(),
- |edges| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
- },
+ dedup,
)
.into_iter()
.map(|e| FullEdge {
@@ -581,32 +611,47 @@ impl Webgraph {
}
pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
- self.inner_edges(
- |segment| segment.ingoing_edges(node).collect_vec(),
- |edges| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
- },
- )
+ let dedup = match self.deduplication {
+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {
+ |edges: &mut Vec<Edge<()>>| {
+ edges.sort_by_key(|e| e.from);
+ edges.dedup_by_key(|e| e.from);
+ }
+ }
+ };
+
+ self.inner_edges(|segment| segment.ingoing_edges(node).collect_vec(), dedup)
}
pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {
+ let dedup = match self.deduplication {
+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {
+ |edges: &mut Vec<Edge<String>>| {
+ edges.sort_by_key(|e| e.from);
+ edges.dedup_by_key(|e| e.from);
+ }
+ }
+ };
+
self.inner_edges(
|segment| segment.ingoing_edges_with_label(node).collect_vec(),
- |edges| {
- edges.sort_by_key(|e| e.from);
- edges.dedup_by_key(|e| e.from);
- },
+ dedup,
)
}
pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {
+ let dedup = match self.deduplication {
+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {
+ |edges: &mut Vec<Edge<String>>| {
+ edges.sort_by_key(|e| e.to);
+ edges.dedup_by_key(|e| e.to);
+ }
+ }
+ };
+
self.inner_edges(
|segment| segment.outgoing_edges_with_label(&node.id()).collect_vec(),
- |edges| {
- edges.sort_by_key(|e| e.to);
- edges.dedup_by_key(|e| e.to);
- },
+ dedup,
)
.into_iter()
.map(|e| FullEdge {
@@ -618,13 +663,16 @@ impl Webgraph {
}
pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {
- self.inner_edges(
- |segment| segment.outgoing_edges(node).collect_vec(),
- |edges| {
- edges.sort_by_key(|e| e.to);
- edges.dedup_by_key(|e| e.to);
- },
- )
+ let dedup = match self.deduplication {
+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {
+ |edges: &mut Vec<Edge<()>>| {
+ edges.sort_by_key(|e| e.to);
+ edges.dedup_by_key(|e| e.to);
+ }
+ }
+ };
+
+ self.inner_edges(|segment| segment.outgoing_edges(node).collect_vec(), dedup)
}
fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>
diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rs
index 17c0a1a1..93f32a0d 100644
--- a/core/src/webgraph/segment.rs
+++ b/core/src/webgraph/segment.rs
@@ -16,7 +16,7 @@
use std::path::{Path, PathBuf};
-use super::{store::EdgeStore, Edge, NodeID};
+use super::{store::EdgeStore, Deduplication, Edge, NodeID};
const ADJACENCY_STORE: &str = "adjacency";
const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";
@@ -29,11 +29,17 @@ pub struct StoredSegment {
}
impl StoredSegment {
- pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {
+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String, dedup: Deduplication) -> Self {
+ let dedup_insert = match dedup {
+ Deduplication::OnlyQuery => false,
+ Deduplication::QueryAndInserts => true,
+ };
+
StoredSegment {
full_adjacency: EdgeStore::open(
folder_path.as_ref().join(&id).join(ADJACENCY_STORE),
false,
+ dedup_insert,
),
full_reversed_adjacency: EdgeStore::open(
folder_path
@@ -41,6 +47,7 @@ impl StoredSegment {
.join(&id)
.join(REVERSED_ADJACENCY_STORE),
true,
+ dedup_insert,
),
folder_path: folder_path
.as_ref()
@@ -137,7 +144,11 @@ mod test {
// ▼ │ │
// 1─────►2◄┘
- let mut store = StoredSegment::open(crate::gen_temp_path(), "test".to_string());
+ let mut store = StoredSegment::open(
+ crate::gen_temp_path(),
+ "test".to_string(),
+ Deduplication::OnlyQuery,
+ );
let mut edges = Vec::new();
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 4c2a129e..11654651 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -25,11 +25,12 @@ const MAX_BATCH_SIZE: usize = 50_000;
pub struct EdgeStore {
reversed: bool,
+ dedup: bool,
db: rocksdb::DB,
}
impl EdgeStore {
- pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {
+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool, dedup: bool) -> Self {
let mut options = rocksdb::Options::default();
options.create_if_missing(true);
@@ -54,7 +55,11 @@ impl EdgeStore {
let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();
- Self { db, reversed }
+ Self {
+ db,
+ reversed,
+ dedup,
+ }
}
pub fn get<L: EdgeLabel>(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {
@@ -119,11 +124,13 @@ impl EdgeStore {
let key_bytes = [prefix_bytes, suffix_bytes].concat();
- if batch_keys.contains(&key_bytes) || self.db.get(&key_bytes).unwrap().is_some() {
- continue;
- }
+ if self.dedup {
+ if batch_keys.contains(&key_bytes) || self.db.get(&key_bytes).unwrap().is_some() {
+ continue;
+ }
- batch_keys.insert(key_bytes.clone());
+ batch_keys.insert(key_bytes.clone());
+ }
let value_bytes = edge.label.to_bytes().unwrap();
@@ -204,7 +211,8 @@ mod tests {
#[test]
fn test_insert() {
- let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), false);
+ let kv: EdgeStore =
+ EdgeStore::open(crate::gen_temp_path().join("test-segment"), false, false);
let e = Edge {
from: NodeID(0),
@@ -228,7 +236,8 @@ mod tests {
#[test]
fn test_reversed() {
- let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), true);
+ let kv: EdgeStore =
+ EdgeStore::open(crate::gen_temp_path().join("test-segment"), true, false);
let e = Edge {
from: NodeID(0),