web search done right
ability to turn off deduplication for page webgraph inserts
Mikkel Denker30e9f12ddc60
diff --git a/core/src/entrypoint/configure.rs b/core/src/entrypoint/configure.rsindex 1ecee632..66933a52 100644--- a/core/src/entrypoint/configure.rs+++ b/core/src/entrypoint/configure.rs@@ -84,8 +84,8 @@ fn create_webgraph() -> Result<()> {let mut worker = webgraph::WebgraphWorker {redirect: None,- host_graph: webgraph::open_graph(&out_path_host),- page_graph: webgraph::open_graph(&out_path_page),+ host_graph: webgraph::open_host_graph(&out_path_host),+ page_graph: webgraph::open_page_graph(&out_path_page),};worker.process_job(&job);diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 1456e05d..b6cf6d24 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -67,10 +67,16 @@ pub struct Job {pub warc_paths: Vec<String>,}-pub fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {+pub fn open_host_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {WebgraphBuilder::new(path).open()}+pub fn open_page_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {+ WebgraphBuilder::new(path)+ .deduplication(webgraph::Deduplication::OnlyQuery)+ .open()+}+pub struct WebgraphWorker {pub host_graph: webgraph::Webgraph,pub page_graph: webgraph::Webgraph,@@ -191,8 +197,8 @@ impl Webgraph {let mut worker = WebgraphWorker {redirect: redirect.clone(),- host_graph: open_graph(host_path),- page_graph: open_graph(page_path),+ host_graph: open_host_graph(host_path),+ page_graph: open_page_graph(page_path),};let jobs = jobs.clone();diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 7fa72b00..2cea3b83 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -231,9 +231,21 @@ pub fn normalize_url(url: &Url) -> String {normalized}+#[derive(Default, Clone, Copy)]+pub enum Deduplication {+ /// Do not deduplicate the inserted edges in the segment. Only deduplicate+ /// the edges when querying all the segments.+ OnlyQuery,+ /// Deduplicate the inserted edges in the segmnt by first checking if the+ /// edge is already present in the segment.+ #[default]+ QueryAndInserts,+}+pub struct WebgraphBuilder {path: Box<Path>,executor: Executor,+ deduplication: Deduplication,}impl WebgraphBuilder {@@ -249,16 +261,22 @@ impl WebgraphBuilder {Self {path: path.as_ref().into(),executor: Executor::multi_thread("webgraph").unwrap(),+ deduplication: Deduplication::default(),}}+ pub fn deduplication(mut self, deduplication: Deduplication) -> Self {+ self.deduplication = deduplication;+ self+ }+pub fn single_threaded(mut self) -> Self {self.executor = Executor::single_thread();self}pub fn open(self) -> Webgraph {- Webgraph::open(self.path, self.executor)+ Webgraph::open(self.path, self.executor, self.deduplication)}}@@ -437,6 +455,7 @@ pub struct Webgraph {insert_batch: Vec<Edge<String>>,id2node: Id2NodeDb,meta: Meta,+ deduplication: Deduplication,}impl Webgraph {@@ -471,16 +490,18 @@ impl Webgraph {writer.write_all(json.as_bytes()).unwrap();}- fn open<P: AsRef<Path>>(path: P, executor: Executor) -> Self {+ fn open<P: AsRef<Path>>(path: P, executor: Executor, deduplication: Deduplication) -> Self {fs::create_dir_all(&path).unwrap();let mut meta = Self::meta(&path);fs::create_dir_all(path.as_ref().join("segments")).unwrap();+let mut segments = Vec::new();for segment in &meta.comitted_segments {segments.push(StoredSegment::open(path.as_ref().join("segments"),segment.clone(),+ deduplication,));}@@ -488,6 +509,7 @@ impl Webgraph {segments.push(StoredSegment::open(path.as_ref().join("segments"),uuid::Uuid::new_v4().to_string(),+ deduplication,));meta.comitted_segments.push(segments[0].id());@@ -495,6 +517,7 @@ impl Webgraph {Self {path: path.as_ref().as_os_str().to_str().unwrap().to_string(),+ deduplication,segments,executor: Arc::new(executor),id2node: Id2NodeDb::open(path.as_ref().join("id2node")),@@ -541,7 +564,8 @@ impl Webgraph {self.meta.comitted_segments.push(segment.id());drop(segment);- self.segments.push(StoredSegment::open(new_path, id));+ self.segments+ .push(StoredSegment::open(new_path, id, self.deduplication));}self.commit();@@ -564,12 +588,18 @@ impl Webgraph {}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {+ let dedup = match self.deduplication {+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {+ |edges: &mut Vec<Edge<String>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);+ }+ }+ };+self.inner_edges(|segment| segment.ingoing_edges_with_label(&node.id()).collect_vec(),- |edges| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);- },+ dedup,).into_iter().map(|e| FullEdge {@@ -581,32 +611,47 @@ impl Webgraph {}pub fn raw_ingoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {- self.inner_edges(- |segment| segment.ingoing_edges(node).collect_vec(),- |edges| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);- },- )+ let dedup = match self.deduplication {+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {+ |edges: &mut Vec<Edge<()>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);+ }+ }+ };++ self.inner_edges(|segment| segment.ingoing_edges(node).collect_vec(), dedup)}pub fn raw_ingoing_edges_with_labels(&self, node: &NodeID) -> Vec<Edge<String>> {+ let dedup = match self.deduplication {+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {+ |edges: &mut Vec<Edge<String>>| {+ edges.sort_by_key(|e| e.from);+ edges.dedup_by_key(|e| e.from);+ }+ }+ };+self.inner_edges(|segment| segment.ingoing_edges_with_label(node).collect_vec(),- |edges| {- edges.sort_by_key(|e| e.from);- edges.dedup_by_key(|e| e.from);- },+ dedup,)}pub fn outgoing_edges(&self, node: Node) -> Vec<FullEdge> {+ let dedup = match self.deduplication {+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {+ |edges: &mut Vec<Edge<String>>| {+ edges.sort_by_key(|e| e.to);+ edges.dedup_by_key(|e| e.to);+ }+ }+ };+self.inner_edges(|segment| segment.outgoing_edges_with_label(&node.id()).collect_vec(),- |edges| {- edges.sort_by_key(|e| e.to);- edges.dedup_by_key(|e| e.to);- },+ dedup,).into_iter().map(|e| FullEdge {@@ -618,13 +663,16 @@ impl Webgraph {}pub fn raw_outgoing_edges(&self, node: &NodeID) -> Vec<Edge<()>> {- self.inner_edges(- |segment| segment.outgoing_edges(node).collect_vec(),- |edges| {- edges.sort_by_key(|e| e.to);- edges.dedup_by_key(|e| e.to);- },- )+ let dedup = match self.deduplication {+ Deduplication::OnlyQuery | Deduplication::QueryAndInserts => {+ |edges: &mut Vec<Edge<()>>| {+ edges.sort_by_key(|e| e.to);+ edges.dedup_by_key(|e| e.to);+ }+ }+ };++ self.inner_edges(|segment| segment.outgoing_edges(node).collect_vec(), dedup)}fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<Edge<L>>diff --git a/core/src/webgraph/segment.rs b/core/src/webgraph/segment.rsindex 17c0a1a1..93f32a0d 100644--- a/core/src/webgraph/segment.rs+++ b/core/src/webgraph/segment.rs@@ -16,7 +16,7 @@use std::path::{Path, PathBuf};-use super::{store::EdgeStore, Edge, NodeID};+use super::{store::EdgeStore, Deduplication, Edge, NodeID};const ADJACENCY_STORE: &str = "adjacency";const REVERSED_ADJACENCY_STORE: &str = "reversed_adjacency";@@ -29,11 +29,17 @@ pub struct StoredSegment {}impl StoredSegment {- pub fn open<P: AsRef<Path>>(folder_path: P, id: String) -> Self {+ pub fn open<P: AsRef<Path>>(folder_path: P, id: String, dedup: Deduplication) -> Self {+ let dedup_insert = match dedup {+ Deduplication::OnlyQuery => false,+ Deduplication::QueryAndInserts => true,+ };+StoredSegment {full_adjacency: EdgeStore::open(folder_path.as_ref().join(&id).join(ADJACENCY_STORE),false,+ dedup_insert,),full_reversed_adjacency: EdgeStore::open(folder_path@@ -41,6 +47,7 @@ impl StoredSegment {.join(&id).join(REVERSED_ADJACENCY_STORE),true,+ dedup_insert,),folder_path: folder_path.as_ref()@@ -137,7 +144,11 @@ mod test {// ▼ │ │// 1─────►2◄┘- let mut store = StoredSegment::open(crate::gen_temp_path(), "test".to_string());+ let mut store = StoredSegment::open(+ crate::gen_temp_path(),+ "test".to_string(),+ Deduplication::OnlyQuery,+ );let mut edges = Vec::new();diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rsindex 4c2a129e..11654651 100644--- a/core/src/webgraph/store.rs+++ b/core/src/webgraph/store.rs@@ -25,11 +25,12 @@ const MAX_BATCH_SIZE: usize = 50_000;pub struct EdgeStore {reversed: bool,+ dedup: bool,db: rocksdb::DB,}impl EdgeStore {- pub fn open<P: AsRef<Path>>(path: P, reversed: bool) -> Self {+ pub fn open<P: AsRef<Path>>(path: P, reversed: bool, dedup: bool) -> Self {let mut options = rocksdb::Options::default();options.create_if_missing(true);@@ -54,7 +55,11 @@ impl EdgeStore {let db = rocksdb::DB::open(&options, path.as_ref().to_str().unwrap()).unwrap();- Self { db, reversed }+ Self {+ db,+ reversed,+ dedup,+ }}pub fn get<L: EdgeLabel>(&self, node: NodeID) -> impl Iterator<Item = Edge<L>> + '_ {@@ -119,11 +124,13 @@ impl EdgeStore {let key_bytes = [prefix_bytes, suffix_bytes].concat();- if batch_keys.contains(&key_bytes) || self.db.get(&key_bytes).unwrap().is_some() {- continue;- }+ if self.dedup {+ if batch_keys.contains(&key_bytes) || self.db.get(&key_bytes).unwrap().is_some() {+ continue;+ }- batch_keys.insert(key_bytes.clone());+ batch_keys.insert(key_bytes.clone());+ }let value_bytes = edge.label.to_bytes().unwrap();@@ -204,7 +211,8 @@ mod tests {#[test]fn test_insert() {- let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), false);+ let kv: EdgeStore =+ EdgeStore::open(crate::gen_temp_path().join("test-segment"), false, false);let e = Edge {from: NodeID(0),@@ -228,7 +236,8 @@ mod tests {#[test]fn test_reversed() {- let kv: EdgeStore = EdgeStore::open(crate::gen_temp_path().join("test-segment"), true);+ let kv: EdgeStore =+ EdgeStore::open(crate::gen_temp_path().join("test-segment"), true, false);let e = Edge {from: NodeID(0),