Skip to content

web search done right

optionally construct only host graph or page graph

Mikkel Denker21d7342ecd3d
diff --git a/assets/licenses.html b/assets/licenses.html
index c42f1dda..d7c9b638 100644
--- a/assets/licenses.html
+++ b/assets/licenses.html
@@ -48,7 +48,7 @@
<li><a href="#MIT">MIT License</a> (186)</li>
<li><a href="#BSD-3-Clause">BSD 3-Clause &quot;New&quot; or &quot;Revised&quot; License</a> (9)</li>
<li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>
- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (7)</li>
+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (8)</li>
<li><a href="#ISC">ISC License</a> (4)</li>
<li><a href="#Unicode-3.0">Unicode License v3</a> (4)</li>
<li><a href="#BSD-2-Clause">BSD 2-Clause &quot;Simplified&quot; License</a> (3)</li>
@@ -69,6 +69,7 @@
<h4>Used by:</h4>
<ul class="license-used-by">
<li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li>
+ <li><a href=" https://crates.io/crates/common ">common 0.1.0</a></li>
<li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li>
<li><a href=" https://crates.io/crates/leechy ">leechy 0.1.0</a></li>
<li><a href=" https://crates.io/crates/lending-iter ">lending-iter 0.1.0</a></li>
diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rs
index 4a25a644..32db06f8 100644
--- a/crates/core/src/config/mod.rs
+++ b/crates/core/src/config/mod.rs
@@ -65,8 +65,8 @@ pub struct IndexerDualEncoderConfig {
#[derive(Debug, serde::Deserialize, Clone)]
pub struct WebgraphConstructConfig {
- pub host_graph_base_path: String,
- pub page_graph_base_path: String,
+ pub host_graph_base_path: Option<String>,
+ pub page_graph_base_path: Option<String>,
pub warc_source: WarcSource,
pub limit_warc_files: Option<usize>,
pub skip_warc_files: Option<usize>,
diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rs
index 36aaef4b..1c4025b2 100644
--- a/crates/core/src/entrypoint/configure.rs
+++ b/crates/core/src/entrypoint/configure.rs
@@ -120,15 +120,15 @@ fn create_webgraph() -> Result<()> {
};
let mut worker = webgraph::WebgraphWorker {
- host_graph: webgraph::open_host_graph_writer(&out_path_host, None),
- page_graph: webgraph::open_page_graph_writer(&out_path_page, None),
+ host_graph: Some(webgraph::open_host_graph_writer(&out_path_host, None)),
+ page_graph: Some(webgraph::open_page_graph_writer(&out_path_page, None)),
canonical_index: None,
};
worker.process_job(&job);
- worker.host_graph.finalize();
- worker.page_graph.finalize();
+ worker.host_graph.unwrap().finalize();
+ worker.page_graph.unwrap().finalize();
Ok(())
}
diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rs
index 946036ae..190f9aba 100644
--- a/crates/core/src/entrypoint/webgraph.rs
+++ b/crates/core/src/entrypoint/webgraph.rs
@@ -21,6 +21,7 @@ use crate::{
webpage::{url_ext::UrlExt, Html},
Result,
};
+use anyhow::bail;
use itertools::Itertools;
use url::Url;
@@ -94,8 +95,8 @@ fn canonical_or_self(index: &CanonicalIndex, url: Url) -> Url {
}
pub struct WebgraphWorker {
- pub host_graph: webgraph::WebgraphWriter,
- pub page_graph: webgraph::WebgraphWriter,
+ pub host_graph: Option<webgraph::WebgraphWriter>,
+ pub page_graph: Option<webgraph::WebgraphWriter>,
pub canonical_index: Option<Arc<CanonicalIndex>>,
}
@@ -141,12 +142,14 @@ impl WebgraphWorker {
let mut destination = Node::from(destination);
trace!("inserting link {:?}", link);
- self.page_graph.insert(
- source.clone(),
- destination.clone(),
- link.text.clone(),
- link.rel,
- );
+ if let Some(graph) = self.page_graph.as_mut() {
+ graph.insert(
+ source.clone(),
+ destination.clone(),
+ link.text.clone(),
+ link.rel,
+ )
+ }
let dest_domain = link.destination.root_domain();
let source_domain = link.source.root_domain();
@@ -157,14 +160,19 @@ impl WebgraphWorker {
source = source.into_host();
destination = destination.into_host();
- self.host_graph
- .insert(source, destination, link.text, link.rel);
+ if let Some(graph) = self.host_graph.as_mut() {
+ graph.insert(source, destination, link.text, link.rel)
+ }
}
}
}
- self.host_graph.commit();
- self.page_graph.commit();
+ if let Some(graph) = self.host_graph.as_mut() {
+ graph.commit()
+ }
+ if let Some(graph) = self.page_graph.as_mut() {
+ graph.commit()
+ }
}
info!("{} done", name);
@@ -175,6 +183,10 @@ pub struct Webgraph {}
impl Webgraph {
pub fn run(config: &WebgraphConstructConfig) -> Result<()> {
+ if config.page_graph_base_path.is_none() && config.host_graph_base_path.is_none() {
+ bail!("either page_graph_base_path or host_graph_base_path must be set");
+ }
+
let warc_paths = config.warc_source.paths()?;
let job_config = JobConfig::from(config.warc_source.clone());
@@ -219,16 +231,22 @@ impl Webgraph {
for i in 0..num_workers {
let host_path = host_path.clone();
- let host_path = Path::new(&host_path);
- let host_path = host_path.join(format!("worker_{i}"));
+ let host_path = host_path
+ .as_ref()
+ .map(|p| Path::new(p).join(format!("worker_{i}")));
let page_path = page_path.clone();
- let page_path = Path::new(&page_path);
- let page_path = page_path.join(format!("worker_{i}"));
+ let page_path = page_path
+ .as_ref()
+ .map(|p| Path::new(p).join(format!("worker_{i}")));
let mut worker = WebgraphWorker {
- host_graph: open_host_graph_writer(host_path, host_centrality_rank_store.clone()),
- page_graph: open_page_graph_writer(page_path, host_centrality_rank_store.clone()),
+ host_graph: host_path
+ .as_ref()
+ .map(|p| open_host_graph_writer(p, host_centrality_rank_store.clone())),
+ page_graph: page_path
+ .as_ref()
+ .map(|p| open_page_graph_writer(p, host_centrality_rank_store.clone())),
canonical_index: canonical_index.clone(),
};
@@ -241,8 +259,8 @@ impl Webgraph {
r.recv().unwrap();
- let host = worker.host_graph.finalize();
- let page = worker.page_graph.finalize();
+ let host = worker.host_graph.map(|graph| graph.finalize());
+ let page = worker.page_graph.map(|graph| graph.finalize());
s.send(()).unwrap();
(host, page)
@@ -257,22 +275,36 @@ impl Webgraph {
let (mut host_graph, mut page_graph) = graphs.pop().unwrap();
for (other_host, other_page) in graphs {
- host_graph.merge(other_host)?;
- page_graph.merge(other_page)?;
+ if let (Some(graph), Some(other)) = (host_graph.as_mut(), other_host) {
+ graph.merge(other)?;
+ }
+
+ if let (Some(graph), Some(other)) = (page_graph.as_mut(), other_page) {
+ graph.merge(other)?;
+ }
}
if config.merge_all_segments {
- host_graph.optimize_read(); // save space in id2node db
- page_graph.optimize_read(); // save space in id2node db
- host_graph.merge_all_segments(Default::default())?;
- page_graph.merge_all_segments(Default::default())?;
+ if let Some(host) = host_graph.as_mut() {
+ host.optimize_read(); // save space in id2node db
+ host.merge_all_segments(Default::default())?;
+ }
+
+ if let Some(page) = page_graph.as_mut() {
+ page.optimize_read(); // save space in id2node db
+ page.merge_all_segments(Default::default())?;
+ }
}
- host_graph.optimize_read();
- page_graph.optimize_read();
+ if let Some(host) = host_graph.as_mut() {
+ host.optimize_read();
+ crate::mv(host.path(), config.host_graph_base_path.as_ref().unwrap())?;
+ }
- crate::mv(host_graph.path(), &config.host_graph_base_path)?;
- crate::mv(page_graph.path(), &config.page_graph_base_path)?;
+ if let Some(page) = page_graph.as_mut() {
+ page.optimize_read();
+ crate::mv(page.path(), config.page_graph_base_path.as_ref().unwrap())?;
+ }
Ok(())
}
diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rs
index d39f28ae..f862182c 100644
--- a/crates/core/src/webgraph/merge.rs
+++ b/crates/core/src/webgraph/merge.rs
@@ -104,18 +104,18 @@ pub struct MergeIter<'a> {
}
impl<'a> MergeIter<'a> {
pub fn new(iters: Vec<impl Iterator<Item = MergeNode> + 'a>) -> Self {
- let mut heap = MinHeap::new();
-
- for item in iters.into_iter().enumerate().map(|(ord, iter)| {
- let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))
- as Box<dyn Iterator<Item = _>>;
-
- Reverse(file_store::Peekable::new(it))
- }) {
- heap.push(item);
- }
-
- Self { iters: heap }
+ let iters = iters
+ .into_iter()
+ .enumerate()
+ .map(|(ord, iter)| {
+ let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))
+ as Box<dyn Iterator<Item = _>>;
+
+ Reverse(file_store::Peekable::new(it))
+ })
+ .collect();
+
+ Self { iters }
}
pub fn advance(&mut self, buf: &mut Vec<MergeNode<MergeSegmentOrd>>) -> bool {
@@ -160,15 +160,15 @@ pub struct EdgeMerger<'a, L = String> {
impl<'a, L> EdgeMerger<'a, L> {
pub fn new(iters: Vec<impl Iterator<Item = StoredEdge<L>> + 'a>) -> Self {
- let mut heap = MinHeap::new();
- for item in iters.into_iter().map(|iter| {
- let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;
- Reverse(file_store::Peekable::new(it))
- }) {
- heap.push(item);
- }
-
- Self { iters: heap }
+ let iters = iters
+ .into_iter()
+ .map(|iter| {
+ let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;
+ Reverse(file_store::Peekable::new(it))
+ })
+ .collect();
+
+ Self { iters }
}
}