web search done right
optionally construct only host graph or page graph
Mikkel Denker21d7342ecd3d
diff --git a/assets/licenses.html b/assets/licenses.htmlindex c42f1dda..d7c9b638 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -48,7 +48,7 @@<li><a href="#MIT">MIT License</a> (186)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (9)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li>- <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (7)</li>+ <li><a href="#AGPL-3.0">GNU Affero General Public License v3.0</a> (8)</li><li><a href="#ISC">ISC License</a> (4)</li><li><a href="#Unicode-3.0">Unicode License v3</a> (4)</li><li><a href="#BSD-2-Clause">BSD 2-Clause "Simplified" License</a> (3)</li>@@ -69,6 +69,7 @@<h4>Used by:</h4><ul class="license-used-by"><li><a href=" https://crates.io/crates/bloom ">bloom 0.1.0</a></li>+ <li><a href=" https://crates.io/crates/common ">common 0.1.0</a></li><li><a href=" https://crates.io/crates/file_store ">file_store 0.1.0</a></li><li><a href=" https://crates.io/crates/leechy ">leechy 0.1.0</a></li><li><a href=" https://crates.io/crates/lending-iter ">lending-iter 0.1.0</a></li>diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 4a25a644..32db06f8 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -65,8 +65,8 @@ pub struct IndexerDualEncoderConfig {#[derive(Debug, serde::Deserialize, Clone)]pub struct WebgraphConstructConfig {- pub host_graph_base_path: String,- pub page_graph_base_path: String,+ pub host_graph_base_path: Option<String>,+ pub page_graph_base_path: Option<String>,pub warc_source: WarcSource,pub limit_warc_files: Option<usize>,pub skip_warc_files: Option<usize>,diff --git a/crates/core/src/entrypoint/configure.rs b/crates/core/src/entrypoint/configure.rsindex 36aaef4b..1c4025b2 100644--- a/crates/core/src/entrypoint/configure.rs+++ b/crates/core/src/entrypoint/configure.rs@@ -120,15 +120,15 @@ fn create_webgraph() -> Result<()> {};let mut worker = webgraph::WebgraphWorker {- host_graph: webgraph::open_host_graph_writer(&out_path_host, None),- page_graph: webgraph::open_page_graph_writer(&out_path_page, None),+ host_graph: Some(webgraph::open_host_graph_writer(&out_path_host, None)),+ page_graph: Some(webgraph::open_page_graph_writer(&out_path_page, None)),canonical_index: None,};worker.process_job(&job);- worker.host_graph.finalize();- worker.page_graph.finalize();+ worker.host_graph.unwrap().finalize();+ worker.page_graph.unwrap().finalize();Ok(())}diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 946036ae..190f9aba 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -21,6 +21,7 @@ use crate::{webpage::{url_ext::UrlExt, Html},Result,};+use anyhow::bail;use itertools::Itertools;use url::Url;@@ -94,8 +95,8 @@ fn canonical_or_self(index: &CanonicalIndex, url: Url) -> Url {}pub struct WebgraphWorker {- pub host_graph: webgraph::WebgraphWriter,- pub page_graph: webgraph::WebgraphWriter,+ pub host_graph: Option<webgraph::WebgraphWriter>,+ pub page_graph: Option<webgraph::WebgraphWriter>,pub canonical_index: Option<Arc<CanonicalIndex>>,}@@ -141,12 +142,14 @@ impl WebgraphWorker {let mut destination = Node::from(destination);trace!("inserting link {:?}", link);- self.page_graph.insert(- source.clone(),- destination.clone(),- link.text.clone(),- link.rel,- );+ if let Some(graph) = self.page_graph.as_mut() {+ graph.insert(+ source.clone(),+ destination.clone(),+ link.text.clone(),+ link.rel,+ )+ }let dest_domain = link.destination.root_domain();let source_domain = link.source.root_domain();@@ -157,14 +160,19 @@ impl WebgraphWorker {source = source.into_host();destination = destination.into_host();- self.host_graph- .insert(source, destination, link.text, link.rel);+ if let Some(graph) = self.host_graph.as_mut() {+ graph.insert(source, destination, link.text, link.rel)+ }}}}- self.host_graph.commit();- self.page_graph.commit();+ if let Some(graph) = self.host_graph.as_mut() {+ graph.commit()+ }+ if let Some(graph) = self.page_graph.as_mut() {+ graph.commit()+ }}info!("{} done", name);@@ -175,6 +183,10 @@ pub struct Webgraph {}impl Webgraph {pub fn run(config: &WebgraphConstructConfig) -> Result<()> {+ if config.page_graph_base_path.is_none() && config.host_graph_base_path.is_none() {+ bail!("either page_graph_base_path or host_graph_base_path must be set");+ }+let warc_paths = config.warc_source.paths()?;let job_config = JobConfig::from(config.warc_source.clone());@@ -219,16 +231,22 @@ impl Webgraph {for i in 0..num_workers {let host_path = host_path.clone();- let host_path = Path::new(&host_path);- let host_path = host_path.join(format!("worker_{i}"));+ let host_path = host_path+ .as_ref()+ .map(|p| Path::new(p).join(format!("worker_{i}")));let page_path = page_path.clone();- let page_path = Path::new(&page_path);- let page_path = page_path.join(format!("worker_{i}"));+ let page_path = page_path+ .as_ref()+ .map(|p| Path::new(p).join(format!("worker_{i}")));let mut worker = WebgraphWorker {- host_graph: open_host_graph_writer(host_path, host_centrality_rank_store.clone()),- page_graph: open_page_graph_writer(page_path, host_centrality_rank_store.clone()),+ host_graph: host_path+ .as_ref()+ .map(|p| open_host_graph_writer(p, host_centrality_rank_store.clone())),+ page_graph: page_path+ .as_ref()+ .map(|p| open_page_graph_writer(p, host_centrality_rank_store.clone())),canonical_index: canonical_index.clone(),};@@ -241,8 +259,8 @@ impl Webgraph {r.recv().unwrap();- let host = worker.host_graph.finalize();- let page = worker.page_graph.finalize();+ let host = worker.host_graph.map(|graph| graph.finalize());+ let page = worker.page_graph.map(|graph| graph.finalize());s.send(()).unwrap();(host, page)@@ -257,22 +275,36 @@ impl Webgraph {let (mut host_graph, mut page_graph) = graphs.pop().unwrap();for (other_host, other_page) in graphs {- host_graph.merge(other_host)?;- page_graph.merge(other_page)?;+ if let (Some(graph), Some(other)) = (host_graph.as_mut(), other_host) {+ graph.merge(other)?;+ }++ if let (Some(graph), Some(other)) = (page_graph.as_mut(), other_page) {+ graph.merge(other)?;+ }}if config.merge_all_segments {- host_graph.optimize_read(); // save space in id2node db- page_graph.optimize_read(); // save space in id2node db- host_graph.merge_all_segments(Default::default())?;- page_graph.merge_all_segments(Default::default())?;+ if let Some(host) = host_graph.as_mut() {+ host.optimize_read(); // save space in id2node db+ host.merge_all_segments(Default::default())?;+ }++ if let Some(page) = page_graph.as_mut() {+ page.optimize_read(); // save space in id2node db+ page.merge_all_segments(Default::default())?;+ }}- host_graph.optimize_read();- page_graph.optimize_read();+ if let Some(host) = host_graph.as_mut() {+ host.optimize_read();+ crate::mv(host.path(), config.host_graph_base_path.as_ref().unwrap())?;+ }- crate::mv(host_graph.path(), &config.host_graph_base_path)?;- crate::mv(page_graph.path(), &config.page_graph_base_path)?;+ if let Some(page) = page_graph.as_mut() {+ page.optimize_read();+ crate::mv(page.path(), config.page_graph_base_path.as_ref().unwrap())?;+ }Ok(())}diff --git a/crates/core/src/webgraph/merge.rs b/crates/core/src/webgraph/merge.rsindex d39f28ae..f862182c 100644--- a/crates/core/src/webgraph/merge.rs+++ b/crates/core/src/webgraph/merge.rs@@ -104,18 +104,18 @@ pub struct MergeIter<'a> {}impl<'a> MergeIter<'a> {pub fn new(iters: Vec<impl Iterator<Item = MergeNode> + 'a>) -> Self {- let mut heap = MinHeap::new();-- for item in iters.into_iter().enumerate().map(|(ord, iter)| {- let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))- as Box<dyn Iterator<Item = _>>;-- Reverse(file_store::Peekable::new(it))- }) {- heap.push(item);- }-- Self { iters: heap }+ let iters = iters+ .into_iter()+ .enumerate()+ .map(|(ord, iter)| {+ let it = Box::new(iter.map(move |node| node.with_ord(MergeSegmentOrd::new(ord))))+ as Box<dyn Iterator<Item = _>>;++ Reverse(file_store::Peekable::new(it))+ })+ .collect();++ Self { iters }}pub fn advance(&mut self, buf: &mut Vec<MergeNode<MergeSegmentOrd>>) -> bool {@@ -160,15 +160,15 @@ pub struct EdgeMerger<'a, L = String> {impl<'a, L> EdgeMerger<'a, L> {pub fn new(iters: Vec<impl Iterator<Item = StoredEdge<L>> + 'a>) -> Self {- let mut heap = MinHeap::new();- for item in iters.into_iter().map(|iter| {- let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;- Reverse(file_store::Peekable::new(it))- }) {- heap.push(item);- }-- Self { iters: heap }+ let iters = iters+ .into_iter()+ .map(|iter| {+ let it = Box::new(iter) as Box<dyn Iterator<Item = _>>;+ Reverse(file_store::Peekable::new(it))+ })+ .collect();++ Self { iters }}}