web search done right
merge webgraph sequentially
Mikkel Denker66ab5248aeac
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 0494397f..c00b55fd 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- directory::DirEntry,entrypoint::async_download_all_warc_files,mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},warc::WarcFile,@@ -128,20 +127,9 @@ impl Map<StatelessWorker, GraphPointer> for Job {}impl Reduce<FrozenWebgraph> for webgraph::Webgraph {- fn reduce(mut self, other: FrozenWebgraph) -> webgraph::Webgraph {- let other_path = match &other.root {- DirEntry::Folder { name, entries: _ } | DirEntry::File { name, content: _ } => {- name.clone()- }- };-- let other = other.into();-- self.merge(other);-- std::fs::remove_dir_all(other_path).unwrap();-- self+ fn reduce(self, other: FrozenWebgraph) -> webgraph::Webgraph {+ let other: webgraph::Webgraph = other.into();+ self.reduce(other)}}@@ -158,22 +146,26 @@ impl Reduce<webgraph::Webgraph> for webgraph::Webgraph {impl Reduce<GraphPointer> for GraphPointer {fn reduce(self, other: GraphPointer) -> Self {- let other_path = other.0.clone();let self_path = self.0.clone();{- let mut graph = open_graph(self.0);+ let graph = open_graph(self.0);let other_graph = open_graph(other.0);- graph.merge(other_graph);+ let _ = graph.reduce(other_graph);}- std::fs::remove_dir_all(other_path).unwrap();-GraphPointer(self_path)}}+impl Reduce<GraphPointer> for webgraph::Webgraph {+ fn reduce(self, other: GraphPointer) -> Self {+ let other = open_graph(other.0);+ self.reduce(other)+ }+}+pub struct Webgraph {}impl Webgraph {@@ -258,7 +250,7 @@ impl Webgraph {};let worker = StatelessWorker::default();- warc_paths+ let mut graphs: Vec<_> = warc_paths.into_iter().take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1))@@ -274,15 +266,14 @@ impl Webgraph {.collect_vec().into_par_iter().map(|job| -> GraphPointer { job.map(&worker) })- .map(Some)- .reduce(- || None,- |a, b| match (a, b) {- (Some(a), Some(b)) => Some(a.reduce(b)),- (Some(graph), None) | (None, Some(graph)) => Some(graph),- (None, None) => None,- },- );+ .collect();++ if graphs.len() > 1 {+ let mut graph = open_graph(graphs.pop().unwrap().0);+ for other in graphs {+ graph = graph.reduce(other);+ }+ }Ok(())}