web search done right
ability to reduce into different type in mapreduce
Mikkel Denker6e959f75f0b2
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rsindex 7823c3c7..37fb6d26 100644--- a/examples/mapreduce_manager.rs+++ b/examples/mapreduce_manager.rs@@ -20,7 +20,7 @@ impl Map<Count> for Job {}}-impl Reduce for Count {+impl Reduce<Count> for Count {fn reduce(self, element: Self) -> Self {Count(self.0 + element.0)}@@ -41,7 +41,7 @@ fn main() {Job { id: 5 },];- let res = jobs+ let res: Count = jobs.into_iter().map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()]).unwrap();diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rsindex 3dafd660..15c25a09 100644--- a/examples/mapreduce_worker.rs+++ b/examples/mapreduce_worker.rs@@ -20,7 +20,7 @@ impl Map<Count> for Job {}}-impl Reduce for Count {+impl Reduce<Count> for Count {fn reduce(self, element: Self) -> Self {Count(self.0 + element.0)}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 71258cf6..a6c9b62e 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -83,10 +83,8 @@ impl Map<FrozenWebgraph> for Job {}}-impl Reduce<FrozenWebgraph> for FrozenWebgraph {- fn reduce(self, other: FrozenWebgraph) -> FrozenWebgraph {- let mut graph: Webgraph = self.into();-+impl Reduce<FrozenWebgraph> for Webgraph {+ fn reduce(mut self, other: FrozenWebgraph) -> Webgraph {let other_path = match &other.root {crate::directory::DirEntry::Folder { name, entries: _ } => name.clone(),crate::directory::DirEntry::File { name, content: _ } => name.clone(),@@ -94,11 +92,18 @@ impl Reduce<FrozenWebgraph> for FrozenWebgraph {let other = other.into();- graph.merge(other);+ self.merge(other);std::fs::remove_dir_all(other_path).unwrap();- graph.into()+ self+ }+}++impl Reduce<Webgraph> for Webgraph {+ fn reduce(mut self, element: Webgraph) -> Self {+ self.merge(element);+ self}}@@ -147,7 +152,7 @@ impl WebgraphEntrypoint {warc_paths = Box::new(warc_paths.take(limit));}- warc_paths+ let _graph: Webgraph = warc_paths.map_reduce(&workers).expect("failed to build webgraph");@@ -180,10 +185,13 @@ impl WebgraphEntrypoint {graph_base_path: "webgraph".to_string(),}).map(|job| job.map())- .fold(None, |acc: Option<FrozenWebgraph>, elem| match acc {- Some(acc) => Some(acc.reduce(elem)),- None => Some(elem),- });+ .fold(+ None,+ |acc: Option<Webgraph>, elem: FrozenWebgraph| match acc {+ Some(acc) => Some(acc.reduce(elem)),+ None => Some(elem.into()),+ },+ );Ok(())}diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 888ed7a0..9cb4372d 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -5,6 +5,8 @@ use crate::mapreduce::Task;use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle};use rayon::iter::ParallelBridge;use rayon::prelude::*;+use serde::de::DeserializeOwned;+use serde::Serialize;use std::io::{Read, Write};use std::net::ToSocketAddrs;use std::net::{SocketAddr, TcpStream};@@ -44,7 +46,7 @@ impl RemoteWorker {fn perform<I, O>(&self, job: &I) -> Result<O>whereI: Map<O> + Send,- O: Reduce<O> + Send,+ O: Serialize + DeserializeOwned + Send,{let mut stream = self.connect()?;let serialized_job = bincode::serialize(&Task::Job(job))?;@@ -78,7 +80,7 @@ impl RemoteWorker {fn stop<I, O>(&self) -> Result<()>whereI: Map<O> + Send,- O: Reduce<O> + Send,+ O: Serialize + DeserializeOwned + Send,{debug!("closing worker {:}", self.addr);let mut stream = self.connect()?;@@ -175,7 +177,7 @@ impl WorkerPool {fn stop_workers<I, O>(&self)whereI: Map<O> + Send,- O: Reduce<O> + Send,+ O: Serialize + DeserializeOwned + Send,{let mut failing_workers = Vec::new();for worker in &self.all_workers {@@ -210,7 +212,7 @@ impl Manager {fn try_map<I, O>(&self, job: &I) -> Result<O>whereI: Map<O> + Send,- O: Reduce<O> + Send,+ O: Serialize + DeserializeOwned + Send,{loop {match self.pool.get_worker()? {@@ -230,7 +232,7 @@ impl Manager {pub fn map<I, O>(&self, job: I) -> OwhereI: Map<O> + Send,- O: Reduce<O> + Send,+ O: Serialize + DeserializeOwned + Send,{loop {match self.try_map(&job) {@@ -243,19 +245,34 @@ impl Manager {}}- fn reduce<O: Reduce<O>>(acc: Option<O>, elem: O) -> O {+ fn reduce<O1, O2>(acc: Option<O2>, elem: O1) -> O2+ where+ O1: Serialize + DeserializeOwned + Send,+ O2: From<O1> + Reduce<O1> + Send,+ {match acc {Some(acc) => acc.reduce(elem),- None => elem,+ None => elem.into(),}}- fn get_results<I, O>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O>+ fn reduce_end<O>(acc: Option<O>, elem: O) -> Owhere- I: Map<O> + Send,O: Reduce<O> + Send,{- let acc = Arc::new(Mutex::new(None));+ match acc {+ Some(acc) => acc.reduce(elem),+ None => elem,+ }+ }++ fn get_results<I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>+ where+ I: Map<O1> + Send,+ O1: Serialize + DeserializeOwned + Send,+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,+ {+ let acc: Arc<Mutex<Option<O2>>> = Arc::new(Mutex::new(None));let size = jobs.size_hint();@@ -270,24 +287,30 @@ impl Manager {.progress_chars("#>-"),);jobs.par_bridge()- .map(|job| self.map::<I, O>(job))+ .map(|job| self.map::<I, O1>(job)).progress_with(pb)- .fold(|| None, |acc, elem| Some(Manager::reduce(acc, elem)))+ .fold(+ || None,+ |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),+ ).for_each(|res| {if let Some(res) = res {let mut lock = acc.lock().unwrap();- *lock = Some(Manager::reduce(lock.take(), res));+ *lock = Some(Manager::reduce_end(lock.take(), res));}});}None => {jobs.par_bridge()- .map(|job| self.map::<I, O>(job))- .fold(|| None, |acc, elem| Some(Manager::reduce(acc, elem)))+ .map(|job| self.map::<I, O1>(job))+ .fold(+ || None,+ |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),+ ).for_each(|res| {if let Some(res) = res {let mut lock = acc.lock().unwrap();- *lock = Some(Manager::reduce(lock.take(), res));+ *lock = Some(Manager::reduce_end(lock.take(), res));}});}@@ -297,13 +320,14 @@ impl Manager {x}- pub fn run<I, O>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O>+ pub fn run<I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>where- I: Map<O> + Send,- O: Reduce<O> + Send,+ I: Map<O1> + Send,+ O1: Serialize + DeserializeOwned + Send,+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{let result = self.get_results(jobs);- self.pool.stop_workers::<I, O>();+ self.pool.stop_workers::<I, O1>();result}diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex fa8c2344..87d56a1b 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -52,29 +52,28 @@ wherefn map(self) -> T;}-pub trait Reduce<T = Self>-where- Self: Serialize + DeserializeOwned,-{- fn reduce(self, element: T) -> T;+pub trait Reduce<T> {+ fn reduce(self, element: T) -> Self;}-pub trait MapReduce<I, O>+pub trait MapReduce<I, O1, O2>whereSelf: Sized + Iterator<Item = I> + Send,- I: Map<O>,- O: Reduce<O> + Send,+ I: Map<O1>,+ O1: Serialize + DeserializeOwned + Send,+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{- fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {+ fn map_reduce(self, workers: &[SocketAddr]) -> Option<O2> {let manager = Manager::new(workers);manager.run(self)}}-impl<I, O, T> MapReduce<I, O> for T+impl<I, O1, O2, T> MapReduce<I, O1, O2> for TwhereT: Iterator<Item = I> + Sized + Send,- I: Map<O>,- O: Reduce<O> + Send,+ I: Map<O1>,+ O1: Serialize + DeserializeOwned + Send,+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,{}