web search done right
mapreduce types doesn't have to be sync
Mikkel Denkerda66ffa2ca6f
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rsindex e7d2cc1c..7823c3c7 100644--- a/examples/mapreduce_manager.rs+++ b/examples/mapreduce_manager.rs@@ -2,7 +2,7 @@ use std::net::SocketAddr;use cuely::mapreduce::{Map, MapReduce, Reduce};use serde::{Deserialize, Serialize};-use tracing::{debug, Level};+use tracing::Level;use tracing_subscriber::FmtSubscriber;#[derive(Serialize, Deserialize, Debug)]@@ -15,18 +15,13 @@ struct Count(usize);impl Map<Count> for Job {fn map(self) -> Count {- debug!("begin map");std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task- debug!("end map");Count(1)}}impl Reduce for Count {fn reduce(self, element: Self) -> Self {- debug!("begin reduce");- std::thread::sleep(std::time::Duration::from_secs(20));- debug!("end reduce");Count(self.0 + element.0)}}diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rsindex d81df6f4..3dafd660 100644--- a/examples/mapreduce_worker.rs+++ b/examples/mapreduce_worker.rs@@ -2,7 +2,7 @@ use std::net::SocketAddr;use cuely::mapreduce::{Map, Reduce, Worker};use serde::{Deserialize, Serialize};-use tracing::{debug, Level};+use tracing::Level;use tracing_subscriber::FmtSubscriber;#[derive(Serialize, Deserialize, Debug)]@@ -15,18 +15,13 @@ struct Count(usize);impl Map<Count> for Job {fn map(self) -> Count {- debug!("begin map {:?}", self);- std::thread::sleep(std::time::Duration::from_secs(20)); // simulate some long running task- debug!("end map");+ std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running taskCount(1)}}impl Reduce for Count {fn reduce(self, element: Self) -> Self {- debug!("begin reduce");- std::thread::sleep(std::time::Duration::from_secs(20));- debug!("end reduce");Count(self.0 + element.0)}}diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 240ad953..8ae2bd75 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -1,6 +1,6 @@use std::io::{Read, Write};use std::net::{SocketAddr, TcpStream};-use std::ops::{Deref, DerefMut};+use std::ops::Deref;use std::sync::atomic::{AtomicU32, Ordering};use std::sync::{Arc, Mutex};use std::time::Duration;@@ -9,7 +9,6 @@ use crate::mapreduce::Task;use super::{Error, Result};use super::{Map, Reduce};-use async_channel::{unbounded, Receiver, Sender};use rayon::iter::ParallelBridge;use rayon::prelude::*;use std::net::ToSocketAddrs;@@ -255,15 +254,18 @@ impl Manager {fn get_results<I, O>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O>whereI: Map<O> + Send,- O: Reduce<O> + Send + Sync,+ O: Reduce<O> + Send,{let acc = Arc::new(Mutex::new(None));jobs.par_bridge().map(|job| self.map::<I, O>(job))+ .fold(|| None, |acc, elem| Some(Manager::reduce(acc, elem))).for_each(|res| {- let mut lock = acc.lock().unwrap();- *lock = Some(Manager::reduce(lock.take(), res));+ if let Some(res) = res {+ let mut lock = acc.lock().unwrap();+ *lock = Some(Manager::reduce(lock.take(), res));+ }});let x = acc.lock().unwrap().take();@@ -272,8 +274,8 @@ impl Manager {pub fn run<I, O>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O>where- I: Map<O> + Send + Sync,- O: Reduce<O> + Send + Sync,+ I: Map<O> + Send,+ O: Reduce<O> + Send,{let result = self.get_results(jobs);self.pool.stop_workers::<I, O>();diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex f6e9bfb4..fa8c2344 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -14,10 +14,8 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::net::SocketAddr;--use async_trait::async_trait;use serde::{de::DeserializeOwned, Deserialize, Serialize};+use std::net::SocketAddr;mod manager;mod worker;@@ -61,12 +59,11 @@ wherefn reduce(self, element: T) -> T;}-#[async_trait]pub trait MapReduce<I, O>whereSelf: Sized + Iterator<Item = I> + Send,- I: Map<O> + Sync,- O: Reduce<O> + Send + Sync,+ I: Map<O>,+ O: Reduce<O> + Send,{fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {let manager = Manager::new(workers);@@ -76,8 +73,8 @@ whereimpl<I, O, T> MapReduce<I, O> for TwhereT: Iterator<Item = I> + Sized + Send,- I: Map<O> + Sync,- O: Reduce<O> + Send + Sync,+ I: Map<O>,+ O: Reduce<O> + Send,{}