Skip to content

web search done right

mapreduce types doesn't have to be sync

Mikkel Denkerda66ffa2ca6f
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rs
index e7d2cc1c..7823c3c7 100644
--- a/examples/mapreduce_manager.rs
+++ b/examples/mapreduce_manager.rs
@@ -2,7 +2,7 @@ use std::net::SocketAddr;
use cuely::mapreduce::{Map, MapReduce, Reduce};
use serde::{Deserialize, Serialize};
-use tracing::{debug, Level};
+use tracing::Level;
use tracing_subscriber::FmtSubscriber;
#[derive(Serialize, Deserialize, Debug)]
@@ -15,18 +15,13 @@ struct Count(usize);
impl Map<Count> for Job {
fn map(self) -> Count {
- debug!("begin map");
std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
- debug!("end map");
Count(1)
}
}
impl Reduce for Count {
fn reduce(self, element: Self) -> Self {
- debug!("begin reduce");
- std::thread::sleep(std::time::Duration::from_secs(20));
- debug!("end reduce");
Count(self.0 + element.0)
}
}
diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rs
index d81df6f4..3dafd660 100644
--- a/examples/mapreduce_worker.rs
+++ b/examples/mapreduce_worker.rs
@@ -2,7 +2,7 @@ use std::net::SocketAddr;
use cuely::mapreduce::{Map, Reduce, Worker};
use serde::{Deserialize, Serialize};
-use tracing::{debug, Level};
+use tracing::Level;
use tracing_subscriber::FmtSubscriber;
#[derive(Serialize, Deserialize, Debug)]
@@ -15,18 +15,13 @@ struct Count(usize);
impl Map<Count> for Job {
fn map(self) -> Count {
- debug!("begin map {:?}", self);
- std::thread::sleep(std::time::Duration::from_secs(20)); // simulate some long running task
- debug!("end map");
+ std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
Count(1)
}
}
impl Reduce for Count {
fn reduce(self, element: Self) -> Self {
- debug!("begin reduce");
- std::thread::sleep(std::time::Duration::from_secs(20));
- debug!("end reduce");
Count(self.0 + element.0)
}
}
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 240ad953..8ae2bd75 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -1,6 +1,6 @@
use std::io::{Read, Write};
use std::net::{SocketAddr, TcpStream};
-use std::ops::{Deref, DerefMut};
+use std::ops::Deref;
use std::sync::atomic::{AtomicU32, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
@@ -9,7 +9,6 @@ use crate::mapreduce::Task;
use super::{Error, Result};
use super::{Map, Reduce};
-use async_channel::{unbounded, Receiver, Sender};
use rayon::iter::ParallelBridge;
use rayon::prelude::*;
use std::net::ToSocketAddrs;
@@ -255,15 +254,18 @@ impl Manager {
fn get_results<I, O>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O>
where
I: Map<O> + Send,
- O: Reduce<O> + Send + Sync,
+ O: Reduce<O> + Send,
{
let acc = Arc::new(Mutex::new(None));
jobs.par_bridge()
.map(|job| self.map::<I, O>(job))
+ .fold(|| None, |acc, elem| Some(Manager::reduce(acc, elem)))
.for_each(|res| {
- let mut lock = acc.lock().unwrap();
- *lock = Some(Manager::reduce(lock.take(), res));
+ if let Some(res) = res {
+ let mut lock = acc.lock().unwrap();
+ *lock = Some(Manager::reduce(lock.take(), res));
+ }
});
let x = acc.lock().unwrap().take();
@@ -272,8 +274,8 @@ impl Manager {
pub fn run<I, O>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O>
where
- I: Map<O> + Send + Sync,
- O: Reduce<O> + Send + Sync,
+ I: Map<O> + Send,
+ O: Reduce<O> + Send,
{
let result = self.get_results(jobs);
self.pool.stop_workers::<I, O>();
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index f6e9bfb4..fa8c2344 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -14,10 +14,8 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::net::SocketAddr;
-
-use async_trait::async_trait;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use std::net::SocketAddr;
mod manager;
mod worker;
@@ -61,12 +59,11 @@ where
fn reduce(self, element: T) -> T;
}
-#[async_trait]
pub trait MapReduce<I, O>
where
Self: Sized + Iterator<Item = I> + Send,
- I: Map<O> + Sync,
- O: Reduce<O> + Send + Sync,
+ I: Map<O>,
+ O: Reduce<O> + Send,
{
fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {
let manager = Manager::new(workers);
@@ -76,8 +73,8 @@ where
impl<I, O, T> MapReduce<I, O> for T
where
T: Iterator<Item = I> + Sized + Send,
- I: Map<O> + Sync,
- O: Reduce<O> + Send + Sync,
+ I: Map<O>,
+ O: Reduce<O> + Send,
{
}