Skip to content

web search done right

ability to reduce into different type in mapreduce

Mikkel Denker6e959f75f0b2
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rs
index 7823c3c7..37fb6d26 100644
--- a/examples/mapreduce_manager.rs
+++ b/examples/mapreduce_manager.rs
@@ -20,7 +20,7 @@ impl Map<Count> for Job {
}
}
-impl Reduce for Count {
+impl Reduce<Count> for Count {
fn reduce(self, element: Self) -> Self {
Count(self.0 + element.0)
}
@@ -41,7 +41,7 @@ fn main() {
Job { id: 5 },
];
- let res = jobs
+ let res: Count = jobs
.into_iter()
.map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])
.unwrap();
diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rs
index 3dafd660..15c25a09 100644
--- a/examples/mapreduce_worker.rs
+++ b/examples/mapreduce_worker.rs
@@ -20,7 +20,7 @@ impl Map<Count> for Job {
}
}
-impl Reduce for Count {
+impl Reduce<Count> for Count {
fn reduce(self, element: Self) -> Self {
Count(self.0 + element.0)
}
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 71258cf6..a6c9b62e 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -83,10 +83,8 @@ impl Map<FrozenWebgraph> for Job {
}
}
-impl Reduce<FrozenWebgraph> for FrozenWebgraph {
- fn reduce(self, other: FrozenWebgraph) -> FrozenWebgraph {
- let mut graph: Webgraph = self.into();
-
+impl Reduce<FrozenWebgraph> for Webgraph {
+ fn reduce(mut self, other: FrozenWebgraph) -> Webgraph {
let other_path = match &other.root {
crate::directory::DirEntry::Folder { name, entries: _ } => name.clone(),
crate::directory::DirEntry::File { name, content: _ } => name.clone(),
@@ -94,11 +92,18 @@ impl Reduce<FrozenWebgraph> for FrozenWebgraph {
let other = other.into();
- graph.merge(other);
+ self.merge(other);
std::fs::remove_dir_all(other_path).unwrap();
- graph.into()
+ self
+ }
+}
+
+impl Reduce<Webgraph> for Webgraph {
+ fn reduce(mut self, element: Webgraph) -> Self {
+ self.merge(element);
+ self
}
}
@@ -147,7 +152,7 @@ impl WebgraphEntrypoint {
warc_paths = Box::new(warc_paths.take(limit));
}
- warc_paths
+ let _graph: Webgraph = warc_paths
.map_reduce(&workers)
.expect("failed to build webgraph");
@@ -180,10 +185,13 @@ impl WebgraphEntrypoint {
graph_base_path: "webgraph".to_string(),
})
.map(|job| job.map())
- .fold(None, |acc: Option<FrozenWebgraph>, elem| match acc {
- Some(acc) => Some(acc.reduce(elem)),
- None => Some(elem),
- });
+ .fold(
+ None,
+ |acc: Option<Webgraph>, elem: FrozenWebgraph| match acc {
+ Some(acc) => Some(acc.reduce(elem)),
+ None => Some(elem.into()),
+ },
+ );
Ok(())
}
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 888ed7a0..9cb4372d 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -5,6 +5,8 @@ use crate::mapreduce::Task;
use indicatif::{ParallelProgressIterator, ProgressBar, ProgressStyle};
use rayon::iter::ParallelBridge;
use rayon::prelude::*;
+use serde::de::DeserializeOwned;
+use serde::Serialize;
use std::io::{Read, Write};
use std::net::ToSocketAddrs;
use std::net::{SocketAddr, TcpStream};
@@ -44,7 +46,7 @@ impl RemoteWorker {
fn perform<I, O>(&self, job: &I) -> Result<O>
where
I: Map<O> + Send,
- O: Reduce<O> + Send,
+ O: Serialize + DeserializeOwned + Send,
{
let mut stream = self.connect()?;
let serialized_job = bincode::serialize(&Task::Job(job))?;
@@ -78,7 +80,7 @@ impl RemoteWorker {
fn stop<I, O>(&self) -> Result<()>
where
I: Map<O> + Send,
- O: Reduce<O> + Send,
+ O: Serialize + DeserializeOwned + Send,
{
debug!("closing worker {:}", self.addr);
let mut stream = self.connect()?;
@@ -175,7 +177,7 @@ impl WorkerPool {
fn stop_workers<I, O>(&self)
where
I: Map<O> + Send,
- O: Reduce<O> + Send,
+ O: Serialize + DeserializeOwned + Send,
{
let mut failing_workers = Vec::new();
for worker in &self.all_workers {
@@ -210,7 +212,7 @@ impl Manager {
fn try_map<I, O>(&self, job: &I) -> Result<O>
where
I: Map<O> + Send,
- O: Reduce<O> + Send,
+ O: Serialize + DeserializeOwned + Send,
{
loop {
match self.pool.get_worker()? {
@@ -230,7 +232,7 @@ impl Manager {
pub fn map<I, O>(&self, job: I) -> O
where
I: Map<O> + Send,
- O: Reduce<O> + Send,
+ O: Serialize + DeserializeOwned + Send,
{
loop {
match self.try_map(&job) {
@@ -243,19 +245,34 @@ impl Manager {
}
}
- fn reduce<O: Reduce<O>>(acc: Option<O>, elem: O) -> O {
+ fn reduce<O1, O2>(acc: Option<O2>, elem: O1) -> O2
+ where
+ O1: Serialize + DeserializeOwned + Send,
+ O2: From<O1> + Reduce<O1> + Send,
+ {
match acc {
Some(acc) => acc.reduce(elem),
- None => elem,
+ None => elem.into(),
}
}
- fn get_results<I, O>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O>
+ fn reduce_end<O>(acc: Option<O>, elem: O) -> O
where
- I: Map<O> + Send,
O: Reduce<O> + Send,
{
- let acc = Arc::new(Mutex::new(None));
+ match acc {
+ Some(acc) => acc.reduce(elem),
+ None => elem,
+ }
+ }
+
+ fn get_results<I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
+ where
+ I: Map<O1> + Send,
+ O1: Serialize + DeserializeOwned + Send,
+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
+ {
+ let acc: Arc<Mutex<Option<O2>>> = Arc::new(Mutex::new(None));
let size = jobs.size_hint();
@@ -270,24 +287,30 @@ impl Manager {
.progress_chars("#>-"),
);
jobs.par_bridge()
- .map(|job| self.map::<I, O>(job))
+ .map(|job| self.map::<I, O1>(job))
.progress_with(pb)
- .fold(|| None, |acc, elem| Some(Manager::reduce(acc, elem)))
+ .fold(
+ || None,
+ |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),
+ )
.for_each(|res| {
if let Some(res) = res {
let mut lock = acc.lock().unwrap();
- *lock = Some(Manager::reduce(lock.take(), res));
+ *lock = Some(Manager::reduce_end(lock.take(), res));
}
});
}
None => {
jobs.par_bridge()
- .map(|job| self.map::<I, O>(job))
- .fold(|| None, |acc, elem| Some(Manager::reduce(acc, elem)))
+ .map(|job| self.map::<I, O1>(job))
+ .fold(
+ || None,
+ |acc: Option<O2>, elem| Some(Manager::reduce(acc, elem)),
+ )
.for_each(|res| {
if let Some(res) = res {
let mut lock = acc.lock().unwrap();
- *lock = Some(Manager::reduce(lock.take(), res));
+ *lock = Some(Manager::reduce_end(lock.take(), res));
}
});
}
@@ -297,13 +320,14 @@ impl Manager {
x
}
- pub fn run<I, O>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O>
+ pub fn run<I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>
where
- I: Map<O> + Send,
- O: Reduce<O> + Send,
+ I: Map<O1> + Send,
+ O1: Serialize + DeserializeOwned + Send,
+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
let result = self.get_results(jobs);
- self.pool.stop_workers::<I, O>();
+ self.pool.stop_workers::<I, O1>();
result
}
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index fa8c2344..87d56a1b 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -52,29 +52,28 @@ where
fn map(self) -> T;
}
-pub trait Reduce<T = Self>
-where
- Self: Serialize + DeserializeOwned,
-{
- fn reduce(self, element: T) -> T;
+pub trait Reduce<T> {
+ fn reduce(self, element: T) -> Self;
}
-pub trait MapReduce<I, O>
+pub trait MapReduce<I, O1, O2>
where
Self: Sized + Iterator<Item = I> + Send,
- I: Map<O>,
- O: Reduce<O> + Send,
+ I: Map<O1>,
+ O1: Serialize + DeserializeOwned + Send,
+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
- fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {
+ fn map_reduce(self, workers: &[SocketAddr]) -> Option<O2> {
let manager = Manager::new(workers);
manager.run(self)
}
}
-impl<I, O, T> MapReduce<I, O> for T
+impl<I, O1, O2, T> MapReduce<I, O1, O2> for T
where
T: Iterator<Item = I> + Sized + Send,
- I: Map<O>,
- O: Reduce<O> + Send,
+ I: Map<O1>,
+ O1: Serialize + DeserializeOwned + Send,
+ O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,
{
}