Skip to content

web search done right

mapreduce is now way more robust to failures

Mikkel Denkere1a8ff33ec5c
diff --git a/Cargo.toml b/Cargo.toml
index cb0c8917..1c44bd0d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,7 @@ authors = ['Cuely ApS. <[email protected]>']
name = "cuely"
[dependencies]
+async-channel = "1.6.1"
async-trait = "0.1.53"
bincode = "1.3.3"
encoding_rs = "0.8.31"
@@ -30,6 +31,7 @@ sled = { version = "0.34.7", features = ["compression"] }
tantivy = "0.18.0"
thiserror = "1.0.31"
tokio = { version = "1.18.2", features = ["full"] }
+tokio-retry = "0.3.0"
toml = "0.5.9"
tracing = "0.1.34"
tracing-subscriber = "0.3.11"
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rs
index baaf03eb..35000b3b 100644
--- a/examples/mapreduce_manager.rs
+++ b/examples/mapreduce_manager.rs
@@ -15,6 +15,7 @@ struct Count(usize);
impl Map<Count> for Job {
fn map(self) -> Count {
+ std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
Count(self.contents.into_iter().filter(|d| *d == 0).count())
}
}
@@ -42,11 +43,24 @@ async fn main() {
Job {
contents: vec![0, 0],
},
+ Job {
+ contents: vec![1, 2, 0, 1, 0, 1, 0],
+ },
+ Job {
+ contents: vec![3, 2, 1],
+ },
+ Job {
+ contents: vec![0, 0],
+ },
];
let res = jobs
.into_iter()
- .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])
+ .map_reduce(&[
+ "0.0.0.0:1337".parse::<SocketAddr>().unwrap(),
+ "0.0.0.0:1338".parse::<SocketAddr>().unwrap(),
+ "0.0.0.0:1339".parse::<SocketAddr>().unwrap(),
+ ])
.await
.unwrap();
println!("{:?}", res.0);
diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rs
index a8871916..5cbda5fa 100644
--- a/examples/mapreduce_worker.rs
+++ b/examples/mapreduce_worker.rs
@@ -15,6 +15,7 @@ struct Count(usize);
impl Map<Count> for Job {
fn map(self) -> Count {
+ std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
Count(self.contents.into_iter().filter(|d| *d == 0).count())
}
}
@@ -32,5 +33,9 @@ async fn main() {
.finish();
tracing::subscriber::set_global_default(subscriber).unwrap();
- Worker::run::<Job, Count>("0.0.0.0:1337".parse::<SocketAddr>().unwrap()).await;
+ let args: Vec<_> = std::env::args().collect();
+
+ Worker::run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())
+ .await
+ .expect("failed to run worker");
}
diff --git a/src/lib.rs b/src/lib.rs
index 44816229..4483457b 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -20,15 +20,8 @@ use tantivy::TantivyError;
use thiserror::Error;
pub mod entrypoint;
-mod index;
-
-/// Stuff to fix for mapreduce:
-/// * we should handle the case where the job exceeds the 4096 bytes buffer.
-/// * manager should keep track of which workers are currently busy
-/// * manager should keep track of failing vs alive workers
-/// * if a job fails, the manager should re-allocate the job to another worker if possible
-/// * functions should return result instead of unwrapping everywhere
-/// * manager should stop workers when done
+pub mod index;
+
pub mod mapreduce;
mod query;
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index 5710efd4..a15b0943 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -1,34 +1,210 @@
use std::net::SocketAddr;
+use std::ops::Deref;
+use std::sync::atomic::{AtomicU32, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
use crate::mapreduce::Task;
+use super::{Error, Result};
use super::{Map, Reduce};
+use async_channel::{unbounded, Receiver, Sender};
use futures::{stream::FuturesUnordered, StreamExt};
-use rand::prelude::SliceRandom;
use std::net::ToSocketAddrs;
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpStream,
};
+use tokio_retry::strategy::ExponentialBackoff;
+use tokio_retry::Retry;
use tracing::debug;
+#[derive(Debug)]
+struct RemoteWorker {
+ addr: SocketAddr,
+}
+
+impl RemoteWorker {
+ fn retry_strategy() -> impl Iterator<Item = Duration> {
+ ExponentialBackoff::from_millis(10).take(4)
+ }
+
+ async fn connect(&self) -> Result<TcpStream> {
+ debug!("connecting to {:}", self.addr);
+ let stream = Retry::spawn(RemoteWorker::retry_strategy(), || async {
+ TcpStream::connect(self.addr).await
+ })
+ .await?;
+ debug!("connected");
+
+ Ok(stream)
+ }
+
+ async fn perform<I, O>(&self, job: &I) -> Result<O>
+ where
+ I: Map<O>,
+ O: Reduce<O> + Send,
+ {
+ let mut stream = self.connect().await?;
+ let serialized_job = bincode::serialize(&Task::Job(job))?;
+ debug!("sending {:?} bytes", serialized_job.len());
+ stream.write(&serialized_job).await?;
+
+ let mut buf = [0; 4096];
+ let mut bytes = Vec::new();
+ loop {
+ if let Ok(size) = stream.read(&mut buf).await {
+ debug!("read {:?} bytes", size);
+ bytes.extend_from_slice(&buf);
+ if size < buf.len() {
+ break;
+ }
+ }
+ }
+
+ Ok(bincode::deserialize(&bytes)?)
+ }
+
+ async fn stop<I, O>(&self) -> Result<()>
+ where
+ I: Map<O>,
+ O: Reduce<O> + Send,
+ {
+ debug!("closing worker {:}", self.addr);
+ let mut stream = self.connect().await?;
+ let serialized_job = bincode::serialize(&Task::<I>::AllFinished)?;
+ debug!("sending {:?} bytes", serialized_job.len());
+ stream.write(&serialized_job).await?;
+ Ok(())
+ }
+}
+
+struct WorkerGuard<'a> {
+ from_pool: &'a WorkerPool,
+ worker: Arc<RemoteWorker>,
+}
+
+impl<'a> WorkerGuard<'a> {
+ fn new(pool: &'a WorkerPool, worker: Arc<RemoteWorker>) -> Self {
+ Self {
+ worker,
+ from_pool: pool,
+ }
+ }
+}
+
+impl<'a> Deref for WorkerGuard<'a> {
+ type Target = Arc<RemoteWorker>;
+
+ fn deref(&self) -> &Self::Target {
+ &self.worker
+ }
+}
+
+impl<'a> Drop for WorkerGuard<'a> {
+ fn drop(&mut self) {
+ self.from_pool.put_back(Arc::clone(&self.worker))
+ }
+}
+
+struct WorkerPool {
+ all_workers: Vec<Arc<RemoteWorker>>,
+ alive_workers: (Sender<Arc<RemoteWorker>>, Receiver<Arc<RemoteWorker>>),
+ running_workers: AtomicU32,
+}
+
+impl WorkerPool {
+ async fn new<A>(workers: &[A]) -> Self
+ where
+ A: ToSocketAddrs + std::fmt::Debug,
+ {
+ let all_workers = workers
+ .iter()
+ .flat_map(|addr| {
+ addr.to_socket_addrs().expect(&format!(
+ "failed to transform {:?} into a socket address",
+ addr
+ ))
+ })
+ .map(|addr| Arc::new(RemoteWorker { addr }))
+ .collect();
+
+ let (s, r) = unbounded();
+
+ for worker in &all_workers {
+ s.send(Arc::clone(worker)).await.unwrap();
+ }
+
+ Self {
+ all_workers,
+ alive_workers: (s, r),
+ running_workers: AtomicU32::new(0),
+ }
+ }
+
+ fn put_back(&self, worker: Arc<RemoteWorker>) {
+ let ch = self.alive_workers.0.clone();
+ tokio::spawn(async move {
+ ch.send(worker).await.unwrap();
+ });
+ self.running_workers.fetch_sub(1, Ordering::SeqCst);
+ }
+
+ async fn get_worker<'a>(&'a self) -> Result<WorkerGuard<'a>> {
+ if self.alive_workers.0.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {
+ return Err(Error::NoAvailableWorker);
+ }
+
+ let worker = self.alive_workers.1.recv().await?;
+ self.running_workers.fetch_add(1, Ordering::SeqCst);
+
+ Ok(WorkerGuard::new(self, worker))
+ }
+
+ async fn stop_workers<I, O>(&self)
+ where
+ I: Map<O>,
+ O: Reduce<O> + Send,
+ {
+ let mut failing_workers = Vec::new();
+ for worker in &self.all_workers {
+ if worker.stop::<I, O>().await.is_err() {
+ failing_workers.push(worker)
+ }
+ }
+
+ if failing_workers.len() > 0 {
+ debug!(
+ "failed to stop the following workers: {:#?}",
+ failing_workers
+ );
+ }
+ }
+}
+
pub struct Manager {
- workers: Vec<SocketAddr>,
+ pool: WorkerPool,
}
impl Manager {
- pub fn new<A>(workers: &[A]) -> Self
+ pub async fn new<A>(workers: &[A]) -> Self
where
- A: ToSocketAddrs,
+ A: ToSocketAddrs + std::fmt::Debug,
{
Self {
- workers: workers
- .iter()
- .flat_map(|addr| addr.to_socket_addrs().unwrap())
- .collect(),
+ pool: WorkerPool::new(workers).await,
}
}
+ async fn try_map<I, O>(&self, job: &I) -> Result<O>
+ where
+ I: Map<O>,
+ O: Reduce<O> + Send,
+ {
+ let worker = self.pool.get_worker().await?;
+ worker.perform(job).await
+ }
+
/// Execute job on one of the remote machines. If the remote machine fails for some reason,
/// the job should be allocated to another machine.
pub async fn map<I, O>(&self, job: I) -> O
@@ -36,17 +212,15 @@ impl Manager {
I: Map<O>,
O: Reduce<O> + Send,
{
- let worker_addr = self.workers.choose(&mut rand::thread_rng()).unwrap();
- debug!("connecting to {:}", worker_addr);
- let mut stream = TcpStream::connect(worker_addr).await.unwrap();
- debug!("connected");
- let serialized_job = bincode::serialize(&Task::Job(job)).unwrap();
- debug!("sending {:?} bytes", serialized_job.len());
- stream.write(&serialized_job).await.unwrap();
-
- let mut buf = vec![0; 4096];
- let size = stream.read(&mut buf).await.unwrap();
- bincode::deserialize(&buf[..size]).unwrap()
+ loop {
+ match self.try_map(&job).await {
+ Ok(res) => return res,
+ Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),
+ Err(_) => {
+ debug!("rescheduling job");
+ }
+ }
+ }
}
fn reduce<O: Reduce<O>>(acc: Option<O>, elem: O) -> O {
@@ -56,7 +230,7 @@ impl Manager {
}
}
- pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>
+ async fn get_results<I, O>(&self, jobs: Vec<I>) -> Option<O>
where
I: Map<O>,
O: Reduce<O> + Send,
@@ -70,4 +244,15 @@ impl Manager {
)
.await
}
+
+ pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>
+ where
+ I: Map<O>,
+ O: Reduce<O> + Send,
+ {
+ let result = self.get_results(jobs).await;
+ self.pool.stop_workers::<I, O>().await;
+
+ result
+ }
}
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index c97a4281..b093d617 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -23,8 +23,29 @@ mod manager;
mod worker;
pub use manager::Manager;
+use thiserror::Error;
pub use worker::Worker;
+pub(crate) type Result<T> = std::result::Result<T, Error>;
+
+#[derive(Error, Debug)]
+pub enum Error {
+ #[error("I/O error")]
+ IO(#[from] std::io::Error),
+
+ #[error("error while serializing/deserializing to/from bytes")]
+ Serialization(#[from] bincode::Error),
+
+ #[error("failed to stop some of the workers")]
+ StopWorker,
+
+ #[error("could not get a working worker")]
+ NoAvailableWorker,
+
+ #[error("failed to get value from channel")]
+ ChannelRecv(#[from] async_channel::RecvError),
+}
+
pub trait Map<T>
where
Self: Serialize + DeserializeOwned + Send,
@@ -41,19 +62,22 @@ where
}
#[async_trait]
-pub trait MapReduce<I, O>: Iterator<Item = I>
+pub trait MapReduce<I, O>
where
- Self: Sized,
- I: Map<O>,
+ Self: Sized + Iterator<Item = I>,
+ I: Map<O> + Sync,
O: Reduce<O> + Send,
{
async fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {
- let manager = Manager::new(workers);
+ let manager = Manager::new(workers).await;
manager.run(self.collect()).await
}
}
-impl<I: Map<O> + Send, O: Reduce<O> + Send, T: Sized> MapReduce<I, O> for T where
- T: Iterator<Item = I>
+impl<I, O, T> MapReduce<I, O> for T
+where
+ T: Iterator<Item = I> + Sized,
+ I: Map<O> + Send + Sync,
+ O: Reduce<O> + Send,
{
}
diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rs
index 6005e157..cae84b28 100644
--- a/src/mapreduce/worker.rs
+++ b/src/mapreduce/worker.rs
@@ -1,54 +1,73 @@
use std::net::SocketAddr;
+use super::Result;
use super::{Map, Task};
use serde::{de::DeserializeOwned, Serialize};
use tokio::io::AsyncReadExt;
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
-use tokio::net::{TcpListener, ToSocketAddrs};
+use tokio::net::TcpListener;
use tracing::{debug, info};
pub struct Worker {}
+enum State {
+ Continue,
+ Finished,
+}
+
impl Worker {
- async fn run_stream<I, O, S>(mut stream: S)
+ async fn run_stream<I, O, S>(mut stream: S) -> Result<State>
where
I: Map<O>,
O: Serialize + DeserializeOwned + Send,
S: AsyncRead + AsyncWrite + Unpin,
{
- let mut buf = vec![0; 4096];
+ let mut buf = [0; 4096];
+ let mut bytes = Vec::new();
loop {
if let Ok(size) = stream.read(&mut buf).await {
- if size == 0 {
+ debug!("read {:?} bytes", size);
+ bytes.extend_from_slice(&buf);
+ if size < buf.len() {
break;
}
- debug!("read {:?} bytes", size);
- match bincode::deserialize::<Task<I>>(&buf[..size]).unwrap() {
- Task::Job(job) => {
- debug!("received job");
- let res = job.map();
- let bytes = bincode::serialize(&res).unwrap();
- stream.write_all(&bytes[..]).await.unwrap();
- }
- Task::AllFinished => break,
- };
}
}
+
+ match bincode::deserialize::<Task<I>>(&bytes)? {
+ Task::Job(job) => {
+ debug!("received job");
+ let res = job.map();
+ let bytes = bincode::serialize(&res)?;
+ stream.write_all(&bytes[..]).await?;
+ }
+ Task::AllFinished => {
+ debug!("shutting down");
+ return Ok(State::Finished);
+ }
+ };
+
+ Ok(State::Continue)
}
- pub async fn run<I, O>(addr: SocketAddr)
+ pub async fn run<I, O>(addr: SocketAddr) -> Result<()>
where
I: Map<O>,
O: Serialize + DeserializeOwned + Send,
{
- let listener = TcpListener::bind(addr).await.unwrap();
+ let listener = TcpListener::bind(addr).await?;
info!("worker listening on: {:}", addr);
loop {
- let (socket, _) = listener.accept().await.unwrap();
+ let (socket, _) = listener.accept().await?;
debug!("received connection");
- Worker::run_stream::<I, O, _>(socket).await;
+ match Worker::run_stream::<I, O, _>(socket).await? {
+ State::Finished => break,
+ State::Continue => {}
+ }
}
+
+ Ok(())
}
}
@@ -95,7 +114,7 @@ mod tests {
self: Pin<&mut Self>,
_: &mut Context<'_>,
buf: &[u8],
- ) -> Poll<Result<usize, std::io::Error>> {
+ ) -> Poll<std::result::Result<usize, std::io::Error>> {
self.get_mut().result = Vec::from(buf);
Poll::Ready(Ok(buf.len()))
}
@@ -103,14 +122,14 @@ mod tests {
fn poll_flush(
self: Pin<&mut Self>,
_: &mut Context<'_>,
- ) -> Poll<Result<(), std::io::Error>> {
+ ) -> Poll<std::result::Result<(), std::io::Error>> {
Poll::Ready(Ok(()))
}
fn poll_shutdown(
self: Pin<&mut Self>,
_: &mut Context<'_>,
- ) -> Poll<Result<(), std::io::Error>> {
+ ) -> Poll<std::result::Result<(), std::io::Error>> {
Poll::Ready(Ok(()))
}
}
@@ -137,7 +156,9 @@ mod tests {
.unwrap();
let mut stream = MockTcpStream::new(job);
- Worker::run_stream::<MockJob, _, _>(&mut stream).await;
+ Worker::run_stream::<MockJob, _, _>(&mut stream)
+ .await
+ .expect("worker failed");
let res: Count = bincode::deserialize(&stream.result).unwrap();