web search done right
mapreduce is now way more robust to failures
Mikkel Denkere1a8ff33ec5c
diff --git a/Cargo.toml b/Cargo.tomlindex cb0c8917..1c44bd0d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,6 +11,7 @@ authors = ['Cuely ApS. <[email protected]>']name = "cuely"[dependencies]+async-channel = "1.6.1"async-trait = "0.1.53"bincode = "1.3.3"encoding_rs = "0.8.31"@@ -30,6 +31,7 @@ sled = { version = "0.34.7", features = ["compression"] }tantivy = "0.18.0"thiserror = "1.0.31"tokio = { version = "1.18.2", features = ["full"] }+tokio-retry = "0.3.0"toml = "0.5.9"tracing = "0.1.34"tracing-subscriber = "0.3.11"diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rsindex baaf03eb..35000b3b 100644--- a/examples/mapreduce_manager.rs+++ b/examples/mapreduce_manager.rs@@ -15,6 +15,7 @@ struct Count(usize);impl Map<Count> for Job {fn map(self) -> Count {+ std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running taskCount(self.contents.into_iter().filter(|d| *d == 0).count())}}@@ -42,11 +43,24 @@ async fn main() {Job {contents: vec![0, 0],},+ Job {+ contents: vec![1, 2, 0, 1, 0, 1, 0],+ },+ Job {+ contents: vec![3, 2, 1],+ },+ Job {+ contents: vec![0, 0],+ },];let res = jobs.into_iter()- .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])+ .map_reduce(&[+ "0.0.0.0:1337".parse::<SocketAddr>().unwrap(),+ "0.0.0.0:1338".parse::<SocketAddr>().unwrap(),+ "0.0.0.0:1339".parse::<SocketAddr>().unwrap(),+ ]).await.unwrap();println!("{:?}", res.0);diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rsindex a8871916..5cbda5fa 100644--- a/examples/mapreduce_worker.rs+++ b/examples/mapreduce_worker.rs@@ -15,6 +15,7 @@ struct Count(usize);impl Map<Count> for Job {fn map(self) -> Count {+ std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running taskCount(self.contents.into_iter().filter(|d| *d == 0).count())}}@@ -32,5 +33,9 @@ async fn main() {.finish();tracing::subscriber::set_global_default(subscriber).unwrap();- Worker::run::<Job, Count>("0.0.0.0:1337".parse::<SocketAddr>().unwrap()).await;+ let args: Vec<_> = std::env::args().collect();++ Worker::run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())+ .await+ .expect("failed to run worker");}diff --git a/src/lib.rs b/src/lib.rsindex 44816229..4483457b 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -20,15 +20,8 @@ use tantivy::TantivyError;use thiserror::Error;pub mod entrypoint;-mod index;--/// Stuff to fix for mapreduce:-/// * we should handle the case where the job exceeds the 4096 bytes buffer.-/// * manager should keep track of which workers are currently busy-/// * manager should keep track of failing vs alive workers-/// * if a job fails, the manager should re-allocate the job to another worker if possible-/// * functions should return result instead of unwrapping everywhere-/// * manager should stop workers when done+pub mod index;+pub mod mapreduce;mod query;diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex 5710efd4..a15b0943 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -1,34 +1,210 @@use std::net::SocketAddr;+use std::ops::Deref;+use std::sync::atomic::{AtomicU32, Ordering};+use std::sync::Arc;+use std::time::Duration;use crate::mapreduce::Task;+use super::{Error, Result};use super::{Map, Reduce};+use async_channel::{unbounded, Receiver, Sender};use futures::{stream::FuturesUnordered, StreamExt};-use rand::prelude::SliceRandom;use std::net::ToSocketAddrs;use tokio::{io::{AsyncReadExt, AsyncWriteExt},net::TcpStream,};+use tokio_retry::strategy::ExponentialBackoff;+use tokio_retry::Retry;use tracing::debug;+#[derive(Debug)]+struct RemoteWorker {+ addr: SocketAddr,+}++impl RemoteWorker {+ fn retry_strategy() -> impl Iterator<Item = Duration> {+ ExponentialBackoff::from_millis(10).take(4)+ }++ async fn connect(&self) -> Result<TcpStream> {+ debug!("connecting to {:}", self.addr);+ let stream = Retry::spawn(RemoteWorker::retry_strategy(), || async {+ TcpStream::connect(self.addr).await+ })+ .await?;+ debug!("connected");++ Ok(stream)+ }++ async fn perform<I, O>(&self, job: &I) -> Result<O>+ where+ I: Map<O>,+ O: Reduce<O> + Send,+ {+ let mut stream = self.connect().await?;+ let serialized_job = bincode::serialize(&Task::Job(job))?;+ debug!("sending {:?} bytes", serialized_job.len());+ stream.write(&serialized_job).await?;++ let mut buf = [0; 4096];+ let mut bytes = Vec::new();+ loop {+ if let Ok(size) = stream.read(&mut buf).await {+ debug!("read {:?} bytes", size);+ bytes.extend_from_slice(&buf);+ if size < buf.len() {+ break;+ }+ }+ }++ Ok(bincode::deserialize(&bytes)?)+ }++ async fn stop<I, O>(&self) -> Result<()>+ where+ I: Map<O>,+ O: Reduce<O> + Send,+ {+ debug!("closing worker {:}", self.addr);+ let mut stream = self.connect().await?;+ let serialized_job = bincode::serialize(&Task::<I>::AllFinished)?;+ debug!("sending {:?} bytes", serialized_job.len());+ stream.write(&serialized_job).await?;+ Ok(())+ }+}++struct WorkerGuard<'a> {+ from_pool: &'a WorkerPool,+ worker: Arc<RemoteWorker>,+}++impl<'a> WorkerGuard<'a> {+ fn new(pool: &'a WorkerPool, worker: Arc<RemoteWorker>) -> Self {+ Self {+ worker,+ from_pool: pool,+ }+ }+}++impl<'a> Deref for WorkerGuard<'a> {+ type Target = Arc<RemoteWorker>;++ fn deref(&self) -> &Self::Target {+ &self.worker+ }+}++impl<'a> Drop for WorkerGuard<'a> {+ fn drop(&mut self) {+ self.from_pool.put_back(Arc::clone(&self.worker))+ }+}++struct WorkerPool {+ all_workers: Vec<Arc<RemoteWorker>>,+ alive_workers: (Sender<Arc<RemoteWorker>>, Receiver<Arc<RemoteWorker>>),+ running_workers: AtomicU32,+}++impl WorkerPool {+ async fn new<A>(workers: &[A]) -> Self+ where+ A: ToSocketAddrs + std::fmt::Debug,+ {+ let all_workers = workers+ .iter()+ .flat_map(|addr| {+ addr.to_socket_addrs().expect(&format!(+ "failed to transform {:?} into a socket address",+ addr+ ))+ })+ .map(|addr| Arc::new(RemoteWorker { addr }))+ .collect();++ let (s, r) = unbounded();++ for worker in &all_workers {+ s.send(Arc::clone(worker)).await.unwrap();+ }++ Self {+ all_workers,+ alive_workers: (s, r),+ running_workers: AtomicU32::new(0),+ }+ }++ fn put_back(&self, worker: Arc<RemoteWorker>) {+ let ch = self.alive_workers.0.clone();+ tokio::spawn(async move {+ ch.send(worker).await.unwrap();+ });+ self.running_workers.fetch_sub(1, Ordering::SeqCst);+ }++ async fn get_worker<'a>(&'a self) -> Result<WorkerGuard<'a>> {+ if self.alive_workers.0.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {+ return Err(Error::NoAvailableWorker);+ }++ let worker = self.alive_workers.1.recv().await?;+ self.running_workers.fetch_add(1, Ordering::SeqCst);++ Ok(WorkerGuard::new(self, worker))+ }++ async fn stop_workers<I, O>(&self)+ where+ I: Map<O>,+ O: Reduce<O> + Send,+ {+ let mut failing_workers = Vec::new();+ for worker in &self.all_workers {+ if worker.stop::<I, O>().await.is_err() {+ failing_workers.push(worker)+ }+ }++ if failing_workers.len() > 0 {+ debug!(+ "failed to stop the following workers: {:#?}",+ failing_workers+ );+ }+ }+}+pub struct Manager {- workers: Vec<SocketAddr>,+ pool: WorkerPool,}impl Manager {- pub fn new<A>(workers: &[A]) -> Self+ pub async fn new<A>(workers: &[A]) -> Selfwhere- A: ToSocketAddrs,+ A: ToSocketAddrs + std::fmt::Debug,{Self {- workers: workers- .iter()- .flat_map(|addr| addr.to_socket_addrs().unwrap())- .collect(),+ pool: WorkerPool::new(workers).await,}}+ async fn try_map<I, O>(&self, job: &I) -> Result<O>+ where+ I: Map<O>,+ O: Reduce<O> + Send,+ {+ let worker = self.pool.get_worker().await?;+ worker.perform(job).await+ }+/// Execute job on one of the remote machines. If the remote machine fails for some reason,/// the job should be allocated to another machine.pub async fn map<I, O>(&self, job: I) -> O@@ -36,17 +212,15 @@ impl Manager {I: Map<O>,O: Reduce<O> + Send,{- let worker_addr = self.workers.choose(&mut rand::thread_rng()).unwrap();- debug!("connecting to {:}", worker_addr);- let mut stream = TcpStream::connect(worker_addr).await.unwrap();- debug!("connected");- let serialized_job = bincode::serialize(&Task::Job(job)).unwrap();- debug!("sending {:?} bytes", serialized_job.len());- stream.write(&serialized_job).await.unwrap();-- let mut buf = vec![0; 4096];- let size = stream.read(&mut buf).await.unwrap();- bincode::deserialize(&buf[..size]).unwrap()+ loop {+ match self.try_map(&job).await {+ Ok(res) => return res,+ Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),+ Err(_) => {+ debug!("rescheduling job");+ }+ }+ }}fn reduce<O: Reduce<O>>(acc: Option<O>, elem: O) -> O {@@ -56,7 +230,7 @@ impl Manager {}}- pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>+ async fn get_results<I, O>(&self, jobs: Vec<I>) -> Option<O>whereI: Map<O>,O: Reduce<O> + Send,@@ -70,4 +244,15 @@ impl Manager {).await}++ pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>+ where+ I: Map<O>,+ O: Reduce<O> + Send,+ {+ let result = self.get_results(jobs).await;+ self.pool.stop_workers::<I, O>().await;++ result+ }}diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex c97a4281..b093d617 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -23,8 +23,29 @@ mod manager;mod worker;pub use manager::Manager;+use thiserror::Error;pub use worker::Worker;+pub(crate) type Result<T> = std::result::Result<T, Error>;++#[derive(Error, Debug)]+pub enum Error {+ #[error("I/O error")]+ IO(#[from] std::io::Error),++ #[error("error while serializing/deserializing to/from bytes")]+ Serialization(#[from] bincode::Error),++ #[error("failed to stop some of the workers")]+ StopWorker,++ #[error("could not get a working worker")]+ NoAvailableWorker,++ #[error("failed to get value from channel")]+ ChannelRecv(#[from] async_channel::RecvError),+}+pub trait Map<T>whereSelf: Serialize + DeserializeOwned + Send,@@ -41,19 +62,22 @@ where}#[async_trait]-pub trait MapReduce<I, O>: Iterator<Item = I>+pub trait MapReduce<I, O>where- Self: Sized,- I: Map<O>,+ Self: Sized + Iterator<Item = I>,+ I: Map<O> + Sync,O: Reduce<O> + Send,{async fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {- let manager = Manager::new(workers);+ let manager = Manager::new(workers).await;manager.run(self.collect()).await}}-impl<I: Map<O> + Send, O: Reduce<O> + Send, T: Sized> MapReduce<I, O> for T where- T: Iterator<Item = I>+impl<I, O, T> MapReduce<I, O> for T+where+ T: Iterator<Item = I> + Sized,+ I: Map<O> + Send + Sync,+ O: Reduce<O> + Send,{}diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rsindex 6005e157..cae84b28 100644--- a/src/mapreduce/worker.rs+++ b/src/mapreduce/worker.rs@@ -1,54 +1,73 @@use std::net::SocketAddr;+use super::Result;use super::{Map, Task};use serde::{de::DeserializeOwned, Serialize};use tokio::io::AsyncReadExt;use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};-use tokio::net::{TcpListener, ToSocketAddrs};+use tokio::net::TcpListener;use tracing::{debug, info};pub struct Worker {}+enum State {+ Continue,+ Finished,+}+impl Worker {- async fn run_stream<I, O, S>(mut stream: S)+ async fn run_stream<I, O, S>(mut stream: S) -> Result<State>whereI: Map<O>,O: Serialize + DeserializeOwned + Send,S: AsyncRead + AsyncWrite + Unpin,{- let mut buf = vec![0; 4096];+ let mut buf = [0; 4096];+ let mut bytes = Vec::new();loop {if let Ok(size) = stream.read(&mut buf).await {- if size == 0 {+ debug!("read {:?} bytes", size);+ bytes.extend_from_slice(&buf);+ if size < buf.len() {break;}- debug!("read {:?} bytes", size);- match bincode::deserialize::<Task<I>>(&buf[..size]).unwrap() {- Task::Job(job) => {- debug!("received job");- let res = job.map();- let bytes = bincode::serialize(&res).unwrap();- stream.write_all(&bytes[..]).await.unwrap();- }- Task::AllFinished => break,- };}}++ match bincode::deserialize::<Task<I>>(&bytes)? {+ Task::Job(job) => {+ debug!("received job");+ let res = job.map();+ let bytes = bincode::serialize(&res)?;+ stream.write_all(&bytes[..]).await?;+ }+ Task::AllFinished => {+ debug!("shutting down");+ return Ok(State::Finished);+ }+ };++ Ok(State::Continue)}- pub async fn run<I, O>(addr: SocketAddr)+ pub async fn run<I, O>(addr: SocketAddr) -> Result<()>whereI: Map<O>,O: Serialize + DeserializeOwned + Send,{- let listener = TcpListener::bind(addr).await.unwrap();+ let listener = TcpListener::bind(addr).await?;info!("worker listening on: {:}", addr);loop {- let (socket, _) = listener.accept().await.unwrap();+ let (socket, _) = listener.accept().await?;debug!("received connection");- Worker::run_stream::<I, O, _>(socket).await;+ match Worker::run_stream::<I, O, _>(socket).await? {+ State::Finished => break,+ State::Continue => {}+ }}++ Ok(())}}@@ -95,7 +114,7 @@ mod tests {self: Pin<&mut Self>,_: &mut Context<'_>,buf: &[u8],- ) -> Poll<Result<usize, std::io::Error>> {+ ) -> Poll<std::result::Result<usize, std::io::Error>> {self.get_mut().result = Vec::from(buf);Poll::Ready(Ok(buf.len()))}@@ -103,14 +122,14 @@ mod tests {fn poll_flush(self: Pin<&mut Self>,_: &mut Context<'_>,- ) -> Poll<Result<(), std::io::Error>> {+ ) -> Poll<std::result::Result<(), std::io::Error>> {Poll::Ready(Ok(()))}fn poll_shutdown(self: Pin<&mut Self>,_: &mut Context<'_>,- ) -> Poll<Result<(), std::io::Error>> {+ ) -> Poll<std::result::Result<(), std::io::Error>> {Poll::Ready(Ok(()))}}@@ -137,7 +156,9 @@ mod tests {.unwrap();let mut stream = MockTcpStream::new(job);- Worker::run_stream::<MockJob, _, _>(&mut stream).await;+ Worker::run_stream::<MockJob, _, _>(&mut stream)+ .await+ .expect("worker failed");let res: Count = bincode::deserialize(&stream.result).unwrap();