web search done right
concurrent reduce and map. Mapreduce doesn't use async anymore
Mikkel Denker7db1d92dba33
diff --git a/Cargo.toml b/Cargo.tomlindex 307b4390..907809ec 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -24,6 +24,7 @@ itertools = "0.10.3"lru = "0.7.6"nom = "7.1.1"rand = "0.8.5"+rayon = "1.5.3"reqwest = "0.11.10"rusoto_core = "0.48.0"rusoto_s3 = "0.48.0"diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rsindex b1fe363d..e7d2cc1c 100644--- a/examples/mapreduce_manager.rs+++ b/examples/mapreduce_manager.rs@@ -2,7 +2,7 @@ use std::net::SocketAddr;use cuely::mapreduce::{Map, MapReduce, Reduce};use serde::{Deserialize, Serialize};-use tracing::Level;+use tracing::{debug, Level};use tracing_subscriber::FmtSubscriber;#[derive(Serialize, Deserialize, Debug)]@@ -10,24 +10,28 @@ struct Job {id: usize,}+#[derive(Serialize, Deserialize, Debug)]+struct Count(usize);+impl Map<Count> for Job {fn map(self) -> Count {+ debug!("begin map");std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task+ debug!("end map");Count(1)}}-#[derive(Serialize, Deserialize, Debug)]-struct Count(usize);-impl Reduce for Count {fn reduce(self, element: Self) -> Self {+ debug!("begin reduce");+ std::thread::sleep(std::time::Duration::from_secs(20));+ debug!("end reduce");Count(self.0 + element.0)}}-#[tokio::main]-async fn main() {+fn main() {let subscriber = FmtSubscriber::builder().with_max_level(Level::DEBUG).finish();@@ -44,12 +48,7 @@ async fn main() {let res = jobs.into_iter()- .map_reduce(&[- "0.0.0.0:1337".parse::<SocketAddr>().unwrap(),- "0.0.0.0:1338".parse::<SocketAddr>().unwrap(),- "0.0.0.0:1339".parse::<SocketAddr>().unwrap(),- ])- .await+ .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()]).unwrap();println!("{:?}", res.0);}diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rsindex ca153f9d..d81df6f4 100644--- a/examples/mapreduce_worker.rs+++ b/examples/mapreduce_worker.rs@@ -2,7 +2,7 @@ use std::net::SocketAddr;use cuely::mapreduce::{Map, Reduce, Worker};use serde::{Deserialize, Serialize};-use tracing::Level;+use tracing::{debug, Level};use tracing_subscriber::FmtSubscriber;#[derive(Serialize, Deserialize, Debug)]@@ -15,19 +15,23 @@ struct Count(usize);impl Map<Count> for Job {fn map(self) -> Count {- std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task+ debug!("begin map {:?}", self);+ std::thread::sleep(std::time::Duration::from_secs(20)); // simulate some long running task+ debug!("end map");Count(1)}}impl Reduce for Count {fn reduce(self, element: Self) -> Self {+ debug!("begin reduce");+ std::thread::sleep(std::time::Duration::from_secs(20));+ debug!("end reduce");Count(self.0 + element.0)}}-#[tokio::main]-async fn main() {+fn main() {let subscriber = FmtSubscriber::builder().with_max_level(Level::DEBUG).finish();@@ -36,6 +40,5 @@ async fn main() {let args: Vec<_> = std::env::args().collect();Worker::run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())- .await.expect("failed to run worker");}diff --git a/src/bin/cuely.rs b/src/bin/cuely.rsindex 931e15ea..0024066b 100644--- a/src/bin/cuely.rs+++ b/src/bin/cuely.rs@@ -47,7 +47,6 @@ async fn main() {}Config::Webgraph(config) => WebgraphBuilder::from(config).run()- .await.expect("Failed to build webgraph"),}}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex 75c85561..ae2cda44 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -36,13 +36,13 @@ impl From<WebgraphConfig> for WebgraphBuilder {}}-#[derive(Serialize, Deserialize, Clone)]+#[derive(Debug, Serialize, Deserialize, Clone)]enum JobConfig {- HTTP(HttpConfig),+ Http(HttpConfig),Local(LocalConfig),}-#[derive(Serialize, Deserialize)]+#[derive(Debug, Serialize, Deserialize)]struct Job {config: JobConfig,warc_path: String,@@ -57,7 +57,7 @@ impl Map<FrozenWebgraph> for Job {let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));let source = match self.config {- JobConfig::HTTP(config) => WarcSource::HTTP(config),+ JobConfig::Http(config) => WarcSource::HTTP(config),JobConfig::Local(config) => WarcSource::Local(config),};@@ -91,18 +91,26 @@ impl Map<FrozenWebgraph> for Job {}impl Reduce<FrozenWebgraph> for FrozenWebgraph {- fn reduce(mut self, other: FrozenWebgraph) -> FrozenWebgraph {+ fn reduce(self, other: FrozenWebgraph) -> FrozenWebgraph {let mut graph: Webgraph = self.into();++ let other_path = match &other.root {+ crate::directory::DirEntry::Folder { name, entries: _ } => name.clone(),+ crate::directory::DirEntry::File { name, content: _ } => name.clone(),+ };+let other = other.into();graph.merge(other);+ std::fs::remove_dir_all(other_path).unwrap();+graph.into()}}impl WebgraphBuilder {- async fn run_master(config: &WebgraphMasterConfig) -> Result<()> {+ fn run_master(config: &WebgraphMasterConfig) -> Result<()> {info!("Running master for webgraph construction");let warc_paths = config.warc_source.paths()?;@@ -115,7 +123,7 @@ impl WebgraphBuilder {let job_config = match config.warc_source.clone() {WarcSource::S3(_) => todo!("s3 not supported yet"),- WarcSource::HTTP(config) => JobConfig::HTTP(config),+ WarcSource::HTTP(config) => JobConfig::Http(config),WarcSource::Local(config) => JobConfig::Local(config),};@@ -130,20 +138,19 @@ impl WebgraphBuilder {warc_paths.into_iter()+ .take(10).map(|warc_path| Job {config: job_config.clone(),warc_path,})- .progress_with(pb).map_reduce(&workers)- .await.expect("failed to build webgraph");Ok(())}- async fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {- Worker::run::<Job, FrozenWebgraph>(config.addr.parse::<SocketAddr>().unwrap()).await?;+ fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {+ Worker::run::<Job, FrozenWebgraph>(config.addr.parse::<SocketAddr>().unwrap())?;Ok(())}@@ -152,7 +159,7 @@ impl WebgraphBuilder {let job_config = match config.warc_source.clone() {WarcSource::S3(_) => todo!("s3 not supported yet"),- WarcSource::HTTP(config) => JobConfig::HTTP(config),+ WarcSource::HTTP(config) => JobConfig::Http(config),WarcSource::Local(config) => JobConfig::Local(config),};@@ -171,10 +178,10 @@ impl WebgraphBuilder {Ok(())}- pub async fn run(&self) -> Result<()> {+ pub fn run(&self) -> Result<()> {match &self.config {- WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config).await,- WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config).await,+ WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),+ WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),WebgraphConfig::Local(config) => WebgraphBuilder::run_locally(config),}}diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsindex e686d6fc..240ad953 100644--- a/src/mapreduce/manager.rs+++ b/src/mapreduce/manager.rs@@ -1,7 +1,8 @@-use std::net::SocketAddr;-use std::ops::Deref;+use std::io::{Read, Write};+use std::net::{SocketAddr, TcpStream};+use std::ops::{Deref, DerefMut};use std::sync::atomic::{AtomicU32, Ordering};-use std::sync::Arc;+use std::sync::{Arc, Mutex};use std::time::Duration;use crate::mapreduce::Task;@@ -9,15 +10,11 @@ use crate::mapreduce::Task;use super::{Error, Result};use super::{Map, Reduce};use async_channel::{unbounded, Receiver, Sender};-use futures::{stream::FuturesUnordered, StreamExt};+use rayon::iter::ParallelBridge;+use rayon::prelude::*;use std::net::ToSocketAddrs;-use tokio::{- io::{AsyncReadExt, AsyncWriteExt},- net::TcpStream,-};use tokio_retry::strategy::ExponentialBackoff;-use tokio_retry::Retry;-use tracing::debug;+use tracing::{debug, warn};#[derive(Debug)]struct RemoteWorker {@@ -32,32 +29,35 @@ impl RemoteWorker {ExponentialBackoff::from_millis(10).take(5)}- async fn connect(&self) -> Result<TcpStream> {+ fn connect(&self) -> Result<TcpStream> {debug!("connecting to {:}", self.addr);- let stream = Retry::spawn(RemoteWorker::retry_strategy(), || async {- TcpStream::connect(self.addr).await- })- .await?;- debug!("connected");+ for dur in RemoteWorker::retry_strategy() {+ if let Ok(stream) = TcpStream::connect(&self.addr) {+ debug!("connected");+ return Ok(stream);+ }++ std::thread::sleep(dur);+ }- Ok(stream)+ Err(Error::NoResponse)}- async fn perform<I, O>(&self, job: &I) -> Result<O>+ fn perform<I, O>(&self, job: &I) -> Result<O>where- I: Map<O>,+ I: Map<O> + Send,O: Reduce<O> + Send,{- let mut stream = self.connect().await?;+ let mut stream = self.connect()?;let serialized_job = bincode::serialize(&Task::Job(job))?;debug!("sending {:?} bytes", serialized_job.len());- stream.write_all(&serialized_job).await?;- stream.write_all(&END_OF_MESSAGE).await?;+ stream.write_all(&serialized_job)?;+ stream.write_all(&END_OF_MESSAGE)?;let mut buf = [0; BUF_SIZE];let mut bytes = Vec::new();loop {- if let Ok(size) = stream.read(&mut buf).await {+ if let Ok(size) = stream.read(&mut buf) {debug!("read {:?} bytes", size);if size == 0 && bytes.is_empty() {return Err(Error::NoResponse);@@ -77,17 +77,17 @@ impl RemoteWorker {Ok(bincode::deserialize(&bytes)?)}- async fn stop<I, O>(&self) -> Result<()>+ fn stop<I, O>(&self) -> Result<()>where- I: Map<O>,+ I: Map<O> + Send,O: Reduce<O> + Send,{debug!("closing worker {:}", self.addr);- let mut stream = self.connect().await?;+ let mut stream = self.connect()?;let serialized_job = bincode::serialize(&Task::<I>::AllFinished)?;debug!("sending {:?} bytes", serialized_job.len());- stream.write_all(&serialized_job).await?;- stream.write_all(&END_OF_MESSAGE).await?;+ stream.write_all(&serialized_job)?;+ stream.write_all(&END_OF_MESSAGE)?;Ok(())}}@@ -105,8 +105,8 @@ impl<'a> WorkerGuard<'a> {}}- async fn success(self) {- self.from_pool.insert(Arc::clone(&self.worker)).await+ fn success(self) {+ self.from_pool.insert(Arc::clone(&self.worker))}}@@ -126,16 +126,16 @@ impl<'a> Drop for WorkerGuard<'a> {struct WorkerPool {all_workers: Vec<Arc<RemoteWorker>>,- alive_workers: (Sender<Arc<RemoteWorker>>, Receiver<Arc<RemoteWorker>>),+ ready_workers: Mutex<Vec<Arc<RemoteWorker>>>,running_workers: AtomicU32,}impl WorkerPool {- async fn new<A>(workers: &[A]) -> Self+ fn new<A>(workers: &[A]) -> SelfwhereA: ToSocketAddrs + std::fmt::Debug,{- let all_workers = workers+ let all_workers: Vec<Arc<RemoteWorker>> = workers.iter().flat_map(|addr| {addr.to_socket_addrs().unwrap_or_else(|_| {@@ -145,15 +145,9 @@ impl WorkerPool {.map(|addr| Arc::new(RemoteWorker { addr })).collect();- let (s, r) = unbounded();-- for worker in &all_workers {- s.send(Arc::clone(worker)).await.unwrap();- }-Self {+ ready_workers: Mutex::new(all_workers.clone()),all_workers,- alive_workers: (s, r),running_workers: AtomicU32::new(0),}}@@ -162,30 +156,32 @@ impl WorkerPool {self.running_workers.fetch_sub(1, Ordering::SeqCst);}- async fn insert(&self, worker: Arc<RemoteWorker>) {- let ch = self.alive_workers.0.clone();- ch.send(worker).await.unwrap();+ fn insert(&self, worker: Arc<RemoteWorker>) {+ self.ready_workers.lock().unwrap().push(worker);}- async fn get_worker(&self) -> Result<WorkerGuard<'_>> {- if self.alive_workers.0.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {+ fn get_worker(&self) -> Result<Option<WorkerGuard<'_>>> {+ let mut ready_workers = self.ready_workers.lock().unwrap();+ if ready_workers.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {return Err(Error::NoAvailableWorker);}- let worker = self.alive_workers.1.recv().await?;- self.running_workers.fetch_add(1, Ordering::SeqCst);-- Ok(WorkerGuard::new(self, worker))+ if let Some(worker) = ready_workers.pop() {+ self.running_workers.fetch_add(1, Ordering::SeqCst);+ Ok(Some(WorkerGuard::new(self, worker)))+ } else {+ Ok(None)+ }}- async fn stop_workers<I, O>(&self)+ fn stop_workers<I, O>(&self)where- I: Map<O>,+ I: Map<O> + Send,O: Reduce<O> + Send,{let mut failing_workers = Vec::new();for worker in &self.all_workers {- if worker.stop::<I, O>().await.is_err() {+ if worker.stop::<I, O>().is_err() {failing_workers.push(worker)}}@@ -204,40 +200,46 @@ pub struct Manager {}impl Manager {- pub async fn new<A>(workers: &[A]) -> Self+ pub fn new<A>(workers: &[A]) -> SelfwhereA: ToSocketAddrs + std::fmt::Debug,{Self {- pool: WorkerPool::new(workers).await,+ pool: WorkerPool::new(workers),}}- async fn try_map<I, O>(&self, job: &I) -> Result<O>+ fn try_map<I, O>(&self, job: &I) -> Result<O>where- I: Map<O>,+ I: Map<O> + Send,O: Reduce<O> + Send,{- let worker = self.pool.get_worker().await?;- let res = worker.perform(job).await?;- worker.success().await;+ loop {+ match self.pool.get_worker()? {+ Some(worker) => {+ let res = worker.perform(job)?;+ worker.success();- Ok(res)+ return Ok(res);+ }+ None => std::thread::sleep(std::time::Duration::from_millis(1000)),+ }+ }}/// Execute job on one of the remote machines. If the remote machine fails for some reason,/// the job should be allocated to another machine.- pub async fn map<I, O>(&self, job: I) -> O+ pub fn map<I, O>(&self, job: I) -> Owhere- I: Map<O>,+ I: Map<O> + Send,O: Reduce<O> + Send,{loop {- match self.try_map(&job).await {+ match self.try_map(&job) {Ok(res) => return res,Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),Err(_) => {- debug!("got err - rescheduling job");+ warn!("Worker failed - rescheduling job");}}}@@ -250,32 +252,31 @@ impl Manager {}}- async fn get_results<I, O>(&self, jobs: Vec<I>) -> Option<O>+ fn get_results<I, O>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O>where- I: Map<O>,- O: Reduce<O> + Send,+ I: Map<O> + Send,+ O: Reduce<O> + Send + Sync,{- jobs.into_iter()+ let acc = Arc::new(Mutex::new(None));++ jobs.par_bridge().map(|job| self.map::<I, O>(job))- .collect::<FuturesUnordered<_>>()- .map(|d| {- println!("TEST");- d- })- .fold(- None,- |acc, elem| async move { Some(Manager::reduce(acc, elem)) },- )- .await+ .for_each(|res| {+ let mut lock = acc.lock().unwrap();+ *lock = Some(Manager::reduce(lock.take(), res));+ });++ let x = acc.lock().unwrap().take();+ x}- pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>+ pub fn run<I, O>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O>where- I: Map<O>,- O: Reduce<O> + Send,+ I: Map<O> + Send + Sync,+ O: Reduce<O> + Send + Sync,{- let result = self.get_results(jobs).await;- self.pool.stop_workers::<I, O>().await;+ let result = self.get_results(jobs);+ self.pool.stop_workers::<I, O>();result}diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsindex d2b06fd4..f6e9bfb4 100644--- a/src/mapreduce/mod.rs+++ b/src/mapreduce/mod.rs@@ -64,20 +64,20 @@ where#[async_trait]pub trait MapReduce<I, O>where- Self: Sized + Iterator<Item = I>,+ Self: Sized + Iterator<Item = I> + Send,I: Map<O> + Sync,- O: Reduce<O> + Send,+ O: Reduce<O> + Send + Sync,{- async fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {- let manager = Manager::new(workers).await;- manager.run(self.collect()).await+ fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {+ let manager = Manager::new(workers);+ manager.run(self)}}impl<I, O, T> MapReduce<I, O> for Twhere- T: Iterator<Item = I> + Sized,+ T: Iterator<Item = I> + Sized + Send,I: Map<O> + Sync,- O: Reduce<O> + Send,+ O: Reduce<O> + Send + Sync,{}diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rsindex 0e38e4dd..35f1d46e 100644--- a/src/mapreduce/worker.rs+++ b/src/mapreduce/worker.rs@@ -1,12 +1,12 @@-use std::net::SocketAddr;+use std::{+ io::{Read, Write},+ net::{SocketAddr, TcpListener},+};use crate::mapreduce::manager::{BUF_SIZE, END_OF_MESSAGE};use super::{Error, Map, Result, Task};use serde::{de::DeserializeOwned, Serialize};-use tokio::io::AsyncReadExt;-use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};-use tokio::net::TcpListener;use tracing::{debug, info};pub struct Worker {}@@ -17,16 +17,16 @@ enum State {}impl Worker {- async fn run_stream<I, O, S>(mut stream: S) -> Result<State>+ fn run_stream<I, O, S>(mut stream: S) -> Result<State>whereI: Map<O>,O: Serialize + DeserializeOwned + Send,- S: AsyncRead + AsyncWrite + Unpin,+ S: Read + Write + Unpin,{let mut buf = [0; BUF_SIZE];let mut bytes = Vec::new();loop {- if let Ok(size) = stream.read(&mut buf).await {+ if let Ok(size) = stream.read(&mut buf) {debug!("read {:?} bytes", size);if size == 0 && bytes.is_empty() {return Err(Error::NoResponse);@@ -50,8 +50,8 @@ impl Worker {let res = job.map();let bytes = bincode::serialize(&res)?;debug!("serialized result into {} bytes", bytes.len());- stream.write_all(&bytes).await?;- stream.write_all(&END_OF_MESSAGE).await?;+ stream.write_all(&bytes)?;+ stream.write_all(&END_OF_MESSAGE)?;}Task::AllFinished => {debug!("shutting down");@@ -62,18 +62,18 @@ impl Worker {Ok(State::Continue)}- pub async fn run<I, O>(addr: SocketAddr) -> Result<()>+ pub fn run<I, O>(addr: SocketAddr) -> Result<()>whereI: Map<O>,O: Serialize + DeserializeOwned + Send,{- let listener = TcpListener::bind(addr).await?;+ let listener = TcpListener::bind(addr)?;info!("worker listening on: {:}", addr);loop {- let (socket, _) = listener.accept().await?;+ let (socket, _) = listener.accept()?;debug!("received connection");- match Worker::run_stream::<I, O, _>(socket).await? {+ match Worker::run_stream::<I, O, _>(socket)? {State::Finished => break,State::Continue => {}}@@ -111,49 +111,28 @@ mod tests {}}- impl AsyncRead for MockTcpStream {- fn poll_read(- self: Pin<&mut Self>,- _: &mut Context<'_>,- buf: &mut ReadBuf<'_>,- ) -> Poll<std::io::Result<()>> {- let mut_self = self.get_mut();-- if mut_self.num_read == 0 {- buf.put_slice(&mut_self.contents[..]);- } else if mut_self.num_read == 1 {- buf.put_slice(&END_OF_MESSAGE);+ impl Read for MockTcpStream {+ fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {+ if self.num_read == 0 {+ buf.write_all(&self.contents[..]);+ } else if self.num_read == 1 {+ buf.write_all(&END_OF_MESSAGE);}- mut_self.contents = Vec::new();- mut_self.num_read += 1;-- Poll::Ready(Ok(()))+ self.contents = Vec::new();+ self.num_read += 1;+ Ok(buf.len())}}- impl AsyncWrite for MockTcpStream {- fn poll_write(- self: Pin<&mut Self>,- _: &mut Context<'_>,- buf: &[u8],- ) -> Poll<std::result::Result<usize, std::io::Error>> {- self.get_mut().result.extend(Vec::from(buf));- Poll::Ready(Ok(buf.len()))- }-- fn poll_flush(- self: Pin<&mut Self>,- _: &mut Context<'_>,- ) -> Poll<std::result::Result<(), std::io::Error>> {- Poll::Ready(Ok(()))+ impl Write for MockTcpStream {+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {+ self.result.extend(Vec::from(buf));+ Ok(buf.len())}- fn poll_shutdown(- self: Pin<&mut Self>,- _: &mut Context<'_>,- ) -> Poll<std::result::Result<(), std::io::Error>> {- Poll::Ready(Ok(()))+ fn flush(&mut self) -> std::io::Result<()> {+ Ok(())}}@@ -171,15 +150,12 @@ mod tests {}}- #[tokio::test]- async fn execute() {+ fn execute() {let contents = vec![1, 2, 0, 1, 0, 1, 0];let job = bincode::serialize(&Task::Job(MockJob { contents })).unwrap();let mut stream = MockTcpStream::new(job);- Worker::run_stream::<MockJob, _, _>(&mut stream)- .await- .expect("worker failed");+ Worker::run_stream::<MockJob, _, _>(&mut stream).expect("worker failed");let result_bytes = &stream.result[..stream.result.len() - END_OF_MESSAGE.len()];let res: Count = bincode::deserialize(result_bytes).unwrap();