Skip to content

web search done right

concurrent reduce and map. Mapreduce doesn't use async anymore

Mikkel Denker7db1d92dba33
diff --git a/Cargo.toml b/Cargo.toml
index 307b4390..907809ec 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -24,6 +24,7 @@ itertools = "0.10.3"
lru = "0.7.6"
nom = "7.1.1"
rand = "0.8.5"
+rayon = "1.5.3"
reqwest = "0.11.10"
rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rs
index b1fe363d..e7d2cc1c 100644
--- a/examples/mapreduce_manager.rs
+++ b/examples/mapreduce_manager.rs
@@ -2,7 +2,7 @@ use std::net::SocketAddr;
use cuely::mapreduce::{Map, MapReduce, Reduce};
use serde::{Deserialize, Serialize};
-use tracing::Level;
+use tracing::{debug, Level};
use tracing_subscriber::FmtSubscriber;
#[derive(Serialize, Deserialize, Debug)]
@@ -10,24 +10,28 @@ struct Job {
id: usize,
}
+#[derive(Serialize, Deserialize, Debug)]
+struct Count(usize);
+
impl Map<Count> for Job {
fn map(self) -> Count {
+ debug!("begin map");
std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
+ debug!("end map");
Count(1)
}
}
-#[derive(Serialize, Deserialize, Debug)]
-struct Count(usize);
-
impl Reduce for Count {
fn reduce(self, element: Self) -> Self {
+ debug!("begin reduce");
+ std::thread::sleep(std::time::Duration::from_secs(20));
+ debug!("end reduce");
Count(self.0 + element.0)
}
}
-#[tokio::main]
-async fn main() {
+fn main() {
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG)
.finish();
@@ -44,12 +48,7 @@ async fn main() {
let res = jobs
.into_iter()
- .map_reduce(&[
- "0.0.0.0:1337".parse::<SocketAddr>().unwrap(),
- "0.0.0.0:1338".parse::<SocketAddr>().unwrap(),
- "0.0.0.0:1339".parse::<SocketAddr>().unwrap(),
- ])
- .await
+ .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])
.unwrap();
println!("{:?}", res.0);
}
diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rs
index ca153f9d..d81df6f4 100644
--- a/examples/mapreduce_worker.rs
+++ b/examples/mapreduce_worker.rs
@@ -2,7 +2,7 @@ use std::net::SocketAddr;
use cuely::mapreduce::{Map, Reduce, Worker};
use serde::{Deserialize, Serialize};
-use tracing::Level;
+use tracing::{debug, Level};
use tracing_subscriber::FmtSubscriber;
#[derive(Serialize, Deserialize, Debug)]
@@ -15,19 +15,23 @@ struct Count(usize);
impl Map<Count> for Job {
fn map(self) -> Count {
- std::thread::sleep(std::time::Duration::from_secs(2)); // simulate some long running task
+ debug!("begin map {:?}", self);
+ std::thread::sleep(std::time::Duration::from_secs(20)); // simulate some long running task
+ debug!("end map");
Count(1)
}
}
impl Reduce for Count {
fn reduce(self, element: Self) -> Self {
+ debug!("begin reduce");
+ std::thread::sleep(std::time::Duration::from_secs(20));
+ debug!("end reduce");
Count(self.0 + element.0)
}
}
-#[tokio::main]
-async fn main() {
+fn main() {
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::DEBUG)
.finish();
@@ -36,6 +40,5 @@ async fn main() {
let args: Vec<_> = std::env::args().collect();
Worker::run::<Job, Count>(args[1].parse::<SocketAddr>().unwrap())
- .await
.expect("failed to run worker");
}
diff --git a/src/bin/cuely.rs b/src/bin/cuely.rs
index 931e15ea..0024066b 100644
--- a/src/bin/cuely.rs
+++ b/src/bin/cuely.rs
@@ -47,7 +47,6 @@ async fn main() {
}
Config::Webgraph(config) => WebgraphBuilder::from(config)
.run()
- .await
.expect("Failed to build webgraph"),
}
}
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index 75c85561..ae2cda44 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -36,13 +36,13 @@ impl From<WebgraphConfig> for WebgraphBuilder {
}
}
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
enum JobConfig {
- HTTP(HttpConfig),
+ Http(HttpConfig),
Local(LocalConfig),
}
-#[derive(Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize)]
struct Job {
config: JobConfig,
warc_path: String,
@@ -57,7 +57,7 @@ impl Map<FrozenWebgraph> for Job {
let mut graph = Webgraph::<SledStore>::open(Path::new("webgraph").join(name));
let source = match self.config {
- JobConfig::HTTP(config) => WarcSource::HTTP(config),
+ JobConfig::Http(config) => WarcSource::HTTP(config),
JobConfig::Local(config) => WarcSource::Local(config),
};
@@ -91,18 +91,26 @@ impl Map<FrozenWebgraph> for Job {
}
impl Reduce<FrozenWebgraph> for FrozenWebgraph {
- fn reduce(mut self, other: FrozenWebgraph) -> FrozenWebgraph {
+ fn reduce(self, other: FrozenWebgraph) -> FrozenWebgraph {
let mut graph: Webgraph = self.into();
+
+ let other_path = match &other.root {
+ crate::directory::DirEntry::Folder { name, entries: _ } => name.clone(),
+ crate::directory::DirEntry::File { name, content: _ } => name.clone(),
+ };
+
let other = other.into();
graph.merge(other);
+ std::fs::remove_dir_all(other_path).unwrap();
+
graph.into()
}
}
impl WebgraphBuilder {
- async fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
+ fn run_master(config: &WebgraphMasterConfig) -> Result<()> {
info!("Running master for webgraph construction");
let warc_paths = config.warc_source.paths()?;
@@ -115,7 +123,7 @@ impl WebgraphBuilder {
let job_config = match config.warc_source.clone() {
WarcSource::S3(_) => todo!("s3 not supported yet"),
- WarcSource::HTTP(config) => JobConfig::HTTP(config),
+ WarcSource::HTTP(config) => JobConfig::Http(config),
WarcSource::Local(config) => JobConfig::Local(config),
};
@@ -130,20 +138,19 @@ impl WebgraphBuilder {
warc_paths
.into_iter()
+ .take(10)
.map(|warc_path| Job {
config: job_config.clone(),
warc_path,
})
- .progress_with(pb)
.map_reduce(&workers)
- .await
.expect("failed to build webgraph");
Ok(())
}
- async fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {
- Worker::run::<Job, FrozenWebgraph>(config.addr.parse::<SocketAddr>().unwrap()).await?;
+ fn run_worker(config: &WebgraphWorkerConfig) -> Result<()> {
+ Worker::run::<Job, FrozenWebgraph>(config.addr.parse::<SocketAddr>().unwrap())?;
Ok(())
}
@@ -152,7 +159,7 @@ impl WebgraphBuilder {
let job_config = match config.warc_source.clone() {
WarcSource::S3(_) => todo!("s3 not supported yet"),
- WarcSource::HTTP(config) => JobConfig::HTTP(config),
+ WarcSource::HTTP(config) => JobConfig::Http(config),
WarcSource::Local(config) => JobConfig::Local(config),
};
@@ -171,10 +178,10 @@ impl WebgraphBuilder {
Ok(())
}
- pub async fn run(&self) -> Result<()> {
+ pub fn run(&self) -> Result<()> {
match &self.config {
- WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config).await,
- WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config).await,
+ WebgraphConfig::Master(config) => WebgraphBuilder::run_master(config),
+ WebgraphConfig::Worker(config) => WebgraphBuilder::run_worker(config),
WebgraphConfig::Local(config) => WebgraphBuilder::run_locally(config),
}
}
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
index e686d6fc..240ad953 100644
--- a/src/mapreduce/manager.rs
+++ b/src/mapreduce/manager.rs
@@ -1,7 +1,8 @@
-use std::net::SocketAddr;
-use std::ops::Deref;
+use std::io::{Read, Write};
+use std::net::{SocketAddr, TcpStream};
+use std::ops::{Deref, DerefMut};
use std::sync::atomic::{AtomicU32, Ordering};
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
use std::time::Duration;
use crate::mapreduce::Task;
@@ -9,15 +10,11 @@ use crate::mapreduce::Task;
use super::{Error, Result};
use super::{Map, Reduce};
use async_channel::{unbounded, Receiver, Sender};
-use futures::{stream::FuturesUnordered, StreamExt};
+use rayon::iter::ParallelBridge;
+use rayon::prelude::*;
use std::net::ToSocketAddrs;
-use tokio::{
- io::{AsyncReadExt, AsyncWriteExt},
- net::TcpStream,
-};
use tokio_retry::strategy::ExponentialBackoff;
-use tokio_retry::Retry;
-use tracing::debug;
+use tracing::{debug, warn};
#[derive(Debug)]
struct RemoteWorker {
@@ -32,32 +29,35 @@ impl RemoteWorker {
ExponentialBackoff::from_millis(10).take(5)
}
- async fn connect(&self) -> Result<TcpStream> {
+ fn connect(&self) -> Result<TcpStream> {
debug!("connecting to {:}", self.addr);
- let stream = Retry::spawn(RemoteWorker::retry_strategy(), || async {
- TcpStream::connect(self.addr).await
- })
- .await?;
- debug!("connected");
+ for dur in RemoteWorker::retry_strategy() {
+ if let Ok(stream) = TcpStream::connect(&self.addr) {
+ debug!("connected");
+ return Ok(stream);
+ }
+
+ std::thread::sleep(dur);
+ }
- Ok(stream)
+ Err(Error::NoResponse)
}
- async fn perform<I, O>(&self, job: &I) -> Result<O>
+ fn perform<I, O>(&self, job: &I) -> Result<O>
where
- I: Map<O>,
+ I: Map<O> + Send,
O: Reduce<O> + Send,
{
- let mut stream = self.connect().await?;
+ let mut stream = self.connect()?;
let serialized_job = bincode::serialize(&Task::Job(job))?;
debug!("sending {:?} bytes", serialized_job.len());
- stream.write_all(&serialized_job).await?;
- stream.write_all(&END_OF_MESSAGE).await?;
+ stream.write_all(&serialized_job)?;
+ stream.write_all(&END_OF_MESSAGE)?;
let mut buf = [0; BUF_SIZE];
let mut bytes = Vec::new();
loop {
- if let Ok(size) = stream.read(&mut buf).await {
+ if let Ok(size) = stream.read(&mut buf) {
debug!("read {:?} bytes", size);
if size == 0 && bytes.is_empty() {
return Err(Error::NoResponse);
@@ -77,17 +77,17 @@ impl RemoteWorker {
Ok(bincode::deserialize(&bytes)?)
}
- async fn stop<I, O>(&self) -> Result<()>
+ fn stop<I, O>(&self) -> Result<()>
where
- I: Map<O>,
+ I: Map<O> + Send,
O: Reduce<O> + Send,
{
debug!("closing worker {:}", self.addr);
- let mut stream = self.connect().await?;
+ let mut stream = self.connect()?;
let serialized_job = bincode::serialize(&Task::<I>::AllFinished)?;
debug!("sending {:?} bytes", serialized_job.len());
- stream.write_all(&serialized_job).await?;
- stream.write_all(&END_OF_MESSAGE).await?;
+ stream.write_all(&serialized_job)?;
+ stream.write_all(&END_OF_MESSAGE)?;
Ok(())
}
}
@@ -105,8 +105,8 @@ impl<'a> WorkerGuard<'a> {
}
}
- async fn success(self) {
- self.from_pool.insert(Arc::clone(&self.worker)).await
+ fn success(self) {
+ self.from_pool.insert(Arc::clone(&self.worker))
}
}
@@ -126,16 +126,16 @@ impl<'a> Drop for WorkerGuard<'a> {
struct WorkerPool {
all_workers: Vec<Arc<RemoteWorker>>,
- alive_workers: (Sender<Arc<RemoteWorker>>, Receiver<Arc<RemoteWorker>>),
+ ready_workers: Mutex<Vec<Arc<RemoteWorker>>>,
running_workers: AtomicU32,
}
impl WorkerPool {
- async fn new<A>(workers: &[A]) -> Self
+ fn new<A>(workers: &[A]) -> Self
where
A: ToSocketAddrs + std::fmt::Debug,
{
- let all_workers = workers
+ let all_workers: Vec<Arc<RemoteWorker>> = workers
.iter()
.flat_map(|addr| {
addr.to_socket_addrs().unwrap_or_else(|_| {
@@ -145,15 +145,9 @@ impl WorkerPool {
.map(|addr| Arc::new(RemoteWorker { addr }))
.collect();
- let (s, r) = unbounded();
-
- for worker in &all_workers {
- s.send(Arc::clone(worker)).await.unwrap();
- }
-
Self {
+ ready_workers: Mutex::new(all_workers.clone()),
all_workers,
- alive_workers: (s, r),
running_workers: AtomicU32::new(0),
}
}
@@ -162,30 +156,32 @@ impl WorkerPool {
self.running_workers.fetch_sub(1, Ordering::SeqCst);
}
- async fn insert(&self, worker: Arc<RemoteWorker>) {
- let ch = self.alive_workers.0.clone();
- ch.send(worker).await.unwrap();
+ fn insert(&self, worker: Arc<RemoteWorker>) {
+ self.ready_workers.lock().unwrap().push(worker);
}
- async fn get_worker(&self) -> Result<WorkerGuard<'_>> {
- if self.alive_workers.0.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {
+ fn get_worker(&self) -> Result<Option<WorkerGuard<'_>>> {
+ let mut ready_workers = self.ready_workers.lock().unwrap();
+ if ready_workers.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {
return Err(Error::NoAvailableWorker);
}
- let worker = self.alive_workers.1.recv().await?;
- self.running_workers.fetch_add(1, Ordering::SeqCst);
-
- Ok(WorkerGuard::new(self, worker))
+ if let Some(worker) = ready_workers.pop() {
+ self.running_workers.fetch_add(1, Ordering::SeqCst);
+ Ok(Some(WorkerGuard::new(self, worker)))
+ } else {
+ Ok(None)
+ }
}
- async fn stop_workers<I, O>(&self)
+ fn stop_workers<I, O>(&self)
where
- I: Map<O>,
+ I: Map<O> + Send,
O: Reduce<O> + Send,
{
let mut failing_workers = Vec::new();
for worker in &self.all_workers {
- if worker.stop::<I, O>().await.is_err() {
+ if worker.stop::<I, O>().is_err() {
failing_workers.push(worker)
}
}
@@ -204,40 +200,46 @@ pub struct Manager {
}
impl Manager {
- pub async fn new<A>(workers: &[A]) -> Self
+ pub fn new<A>(workers: &[A]) -> Self
where
A: ToSocketAddrs + std::fmt::Debug,
{
Self {
- pool: WorkerPool::new(workers).await,
+ pool: WorkerPool::new(workers),
}
}
- async fn try_map<I, O>(&self, job: &I) -> Result<O>
+ fn try_map<I, O>(&self, job: &I) -> Result<O>
where
- I: Map<O>,
+ I: Map<O> + Send,
O: Reduce<O> + Send,
{
- let worker = self.pool.get_worker().await?;
- let res = worker.perform(job).await?;
- worker.success().await;
+ loop {
+ match self.pool.get_worker()? {
+ Some(worker) => {
+ let res = worker.perform(job)?;
+ worker.success();
- Ok(res)
+ return Ok(res);
+ }
+ None => std::thread::sleep(std::time::Duration::from_millis(1000)),
+ }
+ }
}
/// Execute job on one of the remote machines. If the remote machine fails for some reason,
/// the job should be allocated to another machine.
- pub async fn map<I, O>(&self, job: I) -> O
+ pub fn map<I, O>(&self, job: I) -> O
where
- I: Map<O>,
+ I: Map<O> + Send,
O: Reduce<O> + Send,
{
loop {
- match self.try_map(&job).await {
+ match self.try_map(&job) {
Ok(res) => return res,
Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),
Err(_) => {
- debug!("got err - rescheduling job");
+ warn!("Worker failed - rescheduling job");
}
}
}
@@ -250,32 +252,31 @@ impl Manager {
}
}
- async fn get_results<I, O>(&self, jobs: Vec<I>) -> Option<O>
+ fn get_results<I, O>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O>
where
- I: Map<O>,
- O: Reduce<O> + Send,
+ I: Map<O> + Send,
+ O: Reduce<O> + Send + Sync,
{
- jobs.into_iter()
+ let acc = Arc::new(Mutex::new(None));
+
+ jobs.par_bridge()
.map(|job| self.map::<I, O>(job))
- .collect::<FuturesUnordered<_>>()
- .map(|d| {
- println!("TEST");
- d
- })
- .fold(
- None,
- |acc, elem| async move { Some(Manager::reduce(acc, elem)) },
- )
- .await
+ .for_each(|res| {
+ let mut lock = acc.lock().unwrap();
+ *lock = Some(Manager::reduce(lock.take(), res));
+ });
+
+ let x = acc.lock().unwrap().take();
+ x
}
- pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>
+ pub fn run<I, O>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O>
where
- I: Map<O>,
- O: Reduce<O> + Send,
+ I: Map<O> + Send + Sync,
+ O: Reduce<O> + Send + Sync,
{
- let result = self.get_results(jobs).await;
- self.pool.stop_workers::<I, O>().await;
+ let result = self.get_results(jobs);
+ self.pool.stop_workers::<I, O>();
result
}
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
index d2b06fd4..f6e9bfb4 100644
--- a/src/mapreduce/mod.rs
+++ b/src/mapreduce/mod.rs
@@ -64,20 +64,20 @@ where
#[async_trait]
pub trait MapReduce<I, O>
where
- Self: Sized + Iterator<Item = I>,
+ Self: Sized + Iterator<Item = I> + Send,
I: Map<O> + Sync,
- O: Reduce<O> + Send,
+ O: Reduce<O> + Send + Sync,
{
- async fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {
- let manager = Manager::new(workers).await;
- manager.run(self.collect()).await
+ fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {
+ let manager = Manager::new(workers);
+ manager.run(self)
}
}
impl<I, O, T> MapReduce<I, O> for T
where
- T: Iterator<Item = I> + Sized,
+ T: Iterator<Item = I> + Sized + Send,
I: Map<O> + Sync,
- O: Reduce<O> + Send,
+ O: Reduce<O> + Send + Sync,
{
}
diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rs
index 0e38e4dd..35f1d46e 100644
--- a/src/mapreduce/worker.rs
+++ b/src/mapreduce/worker.rs
@@ -1,12 +1,12 @@
-use std::net::SocketAddr;
+use std::{
+ io::{Read, Write},
+ net::{SocketAddr, TcpListener},
+};
use crate::mapreduce::manager::{BUF_SIZE, END_OF_MESSAGE};
use super::{Error, Map, Result, Task};
use serde::{de::DeserializeOwned, Serialize};
-use tokio::io::AsyncReadExt;
-use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
-use tokio::net::TcpListener;
use tracing::{debug, info};
pub struct Worker {}
@@ -17,16 +17,16 @@ enum State {
}
impl Worker {
- async fn run_stream<I, O, S>(mut stream: S) -> Result<State>
+ fn run_stream<I, O, S>(mut stream: S) -> Result<State>
where
I: Map<O>,
O: Serialize + DeserializeOwned + Send,
- S: AsyncRead + AsyncWrite + Unpin,
+ S: Read + Write + Unpin,
{
let mut buf = [0; BUF_SIZE];
let mut bytes = Vec::new();
loop {
- if let Ok(size) = stream.read(&mut buf).await {
+ if let Ok(size) = stream.read(&mut buf) {
debug!("read {:?} bytes", size);
if size == 0 && bytes.is_empty() {
return Err(Error::NoResponse);
@@ -50,8 +50,8 @@ impl Worker {
let res = job.map();
let bytes = bincode::serialize(&res)?;
debug!("serialized result into {} bytes", bytes.len());
- stream.write_all(&bytes).await?;
- stream.write_all(&END_OF_MESSAGE).await?;
+ stream.write_all(&bytes)?;
+ stream.write_all(&END_OF_MESSAGE)?;
}
Task::AllFinished => {
debug!("shutting down");
@@ -62,18 +62,18 @@ impl Worker {
Ok(State::Continue)
}
- pub async fn run<I, O>(addr: SocketAddr) -> Result<()>
+ pub fn run<I, O>(addr: SocketAddr) -> Result<()>
where
I: Map<O>,
O: Serialize + DeserializeOwned + Send,
{
- let listener = TcpListener::bind(addr).await?;
+ let listener = TcpListener::bind(addr)?;
info!("worker listening on: {:}", addr);
loop {
- let (socket, _) = listener.accept().await?;
+ let (socket, _) = listener.accept()?;
debug!("received connection");
- match Worker::run_stream::<I, O, _>(socket).await? {
+ match Worker::run_stream::<I, O, _>(socket)? {
State::Finished => break,
State::Continue => {}
}
@@ -111,49 +111,28 @@ mod tests {
}
}
- impl AsyncRead for MockTcpStream {
- fn poll_read(
- self: Pin<&mut Self>,
- _: &mut Context<'_>,
- buf: &mut ReadBuf<'_>,
- ) -> Poll<std::io::Result<()>> {
- let mut_self = self.get_mut();
-
- if mut_self.num_read == 0 {
- buf.put_slice(&mut_self.contents[..]);
- } else if mut_self.num_read == 1 {
- buf.put_slice(&END_OF_MESSAGE);
+ impl Read for MockTcpStream {
+ fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
+ if self.num_read == 0 {
+ buf.write_all(&self.contents[..]);
+ } else if self.num_read == 1 {
+ buf.write_all(&END_OF_MESSAGE);
}
- mut_self.contents = Vec::new();
- mut_self.num_read += 1;
-
- Poll::Ready(Ok(()))
+ self.contents = Vec::new();
+ self.num_read += 1;
+ Ok(buf.len())
}
}
- impl AsyncWrite for MockTcpStream {
- fn poll_write(
- self: Pin<&mut Self>,
- _: &mut Context<'_>,
- buf: &[u8],
- ) -> Poll<std::result::Result<usize, std::io::Error>> {
- self.get_mut().result.extend(Vec::from(buf));
- Poll::Ready(Ok(buf.len()))
- }
-
- fn poll_flush(
- self: Pin<&mut Self>,
- _: &mut Context<'_>,
- ) -> Poll<std::result::Result<(), std::io::Error>> {
- Poll::Ready(Ok(()))
+ impl Write for MockTcpStream {
+ fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+ self.result.extend(Vec::from(buf));
+ Ok(buf.len())
}
- fn poll_shutdown(
- self: Pin<&mut Self>,
- _: &mut Context<'_>,
- ) -> Poll<std::result::Result<(), std::io::Error>> {
- Poll::Ready(Ok(()))
+ fn flush(&mut self) -> std::io::Result<()> {
+ Ok(())
}
}
@@ -171,15 +150,12 @@ mod tests {
}
}
- #[tokio::test]
- async fn execute() {
+ fn execute() {
let contents = vec![1, 2, 0, 1, 0, 1, 0];
let job = bincode::serialize(&Task::Job(MockJob { contents })).unwrap();
let mut stream = MockTcpStream::new(job);
- Worker::run_stream::<MockJob, _, _>(&mut stream)
- .await
- .expect("worker failed");
+ Worker::run_stream::<MockJob, _, _>(&mut stream).expect("worker failed");
let result_bytes = &stream.result[..stream.result.len() - END_OF_MESSAGE.len()];
let res: Count = bincode::deserialize(result_bytes).unwrap();