Skip to content

web search done right

simple mapreduce without much error handling

Mikkel Denker344dc3fd193b
diff --git a/Cargo.toml b/Cargo.toml
index 9edb9e82..cb0c8917 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,6 +11,7 @@ authors = ['Cuely ApS. <[email protected]>']
name = "cuely"
[dependencies]
+async-trait = "0.1.53"
bincode = "1.3.3"
encoding_rs = "0.8.31"
flate2 = "1.0.23"
@@ -19,6 +20,7 @@ htmlescape = "0.3.1"
indicatif = "0.16.2"
itertools = "0.10.3"
nom = "7.1.1"
+rand = "0.8.5"
reqwest = "0.11.10"
rusoto_core = "0.48.0"
rusoto_s3 = "0.48.0"
@@ -29,6 +31,8 @@ tantivy = "0.18.0"
thiserror = "1.0.31"
tokio = { version = "1.18.2", features = ["full"] }
toml = "0.5.9"
+tracing = "0.1.34"
+tracing-subscriber = "0.3.11"
whatlang = "0.16.0"
[dev-dependencies]
diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rs
new file mode 100644
index 00000000..baaf03eb
--- /dev/null
+++ b/examples/mapreduce_manager.rs
@@ -0,0 +1,53 @@
+use std::net::SocketAddr;
+
+use cuely::mapreduce::{Map, MapReduce, Reduce};
+use serde::{Deserialize, Serialize};
+use tracing::Level;
+use tracing_subscriber::FmtSubscriber;
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Job {
+ contents: Vec<usize>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Count(usize);
+
+impl Map<Count> for Job {
+ fn map(self) -> Count {
+ Count(self.contents.into_iter().filter(|d| *d == 0).count())
+ }
+}
+
+impl Reduce for Count {
+ fn reduce(self, element: Self) -> Self {
+ Count(self.0 + element.0)
+ }
+}
+
+#[tokio::main]
+async fn main() {
+ let subscriber = FmtSubscriber::builder()
+ .with_max_level(Level::DEBUG)
+ .finish();
+ tracing::subscriber::set_global_default(subscriber).unwrap();
+
+ let jobs = vec![
+ Job {
+ contents: vec![1, 2, 0, 1, 0, 1, 0],
+ },
+ Job {
+ contents: vec![3, 2, 1],
+ },
+ Job {
+ contents: vec![0, 0],
+ },
+ ];
+
+ let res = jobs
+ .into_iter()
+ .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])
+ .await
+ .unwrap();
+ println!("{:?}", res.0);
+}
diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rs
new file mode 100644
index 00000000..a8871916
--- /dev/null
+++ b/examples/mapreduce_worker.rs
@@ -0,0 +1,36 @@
+use std::net::SocketAddr;
+
+use cuely::mapreduce::{Map, Reduce, Worker};
+use serde::{Deserialize, Serialize};
+use tracing::Level;
+use tracing_subscriber::FmtSubscriber;
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Job {
+ contents: Vec<usize>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct Count(usize);
+
+impl Map<Count> for Job {
+ fn map(self) -> Count {
+ Count(self.contents.into_iter().filter(|d| *d == 0).count())
+ }
+}
+
+impl Reduce for Count {
+ fn reduce(self, element: Self) -> Self {
+ Count(self.0 + element.0)
+ }
+}
+
+#[tokio::main]
+async fn main() {
+ let subscriber = FmtSubscriber::builder()
+ .with_max_level(Level::DEBUG)
+ .finish();
+ tracing::subscriber::set_global_default(subscriber).unwrap();
+
+ Worker::run::<Job, Count>("0.0.0.0:1337".parse::<SocketAddr>().unwrap()).await;
+}
diff --git a/src/lib.rs b/src/lib.rs
index 5dc0326e..eb2072c6 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -21,6 +21,15 @@ use thiserror::Error;
pub mod entrypoint;
mod index;
+
+/// Stuff to fix for mapreduce:
+/// * we should handle the case where the job exceeds the 4096 bytes buffer.
+/// * manager should keep track of which workers are currently busy
+/// * manager should keep track of failing vs alive workers
+/// * if a job fails, the manager should re-allocate the job to another worker if possible
+/// * functions should return result instead of unwrapping everywhere
+pub mod mapreduce;
+
mod query;
mod ranking;
mod schema;
diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rs
new file mode 100644
index 00000000..5710efd4
--- /dev/null
+++ b/src/mapreduce/manager.rs
@@ -0,0 +1,73 @@
+use std::net::SocketAddr;
+
+use crate::mapreduce::Task;
+
+use super::{Map, Reduce};
+use futures::{stream::FuturesUnordered, StreamExt};
+use rand::prelude::SliceRandom;
+use std::net::ToSocketAddrs;
+use tokio::{
+ io::{AsyncReadExt, AsyncWriteExt},
+ net::TcpStream,
+};
+use tracing::debug;
+
+pub struct Manager {
+ workers: Vec<SocketAddr>,
+}
+
+impl Manager {
+ pub fn new<A>(workers: &[A]) -> Self
+ where
+ A: ToSocketAddrs,
+ {
+ Self {
+ workers: workers
+ .iter()
+ .flat_map(|addr| addr.to_socket_addrs().unwrap())
+ .collect(),
+ }
+ }
+
+ /// Execute job on one of the remote machines. If the remote machine fails for some reason,
+ /// the job should be allocated to another machine.
+ pub async fn map<I, O>(&self, job: I) -> O
+ where
+ I: Map<O>,
+ O: Reduce<O> + Send,
+ {
+ let worker_addr = self.workers.choose(&mut rand::thread_rng()).unwrap();
+ debug!("connecting to {:}", worker_addr);
+ let mut stream = TcpStream::connect(worker_addr).await.unwrap();
+ debug!("connected");
+ let serialized_job = bincode::serialize(&Task::Job(job)).unwrap();
+ debug!("sending {:?} bytes", serialized_job.len());
+ stream.write(&serialized_job).await.unwrap();
+
+ let mut buf = vec![0; 4096];
+ let size = stream.read(&mut buf).await.unwrap();
+ bincode::deserialize(&buf[..size]).unwrap()
+ }
+
+ fn reduce<O: Reduce<O>>(acc: Option<O>, elem: O) -> O {
+ match acc {
+ Some(acc) => acc.reduce(elem),
+ None => elem,
+ }
+ }
+
+ pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>
+ where
+ I: Map<O>,
+ O: Reduce<O> + Send,
+ {
+ jobs.into_iter()
+ .map(|job| self.map::<I, O>(job))
+ .collect::<FuturesUnordered<_>>()
+ .fold(
+ None,
+ |acc, elem| async move { Some(Manager::reduce(acc, elem)) },
+ )
+ .await
+ }
+}
diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rs
new file mode 100644
index 00000000..c97a4281
--- /dev/null
+++ b/src/mapreduce/mod.rs
@@ -0,0 +1,64 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::net::SocketAddr;
+
+use async_trait::async_trait;
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+
+mod manager;
+mod worker;
+
+pub use manager::Manager;
+pub use worker::Worker;
+
+pub trait Map<T>
+where
+ Self: Serialize + DeserializeOwned + Send,
+ T: Serialize + DeserializeOwned + Send,
+{
+ fn map(self) -> T;
+}
+
+pub trait Reduce<T = Self>
+where
+ Self: Serialize + DeserializeOwned,
+{
+ fn reduce(self, element: T) -> T;
+}
+
+#[async_trait]
+pub trait MapReduce<I, O>: Iterator<Item = I>
+where
+ Self: Sized,
+ I: Map<O>,
+ O: Reduce<O> + Send,
+{
+ async fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {
+ let manager = Manager::new(workers);
+ manager.run(self.collect()).await
+ }
+}
+impl<I: Map<O> + Send, O: Reduce<O> + Send, T: Sized> MapReduce<I, O> for T where
+ T: Iterator<Item = I>
+{
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+enum Task<T> {
+ Job(T),
+ AllFinished,
+}
diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rs
new file mode 100644
index 00000000..6005e157
--- /dev/null
+++ b/src/mapreduce/worker.rs
@@ -0,0 +1,146 @@
+use std::net::SocketAddr;
+
+use super::{Map, Task};
+use serde::{de::DeserializeOwned, Serialize};
+use tokio::io::AsyncReadExt;
+use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};
+use tokio::net::{TcpListener, ToSocketAddrs};
+use tracing::{debug, info};
+
+pub struct Worker {}
+
+impl Worker {
+ async fn run_stream<I, O, S>(mut stream: S)
+ where
+ I: Map<O>,
+ O: Serialize + DeserializeOwned + Send,
+ S: AsyncRead + AsyncWrite + Unpin,
+ {
+ let mut buf = vec![0; 4096];
+ loop {
+ if let Ok(size) = stream.read(&mut buf).await {
+ if size == 0 {
+ break;
+ }
+ debug!("read {:?} bytes", size);
+ match bincode::deserialize::<Task<I>>(&buf[..size]).unwrap() {
+ Task::Job(job) => {
+ debug!("received job");
+ let res = job.map();
+ let bytes = bincode::serialize(&res).unwrap();
+ stream.write_all(&bytes[..]).await.unwrap();
+ }
+ Task::AllFinished => break,
+ };
+ }
+ }
+ }
+
+ pub async fn run<I, O>(addr: SocketAddr)
+ where
+ I: Map<O>,
+ O: Serialize + DeserializeOwned + Send,
+ {
+ let listener = TcpListener::bind(addr).await.unwrap();
+ info!("worker listening on: {:}", addr);
+
+ loop {
+ let (socket, _) = listener.accept().await.unwrap();
+ debug!("received connection");
+ Worker::run_stream::<I, O, _>(socket).await;
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use std::{
+ pin::Pin,
+ task::{Context, Poll},
+ };
+
+ use serde::Deserialize;
+ use tokio::io::ReadBuf;
+
+ use super::*;
+
+ struct MockTcpStream {
+ contents: Vec<u8>,
+ result: Vec<u8>,
+ }
+
+ impl MockTcpStream {
+ fn new(contents: Vec<u8>) -> Self {
+ Self {
+ contents,
+ result: Vec::new(),
+ }
+ }
+ }
+
+ impl AsyncRead for MockTcpStream {
+ fn poll_read(
+ self: Pin<&mut Self>,
+ _: &mut Context<'_>,
+ buf: &mut ReadBuf<'_>,
+ ) -> Poll<std::io::Result<()>> {
+ buf.put_slice(&self.contents[..]);
+ self.get_mut().contents = Vec::new();
+ Poll::Ready(Ok(()))
+ }
+ }
+
+ impl AsyncWrite for MockTcpStream {
+ fn poll_write(
+ self: Pin<&mut Self>,
+ _: &mut Context<'_>,
+ buf: &[u8],
+ ) -> Poll<Result<usize, std::io::Error>> {
+ self.get_mut().result = Vec::from(buf);
+ Poll::Ready(Ok(buf.len()))
+ }
+
+ fn poll_flush(
+ self: Pin<&mut Self>,
+ _: &mut Context<'_>,
+ ) -> Poll<Result<(), std::io::Error>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn poll_shutdown(
+ self: Pin<&mut Self>,
+ _: &mut Context<'_>,
+ ) -> Poll<Result<(), std::io::Error>> {
+ Poll::Ready(Ok(()))
+ }
+ }
+
+ #[derive(Serialize, Deserialize, Debug)]
+ struct MockJob {
+ contents: Vec<usize>,
+ }
+
+ #[derive(Serialize, Deserialize, Debug)]
+ struct Count(usize);
+
+ impl Map<Count> for MockJob {
+ fn map(self) -> Count {
+ Count(self.contents.into_iter().filter(|d| *d == 0).count())
+ }
+ }
+
+ #[tokio::test]
+ async fn execute() {
+ let job = bincode::serialize(&Task::Job(MockJob {
+ contents: vec![1, 2, 0, 1, 0, 1, 0],
+ }))
+ .unwrap();
+
+ let mut stream = MockTcpStream::new(job);
+ Worker::run_stream::<MockJob, _, _>(&mut stream).await;
+
+ let res: Count = bincode::deserialize(&stream.result).unwrap();
+
+ assert_eq!(res.0, 3);
+ }
+}