web search done right
simple mapreduce without much error handling
Mikkel Denker344dc3fd193b
diff --git a/Cargo.toml b/Cargo.tomlindex 9edb9e82..cb0c8917 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -11,6 +11,7 @@ authors = ['Cuely ApS. <[email protected]>']name = "cuely"[dependencies]+async-trait = "0.1.53"bincode = "1.3.3"encoding_rs = "0.8.31"flate2 = "1.0.23"@@ -19,6 +20,7 @@ htmlescape = "0.3.1"indicatif = "0.16.2"itertools = "0.10.3"nom = "7.1.1"+rand = "0.8.5"reqwest = "0.11.10"rusoto_core = "0.48.0"rusoto_s3 = "0.48.0"@@ -29,6 +31,8 @@ tantivy = "0.18.0"thiserror = "1.0.31"tokio = { version = "1.18.2", features = ["full"] }toml = "0.5.9"+tracing = "0.1.34"+tracing-subscriber = "0.3.11"whatlang = "0.16.0"[dev-dependencies]diff --git a/examples/mapreduce_manager.rs b/examples/mapreduce_manager.rsnew file mode 100644index 00000000..baaf03eb--- /dev/null+++ b/examples/mapreduce_manager.rs@@ -0,0 +1,53 @@+use std::net::SocketAddr;++use cuely::mapreduce::{Map, MapReduce, Reduce};+use serde::{Deserialize, Serialize};+use tracing::Level;+use tracing_subscriber::FmtSubscriber;++#[derive(Serialize, Deserialize, Debug)]+struct Job {+ contents: Vec<usize>,+}++#[derive(Serialize, Deserialize, Debug)]+struct Count(usize);++impl Map<Count> for Job {+ fn map(self) -> Count {+ Count(self.contents.into_iter().filter(|d| *d == 0).count())+ }+}++impl Reduce for Count {+ fn reduce(self, element: Self) -> Self {+ Count(self.0 + element.0)+ }+}++#[tokio::main]+async fn main() {+ let subscriber = FmtSubscriber::builder()+ .with_max_level(Level::DEBUG)+ .finish();+ tracing::subscriber::set_global_default(subscriber).unwrap();++ let jobs = vec![+ Job {+ contents: vec![1, 2, 0, 1, 0, 1, 0],+ },+ Job {+ contents: vec![3, 2, 1],+ },+ Job {+ contents: vec![0, 0],+ },+ ];++ let res = jobs+ .into_iter()+ .map_reduce(&["0.0.0.0:1337".parse::<SocketAddr>().unwrap()])+ .await+ .unwrap();+ println!("{:?}", res.0);+}diff --git a/examples/mapreduce_worker.rs b/examples/mapreduce_worker.rsnew file mode 100644index 00000000..a8871916--- /dev/null+++ b/examples/mapreduce_worker.rs@@ -0,0 +1,36 @@+use std::net::SocketAddr;++use cuely::mapreduce::{Map, Reduce, Worker};+use serde::{Deserialize, Serialize};+use tracing::Level;+use tracing_subscriber::FmtSubscriber;++#[derive(Serialize, Deserialize, Debug)]+struct Job {+ contents: Vec<usize>,+}++#[derive(Serialize, Deserialize, Debug)]+struct Count(usize);++impl Map<Count> for Job {+ fn map(self) -> Count {+ Count(self.contents.into_iter().filter(|d| *d == 0).count())+ }+}++impl Reduce for Count {+ fn reduce(self, element: Self) -> Self {+ Count(self.0 + element.0)+ }+}++#[tokio::main]+async fn main() {+ let subscriber = FmtSubscriber::builder()+ .with_max_level(Level::DEBUG)+ .finish();+ tracing::subscriber::set_global_default(subscriber).unwrap();++ Worker::run::<Job, Count>("0.0.0.0:1337".parse::<SocketAddr>().unwrap()).await;+}diff --git a/src/lib.rs b/src/lib.rsindex 5dc0326e..eb2072c6 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -21,6 +21,15 @@ use thiserror::Error;pub mod entrypoint;mod index;++/// Stuff to fix for mapreduce:+/// * we should handle the case where the job exceeds the 4096 bytes buffer.+/// * manager should keep track of which workers are currently busy+/// * manager should keep track of failing vs alive workers+/// * if a job fails, the manager should re-allocate the job to another worker if possible+/// * functions should return result instead of unwrapping everywhere+pub mod mapreduce;+mod query;mod ranking;mod schema;diff --git a/src/mapreduce/manager.rs b/src/mapreduce/manager.rsnew file mode 100644index 00000000..5710efd4--- /dev/null+++ b/src/mapreduce/manager.rs@@ -0,0 +1,73 @@+use std::net::SocketAddr;++use crate::mapreduce::Task;++use super::{Map, Reduce};+use futures::{stream::FuturesUnordered, StreamExt};+use rand::prelude::SliceRandom;+use std::net::ToSocketAddrs;+use tokio::{+ io::{AsyncReadExt, AsyncWriteExt},+ net::TcpStream,+};+use tracing::debug;++pub struct Manager {+ workers: Vec<SocketAddr>,+}++impl Manager {+ pub fn new<A>(workers: &[A]) -> Self+ where+ A: ToSocketAddrs,+ {+ Self {+ workers: workers+ .iter()+ .flat_map(|addr| addr.to_socket_addrs().unwrap())+ .collect(),+ }+ }++ /// Execute job on one of the remote machines. If the remote machine fails for some reason,+ /// the job should be allocated to another machine.+ pub async fn map<I, O>(&self, job: I) -> O+ where+ I: Map<O>,+ O: Reduce<O> + Send,+ {+ let worker_addr = self.workers.choose(&mut rand::thread_rng()).unwrap();+ debug!("connecting to {:}", worker_addr);+ let mut stream = TcpStream::connect(worker_addr).await.unwrap();+ debug!("connected");+ let serialized_job = bincode::serialize(&Task::Job(job)).unwrap();+ debug!("sending {:?} bytes", serialized_job.len());+ stream.write(&serialized_job).await.unwrap();++ let mut buf = vec![0; 4096];+ let size = stream.read(&mut buf).await.unwrap();+ bincode::deserialize(&buf[..size]).unwrap()+ }++ fn reduce<O: Reduce<O>>(acc: Option<O>, elem: O) -> O {+ match acc {+ Some(acc) => acc.reduce(elem),+ None => elem,+ }+ }++ pub async fn run<I, O>(&self, jobs: Vec<I>) -> Option<O>+ where+ I: Map<O>,+ O: Reduce<O> + Send,+ {+ jobs.into_iter()+ .map(|job| self.map::<I, O>(job))+ .collect::<FuturesUnordered<_>>()+ .fold(+ None,+ |acc, elem| async move { Some(Manager::reduce(acc, elem)) },+ )+ .await+ }+}diff --git a/src/mapreduce/mod.rs b/src/mapreduce/mod.rsnew file mode 100644index 00000000..c97a4281--- /dev/null+++ b/src/mapreduce/mod.rs@@ -0,0 +1,64 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::net::SocketAddr;++use async_trait::async_trait;+use serde::{de::DeserializeOwned, Deserialize, Serialize};++mod manager;+mod worker;++pub use manager::Manager;+pub use worker::Worker;++pub trait Map<T>+where+ Self: Serialize + DeserializeOwned + Send,+ T: Serialize + DeserializeOwned + Send,+{+ fn map(self) -> T;+}++pub trait Reduce<T = Self>+where+ Self: Serialize + DeserializeOwned,+{+ fn reduce(self, element: T) -> T;+}++#[async_trait]+pub trait MapReduce<I, O>: Iterator<Item = I>+where+ Self: Sized,+ I: Map<O>,+ O: Reduce<O> + Send,+{+ async fn map_reduce(self, workers: &[SocketAddr]) -> Option<O> {+ let manager = Manager::new(workers);+ manager.run(self.collect()).await+ }+}+impl<I: Map<O> + Send, O: Reduce<O> + Send, T: Sized> MapReduce<I, O> for T where+ T: Iterator<Item = I>+{+}++#[derive(Serialize, Deserialize, Debug)]+enum Task<T> {+ Job(T),+ AllFinished,+}diff --git a/src/mapreduce/worker.rs b/src/mapreduce/worker.rsnew file mode 100644index 00000000..6005e157--- /dev/null+++ b/src/mapreduce/worker.rs@@ -0,0 +1,146 @@+use std::net::SocketAddr;++use super::{Map, Task};+use serde::{de::DeserializeOwned, Serialize};+use tokio::io::AsyncReadExt;+use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt};+use tokio::net::{TcpListener, ToSocketAddrs};+use tracing::{debug, info};++pub struct Worker {}++impl Worker {+ async fn run_stream<I, O, S>(mut stream: S)+ where+ I: Map<O>,+ O: Serialize + DeserializeOwned + Send,+ S: AsyncRead + AsyncWrite + Unpin,+ {+ let mut buf = vec![0; 4096];+ loop {+ if let Ok(size) = stream.read(&mut buf).await {+ if size == 0 {+ break;+ }+ debug!("read {:?} bytes", size);+ match bincode::deserialize::<Task<I>>(&buf[..size]).unwrap() {+ Task::Job(job) => {+ debug!("received job");+ let res = job.map();+ let bytes = bincode::serialize(&res).unwrap();+ stream.write_all(&bytes[..]).await.unwrap();+ }+ Task::AllFinished => break,+ };+ }+ }+ }++ pub async fn run<I, O>(addr: SocketAddr)+ where+ I: Map<O>,+ O: Serialize + DeserializeOwned + Send,+ {+ let listener = TcpListener::bind(addr).await.unwrap();+ info!("worker listening on: {:}", addr);++ loop {+ let (socket, _) = listener.accept().await.unwrap();+ debug!("received connection");+ Worker::run_stream::<I, O, _>(socket).await;+ }+ }+}++#[cfg(test)]+mod tests {+ use std::{+ pin::Pin,+ task::{Context, Poll},+ };++ use serde::Deserialize;+ use tokio::io::ReadBuf;++ use super::*;++ struct MockTcpStream {+ contents: Vec<u8>,+ result: Vec<u8>,+ }++ impl MockTcpStream {+ fn new(contents: Vec<u8>) -> Self {+ Self {+ contents,+ result: Vec::new(),+ }+ }+ }++ impl AsyncRead for MockTcpStream {+ fn poll_read(+ self: Pin<&mut Self>,+ _: &mut Context<'_>,+ buf: &mut ReadBuf<'_>,+ ) -> Poll<std::io::Result<()>> {+ buf.put_slice(&self.contents[..]);+ self.get_mut().contents = Vec::new();+ Poll::Ready(Ok(()))+ }+ }++ impl AsyncWrite for MockTcpStream {+ fn poll_write(+ self: Pin<&mut Self>,+ _: &mut Context<'_>,+ buf: &[u8],+ ) -> Poll<Result<usize, std::io::Error>> {+ self.get_mut().result = Vec::from(buf);+ Poll::Ready(Ok(buf.len()))+ }++ fn poll_flush(+ self: Pin<&mut Self>,+ _: &mut Context<'_>,+ ) -> Poll<Result<(), std::io::Error>> {+ Poll::Ready(Ok(()))+ }++ fn poll_shutdown(+ self: Pin<&mut Self>,+ _: &mut Context<'_>,+ ) -> Poll<Result<(), std::io::Error>> {+ Poll::Ready(Ok(()))+ }+ }++ #[derive(Serialize, Deserialize, Debug)]+ struct MockJob {+ contents: Vec<usize>,+ }++ #[derive(Serialize, Deserialize, Debug)]+ struct Count(usize);++ impl Map<Count> for MockJob {+ fn map(self) -> Count {+ Count(self.contents.into_iter().filter(|d| *d == 0).count())+ }+ }++ #[tokio::test]+ async fn execute() {+ let job = bincode::serialize(&Task::Job(MockJob {+ contents: vec![1, 2, 0, 1, 0, 1, 0],+ }))+ .unwrap();++ let mut stream = MockTcpStream::new(job);+ Worker::run_stream::<MockJob, _, _>(&mut stream).await;++ let res: Count = bincode::deserialize(&stream.result).unwrap();++ assert_eq!(res.0, 3);+ }+}