web search done right
// Stract is an open source web search engine.
// Copyright (C) 2024 Stract ApS
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>
use std::ops::DerefMut;
use super::{CoordReq, CoordResp, DhtConn, Job, JobConn, JobReq, JobResp, Req, Resp, Server};
use crate::block_on;
use crate::distributed::sonic;
use crate::Result;
use anyhow::anyhow;
use tokio::net::ToSocketAddrs;
/// A worker is responsible for executing a mapper on its portion of the graph and
/// sending results to the DHT.
pub trait Worker: Send + Sync {
type Remote: RemoteWorker<Job = Self::Job>;
type Request: bincode::Encode + bincode::Decode + Clone + Send + Sync;
type Response: bincode::Encode + bincode::Decode + Send + Sync;
type Job: Job<Worker = Self>;
fn handle(&self, req: Self::Request) -> Self::Response;
fn bind(self, addr: impl ToSocketAddrs) -> Result<Server<Self>>
where
Self: Sized + 'static,
{
Server::bind(addr, self)
}
fn run(self, addr: impl ToSocketAddrs) -> Result<()>
where
Self: Sized + 'static,
{
self.bind(addr)?.run()
}
}
pub trait Message<W: Worker>: std::fmt::Debug + Clone {
type Response;
fn handle(self, worker: &W) -> Self::Response;
}
pub trait RemoteWorker
where
Self: Send + Sync,
{
type Job: Job;
fn pool(&self) -> &sonic::ConnectionPool<JobConn<Self::Job>>;
fn schedule_job(&self, job: &Self::Job, mapper: <Self::Job as Job>::Mapper) -> Result<()> {
self.send_raw(&JobReq::Coordinator(CoordReq::ScheduleJob {
job: job.clone(),
mapper,
}))?;
Ok(())
}
fn send_dht(&self, dht: &DhtConn<<Self::Job as Job>::DhtTables>) -> Result<()> {
self.send_raw(&JobReq::Coordinator(CoordReq::Setup { dht: dht.clone() }))?;
Ok(())
}
fn current_job(&self) -> Result<Option<Self::Job>> {
let req = JobReq::Coordinator(CoordReq::CurrentJob);
let res = self.send_raw(&req)?;
match res {
Resp::Coordinator(CoordResp::CurrentJob(job)) => Ok(job),
_ => Err(anyhow!("unexpected response")),
}
}
fn conn(&self) -> Result<impl DerefMut<Target = JobConn<Self::Job>>> {
let conn = block_on(self.pool().get())?;
Ok(conn)
}
fn send_raw(&self, req: &JobReq<Self::Job>) -> Result<JobResp<Self::Job>> {
let mut conn = self.conn()?;
let res = block_on(conn.send(req))?;
Ok(res)
}
fn send_raw_without_timeout(&self, req: &JobReq<Self::Job>) -> Result<JobResp<Self::Job>> {
let mut conn = self.conn()?;
let res = block_on(conn.send_without_timeout(req))?;
Ok(res)
}
fn send<R>(&self, req: R) -> R::Response
where
R: RequestWrapper<<Self::Job as Job>::Worker>,
{
match self.send_raw(&Req::User(R::wrap(req))).unwrap() {
Resp::Coordinator(_) => panic!("unexpected coordinator response"),
Resp::User(res) => R::unwrap_response(res).unwrap(),
}
}
fn send_without_timeout<R>(&self, req: R) -> R::Response
where
R: RequestWrapper<<Self::Job as Job>::Worker>,
{
match self
.send_raw_without_timeout(&Req::User(R::wrap(req)))
.unwrap()
{
Resp::Coordinator(_) => panic!("unexpected coordinator response"),
Resp::User(res) => R::unwrap_response(res).unwrap(),
}
}
}
pub trait RequestWrapper<W: Worker>: Message<W> {
fn wrap(req: Self) -> W::Request;
fn unwrap_response(res: W::Response) -> Result<Self::Response>;
}
#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)]
pub struct WorkerRef(pub usize);
macro_rules! impl_worker {
($job:ident , $remote:ident => $worker:ident, [$($req:ident),*$(,)?]) => {
mod worker_impl__ {
#![allow(dead_code)]
use super::{$worker, $remote, $job, $($req),*};
use $crate::ampc;
use $crate::ampc::Message;
#[derive(Debug, Clone, ::serde::Serialize, ::serde::Deserialize, ::bincode::Decode, ::strum::EnumDiscriminants)]
pub enum Request {
$($req($req),)*
}
fn req_to_index(req: &Request) -> u32 {
RequestDiscriminants::from(req) as u32
}
fn index_to_req(index: u32) -> RequestDiscriminants {
match index {
$(x if x == RequestDiscriminants::$req as u32 => RequestDiscriminants::$req,)*
_ => panic!("invalid request index"),
}
}
impl ::bincode::Encode for Request {
fn encode<E: ::bincode::enc::Encoder>(
&self,
encoder: &mut E,
) -> Result<(), bincode::error::EncodeError> {
let index = req_to_index(self);
index.encode(encoder)?;
match self {
$(Request::$req(req) => {
req.encode(encoder)
})*
}
}
}
fn res_to_index(res: &Response) -> u32 {
ResponseDiscriminants::from(res) as u32
}
fn index_to_res(index: u32) -> ResponseDiscriminants {
match index {
$(x if x == ResponseDiscriminants::$req as u32 => ResponseDiscriminants::$req,)*
_ => panic!("invalid response index"),
}
}
#[derive(::serde::Serialize, ::serde::Deserialize, ::bincode::Decode, ::strum::EnumDiscriminants)]
pub enum Response {
$($req(<$req as ampc::Message<$worker>>::Response),)*
}
impl ::bincode::Encode for Response {
fn encode<E: ::bincode::enc::Encoder>(
&self,
encoder: &mut E,
) -> Result<(), bincode::error::EncodeError> {
let index = res_to_index(self);
index.encode(encoder)?;
match self {
$(Response::$req(res) => {
res.encode(encoder)
})*
}
}
}
impl ampc::Worker for $worker {
type Remote = $remote;
type Request = Request;
type Response = Response;
type Job = $job;
fn handle(&self, req: Self::Request) -> Self::Response {
match req {
$(Request::$req(req) => Response::$req(req.handle(self)),)*
}
}
}
$(
impl ampc::RequestWrapper<$worker> for $req {
fn wrap(req: Self) -> <$worker as ampc::Worker>::Request {
<$worker as ampc::Worker>::Request::$req(req)
}
fn unwrap_response(res: <$worker as ampc::Worker>::Response) -> anyhow::Result<Self::Response> {
match res {
Response::$req(res) => Ok(res),
_ => Err(anyhow::anyhow!("unexpected response")),
}
}
}
)*
}
};
}
pub(crate) use impl_worker;