web search done right
Remove `#[async_trait]` (#119)
Oliver Bøvingbe9381420dd3
diff --git a/Cargo.lock b/Cargo.lockindex 539f6850..d4818025 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4303,7 +4303,6 @@ dependencies = ["anyhow","async-channel","async-stream",- "async-trait","axum","axum-extra","axum-macros",diff --git a/Cargo.toml b/Cargo.tomlindex 5832a0ae..f9cbe519 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -14,7 +14,6 @@ aes-gcm = "0.10.2"anyhow = {version = "1.0.72", features = ["backtrace"]}async-channel = "1.8.0"async-stream = "0.3.3"-async-trait = "0.1.57"axum = "0.7.2"axum-extra = {version = "0.9.0"}axum-macros = "0.4.0"diff --git a/core/Cargo.toml b/core/Cargo.tomlindex c4bdca95..add68021 100644--- a/core/Cargo.toml+++ b/core/Cargo.toml@@ -25,7 +25,6 @@ aes-gcm = {workspace = true}anyhow = {workspace = true}async-channel = {workspace = true}async-stream = {workspace = true}-async-trait = {workspace = true}axum = {workspace = true}axum-extra = {workspace = true}axum-macros = {workspace = true}diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rsindex bf0be611..841e909f 100644--- a/core/src/crawler/mod.rs+++ b/core/src/crawler/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::{collections::VecDeque, sync::Arc, time::Duration};+use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration};use hashbrown::HashMap;@@ -263,10 +263,9 @@ impl Crawler {}}-#[async_trait::async_trait]pub trait DatumStream: Send + Sync {- async fn write(&self, crawl_datum: CrawlDatum) -> Result<()>;- async fn finish(&self) -> Result<()>;+ fn write(&self, crawl_datum: CrawlDatum) -> impl Future<Output = Result<()>> + Send;+ fn finish(&self) -> impl Future<Output = Result<()>> + Send;}pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rsindex 5a1b6cc5..3d49df4f 100644--- a/core/src/crawler/warc_writer.rs+++ b/core/src/crawler/warc_writer.rs@@ -29,7 +29,6 @@ pub struct WarcWriter {tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,}-#[async_trait::async_trait]impl DatumStream for WarcWriter {async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {self.tx.send(WarcWriterMessage::Crawl(crawl_datum)).await?;diff --git a/core/src/distributed/sonic/service.rs b/core/src/distributed/sonic/service.rsindex 66c5862b..b05c09f7 100644--- a/core/src/distributed/sonic/service.rs+++ b/core/src/distributed/sonic/service.rs@@ -20,16 +20,17 @@ use tokio::net::ToSocketAddrs;use super::Result;-#[async_trait::async_trait]pub trait Service: Sized + Send + Sync + 'static {type Request: serde::de::DeserializeOwned + Send + Sync;type RequestRef<'a>: serde::Serialize + Send + Sync;type Response: serde::Serialize + serde::de::DeserializeOwned + Send + Sync;- async fn handle(req: Self::Request, server: &Self) -> Result<Self::Response>;+ fn handle(+ req: Self::Request,+ server: &Self,+ ) -> impl std::future::Future<Output = Result<Self::Response>> + Send + '_;}-#[async_trait::async_trait]pub trait Message<S: Service> {type Response;async fn handle(self, server: &S) -> Result<Self::Response>;@@ -182,17 +183,22 @@ macro_rules! sonic_service {}})*- #[async_trait::async_trait]impl sonic::service::Service for $service {type Request = Request;type RequestRef<'a> = RequestRef<'a>;type Response = Response;- async fn handle(req: Request, server: &Self) -> sonic::Result<Response> {- match req {- $(- Request::$req(value) => Ok(Response::$req(sonic::service::Message::handle(value, server).await?)),- )*+ // NOTE: This is a workaround for the fact that async functions+ // don't have a Send bound by default, and there's currently no+ // way of specifying that.+ #[allow(clippy::manual_async_fn)]+ fn handle(req: Request, server: &Self) -> impl std::future::Future<Output = sonic::Result<Self::Response>> + Send + '_ {+ async move {+ match req {+ $(+ Request::$req(value) => Ok(Response::$req(sonic::service::Message::handle(value, server).await?)),+ )*+ }}}}@@ -291,7 +297,6 @@ mod tests {#[derive(Debug, Clone, Serialize, Deserialize)]pub struct Reset;- #[async_trait::async_trait]impl Message<CounterService> for Change {type Response = i32;@@ -303,7 +308,6 @@ mod tests {}}- #[async_trait::async_trait]impl Message<CounterService> for Reset {type Response = ();diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rsindex d719fa17..ebba3103 100644--- a/core/src/entrypoint/crawler.rs+++ b/core/src/entrypoint/crawler.rs@@ -100,7 +100,6 @@ pub mod router {#[derive(Debug, Clone, Serialize, Deserialize)]pub struct NewJob {}- #[async_trait::async_trait]impl Message<RouterService> for NewJob {type Response = Option<Job>;@@ -124,7 +123,6 @@ pub mod coordinator {#[derive(Debug, Clone, Serialize, Deserialize)]pub struct GetJob {}- #[async_trait::async_trait]impl Message<CoordinatorService> for GetJob {type Response = Option<Job>;diff --git a/core/src/entrypoint/live_index.rs b/core/src/entrypoint/live_index.rsindex d6e93a1a..c758b5c5 100644--- a/core/src/entrypoint/live_index.rs+++ b/core/src/entrypoint/live_index.rs@@ -79,7 +79,6 @@ impl SearchService {}}-#[async_trait::async_trait]impl sonic::service::Message<SearchService> for RetrieveWebsites {type Response = Option<Vec<inverted_index::RetrievedWebpage>>;async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {@@ -93,7 +92,6 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {}}-#[async_trait::async_trait]impl sonic::service::Message<SearchService> for Search {type Response = Option<InitialWebsiteResult>;async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex be99bb2a..2ba0d9d1 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -113,7 +113,6 @@ pub struct RetrieveWebsites {pub websites: Vec<inverted_index::WebsitePointer>,pub query: String,}-#[async_trait::async_trait]impl sonic::service::Message<SearchService> for RetrieveWebsites {type Response = Option<Vec<inverted_index::RetrievedWebpage>>;async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {@@ -131,7 +130,6 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {pub struct Search {pub query: SearchQuery,}-#[async_trait::async_trait]impl sonic::service::Message<SearchService> for Search {type Response = Option<InitialWebsiteResult>;async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {@@ -146,7 +144,6 @@ impl sonic::service::Message<SearchService> for Search {pub struct GetWebpage {pub url: String,}-#[async_trait::async_trait]impl sonic::service::Message<SearchService> for GetWebpage {type Response = Option<RetrievedWebpage>;async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {@@ -158,7 +155,6 @@ impl sonic::service::Message<SearchService> for GetWebpage {pub struct GetHomepageDescriptions {pub urls: Vec<Url>,}-#[async_trait::async_trait]impl sonic::service::Message<SearchService> for GetHomepageDescriptions {type Response = HashMap<Url, String>;async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {@@ -180,7 +176,6 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {pub struct GetEntityImage {pub image_id: String,}-#[async_trait::async_trait]impl sonic::service::Message<SearchService> for GetEntityImage {type Response = Option<Image>;async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rsindex dd2cc624..c0e94793 100644--- a/core/src/entrypoint/webgraph_server.rs+++ b/core/src/entrypoint/webgraph_server.rs@@ -74,7 +74,6 @@ pub struct Knows {pub host: String,}-#[async_trait::async_trait]impl Message<WebGraphService> for SimilarHosts {type Response = Vec<ScoredHost>;@@ -110,7 +109,6 @@ impl Message<WebGraphService> for SimilarHosts {}}-#[async_trait::async_trait]impl Message<WebGraphService> for Knows {type Response = Option<Node>;@@ -140,7 +138,6 @@ pub struct IngoingLinks {pub level: GraphLevel,}-#[async_trait::async_trait]impl Message<WebGraphService> for IngoingLinks {type Response = Vec<FullEdge>;@@ -161,7 +158,6 @@ pub struct OutgoingLinks {pub level: GraphLevel,}-#[async_trait::async_trait]impl Message<WebGraphService> for OutgoingLinks {type Response = Vec<FullEdge>;diff --git a/core/src/live_index.rs b/core/src/live_index.rsindex f0822a24..53427be2 100644--- a/core/src/live_index.rs+++ b/core/src/live_index.rs@@ -264,7 +264,6 @@ impl Index {}}-#[async_trait::async_trait]impl DatumStream for Indexer {async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {let webpage = self.worker.prepare_webpage(diff --git a/core/src/mapreduce/worker.rs b/core/src/mapreduce/worker.rsindex 46fad548..adf34d0b 100644--- a/core/src/mapreduce/worker.rs+++ b/core/src/mapreduce/worker.rs@@ -14,46 +14,46 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::net::SocketAddr;+use std::{future::Future, net::SocketAddr};use crate::mapreduce::MapReduceServer;use super::{Map, Result, Task};-use async_trait::async_trait;use serde::{de::DeserializeOwned, Serialize};use tracing::{debug, info};#[derive(Default)]pub struct StatelessWorker {}-#[async_trait]pub trait Worker {- async fn run<I, O>(&self, addr: SocketAddr) -> Result<()>+ fn run<I, O>(&self, addr: SocketAddr) -> impl Future<Output = Result<()>>whereSelf: Sized,I: Map<Self, O> + Send + Sync,O: Serialize + DeserializeOwned + Send + Sync,{- let server = MapReduceServer::<I, O>::bind(addr).await?;- info!("worker listening on: {:}", addr);-- loop {- let req = server.accept().await?;- debug!("received request");- match req.body() {- Task::Job(job) => {- debug!("request is a job");- let res = job.map(self);- req.respond(Some(res)).await?;- }- Task::AllFinished => {- req.respond(None).await?;- break;+ async move {+ let server = MapReduceServer::<I, O>::bind(addr).await?;+ info!("worker listening on: {:}", addr);++ loop {+ let req = server.accept().await?;+ debug!("received request");+ match req.body() {+ Task::Job(job) => {+ debug!("request is a job");+ let res = job.map(self);+ req.respond(Some(res)).await?;+ }+ Task::AllFinished => {+ req.respond(None).await?;+ break;+ }}}- }- Ok(())+ Ok(())+ }}}