Skip to content

web search done right

Remove `#[async_trait]` (#119)

Oliver Bøvingbe9381420dd3
diff --git a/Cargo.lock b/Cargo.lock
index 539f6850..d4818025 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4303,7 +4303,6 @@ dependencies = [
"anyhow",
"async-channel",
"async-stream",
- "async-trait",
"axum",
"axum-extra",
"axum-macros",
diff --git a/Cargo.toml b/Cargo.toml
index 5832a0ae..f9cbe519 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -14,7 +14,6 @@ aes-gcm = "0.10.2"
anyhow = {version = "1.0.72", features = ["backtrace"]}
async-channel = "1.8.0"
async-stream = "0.3.3"
-async-trait = "0.1.57"
axum = "0.7.2"
axum-extra = {version = "0.9.0"}
axum-macros = "0.4.0"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index c4bdca95..add68021 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -25,7 +25,6 @@ aes-gcm = {workspace = true}
anyhow = {workspace = true}
async-channel = {workspace = true}
async-stream = {workspace = true}
-async-trait = {workspace = true}
axum = {workspace = true}
axum-extra = {workspace = true}
axum-macros = {workspace = true}
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index bf0be611..841e909f 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::{collections::VecDeque, sync::Arc, time::Duration};
+use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration};
use hashbrown::HashMap;
@@ -263,10 +263,9 @@ impl Crawler {
}
}
-#[async_trait::async_trait]
pub trait DatumStream: Send + Sync {
- async fn write(&self, crawl_datum: CrawlDatum) -> Result<()>;
- async fn finish(&self) -> Result<()>;
+ fn write(&self, crawl_datum: CrawlDatum) -> impl Future<Output = Result<()>> + Send;
+ fn finish(&self) -> impl Future<Output = Result<()>> + Send;
}
pub fn reqwest_client(config: &CrawlerConfig) -> Result<reqwest::Client> {
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index 5a1b6cc5..3d49df4f 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -29,7 +29,6 @@ pub struct WarcWriter {
tx: tokio::sync::mpsc::Sender<WarcWriterMessage>,
}
-#[async_trait::async_trait]
impl DatumStream for WarcWriter {
async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
self.tx.send(WarcWriterMessage::Crawl(crawl_datum)).await?;
diff --git a/core/src/distributed/sonic/service.rs b/core/src/distributed/sonic/service.rs
index 66c5862b..b05c09f7 100644
--- a/core/src/distributed/sonic/service.rs
+++ b/core/src/distributed/sonic/service.rs
@@ -20,16 +20,17 @@ use tokio::net::ToSocketAddrs;
use super::Result;
-#[async_trait::async_trait]
pub trait Service: Sized + Send + Sync + 'static {
type Request: serde::de::DeserializeOwned + Send + Sync;
type RequestRef<'a>: serde::Serialize + Send + Sync;
type Response: serde::Serialize + serde::de::DeserializeOwned + Send + Sync;
- async fn handle(req: Self::Request, server: &Self) -> Result<Self::Response>;
+ fn handle(
+ req: Self::Request,
+ server: &Self,
+ ) -> impl std::future::Future<Output = Result<Self::Response>> + Send + '_;
}
-#[async_trait::async_trait]
pub trait Message<S: Service> {
type Response;
async fn handle(self, server: &S) -> Result<Self::Response>;
@@ -182,17 +183,22 @@ macro_rules! sonic_service {
}
}
)*
- #[async_trait::async_trait]
impl sonic::service::Service for $service {
type Request = Request;
type RequestRef<'a> = RequestRef<'a>;
type Response = Response;
- async fn handle(req: Request, server: &Self) -> sonic::Result<Response> {
- match req {
- $(
- Request::$req(value) => Ok(Response::$req(sonic::service::Message::handle(value, server).await?)),
- )*
+ // NOTE: This is a workaround for the fact that async functions
+ // don't have a Send bound by default, and there's currently no
+ // way of specifying that.
+ #[allow(clippy::manual_async_fn)]
+ fn handle(req: Request, server: &Self) -> impl std::future::Future<Output = sonic::Result<Self::Response>> + Send + '_ {
+ async move {
+ match req {
+ $(
+ Request::$req(value) => Ok(Response::$req(sonic::service::Message::handle(value, server).await?)),
+ )*
+ }
}
}
}
@@ -291,7 +297,6 @@ mod tests {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Reset;
- #[async_trait::async_trait]
impl Message<CounterService> for Change {
type Response = i32;
@@ -303,7 +308,6 @@ mod tests {
}
}
- #[async_trait::async_trait]
impl Message<CounterService> for Reset {
type Response = ();
diff --git a/core/src/entrypoint/crawler.rs b/core/src/entrypoint/crawler.rs
index d719fa17..ebba3103 100644
--- a/core/src/entrypoint/crawler.rs
+++ b/core/src/entrypoint/crawler.rs
@@ -100,7 +100,6 @@ pub mod router {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NewJob {}
- #[async_trait::async_trait]
impl Message<RouterService> for NewJob {
type Response = Option<Job>;
@@ -124,7 +123,6 @@ pub mod coordinator {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct GetJob {}
- #[async_trait::async_trait]
impl Message<CoordinatorService> for GetJob {
type Response = Option<Job>;
diff --git a/core/src/entrypoint/live_index.rs b/core/src/entrypoint/live_index.rs
index d6e93a1a..c758b5c5 100644
--- a/core/src/entrypoint/live_index.rs
+++ b/core/src/entrypoint/live_index.rs
@@ -79,7 +79,6 @@ impl SearchService {
}
}
-#[async_trait::async_trait]
impl sonic::service::Message<SearchService> for RetrieveWebsites {
type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
@@ -93,7 +92,6 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {
}
}
-#[async_trait::async_trait]
impl sonic::service::Message<SearchService> for Search {
type Response = Option<InitialWebsiteResult>;
async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index be99bb2a..2ba0d9d1 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -113,7 +113,6 @@ pub struct RetrieveWebsites {
pub websites: Vec<inverted_index::WebsitePointer>,
pub query: String,
}
-#[async_trait::async_trait]
impl sonic::service::Message<SearchService> for RetrieveWebsites {
type Response = Option<Vec<inverted_index::RetrievedWebpage>>;
async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
@@ -131,7 +130,6 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {
pub struct Search {
pub query: SearchQuery,
}
-#[async_trait::async_trait]
impl sonic::service::Message<SearchService> for Search {
type Response = Option<InitialWebsiteResult>;
async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
@@ -146,7 +144,6 @@ impl sonic::service::Message<SearchService> for Search {
pub struct GetWebpage {
pub url: String,
}
-#[async_trait::async_trait]
impl sonic::service::Message<SearchService> for GetWebpage {
type Response = Option<RetrievedWebpage>;
async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
@@ -158,7 +155,6 @@ impl sonic::service::Message<SearchService> for GetWebpage {
pub struct GetHomepageDescriptions {
pub urls: Vec<Url>,
}
-#[async_trait::async_trait]
impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
type Response = HashMap<Url, String>;
async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
@@ -180,7 +176,6 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {
pub struct GetEntityImage {
pub image_id: String,
}
-#[async_trait::async_trait]
impl sonic::service::Message<SearchService> for GetEntityImage {
type Response = Option<Image>;
async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
index dd2cc624..c0e94793 100644
--- a/core/src/entrypoint/webgraph_server.rs
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -74,7 +74,6 @@ pub struct Knows {
pub host: String,
}
-#[async_trait::async_trait]
impl Message<WebGraphService> for SimilarHosts {
type Response = Vec<ScoredHost>;
@@ -110,7 +109,6 @@ impl Message<WebGraphService> for SimilarHosts {
}
}
-#[async_trait::async_trait]
impl Message<WebGraphService> for Knows {
type Response = Option<Node>;
@@ -140,7 +138,6 @@ pub struct IngoingLinks {
pub level: GraphLevel,
}
-#[async_trait::async_trait]
impl Message<WebGraphService> for IngoingLinks {
type Response = Vec<FullEdge>;
@@ -161,7 +158,6 @@ pub struct OutgoingLinks {
pub level: GraphLevel,
}
-#[async_trait::async_trait]
impl Message<WebGraphService> for OutgoingLinks {
type Response = Vec<FullEdge>;
diff --git a/core/src/live_index.rs b/core/src/live_index.rs
index f0822a24..53427be2 100644
--- a/core/src/live_index.rs
+++ b/core/src/live_index.rs
@@ -264,7 +264,6 @@ impl Index {
}
}
-#[async_trait::async_trait]
impl DatumStream for Indexer {
async fn write(&self, crawl_datum: CrawlDatum) -> Result<()> {
let webpage = self.worker.prepare_webpage(
diff --git a/core/src/mapreduce/worker.rs b/core/src/mapreduce/worker.rs
index 46fad548..adf34d0b 100644
--- a/core/src/mapreduce/worker.rs
+++ b/core/src/mapreduce/worker.rs
@@ -14,46 +14,46 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::net::SocketAddr;
+use std::{future::Future, net::SocketAddr};
use crate::mapreduce::MapReduceServer;
use super::{Map, Result, Task};
-use async_trait::async_trait;
use serde::{de::DeserializeOwned, Serialize};
use tracing::{debug, info};
#[derive(Default)]
pub struct StatelessWorker {}
-#[async_trait]
pub trait Worker {
- async fn run<I, O>(&self, addr: SocketAddr) -> Result<()>
+ fn run<I, O>(&self, addr: SocketAddr) -> impl Future<Output = Result<()>>
where
Self: Sized,
I: Map<Self, O> + Send + Sync,
O: Serialize + DeserializeOwned + Send + Sync,
{
- let server = MapReduceServer::<I, O>::bind(addr).await?;
- info!("worker listening on: {:}", addr);
-
- loop {
- let req = server.accept().await?;
- debug!("received request");
- match req.body() {
- Task::Job(job) => {
- debug!("request is a job");
- let res = job.map(self);
- req.respond(Some(res)).await?;
- }
- Task::AllFinished => {
- req.respond(None).await?;
- break;
+ async move {
+ let server = MapReduceServer::<I, O>::bind(addr).await?;
+ info!("worker listening on: {:}", addr);
+
+ loop {
+ let req = server.accept().await?;
+ debug!("received request");
+ match req.body() {
+ Task::Job(job) => {
+ debug!("request is a job");
+ let res = job.map(self);
+ req.respond(Some(res)).await?;
+ }
+ Task::AllFinished => {
+ req.respond(None).await?;
+ break;
+ }
}
}
- }
- Ok(())
+ Ok(())
+ }
}
}