web search done right
use futures executor for single job instead of tokio since we got an error when trying to build the webgraph. It looked like rayon tried to spawn multiple tokio runtimes on the same thread due to the threadpool
Mikkel Denker3a5b573cc4e5
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 4ee0162a..27c2234d 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -239,11 +239,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {}pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- .block_on(async { async_process_job(job, worker).await })+ futures::executor::block_on(async_process_job(job, worker))}#[derive(Debug, Serialize, Deserialize)]diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex 2ca019f0..b701a469 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -105,11 +105,7 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {}pub fn process_job(job: &Job) -> webgraph::Webgraph {- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- .block_on(async { async_process_job(job).await })+ futures::executor::block_on(async_process_job(job))}impl Map<StatelessWorker, FrozenWebgraph> for Job {