web search done right
sometimes you just gotta say fuck async
Mikkel Denker8277a0521c65
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 27c2234d..5b4c2b39 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use chrono::Utc;-use futures::StreamExt;use std::net::SocketAddr;use std::path::Path;use std::thread;@@ -24,7 +23,7 @@ use serde::{Deserialize, Serialize};use tokio::pin;use tracing::{debug, info, trace};-use crate::entrypoint::async_download_all_warc_files;+use crate::entrypoint::download_all_warc_files;use crate::executor::Executor;use crate::index::{FrozenIndex, Index};use crate::mapreduce::{Manager, Map, Reduce, Worker};@@ -81,7 +80,7 @@ impl IndexingWorker {}}-async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {+pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let name = job.warc_paths.first().unwrap().split('/').last().unwrap();info!("processing {}", name);@@ -93,12 +92,12 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {JobConfig::Local(config) => WarcSource::Local(config),};- let warc_files = async_download_all_warc_files(&job.warc_paths, &source, &job.base_path).await;+ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);pin!(warc_files);let current_timestamp = Utc::now().timestamp().max(0) as usize;- while let Some(file) = warc_files.next().await {+ for file in warc_files.by_ref() {let name = file.split('/').last().unwrap();let path = Path::new(&job.base_path).join("warc_files").join(name);@@ -238,10 +237,6 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {index}-pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {- futures::executor::block_on(async_process_job(job, worker))-}-#[derive(Debug, Serialize, Deserialize)]pub struct IndexPointer(String);diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rsindex 0933d804..f5dff68c 100644--- a/core/src/entrypoint/mod.rs+++ b/core/src/entrypoint/mod.rs@@ -30,18 +30,17 @@ use std::{fs::File, path::Path};pub use centrality::Centrality;pub use entity::EntityIndexer;-use futures::{Stream, StreamExt};pub use indexer::Indexer;-use tracing::debug;+use tracing::{debug, log::error};pub use webgraph::Webgraph;use crate::{warc::WarcFile, WarcSource};-async fn async_download_all_warc_files<'a>(+fn download_all_warc_files<'a>(warc_paths: &'a [String],source: &'a WarcSource,base_path: &'a str,-) -> impl Stream<Item = String> + 'a {+) -> impl Iterator<Item = String> + 'a {let download_path = Path::new(base_path).join("warc_files");if !download_path.exists() {@@ -53,9 +52,7 @@ async fn async_download_all_warc_files<'a>(.map(|warc_path| warc_path.to_string()).collect();- let num_files = warc_paths.len();-- futures::stream::iter(warc_paths.into_iter().map(|warc_path| async {+ warc_paths.into_iter().map(|warc_path| {let download_path = Path::new(base_path).join("warc_files");let name = warc_path.split('/').last().unwrap();let mut file = File::options()@@ -65,15 +62,14 @@ async fn async_download_all_warc_files<'a>(.open(download_path.join(name)).unwrap();debug!("downloading warc file {}", &warc_path);- let res = WarcFile::download_into_buf(source, &warc_path, &mut file).await;+ let res = WarcFile::download_into_buf(source, &warc_path, &mut file);if let Err(err) = res {- debug!("error while downloading: {:?}", err);+ error!("error while downloading: {:?}", err);}debug!("finished downloading");warc_path- }))- .buffer_unordered(num_files)+ })}diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rsindex b701a469..fb1351c1 100644--- a/core/src/entrypoint/webgraph.rs+++ b/core/src/entrypoint/webgraph.rs@@ -14,14 +14,13 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{- entrypoint::async_download_all_warc_files,+ entrypoint::download_all_warc_files,mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},warc::WarcFile,webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,};-use futures::StreamExt;use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};@@ -63,11 +62,10 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {JobConfig::Local(config) => WarcSource::Local(config),};- let warc_files =- async_download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path).await;+ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);pin!(warc_files);- while let Some(warc_path) = warc_files.next().await {+ for warc_path in warc_files.by_ref() {let name = warc_path.split('/').last().unwrap();let path = Path::new(&job.graph_base_path).join("warc_files")@@ -105,7 +103,11 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {}pub fn process_job(job: &Job) -> webgraph::Webgraph {- futures::executor::block_on(async_process_job(job))+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async { async_process_job(job).await })}impl Map<StatelessWorker, FrozenWebgraph> for Job {diff --git a/core/src/warc.rs b/core/src/warc.rsindex 861c9525..16888558 100644--- a/core/src/warc.rs+++ b/core/src/warc.rs@@ -20,11 +20,10 @@ use std::collections::BTreeMap;use std::fs::File;use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};use std::path::Path;+use std::thread::sleep;use std::time::Duration;use flate2::read::MultiGzDecoder;-use futures::StreamExt;-use tokio::time::sleep;use tracing::{debug, trace};pub(crate) struct WarcFile {@@ -81,9 +80,9 @@ impl WarcFile {}#[allow(unused)]- pub(crate) async fn download(source: &WarcSource, warc_path: &str) -> Result<Self> {+ pub(crate) fn download(source: &WarcSource, warc_path: &str) -> Result<Self> {let mut cursor = Cursor::new(Vec::new());- Self::download_into_buf(source, warc_path, &mut cursor).await?;+ Self::download_into_buf(source, warc_path, &mut cursor)?;let mut buf = Vec::new();cursor.read_to_end(&mut buf)?;@@ -91,7 +90,7 @@ impl WarcFile {Ok(Self::new(buf))}- pub(crate) async fn download_into_buf<W: Write + Seek>(+ pub(crate) fn download_into_buf<W: Write + Seek>(source: &WarcSource,warc_path: &str,buf: &mut W,@@ -102,7 +101,7 @@ impl WarcFile {{let res = match source.clone() {WarcSource::HTTP(config) => {- WarcFile::download_from_http(warc_path, config.base_url, buf).await+ WarcFile::download_from_http(warc_path, config.base_url, buf)}WarcSource::Local(config) => {WarcFile::load_from_folder(warc_path, &config.folder, buf)@@ -118,7 +117,7 @@ impl WarcFile {debug!("warc download failed: {:?}", res.err().unwrap());debug!("retrying in {} ms", dur.as_millis());- sleep(dur).await;+ sleep(dur);}Err(Error::DownloadFailed)@@ -133,7 +132,7 @@ impl WarcFile {Ok(())}- async fn download_from_http<W: Write + Seek>(+ fn download_from_http<W: Write + Seek>(warc_path: &str,base_url: String,buf: &mut W,@@ -144,24 +143,22 @@ impl WarcFile {}url += warc_path;- let client = reqwest::ClientBuilder::new()+ let client = reqwest::blocking::ClientBuilder::new().tcp_keepalive(None).pool_idle_timeout(Duration::from_secs(30 * 60)).timeout(Duration::from_secs(30 * 60)).connect_timeout(Duration::from_secs(30 * 60)).build()?;- let res = client.get(url).send().await?;+ let res = client.get(url).send()?;if res.status().as_u16() != 200 {return Err(Error::DownloadFailed);}- let mut stream = res.bytes_stream();+ let bytes = res.bytes()?;buf.rewind()?;- while let Some(chunk) = stream.next().await {- std::io::copy(&mut &chunk?[..], buf)?;- }+ std::io::copy(&mut &bytes[..], buf)?;Ok(())}