Skip to content

web search done right

sometimes you just gotta say fuck async

Mikkel Denker8277a0521c65
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 27c2234d..5b4c2b39 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -14,7 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use chrono::Utc;
-use futures::StreamExt;
use std::net::SocketAddr;
use std::path::Path;
use std::thread;
@@ -24,7 +23,7 @@ use serde::{Deserialize, Serialize};
use tokio::pin;
use tracing::{debug, info, trace};
-use crate::entrypoint::async_download_all_warc_files;
+use crate::entrypoint::download_all_warc_files;
use crate::executor::Executor;
use crate::index::{FrozenIndex, Index};
use crate::mapreduce::{Manager, Map, Reduce, Worker};
@@ -81,7 +80,7 @@ impl IndexingWorker {
}
}
-async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
+pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
info!("processing {}", name);
@@ -93,12 +92,12 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
JobConfig::Local(config) => WarcSource::Local(config),
};
- let warc_files = async_download_all_warc_files(&job.warc_paths, &source, &job.base_path).await;
+ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);
pin!(warc_files);
let current_timestamp = Utc::now().timestamp().max(0) as usize;
- while let Some(file) = warc_files.next().await {
+ for file in warc_files.by_ref() {
let name = file.split('/').last().unwrap();
let path = Path::new(&job.base_path).join("warc_files").join(name);
@@ -238,10 +237,6 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
index
}
-pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
- futures::executor::block_on(async_process_job(job, worker))
-}
-
#[derive(Debug, Serialize, Deserialize)]
pub struct IndexPointer(String);
diff --git a/core/src/entrypoint/mod.rs b/core/src/entrypoint/mod.rs
index 0933d804..f5dff68c 100644
--- a/core/src/entrypoint/mod.rs
+++ b/core/src/entrypoint/mod.rs
@@ -30,18 +30,17 @@ use std::{fs::File, path::Path};
pub use centrality::Centrality;
pub use entity::EntityIndexer;
-use futures::{Stream, StreamExt};
pub use indexer::Indexer;
-use tracing::debug;
+use tracing::{debug, log::error};
pub use webgraph::Webgraph;
use crate::{warc::WarcFile, WarcSource};
-async fn async_download_all_warc_files<'a>(
+fn download_all_warc_files<'a>(
warc_paths: &'a [String],
source: &'a WarcSource,
base_path: &'a str,
-) -> impl Stream<Item = String> + 'a {
+) -> impl Iterator<Item = String> + 'a {
let download_path = Path::new(base_path).join("warc_files");
if !download_path.exists() {
@@ -53,9 +52,7 @@ async fn async_download_all_warc_files<'a>(
.map(|warc_path| warc_path.to_string())
.collect();
- let num_files = warc_paths.len();
-
- futures::stream::iter(warc_paths.into_iter().map(|warc_path| async {
+ warc_paths.into_iter().map(|warc_path| {
let download_path = Path::new(base_path).join("warc_files");
let name = warc_path.split('/').last().unwrap();
let mut file = File::options()
@@ -65,15 +62,14 @@ async fn async_download_all_warc_files<'a>(
.open(download_path.join(name))
.unwrap();
debug!("downloading warc file {}", &warc_path);
- let res = WarcFile::download_into_buf(source, &warc_path, &mut file).await;
+ let res = WarcFile::download_into_buf(source, &warc_path, &mut file);
if let Err(err) = res {
- debug!("error while downloading: {:?}", err);
+ error!("error while downloading: {:?}", err);
}
debug!("finished downloading");
warc_path
- }))
- .buffer_unordered(num_files)
+ })
}
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index b701a469..fb1351c1 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -14,14 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
- entrypoint::async_download_all_warc_files,
+ entrypoint::download_all_warc_files,
mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},
warc::WarcFile,
webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,
};
-use futures::StreamExt;
use itertools::Itertools;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
@@ -63,11 +62,10 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
JobConfig::Local(config) => WarcSource::Local(config),
};
- let warc_files =
- async_download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path).await;
+ let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);
pin!(warc_files);
- while let Some(warc_path) = warc_files.next().await {
+ for warc_path in warc_files.by_ref() {
let name = warc_path.split('/').last().unwrap();
let path = Path::new(&job.graph_base_path)
.join("warc_files")
@@ -105,7 +103,11 @@ async fn async_process_job(job: &Job) -> webgraph::Webgraph {
}
pub fn process_job(job: &Job) -> webgraph::Webgraph {
- futures::executor::block_on(async_process_job(job))
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async { async_process_job(job).await })
}
impl Map<StatelessWorker, FrozenWebgraph> for Job {
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 861c9525..16888558 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -20,11 +20,10 @@ use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufRead, BufReader, Cursor, Read, Seek, Write};
use std::path::Path;
+use std::thread::sleep;
use std::time::Duration;
use flate2::read::MultiGzDecoder;
-use futures::StreamExt;
-use tokio::time::sleep;
use tracing::{debug, trace};
pub(crate) struct WarcFile {
@@ -81,9 +80,9 @@ impl WarcFile {
}
#[allow(unused)]
- pub(crate) async fn download(source: &WarcSource, warc_path: &str) -> Result<Self> {
+ pub(crate) fn download(source: &WarcSource, warc_path: &str) -> Result<Self> {
let mut cursor = Cursor::new(Vec::new());
- Self::download_into_buf(source, warc_path, &mut cursor).await?;
+ Self::download_into_buf(source, warc_path, &mut cursor)?;
let mut buf = Vec::new();
cursor.read_to_end(&mut buf)?;
@@ -91,7 +90,7 @@ impl WarcFile {
Ok(Self::new(buf))
}
- pub(crate) async fn download_into_buf<W: Write + Seek>(
+ pub(crate) fn download_into_buf<W: Write + Seek>(
source: &WarcSource,
warc_path: &str,
buf: &mut W,
@@ -102,7 +101,7 @@ impl WarcFile {
{
let res = match source.clone() {
WarcSource::HTTP(config) => {
- WarcFile::download_from_http(warc_path, config.base_url, buf).await
+ WarcFile::download_from_http(warc_path, config.base_url, buf)
}
WarcSource::Local(config) => {
WarcFile::load_from_folder(warc_path, &config.folder, buf)
@@ -118,7 +117,7 @@ impl WarcFile {
debug!("warc download failed: {:?}", res.err().unwrap());
debug!("retrying in {} ms", dur.as_millis());
- sleep(dur).await;
+ sleep(dur);
}
Err(Error::DownloadFailed)
@@ -133,7 +132,7 @@ impl WarcFile {
Ok(())
}
- async fn download_from_http<W: Write + Seek>(
+ fn download_from_http<W: Write + Seek>(
warc_path: &str,
base_url: String,
buf: &mut W,
@@ -144,24 +143,22 @@ impl WarcFile {
}
url += warc_path;
- let client = reqwest::ClientBuilder::new()
+ let client = reqwest::blocking::ClientBuilder::new()
.tcp_keepalive(None)
.pool_idle_timeout(Duration::from_secs(30 * 60))
.timeout(Duration::from_secs(30 * 60))
.connect_timeout(Duration::from_secs(30 * 60))
.build()?;
- let res = client.get(url).send().await?;
+ let res = client.get(url).send()?;
if res.status().as_u16() != 200 {
return Err(Error::DownloadFailed);
}
- let mut stream = res.bytes_stream();
+ let bytes = res.bytes()?;
buf.rewind()?;
- while let Some(chunk) = stream.next().await {
- std::io::copy(&mut &chunk?[..], buf)?;
- }
+ std::io::copy(&mut &bytes[..], buf)?;
Ok(())
}