Skip to content

web search done right

index warc files as they become available

Mikkel Denker002eb7f79c69
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 38f60af2..4861efe2 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -13,15 +13,17 @@
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use futures::StreamExt;
use std::net::SocketAddr;
use std::path::Path;
use itertools::Itertools;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
+use tokio::pin;
use tracing::{debug, info, trace};
-use crate::entrypoint::download_all_warc_files;
+use crate::entrypoint::async_download_all_warc_files;
use crate::index::{FrozenIndex, Index};
use crate::mapreduce::{Manager, Map, Reduce, Worker};
use crate::ranking::centrality_store::CentralityStore;
@@ -75,7 +77,7 @@ impl IndexingWorker {
}
}
-fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
+async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
info!("processing {}", name);
@@ -87,11 +89,16 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
JobConfig::Local(config) => WarcSource::Local(config),
};
- let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);
+ let warc_files = async_download_all_warc_files(&job.warc_paths, &source, &job.base_path).await;
+ pin!(warc_files);
+
let signal_aggregator = SignalAggregator::default();
- for file in warc_files {
- if let Ok(file) = WarcFile::open(&file) {
+ while let Some(file) = warc_files.next().await {
+ let name = file.split('/').last().unwrap();
+ let path = Path::new(&job.base_path).join("warc_files").join(name);
+
+ if let Ok(file) = WarcFile::open(path) {
for record in
file.records()
.flatten()
@@ -176,6 +183,14 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
index
}
+fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async { async_process_job(job, worker).await })
+}
+
#[derive(Debug, Serialize, Deserialize)]
struct IndexPointer(String);
diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rs
index ba8fcfcd..0102062a 100644
--- a/src/entrypoint/mod.rs
+++ b/src/entrypoint/mod.rs
@@ -24,77 +24,52 @@ use std::{fs::File, path::Path};
pub use centrality::Centrality;
pub use entity::EntityIndexer;
-use futures::StreamExt;
+use futures::{Stream, StreamExt};
pub use indexer::Indexer;
use tracing::debug;
pub use webgraph::Webgraph;
use crate::{warc::WarcFile, WarcSource};
-async fn async_download_all_warc_files(
- warc_paths: &[String],
- source: &WarcSource,
- download_path: &Path,
-) {
- futures::stream::iter(
- warc_paths
- .iter()
- .map(|path| (path.split('/').last().unwrap(), path))
- .map(|(name, path)| {
- let file = File::options()
- .create(true)
- .truncate(true)
- .write(true)
- .open(download_path.join(name))
- .unwrap();
- (file, path)
- })
- .map(|(mut file, warc_path)| async move {
- debug!("downloading warc file {}", &warc_path);
- let res = WarcFile::download_into_buf(source, warc_path, &mut file)
- .await
- .ok();
- debug!("finished downloading");
-
- res
- }),
- )
- .buffer_unordered(warc_paths.len())
- .collect::<Vec<Option<()>>>()
- .await;
-}
-
-fn download_all_warc_files(
- warc_paths: &[String],
- source: &WarcSource,
- base_path: &str,
-) -> Vec<String> {
+async fn async_download_all_warc_files<'a>(
+ warc_paths: &'a [String],
+ source: &'a WarcSource,
+ base_path: &'a str,
+) -> impl Stream<Item = String> + 'a {
let download_path = Path::new(base_path).join("warc_files");
if !download_path.exists() {
std::fs::create_dir_all(&download_path).unwrap();
}
- let file_paths = warc_paths
+ let warc_paths: Vec<_> = warc_paths
.iter()
- .map(|warc_path| warc_path.split('/').last().unwrap())
- .map(|name| {
- download_path
- .join(name)
- .as_os_str()
- .to_str()
- .unwrap()
- .to_string()
- })
+ .map(|warc_path| warc_path.to_string())
.collect();
- tokio::runtime::Builder::new_current_thread()
- .enable_all()
- .build()
- .unwrap()
- .block_on(async {
- async_download_all_warc_files(warc_paths, source, &download_path).await
- });
+ let num_files = warc_paths.len();
+
+ futures::stream::iter(warc_paths.into_iter().map(|warc_path| async {
+ let download_path = Path::new(base_path).join("warc_files");
+ let name = warc_path.split('/').last().unwrap();
+ let mut file = File::options()
+ .create(true)
+ .truncate(true)
+ .write(true)
+ .open(download_path.join(name))
+ .unwrap();
+ debug!("downloading warc file {}", &warc_path);
+ let res = WarcFile::download_into_buf(source, &warc_path, &mut file).await;
+ debug!("finished downloading");
- file_paths
+ (warc_path, res)
+ }))
+ .buffer_unordered(num_files)
+ .filter_map(|(path, res)| async move {
+ if res.is_ok() {
+ Some(path.clone())
+ } else {
+ None
+ }
+ })
}
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index fa7b1edb..f7885465 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -15,17 +15,19 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
directory::DirEntry,
- entrypoint::download_all_warc_files,
+ entrypoint::async_download_all_warc_files,
mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},
warc::WarcFile,
webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},
webpage::Html,
HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,
};
+use futures::StreamExt;
use itertools::Itertools;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::{net::SocketAddr, path::Path};
+use tokio::pin;
use tracing::{info, trace};
#[derive(Debug, Serialize, Deserialize)]
@@ -51,7 +53,7 @@ fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {
.open()
}
-fn process_job(job: &Job) -> webgraph::Webgraph {
+async fn async_process_job(job: &Job) -> webgraph::Webgraph {
let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
info!("processing {}", name);
@@ -63,10 +65,17 @@ fn process_job(job: &Job) -> webgraph::Webgraph {
JobConfig::Local(config) => WarcSource::Local(config),
};
- let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);
+ let warc_files =
+ async_download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path).await;
+ pin!(warc_files);
+
+ while let Some(warc_path) = warc_files.next().await {
+ let name = warc_path.split('/').last().unwrap();
+ let path = Path::new(&job.graph_base_path)
+ .join("warc_files")
+ .join(name);
- for warc_path in &warc_files {
- if let Ok(file) = WarcFile::open(warc_path) {
+ if let Ok(file) = WarcFile::open(path) {
for record in file.records().flatten() {
let webpage = Html::parse_without_text(&record.response.body, &record.request.url);
for link in webpage
@@ -95,6 +104,14 @@ fn process_job(job: &Job) -> webgraph::Webgraph {
graph
}
+fn process_job(job: &Job) -> webgraph::Webgraph {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ .block_on(async { async_process_job(job).await })
+}
+
impl Map<StatelessWorker, FrozenWebgraph> for Job {
fn map(&self, _worker: &StatelessWorker) -> FrozenWebgraph {
let graph = process_job(self);