web search done right
index warc files as they become available
Mikkel Denker002eb7f79c69
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 38f60af2..4861efe2 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -13,15 +13,17 @@//// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use futures::StreamExt;use std::net::SocketAddr;use std::path::Path;use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};+use tokio::pin;use tracing::{debug, info, trace};-use crate::entrypoint::download_all_warc_files;+use crate::entrypoint::async_download_all_warc_files;use crate::index::{FrozenIndex, Index};use crate::mapreduce::{Manager, Map, Reduce, Worker};use crate::ranking::centrality_store::CentralityStore;@@ -75,7 +77,7 @@ impl IndexingWorker {}}-fn process_job(job: &Job, worker: &IndexingWorker) -> Index {+async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {let name = job.warc_paths.first().unwrap().split('/').last().unwrap();info!("processing {}", name);@@ -87,11 +89,16 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {JobConfig::Local(config) => WarcSource::Local(config),};- let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.base_path);+ let warc_files = async_download_all_warc_files(&job.warc_paths, &source, &job.base_path).await;+ pin!(warc_files);+let signal_aggregator = SignalAggregator::default();- for file in warc_files {- if let Ok(file) = WarcFile::open(&file) {+ while let Some(file) = warc_files.next().await {+ let name = file.split('/').last().unwrap();+ let path = Path::new(&job.base_path).join("warc_files").join(name);++ if let Ok(file) = WarcFile::open(path) {for record infile.records().flatten()@@ -176,6 +183,14 @@ fn process_job(job: &Job, worker: &IndexingWorker) -> Index {index}+fn process_job(job: &Job, worker: &IndexingWorker) -> Index {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async { async_process_job(job, worker).await })+}+#[derive(Debug, Serialize, Deserialize)]struct IndexPointer(String);diff --git a/src/entrypoint/mod.rs b/src/entrypoint/mod.rsindex ba8fcfcd..0102062a 100644--- a/src/entrypoint/mod.rs+++ b/src/entrypoint/mod.rs@@ -24,77 +24,52 @@ use std::{fs::File, path::Path};pub use centrality::Centrality;pub use entity::EntityIndexer;-use futures::StreamExt;+use futures::{Stream, StreamExt};pub use indexer::Indexer;use tracing::debug;pub use webgraph::Webgraph;use crate::{warc::WarcFile, WarcSource};-async fn async_download_all_warc_files(- warc_paths: &[String],- source: &WarcSource,- download_path: &Path,-) {- futures::stream::iter(- warc_paths- .iter()- .map(|path| (path.split('/').last().unwrap(), path))- .map(|(name, path)| {- let file = File::options()- .create(true)- .truncate(true)- .write(true)- .open(download_path.join(name))- .unwrap();- (file, path)- })- .map(|(mut file, warc_path)| async move {- debug!("downloading warc file {}", &warc_path);- let res = WarcFile::download_into_buf(source, warc_path, &mut file)- .await- .ok();- debug!("finished downloading");-- res- }),- )- .buffer_unordered(warc_paths.len())- .collect::<Vec<Option<()>>>()- .await;-}--fn download_all_warc_files(- warc_paths: &[String],- source: &WarcSource,- base_path: &str,-) -> Vec<String> {+async fn async_download_all_warc_files<'a>(+ warc_paths: &'a [String],+ source: &'a WarcSource,+ base_path: &'a str,+) -> impl Stream<Item = String> + 'a {let download_path = Path::new(base_path).join("warc_files");if !download_path.exists() {std::fs::create_dir_all(&download_path).unwrap();}- let file_paths = warc_paths+ let warc_paths: Vec<_> = warc_paths.iter()- .map(|warc_path| warc_path.split('/').last().unwrap())- .map(|name| {- download_path- .join(name)- .as_os_str()- .to_str()- .unwrap()- .to_string()- })+ .map(|warc_path| warc_path.to_string()).collect();- tokio::runtime::Builder::new_current_thread()- .enable_all()- .build()- .unwrap()- .block_on(async {- async_download_all_warc_files(warc_paths, source, &download_path).await- });+ let num_files = warc_paths.len();++ futures::stream::iter(warc_paths.into_iter().map(|warc_path| async {+ let download_path = Path::new(base_path).join("warc_files");+ let name = warc_path.split('/').last().unwrap();+ let mut file = File::options()+ .create(true)+ .truncate(true)+ .write(true)+ .open(download_path.join(name))+ .unwrap();+ debug!("downloading warc file {}", &warc_path);+ let res = WarcFile::download_into_buf(source, &warc_path, &mut file).await;+ debug!("finished downloading");- file_paths+ (warc_path, res)+ }))+ .buffer_unordered(num_files)+ .filter_map(|(path, res)| async move {+ if res.is_ok() {+ Some(path.clone())+ } else {+ None+ }+ })}diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex fa7b1edb..f7885465 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -15,17 +15,19 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{directory::DirEntry,- entrypoint::download_all_warc_files,+ entrypoint::async_download_all_warc_files,mapreduce::{Manager, Map, Reduce, StatelessWorker, Worker},warc::WarcFile,webgraph::{self, FrozenWebgraph, Node, WebgraphBuilder},webpage::Html,HttpConfig, LocalConfig, Result, WarcSource, WebgraphLocalConfig, WebgraphMasterConfig,};+use futures::StreamExt;use itertools::Itertools;use rayon::prelude::*;use serde::{Deserialize, Serialize};use std::{net::SocketAddr, path::Path};+use tokio::pin;use tracing::{info, trace};#[derive(Debug, Serialize, Deserialize)]@@ -51,7 +53,7 @@ fn open_graph<P: AsRef<Path>>(path: P) -> webgraph::Webgraph {.open()}-fn process_job(job: &Job) -> webgraph::Webgraph {+async fn async_process_job(job: &Job) -> webgraph::Webgraph {let name = job.warc_paths.first().unwrap().split('/').last().unwrap();info!("processing {}", name);@@ -63,10 +65,17 @@ fn process_job(job: &Job) -> webgraph::Webgraph {JobConfig::Local(config) => WarcSource::Local(config),};- let warc_files = download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path);+ let warc_files =+ async_download_all_warc_files(&job.warc_paths, &source, &job.graph_base_path).await;+ pin!(warc_files);++ while let Some(warc_path) = warc_files.next().await {+ let name = warc_path.split('/').last().unwrap();+ let path = Path::new(&job.graph_base_path)+ .join("warc_files")+ .join(name);- for warc_path in &warc_files {- if let Ok(file) = WarcFile::open(warc_path) {+ if let Ok(file) = WarcFile::open(path) {for record in file.records().flatten() {let webpage = Html::parse_without_text(&record.response.body, &record.request.url);for link in webpage@@ -95,6 +104,14 @@ fn process_job(job: &Job) -> webgraph::Webgraph {graph}+fn process_job(job: &Job) -> webgraph::Webgraph {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(async { async_process_job(job).await })+}+impl Map<StatelessWorker, FrozenWebgraph> for Job {fn map(&self, _worker: &StatelessWorker) -> FrozenWebgraph {let graph = process_job(self);