web search done right
More logging if some things looks off during indexing. Might be due to wrong path for centrality stores, webgraph etc.
Mikkel Denker361e0bbc2f70
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 1f032137..b7962351 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -20,7 +20,7 @@ use std::thread;use itertools::Itertools;use serde::{Deserialize, Serialize};use tokio::pin;-use tracing::{debug, info, trace};+use tracing::{debug, info, trace, warn};use crate::entrypoint::download_all_warc_files;use crate::executor::Executor;@@ -100,6 +100,10 @@ impl IndexingWorker {pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let name = job.warc_paths.first().unwrap().split('/').last().unwrap();+ let mut has_host_centrality = false;+ let mut has_page_centrality = false;+ let mut has_backlinks = false;+info!("processing {}", name);let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();@@ -180,6 +184,18 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {.unwrap_or_default();}+ if host_centrality > 0.0 {+ has_host_centrality = true;+ }++ if page_centrality > 0.0 {+ has_page_centrality = true;+ }++ if !backlinks.is_empty() {+ has_backlinks = true;+ }+let fetch_time_ms = record.metadata.fetch_time_ms as u64;trace!("inserting webpage: {:?}", html.url());@@ -227,6 +243,18 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {std::fs::remove_file(path).unwrap();}+ if !has_host_centrality {+ warn!("no host centrality values found in {}", name);+ }++ if !has_page_centrality {+ warn!("no page centrality values found in {}", name);+ }++ if !has_backlinks {+ warn!("no backlinks found in {}", name);+ }+index.inverted_index.merge_into_max_segments(1).unwrap();info!("{} done", name);