Skip to content

web search done right

More logging if some things looks off during indexing. Might be due to wrong path for centrality stores, webgraph etc.

Mikkel Denker361e0bbc2f70
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 1f032137..b7962351 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -20,7 +20,7 @@ use std::thread;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use tokio::pin;
-use tracing::{debug, info, trace};
+use tracing::{debug, info, trace, warn};
use crate::entrypoint::download_all_warc_files;
use crate::executor::Executor;
@@ -100,6 +100,10 @@ impl IndexingWorker {
pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let name = job.warc_paths.first().unwrap().split('/').last().unwrap();
+ let mut has_host_centrality = false;
+ let mut has_page_centrality = false;
+ let mut has_backlinks = false;
+
info!("processing {}", name);
let mut index = Index::open(Path::new(&job.base_path).join(name)).unwrap();
@@ -180,6 +184,18 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
.unwrap_or_default();
}
+ if host_centrality > 0.0 {
+ has_host_centrality = true;
+ }
+
+ if page_centrality > 0.0 {
+ has_page_centrality = true;
+ }
+
+ if !backlinks.is_empty() {
+ has_backlinks = true;
+ }
+
let fetch_time_ms = record.metadata.fetch_time_ms as u64;
trace!("inserting webpage: {:?}", html.url());
@@ -227,6 +243,18 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
std::fs::remove_file(path).unwrap();
}
+ if !has_host_centrality {
+ warn!("no host centrality values found in {}", name);
+ }
+
+ if !has_page_centrality {
+ warn!("no page centrality values found in {}", name);
+ }
+
+ if !has_backlinks {
+ warn!("no backlinks found in {}", name);
+ }
+
index.inverted_index.merge_into_max_segments(1).unwrap();
info!("{} done", name);