web search done right
Optionally skip some warc files when indexing
Mikkel Denker8b9753d4a0ff
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 66e3eeff..8fe79ea7 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -309,9 +309,11 @@ impl Indexer {WarcSource::Local(config) => JobConfig::Local(config),};- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(+ let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(warc_paths.into_iter()+ .skip(config.skip_num_warc_files.unwrap_or(0))+ .take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1)).into_iter().map(|warc_paths| Job {@@ -329,10 +331,6 @@ impl Indexer {.into_iter(),);- if let Some(limit) = config.limit_warc_files {- warc_paths = Box::new(warc_paths.take(limit));- }-let manager = Manager::new(&workers);let mut index: Index = manager.run::<IndexingWorker, Job, FrozenIndex, Index>(warc_paths)@@ -394,6 +392,7 @@ impl Indexer {let indexes: Vec<_> = warc_paths.into_iter()+ .skip(config.skip_num_warc_files.unwrap_or(0)).take(config.limit_warc_files.unwrap_or(usize::MAX)).chunks(config.batch_size.unwrap_or(1)).into_iter()diff --git a/core/src/lib.rs b/core/src/lib.rsindex 95bf77d7..1d100fe5 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -70,6 +70,7 @@ mod widgets;#[derive(Debug, Deserialize, Clone)]pub struct IndexingMasterConfig {limit_warc_files: Option<usize>,+ skip_num_warc_files: Option<usize>,final_num_segments: Option<u32>,warc_source: WarcSource,workers: Vec<String>,@@ -83,6 +84,7 @@ pub struct IndexingMasterConfig {pub struct IndexingLocalConfig {crawl_stability_path: Option<String>,limit_warc_files: Option<usize>,+ skip_num_warc_files: Option<usize>,final_num_segments: Option<u32>,warc_source: WarcSource,batch_size: Option<usize>,diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 8cba124d..a3e1e55f 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -38,10 +38,10 @@ use crate::webgraph::{Edge, Node};use crate::webgraph::{NodeID, Webgraph};use crate::Result;-const NUM_PROXY_NODES: usize = 50;+const NUM_PROXY_NODES: usize = 500;const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing-const MAX_DIST_PROXY: u8 = 5;+const MAX_DIST_PROXY: u8 = 6;const MAX_NUM_OUT_EDGES: usize = 1000;pub const SCORE_OFFSET: f64 = 1.0;