Skip to content

web search done right

Optionally skip some warc files when indexing

Mikkel Denker8b9753d4a0ff
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 66e3eeff..8fe79ea7 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -309,9 +309,11 @@ impl Indexer {
WarcSource::Local(config) => JobConfig::Local(config),
};
- let mut warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
+ let warc_paths: Box<dyn Iterator<Item = Job> + Send> = Box::new(
warc_paths
.into_iter()
+ .skip(config.skip_num_warc_files.unwrap_or(0))
+ .take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
.into_iter()
.map(|warc_paths| Job {
@@ -329,10 +331,6 @@ impl Indexer {
.into_iter(),
);
- if let Some(limit) = config.limit_warc_files {
- warc_paths = Box::new(warc_paths.take(limit));
- }
-
let manager = Manager::new(&workers);
let mut index: Index = manager
.run::<IndexingWorker, Job, FrozenIndex, Index>(warc_paths)
@@ -394,6 +392,7 @@ impl Indexer {
let indexes: Vec<_> = warc_paths
.into_iter()
+ .skip(config.skip_num_warc_files.unwrap_or(0))
.take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
.into_iter()
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 95bf77d7..1d100fe5 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -70,6 +70,7 @@ mod widgets;
#[derive(Debug, Deserialize, Clone)]
pub struct IndexingMasterConfig {
limit_warc_files: Option<usize>,
+ skip_num_warc_files: Option<usize>,
final_num_segments: Option<u32>,
warc_source: WarcSource,
workers: Vec<String>,
@@ -83,6 +84,7 @@ pub struct IndexingMasterConfig {
pub struct IndexingLocalConfig {
crawl_stability_path: Option<String>,
limit_warc_files: Option<usize>,
+ skip_num_warc_files: Option<usize>,
final_num_segments: Option<u32>,
warc_source: WarcSource,
batch_size: Option<usize>,
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 8cba124d..a3e1e55f 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -38,10 +38,10 @@ use crate::webgraph::{Edge, Node};
use crate::webgraph::{NodeID, Webgraph};
use crate::Result;
-const NUM_PROXY_NODES: usize = 50;
+const NUM_PROXY_NODES: usize = 500;
const BEST_PROXY_NODES_PER_USER_NODE: usize = 3;
const USER_NODES_LIMIT: usize = 100; // if the user specifies more than this number of nodes, the remaining nodes will be merged into existing
-const MAX_DIST_PROXY: u8 = 5;
+const MAX_DIST_PROXY: u8 = 6;
const MAX_NUM_OUT_EDGES: usize = 1000;
pub const SCORE_OFFSET: f64 = 1.0;