Skip to content

web search done right

ability to skip warc files in webgraph construction

Mikkel Denkerc52653abd032
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index ef96315d..659b8fae 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -46,6 +46,7 @@ pub struct WebgraphConstructConfig {
pub page_graph_base_path: String,
pub warc_source: WarcSource,
pub limit_warc_files: Option<usize>,
+ pub skip_warc_files: Option<usize>,
pub batch_size: Option<usize>,
}
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index 1544d1ca..cd1d325a 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -160,6 +160,7 @@ impl Webgraph {
let jobs: Vec<_> = warc_paths
.into_iter()
+ .skip(config.skip_warc_files.unwrap_or(0))
.take(config.limit_warc_files.unwrap_or(usize::MAX))
.chunks(config.batch_size.unwrap_or(1))
.into_iter()
diff --git a/core/src/webgraph/store.rs b/core/src/webgraph/store.rs
index 1061fc1e..dd1729a5 100644
--- a/core/src/webgraph/store.rs
+++ b/core/src/webgraph/store.rs
@@ -459,8 +459,9 @@ impl EdgeStore {
if let Some(last_node) = last_node {
if (reversed && edge.to.id != last_node) || (!reversed && edge.from.id != last_node)
{
- batch
- .sort_by_key(|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id });
+ batch.sort_unstable_by_key(
+ |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },
+ );
batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });
s.put(&batch);
batch.clear();
@@ -472,7 +473,9 @@ impl EdgeStore {
}
if !batch.is_empty() {
- batch.sort_by_key(|e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id });
+ batch.sort_unstable_by_key(
+ |e: &InnerEdge<_>| if reversed { e.from.id } else { e.to.id },
+ );
batch.dedup_by_key(|e| if reversed { e.from.id } else { e.to.id });
s.put(&batch);
}