Skip to content

web search done right

increase randomness of sampled nodes by using reservoir sampling to sample random nodes from graph

Mikkel Denkerc17824321bbe
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 5cc633fc..c73ff160 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -20,7 +20,7 @@ use std::sync::Arc;
use std::{fs, io};
use itertools::Itertools;
-use rand::seq::SliceRandom;
+use rand::seq::{IteratorRandom, SliceRandom};
use rayon::prelude::*;
use self::id_node_db::Id2NodeDb;
@@ -309,10 +309,9 @@ impl Webgraph {
.edges()
.map(|e| e.from)
.unique()
- .take(num)
- .collect::<Vec<_>>();
+ .choose_multiple(&mut rng, num);
nodes.shuffle(&mut rng);
- nodes.into_iter().take(num).collect()
+ nodes
}
pub fn par_nodes(&self) -> impl ParallelIterator<Item = NodeID> + '_ {