web search done right
enforce edgelimit to only be applied in webgraph mod.rs so offset is only applied once
Mikkel Denkerf6bca073dbdb
diff --git a/crates/core/src/iter_ext.rs b/crates/core/src/iter_ext.rsindex 695c9976..8328dada 100644--- a/crates/core/src/iter_ext.rs+++ b/crates/core/src/iter_ext.rs@@ -27,7 +27,7 @@ pub trait IteratorExt: Iterator {pub trait VectorExt {type Iter: Iterator;- fn flat_sorted_by<F>(self, f: F) -> FlatSortedBy<Self::Iter, F>+ fn sort_sorted_by<F>(self, f: F) -> SortSortedBy<Self::Iter, F>whereSelf: Sized,F: FnMut(&<Self::Iter as Iterator>::Item, &<Self::Iter as Iterator>::Item) -> Ordering;@@ -39,12 +39,12 @@ where{type Iter = I;- fn flat_sorted_by<F>(self, f: F) -> FlatSortedBy<Self::Iter, F>+ fn sort_sorted_by<F>(self, f: F) -> SortSortedBy<Self::Iter, F>whereSelf: Sized,F: FnMut(&<Self::Iter as Iterator>::Item, &<Self::Iter as Iterator>::Item) -> Ordering,{- FlatSortedBy::new(self, f)+ SortSortedBy::new(self, f)}}@@ -83,9 +83,11 @@ impl<I: Iterator> Iterator for Peekable<I> {}}-/// Iterator that flattens an iterator of iterators and sorts the items.-/// It assumes that each inner iterator is sorted.-pub struct FlatSortedBy<I, F>+/// Iterator that produces a sorted sequence of items from an a set of+/// iterators where each iterator is sorted.+///+/// This iterator does not verify that the input iterators are sorted.+pub struct SortSortedBy<I, F>whereI: Iterator,{@@ -93,7 +95,7 @@ wheref: F,}-impl<I, F> FlatSortedBy<I, F>+impl<I, F> SortSortedBy<I, F>whereI: Iterator,{@@ -106,7 +108,7 @@ where}}-impl<I, F> Iterator for FlatSortedBy<I, F>+impl<I, F> Iterator for SortSortedBy<I, F>whereI: Iterator,F: FnMut(&<I as Iterator>::Item, &<I as Iterator>::Item) -> Ordering,@@ -139,7 +141,7 @@ mod tests {vec![4, 5, 6].into_iter(),vec![7, 8, 9].into_iter(),];- let sorted: Vec<_> = iters.flat_sorted_by(|a, b| a.cmp(b)).collect();+ let sorted: Vec<_> = iters.sort_sorted_by(|a, b| a.cmp(b)).collect();assert_eq!(sorted, vec![1, 2, 3, 4, 5, 6, 7, 8, 9]);}}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 11687fcb..0ff59837 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -100,10 +100,7 @@ pub enum EdgeLimit {}impl EdgeLimit {- pub fn apply<'a, T>(- &self,- it: impl Iterator<Item = T> + 'a,- ) -> Box<dyn Iterator<Item = T> + 'a> {+ fn apply<'a, T>(&self, it: impl Iterator<Item = T> + 'a) -> Box<dyn Iterator<Item = T> + 'a> {match self {EdgeLimit::Unlimited => Box::new(it),EdgeLimit::Limit(limit) => Box::new(it.take(*limit)),@@ -219,7 +216,7 @@ impl Webgraph {.iter().map(|segment| segment.ingoing_edges_with_label(node_id)).collect::<Vec<_>>() // collects the iterators from each segment, not the edges- .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))+ .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank())).unique_by(|edge| edge.from.node());limit@@ -256,7 +253,7 @@ impl Webgraph {.iter().map(|segment| segment.ingoing_edges(node)).collect::<Vec<_>>() // collects the iterators from each segment, not the edges- .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))+ .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank())).unique_by(|edge| edge.from.node());limit@@ -280,7 +277,7 @@ impl Webgraph {.iter().map(|segment| segment.ingoing_edges_with_label(node)).collect::<Vec<_>>() // collects the iterators from each segment, not the edges- .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))+ .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank())).unique_by(|edge| edge.from.node());limit@@ -304,7 +301,7 @@ impl Webgraph {.iter().map(|segment| segment.outgoing_edges_with_label(node)).collect::<Vec<_>>() // collects the iterators from each segment, not the edges- .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))+ .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank())).unique_by(|edge| edge.to.node());limit@@ -339,7 +336,7 @@ impl Webgraph {.iter().map(|segment| segment.outgoing_edges_with_label(&id)).collect::<Vec<_>>() // collects the iterators from each segment, not the edges- .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))+ .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank())).unique_by(|edge| edge.to.node());limit@@ -358,7 +355,7 @@ impl Webgraph {.iter().map(|segment| segment.outgoing_edges(node)).collect::<Vec<_>>() // collects the iterators from each segment, not the edges- .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))+ .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank())).unique_by(|edge| edge.to.node());limit