Skip to content

web search done right

enforce edgelimit to only be applied in webgraph mod.rs so offset is only applied once

Mikkel Denkerf6bca073dbdb
diff --git a/crates/core/src/iter_ext.rs b/crates/core/src/iter_ext.rs
index 695c9976..8328dada 100644
--- a/crates/core/src/iter_ext.rs
+++ b/crates/core/src/iter_ext.rs
@@ -27,7 +27,7 @@ pub trait IteratorExt: Iterator {
pub trait VectorExt {
type Iter: Iterator;
- fn flat_sorted_by<F>(self, f: F) -> FlatSortedBy<Self::Iter, F>
+ fn sort_sorted_by<F>(self, f: F) -> SortSortedBy<Self::Iter, F>
where
Self: Sized,
F: FnMut(&<Self::Iter as Iterator>::Item, &<Self::Iter as Iterator>::Item) -> Ordering;
@@ -39,12 +39,12 @@ where
{
type Iter = I;
- fn flat_sorted_by<F>(self, f: F) -> FlatSortedBy<Self::Iter, F>
+ fn sort_sorted_by<F>(self, f: F) -> SortSortedBy<Self::Iter, F>
where
Self: Sized,
F: FnMut(&<Self::Iter as Iterator>::Item, &<Self::Iter as Iterator>::Item) -> Ordering,
{
- FlatSortedBy::new(self, f)
+ SortSortedBy::new(self, f)
}
}
@@ -83,9 +83,11 @@ impl<I: Iterator> Iterator for Peekable<I> {
}
}
-/// Iterator that flattens an iterator of iterators and sorts the items.
-/// It assumes that each inner iterator is sorted.
-pub struct FlatSortedBy<I, F>
+/// Iterator that produces a sorted sequence of items from an a set of
+/// iterators where each iterator is sorted.
+///
+/// This iterator does not verify that the input iterators are sorted.
+pub struct SortSortedBy<I, F>
where
I: Iterator,
{
@@ -93,7 +95,7 @@ where
f: F,
}
-impl<I, F> FlatSortedBy<I, F>
+impl<I, F> SortSortedBy<I, F>
where
I: Iterator,
{
@@ -106,7 +108,7 @@ where
}
}
-impl<I, F> Iterator for FlatSortedBy<I, F>
+impl<I, F> Iterator for SortSortedBy<I, F>
where
I: Iterator,
F: FnMut(&<I as Iterator>::Item, &<I as Iterator>::Item) -> Ordering,
@@ -139,7 +141,7 @@ mod tests {
vec![4, 5, 6].into_iter(),
vec![7, 8, 9].into_iter(),
];
- let sorted: Vec<_> = iters.flat_sorted_by(|a, b| a.cmp(b)).collect();
+ let sorted: Vec<_> = iters.sort_sorted_by(|a, b| a.cmp(b)).collect();
assert_eq!(sorted, vec![1, 2, 3, 4, 5, 6, 7, 8, 9]);
}
}
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 11687fcb..0ff59837 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -100,10 +100,7 @@ pub enum EdgeLimit {
}
impl EdgeLimit {
- pub fn apply<'a, T>(
- &self,
- it: impl Iterator<Item = T> + 'a,
- ) -> Box<dyn Iterator<Item = T> + 'a> {
+ fn apply<'a, T>(&self, it: impl Iterator<Item = T> + 'a) -> Box<dyn Iterator<Item = T> + 'a> {
match self {
EdgeLimit::Unlimited => Box::new(it),
EdgeLimit::Limit(limit) => Box::new(it.take(*limit)),
@@ -219,7 +216,7 @@ impl Webgraph {
.iter()
.map(|segment| segment.ingoing_edges_with_label(node_id))
.collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
+ .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
.unique_by(|edge| edge.from.node());
limit
@@ -256,7 +253,7 @@ impl Webgraph {
.iter()
.map(|segment| segment.ingoing_edges(node))
.collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
+ .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
.unique_by(|edge| edge.from.node());
limit
@@ -280,7 +277,7 @@ impl Webgraph {
.iter()
.map(|segment| segment.ingoing_edges_with_label(node))
.collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
+ .sort_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))
.unique_by(|edge| edge.from.node());
limit
@@ -304,7 +301,7 @@ impl Webgraph {
.iter()
.map(|segment| segment.outgoing_edges_with_label(node))
.collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
+ .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
.unique_by(|edge| edge.to.node());
limit
@@ -339,7 +336,7 @@ impl Webgraph {
.iter()
.map(|segment| segment.outgoing_edges_with_label(&id))
.collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
+ .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
.unique_by(|edge| edge.to.node());
limit
@@ -358,7 +355,7 @@ impl Webgraph {
.iter()
.map(|segment| segment.outgoing_edges(node))
.collect::<Vec<_>>() // collects the iterators from each segment, not the edges
- .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
+ .sort_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))
.unique_by(|edge| edge.to.node());
limit