web search done right
implement EdgeLimit::LimitAndOffset in webgraph. to make sure the edges are returned in sorted order, we implement a new FlatSortedBy struct that takes a vec of iters and outputs the sorted order of the iter items. it assumes that each iter is already sorted so it doesn't have to traverse the entire result set
Mikkel Denker9414c4277a6b
diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rsindex 74790c59..f47f9dbd 100644--- a/crates/core/src/image_store.rs+++ b/crates/core/src/image_store.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License asdiff --git a/crates/core/src/iter_ext.rs b/crates/core/src/iter_ext.rsnew file mode 100644index 00000000..695c9976--- /dev/null+++ b/crates/core/src/iter_ext.rs@@ -0,0 +1,145 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::cmp::Ordering;++pub trait IteratorExt: Iterator {+ fn peekable_non_lazy(self) -> Peekable<Self>+ where+ Self: Sized,+ {+ Peekable::new(self)+ }+}++pub trait VectorExt {+ type Iter: Iterator;+ fn flat_sorted_by<F>(self, f: F) -> FlatSortedBy<Self::Iter, F>+ where+ Self: Sized,+ F: FnMut(&<Self::Iter as Iterator>::Item, &<Self::Iter as Iterator>::Item) -> Ordering;+}++impl<I> VectorExt for Vec<I>+where+ I: Iterator,+{+ type Iter = I;++ fn flat_sorted_by<F>(self, f: F) -> FlatSortedBy<Self::Iter, F>+ where+ Self: Sized,+ F: FnMut(&<Self::Iter as Iterator>::Item, &<Self::Iter as Iterator>::Item) -> Ordering,+ {+ FlatSortedBy::new(self, f)+ }+}++impl<I: Iterator> IteratorExt for I {}++pub struct Peekable<I>+where+ I: Iterator,+{+ iter: I,+ peeked: Option<I::Item>,+}++impl<I: Iterator> Peekable<I> {+ pub fn new(iter: I) -> Self {+ let mut iter = iter;+ let peeked = iter.next();+ Self { iter, peeked }+ }++ pub fn peek(&self) -> Option<&I::Item> {+ self.peeked.as_ref()+ }+}++impl<I: Iterator> Iterator for Peekable<I> {+ type Item = I::Item;++ fn next(&mut self) -> Option<Self::Item> {+ let peeked = self.peeked.take();+ if peeked.is_some() {+ self.peeked = self.iter.next();+ }++ peeked+ }+}++/// Iterator that flattens an iterator of iterators and sorts the items.+/// It assumes that each inner iterator is sorted.+pub struct FlatSortedBy<I, F>+where+ I: Iterator,+{+ iters: Vec<Peekable<I>>,+ f: F,+}++impl<I, F> FlatSortedBy<I, F>+where+ I: Iterator,+{+ pub fn new(iters: Vec<I>, f: F) -> Self+ where+ I: Iterator,+ {+ let iters = iters.into_iter().map(Peekable::new).collect();+ Self { iters, f }+ }+}++impl<I, F> Iterator for FlatSortedBy<I, F>+where+ I: Iterator,+ F: FnMut(&<I as Iterator>::Item, &<I as Iterator>::Item) -> Ordering,+{+ type Item = <I as Iterator>::Item;++ fn next(&mut self) -> Option<Self::Item> {+ let mut min = self+ .iters+ .iter_mut()+ .min_by(|a, b| match (a.peek(), b.peek()) {+ (Some(a), Some(b)) => (self.f)(a, b),+ (Some(_), None) => Ordering::Less,+ (None, Some(_)) => Ordering::Greater,+ (None, None) => Ordering::Equal,+ });++ min.as_mut()?.next()+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_flat_sorted_by() {+ let iters = vec![+ vec![1, 2, 3].into_iter(),+ vec![4, 5, 6].into_iter(),+ vec![7, 8, 9].into_iter(),+ ];+ let sorted: Vec<_> = iters.flat_sorted_by(|a, b| a.cmp(b)).collect();+ assert_eq!(sorted, vec![1, 2, 3, 4, 5, 6, 7, 8, 9]);+ }+}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex 37a46a8e..725739ef 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -65,6 +65,7 @@ pub mod image_store;mod improvement;pub mod index;mod intmap;+pub mod iter_ext;mod kahan_sum;mod leaky_queue;mod live_index;diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 70a8695a..11687fcb 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -28,6 +28,7 @@ use self::id_node_db::Id2NodeDb;use self::segment::Segment;use crate::executor::Executor;+use crate::iter_ext::VectorExt;use crate::Result;pub use builder::WebgraphBuilder;pub use compression::Compression;@@ -95,6 +96,7 @@ impl Meta {pub enum EdgeLimit {Unlimited,Limit(usize),+ LimitAndOffset { limit: usize, offset: usize },}impl EdgeLimit {@@ -105,6 +107,7 @@ impl EdgeLimit {match self {EdgeLimit::Unlimited => Box::new(it),EdgeLimit::Limit(limit) => Box::new(it.take(*limit)),+ EdgeLimit::LimitAndOffset { limit, offset } => Box::new(it.skip(*offset).take(*limit)),}}}@@ -211,19 +214,16 @@ impl Webgraph {}pub fn ingoing_edges_by_id(&self, node_id: &NodeID, limit: EdgeLimit) -> Vec<FullEdge> {- let dedup = |edges: &mut Vec<SegmentEdge<String>>| {- edges.sort_by_key(|e| e.from.node());- edges.dedup_by_key(|e| e.from.node());- };-- let mut edges = self.inner_edges(- |segment| segment.ingoing_edges_with_label(node_id, &limit),- dedup,- );- edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));+ let edges = self+ .segments+ .iter()+ .map(|segment| segment.ingoing_edges_with_label(node_id))+ .collect::<Vec<_>>() // collects the iterators from each segment, not the edges+ .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))+ .unique_by(|edge| edge.from.node());limit- .apply(edges.into_iter())+ .apply(edges).map(|e| FullEdge {from: self.id2node(&e.from.node()).unwrap(),to: self.id2node(&e.to.node()).unwrap(),@@ -251,16 +251,16 @@ impl Webgraph {}pub fn raw_ingoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {- let dedup = |edges: &mut Vec<SegmentEdge<()>>| {- edges.sort_by_key(|e| e.from.node());- edges.dedup_by_key(|e| e.from.node());- };-- let mut edges = self.inner_edges(|segment| segment.ingoing_edges(node, &limit), dedup);- edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));+ let edges = self+ .segments+ .iter()+ .map(|segment| segment.ingoing_edges(node))+ .collect::<Vec<_>>() // collects the iterators from each segment, not the edges+ .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))+ .unique_by(|edge| edge.from.node());limit- .apply(edges.into_iter())+ .apply(edges).map(|e| Edge {from: e.from,to: e.to,@@ -275,19 +275,16 @@ impl Webgraph {node: &NodeID,limit: EdgeLimit,) -> Vec<Edge<String>> {- let dedup = |edges: &mut Vec<SegmentEdge<String>>| {- edges.sort_by_key(|e| e.from.node());- edges.dedup_by_key(|e| e.from.node());- };-- let mut edges = self.inner_edges(- |segment| segment.ingoing_edges_with_label(node, &limit),- dedup,- );- edges.sort_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()));+ let edges = self+ .segments+ .iter()+ .map(|segment| segment.ingoing_edges_with_label(node))+ .collect::<Vec<_>>() // collects the iterators from each segment, not the edges+ .flat_sorted_by(|a, b| a.from.host_rank().cmp(&b.from.host_rank()))+ .unique_by(|edge| edge.from.node());limit- .apply(edges.into_iter())+ .apply(edges).map(|e| Edge {from: e.from,to: e.to,@@ -302,20 +299,16 @@ impl Webgraph {node: &NodeID,limit: EdgeLimit,) -> Vec<Edge<String>> {- let dedup = |edges: &mut Vec<SegmentEdge<String>>| {- edges.sort_by_key(|e| e.to.node());- edges.dedup_by_key(|e| e.to.node());- };-- let mut edges = self.inner_edges(- |segment| segment.outgoing_edges_with_label(node, &limit),- dedup,- );-- edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));+ let edges = self+ .segments+ .iter()+ .map(|segment| segment.outgoing_edges_with_label(node))+ .collect::<Vec<_>>() // collects the iterators from each segment, not the edges+ .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))+ .unique_by(|edge| edge.to.node());limit- .apply(edges.into_iter())+ .apply(edges).map(|e| Edge {from: e.from,to: e.to,@@ -340,19 +333,17 @@ impl Webgraph {}pub fn outgoing_edges(&self, node: Node, limit: EdgeLimit) -> Vec<FullEdge> {- let dedup = |edges: &mut Vec<SegmentEdge<String>>| {- edges.sort_by_key(|e| e.to.node());- edges.dedup_by_key(|e| e.to.node());- };-- let mut edges = self.inner_edges(- |segment| segment.outgoing_edges_with_label(&node.id(), &limit),- dedup,- );- edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));+ let id = node.id();+ let edges = self+ .segments+ .iter()+ .map(|segment| segment.outgoing_edges_with_label(&id))+ .collect::<Vec<_>>() // collects the iterators from each segment, not the edges+ .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))+ .unique_by(|edge| edge.to.node());limit- .apply(edges.into_iter())+ .apply(edges).map(|e| FullEdge {from: self.id2node(&e.from.node()).unwrap(),to: self.id2node(&e.to.node()).unwrap(),@@ -362,16 +353,16 @@ impl Webgraph {}pub fn raw_outgoing_edges(&self, node: &NodeID, limit: EdgeLimit) -> Vec<Edge<()>> {- let dedup = |edges: &mut Vec<SegmentEdge<()>>| {- edges.sort_by_key(|e| e.to.node());- edges.dedup_by_key(|e| e.to.node());- };-- let mut edges = self.inner_edges(|segment| segment.outgoing_edges(node, &limit), dedup);- edges.sort_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()));+ let edges = self+ .segments+ .iter()+ .map(|segment| segment.outgoing_edges(node))+ .collect::<Vec<_>>() // collects the iterators from each segment, not the edges+ .flat_sorted_by(|a, b| a.to.host_rank().cmp(&b.to.host_rank()))+ .unique_by(|edge| edge.to.node());limit- .apply(edges.into_iter())+ .apply(edges).map(|e| Edge {from: e.from,to: e.to,@@ -381,25 +372,6 @@ impl Webgraph {.collect()}- fn inner_edges<F1, F2, L>(&self, loader: F1, dedup: F2) -> Vec<SegmentEdge<L>>- where- L: EdgeLabel,- F1: Sized + Sync + Fn(&Segment) -> Vec<SegmentEdge<L>>,- F2: Fn(&mut Vec<SegmentEdge<L>>),- {- let mut edges: Vec<_> = self- .executor- .map(loader, self.segments.iter())- .unwrap()- .into_iter()- .flatten()- .collect();-- dedup(&mut edges);-- edges- }-pub fn id2node(&self, id: &NodeID) -> Option<Node> {self.id2node.get(id)}@@ -1103,4 +1075,28 @@ pub mod tests {RelFlags::IS_IN_FOOTER | RelFlags::TAG,);}++ #[test]+ fn test_limit_and_offset() {+ let (graph, _temp_dir) = test_graph();++ let no_offset = graph.raw_outgoing_edges(+ &Node::from("A").id(),+ EdgeLimit::LimitAndOffset {+ limit: 2,+ offset: 0,+ },+ );+ assert_eq!(no_offset.len(), 2);++ let edges = graph.raw_outgoing_edges(+ &Node::from("A").id(),+ EdgeLimit::LimitAndOffset {+ limit: 2,+ offset: 1,+ },+ );+ assert_eq!(edges.len(), 1);+ assert_eq!(edges[0].to, no_offset[1].to);+ }}diff --git a/crates/core/src/webgraph/segment.rs b/crates/core/src/webgraph/segment.rsindex 614b526f..6c312dbd 100644--- a/crates/core/src/webgraph/segment.rs+++ b/crates/core/src/webgraph/segment.rs@@ -21,8 +21,8 @@ use std::{};use super::{- store::EdgeStore, store_writer::EdgeStoreWriter, Compression, EdgeLimit, InsertableEdge,- NodeID, SegmentEdge,+ store::EdgeStore, store_writer::EdgeStoreWriter, Compression, InsertableEdge, NodeID,+ SegmentEdge,};use crate::Result;@@ -160,32 +160,36 @@ impl Segment {Ok(())}- pub fn outgoing_edges_with_label(- &self,- node: &NodeID,- limit: &EdgeLimit,- ) -> Vec<SegmentEdge<String>> {- self.adjacency.get_with_label(node, limit)+ pub fn outgoing_edges_with_label<'a>(+ &'a self,+ node: &'a NodeID,+ ) -> Box<dyn Iterator<Item = SegmentEdge<String>> + Send + Sync + 'a> {+ self.adjacency.get_with_label(node)}- pub fn outgoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {- self.adjacency.get_without_label(node, limit)+ pub fn outgoing_edges<'a>(+ &'a self,+ node: &'a NodeID,+ ) -> Box<dyn Iterator<Item = SegmentEdge<()>> + Send + Sync + 'a> {+ self.adjacency.get_without_label(node)}pub fn out_degree(&self, node: &NodeID) -> u64 {self.adjacency.degree(node)}- pub fn ingoing_edges_with_label(- &self,- node: &NodeID,- limit: &EdgeLimit,- ) -> Vec<SegmentEdge<String>> {- self.reversed_adjacency.get_with_label(node, limit)+ pub fn ingoing_edges_with_label<'a>(+ &'a self,+ node: &'a NodeID,+ ) -> Box<dyn Iterator<Item = SegmentEdge<String>> + Send + Sync + 'a> {+ self.reversed_adjacency.get_with_label(node)}- pub fn ingoing_edges(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {- self.reversed_adjacency.get_without_label(node, limit)+ pub fn ingoing_edges<'a>(+ &'a self,+ node: &'a NodeID,+ ) -> Box<dyn Iterator<Item = SegmentEdge<()>> + Send + Sync + 'a> {+ self.reversed_adjacency.get_without_label(node)}pub fn in_degree(&self, node: &NodeID) -> u64 {@@ -286,7 +290,7 @@ mod test {}let segment = writer.finalize();- let mut out: Vec<_> = segment.outgoing_edges(&a.id, &EdgeLimit::Unlimited);+ let mut out: Vec<_> = segment.outgoing_edges(&a.id).collect();out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));@@ -310,7 +314,7 @@ mod test {]);- let mut out: Vec<_> = segment.outgoing_edges(&b.id, &EdgeLimit::Unlimited);+ let mut out: Vec<_> = segment.outgoing_edges(&b.id).collect();out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));assert_eq!(out,@@ -323,7 +327,7 @@ mod test {.into(),]);- let mut out: Vec<_> = segment.outgoing_edges(&c.id, &EdgeLimit::Unlimited);+ let mut out: Vec<_> = segment.outgoing_edges(&c.id).collect();out.sort_by(|a, b| a.to.node().cmp(&b.to.node()));assert_eq!(out,@@ -336,7 +340,7 @@ mod test {.into(),]);- let out: Vec<_> = segment.ingoing_edges(&a.id, &EdgeLimit::Unlimited);+ let out: Vec<_> = segment.ingoing_edges(&a.id).collect();assert_eq!(out,vec![Edge {@@ -348,7 +352,7 @@ mod test {.into(),]);- let out: Vec<_> = segment.ingoing_edges(&b.id, &EdgeLimit::Unlimited);+ let out: Vec<_> = segment.ingoing_edges(&b.id).collect();assert_eq!(out,vec![Edge {@@ -360,7 +364,7 @@ mod test {.into(),]);- let mut out: Vec<_> = segment.ingoing_edges(&c.id, &EdgeLimit::Unlimited);+ let mut out: Vec<_> = segment.ingoing_edges(&c.id).collect();out.sort_by(|a, b| a.from.node().cmp(&b.from.node()));assert_eq!(out,diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex e594b600..5428f05f 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -34,7 +34,7 @@ use itertools::Itertools;use super::{merge::{MergeNode, MergeSegmentOrd},- Compression, EdgeLimit, FullNodeID, NodeDatum, NodeID, SegmentEdge, StoredEdge,+ Compression, FullNodeID, NodeDatum, NodeID, SegmentEdge, StoredEdge,};pub struct HostDb {@@ -498,7 +498,10 @@ impl EdgeStore {}}- pub fn get_with_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<String>> {+ pub fn get_with_label<'a>(+ &'a self,+ node: &'a NodeID,+ ) -> Box<dyn Iterator<Item = SegmentEdge<String>> + Send + Sync + 'a> {let node_bytes = node.as_u64().to_be_bytes();match (@@ -515,37 +518,34 @@ impl EdgeStore {.map(|r| r.decompress()).flat_map(|block| block.labels.into_iter());- let labels = limit.apply(labels);-let edges = self.edges.slice(usize_range(node_range.range));- let edges = limit.apply(edges);-- labels- .zip_eq(edges)- .map(|(label, edge)| {- if self.reversed {- SegmentEdge {- from: edge.other,- to: NodeDatum::new(*node, node_range.host_rank),- rel: edge.rel,- label,- }- } else {- SegmentEdge {- from: NodeDatum::new(*node, node_range.host_rank),- to: edge.other,- rel: edge.rel,- label,- }++ Box::new(labels.zip_eq(edges).map(move |(label, edge)| {+ if self.reversed {+ SegmentEdge {+ from: edge.other,+ to: NodeDatum::new(*node, node_range.host_rank),+ rel: edge.rel,+ label,}- })- .collect()+ } else {+ SegmentEdge {+ from: NodeDatum::new(*node, node_range.host_rank),+ to: edge.other,+ rel: edge.rel,+ label,+ }+ }+ }))}- _ => Vec::new(),+ _ => Box::new(std::iter::empty()),}}- pub fn get_without_label(&self, node: &NodeID, limit: &EdgeLimit) -> Vec<SegmentEdge<()>> {+ pub fn get_without_label<'a>(+ &'a self,+ node: &'a NodeID,+ ) -> Box<dyn Iterator<Item = SegmentEdge<()>> + Send + Sync + 'a> {let node_bytes = node.as_u64().to_be_bytes();match self.ranges.edges.get_raw(&node_bytes) {@@ -554,30 +554,25 @@ impl EdgeStore {let edges = self.edges.slice(usize_range(edge_range.range));- let edges = limit.apply(edges);-- edges- .into_iter()- .map(|edge| {- if self.reversed {- SegmentEdge {- from: edge.other,- to: NodeDatum::new(*node, edge_range.host_rank),- rel: edge.rel,- label: (),- }- } else {- SegmentEdge {- from: NodeDatum::new(*node, edge_range.host_rank),- to: edge.other,- rel: edge.rel,- label: (),- }+ Box::new(edges.into_iter().map(move |edge| {+ if self.reversed {+ SegmentEdge {+ from: edge.other,+ to: NodeDatum::new(*node, edge_range.host_rank),+ rel: edge.rel,+ label: (),}- })- .collect()+ } else {+ SegmentEdge {+ from: NodeDatum::new(*node, edge_range.host_rank),+ to: edge.other,+ rel: edge.rel,+ label: (),+ }+ }+ }))}- _ => Vec::new(),+ _ => Box::new(std::iter::empty()),}}@@ -649,12 +644,12 @@ mod tests {let store = kv.finalize();- let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64), &EdgeLimit::Unlimited);+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64)).collect();assert_eq!(edges.len(), 1);assert_eq!(&edges[0].to.node(), &SegmentEdge::from(e.clone()).to.node());- let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);+ let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64)).collect();assert_eq!(edges.len(), 0);@@ -686,43 +681,14 @@ mod tests {let store = kv.finalize();- let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64), &EdgeLimit::Unlimited);+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64)).collect();assert_eq!(edges.len(), 0);- let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Unlimited);+ let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64)).collect();assert_eq!(edges.len(), 1);assert_eq!(&edges[0], &SegmentEdge::from(e.clone()));}- #[test]- fn test_limit() {- let temp_dir = crate::gen_temp_dir().unwrap();- let mut kv: EdgeStoreWriter =- EdgeStoreWriter::new(&temp_dir, Compression::default(), true, None);-- for i in 0..10 {- let e = InsertableEdge {- from: FullNodeID {- id: NodeID::from(i as u64),- host: NodeID::from(0_u64),- },- to: FullNodeID {- id: NodeID::from(1_u64),- host: NodeID::from(0_u64),- },- label: "test".to_string(),- rel: RelFlags::default(),- };-- kv.put(e.clone());- }-- let store = kv.finalize();-- let edges: Vec<_> = store.get_with_label(&NodeID::from(1_u64), &EdgeLimit::Limit(5));- assert_eq!(edges.len(), 5);- }-#[test]fn test_edge_ordering() {let temp_dir = crate::gen_temp_dir().unwrap();@@ -787,7 +753,7 @@ mod tests {let store = kv.finalize();- let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64), &EdgeLimit::Unlimited);+ let edges: Vec<_> = store.get_with_label(&NodeID::from(0_u64)).collect();assert_eq!(edges.len(), 3);