web search done right
[webgraph] exact group_by query
Mikkel Denker16cf814f96f8
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 56ac5b1b..09c0291a 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -40,7 +40,7 @@ mod document;mod node;pub mod query;pub mod remote;-mod schema;+pub mod schema;mod searcher;mod shortest_path;mod store;diff --git a/crates/core/src/webgraph/query/collector/group_exact.rs b/crates/core/src/webgraph/query/collector/group_exact.rsnew file mode 100644index 00000000..5418d6b1--- /dev/null+++ b/crates/core/src/webgraph/query/collector/group_exact.rs@@ -0,0 +1,114 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use anyhow::anyhow;+use rustc_hash::{FxHashMap, FxHashSet};+use tantivy::columnar::Column;++use crate::webgraph::{+ schema::{Field, FieldEnum},+ warmed_column_fields::WarmedColumnFields,+};++use super::Collector;++pub struct GroupExactCollector {+ group_field: FieldEnum,+ value_field: FieldEnum,+ warmed_column_fields: Option<WarmedColumnFields>,+}++impl GroupExactCollector {+ pub fn new<Group: Field, Value: Field>(group_field: Group, value_field: Value) -> Self {+ Self {+ group_field: group_field.into(),+ value_field: value_field.into(),+ warmed_column_fields: None,+ }+ }++ pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {+ self.warmed_column_fields = Some(warmed_column_fields);+ self+ }+}++impl Collector for GroupExactCollector {+ type Fruit = FxHashMap<u64, FxHashSet<u64>>;+ type Child = GroupExactSegmentCollector;++ fn for_segment(+ &self,+ _: tantivy::SegmentOrdinal,+ segment: &tantivy::SegmentReader,+ ) -> crate::Result<Self::Child> {+ let warmed_column_fields = self.warmed_column_fields.as_ref().ok_or(anyhow!(+ "Warmed column fields must be set to construct segment collector"+ ))?;++ let group = warmed_column_fields+ .segment(&segment.segment_id())+ .u64(self.group_field)+ .ok_or(anyhow!("Group field missing from index"))?;++ let value = warmed_column_fields+ .segment(&segment.segment_id())+ .u64(self.value_field)+ .ok_or(anyhow!("Value field missing from index"))?;++ Ok(GroupExactSegmentCollector {+ group,+ value,+ groups: FxHashMap::default(),+ })+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> crate::Result<Self::Fruit> {+ let mut groups: FxHashMap<u64, FxHashSet<u64>> = FxHashMap::default();++ for fruit in segment_fruits {+ for (group, set) in fruit {+ groups.entry(group).or_default().extend(set);+ }+ }++ Ok(groups)+ }+}++pub struct GroupExactSegmentCollector {+ group: Column<u64>,+ value: Column<u64>,+ groups: FxHashMap<u64, FxHashSet<u64>>,+}++impl tantivy::collector::SegmentCollector for GroupExactSegmentCollector {+ type Fruit = FxHashMap<u64, FxHashSet<u64>>;++ fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {+ let group = self.group.first(doc).unwrap();+ let value = self.value.first(doc).unwrap();++ self.groups.entry(group).or_default().insert(value);+ }++ fn harvest(self) -> Self::Fruit {+ self.groups+ }+}diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rsindex 54f05e7a..0f394bfc 100644--- a/crates/core/src/webgraph/query/collector/group_sketch.rs+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs@@ -50,7 +50,7 @@ impl GroupSketchCollector {}impl Collector for GroupSketchCollector {- type Fruit = FxHashMap<u64, HyperLogLog<256>>;+ type Fruit = FxHashMap<u64, HyperLogLog<4069>>;type Child = GroupSketchSegmentCollector;fn for_segment(@@ -83,7 +83,7 @@ impl Collector for GroupSketchCollector {&self,segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,) -> crate::Result<Self::Fruit> {- let mut groups: FxHashMap<u64, HyperLogLog<256>> = FxHashMap::default();+ let mut groups: FxHashMap<u64, HyperLogLog<4069>> = FxHashMap::default();for fruit in segment_fruits {for (group, hll) in fruit {@@ -98,11 +98,11 @@ impl Collector for GroupSketchCollector {pub struct GroupSketchSegmentCollector {group: Column<u64>,value: Column<u64>,- groups: FxHashMap<u64, HyperLogLog<256>>,+ groups: FxHashMap<u64, HyperLogLog<4069>>,}impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {- type Fruit = FxHashMap<u64, HyperLogLog<256>>;+ type Fruit = FxHashMap<u64, HyperLogLog<4069>>;fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {let group = self.group.first(doc).unwrap();diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rsindex eb65ce5a..de4a2433 100644--- a/crates/core/src/webgraph/query/collector/mod.rs+++ b/crates/core/src/webgraph/query/collector/mod.rs@@ -16,11 +16,13 @@mod fast_count;mod first_doc;+mod group_exact;mod group_sketch;mod top_docs;pub use fast_count::{FastCountCollector, FastCountValue};pub use first_doc::FirstDocCollector;+pub use group_exact::GroupExactCollector;pub use group_sketch::GroupSketchCollector;pub use top_docs::{HostDeduplicator, TopDocsCollector};diff --git a/crates/core/src/webgraph/query/group_sketch.rs b/crates/core/src/webgraph/query/group_by.rssimilarity index 55%rename from crates/core/src/webgraph/query/group_sketch.rsrename to crates/core/src/webgraph/query/group_by.rsindex bc25de71..b93210ea 100644--- a/crates/core/src/webgraph/query/group_sketch.rs+++ b/crates/core/src/webgraph/query/group_by.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use rustc_hash::FxHashMap;+use rustc_hash::{FxHashMap, FxHashSet};use crate::{hyperloglog::HyperLogLog,@@ -24,7 +24,10 @@ use crate::{},};-use super::{collector::GroupSketchCollector, raw, Query};+use super::{+ collector::{GroupExactCollector, GroupSketchCollector},+ raw, Query,+};#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub enum LinksDirection {@@ -68,8 +71,8 @@ impl HostGroupSketchQuery {impl Query for HostGroupSketchQuery {type Collector = GroupSketchCollector;type TantivyQuery = raw::HostLinksQuery;- type IntermediateOutput = FxHashMap<u64, HyperLogLog<256>>;- type Output = FxHashMap<u64, HyperLogLog<256>>;+ type IntermediateOutput = FxHashMap<u64, HyperLogLog<4069>>;+ type Output = FxHashMap<u64, HyperLogLog<4069>>;fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {match self.node {@@ -118,6 +121,92 @@ impl Query for HostGroupSketchQuery {}}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct HostGroupQuery {+ node: LinksDirection,+ group: FieldEnum,+ value: FieldEnum,+}++impl HostGroupQuery {+ pub fn new<Group: Field, Value: Field>(+ node: LinksDirection,+ group: Group,+ value: Value,+ ) -> Self {+ Self {+ node,+ group: group.into(),+ value: value.into(),+ }+ }++ pub fn backlinks<Group: Field, Value: Field>(node: NodeID, group: Group, value: Value) -> Self {+ Self::new(LinksDirection::To(node), group, value)+ }++ pub fn forwardlinks<Group: Field, Value: Field>(+ node: NodeID,+ group: Group,+ value: Value,+ ) -> Self {+ Self::new(LinksDirection::From(node), group, value)+ }+}++impl Query for HostGroupQuery {+ type Collector = GroupExactCollector;+ type TantivyQuery = raw::HostLinksQuery;+ type IntermediateOutput = FxHashMap<u64, FxHashSet<u64>>;+ type Output = FxHashMap<u64, FxHashSet<u64>>;++ fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {+ match self.node {+ LinksDirection::From(node) => raw::HostLinksQuery::new(+ node,+ FromId,+ ToId,+ searcher.warmed_column_fields().clone(),+ ),+ LinksDirection::To(node) => raw::HostLinksQuery::new(+ node,+ ToId,+ FromId,+ searcher.warmed_column_fields().clone(),+ ),+ }+ }++ fn collector(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::Collector {+ GroupExactCollector::new(self.group, self.value)+ .with_column_fields(searcher.warmed_column_fields().clone())+ }++ fn remote_collector(&self) -> Self::Collector {+ GroupExactCollector::new(self.group, self.value)+ }++ fn filter_fruit_shards(+ &self,+ _: crate::ampc::dht::ShardId,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> <Self::Collector as super::Collector>::Fruit {+ fruit+ }++ fn retrieve(+ &self,+ _: &crate::webgraph::searcher::Searcher,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> crate::Result<Self::IntermediateOutput> {+ Ok(fruit)+ }++ fn merge_results(mut results: Vec<Self::IntermediateOutput>) -> Self::Output {+ results.pop().unwrap_or_default()+ }+}+#[cfg(test)]mod tests {use crate::webgraph::{tests::test_graph, Node};@@ -135,4 +224,16 @@ mod tests {assert_eq!(result.len(), 1);assert_eq!(result.get(&id.as_u64()).unwrap().size(), 3);}++ #[test]+ fn test_group_exact_query() {+ let (graph, _temp_dir) = test_graph();++ let id = Node::from("C").into_host().id();+ let query = HostGroupQuery::backlinks(id, ToId, FromId);+ let result = graph.search(&query).unwrap();++ assert_eq!(result.len(), 1);+ assert_eq!(result.get(&id.as_u64()).unwrap().len(), 3);+ }}diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rsindex 1f680660..bc294125 100644--- a/crates/core/src/webgraph/query/mod.rs+++ b/crates/core/src/webgraph/query/mod.rs@@ -27,8 +27,8 @@ pub use forwardlink::*;pub mod id2node;pub use id2node::*;-pub mod group_sketch;-pub use group_sketch::*;+pub mod group_by;+pub use group_by::*;pub mod collector;mod degree;