Skip to content

web search done right

[webgraph] exact group_by query

Mikkel Denker16cf814f96f8
diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rs
index 56ac5b1b..09c0291a 100644
--- a/crates/core/src/webgraph/mod.rs
+++ b/crates/core/src/webgraph/mod.rs
@@ -40,7 +40,7 @@ mod document;
mod node;
pub mod query;
pub mod remote;
-mod schema;
+pub mod schema;
mod searcher;
mod shortest_path;
mod store;
diff --git a/crates/core/src/webgraph/query/collector/group_exact.rs b/crates/core/src/webgraph/query/collector/group_exact.rs
new file mode 100644
index 00000000..5418d6b1
--- /dev/null
+++ b/crates/core/src/webgraph/query/collector/group_exact.rs
@@ -0,0 +1,114 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::anyhow;
+use rustc_hash::{FxHashMap, FxHashSet};
+use tantivy::columnar::Column;
+
+use crate::webgraph::{
+ schema::{Field, FieldEnum},
+ warmed_column_fields::WarmedColumnFields,
+};
+
+use super::Collector;
+
+pub struct GroupExactCollector {
+ group_field: FieldEnum,
+ value_field: FieldEnum,
+ warmed_column_fields: Option<WarmedColumnFields>,
+}
+
+impl GroupExactCollector {
+ pub fn new<Group: Field, Value: Field>(group_field: Group, value_field: Value) -> Self {
+ Self {
+ group_field: group_field.into(),
+ value_field: value_field.into(),
+ warmed_column_fields: None,
+ }
+ }
+
+ pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {
+ self.warmed_column_fields = Some(warmed_column_fields);
+ self
+ }
+}
+
+impl Collector for GroupExactCollector {
+ type Fruit = FxHashMap<u64, FxHashSet<u64>>;
+ type Child = GroupExactSegmentCollector;
+
+ fn for_segment(
+ &self,
+ _: tantivy::SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ let warmed_column_fields = self.warmed_column_fields.as_ref().ok_or(anyhow!(
+ "Warmed column fields must be set to construct segment collector"
+ ))?;
+
+ let group = warmed_column_fields
+ .segment(&segment.segment_id())
+ .u64(self.group_field)
+ .ok_or(anyhow!("Group field missing from index"))?;
+
+ let value = warmed_column_fields
+ .segment(&segment.segment_id())
+ .u64(self.value_field)
+ .ok_or(anyhow!("Value field missing from index"))?;
+
+ Ok(GroupExactSegmentCollector {
+ group,
+ value,
+ groups: FxHashMap::default(),
+ })
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ let mut groups: FxHashMap<u64, FxHashSet<u64>> = FxHashMap::default();
+
+ for fruit in segment_fruits {
+ for (group, set) in fruit {
+ groups.entry(group).or_default().extend(set);
+ }
+ }
+
+ Ok(groups)
+ }
+}
+
+pub struct GroupExactSegmentCollector {
+ group: Column<u64>,
+ value: Column<u64>,
+ groups: FxHashMap<u64, FxHashSet<u64>>,
+}
+
+impl tantivy::collector::SegmentCollector for GroupExactSegmentCollector {
+ type Fruit = FxHashMap<u64, FxHashSet<u64>>;
+
+ fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {
+ let group = self.group.first(doc).unwrap();
+ let value = self.value.first(doc).unwrap();
+
+ self.groups.entry(group).or_default().insert(value);
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.groups
+ }
+}
diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rs
index 54f05e7a..0f394bfc 100644
--- a/crates/core/src/webgraph/query/collector/group_sketch.rs
+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs
@@ -50,7 +50,7 @@ impl GroupSketchCollector {
}
impl Collector for GroupSketchCollector {
- type Fruit = FxHashMap<u64, HyperLogLog<256>>;
+ type Fruit = FxHashMap<u64, HyperLogLog<4069>>;
type Child = GroupSketchSegmentCollector;
fn for_segment(
@@ -83,7 +83,7 @@ impl Collector for GroupSketchCollector {
&self,
segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
) -> crate::Result<Self::Fruit> {
- let mut groups: FxHashMap<u64, HyperLogLog<256>> = FxHashMap::default();
+ let mut groups: FxHashMap<u64, HyperLogLog<4069>> = FxHashMap::default();
for fruit in segment_fruits {
for (group, hll) in fruit {
@@ -98,11 +98,11 @@ impl Collector for GroupSketchCollector {
pub struct GroupSketchSegmentCollector {
group: Column<u64>,
value: Column<u64>,
- groups: FxHashMap<u64, HyperLogLog<256>>,
+ groups: FxHashMap<u64, HyperLogLog<4069>>,
}
impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {
- type Fruit = FxHashMap<u64, HyperLogLog<256>>;
+ type Fruit = FxHashMap<u64, HyperLogLog<4069>>;
fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {
let group = self.group.first(doc).unwrap();
diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rs
index eb65ce5a..de4a2433 100644
--- a/crates/core/src/webgraph/query/collector/mod.rs
+++ b/crates/core/src/webgraph/query/collector/mod.rs
@@ -16,11 +16,13 @@
mod fast_count;
mod first_doc;
+mod group_exact;
mod group_sketch;
mod top_docs;
pub use fast_count::{FastCountCollector, FastCountValue};
pub use first_doc::FirstDocCollector;
+pub use group_exact::GroupExactCollector;
pub use group_sketch::GroupSketchCollector;
pub use top_docs::{HostDeduplicator, TopDocsCollector};
diff --git a/crates/core/src/webgraph/query/group_sketch.rs b/crates/core/src/webgraph/query/group_by.rs
similarity index 55%
rename from crates/core/src/webgraph/query/group_sketch.rs
rename to crates/core/src/webgraph/query/group_by.rs
index bc25de71..b93210ea 100644
--- a/crates/core/src/webgraph/query/group_sketch.rs
+++ b/crates/core/src/webgraph/query/group_by.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use rustc_hash::FxHashMap;
+use rustc_hash::{FxHashMap, FxHashSet};
use crate::{
hyperloglog::HyperLogLog,
@@ -24,7 +24,10 @@ use crate::{
},
};
-use super::{collector::GroupSketchCollector, raw, Query};
+use super::{
+ collector::{GroupExactCollector, GroupSketchCollector},
+ raw, Query,
+};
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub enum LinksDirection {
@@ -68,8 +71,8 @@ impl HostGroupSketchQuery {
impl Query for HostGroupSketchQuery {
type Collector = GroupSketchCollector;
type TantivyQuery = raw::HostLinksQuery;
- type IntermediateOutput = FxHashMap<u64, HyperLogLog<256>>;
- type Output = FxHashMap<u64, HyperLogLog<256>>;
+ type IntermediateOutput = FxHashMap<u64, HyperLogLog<4069>>;
+ type Output = FxHashMap<u64, HyperLogLog<4069>>;
fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
match self.node {
@@ -118,6 +121,92 @@ impl Query for HostGroupSketchQuery {
}
}
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct HostGroupQuery {
+ node: LinksDirection,
+ group: FieldEnum,
+ value: FieldEnum,
+}
+
+impl HostGroupQuery {
+ pub fn new<Group: Field, Value: Field>(
+ node: LinksDirection,
+ group: Group,
+ value: Value,
+ ) -> Self {
+ Self {
+ node,
+ group: group.into(),
+ value: value.into(),
+ }
+ }
+
+ pub fn backlinks<Group: Field, Value: Field>(node: NodeID, group: Group, value: Value) -> Self {
+ Self::new(LinksDirection::To(node), group, value)
+ }
+
+ pub fn forwardlinks<Group: Field, Value: Field>(
+ node: NodeID,
+ group: Group,
+ value: Value,
+ ) -> Self {
+ Self::new(LinksDirection::From(node), group, value)
+ }
+}
+
+impl Query for HostGroupQuery {
+ type Collector = GroupExactCollector;
+ type TantivyQuery = raw::HostLinksQuery;
+ type IntermediateOutput = FxHashMap<u64, FxHashSet<u64>>;
+ type Output = FxHashMap<u64, FxHashSet<u64>>;
+
+ fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
+ match self.node {
+ LinksDirection::From(node) => raw::HostLinksQuery::new(
+ node,
+ FromId,
+ ToId,
+ searcher.warmed_column_fields().clone(),
+ ),
+ LinksDirection::To(node) => raw::HostLinksQuery::new(
+ node,
+ ToId,
+ FromId,
+ searcher.warmed_column_fields().clone(),
+ ),
+ }
+ }
+
+ fn collector(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::Collector {
+ GroupExactCollector::new(self.group, self.value)
+ .with_column_fields(searcher.warmed_column_fields().clone())
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ GroupExactCollector::new(self.group, self.value)
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ _: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ }
+
+ fn retrieve(
+ &self,
+ _: &crate::webgraph::searcher::Searcher,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ Ok(fruit)
+ }
+
+ fn merge_results(mut results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ results.pop().unwrap_or_default()
+ }
+}
+
#[cfg(test)]
mod tests {
use crate::webgraph::{tests::test_graph, Node};
@@ -135,4 +224,16 @@ mod tests {
assert_eq!(result.len(), 1);
assert_eq!(result.get(&id.as_u64()).unwrap().size(), 3);
}
+
+ #[test]
+ fn test_group_exact_query() {
+ let (graph, _temp_dir) = test_graph();
+
+ let id = Node::from("C").into_host().id();
+ let query = HostGroupQuery::backlinks(id, ToId, FromId);
+ let result = graph.search(&query).unwrap();
+
+ assert_eq!(result.len(), 1);
+ assert_eq!(result.get(&id.as_u64()).unwrap().len(), 3);
+ }
}
diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rs
index 1f680660..bc294125 100644
--- a/crates/core/src/webgraph/query/mod.rs
+++ b/crates/core/src/webgraph/query/mod.rs
@@ -27,8 +27,8 @@ pub use forwardlink::*;
pub mod id2node;
pub use id2node::*;
-pub mod group_sketch;
-pub use group_sketch::*;
+pub mod group_by;
+pub use group_by::*;
pub mod collector;
mod degree;