Skip to content

web search done right

host group sketch query to compute the in/out degree somewhat accurately

Mikkel Denkerfcf2b9909393
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rs
index 74671b5c..46ba9929 100644
--- a/crates/core/src/entrypoint/webgraph_server.rs
+++ b/crates/core/src/entrypoint/webgraph_server.rs
@@ -37,6 +37,7 @@ use crate::webgraph::query::FullHostBacklinksQuery;
use crate::webgraph::query::FullHostForwardlinksQuery;
use crate::webgraph::query::HostBacklinksQuery;
use crate::webgraph::query::HostForwardlinksQuery;
+use crate::webgraph::query::HostGroupSketchQuery;
use crate::webgraph::query::Id2NodeQuery;
use crate::webgraph::NodeID;
use crate::webgraph::Webgraph;
@@ -144,6 +145,7 @@ impl_search!([
HostForwardlinksQuery,
FullForwardlinksQuery,
FullHostForwardlinksQuery,
+ HostGroupSketchQuery,
]);
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rs
new file mode 100644
index 00000000..54f05e7a
--- /dev/null
+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs
@@ -0,0 +1,117 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use anyhow::anyhow;
+use rustc_hash::FxHashMap;
+use tantivy::columnar::Column;
+
+use crate::{
+ hyperloglog::HyperLogLog,
+ webgraph::{
+ schema::{Field, FieldEnum},
+ warmed_column_fields::WarmedColumnFields,
+ },
+};
+
+use super::Collector;
+
+pub struct GroupSketchCollector {
+ group_field: FieldEnum,
+ value_field: FieldEnum,
+ warmed_column_fields: Option<WarmedColumnFields>,
+}
+
+impl GroupSketchCollector {
+ pub fn new<Group: Field, Value: Field>(group_field: Group, value_field: Value) -> Self {
+ Self {
+ group_field: group_field.into(),
+ value_field: value_field.into(),
+ warmed_column_fields: None,
+ }
+ }
+
+ pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {
+ self.warmed_column_fields = Some(warmed_column_fields);
+ self
+ }
+}
+
+impl Collector for GroupSketchCollector {
+ type Fruit = FxHashMap<u64, HyperLogLog<256>>;
+ type Child = GroupSketchSegmentCollector;
+
+ fn for_segment(
+ &self,
+ _: tantivy::SegmentOrdinal,
+ segment: &tantivy::SegmentReader,
+ ) -> crate::Result<Self::Child> {
+ let warmed_column_fields = self.warmed_column_fields.as_ref().ok_or(anyhow!(
+ "Warmed column fields must be set to construct segment collector"
+ ))?;
+
+ let group = warmed_column_fields
+ .segment(&segment.segment_id())
+ .u64(self.group_field)
+ .ok_or(anyhow!("Group field missing from index"))?;
+
+ let value = warmed_column_fields
+ .segment(&segment.segment_id())
+ .u64(self.value_field)
+ .ok_or(anyhow!("Value field missing from index"))?;
+
+ Ok(GroupSketchSegmentCollector {
+ group,
+ value,
+ groups: FxHashMap::default(),
+ })
+ }
+
+ fn merge_fruits(
+ &self,
+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,
+ ) -> crate::Result<Self::Fruit> {
+ let mut groups: FxHashMap<u64, HyperLogLog<256>> = FxHashMap::default();
+
+ for fruit in segment_fruits {
+ for (group, hll) in fruit {
+ groups.entry(group).or_default().merge(&hll);
+ }
+ }
+
+ Ok(groups)
+ }
+}
+
+pub struct GroupSketchSegmentCollector {
+ group: Column<u64>,
+ value: Column<u64>,
+ groups: FxHashMap<u64, HyperLogLog<256>>,
+}
+
+impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {
+ type Fruit = FxHashMap<u64, HyperLogLog<256>>;
+
+ fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {
+ let group = self.group.first(doc).unwrap();
+ let value = self.value.first(doc).unwrap();
+
+ self.groups.entry(group).or_default().add(value);
+ }
+
+ fn harvest(self) -> Self::Fruit {
+ self.groups
+ }
+}
diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rs
index 0b35246d..eb65ce5a 100644
--- a/crates/core/src/webgraph/query/collector/mod.rs
+++ b/crates/core/src/webgraph/query/collector/mod.rs
@@ -16,10 +16,12 @@
mod fast_count;
mod first_doc;
+mod group_sketch;
mod top_docs;
pub use fast_count::{FastCountCollector, FastCountValue};
pub use first_doc::FirstDocCollector;
+pub use group_sketch::GroupSketchCollector;
pub use top_docs::{HostDeduplicator, TopDocsCollector};
use tantivy::{
diff --git a/crates/core/src/webgraph/query/group_sketch.rs b/crates/core/src/webgraph/query/group_sketch.rs
new file mode 100644
index 00000000..bc25de71
--- /dev/null
+++ b/crates/core/src/webgraph/query/group_sketch.rs
@@ -0,0 +1,138 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use rustc_hash::FxHashMap;
+
+use crate::{
+ hyperloglog::HyperLogLog,
+ webgraph::{
+ schema::{Field, FieldEnum, FromId, ToId},
+ NodeID,
+ },
+};
+
+use super::{collector::GroupSketchCollector, raw, Query};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub enum LinksDirection {
+ From(NodeID),
+ To(NodeID),
+}
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct HostGroupSketchQuery {
+ node: LinksDirection,
+ group: FieldEnum,
+ value: FieldEnum,
+}
+
+impl HostGroupSketchQuery {
+ pub fn new<Group: Field, Value: Field>(
+ node: LinksDirection,
+ group: Group,
+ value: Value,
+ ) -> Self {
+ Self {
+ node,
+ group: group.into(),
+ value: value.into(),
+ }
+ }
+
+ pub fn backlinks<Group: Field, Value: Field>(node: NodeID, group: Group, value: Value) -> Self {
+ Self::new(LinksDirection::To(node), group, value)
+ }
+
+ pub fn forwardlinks<Group: Field, Value: Field>(
+ node: NodeID,
+ group: Group,
+ value: Value,
+ ) -> Self {
+ Self::new(LinksDirection::From(node), group, value)
+ }
+}
+
+impl Query for HostGroupSketchQuery {
+ type Collector = GroupSketchCollector;
+ type TantivyQuery = raw::HostLinksQuery;
+ type IntermediateOutput = FxHashMap<u64, HyperLogLog<256>>;
+ type Output = FxHashMap<u64, HyperLogLog<256>>;
+
+ fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
+ match self.node {
+ LinksDirection::From(node) => raw::HostLinksQuery::new(
+ node,
+ FromId,
+ ToId,
+ searcher.warmed_column_fields().clone(),
+ ),
+ LinksDirection::To(node) => raw::HostLinksQuery::new(
+ node,
+ ToId,
+ FromId,
+ searcher.warmed_column_fields().clone(),
+ ),
+ }
+ }
+
+ fn collector(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::Collector {
+ GroupSketchCollector::new(self.group, self.value)
+ .with_column_fields(searcher.warmed_column_fields().clone())
+ }
+
+ fn remote_collector(&self) -> Self::Collector {
+ GroupSketchCollector::new(self.group, self.value)
+ }
+
+ fn filter_fruit_shards(
+ &self,
+ _: crate::ampc::dht::ShardId,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> <Self::Collector as super::Collector>::Fruit {
+ fruit
+ }
+
+ fn retrieve(
+ &self,
+ _: &crate::webgraph::searcher::Searcher,
+ fruit: <Self::Collector as super::Collector>::Fruit,
+ ) -> crate::Result<Self::IntermediateOutput> {
+ Ok(fruit)
+ }
+
+ fn merge_results(mut results: Vec<Self::IntermediateOutput>) -> Self::Output {
+ results.pop().unwrap_or_default()
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::webgraph::{tests::test_graph, Node};
+
+ use super::*;
+
+ #[test]
+ fn test_group_sketch_query() {
+ let (graph, _temp_dir) = test_graph();
+
+ let id = Node::from("C").into_host().id();
+ let query = HostGroupSketchQuery::backlinks(id, ToId, FromId);
+ let result = graph.search(&query).unwrap();
+
+ assert_eq!(result.len(), 1);
+ assert_eq!(result.get(&id.as_u64()).unwrap().size(), 3);
+ }
+}
diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rs
index 3de5afe9..1f680660 100644
--- a/crates/core/src/webgraph/query/mod.rs
+++ b/crates/core/src/webgraph/query/mod.rs
@@ -27,6 +27,9 @@ pub use forwardlink::*;
pub mod id2node;
pub use id2node::*;
+pub mod group_sketch;
+pub use group_sketch::*;
+
pub mod collector;
mod degree;
mod document_scorer;