web search done right
host group sketch query to compute the in/out degree somewhat accurately
Mikkel Denkerfcf2b9909393
diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex 74671b5c..46ba9929 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -37,6 +37,7 @@ use crate::webgraph::query::FullHostBacklinksQuery;use crate::webgraph::query::FullHostForwardlinksQuery;use crate::webgraph::query::HostBacklinksQuery;use crate::webgraph::query::HostForwardlinksQuery;+use crate::webgraph::query::HostGroupSketchQuery;use crate::webgraph::query::Id2NodeQuery;use crate::webgraph::NodeID;use crate::webgraph::Webgraph;@@ -144,6 +145,7 @@ impl_search!([HostForwardlinksQuery,FullForwardlinksQuery,FullHostForwardlinksQuery,+ HostGroupSketchQuery,]);#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rsnew file mode 100644index 00000000..54f05e7a--- /dev/null+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs@@ -0,0 +1,117 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use anyhow::anyhow;+use rustc_hash::FxHashMap;+use tantivy::columnar::Column;++use crate::{+ hyperloglog::HyperLogLog,+ webgraph::{+ schema::{Field, FieldEnum},+ warmed_column_fields::WarmedColumnFields,+ },+};++use super::Collector;++pub struct GroupSketchCollector {+ group_field: FieldEnum,+ value_field: FieldEnum,+ warmed_column_fields: Option<WarmedColumnFields>,+}++impl GroupSketchCollector {+ pub fn new<Group: Field, Value: Field>(group_field: Group, value_field: Value) -> Self {+ Self {+ group_field: group_field.into(),+ value_field: value_field.into(),+ warmed_column_fields: None,+ }+ }++ pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {+ self.warmed_column_fields = Some(warmed_column_fields);+ self+ }+}++impl Collector for GroupSketchCollector {+ type Fruit = FxHashMap<u64, HyperLogLog<256>>;+ type Child = GroupSketchSegmentCollector;++ fn for_segment(+ &self,+ _: tantivy::SegmentOrdinal,+ segment: &tantivy::SegmentReader,+ ) -> crate::Result<Self::Child> {+ let warmed_column_fields = self.warmed_column_fields.as_ref().ok_or(anyhow!(+ "Warmed column fields must be set to construct segment collector"+ ))?;++ let group = warmed_column_fields+ .segment(&segment.segment_id())+ .u64(self.group_field)+ .ok_or(anyhow!("Group field missing from index"))?;++ let value = warmed_column_fields+ .segment(&segment.segment_id())+ .u64(self.value_field)+ .ok_or(anyhow!("Value field missing from index"))?;++ Ok(GroupSketchSegmentCollector {+ group,+ value,+ groups: FxHashMap::default(),+ })+ }++ fn merge_fruits(+ &self,+ segment_fruits: Vec<<Self::Child as tantivy::collector::SegmentCollector>::Fruit>,+ ) -> crate::Result<Self::Fruit> {+ let mut groups: FxHashMap<u64, HyperLogLog<256>> = FxHashMap::default();++ for fruit in segment_fruits {+ for (group, hll) in fruit {+ groups.entry(group).or_default().merge(&hll);+ }+ }++ Ok(groups)+ }+}++pub struct GroupSketchSegmentCollector {+ group: Column<u64>,+ value: Column<u64>,+ groups: FxHashMap<u64, HyperLogLog<256>>,+}++impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {+ type Fruit = FxHashMap<u64, HyperLogLog<256>>;++ fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {+ let group = self.group.first(doc).unwrap();+ let value = self.value.first(doc).unwrap();++ self.groups.entry(group).or_default().add(value);+ }++ fn harvest(self) -> Self::Fruit {+ self.groups+ }+}diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rsindex 0b35246d..eb65ce5a 100644--- a/crates/core/src/webgraph/query/collector/mod.rs+++ b/crates/core/src/webgraph/query/collector/mod.rs@@ -16,10 +16,12 @@mod fast_count;mod first_doc;+mod group_sketch;mod top_docs;pub use fast_count::{FastCountCollector, FastCountValue};pub use first_doc::FirstDocCollector;+pub use group_sketch::GroupSketchCollector;pub use top_docs::{HostDeduplicator, TopDocsCollector};use tantivy::{diff --git a/crates/core/src/webgraph/query/group_sketch.rs b/crates/core/src/webgraph/query/group_sketch.rsnew file mode 100644index 00000000..bc25de71--- /dev/null+++ b/crates/core/src/webgraph/query/group_sketch.rs@@ -0,0 +1,138 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use rustc_hash::FxHashMap;++use crate::{+ hyperloglog::HyperLogLog,+ webgraph::{+ schema::{Field, FieldEnum, FromId, ToId},+ NodeID,+ },+};++use super::{collector::GroupSketchCollector, raw, Query};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub enum LinksDirection {+ From(NodeID),+ To(NodeID),+}++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct HostGroupSketchQuery {+ node: LinksDirection,+ group: FieldEnum,+ value: FieldEnum,+}++impl HostGroupSketchQuery {+ pub fn new<Group: Field, Value: Field>(+ node: LinksDirection,+ group: Group,+ value: Value,+ ) -> Self {+ Self {+ node,+ group: group.into(),+ value: value.into(),+ }+ }++ pub fn backlinks<Group: Field, Value: Field>(node: NodeID, group: Group, value: Value) -> Self {+ Self::new(LinksDirection::To(node), group, value)+ }++ pub fn forwardlinks<Group: Field, Value: Field>(+ node: NodeID,+ group: Group,+ value: Value,+ ) -> Self {+ Self::new(LinksDirection::From(node), group, value)+ }+}++impl Query for HostGroupSketchQuery {+ type Collector = GroupSketchCollector;+ type TantivyQuery = raw::HostLinksQuery;+ type IntermediateOutput = FxHashMap<u64, HyperLogLog<256>>;+ type Output = FxHashMap<u64, HyperLogLog<256>>;++ fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {+ match self.node {+ LinksDirection::From(node) => raw::HostLinksQuery::new(+ node,+ FromId,+ ToId,+ searcher.warmed_column_fields().clone(),+ ),+ LinksDirection::To(node) => raw::HostLinksQuery::new(+ node,+ ToId,+ FromId,+ searcher.warmed_column_fields().clone(),+ ),+ }+ }++ fn collector(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::Collector {+ GroupSketchCollector::new(self.group, self.value)+ .with_column_fields(searcher.warmed_column_fields().clone())+ }++ fn remote_collector(&self) -> Self::Collector {+ GroupSketchCollector::new(self.group, self.value)+ }++ fn filter_fruit_shards(+ &self,+ _: crate::ampc::dht::ShardId,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> <Self::Collector as super::Collector>::Fruit {+ fruit+ }++ fn retrieve(+ &self,+ _: &crate::webgraph::searcher::Searcher,+ fruit: <Self::Collector as super::Collector>::Fruit,+ ) -> crate::Result<Self::IntermediateOutput> {+ Ok(fruit)+ }++ fn merge_results(mut results: Vec<Self::IntermediateOutput>) -> Self::Output {+ results.pop().unwrap_or_default()+ }+}++#[cfg(test)]+mod tests {+ use crate::webgraph::{tests::test_graph, Node};++ use super::*;++ #[test]+ fn test_group_sketch_query() {+ let (graph, _temp_dir) = test_graph();++ let id = Node::from("C").into_host().id();+ let query = HostGroupSketchQuery::backlinks(id, ToId, FromId);+ let result = graph.search(&query).unwrap();++ assert_eq!(result.len(), 1);+ assert_eq!(result.get(&id.as_u64()).unwrap().size(), 3);+ }+}diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rsindex 3de5afe9..1f680660 100644--- a/crates/core/src/webgraph/query/mod.rs+++ b/crates/core/src/webgraph/query/mod.rs@@ -27,6 +27,9 @@ pub use forwardlink::*;pub mod id2node;pub use id2node::*;+pub mod group_sketch;+pub use group_sketch::*;+pub mod collector;mod degree;mod document_scorer;