web search done right
[webgraph] rel flags filter (nofollow etc)
Mikkel Denkerfb4aec54245b
diff --git a/crates/core/src/webgraph/query/collector/first_doc.rs b/crates/core/src/webgraph/query/collector/first_doc.rsindex 0f4a0a08..42bca480 100644--- a/crates/core/src/webgraph/query/collector/first_doc.rs+++ b/crates/core/src/webgraph/query/collector/first_doc.rs@@ -17,21 +17,54 @@use tantivy::{collector::SegmentCollector, DocId, SegmentOrdinal};use super::Collector;-use crate::{ampc::dht::ShardId, webgraph::doc_address::DocAddress};+use crate::{+ ampc::dht::ShardId,+ webgraph::{+ doc_address::DocAddress,+ query::{ColumnFieldFilter, SegmentColumnFieldFilter},+ warmed_column_fields::WarmedColumnFields,+ },+};pub struct FirstDocCollector {shard_id: Option<ShardId>,+ column_fields: Option<WarmedColumnFields>,+ filter: Option<Box<dyn ColumnFieldFilter>>,}impl FirstDocCollector {+ #[must_use]pub fn with_shard_id(shard_id: ShardId) -> Self {Self {shard_id: Some(shard_id),+ filter: None,+ column_fields: None,}}+ #[must_use]pub fn without_shard_id() -> Self {- Self { shard_id: None }+ Self {+ shard_id: None,+ filter: None,+ column_fields: None,+ }+ }++ #[must_use]+ pub fn with_column_fields(self, warmed_column_fields: WarmedColumnFields) -> Self {+ Self {+ column_fields: Some(warmed_column_fields),+ ..self+ }+ }++ #[must_use]+ pub fn with_filter(self, filter: Box<dyn ColumnFieldFilter>) -> Self {+ Self {+ filter: Some(filter),+ ..self+ }}}@@ -42,10 +75,19 @@ impl Collector for FirstDocCollector {fn for_segment(&self,segment_id: SegmentOrdinal,- _: &tantivy::SegmentReader,+ segment: &tantivy::SegmentReader,) -> crate::Result<Self::Child> {+ let column_fields = self+ .column_fields+ .as_ref()+ .expect("Column fields are required to perform search")+ .segment(&segment.segment_id());++ let filter = self.filter.as_ref().map(|f| f.for_segment(column_fields));+Ok(FirstDocSegmentCollector::new(segment_id,+ filter,self.shard_id.unwrap(),))}@@ -61,14 +103,20 @@ impl Collector for FirstDocCollector {pub struct FirstDocSegmentCollector {first_doc: Option<DocAddress>,segment_id: SegmentOrdinal,+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,shard_id: ShardId,}impl FirstDocSegmentCollector {- pub fn new(segment_id: SegmentOrdinal, shard_id: ShardId) -> Self {+ pub fn new(+ segment_id: SegmentOrdinal,+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,+ shard_id: ShardId,+ ) -> Self {Self {first_doc: None,segment_id,+ filter,shard_id,}}@@ -78,6 +126,12 @@ impl SegmentCollector for FirstDocSegmentCollector {type Fruit = Option<DocAddress>;fn collect(&mut self, doc: DocId, _: tantivy::Score) {+ if let Some(filter) = &self.filter {+ if filter.should_skip(doc) {+ return;+ }+ }+if self.first_doc.is_none() {self.first_doc = Some(DocAddress::new(self.shard_id, self.segment_id, doc));}diff --git a/crates/core/src/webgraph/query/collector/group_exact.rs b/crates/core/src/webgraph/query/collector/group_exact.rsindex 5418d6b1..9a5fcd2f 100644--- a/crates/core/src/webgraph/query/collector/group_exact.rs+++ b/crates/core/src/webgraph/query/collector/group_exact.rs@@ -19,6 +19,7 @@ use rustc_hash::{FxHashMap, FxHashSet};use tantivy::columnar::Column;use crate::webgraph::{+ query::{ColumnFieldFilter, SegmentColumnFieldFilter},schema::{Field, FieldEnum},warmed_column_fields::WarmedColumnFields,};@@ -29,6 +30,7 @@ pub struct GroupExactCollector {group_field: FieldEnum,value_field: FieldEnum,warmed_column_fields: Option<WarmedColumnFields>,+ filter: Option<Box<dyn ColumnFieldFilter>>,}impl GroupExactCollector {@@ -37,13 +39,21 @@ impl GroupExactCollector {group_field: group_field.into(),value_field: value_field.into(),warmed_column_fields: None,+ filter: None,}}+ #[must_use]pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {self.warmed_column_fields = Some(warmed_column_fields);self}++ #[must_use]+ pub fn with_filter(mut self, filter: Box<dyn ColumnFieldFilter>) -> Self {+ self.filter = Some(filter);+ self+ }}impl Collector for GroupExactCollector {@@ -69,7 +79,13 @@ impl Collector for GroupExactCollector {.u64(self.value_field).ok_or(anyhow!("Value field missing from index"))?;+ let filter = self+ .filter+ .as_ref()+ .map(|f| f.for_segment(warmed_column_fields.segment(&segment.segment_id())));+Ok(GroupExactSegmentCollector {+ filter,group,value,groups: FxHashMap::default(),@@ -96,12 +112,19 @@ pub struct GroupExactSegmentCollector {group: Column<u64>,value: Column<u64>,groups: FxHashMap<u64, FxHashSet<u64>>,+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,}impl tantivy::collector::SegmentCollector for GroupExactSegmentCollector {type Fruit = FxHashMap<u64, FxHashSet<u64>>;fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {+ if let Some(filter) = &self.filter {+ if filter.should_skip(doc) {+ return;+ }+ }+let group = self.group.first(doc).unwrap();let value = self.value.first(doc).unwrap();diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rsindex 0f394bfc..5f5eb14c 100644--- a/crates/core/src/webgraph/query/collector/group_sketch.rs+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs@@ -21,6 +21,7 @@ use tantivy::columnar::Column;use crate::{hyperloglog::HyperLogLog,webgraph::{+ query::{ColumnFieldFilter, SegmentColumnFieldFilter},schema::{Field, FieldEnum},warmed_column_fields::WarmedColumnFields,},@@ -32,6 +33,7 @@ pub struct GroupSketchCollector {group_field: FieldEnum,value_field: FieldEnum,warmed_column_fields: Option<WarmedColumnFields>,+ filter: Option<Box<dyn ColumnFieldFilter>>,}impl GroupSketchCollector {@@ -40,13 +42,21 @@ impl GroupSketchCollector {group_field: group_field.into(),value_field: value_field.into(),warmed_column_fields: None,+ filter: None,}}+ #[must_use]pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {self.warmed_column_fields = Some(warmed_column_fields);self}++ #[must_use]+ pub fn with_filter(mut self, filter: Box<dyn ColumnFieldFilter>) -> Self {+ self.filter = Some(filter);+ self+ }}impl Collector for GroupSketchCollector {@@ -72,10 +82,16 @@ impl Collector for GroupSketchCollector {.u64(self.value_field).ok_or(anyhow!("Value field missing from index"))?;+ let filter = self+ .filter+ .as_ref()+ .map(|f| f.for_segment(warmed_column_fields.segment(&segment.segment_id())));+Ok(GroupSketchSegmentCollector {group,value,groups: FxHashMap::default(),+ filter,})}@@ -99,12 +115,19 @@ pub struct GroupSketchSegmentCollector {group: Column<u64>,value: Column<u64>,groups: FxHashMap<u64, HyperLogLog<4069>>,+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,}impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {type Fruit = FxHashMap<u64, HyperLogLog<4069>>;fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {+ if let Some(filter) = &self.filter {+ if filter.should_skip(doc) {+ return;+ }+ }+let group = self.group.first(doc).unwrap();let value = self.value.first(doc).unwrap();diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rsindex 6ad0ae43..f9babee7 100644--- a/crates/core/src/webgraph/query/collector/top_docs.rs+++ b/crates/core/src/webgraph/query/collector/top_docs.rs@@ -313,23 +313,20 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDolet scorer = S::for_segment(segment, &column_fields.warmed_column_fields)?;let segment_id = segment.segment_id();+ let segment_column_fields = column_fields.warmed_column_fields.segment(&segment_id);Ok(TopDocsSegmentCollector {shard_id: self.shard_id.unwrap(),computer: self.computer(),segment_ord,scorer,- host_column: column_fields.host_field.map(|host_field| {- column_fields- .warmed_column_fields- .segment(&segment_id)- .u64_by_enum(host_field)- .unwrap()- }),+ host_column: column_fields+ .host_field+ .map(|host_field| segment_column_fields.u64_by_enum(host_field).unwrap()),filter: self.filter.as_ref()- .map(|f| f.for_segment(&column_fields.warmed_column_fields)),+ .map(|f| f.for_segment(segment_column_fields)),_deduplicator: PhantomData,})}diff --git a/crates/core/src/webgraph/query/filter/and.rs b/crates/core/src/webgraph/query/filter/and.rsindex 464aab29..5aa5621a 100644--- a/crates/core/src/webgraph/query/filter/and.rs+++ b/crates/core/src/webgraph/query/filter/and.rs@@ -16,7 +16,7 @@use tantivy::{query::Occur, DocId};-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};use super::{Filter, FilterEnum};@@ -82,7 +82,7 @@ pub struct AndColumnFieldFilter {impl super::ColumnFieldFilter for AndColumnFieldFilter {fn for_segment(&self,- column_fields: &WarmedColumnFields,+ column_fields: &SegmentColumnFields,) -> Box<dyn super::SegmentColumnFieldFilter> {let mut filters = Vec::with_capacity(self.filters.len());for filter in self.filters.iter() {@@ -98,14 +98,8 @@ pub struct AndSegmentColumnFieldFilter {}impl super::SegmentColumnFieldFilter for AndSegmentColumnFieldFilter {- fn should_skip(&self, doc_id: DocId) -> bool {- for filter in self.filters.iter() {- if filter.should_skip(doc_id) {- return true;- }- }-- false+ fn should_keep(&self, doc_id: DocId) -> bool {+ self.filters.iter().all(|filter| filter.should_keep(doc_id))}}diff --git a/crates/core/src/webgraph/query/filter/mod.rs b/crates/core/src/webgraph/query/filter/mod.rsindex 91130675..90961851 100644--- a/crates/core/src/webgraph/query/filter/mod.rs+++ b/crates/core/src/webgraph/query/filter/mod.rs@@ -26,9 +26,12 @@ pub use not::NotFilter;mod text;pub use text::TextFilter;+mod rel_flag;+pub use rel_flag::RelFlagsFilter;+use tantivy::{query::Occur, DocId};-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};pub trait Filter:Send@@ -50,6 +53,7 @@ pub enum FilterEnum {OrFilter(OrFilter),NotFilter(Box<NotFilter>),TextFilter(TextFilter),+ RelFlags(RelFlagsFilter),}impl Filter for FilterEnum {@@ -59,6 +63,7 @@ impl Filter for FilterEnum {FilterEnum::OrFilter(filter) => filter.column_field_filter(),FilterEnum::NotFilter(filter) => filter.column_field_filter(),FilterEnum::TextFilter(filter) => filter.column_field_filter(),+ FilterEnum::RelFlags(filter) => filter.column_field_filter(),}}@@ -68,16 +73,21 @@ impl Filter for FilterEnum {FilterEnum::OrFilter(filter) => filter.inverted_index_filter(),FilterEnum::NotFilter(filter) => filter.inverted_index_filter(),FilterEnum::TextFilter(filter) => filter.inverted_index_filter(),+ FilterEnum::RelFlags(filter) => filter.inverted_index_filter(),}}}pub trait ColumnFieldFilter: Send + Sync + 'static {- fn for_segment(&self, column_fields: &WarmedColumnFields) -> Box<dyn SegmentColumnFieldFilter>;+ fn for_segment(&self, column_fields: &SegmentColumnFields)+ -> Box<dyn SegmentColumnFieldFilter>;}pub trait SegmentColumnFieldFilter {- fn should_skip(&self, doc_id: DocId) -> bool;+ fn should_keep(&self, doc_id: DocId) -> bool;+ fn should_skip(&self, doc_id: DocId) -> bool {+ !self.should_keep(doc_id)+ }}pub trait InvertedIndexFilter {diff --git a/crates/core/src/webgraph/query/filter/not.rs b/crates/core/src/webgraph/query/filter/not.rsindex 966803ff..c1fd6419 100644--- a/crates/core/src/webgraph/query/filter/not.rs+++ b/crates/core/src/webgraph/query/filter/not.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};use super::{Filter, FilterEnum};use tantivy::{query::Occur, DocId};@@ -57,7 +57,7 @@ struct NotColumnFieldFilter {impl super::ColumnFieldFilter for NotColumnFieldFilter {fn for_segment(&self,- column_fields: &WarmedColumnFields,+ column_fields: &SegmentColumnFields,) -> Box<dyn super::SegmentColumnFieldFilter> {let filter = self.filter.for_segment(column_fields);Box::new(NotSegmentColumnFieldFilter { filter })@@ -69,8 +69,8 @@ struct NotSegmentColumnFieldFilter {}impl super::SegmentColumnFieldFilter for NotSegmentColumnFieldFilter {- fn should_skip(&self, doc_id: DocId) -> bool {- !self.filter.should_skip(doc_id)+ fn should_keep(&self, doc_id: DocId) -> bool {+ !self.filter.should_keep(doc_id)}}diff --git a/crates/core/src/webgraph/query/filter/or.rs b/crates/core/src/webgraph/query/filter/or.rsindex 307150ec..6405b89e 100644--- a/crates/core/src/webgraph/query/filter/or.rs+++ b/crates/core/src/webgraph/query/filter/or.rs@@ -19,7 +19,7 @@ use tantivy::{DocId,};-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};use super::{Filter, FilterEnum};@@ -85,7 +85,7 @@ pub struct OrColumnFieldFilter {impl super::ColumnFieldFilter for OrColumnFieldFilter {fn for_segment(&self,- column_fields: &WarmedColumnFields,+ column_fields: &SegmentColumnFields,) -> Box<dyn super::SegmentColumnFieldFilter> {let mut filters = Vec::with_capacity(self.filters.len());for filter in self.filters.iter() {@@ -101,14 +101,8 @@ pub struct OrSegmentColumnFieldFilter {}impl super::SegmentColumnFieldFilter for OrSegmentColumnFieldFilter {- fn should_skip(&self, doc_id: DocId) -> bool {- for filter in self.filters.iter() {- if !filter.should_skip(doc_id) {- return false;- }- }-- true+ fn should_keep(&self, doc_id: DocId) -> bool {+ self.filters.iter().any(|filter| filter.should_keep(doc_id))}}diff --git a/crates/core/src/webgraph/query/filter/rel_flag.rs b/crates/core/src/webgraph/query/filter/rel_flag.rsnew file mode 100644index 00000000..5679c537--- /dev/null+++ b/crates/core/src/webgraph/query/filter/rel_flag.rs@@ -0,0 +1,146 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::columnar::Column;++use crate::{+ webgraph::{schema, warmed_column_fields::SegmentColumnFields},+ webpage::RelFlags,+};++use super::{ColumnFieldFilter, Filter, FilterEnum, InvertedIndexFilter, SegmentColumnFieldFilter};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct RelFlagsFilter(RelFlags);++impl Filter for RelFlagsFilter {+ fn column_field_filter(&self) -> Option<Box<dyn ColumnFieldFilter>> {+ Some(Box::new(RelFlagsColumnFieldFilter(self.0)))+ }++ fn inverted_index_filter(&self) -> Option<Box<dyn InvertedIndexFilter>> {+ None+ }+}++impl From<RelFlagsFilter> for FilterEnum {+ fn from(filter: RelFlagsFilter) -> Self {+ FilterEnum::RelFlags(filter)+ }+}++struct RelFlagsColumnFieldFilter(RelFlags);++impl ColumnFieldFilter for RelFlagsColumnFieldFilter {+ fn for_segment(+ &self,+ column_fields: &SegmentColumnFields,+ ) -> Box<dyn SegmentColumnFieldFilter> {+ Box::new(RelFlagsSegmentColumnFieldFilter {+ rel_flags: self.0,+ column: column_fields.u64(schema::RelFlags).unwrap(),+ })+ }+}++struct RelFlagsSegmentColumnFieldFilter {+ rel_flags: RelFlags,+ column: Column<u64>,+}++impl SegmentColumnFieldFilter for RelFlagsSegmentColumnFieldFilter {+ fn should_keep(&self, doc_id: tantivy::DocId) -> bool {+ let rel_flag = RelFlags::from(self.column.first(doc_id).unwrap());+ rel_flag.contains(self.rel_flags)+ }+}++#[cfg(test)]+mod tests {+ use file_store::temp::TempDir;++ use crate::{+ webgraph::{+ query::{FullForwardlinksQuery, NotFilter},+ Edge, Node, Webgraph,+ },+ webpage::RelFlags,+ };++ use super::*;++ pub fn test_edges() -> Vec<(Node, Node, RelFlags)> {+ vec![+ (+ Node::from("a.com"),+ Node::from("b.com/123"),+ RelFlags::default(),+ ),+ (+ Node::from("a.com"),+ Node::from("b.dk/123"),+ RelFlags::NOFOLLOW,+ ),+ ]+ }++ pub fn test_graph() -> (Webgraph, TempDir) {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ for (from, to, rel_flags) in test_edges() {+ graph+ .insert(Edge {+ from,+ to,+ rel_flags,+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();+ }++ graph.commit().unwrap();++ (graph, temp_dir)+ }++ #[test]+ fn test_rel_flags_filter() {+ let (graph, _temp_dir) = test_graph();++ let node = Node::from("a.com");++ let res = graph+ .search(&FullForwardlinksQuery::new(node).filter(RelFlagsFilter(RelFlags::NOFOLLOW)))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("b.dk/123"));++ let node = Node::from("a.com");++ let res = graph+ .search(+ &FullForwardlinksQuery::new(node)+ .filter(NotFilter::new(RelFlagsFilter(RelFlags::NOFOLLOW))),+ )+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("b.com/123"));+ }+}diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rsindex fb7287e3..798e0469 100644--- a/crates/core/src/webgraph/query/group_by.rs+++ b/crates/core/src/webgraph/query/group_by.rs@@ -239,8 +239,14 @@ impl Query for HostGroupQuery {}fn collector(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::Collector {- GroupExactCollector::new(self.group, self.value)- .with_column_fields(searcher.warmed_column_fields().clone())+ let mut collector = GroupExactCollector::new(self.group, self.value)+ .with_column_fields(searcher.warmed_column_fields().clone());++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {diff --git a/crates/core/src/webgraph/query/id2node.rs b/crates/core/src/webgraph/query/id2node.rsindex 37780739..1332c57a 100644--- a/crates/core/src/webgraph/query/id2node.rs+++ b/crates/core/src/webgraph/query/id2node.rs@@ -44,6 +44,7 @@ impl Query for Id2NodeQuery {fn collector(&self, searcher: &Searcher) -> Self::Collector {FirstDocCollector::with_shard_id(searcher.shard())+ .with_column_fields(searcher.warmed_column_fields().clone())}fn retrieve(