Skip to content

web search done right

[webgraph] rel flags filter (nofollow etc)

Mikkel Denkerfb4aec54245b
diff --git a/crates/core/src/webgraph/query/collector/first_doc.rs b/crates/core/src/webgraph/query/collector/first_doc.rs
index 0f4a0a08..42bca480 100644
--- a/crates/core/src/webgraph/query/collector/first_doc.rs
+++ b/crates/core/src/webgraph/query/collector/first_doc.rs
@@ -17,21 +17,54 @@
use tantivy::{collector::SegmentCollector, DocId, SegmentOrdinal};
use super::Collector;
-use crate::{ampc::dht::ShardId, webgraph::doc_address::DocAddress};
+use crate::{
+ ampc::dht::ShardId,
+ webgraph::{
+ doc_address::DocAddress,
+ query::{ColumnFieldFilter, SegmentColumnFieldFilter},
+ warmed_column_fields::WarmedColumnFields,
+ },
+};
pub struct FirstDocCollector {
shard_id: Option<ShardId>,
+ column_fields: Option<WarmedColumnFields>,
+ filter: Option<Box<dyn ColumnFieldFilter>>,
}
impl FirstDocCollector {
+ #[must_use]
pub fn with_shard_id(shard_id: ShardId) -> Self {
Self {
shard_id: Some(shard_id),
+ filter: None,
+ column_fields: None,
}
}
+ #[must_use]
pub fn without_shard_id() -> Self {
- Self { shard_id: None }
+ Self {
+ shard_id: None,
+ filter: None,
+ column_fields: None,
+ }
+ }
+
+ #[must_use]
+ pub fn with_column_fields(self, warmed_column_fields: WarmedColumnFields) -> Self {
+ Self {
+ column_fields: Some(warmed_column_fields),
+ ..self
+ }
+ }
+
+ #[must_use]
+ pub fn with_filter(self, filter: Box<dyn ColumnFieldFilter>) -> Self {
+ Self {
+ filter: Some(filter),
+ ..self
+ }
}
}
@@ -42,10 +75,19 @@ impl Collector for FirstDocCollector {
fn for_segment(
&self,
segment_id: SegmentOrdinal,
- _: &tantivy::SegmentReader,
+ segment: &tantivy::SegmentReader,
) -> crate::Result<Self::Child> {
+ let column_fields = self
+ .column_fields
+ .as_ref()
+ .expect("Column fields are required to perform search")
+ .segment(&segment.segment_id());
+
+ let filter = self.filter.as_ref().map(|f| f.for_segment(column_fields));
+
Ok(FirstDocSegmentCollector::new(
segment_id,
+ filter,
self.shard_id.unwrap(),
))
}
@@ -61,14 +103,20 @@ impl Collector for FirstDocCollector {
pub struct FirstDocSegmentCollector {
first_doc: Option<DocAddress>,
segment_id: SegmentOrdinal,
+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,
shard_id: ShardId,
}
impl FirstDocSegmentCollector {
- pub fn new(segment_id: SegmentOrdinal, shard_id: ShardId) -> Self {
+ pub fn new(
+ segment_id: SegmentOrdinal,
+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,
+ shard_id: ShardId,
+ ) -> Self {
Self {
first_doc: None,
segment_id,
+ filter,
shard_id,
}
}
@@ -78,6 +126,12 @@ impl SegmentCollector for FirstDocSegmentCollector {
type Fruit = Option<DocAddress>;
fn collect(&mut self, doc: DocId, _: tantivy::Score) {
+ if let Some(filter) = &self.filter {
+ if filter.should_skip(doc) {
+ return;
+ }
+ }
+
if self.first_doc.is_none() {
self.first_doc = Some(DocAddress::new(self.shard_id, self.segment_id, doc));
}
diff --git a/crates/core/src/webgraph/query/collector/group_exact.rs b/crates/core/src/webgraph/query/collector/group_exact.rs
index 5418d6b1..9a5fcd2f 100644
--- a/crates/core/src/webgraph/query/collector/group_exact.rs
+++ b/crates/core/src/webgraph/query/collector/group_exact.rs
@@ -19,6 +19,7 @@ use rustc_hash::{FxHashMap, FxHashSet};
use tantivy::columnar::Column;
use crate::webgraph::{
+ query::{ColumnFieldFilter, SegmentColumnFieldFilter},
schema::{Field, FieldEnum},
warmed_column_fields::WarmedColumnFields,
};
@@ -29,6 +30,7 @@ pub struct GroupExactCollector {
group_field: FieldEnum,
value_field: FieldEnum,
warmed_column_fields: Option<WarmedColumnFields>,
+ filter: Option<Box<dyn ColumnFieldFilter>>,
}
impl GroupExactCollector {
@@ -37,13 +39,21 @@ impl GroupExactCollector {
group_field: group_field.into(),
value_field: value_field.into(),
warmed_column_fields: None,
+ filter: None,
}
}
+ #[must_use]
pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {
self.warmed_column_fields = Some(warmed_column_fields);
self
}
+
+ #[must_use]
+ pub fn with_filter(mut self, filter: Box<dyn ColumnFieldFilter>) -> Self {
+ self.filter = Some(filter);
+ self
+ }
}
impl Collector for GroupExactCollector {
@@ -69,7 +79,13 @@ impl Collector for GroupExactCollector {
.u64(self.value_field)
.ok_or(anyhow!("Value field missing from index"))?;
+ let filter = self
+ .filter
+ .as_ref()
+ .map(|f| f.for_segment(warmed_column_fields.segment(&segment.segment_id())));
+
Ok(GroupExactSegmentCollector {
+ filter,
group,
value,
groups: FxHashMap::default(),
@@ -96,12 +112,19 @@ pub struct GroupExactSegmentCollector {
group: Column<u64>,
value: Column<u64>,
groups: FxHashMap<u64, FxHashSet<u64>>,
+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,
}
impl tantivy::collector::SegmentCollector for GroupExactSegmentCollector {
type Fruit = FxHashMap<u64, FxHashSet<u64>>;
fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {
+ if let Some(filter) = &self.filter {
+ if filter.should_skip(doc) {
+ return;
+ }
+ }
+
let group = self.group.first(doc).unwrap();
let value = self.value.first(doc).unwrap();
diff --git a/crates/core/src/webgraph/query/collector/group_sketch.rs b/crates/core/src/webgraph/query/collector/group_sketch.rs
index 0f394bfc..5f5eb14c 100644
--- a/crates/core/src/webgraph/query/collector/group_sketch.rs
+++ b/crates/core/src/webgraph/query/collector/group_sketch.rs
@@ -21,6 +21,7 @@ use tantivy::columnar::Column;
use crate::{
hyperloglog::HyperLogLog,
webgraph::{
+ query::{ColumnFieldFilter, SegmentColumnFieldFilter},
schema::{Field, FieldEnum},
warmed_column_fields::WarmedColumnFields,
},
@@ -32,6 +33,7 @@ pub struct GroupSketchCollector {
group_field: FieldEnum,
value_field: FieldEnum,
warmed_column_fields: Option<WarmedColumnFields>,
+ filter: Option<Box<dyn ColumnFieldFilter>>,
}
impl GroupSketchCollector {
@@ -40,13 +42,21 @@ impl GroupSketchCollector {
group_field: group_field.into(),
value_field: value_field.into(),
warmed_column_fields: None,
+ filter: None,
}
}
+ #[must_use]
pub fn with_column_fields(mut self, warmed_column_fields: WarmedColumnFields) -> Self {
self.warmed_column_fields = Some(warmed_column_fields);
self
}
+
+ #[must_use]
+ pub fn with_filter(mut self, filter: Box<dyn ColumnFieldFilter>) -> Self {
+ self.filter = Some(filter);
+ self
+ }
}
impl Collector for GroupSketchCollector {
@@ -72,10 +82,16 @@ impl Collector for GroupSketchCollector {
.u64(self.value_field)
.ok_or(anyhow!("Value field missing from index"))?;
+ let filter = self
+ .filter
+ .as_ref()
+ .map(|f| f.for_segment(warmed_column_fields.segment(&segment.segment_id())));
+
Ok(GroupSketchSegmentCollector {
group,
value,
groups: FxHashMap::default(),
+ filter,
})
}
@@ -99,12 +115,19 @@ pub struct GroupSketchSegmentCollector {
group: Column<u64>,
value: Column<u64>,
groups: FxHashMap<u64, HyperLogLog<4069>>,
+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,
}
impl tantivy::collector::SegmentCollector for GroupSketchSegmentCollector {
type Fruit = FxHashMap<u64, HyperLogLog<4069>>;
fn collect(&mut self, doc: tantivy::DocId, _: tantivy::Score) {
+ if let Some(filter) = &self.filter {
+ if filter.should_skip(doc) {
+ return;
+ }
+ }
+
let group = self.group.first(doc).unwrap();
let value = self.value.first(doc).unwrap();
diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rs
index 6ad0ae43..f9babee7 100644
--- a/crates/core/src/webgraph/query/collector/top_docs.rs
+++ b/crates/core/src/webgraph/query/collector/top_docs.rs
@@ -313,23 +313,20 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDo
let scorer = S::for_segment(segment, &column_fields.warmed_column_fields)?;
let segment_id = segment.segment_id();
+ let segment_column_fields = column_fields.warmed_column_fields.segment(&segment_id);
Ok(TopDocsSegmentCollector {
shard_id: self.shard_id.unwrap(),
computer: self.computer(),
segment_ord,
scorer,
- host_column: column_fields.host_field.map(|host_field| {
- column_fields
- .warmed_column_fields
- .segment(&segment_id)
- .u64_by_enum(host_field)
- .unwrap()
- }),
+ host_column: column_fields
+ .host_field
+ .map(|host_field| segment_column_fields.u64_by_enum(host_field).unwrap()),
filter: self
.filter
.as_ref()
- .map(|f| f.for_segment(&column_fields.warmed_column_fields)),
+ .map(|f| f.for_segment(segment_column_fields)),
_deduplicator: PhantomData,
})
}
diff --git a/crates/core/src/webgraph/query/filter/and.rs b/crates/core/src/webgraph/query/filter/and.rs
index 464aab29..5aa5621a 100644
--- a/crates/core/src/webgraph/query/filter/and.rs
+++ b/crates/core/src/webgraph/query/filter/and.rs
@@ -16,7 +16,7 @@
use tantivy::{query::Occur, DocId};
-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};
use super::{Filter, FilterEnum};
@@ -82,7 +82,7 @@ pub struct AndColumnFieldFilter {
impl super::ColumnFieldFilter for AndColumnFieldFilter {
fn for_segment(
&self,
- column_fields: &WarmedColumnFields,
+ column_fields: &SegmentColumnFields,
) -> Box<dyn super::SegmentColumnFieldFilter> {
let mut filters = Vec::with_capacity(self.filters.len());
for filter in self.filters.iter() {
@@ -98,14 +98,8 @@ pub struct AndSegmentColumnFieldFilter {
}
impl super::SegmentColumnFieldFilter for AndSegmentColumnFieldFilter {
- fn should_skip(&self, doc_id: DocId) -> bool {
- for filter in self.filters.iter() {
- if filter.should_skip(doc_id) {
- return true;
- }
- }
-
- false
+ fn should_keep(&self, doc_id: DocId) -> bool {
+ self.filters.iter().all(|filter| filter.should_keep(doc_id))
}
}
diff --git a/crates/core/src/webgraph/query/filter/mod.rs b/crates/core/src/webgraph/query/filter/mod.rs
index 91130675..90961851 100644
--- a/crates/core/src/webgraph/query/filter/mod.rs
+++ b/crates/core/src/webgraph/query/filter/mod.rs
@@ -26,9 +26,12 @@ pub use not::NotFilter;
mod text;
pub use text::TextFilter;
+mod rel_flag;
+pub use rel_flag::RelFlagsFilter;
+
use tantivy::{query::Occur, DocId};
-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};
pub trait Filter:
Send
@@ -50,6 +53,7 @@ pub enum FilterEnum {
OrFilter(OrFilter),
NotFilter(Box<NotFilter>),
TextFilter(TextFilter),
+ RelFlags(RelFlagsFilter),
}
impl Filter for FilterEnum {
@@ -59,6 +63,7 @@ impl Filter for FilterEnum {
FilterEnum::OrFilter(filter) => filter.column_field_filter(),
FilterEnum::NotFilter(filter) => filter.column_field_filter(),
FilterEnum::TextFilter(filter) => filter.column_field_filter(),
+ FilterEnum::RelFlags(filter) => filter.column_field_filter(),
}
}
@@ -68,16 +73,21 @@ impl Filter for FilterEnum {
FilterEnum::OrFilter(filter) => filter.inverted_index_filter(),
FilterEnum::NotFilter(filter) => filter.inverted_index_filter(),
FilterEnum::TextFilter(filter) => filter.inverted_index_filter(),
+ FilterEnum::RelFlags(filter) => filter.inverted_index_filter(),
}
}
}
pub trait ColumnFieldFilter: Send + Sync + 'static {
- fn for_segment(&self, column_fields: &WarmedColumnFields) -> Box<dyn SegmentColumnFieldFilter>;
+ fn for_segment(&self, column_fields: &SegmentColumnFields)
+ -> Box<dyn SegmentColumnFieldFilter>;
}
pub trait SegmentColumnFieldFilter {
- fn should_skip(&self, doc_id: DocId) -> bool;
+ fn should_keep(&self, doc_id: DocId) -> bool;
+ fn should_skip(&self, doc_id: DocId) -> bool {
+ !self.should_keep(doc_id)
+ }
}
pub trait InvertedIndexFilter {
diff --git a/crates/core/src/webgraph/query/filter/not.rs b/crates/core/src/webgraph/query/filter/not.rs
index 966803ff..c1fd6419 100644
--- a/crates/core/src/webgraph/query/filter/not.rs
+++ b/crates/core/src/webgraph/query/filter/not.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};
use super::{Filter, FilterEnum};
use tantivy::{query::Occur, DocId};
@@ -57,7 +57,7 @@ struct NotColumnFieldFilter {
impl super::ColumnFieldFilter for NotColumnFieldFilter {
fn for_segment(
&self,
- column_fields: &WarmedColumnFields,
+ column_fields: &SegmentColumnFields,
) -> Box<dyn super::SegmentColumnFieldFilter> {
let filter = self.filter.for_segment(column_fields);
Box::new(NotSegmentColumnFieldFilter { filter })
@@ -69,8 +69,8 @@ struct NotSegmentColumnFieldFilter {
}
impl super::SegmentColumnFieldFilter for NotSegmentColumnFieldFilter {
- fn should_skip(&self, doc_id: DocId) -> bool {
- !self.filter.should_skip(doc_id)
+ fn should_keep(&self, doc_id: DocId) -> bool {
+ !self.filter.should_keep(doc_id)
}
}
diff --git a/crates/core/src/webgraph/query/filter/or.rs b/crates/core/src/webgraph/query/filter/or.rs
index 307150ec..6405b89e 100644
--- a/crates/core/src/webgraph/query/filter/or.rs
+++ b/crates/core/src/webgraph/query/filter/or.rs
@@ -19,7 +19,7 @@ use tantivy::{
DocId,
};
-use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::SegmentColumnFields};
use super::{Filter, FilterEnum};
@@ -85,7 +85,7 @@ pub struct OrColumnFieldFilter {
impl super::ColumnFieldFilter for OrColumnFieldFilter {
fn for_segment(
&self,
- column_fields: &WarmedColumnFields,
+ column_fields: &SegmentColumnFields,
) -> Box<dyn super::SegmentColumnFieldFilter> {
let mut filters = Vec::with_capacity(self.filters.len());
for filter in self.filters.iter() {
@@ -101,14 +101,8 @@ pub struct OrSegmentColumnFieldFilter {
}
impl super::SegmentColumnFieldFilter for OrSegmentColumnFieldFilter {
- fn should_skip(&self, doc_id: DocId) -> bool {
- for filter in self.filters.iter() {
- if !filter.should_skip(doc_id) {
- return false;
- }
- }
-
- true
+ fn should_keep(&self, doc_id: DocId) -> bool {
+ self.filters.iter().any(|filter| filter.should_keep(doc_id))
}
}
diff --git a/crates/core/src/webgraph/query/filter/rel_flag.rs b/crates/core/src/webgraph/query/filter/rel_flag.rs
new file mode 100644
index 00000000..5679c537
--- /dev/null
+++ b/crates/core/src/webgraph/query/filter/rel_flag.rs
@@ -0,0 +1,146 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::columnar::Column;
+
+use crate::{
+ webgraph::{schema, warmed_column_fields::SegmentColumnFields},
+ webpage::RelFlags,
+};
+
+use super::{ColumnFieldFilter, Filter, FilterEnum, InvertedIndexFilter, SegmentColumnFieldFilter};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct RelFlagsFilter(RelFlags);
+
+impl Filter for RelFlagsFilter {
+ fn column_field_filter(&self) -> Option<Box<dyn ColumnFieldFilter>> {
+ Some(Box::new(RelFlagsColumnFieldFilter(self.0)))
+ }
+
+ fn inverted_index_filter(&self) -> Option<Box<dyn InvertedIndexFilter>> {
+ None
+ }
+}
+
+impl From<RelFlagsFilter> for FilterEnum {
+ fn from(filter: RelFlagsFilter) -> Self {
+ FilterEnum::RelFlags(filter)
+ }
+}
+
+struct RelFlagsColumnFieldFilter(RelFlags);
+
+impl ColumnFieldFilter for RelFlagsColumnFieldFilter {
+ fn for_segment(
+ &self,
+ column_fields: &SegmentColumnFields,
+ ) -> Box<dyn SegmentColumnFieldFilter> {
+ Box::new(RelFlagsSegmentColumnFieldFilter {
+ rel_flags: self.0,
+ column: column_fields.u64(schema::RelFlags).unwrap(),
+ })
+ }
+}
+
+struct RelFlagsSegmentColumnFieldFilter {
+ rel_flags: RelFlags,
+ column: Column<u64>,
+}
+
+impl SegmentColumnFieldFilter for RelFlagsSegmentColumnFieldFilter {
+ fn should_keep(&self, doc_id: tantivy::DocId) -> bool {
+ let rel_flag = RelFlags::from(self.column.first(doc_id).unwrap());
+ rel_flag.contains(self.rel_flags)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use file_store::temp::TempDir;
+
+ use crate::{
+ webgraph::{
+ query::{FullForwardlinksQuery, NotFilter},
+ Edge, Node, Webgraph,
+ },
+ webpage::RelFlags,
+ };
+
+ use super::*;
+
+ pub fn test_edges() -> Vec<(Node, Node, RelFlags)> {
+ vec![
+ (
+ Node::from("a.com"),
+ Node::from("b.com/123"),
+ RelFlags::default(),
+ ),
+ (
+ Node::from("a.com"),
+ Node::from("b.dk/123"),
+ RelFlags::NOFOLLOW,
+ ),
+ ]
+ }
+
+ pub fn test_graph() -> (Webgraph, TempDir) {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ for (from, to, rel_flags) in test_edges() {
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags,
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ }
+
+ graph.commit().unwrap();
+
+ (graph, temp_dir)
+ }
+
+ #[test]
+ fn test_rel_flags_filter() {
+ let (graph, _temp_dir) = test_graph();
+
+ let node = Node::from("a.com");
+
+ let res = graph
+ .search(&FullForwardlinksQuery::new(node).filter(RelFlagsFilter(RelFlags::NOFOLLOW)))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("b.dk/123"));
+
+ let node = Node::from("a.com");
+
+ let res = graph
+ .search(
+ &FullForwardlinksQuery::new(node)
+ .filter(NotFilter::new(RelFlagsFilter(RelFlags::NOFOLLOW))),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("b.com/123"));
+ }
+}
diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rs
index fb7287e3..798e0469 100644
--- a/crates/core/src/webgraph/query/group_by.rs
+++ b/crates/core/src/webgraph/query/group_by.rs
@@ -239,8 +239,14 @@ impl Query for HostGroupQuery {
}
fn collector(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::Collector {
- GroupExactCollector::new(self.group, self.value)
- .with_column_fields(searcher.warmed_column_fields().clone())
+ let mut collector = GroupExactCollector::new(self.group, self.value)
+ .with_column_fields(searcher.warmed_column_fields().clone());
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
diff --git a/crates/core/src/webgraph/query/id2node.rs b/crates/core/src/webgraph/query/id2node.rs
index 37780739..1332c57a 100644
--- a/crates/core/src/webgraph/query/id2node.rs
+++ b/crates/core/src/webgraph/query/id2node.rs
@@ -44,6 +44,7 @@ impl Query for Id2NodeQuery {
fn collector(&self, searcher: &Searcher) -> Self::Collector {
FirstDocCollector::with_shard_id(searcher.shard())
+ .with_column_fields(searcher.warmed_column_fields().clone())
}
fn retrieve(