Skip to content

web search done right

[webgraph] query filters

Mikkel Denkere4a8a9f2fe1a
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rs
index ac24d151..f590c097 100644
--- a/crates/core/src/webgraph/node.rs
+++ b/crates/core/src/webgraph/node.rs
@@ -169,12 +169,7 @@ impl Node {
#[cfg(test)]
impl From<String> for Node {
fn from(name: String) -> Self {
- let url = if name.contains("://") {
- Url::parse(&name).unwrap()
- } else {
- Url::parse(&("http://".to_string() + name.as_str())).unwrap()
- };
-
+ let url = Url::robust_parse(&name).unwrap();
Node::from(&url)
}
}
@@ -204,6 +199,11 @@ pub fn normalize_url(url: &Url) -> String {
url.normalize_in_place();
let scheme = url.scheme();
+
+ if scheme != "http" && scheme != "https" {
+ return url.to_string();
+ }
+
let mut normalized = url
.as_str()
.strip_prefix(scheme)
@@ -222,3 +222,28 @@ pub fn normalize_url(url: &Url) -> String {
normalized.to_lowercase()
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_normalize_url() {
+ assert_eq!(
+ normalize_url(&Url::robust_parse("https://www.example.com/").unwrap()),
+ "example.com"
+ );
+ }
+
+ #[test]
+ fn test_host_node() {
+ assert_eq!(
+ Node::from("example.com").into_host(),
+ Node::from("example.com")
+ );
+ assert_eq!(
+ Node::from("https://example.com/123").into_host(),
+ Node::from("example.com")
+ );
+ }
+}
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index f0988b4e..2963e33c 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -14,13 +14,13 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::query::ShortCircuitQuery;
+use tantivy::query::{BooleanQuery, Occur, ShortCircuitQuery};
use super::{
collector::{top_docs, HostDeduplicator, TopDocsCollector},
document_scorer::DefaultDocumentScorer,
raw::{HostLinksQuery, LinksQuery},
- Query,
+ AndFilter, Filter, FilterEnum, Query,
};
use crate::{
ampc::dht::ShardId,
@@ -97,6 +97,7 @@ pub fn fetch_edges(searcher: &Searcher, mut doc_ids: Vec<DocAddress>) -> Result<
pub struct BacklinksQuery {
node: NodeID,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl BacklinksQuery {
@@ -104,6 +105,7 @@ impl BacklinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -111,6 +113,26 @@ impl BacklinksQuery {
Self {
node: self.node,
limit,
+ filters: self.filters,
+ }
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+
+ Some(filter)
}
}
}
@@ -121,25 +143,40 @@ impl Query for BacklinksQuery {
type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
match self.limit {
- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, ToId)),
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node, ToId)),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node, ToId)),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
- .with_column_fields(searcher.warmed_column_fields().clone())
+ .with_column_fields(searcher.warmed_column_fields().clone());
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
@@ -190,6 +227,7 @@ impl Query for BacklinksQuery {
pub struct HostBacklinksQuery {
node: NodeID,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl HostBacklinksQuery {
@@ -197,6 +235,7 @@ impl HostBacklinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -204,6 +243,25 @@ impl HostBacklinksQuery {
self.limit = limit;
self
}
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+
+ Some(filter)
+ }
+ }
}
impl Query for HostBacklinksQuery {
@@ -213,41 +271,46 @@ impl Query for HostBacklinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw = Box::new(HostLinksQuery::new(
+ self.node,
+ ToHostId,
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
match self.limit {
- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(
- self.node,
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node,
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node,
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
.with_deduplicator(HostDeduplicator)
.with_column_fields(searcher.warmed_column_fields().clone())
- .with_host_field(FromHostId)
+ .with_host_field(FromHostId);
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
@@ -300,6 +363,7 @@ impl Query for HostBacklinksQuery {
pub struct FullBacklinksQuery {
node: Node,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl FullBacklinksQuery {
@@ -307,6 +371,7 @@ impl FullBacklinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -314,6 +379,26 @@ impl FullBacklinksQuery {
Self {
node: self.node,
limit,
+ filters: self.filters,
+ }
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+
+ Some(filter)
}
}
}
@@ -324,25 +409,41 @@ impl Query for FullBacklinksQuery {
type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw =
+ Box::new(LinksQuery::new(self.node.id(), ToId)) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
match self.limit {
- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), ToId)),
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node.id(), ToId)),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node.id(), ToId)),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
- .with_column_fields(searcher.warmed_column_fields().clone())
+ .with_column_fields(searcher.warmed_column_fields().clone());
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
@@ -381,6 +482,9 @@ impl Query for FullBacklinksQuery {
pub struct FullHostBacklinksQuery {
node: Node,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
+ nodes_as_host: bool,
+ skip_self_links: bool,
}
impl FullHostBacklinksQuery {
@@ -388,13 +492,42 @@ impl FullHostBacklinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
+ nodes_as_host: true,
+ skip_self_links: true,
}
}
pub fn with_limit(self, limit: EdgeLimit) -> Self {
- Self {
- node: self.node,
- limit,
+ Self { limit, ..self }
+ }
+
+ pub fn nodes_as_host(mut self, nodes_as_host: bool) -> Self {
+ self.nodes_as_host = nodes_as_host;
+ self
+ }
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+
+ Some(filter)
}
}
}
@@ -406,41 +539,51 @@ impl Query for FullHostBacklinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- match self.limit {
- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(
+ let mut raw = Box::new(
+ HostLinksQuery::new(
self.node.clone().into_host().id(),
ToHostId,
FromHostId,
searcher.warmed_column_fields().clone(),
- )),
+ )
+ .skip_self_links(self.skip_self_links),
+ ) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
+ match self.limit {
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
.with_deduplicator(HostDeduplicator)
.with_column_fields(searcher.warmed_column_fields().clone())
- .with_host_field(FromHostId)
+ .with_host_field(FromHostId);
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
@@ -457,6 +600,10 @@ impl Query for FullHostBacklinksQuery {
let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();
let edges = fetch_edges(searcher, docs)?;
+ if !self.nodes_as_host {
+ return Ok(edges);
+ }
+
Ok(edges
.into_iter()
.map(|e| Edge {
@@ -484,10 +631,12 @@ impl Query for FullHostBacklinksQuery {
edges
}
}
+
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct BacklinksWithLabelsQuery {
node: NodeID,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl BacklinksWithLabelsQuery {
@@ -495,6 +644,7 @@ impl BacklinksWithLabelsQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -502,6 +652,26 @@ impl BacklinksWithLabelsQuery {
Self {
node: self.node,
limit,
+ filters: self.filters,
+ }
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+
+ Some(filter)
}
}
}
@@ -512,15 +682,30 @@ impl Query for BacklinksWithLabelsQuery {
type IntermediateOutput = Vec<(f64, SmallEdgeWithLabel)>;
type Output = Vec<SmallEdgeWithLabel>;
- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
- Box::new(LinksQuery::new(self.node, ToId))
+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
+ raw
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
- .with_column_fields(searcher.warmed_column_fields().clone())
+ .with_column_fields(searcher.warmed_column_fields().clone());
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rs
index 7813554f..6ad0ae43 100644
--- a/crates/core/src/webgraph/query/collector/top_docs.rs
+++ b/crates/core/src/webgraph/query/collector/top_docs.rs
@@ -26,7 +26,10 @@ use tantivy::{
use crate::webgraph::{
doc_address::DocAddress,
- query::document_scorer::{DefaultDocumentScorer, DocumentScorer},
+ query::{
+ document_scorer::{DefaultDocumentScorer, DocumentScorer},
+ ColumnFieldFilter, SegmentColumnFieldFilter,
+ },
schema::{Field, FieldEnum},
EdgeLimit,
};
@@ -157,6 +160,7 @@ pub struct TopDocsCollector<S = DefaultDocumentScorer, D = NoDeduplicator> {
perform_offset: bool,
deduplicator: D,
column_fields: Option<ColumnFields>,
+ filter: Option<Box<dyn ColumnFieldFilter>>,
_phantom: PhantomData<S>,
}
@@ -192,12 +196,14 @@ impl<S> TopDocsCollector<S, NoDeduplicator> {
perform_offset: true,
deduplicator: NoDeduplicator,
column_fields: None,
+ filter: None,
_phantom: PhantomData,
}
}
}
impl<S, D> TopDocsCollector<S, D> {
+ #[must_use]
pub fn with_shard_id(self, shard_id: ShardId) -> Self {
Self {
shard_id: Some(shard_id),
@@ -205,6 +211,7 @@ impl<S, D> TopDocsCollector<S, D> {
}
}
+ #[must_use]
pub fn with_offset(self, offset: usize) -> Self {
Self {
offset: Some(offset),
@@ -212,6 +219,7 @@ impl<S, D> TopDocsCollector<S, D> {
}
}
+ #[must_use]
pub fn with_limit(self, limit: usize) -> Self {
Self {
limit: Some(limit),
@@ -219,6 +227,7 @@ impl<S, D> TopDocsCollector<S, D> {
}
}
+ #[must_use]
pub fn enable_offset(self) -> Self {
Self {
perform_offset: true,
@@ -226,6 +235,7 @@ impl<S, D> TopDocsCollector<S, D> {
}
}
+ #[must_use]
pub fn disable_offset(self) -> Self {
Self {
perform_offset: false,
@@ -233,6 +243,7 @@ impl<S, D> TopDocsCollector<S, D> {
}
}
+ #[must_use]
pub fn with_deduplicator<D2: Deduplicator>(self, deduplicator: D2) -> TopDocsCollector<S, D2> {
TopDocsCollector {
deduplicator,
@@ -241,10 +252,12 @@ impl<S, D> TopDocsCollector<S, D> {
offset: self.offset,
perform_offset: self.perform_offset,
column_fields: self.column_fields,
+ filter: self.filter,
_phantom: PhantomData,
}
}
+ #[must_use]
pub fn with_column_fields(self, warmed_column_fields: WarmedColumnFields) -> Self {
Self {
column_fields: Some(ColumnFields::new(warmed_column_fields)),
@@ -252,12 +265,21 @@ impl<S, D> TopDocsCollector<S, D> {
}
}
+ #[must_use]
pub fn with_host_field<F: Field>(self, host_field: F) -> Self {
Self {
column_fields: Some(self.column_fields.unwrap().with_host_field(host_field)),
..self
}
}
+
+ #[must_use]
+ pub fn with_filter(self, filter: Box<dyn ColumnFieldFilter>) -> Self {
+ Self {
+ filter: Some(filter),
+ ..self
+ }
+ }
}
impl<S, D> TopDocsCollector<S, D>
@@ -304,6 +326,10 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDo
.u64_by_enum(host_field)
.unwrap()
}),
+ filter: self
+ .filter
+ .as_ref()
+ .map(|f| f.for_segment(&column_fields.warmed_column_fields)),
_deduplicator: PhantomData,
})
}
@@ -394,6 +420,7 @@ pub struct TopDocsSegmentCollector<S: DocumentScorer, D: Deduplicator> {
segment_ord: SegmentOrdinal,
scorer: S,
host_column: Option<Column<u64>>,
+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,
_deduplicator: PhantomData<D>,
}
@@ -407,6 +434,12 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> SegmentCollector
return;
}
+ if let Some(filter) = self.filter.as_ref() {
+ if filter.should_skip(doc) {
+ return;
+ }
+ }
+
let score = self.scorer.score(doc);
self.computer
.push(score, <D::Doc as DeduplicatorDoc>::new(self, doc));
diff --git a/crates/core/src/webgraph/query/filter/and.rs b/crates/core/src/webgraph/query/filter/and.rs
new file mode 100644
index 00000000..464aab29
--- /dev/null
+++ b/crates/core/src/webgraph/query/filter/and.rs
@@ -0,0 +1,188 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{query::Occur, DocId};
+
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+
+use super::{Filter, FilterEnum};
+
+#[derive(Default, Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct AndFilter {
+ filters: Vec<FilterEnum>,
+}
+
+impl AndFilter {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ #[must_use]
+ pub fn and<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+}
+
+impl From<AndFilter> for FilterEnum {
+ fn from(filter: AndFilter) -> Self {
+ Self::AndFilter(filter)
+ }
+}
+
+impl Filter for AndFilter {
+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {
+ let mut filters = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ if let Some(column_field_filter) = filter.column_field_filter() {
+ filters.push(column_field_filter);
+ }
+ }
+
+ if !filters.is_empty() {
+ Some(Box::new(AndColumnFieldFilter { filters }))
+ } else {
+ None
+ }
+ }
+
+ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {
+ let mut filters = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ if let Some(inverted_index_filter) = filter.inverted_index_filter() {
+ filters.push(inverted_index_filter);
+ }
+ }
+
+ if !filters.is_empty() {
+ Some(Box::new(AndInvertedIndexFilter { filters }))
+ } else {
+ None
+ }
+ }
+}
+
+pub struct AndColumnFieldFilter {
+ filters: Vec<Box<dyn super::ColumnFieldFilter>>,
+}
+
+impl super::ColumnFieldFilter for AndColumnFieldFilter {
+ fn for_segment(
+ &self,
+ column_fields: &WarmedColumnFields,
+ ) -> Box<dyn super::SegmentColumnFieldFilter> {
+ let mut filters = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ filters.push(filter.for_segment(column_fields));
+ }
+
+ Box::new(AndSegmentColumnFieldFilter { filters })
+ }
+}
+
+pub struct AndSegmentColumnFieldFilter {
+ filters: Vec<Box<dyn super::SegmentColumnFieldFilter>>,
+}
+
+impl super::SegmentColumnFieldFilter for AndSegmentColumnFieldFilter {
+ fn should_skip(&self, doc_id: DocId) -> bool {
+ for filter in self.filters.iter() {
+ if filter.should_skip(doc_id) {
+ return true;
+ }
+ }
+
+ false
+ }
+}
+
+pub struct AndInvertedIndexFilter {
+ filters: Vec<Box<dyn super::InvertedIndexFilter>>,
+}
+
+impl super::InvertedIndexFilter for AndInvertedIndexFilter {
+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
+ let mut queries = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ for (occur, query) in filter.query(searcher) {
+ queries.push((Occur::compose(Occur::Must, occur), query));
+ }
+ }
+
+ queries
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use file_store::temp::TempDir;
+
+ use crate::{
+ webgraph::{
+ query::{ForwardlinksQuery, TextFilter},
+ schema::ToUrl,
+ Edge, Node, Webgraph,
+ },
+ webpage::RelFlags,
+ };
+
+ pub fn test_edges() -> Vec<(Node, Node, String)> {
+ vec![
+ (Node::from("a.com"), Node::from("b.com/123"), String::new()),
+ (Node::from("a.com"), Node::from("b.dk/123"), String::new()),
+ (Node::from("a.com"), Node::from("b.com/321"), String::new()),
+ (Node::from("a.com"), Node::from("c.com"), String::new()),
+ ]
+ }
+
+ pub fn test_graph() -> (Webgraph, TempDir) {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ for (from, to, label) in test_edges() {
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ }
+
+ graph.commit().unwrap();
+
+ (graph, temp_dir)
+ }
+
+ #[test]
+ fn test_and_filter() {
+ let (graph, _temp_dir) = test_graph();
+ let node = Node::from("a.com");
+
+ let res = graph
+ .search(
+ &ForwardlinksQuery::new(node.id())
+ .filter(TextFilter::new(".com".to_string(), ToUrl))
+ .filter(TextFilter::new(".com/123".to_string(), ToUrl)),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("b.com/123").id());
+ }
+}
diff --git a/crates/core/src/webgraph/query/filter/mod.rs b/crates/core/src/webgraph/query/filter/mod.rs
new file mode 100644
index 00000000..91130675
--- /dev/null
+++ b/crates/core/src/webgraph/query/filter/mod.rs
@@ -0,0 +1,85 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod and;
+pub use and::AndFilter;
+
+mod or;
+pub use or::OrFilter;
+
+mod not;
+pub use not::NotFilter;
+
+mod text;
+pub use text::TextFilter;
+
+use tantivy::{query::Occur, DocId};
+
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+
+pub trait Filter:
+ Send
+ + Sync
+ + bincode::Encode
+ + bincode::Decode
+ + std::fmt::Debug
+ + Clone
+ + Into<FilterEnum>
+ + 'static
+{
+ fn column_field_filter(&self) -> Option<Box<dyn ColumnFieldFilter>>;
+ fn inverted_index_filter(&self) -> Option<Box<dyn InvertedIndexFilter>>;
+}
+
+#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]
+pub enum FilterEnum {
+ AndFilter(AndFilter),
+ OrFilter(OrFilter),
+ NotFilter(Box<NotFilter>),
+ TextFilter(TextFilter),
+}
+
+impl Filter for FilterEnum {
+ fn column_field_filter(&self) -> Option<Box<dyn ColumnFieldFilter>> {
+ match self {
+ FilterEnum::AndFilter(filter) => filter.column_field_filter(),
+ FilterEnum::OrFilter(filter) => filter.column_field_filter(),
+ FilterEnum::NotFilter(filter) => filter.column_field_filter(),
+ FilterEnum::TextFilter(filter) => filter.column_field_filter(),
+ }
+ }
+
+ fn inverted_index_filter(&self) -> Option<Box<dyn InvertedIndexFilter>> {
+ match self {
+ FilterEnum::AndFilter(filter) => filter.inverted_index_filter(),
+ FilterEnum::OrFilter(filter) => filter.inverted_index_filter(),
+ FilterEnum::NotFilter(filter) => filter.inverted_index_filter(),
+ FilterEnum::TextFilter(filter) => filter.inverted_index_filter(),
+ }
+ }
+}
+
+pub trait ColumnFieldFilter: Send + Sync + 'static {
+ fn for_segment(&self, column_fields: &WarmedColumnFields) -> Box<dyn SegmentColumnFieldFilter>;
+}
+
+pub trait SegmentColumnFieldFilter {
+ fn should_skip(&self, doc_id: DocId) -> bool;
+}
+
+pub trait InvertedIndexFilter {
+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;
+}
diff --git a/crates/core/src/webgraph/query/filter/not.rs b/crates/core/src/webgraph/query/filter/not.rs
new file mode 100644
index 00000000..966803ff
--- /dev/null
+++ b/crates/core/src/webgraph/query/filter/not.rs
@@ -0,0 +1,152 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+
+use super::{Filter, FilterEnum};
+use tantivy::{query::Occur, DocId};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct NotFilter {
+ filter: FilterEnum,
+}
+
+impl From<NotFilter> for FilterEnum {
+ fn from(filter: NotFilter) -> Self {
+ FilterEnum::NotFilter(Box::new(filter))
+ }
+}
+
+impl NotFilter {
+ pub fn new<F: Filter>(filter: F) -> Self {
+ Self {
+ filter: filter.into(),
+ }
+ }
+}
+
+impl Filter for NotFilter {
+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {
+ let filter = self.filter.column_field_filter()?;
+ Some(Box::new(NotColumnFieldFilter { filter }))
+ }
+
+ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {
+ let filter = self.filter.inverted_index_filter()?;
+ Some(Box::new(NotInvertedIndexFilter { filter }))
+ }
+}
+
+struct NotColumnFieldFilter {
+ filter: Box<dyn super::ColumnFieldFilter>,
+}
+
+impl super::ColumnFieldFilter for NotColumnFieldFilter {
+ fn for_segment(
+ &self,
+ column_fields: &WarmedColumnFields,
+ ) -> Box<dyn super::SegmentColumnFieldFilter> {
+ let filter = self.filter.for_segment(column_fields);
+ Box::new(NotSegmentColumnFieldFilter { filter })
+ }
+}
+
+struct NotSegmentColumnFieldFilter {
+ filter: Box<dyn super::SegmentColumnFieldFilter>,
+}
+
+impl super::SegmentColumnFieldFilter for NotSegmentColumnFieldFilter {
+ fn should_skip(&self, doc_id: DocId) -> bool {
+ !self.filter.should_skip(doc_id)
+ }
+}
+
+struct NotInvertedIndexFilter {
+ filter: Box<dyn super::InvertedIndexFilter>,
+}
+
+impl super::InvertedIndexFilter for NotInvertedIndexFilter {
+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
+ let mut queries = Vec::new();
+ for (occur, query) in self.filter.query(searcher) {
+ queries.push((Occur::compose(Occur::MustNot, occur), query));
+ }
+
+ queries
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use file_store::temp::TempDir;
+
+ use crate::{
+ webgraph::{
+ query::{FullForwardlinksQuery, TextFilter},
+ schema::ToUrl,
+ Edge, Node, Webgraph,
+ },
+ webpage::RelFlags,
+ };
+
+ use super::*;
+
+ pub fn test_edges() -> Vec<(Node, Node, String)> {
+ vec![
+ (Node::from("a.com"), Node::from("b.com/123"), String::new()),
+ (Node::from("a.com"), Node::from("b.dk/123"), String::new()),
+ (Node::from("a.com"), Node::from("b.com/321"), String::new()),
+ (Node::from("a.com"), Node::from("c.com"), String::new()),
+ ]
+ }
+
+ pub fn test_graph() -> (Webgraph, TempDir) {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ for (from, to, label) in test_edges() {
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ }
+
+ graph.commit().unwrap();
+
+ (graph, temp_dir)
+ }
+
+ #[test]
+ fn test_not_filter() {
+ let (graph, _temp_dir) = test_graph();
+ let node = Node::from("a.com");
+
+ let res = graph
+ .search(
+ &FullForwardlinksQuery::new(node)
+ .filter(NotFilter::new(TextFilter::new(".dk".to_string(), ToUrl))),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 3);
+ assert!(res.iter().all(|r| !r.to.as_str().contains(".dk")));
+ }
+}
diff --git a/crates/core/src/webgraph/query/filter/or.rs b/crates/core/src/webgraph/query/filter/or.rs
new file mode 100644
index 00000000..307150ec
--- /dev/null
+++ b/crates/core/src/webgraph/query/filter/or.rs
@@ -0,0 +1,197 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::{
+ query::{BooleanQuery, Occur},
+ DocId,
+};
+
+use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};
+
+use super::{Filter, FilterEnum};
+
+#[derive(Default, Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct OrFilter {
+ filters: Vec<FilterEnum>,
+}
+
+impl OrFilter {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ #[must_use]
+ pub fn or<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+}
+
+impl From<OrFilter> for FilterEnum {
+ fn from(filter: OrFilter) -> Self {
+ Self::OrFilter(filter)
+ }
+}
+
+impl Filter for OrFilter {
+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {
+ let mut filters = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ if let Some(column_field_filter) = filter.column_field_filter() {
+ filters.push(column_field_filter);
+ }
+ }
+
+ if !filters.is_empty() {
+ Some(Box::new(OrColumnFieldFilter { filters }))
+ } else {
+ None
+ }
+ }
+
+ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {
+ let mut filters = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ if let Some(inverted_index_filter) = filter.inverted_index_filter() {
+ filters.push(inverted_index_filter);
+ }
+ }
+
+ if !filters.is_empty() {
+ Some(Box::new(OrInvertedIndexFilter { filters }))
+ } else {
+ None
+ }
+ }
+}
+
+pub struct OrColumnFieldFilter {
+ filters: Vec<Box<dyn super::ColumnFieldFilter>>,
+}
+
+impl super::ColumnFieldFilter for OrColumnFieldFilter {
+ fn for_segment(
+ &self,
+ column_fields: &WarmedColumnFields,
+ ) -> Box<dyn super::SegmentColumnFieldFilter> {
+ let mut filters = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ filters.push(filter.for_segment(column_fields));
+ }
+
+ Box::new(OrSegmentColumnFieldFilter { filters })
+ }
+}
+
+pub struct OrSegmentColumnFieldFilter {
+ filters: Vec<Box<dyn super::SegmentColumnFieldFilter>>,
+}
+
+impl super::SegmentColumnFieldFilter for OrSegmentColumnFieldFilter {
+ fn should_skip(&self, doc_id: DocId) -> bool {
+ for filter in self.filters.iter() {
+ if !filter.should_skip(doc_id) {
+ return false;
+ }
+ }
+
+ true
+ }
+}
+
+pub struct OrInvertedIndexFilter {
+ filters: Vec<Box<dyn super::InvertedIndexFilter>>,
+}
+
+impl super::InvertedIndexFilter for OrInvertedIndexFilter {
+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
+ let mut queries = Vec::with_capacity(self.filters.len());
+ for filter in self.filters.iter() {
+ for (occur, query) in filter.query(searcher) {
+ queries.push((Occur::compose(Occur::Should, occur), query));
+ }
+ }
+
+ vec![(Occur::Should, Box::new(BooleanQuery::new(queries)))]
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use file_store::temp::TempDir;
+
+ use crate::{
+ webgraph::{
+ query::{FullForwardlinksQuery, TextFilter},
+ schema::ToUrl,
+ Edge, Node, Webgraph,
+ },
+ webpage::RelFlags,
+ };
+
+ use super::*;
+
+ pub fn test_edges() -> Vec<(Node, Node, String)> {
+ vec![
+ (Node::from("a.com"), Node::from("b.com/123"), String::new()),
+ (Node::from("a.com"), Node::from("b.dk/123"), String::new()),
+ (Node::from("a.com"), Node::from("b.se/123"), String::new()),
+ (Node::from("a.com"), Node::from("b.com/321"), String::new()),
+ (Node::from("a.com"), Node::from("c.com"), String::new()),
+ ]
+ }
+
+ pub fn test_graph() -> (Webgraph, TempDir) {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ for (from, to, label) in test_edges() {
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ }
+
+ graph.commit().unwrap();
+
+ (graph, temp_dir)
+ }
+
+ #[test]
+ fn test_or_filter() {
+ let (graph, _temp_dir) = test_graph();
+
+ let node = Node::from("a.com");
+
+ let res = graph
+ .search(
+ &FullForwardlinksQuery::new(node).filter(
+ OrFilter::new()
+ .or(TextFilter::new(".com".to_string(), ToUrl))
+ .or(TextFilter::new(".dk".to_string(), ToUrl)),
+ ),
+ )
+ .unwrap();
+
+ assert_eq!(res.len(), 4);
+ assert!(res.iter().all(|r| !r.to.as_str().contains(".se")));
+ }
+}
diff --git a/crates/core/src/webgraph/query/filter/text.rs b/crates/core/src/webgraph/query/filter/text.rs
new file mode 100644
index 00000000..1285329d
--- /dev/null
+++ b/crates/core/src/webgraph/query/filter/text.rs
@@ -0,0 +1,164 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use tantivy::query::Occur;
+
+use crate::webgraph::{
+ query::raw::PhraseOrTermQuery,
+ schema::{Field, FieldEnum},
+ searcher::Searcher,
+};
+
+use super::{Filter, FilterEnum};
+
+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+pub struct TextFilter {
+ text: String,
+ field: FieldEnum,
+}
+
+impl TextFilter {
+ pub fn new<F: Field>(text: String, field: F) -> Self {
+ Self {
+ text,
+ field: field.into(),
+ }
+ }
+}
+
+impl From<TextFilter> for FilterEnum {
+ fn from(filter: TextFilter) -> Self {
+ FilterEnum::TextFilter(filter)
+ }
+}
+
+impl Filter for TextFilter {
+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {
+ None
+ }
+
+ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {
+ Some(Box::new(TextInvertedIndexFilter {
+ text: self.text.clone(),
+ field: self.field,
+ }))
+ }
+}
+
+struct TextInvertedIndexFilter {
+ text: String,
+ field: FieldEnum,
+}
+
+impl super::InvertedIndexFilter for TextInvertedIndexFilter {
+ fn query(&self, _: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {
+ vec![(
+ Occur::Should,
+ Box::new(PhraseOrTermQuery::new(self.text.clone(), self.field)),
+ )]
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use file_store::temp::TempDir;
+
+ use crate::{
+ webgraph::{
+ query::{BacklinksQuery, ForwardlinksQuery, FullBacklinksQuery, FullForwardlinksQuery},
+ schema::{FromUrl, ToUrl},
+ Edge, Node, Webgraph,
+ },
+ webpage::RelFlags,
+ };
+
+ use super::*;
+
+ pub fn test_edges() -> Vec<(Node, Node, String)> {
+ vec![
+ (Node::from("a.com"), Node::from("b.com"), String::new()),
+ (Node::from("a.com"), Node::from("b.dk"), String::new()),
+ (Node::from("b.com"), Node::from("b.dk"), String::new()),
+ (Node::from("c.dk"), Node::from("b.dk"), String::new()),
+ (Node::from("c.com"), Node::from("a.com"), String::new()),
+ ]
+ }
+
+ pub fn test_graph() -> (Webgraph, TempDir) {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ for (from, to, label) in test_edges() {
+ graph
+ .insert(Edge {
+ from,
+ to,
+ rel_flags: RelFlags::default(),
+ label,
+ sort_score: 0.0,
+ })
+ .unwrap();
+ }
+
+ graph.commit().unwrap();
+
+ (graph, temp_dir)
+ }
+
+ #[test]
+ fn test_text_filter() {
+ let (graph, _temp_dir) = test_graph();
+ let node = Node::from("b.dk");
+
+ let filter = TextFilter::new(".dk".to_string(), FromUrl);
+
+ let res = graph
+ .search(&BacklinksQuery::new(node.id()).filter(filter))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("c.dk").id());
+
+ let filter = TextFilter::new(".dk".to_string(), FromUrl);
+
+ let res = graph
+ .search(&FullBacklinksQuery::new(node.clone()).filter(filter))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].from, Node::from("c.dk"));
+
+ let filter = TextFilter::new(".dk".to_string(), ToUrl);
+
+ let node = Node::from("a.com");
+
+ let res = graph
+ .search(&ForwardlinksQuery::new(node.id()).filter(filter))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("b.dk").id());
+
+ let filter = TextFilter::new(".dk".to_string(), ToUrl);
+
+ let res = graph
+ .search(&FullForwardlinksQuery::new(node.clone()).filter(filter))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert_eq!(res[0].to, Node::from("b.dk"));
+ }
+}
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
index 06b9fa06..75002fb8 100644
--- a/crates/core/src/webgraph/query/forwardlink.rs
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -19,7 +19,7 @@ use super::{
collector::{top_docs, HostDeduplicator, TopDocsCollector},
document_scorer::DefaultDocumentScorer,
raw::{host_links::HostLinksQuery, links::LinksQuery},
- Query,
+ AndFilter, Filter, FilterEnum, Query,
};
use crate::{
webgraph::{
@@ -30,12 +30,13 @@ use crate::{
Result,
};
-use tantivy::query::ShortCircuitQuery;
+use tantivy::query::{BooleanQuery, Occur, ShortCircuitQuery};
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct ForwardlinksQuery {
node: NodeID,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl ForwardlinksQuery {
@@ -43,6 +44,7 @@ impl ForwardlinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -50,6 +52,26 @@ impl ForwardlinksQuery {
Self {
node: self.node,
limit,
+ filters: self.filters,
+ }
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+
+ Some(filter)
}
}
}
@@ -60,25 +82,41 @@ impl Query for ForwardlinksQuery {
type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw =
+ Box::new(LinksQuery::new(self.node, FromId)) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
match self.limit {
- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, FromId)),
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node, FromId)),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node, FromId)),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
- .with_column_fields(searcher.warmed_column_fields().clone())
+ .with_column_fields(searcher.warmed_column_fields().clone());
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
@@ -129,6 +167,7 @@ impl Query for ForwardlinksQuery {
pub struct HostForwardlinksQuery {
node: NodeID,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl HostForwardlinksQuery {
@@ -136,6 +175,7 @@ impl HostForwardlinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -143,6 +183,24 @@ impl HostForwardlinksQuery {
Self {
node: self.node,
limit,
+ filters: self.filters,
+ }
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+ Some(filter)
}
}
}
@@ -154,41 +212,46 @@ impl Query for HostForwardlinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw = Box::new(HostLinksQuery::new(
+ self.node,
+ FromHostId,
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
match self.limit {
- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(
- self.node,
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node,
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node,
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
.with_deduplicator(HostDeduplicator)
.with_column_fields(searcher.warmed_column_fields().clone())
- .with_host_field(ToHostId)
+ .with_host_field(ToHostId);
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
@@ -241,6 +304,7 @@ impl Query for HostForwardlinksQuery {
pub struct FullForwardlinksQuery {
node: Node,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl FullForwardlinksQuery {
@@ -248,6 +312,7 @@ impl FullForwardlinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -255,6 +320,24 @@ impl FullForwardlinksQuery {
Self {
node: self.node,
limit,
+ filters: self.filters,
+ }
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+ Some(filter)
}
}
}
@@ -265,25 +348,41 @@ impl Query for FullForwardlinksQuery {
type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw =
+ Box::new(LinksQuery::new(self.node.id(), FromId)) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
match self.limit {
- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), FromId)),
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node.id(), FromId)),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(LinksQuery::new(self.node.id(), FromId)),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
- .with_column_fields(searcher.warmed_column_fields().clone())
+ .with_column_fields(searcher.warmed_column_fields().clone());
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
@@ -322,6 +421,7 @@ impl Query for FullForwardlinksQuery {
pub struct FullHostForwardlinksQuery {
node: Node,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl FullHostForwardlinksQuery {
@@ -329,6 +429,7 @@ impl FullHostForwardlinksQuery {
Self {
node,
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -336,6 +437,24 @@ impl FullHostForwardlinksQuery {
Self {
node: self.node,
limit,
+ filters: self.filters,
+ }
+ }
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+ Some(filter)
}
}
}
@@ -347,41 +466,46 @@ impl Query for FullHostForwardlinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let mut raw = Box::new(HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ FromHostId,
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ let filter = filter.query(searcher);
+ let mut queries = vec![(Occur::Must, raw)];
+ queries.extend(filter);
+ raw = Box::new(BooleanQuery::new(queries));
+ }
+
match self.limit {
- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ EdgeLimit::Unlimited => raw,
EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(
- Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ raw,
(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,
)),
}
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
.with_deduplicator(HostDeduplicator)
.with_column_fields(searcher.warmed_column_fields().clone())
- .with_host_field(ToHostId)
+ .with_host_field(ToHostId);
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rs
index 01a287e4..c7c450ae 100644
--- a/crates/core/src/webgraph/query/mod.rs
+++ b/crates/core/src/webgraph/query/mod.rs
@@ -18,6 +18,9 @@ use super::searcher::Searcher;
use crate::{ampc::dht::ShardId, Result};
pub use collector::Collector;
+pub mod filter;
+pub use filter::*;
+
pub mod backlink;
pub use backlink::*;
diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rs
index 4d4f0931..16bdf948 100644
--- a/crates/core/src/webgraph/query/raw/host_links.rs
+++ b/crates/core/src/webgraph/query/raw/host_links.rs
@@ -28,6 +28,7 @@ pub struct HostLinksQuery {
field: FieldEnum,
deduplication_field: FieldEnum,
warmed_column_fields: WarmedColumnFields,
+ skip_self_links: bool,
}
impl HostLinksQuery {
@@ -42,8 +43,14 @@ impl HostLinksQuery {
field: lookup_field.into(),
deduplication_field: deduplication_field.into(),
warmed_column_fields,
+ skip_self_links: true,
}
}
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
}
impl tantivy::query::Query for HostLinksQuery {
@@ -56,6 +63,7 @@ impl tantivy::query::Query for HostLinksQuery {
field: self.field,
deduplication_field: self.deduplication_field,
warmed_column_fields: self.warmed_column_fields.clone(),
+ skip_self_links: self.skip_self_links,
}))
}
}
@@ -65,6 +73,7 @@ struct HostLinksWeight {
field: FieldEnum,
deduplication_field: FieldEnum,
warmed_column_fields: WarmedColumnFields,
+ skip_self_links: bool,
}
impl tantivy::query::Weight for HostLinksWeight {
@@ -83,6 +92,7 @@ impl tantivy::query::Weight for HostLinksWeight {
self.deduplication_field,
&self.warmed_column_fields,
self.node.as_u64(),
+ self.skip_self_links,
) {
Ok(Some(scorer)) => Ok(Box::new(scorer)),
_ => Ok(Box::new(tantivy::query::EmptyScorer)),
@@ -103,6 +113,7 @@ struct HostLinksScorer {
host_id_column: Column<u64>,
last_host_id: Option<u64>,
self_host_id: u64,
+ skip_self_links: bool,
}
impl HostLinksScorer {
@@ -112,6 +123,7 @@ impl HostLinksScorer {
deduplication_field: FieldEnum,
warmed_column_fields: &WarmedColumnFields,
self_host_id: u64,
+ skip_self_links: bool,
) -> tantivy::Result<Option<Self>> {
let host_id_column = warmed_column_fields
.segment(&reader.segment_id())
@@ -143,6 +155,7 @@ impl HostLinksScorer {
host_id_column,
postings,
self_host_id,
+ skip_self_links,
}
}))
}
@@ -163,8 +176,8 @@ impl HostLinksScorer {
.unwrap_or(false)
}
- fn is_self_host(&self, doc: tantivy::DocId) -> bool {
- self.host_id(doc) == Some(self.self_host_id)
+ fn skip_self(&self, doc: tantivy::DocId) -> bool {
+ self.skip_self_links && self.host_id(doc) == Some(self.self_host_id)
}
}
@@ -188,7 +201,7 @@ impl tantivy::DocSet for HostLinksScorer {
self.postings.reset_cursor_start_block();
}
- while (self.has_seen_host(self.doc()) || self.is_self_host(self.doc()))
+ while (self.has_seen_host(self.doc()) || self.skip_self(self.doc()))
&& self.doc() != tantivy::TERMINATED
{
self.postings.advance();
diff --git a/crates/core/src/webgraph/tokenizer/mod.rs b/crates/core/src/webgraph/tokenizer/mod.rs
index 2d4f9e8c..121735e4 100644
--- a/crates/core/src/webgraph/tokenizer/mod.rs
+++ b/crates/core/src/webgraph/tokenizer/mod.rs
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-mod url;
+pub mod url;
use enum_dispatch::enum_dispatch;
use strum::{EnumDiscriminants, VariantArray};
diff --git a/crates/core/src/webgraph/tokenizer/url.rs b/crates/core/src/webgraph/tokenizer/url.rs
index 6ecf6884..b0b39d1b 100644
--- a/crates/core/src/webgraph/tokenizer/url.rs
+++ b/crates/core/src/webgraph/tokenizer/url.rs
@@ -29,32 +29,44 @@ struct ParsedUrl {
impl ParsedUrl {
fn parse(url: &str) -> Result<Self> {
- let url = url::Url::robust_parse(url)?;
- let scheme: VecDeque<String> = url
- .scheme()
- .split_preserve(|c| c == ':')
- .map(|s| {
- let mut s = s.to_string();
- s.push(':');
- s
- })
- .add_space_last()
- .collect();
-
- let mut host: VecDeque<String> = url
- .normalized_host()
- .unwrap_or_default()
- .split_preserve(|c| c == '.')
- .map(|s| s.to_string())
- .add_space_last()
- .collect();
-
- let mut path: VecDeque<_> = url
- .path()
- .split_preserve(|c| matches!(c, '/' | '-' | '_'))
- .filter(|s| !(*s).is_empty())
- .map(|s| s.to_string())
- .collect();
+ let parsed_url = url::Url::robust_parse(url)?;
+ let scheme: VecDeque<String> = if url.starts_with(&format!("{}:", parsed_url.scheme())) {
+ parsed_url
+ .scheme()
+ .split_preserve(|c| c == ':')
+ .map(|s| {
+ let mut s = s.to_string();
+ s.push(':');
+ s
+ })
+ .add_space_last()
+ .collect()
+ } else {
+ VecDeque::new()
+ };
+
+ let normalized_host = parsed_url.normalized_host().unwrap_or_default();
+
+ let mut host: VecDeque<String> = if normalized_host != parsed_url.scheme() {
+ normalized_host
+ .split_preserve(|c| c == '.')
+ .map(|s| s.to_string())
+ .add_space_last()
+ .collect()
+ } else {
+ VecDeque::new()
+ };
+
+ let mut path: VecDeque<_> = if normalized_host != parsed_url.scheme() {
+ parsed_url
+ .path()
+ .split_preserve(|c| matches!(c, '/' | '-' | '_'))
+ .filter(|s| !(*s).is_empty())
+ .map(|s| s.to_string())
+ .collect()
+ } else {
+ VecDeque::new()
+ };
if host.is_empty() {
if let Some(maybe_host) = path.pop_front() {
@@ -144,22 +156,7 @@ mod tests {
tokenize("https://example.com/path/to/resource"),
["https: ", "example", ".", "com ", "/", "path", "/", "to", "/", "resource"]
);
- assert_eq!(
- tokenize("example.com"),
- ["https: ", "example", ".", "com ", "/"]
- );
- assert_eq!(tokenize(".com"), ["https: ", ".", "com ", "/"]);
-
- assert_eq!(
- tokenize("mailto:[email protected]"),
- ["mailto: ", "example", ".", "com ", "hello"]
- );
-
- assert_eq!(
- tokenize("mailto:example.com"),
- ["mailto: ", "example", ".", "com "]
- );
-
- assert_eq!(tokenize("tel:+4512345678"), ["tel: ", "+4512345678 "]);
+ assert_eq!(tokenize("example.com"), ["example", ".", "com ", "/"]);
+ assert_eq!(tokenize(".com"), [".", "com ", "/"]);
}
}