web search done right
[webgraph] query filters
Mikkel Denkere4a8a9f2fe1a
diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex ac24d151..f590c097 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -169,12 +169,7 @@ impl Node {#[cfg(test)]impl From<String> for Node {fn from(name: String) -> Self {- let url = if name.contains("://") {- Url::parse(&name).unwrap()- } else {- Url::parse(&("http://".to_string() + name.as_str())).unwrap()- };-+ let url = Url::robust_parse(&name).unwrap();Node::from(&url)}}@@ -204,6 +199,11 @@ pub fn normalize_url(url: &Url) -> String {url.normalize_in_place();let scheme = url.scheme();++ if scheme != "http" && scheme != "https" {+ return url.to_string();+ }+let mut normalized = url.as_str().strip_prefix(scheme)@@ -222,3 +222,28 @@ pub fn normalize_url(url: &Url) -> String {normalized.to_lowercase()}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_normalize_url() {+ assert_eq!(+ normalize_url(&Url::robust_parse("https://www.example.com/").unwrap()),+ "example.com"+ );+ }++ #[test]+ fn test_host_node() {+ assert_eq!(+ Node::from("example.com").into_host(),+ Node::from("example.com")+ );+ assert_eq!(+ Node::from("https://example.com/123").into_host(),+ Node::from("example.com")+ );+ }+}diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rsindex f0988b4e..2963e33c 100644--- a/crates/core/src/webgraph/query/backlink.rs+++ b/crates/core/src/webgraph/query/backlink.rs@@ -14,13 +14,13 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::query::ShortCircuitQuery;+use tantivy::query::{BooleanQuery, Occur, ShortCircuitQuery};use super::{collector::{top_docs, HostDeduplicator, TopDocsCollector},document_scorer::DefaultDocumentScorer,raw::{HostLinksQuery, LinksQuery},- Query,+ AndFilter, Filter, FilterEnum, Query,};use crate::{ampc::dht::ShardId,@@ -97,6 +97,7 @@ pub fn fetch_edges(searcher: &Searcher, mut doc_ids: Vec<DocAddress>) -> Result<pub struct BacklinksQuery {node: NodeID,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl BacklinksQuery {@@ -104,6 +105,7 @@ impl BacklinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -111,6 +113,26 @@ impl BacklinksQuery {Self {node: self.node,limit,+ filters: self.filters,+ }+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();++ for f in self.filters.clone() {+ filter = filter.and(f);+ }++ Some(filter)}}}@@ -121,25 +143,40 @@ impl Query for BacklinksQuery {type IntermediateOutput = Vec<(f64, SmallEdge)>;type Output = Vec<SmallEdge>;- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }+match self.limit {- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, ToId)),+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node, ToId)),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node, ToId)),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset()- .with_column_fields(searcher.warmed_column_fields().clone())+ .with_column_fields(searcher.warmed_column_fields().clone());++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {@@ -190,6 +227,7 @@ impl Query for BacklinksQuery {pub struct HostBacklinksQuery {node: NodeID,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl HostBacklinksQuery {@@ -197,6 +235,7 @@ impl HostBacklinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -204,6 +243,25 @@ impl HostBacklinksQuery {self.limit = limit;self}++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();++ for f in self.filters.clone() {+ filter = filter.and(f);+ }++ Some(filter)+ }+ }}impl Query for HostBacklinksQuery {@@ -213,41 +271,46 @@ impl Query for HostBacklinksQuery {type Output = Vec<SmallEdge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw = Box::new(HostLinksQuery::new(+ self.node,+ ToHostId,+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }+match self.limit {- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(- self.node,- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node,- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node,- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset().with_deduplicator(HostDeduplicator).with_column_fields(searcher.warmed_column_fields().clone())- .with_host_field(FromHostId)+ .with_host_field(FromHostId);++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {@@ -300,6 +363,7 @@ impl Query for HostBacklinksQuery {pub struct FullBacklinksQuery {node: Node,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl FullBacklinksQuery {@@ -307,6 +371,7 @@ impl FullBacklinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -314,6 +379,26 @@ impl FullBacklinksQuery {Self {node: self.node,limit,+ filters: self.filters,+ }+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();++ for f in self.filters.clone() {+ filter = filter.and(f);+ }++ Some(filter)}}}@@ -324,25 +409,41 @@ impl Query for FullBacklinksQuery {type IntermediateOutput = Vec<Edge>;type Output = Vec<Edge>;- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw =+ Box::new(LinksQuery::new(self.node.id(), ToId)) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }+match self.limit {- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), ToId)),+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node.id(), ToId)),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node.id(), ToId)),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset()- .with_column_fields(searcher.warmed_column_fields().clone())+ .with_column_fields(searcher.warmed_column_fields().clone());++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {@@ -381,6 +482,9 @@ impl Query for FullBacklinksQuery {pub struct FullHostBacklinksQuery {node: Node,limit: EdgeLimit,+ filters: Vec<FilterEnum>,+ nodes_as_host: bool,+ skip_self_links: bool,}impl FullHostBacklinksQuery {@@ -388,13 +492,42 @@ impl FullHostBacklinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),+ nodes_as_host: true,+ skip_self_links: true,}}pub fn with_limit(self, limit: EdgeLimit) -> Self {- Self {- node: self.node,- limit,+ Self { limit, ..self }+ }++ pub fn nodes_as_host(mut self, nodes_as_host: bool) -> Self {+ self.nodes_as_host = nodes_as_host;+ self+ }++ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {+ self.skip_self_links = skip_self_links;+ self+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();++ for f in self.filters.clone() {+ filter = filter.and(f);+ }++ Some(filter)}}}@@ -406,41 +539,51 @@ impl Query for FullHostBacklinksQuery {type Output = Vec<Edge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- match self.limit {- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(+ let mut raw = Box::new(+ HostLinksQuery::new(self.node.clone().into_host().id(),ToHostId,FromHostId,searcher.warmed_column_fields().clone(),- )),+ )+ .skip_self_links(self.skip_self_links),+ ) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);++ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);++ raw = Box::new(BooleanQuery::new(queries));+ }++ match self.limit {+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset().with_deduplicator(HostDeduplicator).with_column_fields(searcher.warmed_column_fields().clone())- .with_host_field(FromHostId)+ .with_host_field(FromHostId);++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {@@ -457,6 +600,10 @@ impl Query for FullHostBacklinksQuery {let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();let edges = fetch_edges(searcher, docs)?;+ if !self.nodes_as_host {+ return Ok(edges);+ }+Ok(edges.into_iter().map(|e| Edge {@@ -484,10 +631,12 @@ impl Query for FullHostBacklinksQuery {edges}}+#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct BacklinksWithLabelsQuery {node: NodeID,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl BacklinksWithLabelsQuery {@@ -495,6 +644,7 @@ impl BacklinksWithLabelsQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -502,6 +652,26 @@ impl BacklinksWithLabelsQuery {Self {node: self.node,limit,+ filters: self.filters,+ }+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();++ for f in self.filters.clone() {+ filter = filter.and(f);+ }++ Some(filter)}}}@@ -512,15 +682,30 @@ impl Query for BacklinksWithLabelsQuery {type IntermediateOutput = Vec<(f64, SmallEdgeWithLabel)>;type Output = Vec<SmallEdgeWithLabel>;- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {- Box::new(LinksQuery::new(self.node, ToId))+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }++ raw}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset()- .with_column_fields(searcher.warmed_column_fields().clone())+ .with_column_fields(searcher.warmed_column_fields().clone());++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rsindex 7813554f..6ad0ae43 100644--- a/crates/core/src/webgraph/query/collector/top_docs.rs+++ b/crates/core/src/webgraph/query/collector/top_docs.rs@@ -26,7 +26,10 @@ use tantivy::{use crate::webgraph::{doc_address::DocAddress,- query::document_scorer::{DefaultDocumentScorer, DocumentScorer},+ query::{+ document_scorer::{DefaultDocumentScorer, DocumentScorer},+ ColumnFieldFilter, SegmentColumnFieldFilter,+ },schema::{Field, FieldEnum},EdgeLimit,};@@ -157,6 +160,7 @@ pub struct TopDocsCollector<S = DefaultDocumentScorer, D = NoDeduplicator> {perform_offset: bool,deduplicator: D,column_fields: Option<ColumnFields>,+ filter: Option<Box<dyn ColumnFieldFilter>>,_phantom: PhantomData<S>,}@@ -192,12 +196,14 @@ impl<S> TopDocsCollector<S, NoDeduplicator> {perform_offset: true,deduplicator: NoDeduplicator,column_fields: None,+ filter: None,_phantom: PhantomData,}}}impl<S, D> TopDocsCollector<S, D> {+ #[must_use]pub fn with_shard_id(self, shard_id: ShardId) -> Self {Self {shard_id: Some(shard_id),@@ -205,6 +211,7 @@ impl<S, D> TopDocsCollector<S, D> {}}+ #[must_use]pub fn with_offset(self, offset: usize) -> Self {Self {offset: Some(offset),@@ -212,6 +219,7 @@ impl<S, D> TopDocsCollector<S, D> {}}+ #[must_use]pub fn with_limit(self, limit: usize) -> Self {Self {limit: Some(limit),@@ -219,6 +227,7 @@ impl<S, D> TopDocsCollector<S, D> {}}+ #[must_use]pub fn enable_offset(self) -> Self {Self {perform_offset: true,@@ -226,6 +235,7 @@ impl<S, D> TopDocsCollector<S, D> {}}+ #[must_use]pub fn disable_offset(self) -> Self {Self {perform_offset: false,@@ -233,6 +243,7 @@ impl<S, D> TopDocsCollector<S, D> {}}+ #[must_use]pub fn with_deduplicator<D2: Deduplicator>(self, deduplicator: D2) -> TopDocsCollector<S, D2> {TopDocsCollector {deduplicator,@@ -241,10 +252,12 @@ impl<S, D> TopDocsCollector<S, D> {offset: self.offset,perform_offset: self.perform_offset,column_fields: self.column_fields,+ filter: self.filter,_phantom: PhantomData,}}+ #[must_use]pub fn with_column_fields(self, warmed_column_fields: WarmedColumnFields) -> Self {Self {column_fields: Some(ColumnFields::new(warmed_column_fields)),@@ -252,12 +265,21 @@ impl<S, D> TopDocsCollector<S, D> {}}+ #[must_use]pub fn with_host_field<F: Field>(self, host_field: F) -> Self {Self {column_fields: Some(self.column_fields.unwrap().with_host_field(host_field)),..self}}++ #[must_use]+ pub fn with_filter(self, filter: Box<dyn ColumnFieldFilter>) -> Self {+ Self {+ filter: Some(filter),+ ..self+ }+ }}impl<S, D> TopDocsCollector<S, D>@@ -304,6 +326,10 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDo.u64_by_enum(host_field).unwrap()}),+ filter: self+ .filter+ .as_ref()+ .map(|f| f.for_segment(&column_fields.warmed_column_fields)),_deduplicator: PhantomData,})}@@ -394,6 +420,7 @@ pub struct TopDocsSegmentCollector<S: DocumentScorer, D: Deduplicator> {segment_ord: SegmentOrdinal,scorer: S,host_column: Option<Column<u64>>,+ filter: Option<Box<dyn SegmentColumnFieldFilter>>,_deduplicator: PhantomData<D>,}@@ -407,6 +434,12 @@ impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> SegmentCollectorreturn;}+ if let Some(filter) = self.filter.as_ref() {+ if filter.should_skip(doc) {+ return;+ }+ }+let score = self.scorer.score(doc);self.computer.push(score, <D::Doc as DeduplicatorDoc>::new(self, doc));diff --git a/crates/core/src/webgraph/query/filter/and.rs b/crates/core/src/webgraph/query/filter/and.rsnew file mode 100644index 00000000..464aab29--- /dev/null+++ b/crates/core/src/webgraph/query/filter/and.rs@@ -0,0 +1,188 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::{query::Occur, DocId};++use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};++use super::{Filter, FilterEnum};++#[derive(Default, Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct AndFilter {+ filters: Vec<FilterEnum>,+}++impl AndFilter {+ pub fn new() -> Self {+ Self::default()+ }++ #[must_use]+ pub fn and<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }+}++impl From<AndFilter> for FilterEnum {+ fn from(filter: AndFilter) -> Self {+ Self::AndFilter(filter)+ }+}++impl Filter for AndFilter {+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {+ let mut filters = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ if let Some(column_field_filter) = filter.column_field_filter() {+ filters.push(column_field_filter);+ }+ }++ if !filters.is_empty() {+ Some(Box::new(AndColumnFieldFilter { filters }))+ } else {+ None+ }+ }++ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {+ let mut filters = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ if let Some(inverted_index_filter) = filter.inverted_index_filter() {+ filters.push(inverted_index_filter);+ }+ }++ if !filters.is_empty() {+ Some(Box::new(AndInvertedIndexFilter { filters }))+ } else {+ None+ }+ }+}++pub struct AndColumnFieldFilter {+ filters: Vec<Box<dyn super::ColumnFieldFilter>>,+}++impl super::ColumnFieldFilter for AndColumnFieldFilter {+ fn for_segment(+ &self,+ column_fields: &WarmedColumnFields,+ ) -> Box<dyn super::SegmentColumnFieldFilter> {+ let mut filters = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ filters.push(filter.for_segment(column_fields));+ }++ Box::new(AndSegmentColumnFieldFilter { filters })+ }+}++pub struct AndSegmentColumnFieldFilter {+ filters: Vec<Box<dyn super::SegmentColumnFieldFilter>>,+}++impl super::SegmentColumnFieldFilter for AndSegmentColumnFieldFilter {+ fn should_skip(&self, doc_id: DocId) -> bool {+ for filter in self.filters.iter() {+ if filter.should_skip(doc_id) {+ return true;+ }+ }++ false+ }+}++pub struct AndInvertedIndexFilter {+ filters: Vec<Box<dyn super::InvertedIndexFilter>>,+}++impl super::InvertedIndexFilter for AndInvertedIndexFilter {+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {+ let mut queries = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ for (occur, query) in filter.query(searcher) {+ queries.push((Occur::compose(Occur::Must, occur), query));+ }+ }++ queries+ }+}++#[cfg(test)]+mod tests {+ use file_store::temp::TempDir;++ use crate::{+ webgraph::{+ query::{ForwardlinksQuery, TextFilter},+ schema::ToUrl,+ Edge, Node, Webgraph,+ },+ webpage::RelFlags,+ };++ pub fn test_edges() -> Vec<(Node, Node, String)> {+ vec![+ (Node::from("a.com"), Node::from("b.com/123"), String::new()),+ (Node::from("a.com"), Node::from("b.dk/123"), String::new()),+ (Node::from("a.com"), Node::from("b.com/321"), String::new()),+ (Node::from("a.com"), Node::from("c.com"), String::new()),+ ]+ }++ pub fn test_graph() -> (Webgraph, TempDir) {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ for (from, to, label) in test_edges() {+ graph+ .insert(Edge {+ from,+ to,+ rel_flags: RelFlags::default(),+ label,+ sort_score: 0.0,+ })+ .unwrap();+ }++ graph.commit().unwrap();++ (graph, temp_dir)+ }++ #[test]+ fn test_and_filter() {+ let (graph, _temp_dir) = test_graph();+ let node = Node::from("a.com");++ let res = graph+ .search(+ &ForwardlinksQuery::new(node.id())+ .filter(TextFilter::new(".com".to_string(), ToUrl))+ .filter(TextFilter::new(".com/123".to_string(), ToUrl)),+ )+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("b.com/123").id());+ }+}diff --git a/crates/core/src/webgraph/query/filter/mod.rs b/crates/core/src/webgraph/query/filter/mod.rsnew file mode 100644index 00000000..91130675--- /dev/null+++ b/crates/core/src/webgraph/query/filter/mod.rs@@ -0,0 +1,85 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod and;+pub use and::AndFilter;++mod or;+pub use or::OrFilter;++mod not;+pub use not::NotFilter;++mod text;+pub use text::TextFilter;++use tantivy::{query::Occur, DocId};++use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};++pub trait Filter:+ Send+ + Sync+ + bincode::Encode+ + bincode::Decode+ + std::fmt::Debug+ + Clone+ + Into<FilterEnum>+ + 'static+{+ fn column_field_filter(&self) -> Option<Box<dyn ColumnFieldFilter>>;+ fn inverted_index_filter(&self) -> Option<Box<dyn InvertedIndexFilter>>;+}++#[derive(Clone, Debug, bincode::Encode, bincode::Decode)]+pub enum FilterEnum {+ AndFilter(AndFilter),+ OrFilter(OrFilter),+ NotFilter(Box<NotFilter>),+ TextFilter(TextFilter),+}++impl Filter for FilterEnum {+ fn column_field_filter(&self) -> Option<Box<dyn ColumnFieldFilter>> {+ match self {+ FilterEnum::AndFilter(filter) => filter.column_field_filter(),+ FilterEnum::OrFilter(filter) => filter.column_field_filter(),+ FilterEnum::NotFilter(filter) => filter.column_field_filter(),+ FilterEnum::TextFilter(filter) => filter.column_field_filter(),+ }+ }++ fn inverted_index_filter(&self) -> Option<Box<dyn InvertedIndexFilter>> {+ match self {+ FilterEnum::AndFilter(filter) => filter.inverted_index_filter(),+ FilterEnum::OrFilter(filter) => filter.inverted_index_filter(),+ FilterEnum::NotFilter(filter) => filter.inverted_index_filter(),+ FilterEnum::TextFilter(filter) => filter.inverted_index_filter(),+ }+ }+}++pub trait ColumnFieldFilter: Send + Sync + 'static {+ fn for_segment(&self, column_fields: &WarmedColumnFields) -> Box<dyn SegmentColumnFieldFilter>;+}++pub trait SegmentColumnFieldFilter {+ fn should_skip(&self, doc_id: DocId) -> bool;+}++pub trait InvertedIndexFilter {+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)>;+}diff --git a/crates/core/src/webgraph/query/filter/not.rs b/crates/core/src/webgraph/query/filter/not.rsnew file mode 100644index 00000000..966803ff--- /dev/null+++ b/crates/core/src/webgraph/query/filter/not.rs@@ -0,0 +1,152 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};++use super::{Filter, FilterEnum};+use tantivy::{query::Occur, DocId};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct NotFilter {+ filter: FilterEnum,+}++impl From<NotFilter> for FilterEnum {+ fn from(filter: NotFilter) -> Self {+ FilterEnum::NotFilter(Box::new(filter))+ }+}++impl NotFilter {+ pub fn new<F: Filter>(filter: F) -> Self {+ Self {+ filter: filter.into(),+ }+ }+}++impl Filter for NotFilter {+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {+ let filter = self.filter.column_field_filter()?;+ Some(Box::new(NotColumnFieldFilter { filter }))+ }++ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {+ let filter = self.filter.inverted_index_filter()?;+ Some(Box::new(NotInvertedIndexFilter { filter }))+ }+}++struct NotColumnFieldFilter {+ filter: Box<dyn super::ColumnFieldFilter>,+}++impl super::ColumnFieldFilter for NotColumnFieldFilter {+ fn for_segment(+ &self,+ column_fields: &WarmedColumnFields,+ ) -> Box<dyn super::SegmentColumnFieldFilter> {+ let filter = self.filter.for_segment(column_fields);+ Box::new(NotSegmentColumnFieldFilter { filter })+ }+}++struct NotSegmentColumnFieldFilter {+ filter: Box<dyn super::SegmentColumnFieldFilter>,+}++impl super::SegmentColumnFieldFilter for NotSegmentColumnFieldFilter {+ fn should_skip(&self, doc_id: DocId) -> bool {+ !self.filter.should_skip(doc_id)+ }+}++struct NotInvertedIndexFilter {+ filter: Box<dyn super::InvertedIndexFilter>,+}++impl super::InvertedIndexFilter for NotInvertedIndexFilter {+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {+ let mut queries = Vec::new();+ for (occur, query) in self.filter.query(searcher) {+ queries.push((Occur::compose(Occur::MustNot, occur), query));+ }++ queries+ }+}++#[cfg(test)]+mod tests {+ use file_store::temp::TempDir;++ use crate::{+ webgraph::{+ query::{FullForwardlinksQuery, TextFilter},+ schema::ToUrl,+ Edge, Node, Webgraph,+ },+ webpage::RelFlags,+ };++ use super::*;++ pub fn test_edges() -> Vec<(Node, Node, String)> {+ vec![+ (Node::from("a.com"), Node::from("b.com/123"), String::new()),+ (Node::from("a.com"), Node::from("b.dk/123"), String::new()),+ (Node::from("a.com"), Node::from("b.com/321"), String::new()),+ (Node::from("a.com"), Node::from("c.com"), String::new()),+ ]+ }++ pub fn test_graph() -> (Webgraph, TempDir) {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ for (from, to, label) in test_edges() {+ graph+ .insert(Edge {+ from,+ to,+ rel_flags: RelFlags::default(),+ label,+ sort_score: 0.0,+ })+ .unwrap();+ }++ graph.commit().unwrap();++ (graph, temp_dir)+ }++ #[test]+ fn test_not_filter() {+ let (graph, _temp_dir) = test_graph();+ let node = Node::from("a.com");++ let res = graph+ .search(+ &FullForwardlinksQuery::new(node)+ .filter(NotFilter::new(TextFilter::new(".dk".to_string(), ToUrl))),+ )+ .unwrap();++ assert_eq!(res.len(), 3);+ assert!(res.iter().all(|r| !r.to.as_str().contains(".dk")));+ }+}diff --git a/crates/core/src/webgraph/query/filter/or.rs b/crates/core/src/webgraph/query/filter/or.rsnew file mode 100644index 00000000..307150ec--- /dev/null+++ b/crates/core/src/webgraph/query/filter/or.rs@@ -0,0 +1,197 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::{+ query::{BooleanQuery, Occur},+ DocId,+};++use crate::webgraph::{searcher::Searcher, warmed_column_fields::WarmedColumnFields};++use super::{Filter, FilterEnum};++#[derive(Default, Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct OrFilter {+ filters: Vec<FilterEnum>,+}++impl OrFilter {+ pub fn new() -> Self {+ Self::default()+ }++ #[must_use]+ pub fn or<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }+}++impl From<OrFilter> for FilterEnum {+ fn from(filter: OrFilter) -> Self {+ Self::OrFilter(filter)+ }+}++impl Filter for OrFilter {+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {+ let mut filters = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ if let Some(column_field_filter) = filter.column_field_filter() {+ filters.push(column_field_filter);+ }+ }++ if !filters.is_empty() {+ Some(Box::new(OrColumnFieldFilter { filters }))+ } else {+ None+ }+ }++ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {+ let mut filters = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ if let Some(inverted_index_filter) = filter.inverted_index_filter() {+ filters.push(inverted_index_filter);+ }+ }++ if !filters.is_empty() {+ Some(Box::new(OrInvertedIndexFilter { filters }))+ } else {+ None+ }+ }+}++pub struct OrColumnFieldFilter {+ filters: Vec<Box<dyn super::ColumnFieldFilter>>,+}++impl super::ColumnFieldFilter for OrColumnFieldFilter {+ fn for_segment(+ &self,+ column_fields: &WarmedColumnFields,+ ) -> Box<dyn super::SegmentColumnFieldFilter> {+ let mut filters = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ filters.push(filter.for_segment(column_fields));+ }++ Box::new(OrSegmentColumnFieldFilter { filters })+ }+}++pub struct OrSegmentColumnFieldFilter {+ filters: Vec<Box<dyn super::SegmentColumnFieldFilter>>,+}++impl super::SegmentColumnFieldFilter for OrSegmentColumnFieldFilter {+ fn should_skip(&self, doc_id: DocId) -> bool {+ for filter in self.filters.iter() {+ if !filter.should_skip(doc_id) {+ return false;+ }+ }++ true+ }+}++pub struct OrInvertedIndexFilter {+ filters: Vec<Box<dyn super::InvertedIndexFilter>>,+}++impl super::InvertedIndexFilter for OrInvertedIndexFilter {+ fn query(&self, searcher: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {+ let mut queries = Vec::with_capacity(self.filters.len());+ for filter in self.filters.iter() {+ for (occur, query) in filter.query(searcher) {+ queries.push((Occur::compose(Occur::Should, occur), query));+ }+ }++ vec![(Occur::Should, Box::new(BooleanQuery::new(queries)))]+ }+}++#[cfg(test)]+mod tests {+ use file_store::temp::TempDir;++ use crate::{+ webgraph::{+ query::{FullForwardlinksQuery, TextFilter},+ schema::ToUrl,+ Edge, Node, Webgraph,+ },+ webpage::RelFlags,+ };++ use super::*;++ pub fn test_edges() -> Vec<(Node, Node, String)> {+ vec![+ (Node::from("a.com"), Node::from("b.com/123"), String::new()),+ (Node::from("a.com"), Node::from("b.dk/123"), String::new()),+ (Node::from("a.com"), Node::from("b.se/123"), String::new()),+ (Node::from("a.com"), Node::from("b.com/321"), String::new()),+ (Node::from("a.com"), Node::from("c.com"), String::new()),+ ]+ }++ pub fn test_graph() -> (Webgraph, TempDir) {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ for (from, to, label) in test_edges() {+ graph+ .insert(Edge {+ from,+ to,+ rel_flags: RelFlags::default(),+ label,+ sort_score: 0.0,+ })+ .unwrap();+ }++ graph.commit().unwrap();++ (graph, temp_dir)+ }++ #[test]+ fn test_or_filter() {+ let (graph, _temp_dir) = test_graph();++ let node = Node::from("a.com");++ let res = graph+ .search(+ &FullForwardlinksQuery::new(node).filter(+ OrFilter::new()+ .or(TextFilter::new(".com".to_string(), ToUrl))+ .or(TextFilter::new(".dk".to_string(), ToUrl)),+ ),+ )+ .unwrap();++ assert_eq!(res.len(), 4);+ assert!(res.iter().all(|r| !r.to.as_str().contains(".se")));+ }+}diff --git a/crates/core/src/webgraph/query/filter/text.rs b/crates/core/src/webgraph/query/filter/text.rsnew file mode 100644index 00000000..1285329d--- /dev/null+++ b/crates/core/src/webgraph/query/filter/text.rs@@ -0,0 +1,164 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use tantivy::query::Occur;++use crate::webgraph::{+ query::raw::PhraseOrTermQuery,+ schema::{Field, FieldEnum},+ searcher::Searcher,+};++use super::{Filter, FilterEnum};++#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+pub struct TextFilter {+ text: String,+ field: FieldEnum,+}++impl TextFilter {+ pub fn new<F: Field>(text: String, field: F) -> Self {+ Self {+ text,+ field: field.into(),+ }+ }+}++impl From<TextFilter> for FilterEnum {+ fn from(filter: TextFilter) -> Self {+ FilterEnum::TextFilter(filter)+ }+}++impl Filter for TextFilter {+ fn column_field_filter(&self) -> Option<Box<dyn super::ColumnFieldFilter>> {+ None+ }++ fn inverted_index_filter(&self) -> Option<Box<dyn super::InvertedIndexFilter>> {+ Some(Box::new(TextInvertedIndexFilter {+ text: self.text.clone(),+ field: self.field,+ }))+ }+}++struct TextInvertedIndexFilter {+ text: String,+ field: FieldEnum,+}++impl super::InvertedIndexFilter for TextInvertedIndexFilter {+ fn query(&self, _: &Searcher) -> Vec<(Occur, Box<dyn tantivy::query::Query>)> {+ vec![(+ Occur::Should,+ Box::new(PhraseOrTermQuery::new(self.text.clone(), self.field)),+ )]+ }+}++#[cfg(test)]+mod tests {+ use file_store::temp::TempDir;++ use crate::{+ webgraph::{+ query::{BacklinksQuery, ForwardlinksQuery, FullBacklinksQuery, FullForwardlinksQuery},+ schema::{FromUrl, ToUrl},+ Edge, Node, Webgraph,+ },+ webpage::RelFlags,+ };++ use super::*;++ pub fn test_edges() -> Vec<(Node, Node, String)> {+ vec![+ (Node::from("a.com"), Node::from("b.com"), String::new()),+ (Node::from("a.com"), Node::from("b.dk"), String::new()),+ (Node::from("b.com"), Node::from("b.dk"), String::new()),+ (Node::from("c.dk"), Node::from("b.dk"), String::new()),+ (Node::from("c.com"), Node::from("a.com"), String::new()),+ ]+ }++ pub fn test_graph() -> (Webgraph, TempDir) {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ for (from, to, label) in test_edges() {+ graph+ .insert(Edge {+ from,+ to,+ rel_flags: RelFlags::default(),+ label,+ sort_score: 0.0,+ })+ .unwrap();+ }++ graph.commit().unwrap();++ (graph, temp_dir)+ }++ #[test]+ fn test_text_filter() {+ let (graph, _temp_dir) = test_graph();+ let node = Node::from("b.dk");++ let filter = TextFilter::new(".dk".to_string(), FromUrl);++ let res = graph+ .search(&BacklinksQuery::new(node.id()).filter(filter))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].from, Node::from("c.dk").id());++ let filter = TextFilter::new(".dk".to_string(), FromUrl);++ let res = graph+ .search(&FullBacklinksQuery::new(node.clone()).filter(filter))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].from, Node::from("c.dk"));++ let filter = TextFilter::new(".dk".to_string(), ToUrl);++ let node = Node::from("a.com");++ let res = graph+ .search(&ForwardlinksQuery::new(node.id()).filter(filter))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("b.dk").id());++ let filter = TextFilter::new(".dk".to_string(), ToUrl);++ let res = graph+ .search(&FullForwardlinksQuery::new(node.clone()).filter(filter))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert_eq!(res[0].to, Node::from("b.dk"));+ }+}diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rsindex 06b9fa06..75002fb8 100644--- a/crates/core/src/webgraph/query/forwardlink.rs+++ b/crates/core/src/webgraph/query/forwardlink.rs@@ -19,7 +19,7 @@ use super::{collector::{top_docs, HostDeduplicator, TopDocsCollector},document_scorer::DefaultDocumentScorer,raw::{host_links::HostLinksQuery, links::LinksQuery},- Query,+ AndFilter, Filter, FilterEnum, Query,};use crate::{webgraph::{@@ -30,12 +30,13 @@ use crate::{Result,};-use tantivy::query::ShortCircuitQuery;+use tantivy::query::{BooleanQuery, Occur, ShortCircuitQuery};#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct ForwardlinksQuery {node: NodeID,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl ForwardlinksQuery {@@ -43,6 +44,7 @@ impl ForwardlinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -50,6 +52,26 @@ impl ForwardlinksQuery {Self {node: self.node,limit,+ filters: self.filters,+ }+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();++ for f in self.filters.clone() {+ filter = filter.and(f);+ }++ Some(filter)}}}@@ -60,25 +82,41 @@ impl Query for ForwardlinksQuery {type IntermediateOutput = Vec<(f64, SmallEdge)>;type Output = Vec<SmallEdge>;- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw =+ Box::new(LinksQuery::new(self.node, FromId)) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }+match self.limit {- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node, FromId)),+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node, FromId)),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node, FromId)),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset()- .with_column_fields(searcher.warmed_column_fields().clone())+ .with_column_fields(searcher.warmed_column_fields().clone());++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {@@ -129,6 +167,7 @@ impl Query for ForwardlinksQuery {pub struct HostForwardlinksQuery {node: NodeID,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl HostForwardlinksQuery {@@ -136,6 +175,7 @@ impl HostForwardlinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -143,6 +183,24 @@ impl HostForwardlinksQuery {Self {node: self.node,limit,+ filters: self.filters,+ }+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();+ for f in self.filters.clone() {+ filter = filter.and(f);+ }+ Some(filter)}}}@@ -154,41 +212,46 @@ impl Query for HostForwardlinksQuery {type Output = Vec<SmallEdge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw = Box::new(HostLinksQuery::new(+ self.node,+ FromHostId,+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }+match self.limit {- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(- self.node,- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node,- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node,- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset().with_deduplicator(HostDeduplicator).with_column_fields(searcher.warmed_column_fields().clone())- .with_host_field(ToHostId)+ .with_host_field(ToHostId);++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {@@ -241,6 +304,7 @@ impl Query for HostForwardlinksQuery {pub struct FullForwardlinksQuery {node: Node,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl FullForwardlinksQuery {@@ -248,6 +312,7 @@ impl FullForwardlinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -255,6 +320,24 @@ impl FullForwardlinksQuery {Self {node: self.node,limit,+ filters: self.filters,+ }+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();+ for f in self.filters.clone() {+ filter = filter.and(f);+ }+ Some(filter)}}}@@ -265,25 +348,41 @@ impl Query for FullForwardlinksQuery {type IntermediateOutput = Vec<Edge>;type Output = Vec<Edge>;- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw =+ Box::new(LinksQuery::new(self.node.id(), FromId)) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }+match self.limit {- EdgeLimit::Unlimited => Box::new(LinksQuery::new(self.node.id(), FromId)),+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node.id(), FromId)),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(LinksQuery::new(self.node.id(), FromId)),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset()- .with_column_fields(searcher.warmed_column_fields().clone())+ .with_column_fields(searcher.warmed_column_fields().clone());++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {@@ -322,6 +421,7 @@ impl Query for FullForwardlinksQuery {pub struct FullHostForwardlinksQuery {node: Node,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl FullHostForwardlinksQuery {@@ -329,6 +429,7 @@ impl FullHostForwardlinksQuery {Self {node,limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -336,6 +437,24 @@ impl FullHostForwardlinksQuery {Self {node: self.node,limit,+ filters: self.filters,+ }+ }++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();+ for f in self.filters.clone() {+ filter = filter.and(f);+ }+ Some(filter)}}}@@ -347,41 +466,46 @@ impl Query for FullHostForwardlinksQuery {type Output = Vec<Edge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let mut raw = Box::new(HostLinksQuery::new(+ self.node.clone().into_host().id(),+ FromHostId,+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )) as Box<dyn tantivy::query::Query>;++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ let filter = filter.query(searcher);+ let mut queries = vec![(Occur::Must, raw)];+ queries.extend(filter);+ raw = Box::new(BooleanQuery::new(queries));+ }+match self.limit {- EdgeLimit::Unlimited => Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),+ EdgeLimit::Unlimited => raw,EdgeLimit::Limit(limit) => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + top_docs::DEDUPLICATION_BUFFER) as u64,)),EdgeLimit::LimitAndOffset { limit, offset } => Box::new(ShortCircuitQuery::new(- Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),+ raw,(limit + offset + top_docs::DEDUPLICATION_BUFFER) as u64,)),}}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset().with_deduplicator(HostDeduplicator).with_column_fields(searcher.warmed_column_fields().clone())- .with_host_field(ToHostId)+ .with_host_field(ToHostId);++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rsindex 01a287e4..c7c450ae 100644--- a/crates/core/src/webgraph/query/mod.rs+++ b/crates/core/src/webgraph/query/mod.rs@@ -18,6 +18,9 @@ use super::searcher::Searcher;use crate::{ampc::dht::ShardId, Result};pub use collector::Collector;+pub mod filter;+pub use filter::*;+pub mod backlink;pub use backlink::*;diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rsindex 4d4f0931..16bdf948 100644--- a/crates/core/src/webgraph/query/raw/host_links.rs+++ b/crates/core/src/webgraph/query/raw/host_links.rs@@ -28,6 +28,7 @@ pub struct HostLinksQuery {field: FieldEnum,deduplication_field: FieldEnum,warmed_column_fields: WarmedColumnFields,+ skip_self_links: bool,}impl HostLinksQuery {@@ -42,8 +43,14 @@ impl HostLinksQuery {field: lookup_field.into(),deduplication_field: deduplication_field.into(),warmed_column_fields,+ skip_self_links: true,}}++ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {+ self.skip_self_links = skip_self_links;+ self+ }}impl tantivy::query::Query for HostLinksQuery {@@ -56,6 +63,7 @@ impl tantivy::query::Query for HostLinksQuery {field: self.field,deduplication_field: self.deduplication_field,warmed_column_fields: self.warmed_column_fields.clone(),+ skip_self_links: self.skip_self_links,}))}}@@ -65,6 +73,7 @@ struct HostLinksWeight {field: FieldEnum,deduplication_field: FieldEnum,warmed_column_fields: WarmedColumnFields,+ skip_self_links: bool,}impl tantivy::query::Weight for HostLinksWeight {@@ -83,6 +92,7 @@ impl tantivy::query::Weight for HostLinksWeight {self.deduplication_field,&self.warmed_column_fields,self.node.as_u64(),+ self.skip_self_links,) {Ok(Some(scorer)) => Ok(Box::new(scorer)),_ => Ok(Box::new(tantivy::query::EmptyScorer)),@@ -103,6 +113,7 @@ struct HostLinksScorer {host_id_column: Column<u64>,last_host_id: Option<u64>,self_host_id: u64,+ skip_self_links: bool,}impl HostLinksScorer {@@ -112,6 +123,7 @@ impl HostLinksScorer {deduplication_field: FieldEnum,warmed_column_fields: &WarmedColumnFields,self_host_id: u64,+ skip_self_links: bool,) -> tantivy::Result<Option<Self>> {let host_id_column = warmed_column_fields.segment(&reader.segment_id())@@ -143,6 +155,7 @@ impl HostLinksScorer {host_id_column,postings,self_host_id,+ skip_self_links,}}))}@@ -163,8 +176,8 @@ impl HostLinksScorer {.unwrap_or(false)}- fn is_self_host(&self, doc: tantivy::DocId) -> bool {- self.host_id(doc) == Some(self.self_host_id)+ fn skip_self(&self, doc: tantivy::DocId) -> bool {+ self.skip_self_links && self.host_id(doc) == Some(self.self_host_id)}}@@ -188,7 +201,7 @@ impl tantivy::DocSet for HostLinksScorer {self.postings.reset_cursor_start_block();}- while (self.has_seen_host(self.doc()) || self.is_self_host(self.doc()))+ while (self.has_seen_host(self.doc()) || self.skip_self(self.doc()))&& self.doc() != tantivy::TERMINATED{self.postings.advance();diff --git a/crates/core/src/webgraph/tokenizer/mod.rs b/crates/core/src/webgraph/tokenizer/mod.rsindex 2d4f9e8c..121735e4 100644--- a/crates/core/src/webgraph/tokenizer/mod.rs+++ b/crates/core/src/webgraph/tokenizer/mod.rs@@ -14,7 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-mod url;+pub mod url;use enum_dispatch::enum_dispatch;use strum::{EnumDiscriminants, VariantArray};diff --git a/crates/core/src/webgraph/tokenizer/url.rs b/crates/core/src/webgraph/tokenizer/url.rsindex 6ecf6884..b0b39d1b 100644--- a/crates/core/src/webgraph/tokenizer/url.rs+++ b/crates/core/src/webgraph/tokenizer/url.rs@@ -29,32 +29,44 @@ struct ParsedUrl {impl ParsedUrl {fn parse(url: &str) -> Result<Self> {- let url = url::Url::robust_parse(url)?;- let scheme: VecDeque<String> = url- .scheme()- .split_preserve(|c| c == ':')- .map(|s| {- let mut s = s.to_string();- s.push(':');- s- })- .add_space_last()- .collect();-- let mut host: VecDeque<String> = url- .normalized_host()- .unwrap_or_default()- .split_preserve(|c| c == '.')- .map(|s| s.to_string())- .add_space_last()- .collect();-- let mut path: VecDeque<_> = url- .path()- .split_preserve(|c| matches!(c, '/' | '-' | '_'))- .filter(|s| !(*s).is_empty())- .map(|s| s.to_string())- .collect();+ let parsed_url = url::Url::robust_parse(url)?;+ let scheme: VecDeque<String> = if url.starts_with(&format!("{}:", parsed_url.scheme())) {+ parsed_url+ .scheme()+ .split_preserve(|c| c == ':')+ .map(|s| {+ let mut s = s.to_string();+ s.push(':');+ s+ })+ .add_space_last()+ .collect()+ } else {+ VecDeque::new()+ };++ let normalized_host = parsed_url.normalized_host().unwrap_or_default();++ let mut host: VecDeque<String> = if normalized_host != parsed_url.scheme() {+ normalized_host+ .split_preserve(|c| c == '.')+ .map(|s| s.to_string())+ .add_space_last()+ .collect()+ } else {+ VecDeque::new()+ };++ let mut path: VecDeque<_> = if normalized_host != parsed_url.scheme() {+ parsed_url+ .path()+ .split_preserve(|c| matches!(c, '/' | '-' | '_'))+ .filter(|s| !(*s).is_empty())+ .map(|s| s.to_string())+ .collect()+ } else {+ VecDeque::new()+ };if host.is_empty() {if let Some(maybe_host) = path.pop_front() {@@ -144,22 +156,7 @@ mod tests {tokenize("https://example.com/path/to/resource"),["https: ", "example", ".", "com ", "/", "path", "/", "to", "/", "resource"]);- assert_eq!(- tokenize("example.com"),- ["https: ", "example", ".", "com ", "/"]- );- assert_eq!(tokenize(".com"), ["https: ", ".", "com ", "/"]);-- assert_eq!(- tokenize("mailto:[email protected]"),- ["mailto: ", "example", ".", "com ", "hello"]- );-- assert_eq!(- tokenize("mailto:example.com"),- ["mailto: ", "example", ".", "com "]- );-- assert_eq!(tokenize("tel:+4512345678"), ["tel: ", "+4512345678 "]);+ assert_eq!(tokenize("example.com"), ["example", ".", "com ", "/"]);+ assert_eq!(tokenize(".com"), [".", "com ", "/"]);}}