web search done right
[webgraph] apply filters in between query
Mikkel Denker369ab36fe08a
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rsindex 2963e33c..b32bfcb7 100644--- a/crates/core/src/webgraph/query/backlink.rs+++ b/crates/core/src/webgraph/query/backlink.rs@@ -271,12 +271,10 @@ impl Query for HostBacklinksQuery {type Output = Vec<SmallEdge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw = Box::new(HostLinksQuery::new(- self.node,- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(+ HostLinksQuery::new(self.node, ToHostId, searcher.warmed_column_fields().clone())+ .with_deduplication_field(FromHostId),+ ) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);@@ -543,10 +541,10 @@ impl Query for FullHostBacklinksQuery {HostLinksQuery::new(self.node.clone().into_host().id(),ToHostId,- FromHostId,searcher.warmed_column_fields().clone(),)- .skip_self_links(self.skip_self_links),+ .skip_self_links(self.skip_self_links)+ .with_deduplication_field(FromHostId),) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rsindex 0c11b462..dd58f7d3 100644--- a/crates/core/src/webgraph/query/between.rs+++ b/crates/core/src/webgraph/query/between.rs@@ -14,33 +14,37 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::query::BooleanQuery;+use tantivy::query::{BooleanQuery, Occur};use crate::{ampc::dht::ShardId,webgraph::{- schema::{FromUrl, ToUrl},+ schema::{FromHostId, ToHostId},searcher::Searcher,Edge, EdgeLimit, Node, Query,},Result,};-use super::{collector::TopDocsCollector, fetch_edges, raw::PhraseOrTermQuery};+use super::{+ collector::TopDocsCollector, fetch_edges, raw::HostLinksQuery, AndFilter, Filter, FilterEnum,+};#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]pub struct FullLinksBetweenQuery {from: Node,to: Node,limit: EdgeLimit,+ filters: Vec<FilterEnum>,}impl FullLinksBetweenQuery {pub fn new(from: Node, to: Node) -> Self {Self {- from,- to,+ from: from.into_host(),+ to: to.into_host(),limit: EdgeLimit::Unlimited,+ filters: Vec::new(),}}@@ -48,6 +52,23 @@ impl FullLinksBetweenQuery {self.limit = limit;self}++ pub fn filter<F: Filter>(mut self, filter: F) -> Self {+ self.filters.push(filter.into());+ self+ }++ fn filter_as_and(&self) -> Option<AndFilter> {+ if self.filters.is_empty() {+ None+ } else {+ let mut filter = AndFilter::new();+ for f in self.filters.clone() {+ filter = filter.and(f);+ }+ Some(filter)+ }+ }}impl Query for FullLinksBetweenQuery {@@ -56,18 +77,40 @@ impl Query for FullLinksBetweenQuery {type IntermediateOutput = Vec<Edge>;type Output = Vec<Edge>;- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {- let from_query = PhraseOrTermQuery::new(self.from.as_str().to_string(), FromUrl);- let to_query = PhraseOrTermQuery::new(self.to.as_str().to_string(), ToUrl);+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {+ let from_query = Box::new(HostLinksQuery::new(+ self.from.id(),+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )) as Box<dyn tantivy::query::Query>;+ let to_query = Box::new(HostLinksQuery::new(+ self.to.id(),+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )) as Box<dyn tantivy::query::Query>;++ let mut queries = vec![(Occur::Must, from_query), (Occur::Must, to_query)];++ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {+ for (occur, q) in filter.query(searcher) {+ queries.push((occur, q));+ }+ }- BooleanQuery::intersection(vec![Box::new(from_query), Box::new(to_query)])+ BooleanQuery::new(queries)}fn collector(&self, searcher: &Searcher) -> Self::Collector {- TopDocsCollector::from(self.limit)+ let mut collector = TopDocsCollector::from(self.limit).with_shard_id(searcher.shard()).disable_offset()- .with_column_fields(searcher.warmed_column_fields().clone())+ .with_column_fields(searcher.warmed_column_fields().clone());++ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {+ collector = collector.with_filter(filter);+ }++ collector}fn remote_collector(&self) -> Self::Collector {diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rsindex 75002fb8..37c13337 100644--- a/crates/core/src/webgraph/query/forwardlink.rs+++ b/crates/core/src/webgraph/query/forwardlink.rs@@ -212,12 +212,14 @@ impl Query for HostForwardlinksQuery {type Output = Vec<SmallEdge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw = Box::new(HostLinksQuery::new(- self.node,- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(+ HostLinksQuery::new(+ self.node,+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )+ .with_deduplication_field(ToHostId),+ ) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);@@ -466,12 +468,14 @@ impl Query for FullHostForwardlinksQuery {type Output = Vec<Edge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw = Box::new(HostLinksQuery::new(- self.node.clone().into_host().id(),- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(+ HostLinksQuery::new(+ self.node.clone().into_host().id(),+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )+ .with_deduplication_field(ToHostId),+ ) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rsindex 7790f4a8..05e4b7dc 100644--- a/crates/core/src/webgraph/query/group_by.rs+++ b/crates/core/src/webgraph/query/group_by.rs@@ -98,18 +98,14 @@ impl Query for HostGroupSketchQuery {fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {let mut raw: Self::TantivyQuery = match self.node {- LinksDirection::From(node) => Box::new(raw::HostLinksQuery::new(- node,- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),- LinksDirection::To(node) => Box::new(raw::HostLinksQuery::new(- node,- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),+ LinksDirection::From(node) => Box::new(+ raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())+ .with_deduplication_field(ToHostId),+ ),+ LinksDirection::To(node) => Box::new(+ raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())+ .with_deduplication_field(FromHostId),+ ),};if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {@@ -220,18 +216,14 @@ impl Query for HostGroupQuery {fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {let mut raw: Self::TantivyQuery = match self.node {- LinksDirection::From(node) => Box::new(raw::HostLinksQuery::new(- node,- FromHostId,- ToHostId,- searcher.warmed_column_fields().clone(),- )),- LinksDirection::To(node) => Box::new(raw::HostLinksQuery::new(- node,- ToHostId,- FromHostId,- searcher.warmed_column_fields().clone(),- )),+ LinksDirection::From(node) => Box::new(+ raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())+ .with_deduplication_field(ToHostId),+ ),+ LinksDirection::To(node) => Box::new(+ raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())+ .with_deduplication_field(FromHostId),+ ),};if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rsindex 16bdf948..d8d393b4 100644--- a/crates/core/src/webgraph/query/raw/host_links.rs+++ b/crates/core/src/webgraph/query/raw/host_links.rs@@ -26,22 +26,21 @@ use crate::webgraph::{pub struct HostLinksQuery {node: NodeID,field: FieldEnum,- deduplication_field: FieldEnum,+ deduplication_field: Option<FieldEnum>,warmed_column_fields: WarmedColumnFields,skip_self_links: bool,}impl HostLinksQuery {- pub fn new<F: Field, FDedup: Field>(+ pub fn new<F: Field>(node: NodeID,lookup_field: F,- deduplication_field: FDedup,warmed_column_fields: WarmedColumnFields,) -> Self {Self {node,field: lookup_field.into(),- deduplication_field: deduplication_field.into(),+ deduplication_field: None,warmed_column_fields,skip_self_links: true,}@@ -51,6 +50,11 @@ impl HostLinksQuery {self.skip_self_links = skip_self_links;self}++ pub fn with_deduplication_field<F: Field>(mut self, deduplication_field: F) -> Self {+ self.deduplication_field = Some(deduplication_field.into());+ self+ }}impl tantivy::query::Query for HostLinksQuery {@@ -71,7 +75,7 @@ impl tantivy::query::Query for HostLinksQuery {struct HostLinksWeight {node: NodeID,field: FieldEnum,- deduplication_field: FieldEnum,+ deduplication_field: Option<FieldEnum>,warmed_column_fields: WarmedColumnFields,skip_self_links: bool,}@@ -110,7 +114,7 @@ impl tantivy::query::Weight for HostLinksWeight {struct HostLinksScorer {postings: SegmentPostings,- host_id_column: Column<u64>,+ host_id_column: Option<Column<u64>>,last_host_id: Option<u64>,self_host_id: u64,skip_self_links: bool,@@ -120,15 +124,17 @@ impl HostLinksScorer {fn new(reader: &tantivy::SegmentReader,term: tantivy::Term,- deduplication_field: FieldEnum,+ deduplication_field: Option<FieldEnum>,warmed_column_fields: &WarmedColumnFields,self_host_id: u64,skip_self_links: bool,) -> tantivy::Result<Option<Self>> {- let host_id_column = warmed_column_fields- .segment(&reader.segment_id())- .u64(deduplication_field)- .unwrap();+ let host_id_column = deduplication_field.map(|f| {+ warmed_column_fields+ .segment(&reader.segment_id())+ .u64(f)+ .unwrap()+ });Ok(reader.inverted_index(term.field())?@@ -137,14 +143,18 @@ impl HostLinksScorer {let mut last_host_id = None;if postings.doc() != tantivy::TERMINATED {- last_host_id = host_id_column.first(postings.doc());+ last_host_id = host_id_column+ .as_ref()+ .map(|c| c.first(postings.doc()).unwrap());}while postings.doc() != tantivy::TERMINATED && last_host_id == Some(self_host_id) {postings.advance();if postings.doc() != tantivy::TERMINATED {- last_host_id = host_id_column.first(postings.doc());+ last_host_id = host_id_column+ .as_ref()+ .map(|c| c.first(postings.doc()).unwrap());} else {last_host_id = None;}@@ -167,7 +177,7 @@ impl HostLinksScorer {return None;}- self.host_id_column.first(doc)+ self.host_id_column.as_ref().map(|c| c.first(doc).unwrap())}fn has_seen_host(&self, doc: tantivy::DocId) -> bool {