Skip to content

web search done right

[webgraph] apply filters in between query

Mikkel Denker369ab36fe08a
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index 2963e33c..b32bfcb7 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -271,12 +271,10 @@ impl Query for HostBacklinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(HostLinksQuery::new(
- self.node,
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(
+ HostLinksQuery::new(self.node, ToHostId, searcher.warmed_column_fields().clone())
+ .with_deduplication_field(FromHostId),
+ ) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -543,10 +541,10 @@ impl Query for FullHostBacklinksQuery {
HostLinksQuery::new(
self.node.clone().into_host().id(),
ToHostId,
- FromHostId,
searcher.warmed_column_fields().clone(),
)
- .skip_self_links(self.skip_self_links),
+ .skip_self_links(self.skip_self_links)
+ .with_deduplication_field(FromHostId),
) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rs
index 0c11b462..dd58f7d3 100644
--- a/crates/core/src/webgraph/query/between.rs
+++ b/crates/core/src/webgraph/query/between.rs
@@ -14,33 +14,37 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::query::BooleanQuery;
+use tantivy::query::{BooleanQuery, Occur};
use crate::{
ampc::dht::ShardId,
webgraph::{
- schema::{FromUrl, ToUrl},
+ schema::{FromHostId, ToHostId},
searcher::Searcher,
Edge, EdgeLimit, Node, Query,
},
Result,
};
-use super::{collector::TopDocsCollector, fetch_edges, raw::PhraseOrTermQuery};
+use super::{
+ collector::TopDocsCollector, fetch_edges, raw::HostLinksQuery, AndFilter, Filter, FilterEnum,
+};
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
pub struct FullLinksBetweenQuery {
from: Node,
to: Node,
limit: EdgeLimit,
+ filters: Vec<FilterEnum>,
}
impl FullLinksBetweenQuery {
pub fn new(from: Node, to: Node) -> Self {
Self {
- from,
- to,
+ from: from.into_host(),
+ to: to.into_host(),
limit: EdgeLimit::Unlimited,
+ filters: Vec::new(),
}
}
@@ -48,6 +52,23 @@ impl FullLinksBetweenQuery {
self.limit = limit;
self
}
+
+ pub fn filter<F: Filter>(mut self, filter: F) -> Self {
+ self.filters.push(filter.into());
+ self
+ }
+
+ fn filter_as_and(&self) -> Option<AndFilter> {
+ if self.filters.is_empty() {
+ None
+ } else {
+ let mut filter = AndFilter::new();
+ for f in self.filters.clone() {
+ filter = filter.and(f);
+ }
+ Some(filter)
+ }
+ }
}
impl Query for FullLinksBetweenQuery {
@@ -56,18 +77,40 @@ impl Query for FullLinksBetweenQuery {
type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
- fn tantivy_query(&self, _: &Searcher) -> Self::TantivyQuery {
- let from_query = PhraseOrTermQuery::new(self.from.as_str().to_string(), FromUrl);
- let to_query = PhraseOrTermQuery::new(self.to.as_str().to_string(), ToUrl);
+ fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
+ let from_query = Box::new(HostLinksQuery::new(
+ self.from.id(),
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
+ let to_query = Box::new(HostLinksQuery::new(
+ self.to.id(),
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
+
+ let mut queries = vec![(Occur::Must, from_query), (Occur::Must, to_query)];
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
+ for (occur, q) in filter.query(searcher) {
+ queries.push((occur, q));
+ }
+ }
- BooleanQuery::intersection(vec![Box::new(from_query), Box::new(to_query)])
+ BooleanQuery::new(queries)
}
fn collector(&self, searcher: &Searcher) -> Self::Collector {
- TopDocsCollector::from(self.limit)
+ let mut collector = TopDocsCollector::from(self.limit)
.with_shard_id(searcher.shard())
.disable_offset()
- .with_column_fields(searcher.warmed_column_fields().clone())
+ .with_column_fields(searcher.warmed_column_fields().clone());
+
+ if let Some(filter) = self.filter_as_and().and_then(|f| f.column_field_filter()) {
+ collector = collector.with_filter(filter);
+ }
+
+ collector
}
fn remote_collector(&self) -> Self::Collector {
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
index 75002fb8..37c13337 100644
--- a/crates/core/src/webgraph/query/forwardlink.rs
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -212,12 +212,14 @@ impl Query for HostForwardlinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(HostLinksQuery::new(
- self.node,
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(
+ HostLinksQuery::new(
+ self.node,
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .with_deduplication_field(ToHostId),
+ ) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -466,12 +468,14 @@ impl Query for FullHostForwardlinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(HostLinksQuery::new(
- self.node.clone().into_host().id(),
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(
+ HostLinksQuery::new(
+ self.node.clone().into_host().id(),
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .with_deduplication_field(ToHostId),
+ ) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rs
index 7790f4a8..05e4b7dc 100644
--- a/crates/core/src/webgraph/query/group_by.rs
+++ b/crates/core/src/webgraph/query/group_by.rs
@@ -98,18 +98,14 @@ impl Query for HostGroupSketchQuery {
fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
let mut raw: Self::TantivyQuery = match self.node {
- LinksDirection::From(node) => Box::new(raw::HostLinksQuery::new(
- node,
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
- LinksDirection::To(node) => Box::new(raw::HostLinksQuery::new(
- node,
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ LinksDirection::From(node) => Box::new(
+ raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())
+ .with_deduplication_field(ToHostId),
+ ),
+ LinksDirection::To(node) => Box::new(
+ raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())
+ .with_deduplication_field(FromHostId),
+ ),
};
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
@@ -220,18 +216,14 @@ impl Query for HostGroupQuery {
fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
let mut raw: Self::TantivyQuery = match self.node {
- LinksDirection::From(node) => Box::new(raw::HostLinksQuery::new(
- node,
- FromHostId,
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )),
- LinksDirection::To(node) => Box::new(raw::HostLinksQuery::new(
- node,
- ToHostId,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )),
+ LinksDirection::From(node) => Box::new(
+ raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())
+ .with_deduplication_field(ToHostId),
+ ),
+ LinksDirection::To(node) => Box::new(
+ raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())
+ .with_deduplication_field(FromHostId),
+ ),
};
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rs
index 16bdf948..d8d393b4 100644
--- a/crates/core/src/webgraph/query/raw/host_links.rs
+++ b/crates/core/src/webgraph/query/raw/host_links.rs
@@ -26,22 +26,21 @@ use crate::webgraph::{
pub struct HostLinksQuery {
node: NodeID,
field: FieldEnum,
- deduplication_field: FieldEnum,
+ deduplication_field: Option<FieldEnum>,
warmed_column_fields: WarmedColumnFields,
skip_self_links: bool,
}
impl HostLinksQuery {
- pub fn new<F: Field, FDedup: Field>(
+ pub fn new<F: Field>(
node: NodeID,
lookup_field: F,
- deduplication_field: FDedup,
warmed_column_fields: WarmedColumnFields,
) -> Self {
Self {
node,
field: lookup_field.into(),
- deduplication_field: deduplication_field.into(),
+ deduplication_field: None,
warmed_column_fields,
skip_self_links: true,
}
@@ -51,6 +50,11 @@ impl HostLinksQuery {
self.skip_self_links = skip_self_links;
self
}
+
+ pub fn with_deduplication_field<F: Field>(mut self, deduplication_field: F) -> Self {
+ self.deduplication_field = Some(deduplication_field.into());
+ self
+ }
}
impl tantivy::query::Query for HostLinksQuery {
@@ -71,7 +75,7 @@ impl tantivy::query::Query for HostLinksQuery {
struct HostLinksWeight {
node: NodeID,
field: FieldEnum,
- deduplication_field: FieldEnum,
+ deduplication_field: Option<FieldEnum>,
warmed_column_fields: WarmedColumnFields,
skip_self_links: bool,
}
@@ -110,7 +114,7 @@ impl tantivy::query::Weight for HostLinksWeight {
struct HostLinksScorer {
postings: SegmentPostings,
- host_id_column: Column<u64>,
+ host_id_column: Option<Column<u64>>,
last_host_id: Option<u64>,
self_host_id: u64,
skip_self_links: bool,
@@ -120,15 +124,17 @@ impl HostLinksScorer {
fn new(
reader: &tantivy::SegmentReader,
term: tantivy::Term,
- deduplication_field: FieldEnum,
+ deduplication_field: Option<FieldEnum>,
warmed_column_fields: &WarmedColumnFields,
self_host_id: u64,
skip_self_links: bool,
) -> tantivy::Result<Option<Self>> {
- let host_id_column = warmed_column_fields
- .segment(&reader.segment_id())
- .u64(deduplication_field)
- .unwrap();
+ let host_id_column = deduplication_field.map(|f| {
+ warmed_column_fields
+ .segment(&reader.segment_id())
+ .u64(f)
+ .unwrap()
+ });
Ok(reader
.inverted_index(term.field())?
@@ -137,14 +143,18 @@ impl HostLinksScorer {
let mut last_host_id = None;
if postings.doc() != tantivy::TERMINATED {
- last_host_id = host_id_column.first(postings.doc());
+ last_host_id = host_id_column
+ .as_ref()
+ .map(|c| c.first(postings.doc()).unwrap());
}
while postings.doc() != tantivy::TERMINATED && last_host_id == Some(self_host_id) {
postings.advance();
if postings.doc() != tantivy::TERMINATED {
- last_host_id = host_id_column.first(postings.doc());
+ last_host_id = host_id_column
+ .as_ref()
+ .map(|c| c.first(postings.doc()).unwrap());
} else {
last_host_id = None;
}
@@ -167,7 +177,7 @@ impl HostLinksScorer {
return None;
}
- self.host_id_column.first(doc)
+ self.host_id_column.as_ref().map(|c| c.first(doc).unwrap())
}
fn has_seen_host(&self, doc: tantivy::DocId) -> bool {