Skip to content

web search done right

[webgraph] optionally deduplicate edges

Mikkel Denker6deb599d78e5
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index f2224563..0b22772a 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -492,6 +492,7 @@ pub struct FullHostBacklinksQuery {
filters: Vec<FilterEnum>,
nodes_as_host: bool,
skip_self_links: bool,
+ deduplicate: bool,
}
impl FullHostBacklinksQuery {
@@ -502,6 +503,7 @@ impl FullHostBacklinksQuery {
filters: Vec::new(),
nodes_as_host: true,
skip_self_links: true,
+ deduplicate: true,
}
}
@@ -519,6 +521,11 @@ impl FullHostBacklinksQuery {
self
}
+ pub fn deduplicate(mut self, deduplicate: bool) -> Self {
+ self.deduplicate = deduplicate;
+ self
+ }
+
pub fn filter<F: Filter>(mut self, filter: F) -> Self {
self.filters.push(filter.into());
self
@@ -546,15 +553,18 @@ impl Query for FullHostBacklinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(
- LinksQuery::new(
- self.node.clone().into_host().id(),
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )
- .skip_self_links(self.skip_self_links)
- .with_deduplication_field(FromHostId),
- ) as Box<dyn tantivy::query::Query>;
+ let mut query = LinksQuery::new(
+ self.node.clone().into_host().id(),
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .skip_self_links(self.skip_self_links);
+
+ if self.deduplicate {
+ query = query.with_deduplication_field(FromHostId);
+ }
+
+ let mut raw = Box::new(query) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);