Skip to content

web search done right

[webgraph] control whether or not to skip self links in more queries

Mikkel Denker9d507debce58
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index 04c164d5..17d08920 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -365,6 +365,8 @@ pub struct FullBacklinksQuery {
node: Node,
limit: EdgeLimit,
filters: Vec<FilterEnum>,
+ skip_self_links: bool,
+ deduplicate: bool,
}
impl FullBacklinksQuery {
@@ -373,15 +375,23 @@ impl FullBacklinksQuery {
node,
limit: EdgeLimit::Unlimited,
filters: Vec::new(),
+ skip_self_links: true,
+ deduplicate: true,
}
}
pub fn with_limit(self, limit: EdgeLimit) -> Self {
- Self {
- node: self.node,
- limit,
- filters: self.filters,
- }
+ Self { limit, ..self }
+ }
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
+
+ pub fn deduplicate(mut self, deduplicate: bool) -> Self {
+ self.deduplicate = deduplicate;
+ self
}
pub fn filter<F: Filter>(mut self, filter: F) -> Self {
@@ -411,14 +421,18 @@ impl Query for FullBacklinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(
- LinksQuery::new(
- self.node.id(),
- ToId,
- searcher.warmed_column_fields().clone(),
- )
- .with_deduplication_field(FromId),
- ) as Box<dyn tantivy::query::Query>;
+ let mut q = LinksQuery::new(
+ self.node.id(),
+ ToId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .skip_self_links(self.skip_self_links);
+
+ if self.deduplicate {
+ q = q.with_deduplication_field(FromId);
+ }
+
+ let mut raw = Box::new(q) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rs
index 0c151e87..c6fe43e0 100644
--- a/crates/core/src/webgraph/query/between.rs
+++ b/crates/core/src/webgraph/query/between.rs
@@ -19,7 +19,7 @@ use tantivy::query::{BooleanQuery, Occur};
use crate::{
ampc::dht::ShardId,
webgraph::{
- schema::{FromHostId, FromId, ToHostId, ToId},
+ schema::{FromHostId, ToHostId},
searcher::Searcher,
Edge, EdgeLimit, Node, Query,
},
@@ -78,22 +78,16 @@ impl Query for FullLinksBetweenQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let from_query = Box::new(
- LinksQuery::new(
- self.from.id(),
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )
- .with_deduplication_field(ToId),
- ) as Box<dyn tantivy::query::Query>;
- let to_query = Box::new(
- LinksQuery::new(
- self.to.id(),
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )
- .with_deduplication_field(FromId),
- ) as Box<dyn tantivy::query::Query>;
+ let from_query = Box::new(LinksQuery::new(
+ self.from.id(),
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
+ let to_query = Box::new(LinksQuery::new(
+ self.to.id(),
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
let mut queries = vec![(Occur::Must, from_query), (Occur::Must, to_query)];
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
index 23500287..9abff18b 100644
--- a/crates/core/src/webgraph/query/forwardlink.rs
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -37,6 +37,8 @@ pub struct ForwardlinksQuery {
node: NodeID,
limit: EdgeLimit,
filters: Vec<FilterEnum>,
+ skip_self_links: bool,
+ deduplicate: bool,
}
impl ForwardlinksQuery {
@@ -45,15 +47,23 @@ impl ForwardlinksQuery {
node,
limit: EdgeLimit::Unlimited,
filters: Vec::new(),
+ skip_self_links: true,
+ deduplicate: true,
}
}
pub fn with_limit(self, limit: EdgeLimit) -> Self {
- Self {
- node: self.node,
- limit,
- filters: self.filters,
- }
+ Self { limit, ..self }
+ }
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
+
+ pub fn deduplicate(mut self, deduplicate: bool) -> Self {
+ self.deduplicate = deduplicate;
+ self
}
pub fn filter<F: Filter>(mut self, filter: F) -> Self {
@@ -83,11 +93,14 @@ impl Query for ForwardlinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(LinksQuery::new(
- self.node,
- FromId,
- searcher.warmed_column_fields().clone(),
- )) as Box<dyn tantivy::query::Query>;
+ let mut q = LinksQuery::new(self.node, FromId, searcher.warmed_column_fields().clone())
+ .skip_self_links(self.skip_self_links);
+
+ if self.deduplicate {
+ q = q.with_deduplication_field(ToId);
+ }
+
+ let mut raw = Box::new(q) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -171,6 +184,8 @@ pub struct HostForwardlinksQuery {
node: NodeID,
limit: EdgeLimit,
filters: Vec<FilterEnum>,
+ skip_self_links: bool,
+ deduplicate: bool,
}
impl HostForwardlinksQuery {
@@ -179,15 +194,23 @@ impl HostForwardlinksQuery {
node,
limit: EdgeLimit::Unlimited,
filters: Vec::new(),
+ skip_self_links: true,
+ deduplicate: true,
}
}
pub fn with_limit(self, limit: EdgeLimit) -> Self {
- Self {
- node: self.node,
- limit,
- filters: self.filters,
- }
+ Self { limit, ..self }
+ }
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
+
+ pub fn deduplicate(mut self, deduplicate: bool) -> Self {
+ self.deduplicate = deduplicate;
+ self
}
pub fn filter<F: Filter>(mut self, filter: F) -> Self {
@@ -215,14 +238,18 @@ impl Query for HostForwardlinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(
- LinksQuery::new(
- self.node,
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )
- .with_deduplication_field(ToHostId),
- ) as Box<dyn tantivy::query::Query>;
+ let mut q = LinksQuery::new(
+ self.node,
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .skip_self_links(self.skip_self_links);
+
+ if self.deduplicate {
+ q = q.with_deduplication_field(ToHostId);
+ }
+
+ let mut raw = Box::new(q) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -310,6 +337,8 @@ pub struct FullForwardlinksQuery {
node: Node,
limit: EdgeLimit,
filters: Vec<FilterEnum>,
+ skip_self_links: bool,
+ deduplicate: bool,
}
impl FullForwardlinksQuery {
@@ -318,15 +347,23 @@ impl FullForwardlinksQuery {
node,
limit: EdgeLimit::Unlimited,
filters: Vec::new(),
+ skip_self_links: true,
+ deduplicate: true,
}
}
pub fn with_limit(self, limit: EdgeLimit) -> Self {
- Self {
- node: self.node,
- limit,
- filters: self.filters,
- }
+ Self { limit, ..self }
+ }
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
+
+ pub fn deduplicate(mut self, deduplicate: bool) -> Self {
+ self.deduplicate = deduplicate;
+ self
}
pub fn filter<F: Filter>(mut self, filter: F) -> Self {
@@ -354,14 +391,18 @@ impl Query for FullForwardlinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(
- LinksQuery::new(
- self.node.id(),
- FromId,
- searcher.warmed_column_fields().clone(),
- )
- .with_deduplication_field(ToId),
- ) as Box<dyn tantivy::query::Query>;
+ let mut q = LinksQuery::new(
+ self.node.id(),
+ FromId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .skip_self_links(self.skip_self_links);
+
+ if self.deduplicate {
+ q = q.with_deduplication_field(ToId);
+ }
+
+ let mut raw = Box::new(q) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -433,6 +474,8 @@ pub struct FullHostForwardlinksQuery {
node: Node,
limit: EdgeLimit,
filters: Vec<FilterEnum>,
+ skip_self_links: bool,
+ deduplicate: bool,
}
impl FullHostForwardlinksQuery {
@@ -441,15 +484,23 @@ impl FullHostForwardlinksQuery {
node,
limit: EdgeLimit::Unlimited,
filters: Vec::new(),
+ skip_self_links: true,
+ deduplicate: true,
}
}
pub fn with_limit(self, limit: EdgeLimit) -> Self {
- Self {
- node: self.node,
- limit,
- filters: self.filters,
- }
+ Self { limit, ..self }
+ }
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
+
+ pub fn deduplicate(mut self, deduplicate: bool) -> Self {
+ self.deduplicate = deduplicate;
+ self
}
pub fn filter<F: Filter>(mut self, filter: F) -> Self {
@@ -477,14 +528,18 @@ impl Query for FullHostForwardlinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(
- LinksQuery::new(
- self.node.clone().into_host().id(),
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )
- .with_deduplication_field(ToHostId),
- ) as Box<dyn tantivy::query::Query>;
+ let mut q = LinksQuery::new(
+ self.node.clone().into_host().id(),
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .skip_self_links(self.skip_self_links);
+
+ if self.deduplicate {
+ q = q.with_deduplication_field(ToHostId);
+ }
+
+ let mut raw = Box::new(q) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);