Skip to content

web search done right

[webgraph] rename HostLinkQuery to LinksQuery

Mikkel Denkere94cd2de11d5
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index b32bfcb7..f2224563 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -19,7 +19,7 @@ use tantivy::query::{BooleanQuery, Occur, ShortCircuitQuery};
use super::{
collector::{top_docs, HostDeduplicator, TopDocsCollector},
document_scorer::DefaultDocumentScorer,
- raw::{HostLinksQuery, LinksQuery},
+ raw::LinksQuery,
AndFilter, Filter, FilterEnum, Query,
};
use crate::{
@@ -144,7 +144,10 @@ impl Query for BacklinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(
+ LinksQuery::new(self.node, ToId, searcher.warmed_column_fields().clone())
+ .with_deduplication_field(FromId),
+ ) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -272,7 +275,7 @@ impl Query for HostBacklinksQuery {
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
let mut raw = Box::new(
- HostLinksQuery::new(self.node, ToHostId, searcher.warmed_column_fields().clone())
+ LinksQuery::new(self.node, ToHostId, searcher.warmed_column_fields().clone())
.with_deduplication_field(FromHostId),
) as Box<dyn tantivy::query::Query>;
@@ -408,8 +411,14 @@ impl Query for FullBacklinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw =
- Box::new(LinksQuery::new(self.node.id(), ToId)) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(
+ LinksQuery::new(
+ self.node.id(),
+ ToId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .with_deduplication_field(FromId),
+ ) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -538,7 +547,7 @@ impl Query for FullHostBacklinksQuery {
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
let mut raw = Box::new(
- HostLinksQuery::new(
+ LinksQuery::new(
self.node.clone().into_host().id(),
ToHostId,
searcher.warmed_column_fields().clone(),
@@ -681,7 +690,10 @@ impl Query for BacklinksWithLabelsQuery {
type Output = Vec<SmallEdgeWithLabel>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(
+ LinksQuery::new(self.node, ToId, searcher.warmed_column_fields().clone())
+ .with_deduplication_field(FromId),
+ ) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rs
index dd58f7d3..94aabbef 100644
--- a/crates/core/src/webgraph/query/between.rs
+++ b/crates/core/src/webgraph/query/between.rs
@@ -19,7 +19,7 @@ use tantivy::query::{BooleanQuery, Occur};
use crate::{
ampc::dht::ShardId,
webgraph::{
- schema::{FromHostId, ToHostId},
+ schema::{FromHostId, FromId, ToHostId, ToId},
searcher::Searcher,
Edge, EdgeLimit, Node, Query,
},
@@ -27,7 +27,7 @@ use crate::{
};
use super::{
- collector::TopDocsCollector, fetch_edges, raw::HostLinksQuery, AndFilter, Filter, FilterEnum,
+ collector::TopDocsCollector, fetch_edges, raw::LinksQuery, AndFilter, Filter, FilterEnum,
};
#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
@@ -78,16 +78,22 @@ impl Query for FullLinksBetweenQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let from_query = Box::new(HostLinksQuery::new(
- self.from.id(),
- FromHostId,
- searcher.warmed_column_fields().clone(),
- )) as Box<dyn tantivy::query::Query>;
- let to_query = Box::new(HostLinksQuery::new(
- self.to.id(),
- ToHostId,
- searcher.warmed_column_fields().clone(),
- )) as Box<dyn tantivy::query::Query>;
+ let from_query = Box::new(
+ LinksQuery::new(
+ self.from.id(),
+ FromHostId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .with_deduplication_field(ToId),
+ ) as Box<dyn tantivy::query::Query>;
+ let to_query = Box::new(
+ LinksQuery::new(
+ self.to.id(),
+ ToHostId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .with_deduplication_field(FromId),
+ ) as Box<dyn tantivy::query::Query>;
let mut queries = vec![(Occur::Must, from_query), (Occur::Must, to_query)];
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
index 37c13337..23500287 100644
--- a/crates/core/src/webgraph/query/forwardlink.rs
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -18,7 +18,7 @@ use super::{
backlink::{fetch_edges, fetch_small_edges},
collector::{top_docs, HostDeduplicator, TopDocsCollector},
document_scorer::DefaultDocumentScorer,
- raw::{host_links::HostLinksQuery, links::LinksQuery},
+ raw::LinksQuery,
AndFilter, Filter, FilterEnum, Query,
};
use crate::{
@@ -83,8 +83,11 @@ impl Query for ForwardlinksQuery {
type Output = Vec<SmallEdge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw =
- Box::new(LinksQuery::new(self.node, FromId)) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(LinksQuery::new(
+ self.node,
+ FromId,
+ searcher.warmed_column_fields().clone(),
+ )) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -213,7 +216,7 @@ impl Query for HostForwardlinksQuery {
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
let mut raw = Box::new(
- HostLinksQuery::new(
+ LinksQuery::new(
self.node,
FromHostId,
searcher.warmed_column_fields().clone(),
@@ -351,8 +354,14 @@ impl Query for FullForwardlinksQuery {
type Output = Vec<Edge>;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
- let mut raw =
- Box::new(LinksQuery::new(self.node.id(), FromId)) as Box<dyn tantivy::query::Query>;
+ let mut raw = Box::new(
+ LinksQuery::new(
+ self.node.id(),
+ FromId,
+ searcher.warmed_column_fields().clone(),
+ )
+ .with_deduplication_field(ToId),
+ ) as Box<dyn tantivy::query::Query>;
if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {
let filter = filter.query(searcher);
@@ -469,7 +478,7 @@ impl Query for FullHostForwardlinksQuery {
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {
let mut raw = Box::new(
- HostLinksQuery::new(
+ LinksQuery::new(
self.node.clone().into_host().id(),
FromHostId,
searcher.warmed_column_fields().clone(),
diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rs
index 05e4b7dc..2eedb609 100644
--- a/crates/core/src/webgraph/query/group_by.rs
+++ b/crates/core/src/webgraph/query/group_by.rs
@@ -99,11 +99,11 @@ impl Query for HostGroupSketchQuery {
fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
let mut raw: Self::TantivyQuery = match self.node {
LinksDirection::From(node) => Box::new(
- raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())
+ raw::LinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())
.with_deduplication_field(ToHostId),
),
LinksDirection::To(node) => Box::new(
- raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())
+ raw::LinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())
.with_deduplication_field(FromHostId),
),
};
@@ -217,11 +217,11 @@ impl Query for HostGroupQuery {
fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {
let mut raw: Self::TantivyQuery = match self.node {
LinksDirection::From(node) => Box::new(
- raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())
+ raw::LinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())
.with_deduplication_field(ToHostId),
),
LinksDirection::To(node) => Box::new(
- raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())
+ raw::LinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())
.with_deduplication_field(FromHostId),
),
};
diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rs
deleted file mode 100644
index d8d393b4..00000000
--- a/crates/core/src/webgraph/query/raw/host_links.rs
+++ /dev/null
@@ -1,363 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2024 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};
-
-use crate::webgraph::{
- schema::{Field, FieldEnum},
- warmed_column_fields::WarmedColumnFields,
- NodeID,
-};
-
-#[derive(Debug, Clone)]
-pub struct HostLinksQuery {
- node: NodeID,
- field: FieldEnum,
- deduplication_field: Option<FieldEnum>,
- warmed_column_fields: WarmedColumnFields,
- skip_self_links: bool,
-}
-
-impl HostLinksQuery {
- pub fn new<F: Field>(
- node: NodeID,
- lookup_field: F,
- warmed_column_fields: WarmedColumnFields,
- ) -> Self {
- Self {
- node,
- field: lookup_field.into(),
- deduplication_field: None,
- warmed_column_fields,
- skip_self_links: true,
- }
- }
-
- pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
- self.skip_self_links = skip_self_links;
- self
- }
-
- pub fn with_deduplication_field<F: Field>(mut self, deduplication_field: F) -> Self {
- self.deduplication_field = Some(deduplication_field.into());
- self
- }
-}
-
-impl tantivy::query::Query for HostLinksQuery {
- fn weight(
- &self,
- _: tantivy::query::EnableScoring<'_>,
- ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {
- Ok(Box::new(HostLinksWeight {
- node: self.node,
- field: self.field,
- deduplication_field: self.deduplication_field,
- warmed_column_fields: self.warmed_column_fields.clone(),
- skip_self_links: self.skip_self_links,
- }))
- }
-}
-
-struct HostLinksWeight {
- node: NodeID,
- field: FieldEnum,
- deduplication_field: Option<FieldEnum>,
- warmed_column_fields: WarmedColumnFields,
- skip_self_links: bool,
-}
-
-impl tantivy::query::Weight for HostLinksWeight {
- fn scorer(
- &self,
- reader: &tantivy::SegmentReader,
- _: tantivy::Score,
- ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
- let schema = reader.schema();
- let field = schema.get_field(self.field.name())?;
- let term = tantivy::Term::from_field_u64(field, self.node.as_u64());
-
- match HostLinksScorer::new(
- reader,
- term,
- self.deduplication_field,
- &self.warmed_column_fields,
- self.node.as_u64(),
- self.skip_self_links,
- ) {
- Ok(Some(scorer)) => Ok(Box::new(scorer)),
- _ => Ok(Box::new(tantivy::query::EmptyScorer)),
- }
- }
-
- fn explain(
- &self,
- _: &tantivy::SegmentReader,
- _: tantivy::DocId,
- ) -> tantivy::Result<tantivy::query::Explanation> {
- Ok(tantivy::query::Explanation::new("HostLinksWeight", 0.0))
- }
-}
-
-struct HostLinksScorer {
- postings: SegmentPostings,
- host_id_column: Option<Column<u64>>,
- last_host_id: Option<u64>,
- self_host_id: u64,
- skip_self_links: bool,
-}
-
-impl HostLinksScorer {
- fn new(
- reader: &tantivy::SegmentReader,
- term: tantivy::Term,
- deduplication_field: Option<FieldEnum>,
- warmed_column_fields: &WarmedColumnFields,
- self_host_id: u64,
- skip_self_links: bool,
- ) -> tantivy::Result<Option<Self>> {
- let host_id_column = deduplication_field.map(|f| {
- warmed_column_fields
- .segment(&reader.segment_id())
- .u64(f)
- .unwrap()
- });
-
- Ok(reader
- .inverted_index(term.field())?
- .read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?
- .map(|mut postings| {
- let mut last_host_id = None;
-
- if postings.doc() != tantivy::TERMINATED {
- last_host_id = host_id_column
- .as_ref()
- .map(|c| c.first(postings.doc()).unwrap());
- }
-
- while postings.doc() != tantivy::TERMINATED && last_host_id == Some(self_host_id) {
- postings.advance();
-
- if postings.doc() != tantivy::TERMINATED {
- last_host_id = host_id_column
- .as_ref()
- .map(|c| c.first(postings.doc()).unwrap());
- } else {
- last_host_id = None;
- }
- }
-
- Self {
- last_host_id,
- host_id_column,
- postings,
- self_host_id,
- skip_self_links,
- }
- }))
- }
-}
-
-impl HostLinksScorer {
- fn host_id(&self, doc: tantivy::DocId) -> Option<u64> {
- if doc == tantivy::TERMINATED {
- return None;
- }
-
- self.host_id_column.as_ref().map(|c| c.first(doc).unwrap())
- }
-
- fn has_seen_host(&self, doc: tantivy::DocId) -> bool {
- self.host_id(doc)
- .map(|host_id| self.last_host_id == Some(host_id))
- .unwrap_or(false)
- }
-
- fn skip_self(&self, doc: tantivy::DocId) -> bool {
- self.skip_self_links && self.host_id(doc) == Some(self.self_host_id)
- }
-}
-
-impl tantivy::query::Scorer for HostLinksScorer {
- fn score(&mut self) -> tantivy::Score {
- unimplemented!()
- }
-}
-impl tantivy::DocSet for HostLinksScorer {
- fn advance(&mut self) -> tantivy::DocId {
- self.postings.advance();
-
- while self.has_seen_host(
- self.postings
- .block_cursor()
- .skip_reader()
- .last_doc_in_block(),
- ) && self.doc() != tantivy::TERMINATED
- {
- self.postings.mut_block_cursor().advance();
- self.postings.reset_cursor_start_block();
- }
-
- while (self.has_seen_host(self.doc()) || self.skip_self(self.doc()))
- && self.doc() != tantivy::TERMINATED
- {
- self.postings.advance();
- }
-
- if let Some(host_id) = self.host_id(self.doc()) {
- self.last_host_id = Some(host_id);
- }
-
- self.doc()
- }
-
- fn doc(&self) -> tantivy::DocId {
- self.postings.doc()
- }
-
- fn size_hint(&self) -> u32 {
- self.postings.size_hint()
- }
-}
-
-#[cfg(test)]
-mod tests {
- use crate::{
- webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph},
- webpage::RelFlags,
- };
-
- #[test]
- fn test_simple() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
-
- let node_a = Node::from("A");
- let node_b = Node::from("B");
- let node_c = Node::from("C");
-
- graph
- .insert(Edge {
- from: node_a.clone(),
- to: node_b.clone(),
- rel_flags: RelFlags::default(),
- label: String::new(),
- sort_score: 0.0,
- })
- .unwrap();
-
- graph
- .insert(Edge {
- from: node_c.clone(),
- to: node_b.clone(),
- rel_flags: RelFlags::default(),
- label: String::new(),
- sort_score: 0.0,
- })
- .unwrap();
-
- graph.commit().unwrap();
-
- let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();
-
- assert_eq!(res.len(), 2);
- assert!(res.iter().any(|r| r.from == node_a.id()));
- assert!(res.iter().any(|r| r.from == node_c.id()));
- }
-
- #[test]
- fn test_self_host_skipped() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
-
- let node_a = Node::from("A");
- let node_b = Node::from("B");
-
- graph
- .insert(Edge {
- from: node_a.clone(),
- to: node_b.clone(),
- rel_flags: RelFlags::default(),
- label: String::new(),
- sort_score: 0.0,
- })
- .unwrap();
-
- graph
- .insert(Edge {
- from: node_b.clone(),
- to: node_b.clone(),
- rel_flags: RelFlags::default(),
- label: String::new(),
- sort_score: 0.0,
- })
- .unwrap();
-
- graph.commit().unwrap();
-
- let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();
-
- assert_eq!(res.len(), 1);
- assert!(res[0].from == node_a.id());
- }
-
- #[test]
- fn test_deduplication() {
- let temp_dir = crate::gen_temp_dir().unwrap();
- let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
-
- graph
- .insert(Edge {
- from: Node::from("https://A.com/1"),
- to: Node::from("https://B.com/1"),
- rel_flags: RelFlags::default(),
- label: String::new(),
- sort_score: 0.0,
- })
- .unwrap();
-
- graph
- .insert(Edge {
- from: Node::from("https://A.com/2"),
- to: Node::from("https://B.com/2"),
- rel_flags: RelFlags::default(),
- label: String::new(),
- sort_score: 0.0,
- })
- .unwrap();
-
- graph
- .insert(Edge {
- from: Node::from("https://A.com/3"),
- to: Node::from("https://B.com/3"),
- rel_flags: RelFlags::default(),
- label: String::new(),
- sort_score: 0.0,
- })
- .unwrap();
-
- graph.commit().unwrap();
-
- let res = graph
- .search(&HostBacklinksQuery::new(
- Node::from("https://B.com/").into_host().id(),
- ))
- .unwrap();
-
- assert_eq!(res.len(), 1);
- assert!(res[0].from == Node::from("https://A.com/").into_host().id());
- }
-}
diff --git a/crates/core/src/webgraph/query/raw/links.rs b/crates/core/src/webgraph/query/raw/links.rs
index 01288f4c..6bff9554 100644
--- a/crates/core/src/webgraph/query/raw/links.rs
+++ b/crates/core/src/webgraph/query/raw/links.rs
@@ -14,26 +14,47 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use tantivy::{postings::SegmentPostings, query::EmptyScorer, Term};
+use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};
use crate::webgraph::{
schema::{Field, FieldEnum},
+ warmed_column_fields::WarmedColumnFields,
NodeID,
};
-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]
+#[derive(Debug, Clone)]
pub struct LinksQuery {
node: NodeID,
field: FieldEnum,
+ deduplication_field: Option<FieldEnum>,
+ warmed_column_fields: WarmedColumnFields,
+ skip_self_links: bool,
}
impl LinksQuery {
- pub fn new<F: Field>(node: NodeID, field: F) -> Self {
+ pub fn new<F: Field>(
+ node: NodeID,
+ lookup_field: F,
+ warmed_column_fields: WarmedColumnFields,
+ ) -> Self {
Self {
node,
- field: field.into(),
+ field: lookup_field.into(),
+ deduplication_field: None,
+ warmed_column_fields,
+ skip_self_links: true,
}
}
+
+ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {
+ self.skip_self_links = skip_self_links;
+ self
+ }
+
+ pub fn with_deduplication_field<F: Field>(mut self, deduplication_field: F) -> Self {
+ self.deduplication_field = Some(deduplication_field.into());
+ self
+ }
}
impl tantivy::query::Query for LinksQuery {
@@ -44,6 +65,9 @@ impl tantivy::query::Query for LinksQuery {
Ok(Box::new(LinksWeight {
node: self.node,
field: self.field,
+ deduplication_field: self.deduplication_field,
+ warmed_column_fields: self.warmed_column_fields.clone(),
+ skip_self_links: self.skip_self_links,
}))
}
}
@@ -51,6 +75,9 @@ impl tantivy::query::Query for LinksQuery {
struct LinksWeight {
node: NodeID,
field: FieldEnum,
+ deduplication_field: Option<FieldEnum>,
+ warmed_column_fields: WarmedColumnFields,
+ skip_self_links: bool,
}
impl tantivy::query::Weight for LinksWeight {
@@ -59,13 +86,20 @@ impl tantivy::query::Weight for LinksWeight {
reader: &tantivy::SegmentReader,
_: tantivy::Score,
) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {
- let field = reader.schema().get_field(self.field.name())?;
- let term = Term::from_field_u64(field, self.node.as_u64());
+ let schema = reader.schema();
+ let field = schema.get_field(self.field.name())?;
+ let term = tantivy::Term::from_field_u64(field, self.node.as_u64());
- let index = reader.inverted_index(field)?;
- match index.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)? {
- Some(postings) => Ok(Box::new(LinksScorer { postings })),
- None => Ok(Box::new(EmptyScorer)),
+ match LinksScorer::new(
+ reader,
+ term,
+ self.deduplication_field,
+ &self.warmed_column_fields,
+ self.node.as_u64(),
+ self.skip_self_links,
+ ) {
+ Ok(Some(scorer)) => Ok(Box::new(scorer)),
+ _ => Ok(Box::new(tantivy::query::EmptyScorer)),
}
}
@@ -74,13 +108,90 @@ impl tantivy::query::Weight for LinksWeight {
_: &tantivy::SegmentReader,
_: tantivy::DocId,
) -> tantivy::Result<tantivy::query::Explanation> {
- let explanation = tantivy::query::Explanation::new("LinksQuery", 0.0);
- Ok(explanation)
+ Ok(tantivy::query::Explanation::new("LinksWeight", 0.0))
}
}
struct LinksScorer {
postings: SegmentPostings,
+ dedup_column: Option<Column<u64>>,
+ last_dedup_val: Option<u64>,
+ self_dedup_val: u64,
+ skip_self_links: bool,
+}
+
+impl LinksScorer {
+ fn new(
+ reader: &tantivy::SegmentReader,
+ term: tantivy::Term,
+ deduplication_field: Option<FieldEnum>,
+ warmed_column_fields: &WarmedColumnFields,
+ self_dedup_val: u64,
+ skip_self_links: bool,
+ ) -> tantivy::Result<Option<Self>> {
+ let dedup_column = deduplication_field.map(|f| {
+ warmed_column_fields
+ .segment(&reader.segment_id())
+ .u64(f)
+ .unwrap()
+ });
+
+ Ok(reader
+ .inverted_index(term.field())?
+ .read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?
+ .map(|mut postings| {
+ let mut last_dedup_val = None;
+
+ if postings.doc() != tantivy::TERMINATED {
+ last_dedup_val = dedup_column
+ .as_ref()
+ .map(|c| c.first(postings.doc()).unwrap());
+ }
+
+ while postings.doc() != tantivy::TERMINATED
+ && skip_self_links
+ && last_dedup_val == Some(self_dedup_val)
+ {
+ postings.advance();
+
+ if postings.doc() != tantivy::TERMINATED {
+ last_dedup_val = dedup_column
+ .as_ref()
+ .map(|c| c.first(postings.doc()).unwrap());
+ } else {
+ last_dedup_val = None;
+ }
+ }
+
+ Self {
+ last_dedup_val,
+ dedup_column,
+ postings,
+ self_dedup_val,
+ skip_self_links,
+ }
+ }))
+ }
+}
+
+impl LinksScorer {
+ fn dedup_val(&self, doc: tantivy::DocId) -> Option<u64> {
+ if doc == tantivy::TERMINATED {
+ return None;
+ }
+
+ self.dedup_column.as_ref().map(|c| c.first(doc).unwrap())
+ }
+
+ fn has_seen(&self, doc: tantivy::DocId) -> bool {
+ self.dedup_val(doc)
+ .map(|dedup_val| self.last_dedup_val == Some(dedup_val))
+ .unwrap_or(false)
+ }
+
+ fn skip_self(&self, doc: tantivy::DocId) -> bool {
+ self.skip_self_links && self.dedup_val(doc) == Some(self.self_dedup_val)
+ }
}
impl tantivy::query::Scorer for LinksScorer {
@@ -88,10 +199,32 @@ impl tantivy::query::Scorer for LinksScorer {
unimplemented!()
}
}
-
impl tantivy::DocSet for LinksScorer {
fn advance(&mut self) -> tantivy::DocId {
- self.postings.advance()
+ self.postings.advance();
+
+ while self.has_seen(
+ self.postings
+ .block_cursor()
+ .skip_reader()
+ .last_doc_in_block(),
+ ) && self.doc() != tantivy::TERMINATED
+ {
+ self.postings.mut_block_cursor().advance();
+ self.postings.reset_cursor_start_block();
+ }
+
+ while (self.has_seen(self.doc()) || self.skip_self(self.doc()))
+ && self.doc() != tantivy::TERMINATED
+ {
+ self.postings.advance();
+ }
+
+ if let Some(dedup_val) = self.dedup_val(self.doc()) {
+ self.last_dedup_val = Some(dedup_val);
+ }
+
+ self.doc()
}
fn doc(&self) -> tantivy::DocId {
@@ -102,3 +235,132 @@ impl tantivy::DocSet for LinksScorer {
self.postings.size_hint()
}
}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph},
+ webpage::RelFlags,
+ };
+
+ #[test]
+ fn test_simple() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ let node_a = Node::from("A");
+ let node_b = Node::from("B");
+ let node_c = Node::from("C");
+
+ graph
+ .insert(Edge {
+ from: node_a.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: node_c.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();
+
+ assert_eq!(res.len(), 2);
+ assert!(res.iter().any(|r| r.from == node_a.id()));
+ assert!(res.iter().any(|r| r.from == node_c.id()));
+ }
+
+ #[test]
+ fn test_self_host_skipped() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ let node_a = Node::from("A");
+ let node_b = Node::from("B");
+
+ graph
+ .insert(Edge {
+ from: node_a.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: node_b.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert!(res[0].from == node_a.id());
+ }
+
+ #[test]
+ fn test_deduplication() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/1"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/2"),
+ to: Node::from("https://B.com/2"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/3"),
+ to: Node::from("https://B.com/3"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph
+ .search(&HostBacklinksQuery::new(
+ Node::from("https://B.com/").into_host().id(),
+ ))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert!(res[0].from == Node::from("https://A.com/").into_host().id());
+ }
+}
diff --git a/crates/core/src/webgraph/query/raw/mod.rs b/crates/core/src/webgraph/query/raw/mod.rs
index 5c940356..ad9ce07e 100644
--- a/crates/core/src/webgraph/query/raw/mod.rs
+++ b/crates/core/src/webgraph/query/raw/mod.rs
@@ -17,12 +17,10 @@
pub mod dummy;
pub use dummy::DummyQuery;
-pub mod host_links;
pub mod id2node;
pub mod links;
pub mod phrase_or_term;
-pub use host_links::HostLinksQuery;
pub use id2node::Id2NodeQuery;
pub use links::LinksQuery;
pub use phrase_or_term::PhraseOrTermQuery;
diff --git a/crates/core/src/webgraph/tests.rs b/crates/core/src/webgraph/tests.rs
index e192c17f..22e1a4cd 100644
--- a/crates/core/src/webgraph/tests.rs
+++ b/crates/core/src/webgraph/tests.rs
@@ -277,7 +277,7 @@ fn merge_cycle() {
let temp_dir = crate::gen_temp_dir().unwrap();
for (i, (from, to, label)) in (0..).zip([
(Node::from("A"), Node::from("B"), String::new()),
- (Node::from("B"), Node::from("B"), String::new()),
+ (Node::from("B"), Node::from("A"), String::new()),
(Node::from("B"), Node::from("C"), String::new()),
(Node::from("C"), Node::from("A"), String::new()),
]) {