web search done right
[webgraph] rename HostLinkQuery to LinksQuery
Mikkel Denkere94cd2de11d5
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rsindex b32bfcb7..f2224563 100644--- a/crates/core/src/webgraph/query/backlink.rs+++ b/crates/core/src/webgraph/query/backlink.rs@@ -19,7 +19,7 @@ use tantivy::query::{BooleanQuery, Occur, ShortCircuitQuery};use super::{collector::{top_docs, HostDeduplicator, TopDocsCollector},document_scorer::DefaultDocumentScorer,- raw::{HostLinksQuery, LinksQuery},+ raw::LinksQuery,AndFilter, Filter, FilterEnum, Query,};use crate::{@@ -144,7 +144,10 @@ impl Query for BacklinksQuery {type Output = Vec<SmallEdge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(+ LinksQuery::new(self.node, ToId, searcher.warmed_column_fields().clone())+ .with_deduplication_field(FromId),+ ) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);@@ -272,7 +275,7 @@ impl Query for HostBacklinksQuery {fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {let mut raw = Box::new(- HostLinksQuery::new(self.node, ToHostId, searcher.warmed_column_fields().clone())+ LinksQuery::new(self.node, ToHostId, searcher.warmed_column_fields().clone()).with_deduplication_field(FromHostId),) as Box<dyn tantivy::query::Query>;@@ -408,8 +411,14 @@ impl Query for FullBacklinksQuery {type Output = Vec<Edge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw =- Box::new(LinksQuery::new(self.node.id(), ToId)) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(+ LinksQuery::new(+ self.node.id(),+ ToId,+ searcher.warmed_column_fields().clone(),+ )+ .with_deduplication_field(FromId),+ ) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);@@ -538,7 +547,7 @@ impl Query for FullHostBacklinksQuery {fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {let mut raw = Box::new(- HostLinksQuery::new(+ LinksQuery::new(self.node.clone().into_host().id(),ToHostId,searcher.warmed_column_fields().clone(),@@ -681,7 +690,10 @@ impl Query for BacklinksWithLabelsQuery {type Output = Vec<SmallEdgeWithLabel>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw = Box::new(LinksQuery::new(self.node, ToId)) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(+ LinksQuery::new(self.node, ToId, searcher.warmed_column_fields().clone())+ .with_deduplication_field(FromId),+ ) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);diff --git a/crates/core/src/webgraph/query/between.rs b/crates/core/src/webgraph/query/between.rsindex dd58f7d3..94aabbef 100644--- a/crates/core/src/webgraph/query/between.rs+++ b/crates/core/src/webgraph/query/between.rs@@ -19,7 +19,7 @@ use tantivy::query::{BooleanQuery, Occur};use crate::{ampc::dht::ShardId,webgraph::{- schema::{FromHostId, ToHostId},+ schema::{FromHostId, FromId, ToHostId, ToId},searcher::Searcher,Edge, EdgeLimit, Node, Query,},@@ -27,7 +27,7 @@ use crate::{};use super::{- collector::TopDocsCollector, fetch_edges, raw::HostLinksQuery, AndFilter, Filter, FilterEnum,+ collector::TopDocsCollector, fetch_edges, raw::LinksQuery, AndFilter, Filter, FilterEnum,};#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]@@ -78,16 +78,22 @@ impl Query for FullLinksBetweenQuery {type Output = Vec<Edge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let from_query = Box::new(HostLinksQuery::new(- self.from.id(),- FromHostId,- searcher.warmed_column_fields().clone(),- )) as Box<dyn tantivy::query::Query>;- let to_query = Box::new(HostLinksQuery::new(- self.to.id(),- ToHostId,- searcher.warmed_column_fields().clone(),- )) as Box<dyn tantivy::query::Query>;+ let from_query = Box::new(+ LinksQuery::new(+ self.from.id(),+ FromHostId,+ searcher.warmed_column_fields().clone(),+ )+ .with_deduplication_field(ToId),+ ) as Box<dyn tantivy::query::Query>;+ let to_query = Box::new(+ LinksQuery::new(+ self.to.id(),+ ToHostId,+ searcher.warmed_column_fields().clone(),+ )+ .with_deduplication_field(FromId),+ ) as Box<dyn tantivy::query::Query>;let mut queries = vec![(Occur::Must, from_query), (Occur::Must, to_query)];diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rsindex 37c13337..23500287 100644--- a/crates/core/src/webgraph/query/forwardlink.rs+++ b/crates/core/src/webgraph/query/forwardlink.rs@@ -18,7 +18,7 @@ use super::{backlink::{fetch_edges, fetch_small_edges},collector::{top_docs, HostDeduplicator, TopDocsCollector},document_scorer::DefaultDocumentScorer,- raw::{host_links::HostLinksQuery, links::LinksQuery},+ raw::LinksQuery,AndFilter, Filter, FilterEnum, Query,};use crate::{@@ -83,8 +83,11 @@ impl Query for ForwardlinksQuery {type Output = Vec<SmallEdge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw =- Box::new(LinksQuery::new(self.node, FromId)) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(LinksQuery::new(+ self.node,+ FromId,+ searcher.warmed_column_fields().clone(),+ )) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);@@ -213,7 +216,7 @@ impl Query for HostForwardlinksQuery {fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {let mut raw = Box::new(- HostLinksQuery::new(+ LinksQuery::new(self.node,FromHostId,searcher.warmed_column_fields().clone(),@@ -351,8 +354,14 @@ impl Query for FullForwardlinksQuery {type Output = Vec<Edge>;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {- let mut raw =- Box::new(LinksQuery::new(self.node.id(), FromId)) as Box<dyn tantivy::query::Query>;+ let mut raw = Box::new(+ LinksQuery::new(+ self.node.id(),+ FromId,+ searcher.warmed_column_fields().clone(),+ )+ .with_deduplication_field(ToId),+ ) as Box<dyn tantivy::query::Query>;if let Some(filter) = self.filter_as_and().and_then(|f| f.inverted_index_filter()) {let filter = filter.query(searcher);@@ -469,7 +478,7 @@ impl Query for FullHostForwardlinksQuery {fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery {let mut raw = Box::new(- HostLinksQuery::new(+ LinksQuery::new(self.node.clone().into_host().id(),FromHostId,searcher.warmed_column_fields().clone(),diff --git a/crates/core/src/webgraph/query/group_by.rs b/crates/core/src/webgraph/query/group_by.rsindex 05e4b7dc..2eedb609 100644--- a/crates/core/src/webgraph/query/group_by.rs+++ b/crates/core/src/webgraph/query/group_by.rs@@ -99,11 +99,11 @@ impl Query for HostGroupSketchQuery {fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {let mut raw: Self::TantivyQuery = match self.node {LinksDirection::From(node) => Box::new(- raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())+ raw::LinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone()).with_deduplication_field(ToHostId),),LinksDirection::To(node) => Box::new(- raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())+ raw::LinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone()).with_deduplication_field(FromHostId),),};@@ -217,11 +217,11 @@ impl Query for HostGroupQuery {fn tantivy_query(&self, searcher: &crate::webgraph::searcher::Searcher) -> Self::TantivyQuery {let mut raw: Self::TantivyQuery = match self.node {LinksDirection::From(node) => Box::new(- raw::HostLinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone())+ raw::LinksQuery::new(node, FromHostId, searcher.warmed_column_fields().clone()).with_deduplication_field(ToHostId),),LinksDirection::To(node) => Box::new(- raw::HostLinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone())+ raw::LinksQuery::new(node, ToHostId, searcher.warmed_column_fields().clone()).with_deduplication_field(FromHostId),),};diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rsdeleted file mode 100644index d8d393b4..00000000--- a/crates/core/src/webgraph/query/raw/host_links.rs+++ /dev/null@@ -1,363 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};--use crate::webgraph::{- schema::{Field, FieldEnum},- warmed_column_fields::WarmedColumnFields,- NodeID,-};--#[derive(Debug, Clone)]-pub struct HostLinksQuery {- node: NodeID,- field: FieldEnum,- deduplication_field: Option<FieldEnum>,- warmed_column_fields: WarmedColumnFields,- skip_self_links: bool,-}--impl HostLinksQuery {- pub fn new<F: Field>(- node: NodeID,- lookup_field: F,- warmed_column_fields: WarmedColumnFields,- ) -> Self {- Self {- node,- field: lookup_field.into(),- deduplication_field: None,- warmed_column_fields,- skip_self_links: true,- }- }-- pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {- self.skip_self_links = skip_self_links;- self- }-- pub fn with_deduplication_field<F: Field>(mut self, deduplication_field: F) -> Self {- self.deduplication_field = Some(deduplication_field.into());- self- }-}--impl tantivy::query::Query for HostLinksQuery {- fn weight(- &self,- _: tantivy::query::EnableScoring<'_>,- ) -> tantivy::Result<Box<dyn tantivy::query::Weight>> {- Ok(Box::new(HostLinksWeight {- node: self.node,- field: self.field,- deduplication_field: self.deduplication_field,- warmed_column_fields: self.warmed_column_fields.clone(),- skip_self_links: self.skip_self_links,- }))- }-}--struct HostLinksWeight {- node: NodeID,- field: FieldEnum,- deduplication_field: Option<FieldEnum>,- warmed_column_fields: WarmedColumnFields,- skip_self_links: bool,-}--impl tantivy::query::Weight for HostLinksWeight {- fn scorer(- &self,- reader: &tantivy::SegmentReader,- _: tantivy::Score,- ) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {- let schema = reader.schema();- let field = schema.get_field(self.field.name())?;- let term = tantivy::Term::from_field_u64(field, self.node.as_u64());-- match HostLinksScorer::new(- reader,- term,- self.deduplication_field,- &self.warmed_column_fields,- self.node.as_u64(),- self.skip_self_links,- ) {- Ok(Some(scorer)) => Ok(Box::new(scorer)),- _ => Ok(Box::new(tantivy::query::EmptyScorer)),- }- }-- fn explain(- &self,- _: &tantivy::SegmentReader,- _: tantivy::DocId,- ) -> tantivy::Result<tantivy::query::Explanation> {- Ok(tantivy::query::Explanation::new("HostLinksWeight", 0.0))- }-}--struct HostLinksScorer {- postings: SegmentPostings,- host_id_column: Option<Column<u64>>,- last_host_id: Option<u64>,- self_host_id: u64,- skip_self_links: bool,-}--impl HostLinksScorer {- fn new(- reader: &tantivy::SegmentReader,- term: tantivy::Term,- deduplication_field: Option<FieldEnum>,- warmed_column_fields: &WarmedColumnFields,- self_host_id: u64,- skip_self_links: bool,- ) -> tantivy::Result<Option<Self>> {- let host_id_column = deduplication_field.map(|f| {- warmed_column_fields- .segment(&reader.segment_id())- .u64(f)- .unwrap()- });-- Ok(reader- .inverted_index(term.field())?- .read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?- .map(|mut postings| {- let mut last_host_id = None;-- if postings.doc() != tantivy::TERMINATED {- last_host_id = host_id_column- .as_ref()- .map(|c| c.first(postings.doc()).unwrap());- }-- while postings.doc() != tantivy::TERMINATED && last_host_id == Some(self_host_id) {- postings.advance();-- if postings.doc() != tantivy::TERMINATED {- last_host_id = host_id_column- .as_ref()- .map(|c| c.first(postings.doc()).unwrap());- } else {- last_host_id = None;- }- }-- Self {- last_host_id,- host_id_column,- postings,- self_host_id,- skip_self_links,- }- }))- }-}--impl HostLinksScorer {- fn host_id(&self, doc: tantivy::DocId) -> Option<u64> {- if doc == tantivy::TERMINATED {- return None;- }-- self.host_id_column.as_ref().map(|c| c.first(doc).unwrap())- }-- fn has_seen_host(&self, doc: tantivy::DocId) -> bool {- self.host_id(doc)- .map(|host_id| self.last_host_id == Some(host_id))- .unwrap_or(false)- }-- fn skip_self(&self, doc: tantivy::DocId) -> bool {- self.skip_self_links && self.host_id(doc) == Some(self.self_host_id)- }-}--impl tantivy::query::Scorer for HostLinksScorer {- fn score(&mut self) -> tantivy::Score {- unimplemented!()- }-}-impl tantivy::DocSet for HostLinksScorer {- fn advance(&mut self) -> tantivy::DocId {- self.postings.advance();-- while self.has_seen_host(- self.postings- .block_cursor()- .skip_reader()- .last_doc_in_block(),- ) && self.doc() != tantivy::TERMINATED- {- self.postings.mut_block_cursor().advance();- self.postings.reset_cursor_start_block();- }-- while (self.has_seen_host(self.doc()) || self.skip_self(self.doc()))- && self.doc() != tantivy::TERMINATED- {- self.postings.advance();- }-- if let Some(host_id) = self.host_id(self.doc()) {- self.last_host_id = Some(host_id);- }-- self.doc()- }-- fn doc(&self) -> tantivy::DocId {- self.postings.doc()- }-- fn size_hint(&self) -> u32 {- self.postings.size_hint()- }-}--#[cfg(test)]-mod tests {- use crate::{- webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph},- webpage::RelFlags,- };-- #[test]- fn test_simple() {- let temp_dir = crate::gen_temp_dir().unwrap();- let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();-- let node_a = Node::from("A");- let node_b = Node::from("B");- let node_c = Node::from("C");-- graph- .insert(Edge {- from: node_a.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })- .unwrap();-- graph- .insert(Edge {- from: node_c.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })- .unwrap();-- graph.commit().unwrap();-- let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();-- assert_eq!(res.len(), 2);- assert!(res.iter().any(|r| r.from == node_a.id()));- assert!(res.iter().any(|r| r.from == node_c.id()));- }-- #[test]- fn test_self_host_skipped() {- let temp_dir = crate::gen_temp_dir().unwrap();- let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();-- let node_a = Node::from("A");- let node_b = Node::from("B");-- graph- .insert(Edge {- from: node_a.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })- .unwrap();-- graph- .insert(Edge {- from: node_b.clone(),- to: node_b.clone(),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })- .unwrap();-- graph.commit().unwrap();-- let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();-- assert_eq!(res.len(), 1);- assert!(res[0].from == node_a.id());- }-- #[test]- fn test_deduplication() {- let temp_dir = crate::gen_temp_dir().unwrap();- let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();-- graph- .insert(Edge {- from: Node::from("https://A.com/1"),- to: Node::from("https://B.com/1"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })- .unwrap();-- graph- .insert(Edge {- from: Node::from("https://A.com/2"),- to: Node::from("https://B.com/2"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })- .unwrap();-- graph- .insert(Edge {- from: Node::from("https://A.com/3"),- to: Node::from("https://B.com/3"),- rel_flags: RelFlags::default(),- label: String::new(),- sort_score: 0.0,- })- .unwrap();-- graph.commit().unwrap();-- let res = graph- .search(&HostBacklinksQuery::new(- Node::from("https://B.com/").into_host().id(),- ))- .unwrap();-- assert_eq!(res.len(), 1);- assert!(res[0].from == Node::from("https://A.com/").into_host().id());- }-}diff --git a/crates/core/src/webgraph/query/raw/links.rs b/crates/core/src/webgraph/query/raw/links.rsindex 01288f4c..6bff9554 100644--- a/crates/core/src/webgraph/query/raw/links.rs+++ b/crates/core/src/webgraph/query/raw/links.rs@@ -14,26 +14,47 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use tantivy::{postings::SegmentPostings, query::EmptyScorer, Term};+use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};use crate::webgraph::{schema::{Field, FieldEnum},+ warmed_column_fields::WarmedColumnFields,NodeID,};-#[derive(Debug, Clone, bincode::Encode, bincode::Decode)]+#[derive(Debug, Clone)]pub struct LinksQuery {node: NodeID,field: FieldEnum,+ deduplication_field: Option<FieldEnum>,+ warmed_column_fields: WarmedColumnFields,+ skip_self_links: bool,}impl LinksQuery {- pub fn new<F: Field>(node: NodeID, field: F) -> Self {+ pub fn new<F: Field>(+ node: NodeID,+ lookup_field: F,+ warmed_column_fields: WarmedColumnFields,+ ) -> Self {Self {node,- field: field.into(),+ field: lookup_field.into(),+ deduplication_field: None,+ warmed_column_fields,+ skip_self_links: true,}}++ pub fn skip_self_links(mut self, skip_self_links: bool) -> Self {+ self.skip_self_links = skip_self_links;+ self+ }++ pub fn with_deduplication_field<F: Field>(mut self, deduplication_field: F) -> Self {+ self.deduplication_field = Some(deduplication_field.into());+ self+ }}impl tantivy::query::Query for LinksQuery {@@ -44,6 +65,9 @@ impl tantivy::query::Query for LinksQuery {Ok(Box::new(LinksWeight {node: self.node,field: self.field,+ deduplication_field: self.deduplication_field,+ warmed_column_fields: self.warmed_column_fields.clone(),+ skip_self_links: self.skip_self_links,}))}}@@ -51,6 +75,9 @@ impl tantivy::query::Query for LinksQuery {struct LinksWeight {node: NodeID,field: FieldEnum,+ deduplication_field: Option<FieldEnum>,+ warmed_column_fields: WarmedColumnFields,+ skip_self_links: bool,}impl tantivy::query::Weight for LinksWeight {@@ -59,13 +86,20 @@ impl tantivy::query::Weight for LinksWeight {reader: &tantivy::SegmentReader,_: tantivy::Score,) -> tantivy::Result<Box<dyn tantivy::query::Scorer>> {- let field = reader.schema().get_field(self.field.name())?;- let term = Term::from_field_u64(field, self.node.as_u64());+ let schema = reader.schema();+ let field = schema.get_field(self.field.name())?;+ let term = tantivy::Term::from_field_u64(field, self.node.as_u64());- let index = reader.inverted_index(field)?;- match index.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)? {- Some(postings) => Ok(Box::new(LinksScorer { postings })),- None => Ok(Box::new(EmptyScorer)),+ match LinksScorer::new(+ reader,+ term,+ self.deduplication_field,+ &self.warmed_column_fields,+ self.node.as_u64(),+ self.skip_self_links,+ ) {+ Ok(Some(scorer)) => Ok(Box::new(scorer)),+ _ => Ok(Box::new(tantivy::query::EmptyScorer)),}}@@ -74,13 +108,90 @@ impl tantivy::query::Weight for LinksWeight {_: &tantivy::SegmentReader,_: tantivy::DocId,) -> tantivy::Result<tantivy::query::Explanation> {- let explanation = tantivy::query::Explanation::new("LinksQuery", 0.0);- Ok(explanation)+ Ok(tantivy::query::Explanation::new("LinksWeight", 0.0))}}struct LinksScorer {postings: SegmentPostings,+ dedup_column: Option<Column<u64>>,+ last_dedup_val: Option<u64>,+ self_dedup_val: u64,+ skip_self_links: bool,+}++impl LinksScorer {+ fn new(+ reader: &tantivy::SegmentReader,+ term: tantivy::Term,+ deduplication_field: Option<FieldEnum>,+ warmed_column_fields: &WarmedColumnFields,+ self_dedup_val: u64,+ skip_self_links: bool,+ ) -> tantivy::Result<Option<Self>> {+ let dedup_column = deduplication_field.map(|f| {+ warmed_column_fields+ .segment(&reader.segment_id())+ .u64(f)+ .unwrap()+ });++ Ok(reader+ .inverted_index(term.field())?+ .read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?+ .map(|mut postings| {+ let mut last_dedup_val = None;++ if postings.doc() != tantivy::TERMINATED {+ last_dedup_val = dedup_column+ .as_ref()+ .map(|c| c.first(postings.doc()).unwrap());+ }++ while postings.doc() != tantivy::TERMINATED+ && skip_self_links+ && last_dedup_val == Some(self_dedup_val)+ {+ postings.advance();++ if postings.doc() != tantivy::TERMINATED {+ last_dedup_val = dedup_column+ .as_ref()+ .map(|c| c.first(postings.doc()).unwrap());+ } else {+ last_dedup_val = None;+ }+ }++ Self {+ last_dedup_val,+ dedup_column,+ postings,+ self_dedup_val,+ skip_self_links,+ }+ }))+ }+}++impl LinksScorer {+ fn dedup_val(&self, doc: tantivy::DocId) -> Option<u64> {+ if doc == tantivy::TERMINATED {+ return None;+ }++ self.dedup_column.as_ref().map(|c| c.first(doc).unwrap())+ }++ fn has_seen(&self, doc: tantivy::DocId) -> bool {+ self.dedup_val(doc)+ .map(|dedup_val| self.last_dedup_val == Some(dedup_val))+ .unwrap_or(false)+ }++ fn skip_self(&self, doc: tantivy::DocId) -> bool {+ self.skip_self_links && self.dedup_val(doc) == Some(self.self_dedup_val)+ }}impl tantivy::query::Scorer for LinksScorer {@@ -88,10 +199,32 @@ impl tantivy::query::Scorer for LinksScorer {unimplemented!()}}-impl tantivy::DocSet for LinksScorer {fn advance(&mut self) -> tantivy::DocId {- self.postings.advance()+ self.postings.advance();++ while self.has_seen(+ self.postings+ .block_cursor()+ .skip_reader()+ .last_doc_in_block(),+ ) && self.doc() != tantivy::TERMINATED+ {+ self.postings.mut_block_cursor().advance();+ self.postings.reset_cursor_start_block();+ }++ while (self.has_seen(self.doc()) || self.skip_self(self.doc()))+ && self.doc() != tantivy::TERMINATED+ {+ self.postings.advance();+ }++ if let Some(dedup_val) = self.dedup_val(self.doc()) {+ self.last_dedup_val = Some(dedup_val);+ }++ self.doc()}fn doc(&self) -> tantivy::DocId {@@ -102,3 +235,132 @@ impl tantivy::DocSet for LinksScorer {self.postings.size_hint()}}++#[cfg(test)]+mod tests {+ use crate::{+ webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph},+ webpage::RelFlags,+ };++ #[test]+ fn test_simple() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ let node_a = Node::from("A");+ let node_b = Node::from("B");+ let node_c = Node::from("C");++ graph+ .insert(Edge {+ from: node_a.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: node_c.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();++ assert_eq!(res.len(), 2);+ assert!(res.iter().any(|r| r.from == node_a.id()));+ assert!(res.iter().any(|r| r.from == node_c.id()));+ }++ #[test]+ fn test_self_host_skipped() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ let node_a = Node::from("A");+ let node_b = Node::from("B");++ graph+ .insert(Edge {+ from: node_a.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: node_b.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();++ assert_eq!(res.len(), 1);+ assert!(res[0].from == node_a.id());+ }++ #[test]+ fn test_deduplication() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/1"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/2"),+ to: Node::from("https://B.com/2"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/3"),+ to: Node::from("https://B.com/3"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph+ .search(&HostBacklinksQuery::new(+ Node::from("https://B.com/").into_host().id(),+ ))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert!(res[0].from == Node::from("https://A.com/").into_host().id());+ }+}diff --git a/crates/core/src/webgraph/query/raw/mod.rs b/crates/core/src/webgraph/query/raw/mod.rsindex 5c940356..ad9ce07e 100644--- a/crates/core/src/webgraph/query/raw/mod.rs+++ b/crates/core/src/webgraph/query/raw/mod.rs@@ -17,12 +17,10 @@pub mod dummy;pub use dummy::DummyQuery;-pub mod host_links;pub mod id2node;pub mod links;pub mod phrase_or_term;-pub use host_links::HostLinksQuery;pub use id2node::Id2NodeQuery;pub use links::LinksQuery;pub use phrase_or_term::PhraseOrTermQuery;diff --git a/crates/core/src/webgraph/tests.rs b/crates/core/src/webgraph/tests.rsindex e192c17f..22e1a4cd 100644--- a/crates/core/src/webgraph/tests.rs+++ b/crates/core/src/webgraph/tests.rs@@ -277,7 +277,7 @@ fn merge_cycle() {let temp_dir = crate::gen_temp_dir().unwrap();for (i, (from, to, label)) in (0..).zip([(Node::from("A"), Node::from("B"), String::new()),- (Node::from("B"), Node::from("B"), String::new()),+ (Node::from("B"), Node::from("A"), String::new()),(Node::from("B"), Node::from("C"), String::new()),(Node::from("C"), Node::from("A"), String::new()),]) {