web search done right
deduplicate host edges across segment+shard results
Mikkel Denkerc5bd49df5122
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rsindex 226e5a55..78c7d24c 100644--- a/crates/core/src/webgraph/query/backlink.rs+++ b/crates/core/src/webgraph/query/backlink.rs@@ -17,7 +17,8 @@use tantivy::query::ShortCircuitQuery;use super::{- collector::TopDocsCollector,+ collector::{HostDeduplicator, TopDocsCollector},+ document_scorer::DefaultDocumentScorer,raw::{HostLinksQuery, LinksQuery},Query,};@@ -129,9 +130,9 @@ impl Query for BacklinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()}@@ -200,7 +201,7 @@ impl HostBacklinksQuery {}impl Query for HostBacklinksQuery {- type Collector = TopDocsCollector;+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;type TantivyQuery = Box<dyn tantivy::query::Query>;type IntermediateOutput = Vec<(f64, SmallEdge)>;type Output = Vec<SmallEdge>;@@ -227,14 +228,18 @@ impl Query for HostBacklinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()+ .with_deduplicator(HostDeduplicator)+ .with_column_fields(searcher.warmed_column_fields().clone(), FromHostId)}fn remote_collector(&self) -> Self::Collector {- TopDocsCollector::from(self.limit).enable_offset()+ TopDocsCollector::from(self.limit)+ .enable_offset()+ .with_deduplicator(HostDeduplicator)}fn retrieve(@@ -242,7 +247,7 @@ impl Query for HostBacklinksQuery {searcher: &Searcher,fruit: <Self::Collector as super::collector::Collector>::Fruit,) -> Result<Self::IntermediateOutput> {- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();let nodes = fetch_small_edges(searcher, docs, FromHostId)?;Ok(nodes.into_iter()@@ -266,7 +271,7 @@ impl Query for HostBacklinksQuery {) -> <Self::Collector as super::Collector>::Fruit {fruit.into_iter()- .filter(|(_, doc)| doc.shard_id == shard_id)+ .filter(|(_, doc)| doc.address.shard_id == shard_id).collect()}@@ -317,9 +322,9 @@ impl Query for FullBacklinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()}@@ -378,7 +383,7 @@ impl FullHostBacklinksQuery {}impl Query for FullHostBacklinksQuery {- type Collector = TopDocsCollector;+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;type TantivyQuery = Box<dyn tantivy::query::Query>;type IntermediateOutput = Vec<Edge>;type Output = Vec<Edge>;@@ -405,14 +410,18 @@ impl Query for FullHostBacklinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()+ .with_deduplicator(HostDeduplicator)+ .with_column_fields(searcher.warmed_column_fields().clone(), FromHostId)}fn remote_collector(&self) -> Self::Collector {- TopDocsCollector::from(self.limit).enable_offset()+ TopDocsCollector::from(self.limit)+ .enable_offset()+ .with_deduplicator(HostDeduplicator)}fn retrieve(@@ -420,7 +429,7 @@ impl Query for FullHostBacklinksQuery {searcher: &Searcher,fruit: <Self::Collector as super::collector::Collector>::Fruit,) -> Result<Self::IntermediateOutput> {- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();let edges = fetch_edges(searcher, docs)?;Ok(edges@@ -440,7 +449,7 @@ impl Query for FullHostBacklinksQuery {) -> <Self::Collector as super::Collector>::Fruit {fruit.into_iter()- .filter(|(_, doc)| doc.shard_id == shard_id)+ .filter(|(_, doc)| doc.address.shard_id == shard_id).collect()}@@ -482,9 +491,9 @@ impl Query for BacklinksWithLabelsQuery {Box::new(LinksQuery::new(self.node, ToId))}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()}diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rsindex b1478590..0b35246d 100644--- a/crates/core/src/webgraph/query/collector/mod.rs+++ b/crates/core/src/webgraph/query/collector/mod.rs@@ -20,7 +20,7 @@ mod top_docs;pub use fast_count::{FastCountCollector, FastCountValue};pub use first_doc::FirstDocCollector;-pub use top_docs::TopDocsCollector;+pub use top_docs::{HostDeduplicator, TopDocsCollector};use tantivy::{collector::{Fruit, SegmentCollector},diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rsindex dda3d705..15f4dae0 100644--- a/crates/core/src/webgraph/query/collector/top_docs.rs+++ b/crates/core/src/webgraph/query/collector/top_docs.rs@@ -16,29 +16,141 @@use std::marker::PhantomData;+use anyhow::anyhow;+use itertools::Itertools;use tantivy::{collector::{SegmentCollector, TopNComputer},+ columnar::Column,DocId, SegmentOrdinal,};-use crate::distributed::member::ShardId;use crate::webgraph::{doc_address::DocAddress,query::document_scorer::{DefaultDocumentScorer, DocumentScorer},+ schema::{Field, FieldEnum},EdgeLimit,};+use crate::{distributed::member::ShardId, webgraph::warmed_column_fields::WarmedColumnFields};use super::Collector;-pub struct TopDocsCollector<S: DocumentScorer = DefaultDocumentScorer> {+pub trait DeduplicatorDoc+where+ Self: Send + Sync + serde::Serialize + serde::de::DeserializeOwned + Ord + Clone,+{+ fn new<S, D>(collector: &TopDocsSegmentCollector<S, D>, doc: DocId) -> Self+ where+ S: DocumentScorer + 'static,+ D: Deduplicator + 'static;+}++pub trait Deduplicator: Clone + Send + Sync {+ type Doc: DeduplicatorDoc;++ fn deduplicate(+ &self,+ docs: Vec<(tantivy::Score, Self::Doc)>,+ ) -> Vec<(tantivy::Score, Self::Doc)>;+}++impl DeduplicatorDoc for DocAddress {+ fn new<S, D>(collector: &TopDocsSegmentCollector<S, D>, doc: DocId) -> Self+ where+ S: DocumentScorer + 'static,+ D: Deduplicator + 'static,+ {+ DocAddress::new(collector.shard_id, collector.segment_ord, doc)+ }+}++#[derive(Clone)]+pub struct NoDeduplicator;++impl Deduplicator for NoDeduplicator {+ type Doc = DocAddress;++ fn deduplicate(+ &self,+ docs: Vec<(tantivy::Score, Self::Doc)>,+ ) -> Vec<(tantivy::Score, Self::Doc)> {+ docs+ }+}++#[derive(+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Ord,+ PartialOrd,+ Eq,+ PartialEq,+)]+pub struct DocAddressWithHost {+ pub address: DocAddress,+ pub host: u64,+}++impl DeduplicatorDoc for DocAddressWithHost {+ fn new<S, D>(collector: &TopDocsSegmentCollector<S, D>, doc: DocId) -> Self+ where+ S: DocumentScorer + 'static,+ D: Deduplicator + 'static,+ {+ let host = collector+ .host_column+ .as_ref()+ .and_then(|col| col.first(doc))+ .ok_or_else(|| anyhow!("ColumnFields must be set to use HostDeduplicator"))+ .unwrap();+ Self {+ address: DocAddress::new(collector.shard_id, collector.segment_ord, doc),+ host,+ }+ }+}++#[derive(Clone)]+pub struct HostDeduplicator;++impl Deduplicator for HostDeduplicator {+ type Doc = DocAddressWithHost;++ fn deduplicate(+ &self,+ docs: Vec<(tantivy::Score, Self::Doc)>,+ ) -> Vec<(tantivy::Score, Self::Doc)> {+ docs.into_iter().unique_by(|(_, doc)| doc.host).collect()+ }+}++pub struct ColumnFields {+ warmed_column_fields: WarmedColumnFields,+ host_field: FieldEnum,+}++impl ColumnFields {+ pub fn new<F: Field>(warmed_column_fields: WarmedColumnFields, host_field: F) -> Self {+ Self {+ warmed_column_fields,+ host_field: host_field.into(),+ }+ }+}++pub struct TopDocsCollector<S = DefaultDocumentScorer, D = NoDeduplicator> {shard_id: Option<ShardId>,limit: Option<usize>,offset: Option<usize>,perform_offset: bool,+ deduplicator: D,+ column_fields: Option<ColumnFields>,_phantom: PhantomData<S>,}-impl<S: DocumentScorer> From<EdgeLimit> for TopDocsCollector<S> {+impl<S> From<EdgeLimit> for TopDocsCollector<S, NoDeduplicator> {fn from(limit: EdgeLimit) -> Self {let mut collector = TopDocsCollector::new().disable_offset();@@ -55,23 +167,27 @@ impl<S: DocumentScorer> From<EdgeLimit> for TopDocsCollector<S> {}}-impl<S: DocumentScorer> Default for TopDocsCollector<S> {+impl<S> Default for TopDocsCollector<S, NoDeduplicator> {fn default() -> Self {Self::new()}}-impl<S: DocumentScorer> TopDocsCollector<S> {+impl<S> TopDocsCollector<S, NoDeduplicator> {pub fn new() -> Self {Self {shard_id: None,limit: None,offset: None,perform_offset: true,+ deduplicator: NoDeduplicator,+ column_fields: None,_phantom: PhantomData,}}+}+impl<S, D> TopDocsCollector<S, D> {pub fn with_shard_id(self, shard_id: ShardId) -> Self {Self {shard_id: Some(shard_id),@@ -107,7 +223,35 @@ impl<S: DocumentScorer> TopDocsCollector<S> {}}- fn computer(&self) -> Computer {+ pub fn with_deduplicator<D2: Deduplicator>(self, deduplicator: D2) -> TopDocsCollector<S, D2> {+ TopDocsCollector {+ deduplicator,+ shard_id: self.shard_id,+ limit: self.limit,+ offset: self.offset,+ perform_offset: self.perform_offset,+ column_fields: self.column_fields,+ _phantom: PhantomData,+ }+ }++ pub fn with_column_fields<F: Field>(+ self,+ warmed_column_fields: WarmedColumnFields,+ host_field: F,+ ) -> Self {+ Self {+ column_fields: Some(ColumnFields::new(warmed_column_fields, host_field)),+ ..self+ }+ }+}++impl<S, D> TopDocsCollector<S, D>+where+ D: Deduplicator,+{+ fn computer(&self) -> Computer<D> {match (self.offset, self.limit) {(Some(offset), Some(limit)) => Computer::TopN(TopNComputer::new(limit + offset)),(Some(_), None) => Computer::All(AllComputer::new()),@@ -117,10 +261,10 @@ impl<S: DocumentScorer> TopDocsCollector<S> {}}-impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {- type Fruit = Vec<(tantivy::Score, DocAddress)>;+impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDocsCollector<S, D> {+ type Fruit = Vec<(tantivy::Score, <D as Deduplicator>::Doc)>;- type Child = TopDocsSegmentCollector<S>;+ type Child = TopDocsSegmentCollector<S, D>;fn for_segment(&self,@@ -129,11 +273,20 @@ impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {) -> crate::Result<Self::Child> {let scorer = S::for_segment(segment)?;+ let segment_id = segment.segment_id();+Ok(TopDocsSegmentCollector {shard_id: self.shard_id.unwrap(),computer: self.computer(),segment_ord,scorer,+ host_column: self.column_fields.as_ref().map(|cf| {+ cf.warmed_column_fields+ .segment(&segment_id)+ .u64_by_enum(cf.host_field)+ .unwrap()+ }),+ _deduplicator: PhantomData,})}@@ -143,10 +296,11 @@ impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {) -> crate::Result<Self::Fruit> {let mut computer = self.computer();- for fruit in segment_fruits {- for (score, doc) in fruit {- computer.push(score, doc);- }+ for (score, doc) in self+ .deduplicator+ .deduplicate(segment_fruits.into_iter().flatten().collect())+ {+ computer.push(score, doc);}let mut result = computer.harvest();@@ -159,20 +313,20 @@ impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {}}-enum Computer {- TopN(TopNComputer<tantivy::Score, DocAddress>),- All(AllComputer),+enum Computer<D: Deduplicator> {+ TopN(TopNComputer<tantivy::Score, <D as Deduplicator>::Doc>),+ All(AllComputer<D>),}-impl Computer {- fn push(&mut self, score: tantivy::Score, doc: DocAddress) {+impl<D: Deduplicator> Computer<D> {+ fn push(&mut self, score: tantivy::Score, doc: <D as Deduplicator>::Doc) {match self {Computer::TopN(computer) => computer.push(score, doc),Computer::All(computer) => computer.push(score, doc),}}- fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {+ fn harvest(self) -> Vec<(tantivy::Score, <D as Deduplicator>::Doc)> {match self {Computer::TopN(computer) => computer.into_sorted_vec()@@ -184,35 +338,39 @@ impl Computer {}}-struct AllComputer {- docs: Vec<(tantivy::Score, DocAddress)>,+struct AllComputer<D: Deduplicator> {+ docs: Vec<(tantivy::Score, <D as Deduplicator>::Doc)>,}-impl AllComputer {+impl<D: Deduplicator> AllComputer<D> {fn new() -> Self {Self { docs: Vec::new() }}- fn push(&mut self, score: tantivy::Score, doc: DocAddress) {+ fn push(&mut self, score: tantivy::Score, doc: <D as Deduplicator>::Doc) {self.docs.push((score, doc));}- fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {+ fn harvest(self) -> Vec<(tantivy::Score, <D as Deduplicator>::Doc)> {let mut docs = self.docs;docs.sort_by(|(score1, _), (score2, _)| score2.total_cmp(score1));docs}}-pub struct TopDocsSegmentCollector<S: DocumentScorer> {+pub struct TopDocsSegmentCollector<S: DocumentScorer, D: Deduplicator> {shard_id: ShardId,- computer: Computer,+ computer: Computer<D>,segment_ord: SegmentOrdinal,scorer: S,+ host_column: Option<Column<u64>>,+ _deduplicator: PhantomData<D>,}-impl<S: DocumentScorer + 'static> SegmentCollector for TopDocsSegmentCollector<S> {- type Fruit = Vec<(tantivy::Score, DocAddress)>;+impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> SegmentCollector+ for TopDocsSegmentCollector<S, D>+{+ type Fruit = Vec<(tantivy::Score, <D as Deduplicator>::Doc)>;fn collect(&mut self, doc: DocId, _: tantivy::Score) {if doc == tantivy::TERMINATED {@@ -221,10 +379,124 @@ impl<S: DocumentScorer + 'static> SegmentCollector for TopDocsSegmentCollector<Slet score = self.scorer.score(doc);self.computer- .push(score, DocAddress::new(self.shard_id, self.segment_ord, doc));+ .push(score, <D::Doc as DeduplicatorDoc>::new(self, doc));}fn harvest(self) -> Self::Fruit {self.computer.harvest()}}++#[cfg(test)]+mod tests {+ use crate::{+ webgraph::{+ query::{BacklinksQuery, HostBacklinksQuery},+ Edge, Node, Webgraph,+ },+ webpage::RelFlags,+ };++ #[test]+ fn test_simple() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/1"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph+ .search(&BacklinksQuery::new(Node::from("https://B.com/1").id()))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert!(res[0].from == Node::from("https://A.com/1").id());+ }++ #[test]+ fn test_deduplication() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/1"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();+ graph+ .insert(Edge {+ from: Node::from("https://A.com/2"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph+ .search(&BacklinksQuery::new(Node::from("https://B.com/1").id()))+ .unwrap();++ assert_eq!(res.len(), 2);++ let res = graph+ .search(&HostBacklinksQuery::new(+ Node::from("https://B.com/").into_host().id(),+ ))+ .unwrap();++ assert_eq!(res.len(), 1);+ }++ #[test]+ fn test_deduplication_across_segments() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/1"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();+ graph.commit().unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/2"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph+ .search(&HostBacklinksQuery::new(+ Node::from("https://B.com/").into_host().id(),+ ))+ .unwrap();++ assert_eq!(res.len(), 1);+ }+}diff --git a/crates/core/src/webgraph/query/degree.rs b/crates/core/src/webgraph/query/degree.rsindex 7f3d1f7a..ec8947e0 100644--- a/crates/core/src/webgraph/query/degree.rs+++ b/crates/core/src/webgraph/query/degree.rs@@ -45,12 +45,12 @@ impl Query for InDegreeQuery {raw::DummyQuery}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {FastCountCollector::new(ToId.name().to_string(),FastCountValue::U64(self.node.as_u64()),)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard())}fn remote_collector(&self) -> Self::Collector {diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rsindex bef7dc2d..54cb89e5 100644--- a/crates/core/src/webgraph/query/forwardlink.rs+++ b/crates/core/src/webgraph/query/forwardlink.rs@@ -16,12 +16,12 @@use super::{backlink::{fetch_edges, fetch_small_edges},- collector::TopDocsCollector,+ collector::{HostDeduplicator, TopDocsCollector},+ document_scorer::DefaultDocumentScorer,raw::{host_links::HostLinksQuery, links::LinksQuery},Query,};use crate::{- ampc::dht::ShardId,webgraph::{document::Edge,schema::{FromHostId, FromId, ToHostId, ToId},@@ -69,9 +69,9 @@ impl Query for ForwardlinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()}@@ -142,7 +142,7 @@ impl HostForwardlinksQuery {}impl Query for HostForwardlinksQuery {- type Collector = TopDocsCollector;+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;type TantivyQuery = Box<dyn tantivy::query::Query>;type IntermediateOutput = Vec<(f64, SmallEdge)>;type Output = Vec<SmallEdge>;@@ -169,14 +169,18 @@ impl Query for HostForwardlinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()+ .with_deduplicator(HostDeduplicator)+ .with_column_fields(searcher.warmed_column_fields().clone(), ToHostId)}fn remote_collector(&self) -> Self::Collector {- TopDocsCollector::from(self.limit).enable_offset()+ TopDocsCollector::from(self.limit)+ .enable_offset()+ .with_deduplicator(HostDeduplicator)}fn filter_fruit_shards(@@ -186,7 +190,7 @@ impl Query for HostForwardlinksQuery {) -> <Self::Collector as super::Collector>::Fruit {fruit.into_iter()- .filter(|(_, doc)| doc.shard_id == shard)+ .filter(|(_, doc)| doc.address.shard_id == shard).collect()}@@ -195,7 +199,7 @@ impl Query for HostForwardlinksQuery {searcher: &Searcher,fruit: <Self::Collector as super::collector::Collector>::Fruit,) -> Result<Self::IntermediateOutput> {- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();let nodes = fetch_small_edges(searcher, docs, ToHostId)?;Ok(nodes.into_iter()@@ -259,9 +263,9 @@ impl Query for FullForwardlinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()}@@ -320,7 +324,7 @@ impl FullHostForwardlinksQuery {}impl Query for FullHostForwardlinksQuery {- type Collector = TopDocsCollector;+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;type TantivyQuery = Box<dyn tantivy::query::Query>;type IntermediateOutput = Vec<Edge>;type Output = Vec<Edge>;@@ -347,14 +351,18 @@ impl Query for FullHostForwardlinksQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {+ fn collector(&self, searcher: &Searcher) -> Self::Collector {TopDocsCollector::from(self.limit)- .with_shard_id(shard_id)+ .with_shard_id(searcher.shard()).disable_offset()+ .with_deduplicator(HostDeduplicator)+ .with_column_fields(searcher.warmed_column_fields().clone(), ToHostId)}fn remote_collector(&self) -> Self::Collector {- TopDocsCollector::from(self.limit).enable_offset()+ TopDocsCollector::from(self.limit)+ .enable_offset()+ .with_deduplicator(HostDeduplicator)}fn filter_fruit_shards(@@ -364,7 +372,7 @@ impl Query for FullHostForwardlinksQuery {) -> <Self::Collector as super::Collector>::Fruit {fruit.into_iter()- .filter(|(_, doc)| doc.shard_id == shard)+ .filter(|(_, doc)| doc.address.shard_id == shard).collect()}@@ -373,7 +381,7 @@ impl Query for FullHostForwardlinksQuery {searcher: &Searcher,fruit: <Self::Collector as super::collector::Collector>::Fruit,) -> Result<Self::IntermediateOutput> {- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();let edges = fetch_edges(searcher, docs)?;Ok(edges.into_iter()diff --git a/crates/core/src/webgraph/query/id2node.rs b/crates/core/src/webgraph/query/id2node.rsindex 58d95910..37780739 100644--- a/crates/core/src/webgraph/query/id2node.rs+++ b/crates/core/src/webgraph/query/id2node.rs@@ -16,7 +16,6 @@use super::collector::FirstDocCollector;use super::Query;-use crate::ampc::dht::ShardId;use crate::webgraph::query::raw;use crate::webgraph::schema::{FromHostId, FromId, ToHostId, ToId};use crate::webgraph::searcher::Searcher;@@ -43,8 +42,8 @@ impl Query for Id2NodeQuery {}}- fn collector(&self, shard_id: ShardId) -> Self::Collector {- FirstDocCollector::with_shard_id(shard_id)+ fn collector(&self, searcher: &Searcher) -> Self::Collector {+ FirstDocCollector::with_shard_id(searcher.shard())}fn retrieve(diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rsindex a826dd7b..3de5afe9 100644--- a/crates/core/src/webgraph/query/mod.rs+++ b/crates/core/src/webgraph/query/mod.rs@@ -39,7 +39,7 @@ pub trait Query: Send + Sync + bincode::Encode + bincode::Decode + Clone {type Output;fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery;- fn collector(&self, shard_id: ShardId) -> Self::Collector;+ fn collector(&self, searcher: &Searcher) -> Self::Collector;fn remote_collector(&self) -> Self::Collector;fn filter_fruit_shards(diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rsindex 91118035..4617d60b 100644--- a/crates/core/src/webgraph/query/raw/host_links.rs+++ b/crates/core/src/webgraph/query/raw/host_links.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use rustc_hash::FxHashSet;use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};use crate::webgraph::{@@ -83,6 +82,7 @@ impl tantivy::query::Weight for HostLinksWeight {term,self.deduplication_field,&self.warmed_column_fields,+ self.node.as_u64(),) {Ok(Some(scorer)) => Ok(Box::new(scorer)),_ => Ok(Box::new(tantivy::query::EmptyScorer)),@@ -101,7 +101,8 @@ impl tantivy::query::Weight for HostLinksWeight {struct HostLinksScorer {postings: SegmentPostings,host_id_column: Column<u64>,- seen_host_ids: FxHashSet<u64>,+ last_host_id: Option<u64>,+ self_host_id: u64,}impl HostLinksScorer {@@ -110,6 +111,7 @@ impl HostLinksScorer {term: tantivy::Term,deduplication_field: FieldEnum,warmed_column_fields: &WarmedColumnFields,+ self_host_id: u64,) -> tantivy::Result<Option<Self>> {let host_id_column = warmed_column_fields.segment(&reader.segment_id())@@ -119,18 +121,11 @@ impl HostLinksScorer {Ok(reader.inverted_index(term.field())?.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?- .map(|postings| {- let mut seen_host_ids = FxHashSet::default();-- if let Some(host_id) = host_id_column.first(postings.doc()) {- seen_host_ids.insert(host_id);- }-- Self {- seen_host_ids,- host_id_column,- postings,- }+ .map(|postings| Self {+ last_host_id: host_id_column.first(postings.doc()),+ host_id_column,+ postings,+ self_host_id,}))}}@@ -145,9 +140,13 @@ impl HostLinksScorer {fn has_seen_host(&self, doc: tantivy::DocId) -> bool {self.host_id(doc)- .map(|host_id| self.seen_host_ids.contains(&host_id))+ .map(|host_id| self.last_host_id == Some(host_id)).unwrap_or(false)}++ fn is_self_host(&self, doc: tantivy::DocId) -> bool {+ self.host_id(doc) == Some(self.self_host_id)+ }}impl tantivy::query::Scorer for HostLinksScorer {@@ -170,15 +169,17 @@ impl tantivy::DocSet for HostLinksScorer {self.postings.reset_cursor_start_block();}- while self.has_seen_host(self.postings.doc()) && self.doc() != tantivy::TERMINATED {+ while (self.has_seen_host(self.doc()) || self.is_self_host(self.doc()))+ && self.doc() != tantivy::TERMINATED+ {self.postings.advance();}- if let Some(host_id) = self.host_id(self.postings.doc()) {- self.seen_host_ids.insert(host_id);+ if let Some(host_id) = self.host_id(self.doc()) {+ self.last_host_id = Some(host_id);}- self.postings.doc()+ self.doc()}fn doc(&self) -> tantivy::DocId {@@ -189,3 +190,132 @@ impl tantivy::DocSet for HostLinksScorer {self.postings.size_hint()}}++#[cfg(test)]+mod tests {+ use crate::{+ webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph},+ webpage::RelFlags,+ };++ #[test]+ fn test_simple() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ let node_a = Node::from("A");+ let node_b = Node::from("B");+ let node_c = Node::from("C");++ graph+ .insert(Edge {+ from: node_a.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: node_c.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();++ assert_eq!(res.len(), 2);+ assert!(res.iter().any(|r| r.from == node_a.id()));+ assert!(res.iter().any(|r| r.from == node_c.id()));+ }++ #[test]+ fn test_self_host_skipped() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ let node_a = Node::from("A");+ let node_b = Node::from("B");++ graph+ .insert(Edge {+ from: node_a.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: node_b.clone(),+ to: node_b.clone(),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();++ assert_eq!(res.len(), 1);+ assert!(res[0].from == node_a.id());+ }++ #[test]+ fn test_deduplication() {+ let temp_dir = crate::gen_temp_dir().unwrap();+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/1"),+ to: Node::from("https://B.com/1"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/2"),+ to: Node::from("https://B.com/2"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph+ .insert(Edge {+ from: Node::from("https://A.com/3"),+ to: Node::from("https://B.com/3"),+ rel_flags: RelFlags::default(),+ label: String::new(),+ sort_score: 0.0,+ })+ .unwrap();++ graph.commit().unwrap();++ let res = graph+ .search(&HostBacklinksQuery::new(+ Node::from("https://B.com/").into_host().id(),+ ))+ .unwrap();++ assert_eq!(res.len(), 1);+ assert!(res[0].from == Node::from("https://A.com/").into_host().id());+ }+}diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex d850d973..ce55660a 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -240,7 +240,7 @@ impl EdgeStore {let res = searcher.tantivy_searcher().search(&query.tantivy_query(&searcher),- &TantivyCollector::from(&query.collector(self.shard_id)),+ &TantivyCollector::from(&query.collector(&searcher)),)?;Ok(res)diff --git a/crates/core/src/webgraph/warmed_column_fields.rs b/crates/core/src/webgraph/warmed_column_fields.rsindex fd9980a0..5da45820 100644--- a/crates/core/src/webgraph/warmed_column_fields.rs+++ b/crates/core/src/webgraph/warmed_column_fields.rs@@ -29,16 +29,22 @@ pub struct SegmentColumnFields {impl SegmentColumnFields {pub fn u64<F: Field>(&self, field: F) -> Option<Column<u64>> {- let field_enum: FieldEnum = field.into();+ self.u64_by_enum(field.into())+ }++ pub fn u64_by_enum(&self, field: FieldEnum) -> Option<Column<u64>> {self.u64_fields- .get(FieldEnumDiscriminants::from(field_enum))+ .get(FieldEnumDiscriminants::from(field)).cloned()}pub fn f64<F: Field>(&self, field: F) -> Option<Column<f64>> {- let field_enum: FieldEnum = field.into();+ self.f64_by_enum(field.into())+ }++ pub fn f64_by_enum(&self, field: FieldEnum) -> Option<Column<f64>> {self.f64_fields- .get(FieldEnumDiscriminants::from(field_enum))+ .get(FieldEnumDiscriminants::from(field)).cloned()}}