Skip to content

web search done right

deduplicate host edges across segment+shard results

Mikkel Denkerc5bd49df5122
diff --git a/crates/core/src/webgraph/query/backlink.rs b/crates/core/src/webgraph/query/backlink.rs
index 226e5a55..78c7d24c 100644
--- a/crates/core/src/webgraph/query/backlink.rs
+++ b/crates/core/src/webgraph/query/backlink.rs
@@ -17,7 +17,8 @@
use tantivy::query::ShortCircuitQuery;
use super::{
- collector::TopDocsCollector,
+ collector::{HostDeduplicator, TopDocsCollector},
+ document_scorer::DefaultDocumentScorer,
raw::{HostLinksQuery, LinksQuery},
Query,
};
@@ -129,9 +130,9 @@ impl Query for BacklinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
}
@@ -200,7 +201,7 @@ impl HostBacklinksQuery {
}
impl Query for HostBacklinksQuery {
- type Collector = TopDocsCollector;
+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;
type TantivyQuery = Box<dyn tantivy::query::Query>;
type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
@@ -227,14 +228,18 @@ impl Query for HostBacklinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
+ .with_deduplicator(HostDeduplicator)
+ .with_column_fields(searcher.warmed_column_fields().clone(), FromHostId)
}
fn remote_collector(&self) -> Self::Collector {
- TopDocsCollector::from(self.limit).enable_offset()
+ TopDocsCollector::from(self.limit)
+ .enable_offset()
+ .with_deduplicator(HostDeduplicator)
}
fn retrieve(
@@ -242,7 +247,7 @@ impl Query for HostBacklinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();
let nodes = fetch_small_edges(searcher, docs, FromHostId)?;
Ok(nodes
.into_iter()
@@ -266,7 +271,7 @@ impl Query for HostBacklinksQuery {
) -> <Self::Collector as super::Collector>::Fruit {
fruit
.into_iter()
- .filter(|(_, doc)| doc.shard_id == shard_id)
+ .filter(|(_, doc)| doc.address.shard_id == shard_id)
.collect()
}
@@ -317,9 +322,9 @@ impl Query for FullBacklinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
}
@@ -378,7 +383,7 @@ impl FullHostBacklinksQuery {
}
impl Query for FullHostBacklinksQuery {
- type Collector = TopDocsCollector;
+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;
type TantivyQuery = Box<dyn tantivy::query::Query>;
type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
@@ -405,14 +410,18 @@ impl Query for FullHostBacklinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
+ .with_deduplicator(HostDeduplicator)
+ .with_column_fields(searcher.warmed_column_fields().clone(), FromHostId)
}
fn remote_collector(&self) -> Self::Collector {
- TopDocsCollector::from(self.limit).enable_offset()
+ TopDocsCollector::from(self.limit)
+ .enable_offset()
+ .with_deduplicator(HostDeduplicator)
}
fn retrieve(
@@ -420,7 +429,7 @@ impl Query for FullHostBacklinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();
let edges = fetch_edges(searcher, docs)?;
Ok(edges
@@ -440,7 +449,7 @@ impl Query for FullHostBacklinksQuery {
) -> <Self::Collector as super::Collector>::Fruit {
fruit
.into_iter()
- .filter(|(_, doc)| doc.shard_id == shard_id)
+ .filter(|(_, doc)| doc.address.shard_id == shard_id)
.collect()
}
@@ -482,9 +491,9 @@ impl Query for BacklinksWithLabelsQuery {
Box::new(LinksQuery::new(self.node, ToId))
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
}
diff --git a/crates/core/src/webgraph/query/collector/mod.rs b/crates/core/src/webgraph/query/collector/mod.rs
index b1478590..0b35246d 100644
--- a/crates/core/src/webgraph/query/collector/mod.rs
+++ b/crates/core/src/webgraph/query/collector/mod.rs
@@ -20,7 +20,7 @@ mod top_docs;
pub use fast_count::{FastCountCollector, FastCountValue};
pub use first_doc::FirstDocCollector;
-pub use top_docs::TopDocsCollector;
+pub use top_docs::{HostDeduplicator, TopDocsCollector};
use tantivy::{
collector::{Fruit, SegmentCollector},
diff --git a/crates/core/src/webgraph/query/collector/top_docs.rs b/crates/core/src/webgraph/query/collector/top_docs.rs
index dda3d705..15f4dae0 100644
--- a/crates/core/src/webgraph/query/collector/top_docs.rs
+++ b/crates/core/src/webgraph/query/collector/top_docs.rs
@@ -16,29 +16,141 @@
use std::marker::PhantomData;
+use anyhow::anyhow;
+use itertools::Itertools;
use tantivy::{
collector::{SegmentCollector, TopNComputer},
+ columnar::Column,
DocId, SegmentOrdinal,
};
-use crate::distributed::member::ShardId;
use crate::webgraph::{
doc_address::DocAddress,
query::document_scorer::{DefaultDocumentScorer, DocumentScorer},
+ schema::{Field, FieldEnum},
EdgeLimit,
};
+use crate::{distributed::member::ShardId, webgraph::warmed_column_fields::WarmedColumnFields};
use super::Collector;
-pub struct TopDocsCollector<S: DocumentScorer = DefaultDocumentScorer> {
+pub trait DeduplicatorDoc
+where
+ Self: Send + Sync + serde::Serialize + serde::de::DeserializeOwned + Ord + Clone,
+{
+ fn new<S, D>(collector: &TopDocsSegmentCollector<S, D>, doc: DocId) -> Self
+ where
+ S: DocumentScorer + 'static,
+ D: Deduplicator + 'static;
+}
+
+pub trait Deduplicator: Clone + Send + Sync {
+ type Doc: DeduplicatorDoc;
+
+ fn deduplicate(
+ &self,
+ docs: Vec<(tantivy::Score, Self::Doc)>,
+ ) -> Vec<(tantivy::Score, Self::Doc)>;
+}
+
+impl DeduplicatorDoc for DocAddress {
+ fn new<S, D>(collector: &TopDocsSegmentCollector<S, D>, doc: DocId) -> Self
+ where
+ S: DocumentScorer + 'static,
+ D: Deduplicator + 'static,
+ {
+ DocAddress::new(collector.shard_id, collector.segment_ord, doc)
+ }
+}
+
+#[derive(Clone)]
+pub struct NoDeduplicator;
+
+impl Deduplicator for NoDeduplicator {
+ type Doc = DocAddress;
+
+ fn deduplicate(
+ &self,
+ docs: Vec<(tantivy::Score, Self::Doc)>,
+ ) -> Vec<(tantivy::Score, Self::Doc)> {
+ docs
+ }
+}
+
+#[derive(
+ Clone,
+ serde::Serialize,
+ serde::Deserialize,
+ bincode::Encode,
+ bincode::Decode,
+ Ord,
+ PartialOrd,
+ Eq,
+ PartialEq,
+)]
+pub struct DocAddressWithHost {
+ pub address: DocAddress,
+ pub host: u64,
+}
+
+impl DeduplicatorDoc for DocAddressWithHost {
+ fn new<S, D>(collector: &TopDocsSegmentCollector<S, D>, doc: DocId) -> Self
+ where
+ S: DocumentScorer + 'static,
+ D: Deduplicator + 'static,
+ {
+ let host = collector
+ .host_column
+ .as_ref()
+ .and_then(|col| col.first(doc))
+ .ok_or_else(|| anyhow!("ColumnFields must be set to use HostDeduplicator"))
+ .unwrap();
+ Self {
+ address: DocAddress::new(collector.shard_id, collector.segment_ord, doc),
+ host,
+ }
+ }
+}
+
+#[derive(Clone)]
+pub struct HostDeduplicator;
+
+impl Deduplicator for HostDeduplicator {
+ type Doc = DocAddressWithHost;
+
+ fn deduplicate(
+ &self,
+ docs: Vec<(tantivy::Score, Self::Doc)>,
+ ) -> Vec<(tantivy::Score, Self::Doc)> {
+ docs.into_iter().unique_by(|(_, doc)| doc.host).collect()
+ }
+}
+
+pub struct ColumnFields {
+ warmed_column_fields: WarmedColumnFields,
+ host_field: FieldEnum,
+}
+
+impl ColumnFields {
+ pub fn new<F: Field>(warmed_column_fields: WarmedColumnFields, host_field: F) -> Self {
+ Self {
+ warmed_column_fields,
+ host_field: host_field.into(),
+ }
+ }
+}
+
+pub struct TopDocsCollector<S = DefaultDocumentScorer, D = NoDeduplicator> {
shard_id: Option<ShardId>,
limit: Option<usize>,
offset: Option<usize>,
perform_offset: bool,
+ deduplicator: D,
+ column_fields: Option<ColumnFields>,
_phantom: PhantomData<S>,
}
-impl<S: DocumentScorer> From<EdgeLimit> for TopDocsCollector<S> {
+impl<S> From<EdgeLimit> for TopDocsCollector<S, NoDeduplicator> {
fn from(limit: EdgeLimit) -> Self {
let mut collector = TopDocsCollector::new().disable_offset();
@@ -55,23 +167,27 @@ impl<S: DocumentScorer> From<EdgeLimit> for TopDocsCollector<S> {
}
}
-impl<S: DocumentScorer> Default for TopDocsCollector<S> {
+impl<S> Default for TopDocsCollector<S, NoDeduplicator> {
fn default() -> Self {
Self::new()
}
}
-impl<S: DocumentScorer> TopDocsCollector<S> {
+impl<S> TopDocsCollector<S, NoDeduplicator> {
pub fn new() -> Self {
Self {
shard_id: None,
limit: None,
offset: None,
perform_offset: true,
+ deduplicator: NoDeduplicator,
+ column_fields: None,
_phantom: PhantomData,
}
}
+}
+impl<S, D> TopDocsCollector<S, D> {
pub fn with_shard_id(self, shard_id: ShardId) -> Self {
Self {
shard_id: Some(shard_id),
@@ -107,7 +223,35 @@ impl<S: DocumentScorer> TopDocsCollector<S> {
}
}
- fn computer(&self) -> Computer {
+ pub fn with_deduplicator<D2: Deduplicator>(self, deduplicator: D2) -> TopDocsCollector<S, D2> {
+ TopDocsCollector {
+ deduplicator,
+ shard_id: self.shard_id,
+ limit: self.limit,
+ offset: self.offset,
+ perform_offset: self.perform_offset,
+ column_fields: self.column_fields,
+ _phantom: PhantomData,
+ }
+ }
+
+ pub fn with_column_fields<F: Field>(
+ self,
+ warmed_column_fields: WarmedColumnFields,
+ host_field: F,
+ ) -> Self {
+ Self {
+ column_fields: Some(ColumnFields::new(warmed_column_fields, host_field)),
+ ..self
+ }
+ }
+}
+
+impl<S, D> TopDocsCollector<S, D>
+where
+ D: Deduplicator,
+{
+ fn computer(&self) -> Computer<D> {
match (self.offset, self.limit) {
(Some(offset), Some(limit)) => Computer::TopN(TopNComputer::new(limit + offset)),
(Some(_), None) => Computer::All(AllComputer::new()),
@@ -117,10 +261,10 @@ impl<S: DocumentScorer> TopDocsCollector<S> {
}
}
-impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {
- type Fruit = Vec<(tantivy::Score, DocAddress)>;
+impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> Collector for TopDocsCollector<S, D> {
+ type Fruit = Vec<(tantivy::Score, <D as Deduplicator>::Doc)>;
- type Child = TopDocsSegmentCollector<S>;
+ type Child = TopDocsSegmentCollector<S, D>;
fn for_segment(
&self,
@@ -129,11 +273,20 @@ impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {
) -> crate::Result<Self::Child> {
let scorer = S::for_segment(segment)?;
+ let segment_id = segment.segment_id();
+
Ok(TopDocsSegmentCollector {
shard_id: self.shard_id.unwrap(),
computer: self.computer(),
segment_ord,
scorer,
+ host_column: self.column_fields.as_ref().map(|cf| {
+ cf.warmed_column_fields
+ .segment(&segment_id)
+ .u64_by_enum(cf.host_field)
+ .unwrap()
+ }),
+ _deduplicator: PhantomData,
})
}
@@ -143,10 +296,11 @@ impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {
) -> crate::Result<Self::Fruit> {
let mut computer = self.computer();
- for fruit in segment_fruits {
- for (score, doc) in fruit {
- computer.push(score, doc);
- }
+ for (score, doc) in self
+ .deduplicator
+ .deduplicate(segment_fruits.into_iter().flatten().collect())
+ {
+ computer.push(score, doc);
}
let mut result = computer.harvest();
@@ -159,20 +313,20 @@ impl<S: DocumentScorer + 'static> Collector for TopDocsCollector<S> {
}
}
-enum Computer {
- TopN(TopNComputer<tantivy::Score, DocAddress>),
- All(AllComputer),
+enum Computer<D: Deduplicator> {
+ TopN(TopNComputer<tantivy::Score, <D as Deduplicator>::Doc>),
+ All(AllComputer<D>),
}
-impl Computer {
- fn push(&mut self, score: tantivy::Score, doc: DocAddress) {
+impl<D: Deduplicator> Computer<D> {
+ fn push(&mut self, score: tantivy::Score, doc: <D as Deduplicator>::Doc) {
match self {
Computer::TopN(computer) => computer.push(score, doc),
Computer::All(computer) => computer.push(score, doc),
}
}
- fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {
+ fn harvest(self) -> Vec<(tantivy::Score, <D as Deduplicator>::Doc)> {
match self {
Computer::TopN(computer) => computer
.into_sorted_vec()
@@ -184,35 +338,39 @@ impl Computer {
}
}
-struct AllComputer {
- docs: Vec<(tantivy::Score, DocAddress)>,
+struct AllComputer<D: Deduplicator> {
+ docs: Vec<(tantivy::Score, <D as Deduplicator>::Doc)>,
}
-impl AllComputer {
+impl<D: Deduplicator> AllComputer<D> {
fn new() -> Self {
Self { docs: Vec::new() }
}
- fn push(&mut self, score: tantivy::Score, doc: DocAddress) {
+ fn push(&mut self, score: tantivy::Score, doc: <D as Deduplicator>::Doc) {
self.docs.push((score, doc));
}
- fn harvest(self) -> Vec<(tantivy::Score, DocAddress)> {
+ fn harvest(self) -> Vec<(tantivy::Score, <D as Deduplicator>::Doc)> {
let mut docs = self.docs;
docs.sort_by(|(score1, _), (score2, _)| score2.total_cmp(score1));
docs
}
}
-pub struct TopDocsSegmentCollector<S: DocumentScorer> {
+pub struct TopDocsSegmentCollector<S: DocumentScorer, D: Deduplicator> {
shard_id: ShardId,
- computer: Computer,
+ computer: Computer<D>,
segment_ord: SegmentOrdinal,
scorer: S,
+ host_column: Option<Column<u64>>,
+ _deduplicator: PhantomData<D>,
}
-impl<S: DocumentScorer + 'static> SegmentCollector for TopDocsSegmentCollector<S> {
- type Fruit = Vec<(tantivy::Score, DocAddress)>;
+impl<S: DocumentScorer + 'static, D: Deduplicator + 'static> SegmentCollector
+ for TopDocsSegmentCollector<S, D>
+{
+ type Fruit = Vec<(tantivy::Score, <D as Deduplicator>::Doc)>;
fn collect(&mut self, doc: DocId, _: tantivy::Score) {
if doc == tantivy::TERMINATED {
@@ -221,10 +379,124 @@ impl<S: DocumentScorer + 'static> SegmentCollector for TopDocsSegmentCollector<S
let score = self.scorer.score(doc);
self.computer
- .push(score, DocAddress::new(self.shard_id, self.segment_ord, doc));
+ .push(score, <D::Doc as DeduplicatorDoc>::new(self, doc));
}
fn harvest(self) -> Self::Fruit {
self.computer.harvest()
}
}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ webgraph::{
+ query::{BacklinksQuery, HostBacklinksQuery},
+ Edge, Node, Webgraph,
+ },
+ webpage::RelFlags,
+ };
+
+ #[test]
+ fn test_simple() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/1"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph
+ .search(&BacklinksQuery::new(Node::from("https://B.com/1").id()))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert!(res[0].from == Node::from("https://A.com/1").id());
+ }
+
+ #[test]
+ fn test_deduplication() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/1"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/2"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph
+ .search(&BacklinksQuery::new(Node::from("https://B.com/1").id()))
+ .unwrap();
+
+ assert_eq!(res.len(), 2);
+
+ let res = graph
+ .search(&HostBacklinksQuery::new(
+ Node::from("https://B.com/").into_host().id(),
+ ))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ }
+
+ #[test]
+ fn test_deduplication_across_segments() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/1"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+ graph.commit().unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/2"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph
+ .search(&HostBacklinksQuery::new(
+ Node::from("https://B.com/").into_host().id(),
+ ))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ }
+}
diff --git a/crates/core/src/webgraph/query/degree.rs b/crates/core/src/webgraph/query/degree.rs
index 7f3d1f7a..ec8947e0 100644
--- a/crates/core/src/webgraph/query/degree.rs
+++ b/crates/core/src/webgraph/query/degree.rs
@@ -45,12 +45,12 @@ impl Query for InDegreeQuery {
raw::DummyQuery
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
FastCountCollector::new(
ToId.name().to_string(),
FastCountValue::U64(self.node.as_u64()),
)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
}
fn remote_collector(&self) -> Self::Collector {
diff --git a/crates/core/src/webgraph/query/forwardlink.rs b/crates/core/src/webgraph/query/forwardlink.rs
index bef7dc2d..54cb89e5 100644
--- a/crates/core/src/webgraph/query/forwardlink.rs
+++ b/crates/core/src/webgraph/query/forwardlink.rs
@@ -16,12 +16,12 @@
use super::{
backlink::{fetch_edges, fetch_small_edges},
- collector::TopDocsCollector,
+ collector::{HostDeduplicator, TopDocsCollector},
+ document_scorer::DefaultDocumentScorer,
raw::{host_links::HostLinksQuery, links::LinksQuery},
Query,
};
use crate::{
- ampc::dht::ShardId,
webgraph::{
document::Edge,
schema::{FromHostId, FromId, ToHostId, ToId},
@@ -69,9 +69,9 @@ impl Query for ForwardlinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
}
@@ -142,7 +142,7 @@ impl HostForwardlinksQuery {
}
impl Query for HostForwardlinksQuery {
- type Collector = TopDocsCollector;
+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;
type TantivyQuery = Box<dyn tantivy::query::Query>;
type IntermediateOutput = Vec<(f64, SmallEdge)>;
type Output = Vec<SmallEdge>;
@@ -169,14 +169,18 @@ impl Query for HostForwardlinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
+ .with_deduplicator(HostDeduplicator)
+ .with_column_fields(searcher.warmed_column_fields().clone(), ToHostId)
}
fn remote_collector(&self) -> Self::Collector {
- TopDocsCollector::from(self.limit).enable_offset()
+ TopDocsCollector::from(self.limit)
+ .enable_offset()
+ .with_deduplicator(HostDeduplicator)
}
fn filter_fruit_shards(
@@ -186,7 +190,7 @@ impl Query for HostForwardlinksQuery {
) -> <Self::Collector as super::Collector>::Fruit {
fruit
.into_iter()
- .filter(|(_, doc)| doc.shard_id == shard)
+ .filter(|(_, doc)| doc.address.shard_id == shard)
.collect()
}
@@ -195,7 +199,7 @@ impl Query for HostForwardlinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();
let nodes = fetch_small_edges(searcher, docs, ToHostId)?;
Ok(nodes
.into_iter()
@@ -259,9 +263,9 @@ impl Query for FullForwardlinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
}
@@ -320,7 +324,7 @@ impl FullHostForwardlinksQuery {
}
impl Query for FullHostForwardlinksQuery {
- type Collector = TopDocsCollector;
+ type Collector = TopDocsCollector<DefaultDocumentScorer, HostDeduplicator>;
type TantivyQuery = Box<dyn tantivy::query::Query>;
type IntermediateOutput = Vec<Edge>;
type Output = Vec<Edge>;
@@ -347,14 +351,18 @@ impl Query for FullHostForwardlinksQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
TopDocsCollector::from(self.limit)
- .with_shard_id(shard_id)
+ .with_shard_id(searcher.shard())
.disable_offset()
+ .with_deduplicator(HostDeduplicator)
+ .with_column_fields(searcher.warmed_column_fields().clone(), ToHostId)
}
fn remote_collector(&self) -> Self::Collector {
- TopDocsCollector::from(self.limit).enable_offset()
+ TopDocsCollector::from(self.limit)
+ .enable_offset()
+ .with_deduplicator(HostDeduplicator)
}
fn filter_fruit_shards(
@@ -364,7 +372,7 @@ impl Query for FullHostForwardlinksQuery {
) -> <Self::Collector as super::Collector>::Fruit {
fruit
.into_iter()
- .filter(|(_, doc)| doc.shard_id == shard)
+ .filter(|(_, doc)| doc.address.shard_id == shard)
.collect()
}
@@ -373,7 +381,7 @@ impl Query for FullHostForwardlinksQuery {
searcher: &Searcher,
fruit: <Self::Collector as super::collector::Collector>::Fruit,
) -> Result<Self::IntermediateOutput> {
- let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc).collect();
+ let docs: Vec<_> = fruit.into_iter().map(|(_, doc)| doc.address).collect();
let edges = fetch_edges(searcher, docs)?;
Ok(edges
.into_iter()
diff --git a/crates/core/src/webgraph/query/id2node.rs b/crates/core/src/webgraph/query/id2node.rs
index 58d95910..37780739 100644
--- a/crates/core/src/webgraph/query/id2node.rs
+++ b/crates/core/src/webgraph/query/id2node.rs
@@ -16,7 +16,6 @@
use super::collector::FirstDocCollector;
use super::Query;
-use crate::ampc::dht::ShardId;
use crate::webgraph::query::raw;
use crate::webgraph::schema::{FromHostId, FromId, ToHostId, ToId};
use crate::webgraph::searcher::Searcher;
@@ -43,8 +42,8 @@ impl Query for Id2NodeQuery {
}
}
- fn collector(&self, shard_id: ShardId) -> Self::Collector {
- FirstDocCollector::with_shard_id(shard_id)
+ fn collector(&self, searcher: &Searcher) -> Self::Collector {
+ FirstDocCollector::with_shard_id(searcher.shard())
}
fn retrieve(
diff --git a/crates/core/src/webgraph/query/mod.rs b/crates/core/src/webgraph/query/mod.rs
index a826dd7b..3de5afe9 100644
--- a/crates/core/src/webgraph/query/mod.rs
+++ b/crates/core/src/webgraph/query/mod.rs
@@ -39,7 +39,7 @@ pub trait Query: Send + Sync + bincode::Encode + bincode::Decode + Clone {
type Output;
fn tantivy_query(&self, searcher: &Searcher) -> Self::TantivyQuery;
- fn collector(&self, shard_id: ShardId) -> Self::Collector;
+ fn collector(&self, searcher: &Searcher) -> Self::Collector;
fn remote_collector(&self) -> Self::Collector;
fn filter_fruit_shards(
diff --git a/crates/core/src/webgraph/query/raw/host_links.rs b/crates/core/src/webgraph/query/raw/host_links.rs
index 91118035..4617d60b 100644
--- a/crates/core/src/webgraph/query/raw/host_links.rs
+++ b/crates/core/src/webgraph/query/raw/host_links.rs
@@ -14,7 +14,6 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use rustc_hash::FxHashSet;
use tantivy::{columnar::Column, postings::SegmentPostings, DocSet};
use crate::webgraph::{
@@ -83,6 +82,7 @@ impl tantivy::query::Weight for HostLinksWeight {
term,
self.deduplication_field,
&self.warmed_column_fields,
+ self.node.as_u64(),
) {
Ok(Some(scorer)) => Ok(Box::new(scorer)),
_ => Ok(Box::new(tantivy::query::EmptyScorer)),
@@ -101,7 +101,8 @@ impl tantivy::query::Weight for HostLinksWeight {
struct HostLinksScorer {
postings: SegmentPostings,
host_id_column: Column<u64>,
- seen_host_ids: FxHashSet<u64>,
+ last_host_id: Option<u64>,
+ self_host_id: u64,
}
impl HostLinksScorer {
@@ -110,6 +111,7 @@ impl HostLinksScorer {
term: tantivy::Term,
deduplication_field: FieldEnum,
warmed_column_fields: &WarmedColumnFields,
+ self_host_id: u64,
) -> tantivy::Result<Option<Self>> {
let host_id_column = warmed_column_fields
.segment(&reader.segment_id())
@@ -119,18 +121,11 @@ impl HostLinksScorer {
Ok(reader
.inverted_index(term.field())?
.read_postings(&term, tantivy::schema::IndexRecordOption::Basic)?
- .map(|postings| {
- let mut seen_host_ids = FxHashSet::default();
-
- if let Some(host_id) = host_id_column.first(postings.doc()) {
- seen_host_ids.insert(host_id);
- }
-
- Self {
- seen_host_ids,
- host_id_column,
- postings,
- }
+ .map(|postings| Self {
+ last_host_id: host_id_column.first(postings.doc()),
+ host_id_column,
+ postings,
+ self_host_id,
}))
}
}
@@ -145,9 +140,13 @@ impl HostLinksScorer {
fn has_seen_host(&self, doc: tantivy::DocId) -> bool {
self.host_id(doc)
- .map(|host_id| self.seen_host_ids.contains(&host_id))
+ .map(|host_id| self.last_host_id == Some(host_id))
.unwrap_or(false)
}
+
+ fn is_self_host(&self, doc: tantivy::DocId) -> bool {
+ self.host_id(doc) == Some(self.self_host_id)
+ }
}
impl tantivy::query::Scorer for HostLinksScorer {
@@ -170,15 +169,17 @@ impl tantivy::DocSet for HostLinksScorer {
self.postings.reset_cursor_start_block();
}
- while self.has_seen_host(self.postings.doc()) && self.doc() != tantivy::TERMINATED {
+ while (self.has_seen_host(self.doc()) || self.is_self_host(self.doc()))
+ && self.doc() != tantivy::TERMINATED
+ {
self.postings.advance();
}
- if let Some(host_id) = self.host_id(self.postings.doc()) {
- self.seen_host_ids.insert(host_id);
+ if let Some(host_id) = self.host_id(self.doc()) {
+ self.last_host_id = Some(host_id);
}
- self.postings.doc()
+ self.doc()
}
fn doc(&self) -> tantivy::DocId {
@@ -189,3 +190,132 @@ impl tantivy::DocSet for HostLinksScorer {
self.postings.size_hint()
}
}
+
+#[cfg(test)]
+mod tests {
+ use crate::{
+ webgraph::{query::HostBacklinksQuery, Edge, Node, Webgraph},
+ webpage::RelFlags,
+ };
+
+ #[test]
+ fn test_simple() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ let node_a = Node::from("A");
+ let node_b = Node::from("B");
+ let node_c = Node::from("C");
+
+ graph
+ .insert(Edge {
+ from: node_a.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: node_c.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();
+
+ assert_eq!(res.len(), 2);
+ assert!(res.iter().any(|r| r.from == node_a.id()));
+ assert!(res.iter().any(|r| r.from == node_c.id()));
+ }
+
+ #[test]
+ fn test_self_host_skipped() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ let node_a = Node::from("A");
+ let node_b = Node::from("B");
+
+ graph
+ .insert(Edge {
+ from: node_a.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: node_b.clone(),
+ to: node_b.clone(),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph.search(&HostBacklinksQuery::new(node_b.id())).unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert!(res[0].from == node_a.id());
+ }
+
+ #[test]
+ fn test_deduplication() {
+ let temp_dir = crate::gen_temp_dir().unwrap();
+ let mut graph = Webgraph::builder(&temp_dir, 0u64.into()).open().unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/1"),
+ to: Node::from("https://B.com/1"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/2"),
+ to: Node::from("https://B.com/2"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph
+ .insert(Edge {
+ from: Node::from("https://A.com/3"),
+ to: Node::from("https://B.com/3"),
+ rel_flags: RelFlags::default(),
+ label: String::new(),
+ sort_score: 0.0,
+ })
+ .unwrap();
+
+ graph.commit().unwrap();
+
+ let res = graph
+ .search(&HostBacklinksQuery::new(
+ Node::from("https://B.com/").into_host().id(),
+ ))
+ .unwrap();
+
+ assert_eq!(res.len(), 1);
+ assert!(res[0].from == Node::from("https://A.com/").into_host().id());
+ }
+}
diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rs
index d850d973..ce55660a 100644
--- a/crates/core/src/webgraph/store.rs
+++ b/crates/core/src/webgraph/store.rs
@@ -240,7 +240,7 @@ impl EdgeStore {
let res = searcher.tantivy_searcher().search(
&query.tantivy_query(&searcher),
- &TantivyCollector::from(&query.collector(self.shard_id)),
+ &TantivyCollector::from(&query.collector(&searcher)),
)?;
Ok(res)
diff --git a/crates/core/src/webgraph/warmed_column_fields.rs b/crates/core/src/webgraph/warmed_column_fields.rs
index fd9980a0..5da45820 100644
--- a/crates/core/src/webgraph/warmed_column_fields.rs
+++ b/crates/core/src/webgraph/warmed_column_fields.rs
@@ -29,16 +29,22 @@ pub struct SegmentColumnFields {
impl SegmentColumnFields {
pub fn u64<F: Field>(&self, field: F) -> Option<Column<u64>> {
- let field_enum: FieldEnum = field.into();
+ self.u64_by_enum(field.into())
+ }
+
+ pub fn u64_by_enum(&self, field: FieldEnum) -> Option<Column<u64>> {
self.u64_fields
- .get(FieldEnumDiscriminants::from(field_enum))
+ .get(FieldEnumDiscriminants::from(field))
.cloned()
}
pub fn f64<F: Field>(&self, field: F) -> Option<Column<f64>> {
- let field_enum: FieldEnum = field.into();
+ self.f64_by_enum(field.into())
+ }
+
+ pub fn f64_by_enum(&self, field: FieldEnum) -> Option<Column<f64>> {
self.f64_fields
- .get(FieldEnumDiscriminants::from(field_enum))
+ .get(FieldEnumDiscriminants::from(field))
.cloned()
}
}