Skip to content

web search done right

Collector didn't correctly stop when max_docs was reached. Also faster similar-sites as we now use the scorer to estimate the importance for the backlink nodes as well.

Mikkel Denker7fef2b7b0db9
diff --git a/core/src/collector.rs b/core/src/collector.rs
index 36ffe6f3..3bafff13 100644
--- a/core/src/collector.rs
+++ b/core/src/collector.rs
@@ -161,15 +161,20 @@ impl TopSegmentCollector {
}
impl TopSegmentCollector {
- fn collect(&mut self, doc: DocId, score: Score) {
+ fn is_done(&self) -> bool {
if let Some(max_docs) = &self.max_docs {
- if self.num_docs_taken >= *max_docs {
- return;
- }
-
- self.num_docs_taken += 1;
+ self.num_docs_taken >= *max_docs
+ } else {
+ false
+ }
+ }
+ fn collect(&mut self, doc: DocId, score: Score) {
+ if self.is_done() {
+ return;
}
+ self.num_docs_taken += 1;
+
let simhash: Option<u64> = self
.fastfield_segment_reader
.get_field_reader(&FastField::SimHash)
@@ -458,6 +463,10 @@ where
type Fruit = Vec<SegmentDoc>;
fn collect(&mut self, doc: DocId, score: tantivy::Score) {
+ if self.segment_collector.is_done() {
+ return;
+ }
+
let score = self.segment_scorer.score(doc, score);
self.segment_collector.collect(doc, score);
}
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 59b2d502..3bc624e9 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -318,7 +318,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
model,
signal_coefficients: None,
}),
- stage_top_n: 10_000,
+ stage_top_n: 100,
derank_similar: true,
};
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 4778f7b3..c9156299 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -161,7 +161,7 @@ impl LocalSearcher {
}
Ok(ranker
- .with_max_docs(10_000_000, self.index.num_segments())
+ .with_max_docs(250_000, self.index.num_segments())
.with_num_results(query.num_results())
.with_offset(query.offset()))
}
diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rs
index 6d593546..2f9009c5 100644
--- a/core/src/similar_sites.rs
+++ b/core/src/similar_sites.rs
@@ -83,55 +83,61 @@ impl SimilarSitesFinder {
let scorer = self.inbound_similarity.scorer(&nodes, &[]);
- let mut backlink_count: IntMap<NodeID, usize> = IntMap::new();
+ let mut backlinks: IntMap<NodeID, f64> = IntMap::new();
for node in &nodes {
for edge in self.webgraph.raw_ingoing_edges(node) {
- if !backlink_count.contains_key(&edge.from) {
- backlink_count.insert(edge.from, 0);
+ if !backlinks.contains_key(&edge.from) {
+ backlinks.insert(edge.from, scorer.score(&edge.from));
}
-
- *backlink_count.get_mut(&edge.from).unwrap() += 1;
}
}
- let mut top_backlink_nodes: Vec<_> = backlink_count
+ let mut top_backlink_nodes: Vec<_> = backlinks
.into_iter()
- .map(|(node, count)| (node, count))
+ .map(|(node, score)| (node, score))
.collect();
- top_backlink_nodes.sort_unstable_by_key(|(_, count)| *count);
+ top_backlink_nodes.sort_unstable_by(|(_, score_a), (_, score_b)| {
+ score_a
+ .partial_cmp(score_b)
+ .unwrap_or(std::cmp::Ordering::Equal)
+ });
top_backlink_nodes.reverse();
- let mut potential_nodes = IntSet::new();
- for (backlink_node, _) in top_backlink_nodes {
+ let mut scored_nodes = BinaryHeap::with_capacity(limit);
+ let mut checked_nodes = IntSet::new();
+
+ for (backlink_node, _) in top_backlink_nodes.into_iter().take(limit) {
for edge in self.webgraph.raw_outgoing_edges(&backlink_node) {
- potential_nodes.insert(edge.to);
- }
- }
+ let potential_node = edge.to;
- let mut scored_nodes = BinaryHeap::with_capacity(limit);
+ if checked_nodes.contains(&potential_node) {
+ continue;
+ }
+
+ checked_nodes.insert(potential_node);
- for potential_node in potential_nodes.into_iter() {
- let score = scorer.score(&potential_node);
- let scored_node_id = ScoredNodeID {
- node_id: potential_node,
- score,
- };
+ let score = scorer.score(&potential_node);
+ let scored_node_id = ScoredNodeID {
+ node_id: potential_node,
+ score,
+ };
- if scored_nodes.len() < limit {
- scored_nodes.push(Reverse(scored_node_id));
- } else {
- let mut min_scored_node = scored_nodes.peek_mut().unwrap();
+ if scored_nodes.len() < limit {
+ scored_nodes.push(Reverse(scored_node_id));
+ } else {
+ let mut min_scored_node = scored_nodes.peek_mut().unwrap();
- if scored_node_id > min_scored_node.0 {
- *min_scored_node = Reverse(scored_node_id);
+ if scored_node_id > min_scored_node.0 {
+ *min_scored_node = Reverse(scored_node_id);
+ }
}
}
}
let mut scored_nodes: Vec<_> = scored_nodes.into_iter().take(limit).map(|n| n.0).collect();
- scored_nodes.sort();
+ scored_nodes.sort_unstable();
scored_nodes.reverse();
scored_nodes