web search done right
Collector didn't correctly stop when max_docs was reached. Also faster similar-sites as we now use the scorer to estimate the importance for the backlink nodes as well.
Mikkel Denker7fef2b7b0db9
diff --git a/core/src/collector.rs b/core/src/collector.rsindex 36ffe6f3..3bafff13 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -161,15 +161,20 @@ impl TopSegmentCollector {}impl TopSegmentCollector {- fn collect(&mut self, doc: DocId, score: Score) {+ fn is_done(&self) -> bool {if let Some(max_docs) = &self.max_docs {- if self.num_docs_taken >= *max_docs {- return;- }-- self.num_docs_taken += 1;+ self.num_docs_taken >= *max_docs+ } else {+ false+ }+ }+ fn collect(&mut self, doc: DocId, score: Score) {+ if self.is_done() {+ return;}+ self.num_docs_taken += 1;+let simhash: Option<u64> = self.fastfield_segment_reader.get_field_reader(&FastField::SimHash)@@ -458,6 +463,10 @@ wheretype Fruit = Vec<SegmentDoc>;fn collect(&mut self, doc: DocId, score: tantivy::Score) {+ if self.segment_collector.is_done() {+ return;+ }+let score = self.segment_scorer.score(doc, score);self.segment_collector.collect(doc, score);}diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 59b2d502..3bc624e9 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -318,7 +318,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {model,signal_coefficients: None,}),- stage_top_n: 10_000,+ stage_top_n: 100,derank_similar: true,};diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 4778f7b3..c9156299 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -161,7 +161,7 @@ impl LocalSearcher {}Ok(ranker- .with_max_docs(10_000_000, self.index.num_segments())+ .with_max_docs(250_000, self.index.num_segments()).with_num_results(query.num_results()).with_offset(query.offset()))}diff --git a/core/src/similar_sites.rs b/core/src/similar_sites.rsindex 6d593546..2f9009c5 100644--- a/core/src/similar_sites.rs+++ b/core/src/similar_sites.rs@@ -83,55 +83,61 @@ impl SimilarSitesFinder {let scorer = self.inbound_similarity.scorer(&nodes, &[]);- let mut backlink_count: IntMap<NodeID, usize> = IntMap::new();+ let mut backlinks: IntMap<NodeID, f64> = IntMap::new();for node in &nodes {for edge in self.webgraph.raw_ingoing_edges(node) {- if !backlink_count.contains_key(&edge.from) {- backlink_count.insert(edge.from, 0);+ if !backlinks.contains_key(&edge.from) {+ backlinks.insert(edge.from, scorer.score(&edge.from));}-- *backlink_count.get_mut(&edge.from).unwrap() += 1;}}- let mut top_backlink_nodes: Vec<_> = backlink_count+ let mut top_backlink_nodes: Vec<_> = backlinks.into_iter()- .map(|(node, count)| (node, count))+ .map(|(node, score)| (node, score)).collect();- top_backlink_nodes.sort_unstable_by_key(|(_, count)| *count);+ top_backlink_nodes.sort_unstable_by(|(_, score_a), (_, score_b)| {+ score_a+ .partial_cmp(score_b)+ .unwrap_or(std::cmp::Ordering::Equal)+ });top_backlink_nodes.reverse();- let mut potential_nodes = IntSet::new();- for (backlink_node, _) in top_backlink_nodes {+ let mut scored_nodes = BinaryHeap::with_capacity(limit);+ let mut checked_nodes = IntSet::new();++ for (backlink_node, _) in top_backlink_nodes.into_iter().take(limit) {for edge in self.webgraph.raw_outgoing_edges(&backlink_node) {- potential_nodes.insert(edge.to);- }- }+ let potential_node = edge.to;- let mut scored_nodes = BinaryHeap::with_capacity(limit);+ if checked_nodes.contains(&potential_node) {+ continue;+ }++ checked_nodes.insert(potential_node);- for potential_node in potential_nodes.into_iter() {- let score = scorer.score(&potential_node);- let scored_node_id = ScoredNodeID {- node_id: potential_node,- score,- };+ let score = scorer.score(&potential_node);+ let scored_node_id = ScoredNodeID {+ node_id: potential_node,+ score,+ };- if scored_nodes.len() < limit {- scored_nodes.push(Reverse(scored_node_id));- } else {- let mut min_scored_node = scored_nodes.peek_mut().unwrap();+ if scored_nodes.len() < limit {+ scored_nodes.push(Reverse(scored_node_id));+ } else {+ let mut min_scored_node = scored_nodes.peek_mut().unwrap();- if scored_node_id > min_scored_node.0 {- *min_scored_node = Reverse(scored_node_id);+ if scored_node_id > min_scored_node.0 {+ *min_scored_node = Reverse(scored_node_id);+ }}}}let mut scored_nodes: Vec<_> = scored_nodes.into_iter().take(limit).map(|n| n.0).collect();- scored_nodes.sort();+ scored_nodes.sort_unstable();scored_nodes.reverse();scored_nodes