web search done right
simplify ranking pipeline and fix offsets
Mikkel Denkerdef23ac59ad8
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 48420441..a11c2030 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -227,45 +227,19 @@ impl<T: AsRankingWebsite> Scorer<T> for Initial {}}-enum Prev<T: AsRankingWebsite> {- Initial,- #[allow(dead_code)]- Node(Box<RankingStage<T>>),-}-struct RankingStage<T: AsRankingWebsite> {scorer: Box<dyn Scorer<T>>,- prev: Prev<T>,- top_n: usize,- memory: Option<Vec<T>>,+ stage_top_n: usize,derank_similar: bool,}impl<T: AsRankingWebsite> RankingStage<T> {- fn initial_top_n(&self) -> usize {- match &self.prev {- Prev::Initial => self.top_n,- Prev::Node(n) => n.initial_top_n(),- }- }-- pub fn populate(&mut self, websites: Vec<T>) {- match &mut self.prev {- Prev::Initial => {- self.memory = Some(websites);- }- Prev::Node(n) => n.populate(websites),- }- }-- fn apply(&self, top_n: usize, page: usize) -> Vec<T> {- let next_page = (page * top_n) / self.top_n;- let mut websites = match &self.prev {- Prev::Initial => self.memory.clone().unwrap(),- Prev::Node(n) => n.apply(self.top_n, next_page),- };-- let page = page - next_page;+ fn apply(&self, websites: Vec<T>, top_n: usize, offset: usize) -> Vec<T> {+ let mut websites = websites+ .into_iter()+ .skip(offset)+ .take(self.stage_top_n.max(top_n))+ .collect::<Vec<_>>();self.scorer.score(&mut websites);for website in websites.iter_mut() {@@ -277,31 +251,22 @@ impl<T: AsRankingWebsite> RankingStage<T> {}}- let mut collector = BucketCollector::new((page + 1) * top_n);+ let mut collector = BucketCollector::new(self.stage_top_n.max(top_n) + offset);for website in websites {collector.insert(website);}- collector- .into_sorted_vec(self.derank_similar)- .into_iter()- .skip(page * top_n)- .take(top_n)- .collect()+ collector.into_sorted_vec(self.derank_similar)}fn set_query_info(&mut self, query: &SearchQuery) {self.scorer.set_query_info(query);- match &mut self.prev {- Prev::Initial => {}- Prev::Node(prev) => prev.set_query_info(query),- }}}pub struct RankingPipeline<T: AsRankingWebsite> {- last_stage: RankingStage<T>,+ stage: RankingStage<T>,page: usize,pub top_n: usize,}@@ -311,16 +276,14 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {crossencoder: Arc<M>,lambda: Option<Arc<LambdaMART>>,) -> Result<Self> {- let last_stage = RankingStage {+ let stage = RankingStage {scorer: Box::new(ReRanker::new(crossencoder, lambda)),- prev: Prev::Initial,- memory: None,- top_n: 20,+ stage_top_n: 20,derank_similar: false,};Ok(Self {- last_stage,+ stage,page: 0,top_n: 0,})@@ -343,14 +306,12 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {model,signal_coefficients: None,}),- prev: Prev::Initial,- memory: None,- top_n: 10_000,+ stage_top_n: 10_000,derank_similar: true,};Self {- last_stage,+ stage: last_stage,page: 0,top_n: 0,}@@ -364,34 +325,28 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}fn set_query_info(&mut self, query: &mut SearchQuery) {- self.last_stage.set_query_info(query);+ self.stage.set_query_info(query);self.page = query.page;self.top_n = query.num_results;query.num_results = self.collector_top_n();- query.page = self.collector_page();+ query.page = 0;}- pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {- self.last_stage.populate(websites);+ pub fn offset(&self) -> usize {+ self.top_n * self.page+ }- self.last_stage.apply(self.top_n, self.pipeline_page())+ pub fn apply(self, websites: Vec<T>) -> Vec<T> {+ self.stage.apply(websites, self.top_n, self.offset())}pub fn collector_top_n(&self) -> usize {- self.initial_top_n()+ self.initial_top_n().max(self.top_n) + self.top_n * self.page}pub fn initial_top_n(&self) -> usize {- self.last_stage.initial_top_n()- }-- fn pipeline_page(&self) -> usize {- self.page - (self.collector_page() * self.collector_top_n() / self.top_n)- }-- pub fn collector_page(&self) -> usize {- (self.page * self.top_n) / self.initial_top_n()+ self.stage.stage_top_n.max(self.top_n)}}@@ -464,6 +419,38 @@ mod tests {assert_eq!(res, expected);}+ #[test]+ fn top_n() {+ let num_results = 100;+ let pipeline = RankingPipeline::reranking_for_query(+ &mut SearchQuery {+ num_results,+ ..Default::default()+ },+ Arc::new(DummyCrossEncoder {}),+ None,+ )+ .unwrap();++ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());++ let expected: Vec<_> = sample+ .clone()+ .into_iter()+ .take(num_results)+ .map(|w| w.pointer.address)+ .collect();++ let res = pipeline+ .apply(sample)+ .into_iter()+ .map(|w| w.pointer.address)+ .collect_vec();++ assert_eq!(res.len(), num_results);+ assert_eq!(res, expected);+ }+#[test]fn offsets() {let num_results = 20;@@ -478,11 +465,7 @@ mod tests {).unwrap();- let sample: Vec<_> =- sample_websites(pipeline.collector_top_n() + pipeline.collector_page() * num_results)- .into_iter()- .skip(pipeline.collector_page() * num_results)- .collect();+ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());let mut prev: Vec<_> = pipeline.apply(sample);for p in 1..1_000 {let pipeline = RankingPipeline::reranking_for_query(@@ -495,12 +478,7 @@ mod tests {).unwrap();- let sample: Vec<_> = sample_websites(- pipeline.collector_top_n() + pipeline.collector_page() * num_results,- )- .into_iter()- .skip(pipeline.collector_page() * num_results)- .collect();+ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());let res: Vec<_> = pipeline.apply(sample).into_iter().collect();assert_eq!(diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rsindex 43ea8c41..64ede2a4 100644--- a/core/src/searcher/frontend.rs+++ b/core/src/searcher/frontend.rs@@ -80,11 +80,13 @@ impl FrontendSearcher {&self,initial_results: Vec<InitialSearchResultShard>,pipeline: RankingPipeline<ScoredWebsitePointer>,- ) -> Vec<ScoredWebsitePointer> {+ ) -> (Vec<ScoredWebsitePointer>, bool) {let mut collector = BucketCollector::new(pipeline.collector_top_n());+ let mut num_sites: usize = 0;for result in initial_results {for website in result.local_result.websites {+ num_sites += 1;let pointer = ScoredWebsitePointer {website,shard: result.shard.clone(),@@ -100,7 +102,12 @@ impl FrontendSearcher {.take(pipeline.collector_top_n()).collect::<Vec<_>>();- pipeline.apply(top_websites)+ let offset = pipeline.offset();++ let res = pipeline.apply(top_websites);+ let has_more = num_sites.saturating_sub(offset) > res.len();++ (res, has_more)}async fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {@@ -238,7 +245,7 @@ impl FrontendSearcher {return Ok(None);}- let top_websites = self.combine_results(initial_results, pipeline);+ let (top_websites, _) = self.combine_results(initial_results, pipeline);let scores: Vec<_> = top_websites.iter()@@ -327,8 +334,7 @@ impl FrontendSearcher {.map(|result| result.local_result.num_websites).sum();- let has_more_results = initial_results.iter().any(|res| res.local_result.has_more);- let top_websites = self.combine_results(initial_results, pipeline);+ let (top_websites, has_more_results) = self.combine_results(initial_results, pipeline);// retrieve webpageslet mut retrieved_webpages: Vec<_> = selfdiff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rsindex 69145856..d31fe3d9 100644--- a/core/src/spell/mod.rs+++ b/core/src/spell/mod.rs@@ -119,6 +119,7 @@ impl Spell {let searcher = index.inverted_index.tv_searcher();let schema = searcher.schema();let mut dict = DictionaryBuilder::new(1_000_000);+#[allow(unused_assignments, unused_mut)]let mut limit_terms: Option<usize> = None;#[cfg(debug_assertions)]