Skip to content

web search done right

simplify ranking pipeline and fix offsets

Mikkel Denkerdef23ac59ad8
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 48420441..a11c2030 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -227,45 +227,19 @@ impl<T: AsRankingWebsite> Scorer<T> for Initial {
}
}
-enum Prev<T: AsRankingWebsite> {
- Initial,
- #[allow(dead_code)]
- Node(Box<RankingStage<T>>),
-}
-
struct RankingStage<T: AsRankingWebsite> {
scorer: Box<dyn Scorer<T>>,
- prev: Prev<T>,
- top_n: usize,
- memory: Option<Vec<T>>,
+ stage_top_n: usize,
derank_similar: bool,
}
impl<T: AsRankingWebsite> RankingStage<T> {
- fn initial_top_n(&self) -> usize {
- match &self.prev {
- Prev::Initial => self.top_n,
- Prev::Node(n) => n.initial_top_n(),
- }
- }
-
- pub fn populate(&mut self, websites: Vec<T>) {
- match &mut self.prev {
- Prev::Initial => {
- self.memory = Some(websites);
- }
- Prev::Node(n) => n.populate(websites),
- }
- }
-
- fn apply(&self, top_n: usize, page: usize) -> Vec<T> {
- let next_page = (page * top_n) / self.top_n;
- let mut websites = match &self.prev {
- Prev::Initial => self.memory.clone().unwrap(),
- Prev::Node(n) => n.apply(self.top_n, next_page),
- };
-
- let page = page - next_page;
+ fn apply(&self, websites: Vec<T>, top_n: usize, offset: usize) -> Vec<T> {
+ let mut websites = websites
+ .into_iter()
+ .skip(offset)
+ .take(self.stage_top_n.max(top_n))
+ .collect::<Vec<_>>();
self.scorer.score(&mut websites);
for website in websites.iter_mut() {
@@ -277,31 +251,22 @@ impl<T: AsRankingWebsite> RankingStage<T> {
}
}
- let mut collector = BucketCollector::new((page + 1) * top_n);
+ let mut collector = BucketCollector::new(self.stage_top_n.max(top_n) + offset);
for website in websites {
collector.insert(website);
}
- collector
- .into_sorted_vec(self.derank_similar)
- .into_iter()
- .skip(page * top_n)
- .take(top_n)
- .collect()
+ collector.into_sorted_vec(self.derank_similar)
}
fn set_query_info(&mut self, query: &SearchQuery) {
self.scorer.set_query_info(query);
- match &mut self.prev {
- Prev::Initial => {}
- Prev::Node(prev) => prev.set_query_info(query),
- }
}
}
pub struct RankingPipeline<T: AsRankingWebsite> {
- last_stage: RankingStage<T>,
+ stage: RankingStage<T>,
page: usize,
pub top_n: usize,
}
@@ -311,16 +276,14 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
crossencoder: Arc<M>,
lambda: Option<Arc<LambdaMART>>,
) -> Result<Self> {
- let last_stage = RankingStage {
+ let stage = RankingStage {
scorer: Box::new(ReRanker::new(crossencoder, lambda)),
- prev: Prev::Initial,
- memory: None,
- top_n: 20,
+ stage_top_n: 20,
derank_similar: false,
};
Ok(Self {
- last_stage,
+ stage,
page: 0,
top_n: 0,
})
@@ -343,14 +306,12 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
model,
signal_coefficients: None,
}),
- prev: Prev::Initial,
- memory: None,
- top_n: 10_000,
+ stage_top_n: 10_000,
derank_similar: true,
};
Self {
- last_stage,
+ stage: last_stage,
page: 0,
top_n: 0,
}
@@ -364,34 +325,28 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
fn set_query_info(&mut self, query: &mut SearchQuery) {
- self.last_stage.set_query_info(query);
+ self.stage.set_query_info(query);
self.page = query.page;
self.top_n = query.num_results;
query.num_results = self.collector_top_n();
- query.page = self.collector_page();
+ query.page = 0;
}
- pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {
- self.last_stage.populate(websites);
+ pub fn offset(&self) -> usize {
+ self.top_n * self.page
+ }
- self.last_stage.apply(self.top_n, self.pipeline_page())
+ pub fn apply(self, websites: Vec<T>) -> Vec<T> {
+ self.stage.apply(websites, self.top_n, self.offset())
}
pub fn collector_top_n(&self) -> usize {
- self.initial_top_n()
+ self.initial_top_n().max(self.top_n) + self.top_n * self.page
}
pub fn initial_top_n(&self) -> usize {
- self.last_stage.initial_top_n()
- }
-
- fn pipeline_page(&self) -> usize {
- self.page - (self.collector_page() * self.collector_top_n() / self.top_n)
- }
-
- pub fn collector_page(&self) -> usize {
- (self.page * self.top_n) / self.initial_top_n()
+ self.stage.stage_top_n.max(self.top_n)
}
}
@@ -464,6 +419,38 @@ mod tests {
assert_eq!(res, expected);
}
+ #[test]
+ fn top_n() {
+ let num_results = 100;
+ let pipeline = RankingPipeline::reranking_for_query(
+ &mut SearchQuery {
+ num_results,
+ ..Default::default()
+ },
+ Arc::new(DummyCrossEncoder {}),
+ None,
+ )
+ .unwrap();
+
+ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());
+
+ let expected: Vec<_> = sample
+ .clone()
+ .into_iter()
+ .take(num_results)
+ .map(|w| w.pointer.address)
+ .collect();
+
+ let res = pipeline
+ .apply(sample)
+ .into_iter()
+ .map(|w| w.pointer.address)
+ .collect_vec();
+
+ assert_eq!(res.len(), num_results);
+ assert_eq!(res, expected);
+ }
+
#[test]
fn offsets() {
let num_results = 20;
@@ -478,11 +465,7 @@ mod tests {
)
.unwrap();
- let sample: Vec<_> =
- sample_websites(pipeline.collector_top_n() + pipeline.collector_page() * num_results)
- .into_iter()
- .skip(pipeline.collector_page() * num_results)
- .collect();
+ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());
let mut prev: Vec<_> = pipeline.apply(sample);
for p in 1..1_000 {
let pipeline = RankingPipeline::reranking_for_query(
@@ -495,12 +478,7 @@ mod tests {
)
.unwrap();
- let sample: Vec<_> = sample_websites(
- pipeline.collector_top_n() + pipeline.collector_page() * num_results,
- )
- .into_iter()
- .skip(pipeline.collector_page() * num_results)
- .collect();
+ let sample: Vec<_> = sample_websites(pipeline.collector_top_n());
let res: Vec<_> = pipeline.apply(sample).into_iter().collect();
assert_eq!(
diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rs
index 43ea8c41..64ede2a4 100644
--- a/core/src/searcher/frontend.rs
+++ b/core/src/searcher/frontend.rs
@@ -80,11 +80,13 @@ impl FrontendSearcher {
&self,
initial_results: Vec<InitialSearchResultShard>,
pipeline: RankingPipeline<ScoredWebsitePointer>,
- ) -> Vec<ScoredWebsitePointer> {
+ ) -> (Vec<ScoredWebsitePointer>, bool) {
let mut collector = BucketCollector::new(pipeline.collector_top_n());
+ let mut num_sites: usize = 0;
for result in initial_results {
for website in result.local_result.websites {
+ num_sites += 1;
let pointer = ScoredWebsitePointer {
website,
shard: result.shard.clone(),
@@ -100,7 +102,12 @@ impl FrontendSearcher {
.take(pipeline.collector_top_n())
.collect::<Vec<_>>();
- pipeline.apply(top_websites)
+ let offset = pipeline.offset();
+
+ let res = pipeline.apply(top_websites);
+ let has_more = num_sites.saturating_sub(offset) > res.len();
+
+ (res, has_more)
}
async fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {
@@ -238,7 +245,7 @@ impl FrontendSearcher {
return Ok(None);
}
- let top_websites = self.combine_results(initial_results, pipeline);
+ let (top_websites, _) = self.combine_results(initial_results, pipeline);
let scores: Vec<_> = top_websites
.iter()
@@ -327,8 +334,7 @@ impl FrontendSearcher {
.map(|result| result.local_result.num_websites)
.sum();
- let has_more_results = initial_results.iter().any(|res| res.local_result.has_more);
- let top_websites = self.combine_results(initial_results, pipeline);
+ let (top_websites, has_more_results) = self.combine_results(initial_results, pipeline);
// retrieve webpages
let mut retrieved_webpages: Vec<_> = self
diff --git a/core/src/spell/mod.rs b/core/src/spell/mod.rs
index 69145856..d31fe3d9 100644
--- a/core/src/spell/mod.rs
+++ b/core/src/spell/mod.rs
@@ -119,6 +119,7 @@ impl Spell {
let searcher = index.inverted_index.tv_searcher();
let schema = searcher.schema();
let mut dict = DictionaryBuilder::new(1_000_000);
+
#[allow(unused_assignments, unused_mut)]
let mut limit_terms: Option<usize> = None;
#[cfg(debug_assertions)]