Skip to content

web search done right

change ranking pipeline names to recall and reranking stage these names better represent what the different pipelines are used for

Mikkel Denker1f4dc0ac5755
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 06d55e93..7b328780 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -340,7 +340,7 @@ pub struct RankingPipeline<T: AsRankingWebsite> {
}
impl<T: AsRankingWebsite> RankingPipeline<T> {
- fn create_second_stage<M: CrossEncoder + 'static>(
+ fn create_reranking_stage<M: CrossEncoder + 'static>(
crossencoder: Option<Arc<M>>,
lambda: Option<Arc<LambdaMART>>,
collector_config: CollectorConfig,
@@ -375,13 +375,13 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
top_n_considered: usize,
) -> Result<Self> {
let mut pipeline =
- Self::create_second_stage(crossencoder, lambda, collector_config, top_n_considered)?;
+ Self::create_reranking_stage(crossencoder, lambda, collector_config, top_n_considered)?;
pipeline.set_query_info(query);
Ok(pipeline)
}
- fn create_first_stage(
+ fn create_recall_stage(
model: Option<Arc<LambdaMART>>,
collector_config: CollectorConfig,
stage_top_n: usize,
@@ -403,13 +403,13 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
}
- pub fn first_stage(
+ pub fn recall_stage(
query: &mut SearchQuery,
model: Option<Arc<LambdaMART>>,
collector_config: CollectorConfig,
top_n_considered: usize,
) -> Self {
- let mut pipeline = Self::create_first_stage(model, collector_config, top_n_considered);
+ let mut pipeline = Self::create_recall_stage(model, collector_config, top_n_considered);
pipeline.set_query_info(query);
pipeline
diff --git a/core/src/searcher/api/mod.rs b/core/src/searcher/api/mod.rs
index 332e375b..7f42e962 100644
--- a/core/src/searcher/api/mod.rs
+++ b/core/src/searcher/api/mod.rs
@@ -330,7 +330,7 @@ impl ApiSearcher {
// This pipeline should be created before the first search is performed
// so the query knows how many results to fetch from the indices
- let first_pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::first_stage(
+ let recall_pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::recall_stage(
&mut search_query,
self.lambda_model.clone(),
self.collector_config.clone(),
@@ -369,7 +369,7 @@ impl ApiSearcher {
self.collector_config.clone(),
initial_results,
live_results,
- first_pipeline,
+ recall_pipeline,
);
let retrieved_webpages = self
@@ -382,16 +382,17 @@ impl ApiSearcher {
};
#[cfg(feature = "libtorch")]
- let pipeline: RankingPipeline<RetrievedWebpageRanking> = RankingPipeline::reranker(
- &mut search_query,
- self.cross_encoder.clone(),
- self.lambda_model.clone(),
- self.collector_config.clone(),
- query.num_results,
- )?;
+ let reranking_pipeline: RankingPipeline<RetrievedWebpageRanking> =
+ RankingPipeline::reranker(
+ &mut search_query,
+ self.cross_encoder.clone(),
+ self.lambda_model.clone(),
+ self.collector_config.clone(),
+ query.num_results,
+ )?;
#[cfg(not(feature = "libtorch"))]
- let pipeline: RankingPipeline<RetrievedWebpageRanking> =
+ let reranking_pipeline: RankingPipeline<RetrievedWebpageRanking> =
RankingPipeline::reranker::<DummyCrossEncoder>(
&mut search_query,
None,
@@ -400,7 +401,7 @@ impl ApiSearcher {
query.num_results,
)?;
- let retrieved_webpages = pipeline.apply(retrieved_webpages);
+ let retrieved_webpages = reranking_pipeline.apply(retrieved_webpages);
let mut retrieved_webpages: Vec<_> = retrieved_webpages
.into_iter()
diff --git a/core/src/searcher/api/widget.rs b/core/src/searcher/api/widget.rs
index da15792e..20664b92 100644
--- a/core/src/searcher/api/widget.rs
+++ b/core/src/searcher/api/widget.rs
@@ -109,7 +109,7 @@ impl WidgetManager {
// This pipeline should be created before the first search is performed
// so the query knows how many results to fetch from the indices
- let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::first_stage(
+ let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::recall_stage(
&mut query,
self.lambda_model.clone(),
self.collector_config.clone(),
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index aa36bcf9..b04534ce 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -249,7 +249,7 @@ where
de_rank_similar: bool,
) -> Result<InvertedIndexResult> {
let mut query = query.clone();
- let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::first_stage(
+ let pipeline: RankingPipeline<RankingWebsite> = RankingPipeline::recall_stage(
&mut query,
self.lambda_model.clone(),
self.collector_config.clone(),