web search done right
fix offset bug in distributed searcher
Mikkel Denker58a77e60bd76
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 7ae2c156..666d9363 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -70,7 +70,7 @@ fn proximity_queries(let tantivy_field = schema.get_field(field.name()).unwrap();let tantivy_entry = schema.get_field_entry(tantivy_field);- for (boost, slop) in [(32, 0), (16, 1), (8, 2), (4, 4), (2, 16), (1, 32)] {+ for (boost, slop) in [(32, 0), (16, 1), (8, 2), (4, 4), (2, 8), (1, 32)] {let mut terms = Vec::new();let mut num_terms = 0;diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rsindex 7c0bd050..ba6a6dad 100644--- a/core/src/ranking/models/cross_encoder.rs+++ b/core/src/ranking/models/cross_encoder.rs@@ -22,6 +22,7 @@ use onnxruntime::{ndarray::ArrayBase, tensor::OrtOwnedTensor, GraphOptimizationLuse crate::Result;use super::ONNX_ENVIRONMENT;+const TRUNCATE_INPUT: usize = 128;pub struct CrossEncoderModel {tokenizer: tokenizers::Tokenizer,@@ -58,26 +59,27 @@ impl CrossEncoder for DummyCrossEncoder {impl CrossEncoder for CrossEncoderModel {fn run(&self, query: &str, body: &str) -> f64 {+ let body: String = body.split_whitespace().take(TRUNCATE_INPUT).collect();let encoded = self.tokenizer.encode((query, body), true).unwrap();let ids = encoded.get_ids().iter().map(|i| *i as i64)- .take(512)+ .take(TRUNCATE_INPUT).collect_vec();let attention_mask = encoded.get_attention_mask().iter()- .take(512)+ .take(TRUNCATE_INPUT).map(|i| *i as i64).collect_vec();let type_ids = encoded.get_type_ids().iter()- .take(512)+ .take(TRUNCATE_INPUT).map(|i| *i as i64).collect_vec();diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 87887012..38ca29ad 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -286,10 +286,10 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {scorer: Box::<PrioritizeText>::default(),prev: Prev::Initial,memory: None,- top_n: 100,+ top_n: 40,})),memory: None,- top_n: 50,+ top_n: 20,};Ok(Self {@@ -340,9 +340,11 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}pub fn apply(mut self, websites: Vec<T>) -> Vec<T> {+ dbg!(websites.len());self.last_stage.populate(websites);- self.last_stage.apply(20, self.offset)+ self.last_stage+ .apply(self.top_n, dbg!(self.pipeline_offset()))}pub fn collector_top_n(&self) -> usize {@@ -353,8 +355,12 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {self.last_stage.initial_top_n()}+ fn pipeline_offset(&self) -> usize {+ self.offset - self.collector_offset()+ }+pub fn collector_offset(&self) -> usize {- (self.offset / self.initial_top_n()) * self.initial_top_n()+ dbg!((self.offset / self.initial_top_n()) * self.initial_top_n())}}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex b816d20b..c08e3937 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -113,7 +113,7 @@ impl RemoteSearcher {}}-#[derive(Clone, PartialEq, Eq)]+#[derive(Debug, Clone, PartialEq, Eq)]struct ShardId(u64);pub struct Shard {@@ -173,6 +173,7 @@ impl Shard {}}+#[derive(Debug)]struct InitialSearchResultShard {local_result: InitialWebsiteResult,shard: ShardId,@@ -235,13 +236,9 @@ impl DistributedSearcher {fn combine_results(&self,initial_results: Vec<InitialSearchResultShard>,- search_query: &mut SearchQuery,- ) -> Result<Vec<ScoredWebsitePointer>> {- let pipeline: RankingPipeline<ScoredWebsitePointer> =- RankingPipeline::for_query(search_query, self.cross_encoder.clone())?;-- let mut collector =- BucketCollector::new(pipeline.collector_top_n() + pipeline.collector_offset());+ pipeline: RankingPipeline<ScoredWebsitePointer>,+ ) -> Vec<ScoredWebsitePointer> {+ let mut collector = BucketCollector::new(pipeline.collector_top_n());for result in initial_results {for website in result.local_result.websites {@@ -257,11 +254,10 @@ impl DistributedSearcher {let top_websites = collector.into_sorted_vec(true).into_iter()- .skip(pipeline.collector_offset()).take(pipeline.collector_top_n()).collect::<Vec<_>>();- Ok(pipeline.apply(top_websites))+ pipeline.apply(top_websites)}async fn retrieve_webpages(@@ -385,6 +381,9 @@ impl DistributedSearcher {..Default::default()};+ let pipeline: RankingPipeline<ScoredWebsitePointer> =+ RankingPipeline::for_query(&mut query, self.cross_encoder.clone())?;+let initial_results = self.search_initial(&query).await;if initial_results.is_empty() {@@ -400,7 +399,7 @@ impl DistributedSearcher {return Ok(None);}- let results = self.combine_results(initial_results, &mut query)?;+ let results = self.combine_results(initial_results, pipeline);let scores: Vec<_> = results.iter()@@ -439,8 +438,10 @@ impl DistributedSearcher {}let mut search_query = query.clone();+ let pipeline: RankingPipeline<ScoredWebsitePointer> =+ RankingPipeline::for_query(&mut search_query, self.cross_encoder.clone())?;- let initial_results = self.search_initial(query).await;+ let initial_results = self.search_initial(&search_query).await;let sidebar = self.sidebar(&initial_results, query).await?;let discussions = self.discussions_widget(query).await?;@@ -455,7 +456,7 @@ impl DistributedSearcher {.map(|result| result.local_result.num_websites).sum();- let top_websites = self.combine_results(initial_results, &mut search_query)?;+ let top_websites = self.combine_results(initial_results, pipeline);// retrieve webpageslet retrieved_webpages: Vec<_> = self@@ -469,13 +470,15 @@ impl DistributedSearcher {return Err(Error::SearchFailed);}+ let search_duration_ms = start.elapsed().as_millis();+Ok(SearchResult::Websites(WebsitesResult {spell_corrected_query,num_hits: num_docs,webpages: retrieved_webpages,sidebar,discussions,- search_duration_ms: start.elapsed().as_millis(),+ search_duration_ms,}))}}diff --git a/frontend/src/components/Discussions.astro b/frontend/src/components/Discussions.astroindex 03980b27..8b7f2976 100644--- a/frontend/src/components/Discussions.astro+++ b/frontend/src/components/Discussions.astro@@ -28,7 +28,7 @@ askama.if_("let Some(discussions) = discussions", () => ())}</div>- <div id="discussion-more-btn" class="hover:cursor-pointer hover:bg-neutral-100 w-fit px-2 py-1 mt-2 border rounded-full">More discussions</div>+ <div id="discussion-more-btn" class="hover:cursor-pointer hover:bg-neutral-100 w-fit px-2 py-1 mt-3 border rounded-full">More discussions</div></div></div>))