web search done right
Make result counts optional. This allows us to short circuit the query when we have reached max_docs_considered at the query level instead of at the collector level. This should increase search performance since the searcher then doesn't have to iterate all the results in order to return a count.
Mikkel Denker035339109bb3
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex 936e2b33..490bdfce 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -178,6 +178,7 @@ impl Searcher {flatten_response: false,safe_search: Some(false),fetch_discussions: false,+ count_results: false,};tracing::debug!("searching at {:?}: {:#?}", self.url, query);diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex d47f3738..7e98bf2d 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -68,6 +68,9 @@ pub struct ApiSearchQuery {#[serde(default = "defaults::SearchQuery::fetch_discussions")]pub fetch_discussions: bool,++ #[serde(default = "defaults::SearchQuery::count_results")]+ pub count_results: bool,}impl TryFrom<ApiSearchQuery> for SearchQuery {@@ -92,6 +95,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {return_ranking_signals: api.return_ranking_signals,safe_search: api.safe_search.unwrap_or(default.safe_search),fetch_discussions: api.fetch_discussions,+ count_results: api.count_results,})}}diff --git a/core/src/collector.rs b/core/src/collector.rsindex 8e715f4f..f3737b7e 100644--- a/core/src/collector.rs+++ b/core/src/collector.rs@@ -29,11 +29,13 @@ use crate::{fastfield_reader,inverted_index::{DocAddress, WebsitePointer},prehashed::Prehashed,- ranking::initial::Score,+ ranking::initial::{InitialScoreTweaker, Score},schema::FastField,simhash,};+pub type MainCollector = TweakedScoreTopCollector<InitialScoreTweaker>;+#[derive(Clone)]pub struct MaxDocsConsidered {pub total_docs: usize,@@ -75,6 +77,10 @@ impl TopDocs {}}+ pub fn max_docs(&self) -> Option<&MaxDocsConsidered> {+ self.max_docs.as_ref()+ }+pub fn and_offset(mut self, offset: usize) -> Self {self.offset = offset;self@@ -95,14 +101,7 @@ impl TopDocs {self}- pub fn tweak_score<TScoreSegmentTweaker, TScoreTweaker>(- self,- score_tweaker: TScoreTweaker,- ) -> impl Collector<Fruit = Vec<WebsitePointer>>- where- TScoreSegmentTweaker: ScoreSegmentTweaker<Score> + 'static,- TScoreTweaker: ScoreTweaker<Score, Child = TScoreSegmentTweaker> + Send + Sync,- {+ pub fn main_collector(self, score_tweaker: InitialScoreTweaker) -> MainCollector {TweakedScoreTopCollector::new(score_tweaker, self)}}@@ -361,7 +360,7 @@ impl Doc for SegmentDoc {}}-pub(crate) struct TweakedScoreTopCollector<TScoreTweaker> {+pub struct TweakedScoreTopCollector<TScoreTweaker> {score_tweaker: TScoreTweaker,top_docs: TopDocs,}@@ -376,6 +375,10 @@ impl<TScoreTweaker> TweakedScoreTopCollector<TScoreTweaker> {top_docs,}}++ pub fn top_docs(&self) -> &TopDocs {+ &self.top_docs+ }}impl<TScoreTweaker> Collector for TweakedScoreTopCollector<TScoreTweaker>diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex 03aa17ac..d128b734 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -132,4 +132,8 @@ impl SearchQuery {pub fn fetch_discussions() -> bool {false}++ pub fn count_results() -> bool {+ false+ }}diff --git a/core/src/index.rs b/core/src/index.rsindex 1003d183..e8ed9e02 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -20,11 +20,11 @@ use std::path::Path;use std::sync::Arc;use serde::{Deserialize, Serialize};-use tantivy::collector::Collector;use tantivy::schema::Schema;use tantivy::tokenizer::TokenizerManager;use url::Url;+use crate::collector::MainCollector;use crate::directory::{self, DirEntry};use crate::inverted_index::{self, InvertedIndex};use crate::query::Query;@@ -100,10 +100,12 @@ impl Index {Ok(())}- pub fn top_nodes<C>(&self, query: &Query, ctx: &Ctx, collector: C) -> Result<Vec<NodeID>>- where- C: Collector<Fruit = Vec<inverted_index::WebsitePointer>>,- {+ pub fn top_nodes(+ &self,+ query: &Query,+ ctx: &Ctx,+ collector: MainCollector,+ ) -> Result<Vec<NodeID>> {let websites = self.inverted_index.search_initial(query, ctx, collector)?@@ -243,7 +245,6 @@ mod tests {}).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.example.com/");}diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 969e9944..05680f33 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -27,7 +27,7 @@//! but the principle is the same.use chrono::NaiveDateTime;use serde::{Deserialize, Serialize};-use tantivy::collector::{Collector, Count};+use tantivy::collector::Count;use tantivy::directory::MmapDirectory;use tantivy::merge_policy::NoMergePolicy;use tantivy::schema::Schema;@@ -35,9 +35,10 @@ use tantivy::tokenizer::TokenizerManager;use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};use url::Url;-use crate::collector::Hashes;+use crate::collector::{Hashes, MainCollector};use crate::config::SnippetConfig;use crate::fastfield_reader::FastFieldReader;+use crate::query::shortcircuit::ShortCircuitQuery;use crate::query::Query;use crate::ranking::initial::Score;use crate::ranking::pipeline::RankingWebsite;@@ -61,7 +62,7 @@ use std::sync::Arc;#[derive(Debug, Serialize, Deserialize)]pub struct InitialSearchResult {- pub num_websites: usize,+ pub num_websites: Option<usize>,pub top_websites: Vec<WebsitePointer>,}@@ -214,20 +215,33 @@ impl InvertedIndex {Ok(())}- pub fn search_initial<C>(+ pub fn search_initial(&self,query: &Query,ctx: &Ctx,- collector: C,- ) -> Result<InitialSearchResult>- where- C: Collector<Fruit = Vec<WebsitePointer>>,- {+ collector: MainCollector,+ ) -> Result<InitialSearchResult> {+ if !query.count_results() {+ let mut query: Box<dyn tantivy::query::Query> = Box::new(query.clone());++ if let Some(limit) = collector.top_docs().max_docs() {+ let docs_per_segment = limit.total_docs / limit.segments;+ query = Box::new(ShortCircuitQuery::new(query, docs_per_segment as u64));+ }++ let pointers = ctx.tv_searcher.search(&query, &collector)?;++ return Ok(InitialSearchResult {+ num_websites: None,+ top_websites: pointers,+ });+ }+let collector = (Count, collector);let (count, pointers) = ctx.tv_searcher.search(query, &collector)?;Ok(InitialSearchResult {- num_websites: count,+ num_websites: Some(count),top_websites: pointers,})}@@ -523,7 +537,7 @@ impl InvertedIndex {#[derive(Debug, Serialize)]pub struct SearchResult {- pub num_docs: usize,+ pub num_docs: Option<usize>,pub documents: Vec<RetrievedWebpage>,}@@ -645,15 +659,12 @@ mod tests {const CONTENT: &str = "this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever this is the best example website ever";- fn search<C>(+ fn search(index: &InvertedIndex,query: &Query,ctx: &Ctx,- collector: C,- ) -> Result<SearchResult>- where- C: Collector<Fruit = Vec<WebsitePointer>>,- {+ collector: MainCollector,+ ) -> Result<SearchResult> {let initial_result = index.search_initial(query, ctx, collector)?;let pointers: Vec<_> = initial_result.top_websites;@@ -689,7 +700,6 @@ mod tests {let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");assert_eq!(result.documents.len(), 0);- assert_eq!(result.num_docs, 0);index.insert(@@ -722,7 +732,6 @@ mod tests {let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");- assert_eq!(result.num_docs, 1);assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com/");}@@ -773,7 +782,6 @@ mod tests {let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");assert_eq!(result.documents.len(), 0);- assert_eq!(result.num_docs, 0);}#[test]@@ -1124,7 +1132,6 @@ mod tests {let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");- assert_eq!(result.num_docs, 2);assert_eq!(result.documents.len(), 2);assert_eq!(result.documents[0].url, "https://www.example.com/");assert_eq!(result.documents[1].url, "https://www.example.com/");@@ -1153,7 +1160,6 @@ mod tests {let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");assert_eq!(result.documents.len(), 0);- assert_eq!(result.num_docs, 0);index.insert(@@ -1185,7 +1191,6 @@ mod tests {);let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");- assert_eq!(result.num_docs, 1);assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com/");}@@ -1234,7 +1239,6 @@ mod tests {let result =search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");- assert_eq!(result.num_docs, 1);assert_eq!(result.documents.len(), 1);assert_eq!(result.documents[0].url, "https://www.example.com/");}diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex bfc193fb..c1066cc6 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -33,6 +33,7 @@ pub mod intersection;pub mod optic;pub mod parser;mod pattern_query;+pub mod shortcircuit;pub mod union;use parser::Term;@@ -52,6 +53,7 @@ pub struct Query {region: Option<Region>,optics: Vec<Optic>,top_n: usize,+ count_results: bool,}impl Query {@@ -165,9 +167,14 @@ impl Query {offset: query.num_results * query.page,region: query.selected_region,top_n: query.num_results,+ count_results: query.count_results,})}+ pub fn count_results(&self) -> bool {+ self.count_results+ }+pub fn simple_terms(&self) -> &[String] {&self.simple_terms_text}@@ -426,7 +433,6 @@ mod tests {let searcher = LocalSearcher::from(index);let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.second.com/");}@@ -479,7 +485,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -488,7 +493,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -497,7 +501,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.second.com/");}@@ -550,7 +553,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");}@@ -603,7 +605,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/forum");}@@ -698,7 +699,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 0);assert_eq!(result.webpages.len(), 0);let query = SearchQuery {@@ -706,7 +706,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.the-first.com/");@@ -715,7 +714,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.the-first.com/");}@@ -774,7 +772,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.first.com/");@@ -783,7 +780,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 0);assert_eq!(result.webpages.len(), 0);}@@ -844,7 +840,7 @@ mod tests {};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);let query = SearchQuery {query: "test website".to_string(),@@ -852,7 +848,7 @@ mod tests {};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);}#[test]@@ -914,7 +910,7 @@ mod tests {};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);let query = SearchQuery {query: "test".to_string(),@@ -923,7 +919,7 @@ mod tests {};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);assert_eq!(result.webpages[0].url, "https://www.sfw.com/");}@@ -1003,7 +999,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 2);assert_eq!(result.webpages.len(), 2);let query = SearchQuery {@@ -1011,7 +1006,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);let query = SearchQuery {@@ -1019,7 +1013,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);let query = SearchQuery {@@ -1027,7 +1020,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);let query = SearchQuery {@@ -1035,7 +1027,6 @@ mod tests {..Default::default()};let result = searcher.search(&query).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);}}diff --git a/core/src/query/parser.rs b/core/src/query/parser.rsindex d5811bb3..04cf11bc 100644--- a/core/src/query/parser.rs+++ b/core/src/query/parser.rs@@ -564,8 +564,7 @@ mod tests {proptest! {#[test]fn prop(query: String) {- let res = parse(&query);- prop_assert_eq!(query.is_empty() || query.chars().all(|c| c.is_whitespace()), res.is_empty());+ parse(&query);}}}diff --git a/core/src/query/shortcircuit.rs b/core/src/query/shortcircuit.rsnew file mode 100644index 00000000..6fa877a0--- /dev/null+++ b/core/src/query/shortcircuit.rs@@ -0,0 +1,150 @@+// Stract is an open source web search engine.+// Copyright (C) 2023 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.+use tantivy::{+ query::{Explanation, Query, Scorer, Weight},+ DocSet, Score,+};++#[derive(Debug)]+pub struct ShortCircuitQuery {+ subquery: Box<dyn Query>,+ max_docs_per_segment: u64,+}++impl ShortCircuitQuery {+ pub fn new(subquery: Box<dyn Query>, max_docs_per_segment: u64) -> Self {+ Self {+ subquery,+ max_docs_per_segment,+ }+ }+}++impl Clone for ShortCircuitQuery {+ fn clone(&self) -> Self {+ Self {+ subquery: self.subquery.box_clone(),+ max_docs_per_segment: self.max_docs_per_segment,+ }+ }+}++impl Query for ShortCircuitQuery {+ fn weight(+ &self,+ enable_scoring: tantivy::query::EnableScoring,+ ) -> tantivy::Result<Box<dyn Weight>> {+ let schema = match enable_scoring {+ tantivy::query::EnableScoring::Enabled {+ searcher,+ statistics_provider: _,+ } => searcher.schema(),+ tantivy::query::EnableScoring::Disabled {+ schema,+ searcher_opt: _,+ } => schema,+ };++ Ok(Box::new(ShortCircuitWeight {+ subweight: self+ .subquery+ .weight(tantivy::query::EnableScoring::Disabled {+ schema,+ searcher_opt: None,+ })?,+ max_docs_per_segment: self.max_docs_per_segment,+ }))+ }+}++struct ShortCircuitWeight {+ subweight: Box<dyn Weight>,+ max_docs_per_segment: u64,+}++impl Weight for ShortCircuitWeight {+ fn scorer(+ &self,+ reader: &tantivy::SegmentReader,+ boost: Score,+ ) -> tantivy::Result<Box<dyn Scorer>> {+ let subscorer = self.subweight.scorer(reader, boost)?;+ Ok(Box::new(ShortCircuitScorer {+ subscorer,+ num_docs: 0,+ max_docs_per_segment: self.max_docs_per_segment,+ }))+ }++ fn explain(+ &self,+ reader: &tantivy::SegmentReader,+ doc: tantivy::DocId,+ ) -> tantivy::Result<tantivy::query::Explanation> {+ let mut scorer = self.scorer(reader, 1.0)?;+ if scorer.seek(doc) != doc {+ return Err(tantivy::TantivyError::InvalidArgument(format!(+ "Document #({doc}) does not match"+ )));+ }+ let score = scorer.score();++ let mut expl = Explanation::new("Short circuited query", score);+ expl.add_context(format!(+ "Max docs per segment: {}",+ self.max_docs_per_segment+ ));+ expl.add_detail(self.subweight.explain(reader, doc)?);+ Ok(expl)+ }+}++struct ShortCircuitScorer {+ subscorer: Box<dyn Scorer>,+ num_docs: u64,+ max_docs_per_segment: u64,+}++impl Scorer for ShortCircuitScorer {+ fn score(&mut self) -> Score {+ self.subscorer.score()+ }+}++impl DocSet for ShortCircuitScorer {+ fn advance(&mut self) -> tantivy::DocId {+ if self.num_docs >= self.max_docs_per_segment {+ return tantivy::TERMINATED;+ }++ self.num_docs += 1;+ self.subscorer.advance()+ }++ fn doc(&self) -> tantivy::DocId {+ if self.num_docs >= self.max_docs_per_segment {+ return tantivy::TERMINATED;+ }++ self.subscorer.doc()+ }++ fn size_hint(&self) -> u32 {+ self.subscorer+ .size_hint()+ .min(self.max_docs_per_segment as u32)+ }+}diff --git a/core/src/ranking/initial.rs b/core/src/ranking/initial.rsindex 8b06e713..beece4c0 100644--- a/core/src/ranking/initial.rs+++ b/core/src/ranking/initial.rs@@ -22,7 +22,7 @@ use tantivy::{DocId, SegmentReader};use super::SignalAggregator;-pub(crate) struct InitialScoreTweaker {+pub struct InitialScoreTweaker {tv_searcher: tantivy::Searcher,fastfield_reader: FastFieldReader,aggregator: SignalAggregator,@@ -65,7 +65,7 @@ impl ScoreTweaker<Score> for InitialScoreTweaker {}}-pub(crate) struct InitialSegmentScoreTweaker {+pub struct InitialSegmentScoreTweaker {aggregator: SignalAggregator,}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 237a5190..a6081471 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -26,13 +26,11 @@ pub mod query_centrality;pub mod signal;use initial::InitialScoreTweaker;-use tantivy::collector::Collector;use crate::{- collector::{MaxDocsConsidered, TopDocs},+ collector::{MaxDocsConsidered, TopDocs, MainCollector},config::CollectorConfig,fastfield_reader::FastFieldReader,- inverted_index,search_ctx::Ctx,searcher::NUM_RESULTS_PER_PAGE,webpage::region::Region,@@ -102,7 +100,7 @@ impl Ranker {pub fn collector(&self,ctx: Ctx,- ) -> impl Collector<Fruit = Vec<inverted_index::WebsitePointer>> {+ ) -> MainCollector {let aggregator = self.aggregator();let score_tweaker =@@ -127,7 +125,7 @@ impl Ranker {collector = collector.and_collector_config(self.collector_config.clone());- collector.tweak_score(score_tweaker)+ collector.main_collector(score_tweaker)}pub fn set_query_centrality(&mut self, query_centrality: query_centrality::Scorer) {@@ -648,7 +646,7 @@ mod tests {}).unwrap();- assert_eq!(res.num_hits, 3);+ assert_eq!(res.webpages.len(), 3);assert_eq!(&res.webpages[0].url, "https://www.title.com/");let res = searcher@@ -666,7 +664,7 @@ mod tests {}).unwrap();- assert_eq!(res.num_hits, 3);+ assert_eq!(res.webpages.len(), 3);assert_eq!(&res.webpages[0].url, "https://www.centrality.com/");}@@ -786,7 +784,6 @@ mod tests {}).expect("Search failed");- assert_eq!(result.num_hits, 3);assert_eq!(result.webpages.len(), 3);assert_eq!(result.webpages[0].url, "https://www.first.com/");assert_eq!(result.webpages[1].url, "https://www.second.com/");@@ -867,7 +864,6 @@ mod tests {}).expect("Search failed");- assert_eq!(result.num_hits, 2);assert_eq!(result.webpages.len(), 2);assert_eq!(result.webpages[0].url, "https://www.first.com/");assert_eq!(result.webpages[1].url, "https://www.second.com/");@@ -977,7 +973,6 @@ mod tests {}).expect("Search failed");- assert_eq!(result.num_hits, 3);assert_eq!(result.webpages.len(), 3);assert_eq!(result.webpages[0].url, "https://www.first.com/one");assert_eq!(result.webpages[1].url, "https://www.second.com/one/two");diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 3ed1b3b3..029e7962 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -188,7 +188,6 @@ mod tests {}).expect("Search failed");- assert_eq!(result.num_hits, 3);assert_eq!(result.webpages.len(), 3);assert_eq!(result.webpages[0].url, "https://www.first.com/");assert_eq!(result.webpages[1].url, "https://www.second.com/");@@ -206,7 +205,6 @@ mod tests {}).expect("Search failed");- assert_eq!(result.num_hits, 3);assert_eq!(result.webpages.len(), 3);assert_eq!(result.webpages[0].url, "https://www.first.com/");assert_eq!(result.webpages[1].url, "https://www.second.com/");@@ -225,7 +223,6 @@ mod tests {}).expect("Search failed");- assert_eq!(result.num_hits, 2);assert_eq!(result.webpages.len(), 2);assert_eq!(result.webpages[0].url, "https://www.third.com/");assert_eq!(result.webpages[1].url, "https://www.second.com/");diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex ed8c8302..91493f5c 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -256,7 +256,7 @@ impl ApiSearcher {let num_results: usize = initial_results.iter()- .map(|res| res.local_result.num_websites)+ .map(|res| res.local_result.websites.len()).sum();if num_results < NUM_RESULTS / 2 {diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 13c74180..edf41aa5 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -58,7 +58,7 @@ impl From<Index> for LocalSearcher {struct InvertedIndexResult {webpages: Vec<RankingWebsite>,- num_hits: usize,+ num_hits: Option<usize>,has_more: bool,}diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 63a80860..592d2552 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -50,7 +50,7 @@ pub enum SearchResult {pub struct WebsitesResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,- pub num_hits: usize,+ pub num_hits: Option<usize>,pub sidebar: Option<DisplayedSidebar>,pub widget: Option<Widget>,pub direct_answer: Option<DisplayedAnswer>,@@ -70,12 +70,13 @@ pub struct SearchQuery {pub return_ranking_signals: bool,pub safe_search: bool,pub fetch_discussions: bool,+ pub count_results: bool,}#[derive(Debug, Clone, Serialize, Deserialize)]pub struct InitialWebsiteResult {pub spell_corrected_query: Option<Correction>,- pub num_websites: usize,+ pub num_websites: Option<usize>,pub websites: Vec<RankingWebsite>,pub has_more: bool,pub entity_sidebar: Option<EntityMatch>,@@ -96,6 +97,7 @@ impl Default for SearchQuery {return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),safe_search: defaults::SearchQuery::safe_search(),fetch_discussions: defaults::SearchQuery::fetch_discussions(),+ count_results: defaults::SearchQuery::count_results(),}}}diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex b205d49a..5ea033ec 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -393,7 +393,6 @@ Survey in 2016, 2017, and 2018."#;}).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(highlight(result.webpages[0].snippet.clone()), format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?"));}@@ -433,7 +432,6 @@ Survey in 2016, 2017, and 2018."#;}).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(highlight(result.webpages[0].snippet.clone()), format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?"));}@@ -473,7 +471,6 @@ Survey in 2016, 2017, and 2018."#;}).expect("Search failed");- assert_eq!(result.num_hits, 1);assert_eq!(result.webpages.len(), 1);assert_eq!(highlight(result.webpages[0].snippet.clone()),diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex 221d86bc..458c11fb 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -192,6 +192,7 @@ export const api = {};export type ApiSearchQuery = {+ countResults?: boolean;fetchDiscussions?: boolean;flattenResponse?: boolean;numResults?: number;@@ -426,7 +427,7 @@ export type WebsitesResult = {directAnswer?: DisplayedAnswer;discussions?: DisplayedWebpage[];hasMoreResults: boolean;- numHits: number;+ numHits?: number;searchDurationMs: number;sidebar?: DisplayedSidebar;spellCorrectedQuery?: HighlightedSpellCorrection;diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelteindex 24bc8132..8f5504c3 100644--- a/frontend/src/routes/search/+page.svelte+++ b/frontend/src/routes/search/+page.svelte@@ -153,10 +153,14 @@<div class="mx-auto flex w-full justify-between"><div class="flex h-full flex-col justify-center space-x-2 text-sm text-neutral"><p class="h-fit">- Found {results.numHits.toLocaleString()} results in {(- (results.searchDurationMs ?? 0) / 1000- ).toFixed(2)}- seconds+ {#if results.numHits }+ Found {results.numHits.toLocaleString()} results in {(+ (results.searchDurationMs ?? 0) / 1000+ ).toFixed(2)}+ seconds+ {:else}+ Search took {((results.searchDurationMs ?? 0) / 1000).toFixed(2)} seconds+ {/if}</p></div><div class="flex space-x-2">diff --git a/frontend/src/routes/search/+page.ts b/frontend/src/routes/search/+page.tsindex d3a5a3f6..4f52f0b2 100644--- a/frontend/src/routes/search/+page.ts+++ b/frontend/src/routes/search/+page.ts@@ -32,6 +32,7 @@ export const load: PageLoad = async (req) => {selectedRegion: params.selectedRegion,siteRankings: params.siteRankings,fetchDiscussions: true,+ countResults: true,},{ fetch },);