web search done right
refactor searcher to have separate paths for api and html
Mikkel Denker2064cc4ad51a
diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rsindex 9258cdc5..54eba9f4 100644--- a/benches/search-preindexed-goggle.rs+++ b/benches/search-preindexed-goggle.rs@@ -1,5 +1,8 @@use criterion::{criterion_group, criterion_main, Criterion};-use cuely::{index::Index, searcher::LocalSearcher};+use cuely::{+ index::Index,+ searcher::{LocalSearcher, SearchQuery},+};const INDEX_PATH: &str = "data/index";@@ -12,7 +15,12 @@ macro_rules! bench {$c.bench_function(desc.as_str(), |b| {b.iter(|| {$searcher- .search($query, None, Some($goggle.to_string()), None)+ .search(&SearchQuery {+ original: $query.to_string(),+ selected_region: None,+ goggle_program: Some($goggle.to_string()),+ skip_pages: None,+ }).unwrap()})});diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rsindex bbcdee67..87dfa85e 100644--- a/benches/search-preindexed.rs+++ b/benches/search-preindexed.rs@@ -1,5 +1,8 @@use criterion::{criterion_group, criterion_main, Criterion};-use cuely::{index::Index, searcher::LocalSearcher};+use cuely::{+ index::Index,+ searcher::{LocalSearcher, SearchQuery},+};const INDEX_PATH: &str = "data/index";@@ -9,7 +12,16 @@ macro_rules! bench {desc.push_str($query);desc.push('\'');$c.bench_function(desc.as_str(), |b| {- b.iter(|| $searcher.search($query, None, None, None).unwrap())+ b.iter(|| {+ $searcher+ .search(&SearchQuery {+ original: $query.to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ })+ .unwrap()+ })});};}diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rsindex 63f90317..08c8209f 100644--- a/src/entrypoint/search_server.rs+++ b/src/entrypoint/search_server.rs@@ -42,14 +42,8 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {loop {if let Ok(req) = server.accept::<searcher::distributed::Request>().await {match &req.body {- searcher::Request::Search(search) => {- match local_searcher.search_initial(- &search.original,- search.selected_region,- search.goggle_program.clone(),- search.skip_pages,- false,- ) {+ searcher::Request::Search(query) => {+ match local_searcher.search_initial(query, false) {Ok(response) => {req.respond(sonic::Response::Content(response)).await.ok();}@@ -74,30 +68,19 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {}}}- searcher::Request::SearchPrettified(search) => {- match local_searcher.search_initial(- &search.original,- search.selected_region,- search.goggle_program.clone(),- search.skip_pages,- false,- ) {+ searcher::Request::SearchPrettified(query) => {+ match local_searcher.search_initial(query, false) {Ok(result) => match result {- searcher::InitialSearchResult::Websites(- searcher::InitialWebsitesFormatting::Raw(result),- ) => {+ searcher::InitialSearchResult::Websites(result) => {let res = search_prettifier::initial(result, &local_searcher);req.respond(sonic::Response::Content(- searcher::InitialSearchResult::Websites(- searcher::InitialWebsitesFormatting::Prettified(res),- ),+ searcher::InitialPrettifiedSearchResult::Websites(res),)).await.ok();}searcher::InitialSearchResult::Bang(_) => todo!(),- _ => unreachable!(),},Err(_) => {req.respond::<inverted_index::SearchResult>(sonic::Response::Empty)diff --git a/src/frontend/api.rs b/src/frontend/api.rsindex f935af6d..2fe812f5 100644--- a/src/frontend/api.rs+++ b/src/frontend/api.rs@@ -16,7 +16,7 @@use std::{collections::HashMap, sync::Arc};-use crate::webpage::region::Region;+use crate::{searcher::SearchQuery, webpage::region::Region};use super::State;use axum::{extract, response::IntoResponse, Extension, Json};@@ -40,7 +40,12 @@ pub async fn search(match state.searcher- .search_api(query.as_str(), selected_region, None, skip_pages)+ .search_api(&SearchQuery {+ original: query.to_string(),+ selected_region,+ goggle_program: None,+ skip_pages,+ }).await{Ok(result) => Json(result),diff --git a/src/frontend/search.rs b/src/frontend/search.rsindex d326921c..c0f52557 100644--- a/src/frontend/search.rs+++ b/src/frontend/search.rs@@ -20,7 +20,7 @@ use axum::Extension;use crate::{search_prettifier::{thousand_sep_number, DisplayedEntity, DisplayedWebpage},- searcher::{self, PrettifiedSearchResult},+ searcher::{self, PrettifiedSearchResult, SearchQuery},webpage::region::{Region, ALL_REGIONS},};@@ -91,7 +91,12 @@ pub async fn route(match state.searcher- .search_prettified(query.as_str(), selected_region, goggle, skip_pages)+ .search_prettified(&SearchQuery {+ original: query.clone(),+ selected_region,+ goggle_program: goggle,+ skip_pages,+ }).await{Ok(result) => match result {diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rsindex 7fe3f21f..485267f7 100644--- a/src/ranking/goggles/mod.rs+++ b/src/ranking/goggles/mod.rs@@ -294,7 +294,7 @@ mod tests {use crate::{index::Index,schema::create_schema,- searcher::LocalSearcher,+ searcher::{LocalSearcher, SearchQuery},webpage::{Html, Webpage},};@@ -361,7 +361,12 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search("website", None, None, None)+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).unwrap().into_websites().unwrap()@@ -373,17 +378,17 @@ mod tests {assert_eq!(res[1].url, "https://www.a.com");let res = searcher- .search(- "website",- None,- Some(+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: Some(r#"- $discard,site=b.com- "#+ $discard,site=b.com+ "#.to_string(),),- None,- )+ skip_pages: None,+ }).unwrap().into_websites().unwrap()@@ -394,17 +399,17 @@ mod tests {assert_eq!(res[0].url, "https://www.a.com");let res = searcher- .search(- "website",- None,- Some(+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: Some(r#"- $boost=10,site=a.com- "#+ $boost=10,site=a.com+ "#.to_string(),),- None,- )+ skip_pages: None,+ }).unwrap().into_websites().unwrap()@@ -483,12 +488,14 @@ mod tests {let searcher = LocalSearcher::from(index);let _ = searcher- .search(- "website",- None,- Some(include_str!("../../../testcases/goggles/quickstart.goggle").to_string()),- None,- )+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: Some(+ include_str!("../../../testcases/goggles/quickstart.goggle").to_string(),+ ),+ skip_pages: None,+ }).unwrap().into_websites().unwrap()@@ -496,12 +503,14 @@ mod tests {.documents;let _ = searcher- .search(- "website",- None,- Some(include_str!("../../../testcases/goggles/hacker_news.goggle").to_string()),- None,- )+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: Some(+ include_str!("../../../testcases/goggles/hacker_news.goggle").to_string(),+ ),+ skip_pages: None,+ }).unwrap().into_websites().unwrap()@@ -509,14 +518,14 @@ mod tests {.documents;let _ = searcher- .search(- "website",- None,- Some(+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: Some(include_str!("../../../testcases/goggles/copycats_removal.goggle").to_string(),),- None,- )+ skip_pages: None,+ }).unwrap().into_websites().unwrap()@@ -609,10 +618,10 @@ mod tests {let searcher = LocalSearcher::from(index);let res = searcher- .search(- "website",- None,- Some(+ .search(&SearchQuery {+ original: "website".to_string(),+ selected_region: None,+ goggle_program: Some(r#"$discard$site=a.com,boost=6@@ -620,8 +629,8 @@ mod tests {"#.to_string(),),- None,- )+ skip_pages: None,+ }).unwrap().into_websites().unwrap()diff --git a/src/ranking/mod.rs b/src/ranking/mod.rsindex 5151301b..8a5f5683 100644--- a/src/ranking/mod.rs+++ b/src/ranking/mod.rs@@ -106,7 +106,7 @@ impl Ranker {mod tests {use crate::{index::Index,- searcher::LocalSearcher,+ searcher::{LocalSearcher, SearchQuery},webpage::{Html, Link, Webpage},};@@ -171,7 +171,12 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search("example", None, None, None)+ .search(&SearchQuery {+ original: "example".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -239,7 +244,12 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search("example", None, None, None)+ .search(&SearchQuery {+ original: "example".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -329,7 +339,12 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search("dr dk", None, None, None)+ .search(&SearchQuery {+ original: "dr dk".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -398,7 +413,12 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search("title", None, None, None)+ .search(&SearchQuery {+ original: "title".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -472,7 +492,12 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search("test", None, None, None)+ .search(&SearchQuery {+ original: "test".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -539,7 +564,12 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);let result = searcher- .search("test", None, None, None)+ .search(&SearchQuery {+ original: "test".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -630,18 +660,18 @@ mod tests {let searcher = LocalSearcher::new(index, None, None);let res = searcher- .search(- "example",- None,- Some(+ .search(&SearchQuery {+ original: "example".to_string(),+ selected_region: None,+ goggle_program: Some(r#"@field_title = 20000000@host_centrality = 0"#.to_string(),),- None,- )+ skip_pages: None,+ }).unwrap().into_websites().unwrap();@@ -650,18 +680,18 @@ mod tests {assert_eq!(&res.webpages.documents[0].url, "https://www.title.com");let res = searcher- .search(- "example",- None,- Some(+ .search(&SearchQuery {+ original: "example".to_string(),+ selected_region: None,+ goggle_program: Some(r#"- @field_all_body= 20000000+ @field_all_body = 20000000@host_centrality = 0"#.to_string(),),- None,- )+ skip_pages: None,+ }).unwrap().into_websites().unwrap();@@ -670,12 +700,17 @@ mod tests {assert_eq!(&res.webpages.documents[0].url, "https://www.body.com");let res = searcher- .search(- "example",- None,- Some("@host_centrality= 2000000".to_string()),- None,- )+ .search(&SearchQuery {+ original: "example".to_string(),+ selected_region: None,+ goggle_program: Some(+ r#"+ @host_centrality = 2000000+ "#+ .to_string(),+ ),+ skip_pages: None,+ }).unwrap().into_websites().unwrap();@@ -766,7 +801,12 @@ mod tests {let searcher = LocalSearcher::new(index, None, None);let result = searcher- .search("termA termB", None, None, None)+ .search(&SearchQuery {+ original: "termA termB".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap()@@ -837,7 +877,12 @@ mod tests {let searcher = LocalSearcher::new(index, None, None);let result = searcher- .search("test", None, None, None)+ .search(&SearchQuery {+ original: "test".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap()diff --git a/src/searcher/distributed.rs b/src/searcher/distributed.rsindex f9e17a72..745464eb 100644--- a/src/searcher/distributed.rs+++ b/src/searcher/distributed.rs@@ -19,10 +19,7 @@ use crate::{exponential_backoff::ExponentialBackoff,inverted_index::{self, RetrievedWebpage},search_prettifier::DisplayedWebpage,- searcher::{- InitialWebsitesFormatting, PrettifiedWebsitesResult, SearchResult, WebsitesResult,- NUM_RESULTS_PER_PAGE,- },+ searcher::{PrettifiedWebsitesResult, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE},};use std::{cmp::Ordering, net::SocketAddr, time::Instant};@@ -32,9 +29,11 @@ use futures::StreamExt;use serde::{Deserialize, Serialize};use thiserror::Error;-use crate::{sonic, webpage::region::Region};+use crate::sonic;-use super::{InitialSearchResult, PrettifiedSearchResult};+use super::{+ InitialPrettifiedSearchResult, InitialSearchResult, PrettifiedSearchResult, SearchQuery,+};type Result<T> = std::result::Result<T, Error>;@@ -67,7 +66,10 @@ impl RemoteSearcher {Err(Error::SearchFailed)}- async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResult> {+ async fn search_prettified(+ &self,+ query: &SearchQuery,+ ) -> Result<InitialPrettifiedSearchResult> {for timeout in ExponentialBackoff::from_millis(30).take(5) {if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await{@@ -171,7 +173,10 @@ impl Shard {}}- async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {+ async fn search_prettified(+ &self,+ query: &SearchQuery,+ ) -> Result<InitialPrettifiedSearchResultShard> {match self.replicas.iter()@@ -180,7 +185,7 @@ impl Shard {.next().await{- Some(result) => Ok(InitialSearchResultShard {+ Some(result) => Ok(InitialPrettifiedSearchResultShard {local_result: result?,shard: self.id.clone(),}),@@ -230,12 +235,9 @@ struct InitialSearchResultShard {shard: ShardId,}-#[derive(Serialize, Deserialize, Clone)]-pub struct SearchQuery {- pub original: String,- pub selected_region: Option<Region>,- pub goggle_program: Option<String>,- pub skip_pages: Option<usize>,+struct InitialPrettifiedSearchResultShard {+ local_result: InitialPrettifiedSearchResult,+ shard: ShardId,}#[derive(Serialize, Deserialize)]@@ -323,31 +325,18 @@ impl DistributedSearcher {Self { shards }}- pub async fn search_api(- &self,- query: &str,- selected_region: Option<Region>,- goggle_program: Option<String>,- skip_pages: Option<usize>,- ) -> Result<SearchResult> {+ pub async fn search_api(&self, query: &SearchQuery) -> Result<SearchResult> {let start = Instant::now();if query.is_empty() {return Err(Error::EmptyQuery);}- let query = SearchQuery {- original: query.to_string(),- selected_region,- goggle_program,- skip_pages,- };-// search shardslet initial_results = self.shards.iter()- .map(|shard| shard.search(&query))+ .map(|shard| shard.search(query)).collect::<FuturesUnordered<_>>().collect::<Vec<_>>().await@@ -366,9 +355,7 @@ impl DistributedSearcher {}let spell_corrected_query = initial_results.first().and_then(|result| {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =- &result.local_result- {+ if let InitialSearchResult::Websites(result) = &result.local_result {result.spell_corrected_query.clone()} else {None@@ -376,9 +363,7 @@ impl DistributedSearcher {});let entity = initial_results.first().and_then(|result| {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =- &result.local_result- {+ if let InitialSearchResult::Websites(result) = &result.local_result {result.entity.clone()} else {None@@ -388,9 +373,7 @@ impl DistributedSearcher {let num_docs = initial_results.iter().map(|result| {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =- &result.local_result- {+ if let InitialSearchResult::Websites(result) = &result.local_result {result.websites.num_websites} else {0@@ -399,13 +382,11 @@ impl DistributedSearcher {.sum();// combine results- let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);+ let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);let mut collector = BucketCollector::new(top_n);for result in initial_results {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(local_result)) =- result.local_result- {+ if let InitialSearchResult::Websites(local_result) = result.local_result {for website in local_result.websites.top_websites {let pointer = ScoredWebsitePointer {local_pointer: website,@@ -420,7 +401,7 @@ impl DistributedSearcher {let top_websites = collector.into_sorted_vec(true).into_iter()- .skip(skip_pages.unwrap_or(0))+ .skip(query.skip_pages.unwrap_or(0)).take(NUM_RESULTS_PER_PAGE).collect::<Vec<_>>();@@ -465,31 +446,18 @@ impl DistributedSearcher {}))}- pub async fn search_prettified(- &self,- query: &str,- selected_region: Option<Region>,- goggle_program: Option<String>,- skip_pages: Option<usize>,- ) -> Result<PrettifiedSearchResult> {+ pub async fn search_prettified(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {let start = Instant::now();if query.is_empty() {return Err(Error::EmptyQuery);}- let query = SearchQuery {- original: query.to_string(),- selected_region,- goggle_program,- skip_pages,- };-// search shardslet initial_results = self.shards.iter()- .map(|shard| shard.search_prettified(&query))+ .map(|shard| shard.search_prettified(query)).collect::<FuturesUnordered<_>>().collect::<Vec<_>>().await@@ -500,17 +468,15 @@ impl DistributedSearcher {// check if any result has a bang hitif let Some(result) = initial_results.iter()- .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))+ .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_))){- if let InitialSearchResult::Bang(bang) = &result.local_result {+ if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {return Ok(PrettifiedSearchResult::Bang(bang.clone()));}}let spell_corrected_query = initial_results.first().and_then(|result| {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =- &result.local_result- {+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {result.spell_corrected_query.clone()} else {None@@ -518,9 +484,7 @@ impl DistributedSearcher {});let entity = initial_results.first().and_then(|result| {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =- &result.local_result- {+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {result.entity.clone()} else {None@@ -530,10 +494,7 @@ impl DistributedSearcher {let num_docs = initial_results.iter().map(|result| {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(- result,- )) = &result.local_result- {+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {result.websites.num_websites} else {0@@ -542,14 +503,11 @@ impl DistributedSearcher {.sum();// combine results- let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);+ let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);let mut collector = BucketCollector::new(top_n);for result in initial_results {- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(- local_result,- )) = result.local_result- {+ if let InitialPrettifiedSearchResult::Websites(local_result) = result.local_result {for website in local_result.websites.top_websites {let pointer = ScoredWebsitePointer {local_pointer: website,@@ -564,7 +522,7 @@ impl DistributedSearcher {let top_websites = collector.into_sorted_vec(true).into_iter()- .skip(skip_pages.unwrap_or(0))+ .skip(query.skip_pages.unwrap_or(0)).take(NUM_RESULTS_PER_PAGE).collect::<Vec<_>>();diff --git a/src/searcher/local.rs b/src/searcher/local.rsindex f49ed41e..f352423c 100644--- a/src/searcher/local.rs+++ b/src/searcher/local.rs@@ -31,10 +31,7 @@ use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};-use super::{- InitialSearchResult, InitialWebsitesFormatting, SearchResult, WebsitesResult,- NUM_RESULTS_PER_PAGE,-};+use super::{InitialSearchResult, SearchQuery, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE};pub struct LocalSearcher {index: Index,@@ -66,17 +63,17 @@ impl LocalSearcher {pub fn search_initial(&self,- query: &str,- selected_region: Option<Region>,- goggle_program: Option<String>,- skip_pages: Option<usize>,+ query: &SearchQuery,de_rank_similar: bool,) -> Result<InitialSearchResult> {- let raw_query = query.to_string();- let goggle = goggle_program.and_then(|program| goggles::parse(&program).ok());+ let raw_query = query.original.clone();+ let goggle = query+ .goggle_program+ .as_ref()+ .and_then(|program| goggles::parse(program).ok());- let mut query = Query::parse(- query,+ let mut parsed_query = Query::parse(+ &query.original,self.index.schema(),self.index.tokenizers(),goggle@@ -85,18 +82,18 @@ impl LocalSearcher {.unwrap_or(&SignalAggregator::default()),)?;- if query.is_empty() {+ if parsed_query.is_empty() {return Err(Error::EmptyQuery);}if let Some(bangs) = self.bangs.as_ref() {- if let Some(bang) = bangs.get(&query) {+ if let Some(bang) = bangs.get(&parsed_query) {return Ok(InitialSearchResult::Bang(bang));}}if let Some(goggle) = &goggle {- query.set_goggle(goggle, &self.index.schema());+ parsed_query.set_goggle(goggle, &self.index.schema());}let mut ranker = Ranker::new(@@ -104,11 +101,11 @@ impl LocalSearcher {goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),);- if let Some(skip_pages) = skip_pages {+ if let Some(skip_pages) = query.skip_pages {ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);}- if let Some(region) = selected_region {+ if let Some(region) = query.selected_region {if region != Region::All {ranker = ranker.with_region(region);}@@ -117,21 +114,21 @@ impl LocalSearcher {ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());ranker.de_rank_similar(de_rank_similar);- let webpages = self.index.search_initial(&query, ranker.collector())?;- let correction = self.index.spell_correction(&query.simple_terms());+ let webpages = self+ .index+ .search_initial(&parsed_query, ranker.collector())?;+ let correction = self.index.spell_correction(&parsed_query.simple_terms());let entity = self.entity_index.as_ref().and_then(|index| index.search(&raw_query));- Ok(InitialSearchResult::Websites(- InitialWebsitesFormatting::Raw(InitialWebsiteResult {- spell_corrected_query: correction,- websites: webpages,- entity,- }),- ))+ Ok(InitialSearchResult::Websites(InitialWebsiteResult {+ spell_corrected_query: correction,+ websites: webpages,+ entity,+ }))}pub fn retrieve_websites(@@ -153,22 +150,16 @@ impl LocalSearcher {self.index.retrieve_websites(websites, &query)}- pub fn search(- &self,- query: &str,- selected_region: Option<Region>,- goggle_program: Option<String>,- skip_pages: Option<usize>,- ) -> Result<SearchResult> {+ pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {let start = Instant::now();- let initial_result =- self.search_initial(query, selected_region, goggle_program, skip_pages, true)?;+ let query_text = query.original.clone();+ let initial_result = self.search_initial(query, true)?;match initial_result {- InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(search_result)) => {+ InitialSearchResult::Websites(search_result) => {let retrieved_sites =- self.retrieve_websites(&search_result.websites.top_websites, query)?;+ self.retrieve_websites(&search_result.websites.top_websites, &query_text)?;Ok(SearchResult::Websites(WebsitesResult {spell_corrected_query: search_result.spell_corrected_query,@@ -181,7 +172,6 @@ impl LocalSearcher {}))}InitialSearchResult::Bang(bang) => Ok(SearchResult::Bang(bang)),- _ => unreachable!(),}}@@ -264,7 +254,12 @@ mod tests {for p in 0..NUM_PAGES {let urls: Vec<_> = searcher- .search("test", None, None, Some(p))+ .search(&SearchQuery {+ original: "test".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: Some(p),+ }).unwrap().into_websites().unwrap()diff --git a/src/searcher/mod.rs b/src/searcher/mod.rsindex 6f9c3f04..61f0541a 100644--- a/src/searcher/mod.rs+++ b/src/searcher/mod.rs@@ -26,6 +26,7 @@ use crate::{entity_index::StoredEntity,inverted_index,search_prettifier::{self, DisplayedEntity, DisplayedWebpage},+ webpage::region::Region,};pub const NUM_RESULTS_PER_PAGE: usize = 20;@@ -61,12 +62,26 @@ pub struct PrettifiedWebsitesResult {#[derive(Debug, Serialize, Deserialize)]pub enum InitialSearchResult {- Websites(InitialWebsitesFormatting),+ Websites(local::InitialWebsiteResult),Bang(BangHit),}#[derive(Debug, Serialize, Deserialize)]-pub enum InitialWebsitesFormatting {- Raw(local::InitialWebsiteResult),- Prettified(search_prettifier::InitialWebsiteResult),+pub enum InitialPrettifiedSearchResult {+ Websites(search_prettifier::InitialWebsiteResult),+ Bang(BangHit),+}++#[derive(Serialize, Deserialize, Clone)]+pub struct SearchQuery {+ pub original: String,+ pub selected_region: Option<Region>,+ pub goggle_program: Option<String>,+ pub skip_pages: Option<usize>,+}++impl SearchQuery {+ pub fn is_empty(&self) -> bool {+ self.original.is_empty()+ }}diff --git a/src/snippet.rs b/src/snippet.rsindex 264ca9c4..462befd4 100644--- a/src/snippet.rs+++ b/src/snippet.rs@@ -320,7 +320,11 @@ pub fn generate(#[cfg(test)]mod tests {use super::*;- use crate::{index::Index, searcher::LocalSearcher, webpage::Webpage};+ use crate::{+ index::Index,+ searcher::{LocalSearcher, SearchQuery},+ webpage::Webpage,+ };use maplit::btreemap;use tantivy::tokenizer::SimpleTokenizer;@@ -363,7 +367,12 @@ Survey in 2016, 2017, and 2018."#;let searcher = LocalSearcher::from(index);let result = searcher- .search("rust language", None, None, None)+ .search(&SearchQuery {+ original: "rust language".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -400,7 +409,12 @@ Survey in 2016, 2017, and 2018."#;let searcher = LocalSearcher::from(index);let result = searcher- .search("describe", None, None, None)+ .search(&SearchQuery {+ original: "describe".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();@@ -592,7 +606,12 @@ Survey in 2016, 2017, and 2018."#;let searcher = LocalSearcher::from(index);let result = searcher- .search("paradigms", None, None, None)+ .search(&SearchQuery {+ original: "paradigms".to_string(),+ selected_region: None,+ goggle_program: None,+ skip_pages: None,+ }).expect("Search failed").into_websites().unwrap();