web search done right
Cleanup search result types
Mikkel Denker9986d05629de
diff --git a/configs/frontend.toml b/configs/frontend.tomlindex 3abc2979..e17cda94 100644--- a/configs/frontend.toml+++ b/configs/frontend.toml@@ -1,4 +1,5 @@queries_csv_path = "data/queries_us.csv"host = "0.0.0.0:3000"search_servers = [["0.0.0.0:3001"]]-crossencoder_model_path = "data/cross_encoder"\ No newline at end of file+crossencoder_model_path = "data/cross_encoder"+bangs_path = "data/bangs.json"\ No newline at end of filediff --git a/configs/search_server.toml b/configs/search_server.tomlindex c35d3057..d2692b76 100644--- a/configs/search_server.toml+++ b/configs/search_server.toml@@ -1,5 +1,4 @@host = "0.0.0.0:3001"index_path = "data/index"entity_index_path = "data/entity"-bangs_path = "data/bangs.json"centrality_store_path = "data/centrality"\ No newline at end of filediff --git a/core/src/bangs.rs b/core/src/bangs.rsindex 18a41903..05175068 100644--- a/core/src/bangs.rs+++ b/core/src/bangs.rs@@ -19,10 +19,7 @@ use std::{collections::HashMap, fs, path::Path};use itertools::intersperse;use serde::{Deserialize, Serialize};-use crate::{- query::{parser::Term, Query},- webpage::Url,-};+use crate::{query::parser::Term, webpage::Url};pub const BANG_PREFIX: char = '!';@@ -78,8 +75,8 @@ impl Bangs {}}- pub fn get(&self, query: &Query) -> Option<BangHit> {- for possible_bang in query.terms().iter().filter_map(|term| {+ pub fn get(&self, terms: &[Box<Term>]) -> Option<BangHit> {+ for possible_bang in terms.iter().filter_map(|term| {if let Term::PossibleBang(possible_bang) = term.as_ref() {Some(possible_bang)} else {@@ -94,8 +91,7 @@ impl Bangs {.replace("{{{s}}}",intersperse(- query- .terms()+ terms.iter().filter(|term| {if let Term::PossibleBang(bang) = term.as_ref() {@@ -121,19 +117,13 @@ impl Bangs {#[cfg(test)]mod tests {- use std::sync::Arc;- use tantivy::tokenizer::TokenizerManager;-- use crate::{ranking::SignalAggregator, schema::create_schema};+ use crate::query::parser::parse;use super::*;#[test]fn simple_bang() {- let schema = Arc::new(create_schema());- let tokenizer_manager = TokenizerManager::new();-let bangs = Bangs::from_json(r#"[{"c": "Multimedia",@@ -146,33 +136,11 @@ mod tests {}]"#,);- let query = Query::parse(- "no bangs",- schema.clone(),- &tokenizer_manager,- &SignalAggregator::default(),- )- .unwrap();- assert_eq!(bangs.get(&query), None);-- let query = Query::parse(- "!no bangs",- schema.clone(),- &tokenizer_manager,- &SignalAggregator::default(),- )- .unwrap();- assert_eq!(bangs.get(&query), None);-- let query = Query::parse(- "!ty bangs",- schema,- &tokenizer_manager,- &SignalAggregator::default(),- )- .unwrap();+ assert_eq!(bangs.get(&parse("no bangs")), None);+ assert_eq!(bangs.get(&parse("!no bangs")), None);+assert_eq!(- bangs.get(&query),+ bangs.get(&parse("!ty bangs")),Some(BangHit {bang: Bang {category: Some("Multimedia".to_string()),diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rsindex 64f633dd..e3f0003f 100644--- a/core/src/entrypoint/frontend.rs+++ b/core/src/entrypoint/frontend.rs@@ -16,16 +16,16 @@use anyhow::Result;-use crate::frontend::router;+use crate::{frontend::router, FrontendConfig};-pub async fn run(- queries_csv_path: &str,- crossencoder_model_path: &str,- host: &str,- shards: Vec<Vec<String>>,-) -> Result<()> {- let app = router(queries_csv_path, crossencoder_model_path, shards)?;- let addr = host.parse()?;+pub async fn run(config: FrontendConfig) -> Result<()> {+ let app = router(+ &config.queries_csv_path,+ &config.crossencoder_model_path,+ &config.bangs_path,+ config.search_servers,+ )?;+ let addr = config.host.parse()?;tracing::info!("listening on {}", addr);axum::Server::bind(&addr).serve(app.into_make_service())diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 7bd9768b..4f0939ad 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -17,12 +17,10 @@use std::net::SocketAddr;use crate::{- bangs::Bangs,entity_index::EntityIndex,index::Index,inverted_index,ranking::centrality_store::CentralityStore,- search_prettifier::{self},searcher::{self, LocalSearcher},sonic, Result, SearchServerConfig,};@@ -35,7 +33,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {let entity_index = config.entity_index_path.map(|path| EntityIndex::open(path).unwrap());- let bangs = config.bangs_path.map(Bangs::from_path);let centrality_store = config.centrality_store_path.map(CentralityStore::open);let search_index = Index::open(config.index_path)?;@@ -45,10 +42,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {local_searcher.set_entity_index(entity_index);}- if let Some(bangs) = bangs {- local_searcher.set_bangs(bangs);- }-if let Some(centrality_store) = centrality_store {local_searcher.set_centrality_store(centrality_store);}@@ -56,18 +49,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {loop {if let Ok(req) = server.accept::<searcher::distributed::Request>().await {match &req.body {- searcher::Request::Search(query) => {- match local_searcher.search_initial(query, false) {- Ok(response) => {- req.respond(sonic::Response::Content(response)).await.ok();- }- Err(_) => {- req.respond::<searcher::InitialSearchResult>(sonic::Response::Empty)- .await- .ok();- }- }- }searcher::Request::RetrieveWebsites { websites, query } => {match local_searcher.retrieve_websites(websites, query) {Ok(response) => {@@ -82,26 +63,11 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {}}}- searcher::Request::SearchPrettified(query) => {+ searcher::Request::Search(query) => {match local_searcher.search_initial(query, false) {- Ok(result) => match result {- searcher::InitialSearchResult::Websites(result) => {- let res = search_prettifier::initial(result);-- req.respond(sonic::Response::Content(- searcher::InitialPrettifiedSearchResult::Websites(res),- ))- .await- .ok();- }- searcher::InitialSearchResult::Bang(bang) => {- req.respond(sonic::Response::Content(- searcher::InitialSearchResult::Bang(bang),- ))- .await- .ok();- }- },+ Ok(result) => {+ req.respond(sonic::Response::Content(result)).await.ok();+ }Err(_) => {req.respond::<inverted_index::SearchResult>(sonic::Response::Empty).awaitdiff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rsindex 66f9891d..4cb5f44d 100644--- a/core/src/frontend/mod.rs+++ b/core/src/frontend/mod.rs@@ -19,6 +19,7 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};use crate::{autosuggest::Autosuggest,+ bangs::Bangs,ranking::models::cross_encoder::CrossEncoderModel,searcher::{DistributedSearcher, Shard},};@@ -78,17 +79,19 @@ pub async fn favicon() -> impl IntoResponse {pub fn router(queries_csv_path: &str,crossencoder_model_path: &str,+ bangs_path: &str,shards: Vec<Vec<String>>,) -> Result<Router> {let shards: Vec<_> = shards.into_iter().enumerate()- .map(|(id, replicas)| Shard::new(id as u32, replicas))+ .map(|(id, replicas)| Shard::new(id as u64, replicas)).collect();let autosuggest = Autosuggest::load_csv(queries_csv_path)?;let crossencoder = CrossEncoderModel::open(crossencoder_model_path)?;- let searcher = DistributedSearcher::new(shards, crossencoder);+ let bangs = Bangs::from_path(bangs_path);+ let searcher = DistributedSearcher::new(shards, crossencoder, bangs);let state = Arc::new(State {searcher,diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex 0adc6c2f..d8a11550 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -25,7 +25,7 @@ use crate::{thousand_sep_number, CodeOrText, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,Snippet,},- searcher::{self, PrettifiedSearchResult, SearchQuery, NUM_RESULTS_PER_PAGE},+ searcher::{self, SearchQuery, SearchResult, NUM_RESULTS_PER_PAGE},webpage::region::{Region, ALL_REGIONS},};@@ -131,11 +131,11 @@ pub async fn route(.await{Ok(result) => match result {- PrettifiedSearchResult::Websites(result) => {+ SearchResult::Websites(result) => {let sidebar = result.sidebar;let spell_correction = result.spell_corrected_query;- let num_matches = thousand_sep_number(result.num_docs);+ let num_matches = thousand_sep_number(result.num_hits);let search_duration_sec =format!("{:.2}", result.search_duration_ms as f64 / 1000.0);@@ -196,9 +196,7 @@ pub async fn route(HtmlTemplate(template).into_response()}- PrettifiedSearchResult::Bang(result) => {- Redirect::to(&result.redirect_to.full()).into_response()- }+ SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),},Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),Err(_) => panic!("Search failed"), // TODO: show 500 status to user here@@ -216,10 +214,8 @@ pub async fn api(match state.searcher.search(&query).await {Ok(result) => match result {- PrettifiedSearchResult::Websites(result) => Json(result).into_response(),- PrettifiedSearchResult::Bang(result) => {- Redirect::to(&result.redirect_to.full()).into_response()- }+ SearchResult::Websites(result) => Json(result).into_response(),+ SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),},Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),Err(_) => panic!("Search failed"), // TODO: show 500 status to user herediff --git a/core/src/index.rs b/core/src/index.rsindex f0a09711..dc4e272e 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -402,14 +402,11 @@ mod tests {query: "website".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;-- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.example.com");+ .expect("Search failed");++ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.example.com");}#[test]diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex c58c4af1..fbda560b 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -206,7 +206,7 @@ impl InvertedIndex {for pointer in pointers {let doc = searcher.doc(pointer.address.into())?;- top_websites.push(RankingWebsite::new(doc, pointer, aggregator))+ top_websites.push(RankingWebsite::new(doc, pointer, aggregator));}Ok(top_websites)@@ -237,7 +237,7 @@ impl InvertedIndex {let mut webpages: Vec<RetrievedWebpage> = websites.iter().map(|website| self.retrieve_doc(website.address, &searcher))- .filter_map(|page| page.ok())+ .filter_map(|res| res.ok()).map(|mut doc| {if let Some(image) = doc.primary_image.as_ref() {if !query.simple_terms().into_iter().all(|term| {@@ -438,14 +438,14 @@ impl From<Document> for RetrievedWebpage {.value.as_text().expect("Title field should be text")- .to_string()+ .to_string();}Field::Text(TextField::StemmedCleanBody) => {webpage.body = value.value.as_text().expect("Body field should be text")- .to_string()+ .to_string();}Field::Text(TextField::Description) => {let desc = value@@ -461,7 +461,7 @@ impl From<Document> for RetrievedWebpage {.value.as_text().expect("Url field should be text")- .to_string()+ .to_string();}Field::Text(TextField::PrimaryImage) => {webpage.primary_image = {@@ -488,7 +488,7 @@ impl From<Document> for RetrievedWebpage {.value.as_text().expect("All body field should be text")- .to_string()+ .to_string();}Field::Fast(FastField::Region) => {webpage.region = {@@ -500,7 +500,7 @@ impl From<Document> for RetrievedWebpage {let facet = value.value.as_facet().unwrap();if !facet.is_root() {- webpage.host_topic = Some(facet.clone().into())+ webpage.host_topic = Some(facet.clone().into());}}Field::Text(TextField::DmozDescription) => {diff --git a/core/src/lib.rs b/core/src/lib.rsindex bf9f48a4..5349b34f 100644--- a/core/src/lib.rs+++ b/core/src/lib.rs@@ -152,6 +152,7 @@ pub struct FrontendConfig {pub queries_csv_path: String,pub host: String,pub crossencoder_model_path: String,+ pub bangs_path: String,pub search_servers: Vec<Vec<String>>,}@@ -159,7 +160,6 @@ pub struct FrontendConfig {pub struct SearchServerConfig {pub index_path: String,pub entity_index_path: Option<String>,- pub bangs_path: Option<String>,pub centrality_store_path: Option<String>,pub host: String,}diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 42daf788..ec661909 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -352,15 +352,10 @@ mod tests {index.commit().expect("failed to commit index");let searcher = LocalSearcher::from(index);- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.second.com");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.second.com");}#[test]@@ -404,43 +399,28 @@ mod tests {query: "test site:first.com".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.first.com");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com");let query = SearchQuery {query: "test site:www.first.com".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.first.com");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com");let query = SearchQuery {query: "test -site:first.com".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.second.com");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.second.com");}#[test]@@ -484,15 +464,10 @@ mod tests {query: "intitle:website".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.first.com");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com");}#[test]@@ -536,15 +511,10 @@ mod tests {query: "test inurl:forum".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.first.com/forum");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.first.com/forum");}#[test]@@ -618,41 +588,26 @@ mod tests {query: "test site:first.com".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 0);- assert_eq!(result.documents.len(), 0);+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 0);+ assert_eq!(result.webpages.len(), 0);let query = SearchQuery {query: "test site:the-first.com".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.the-first.com");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.the-first.com");let query = SearchQuery {query: "test site:www.the-first.com".to_string(),..Default::default()};- let result = searcher- .search(&query)- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;- assert_eq!(result.num_docs, 1);- assert_eq!(result.documents.len(), 1);- assert_eq!(result.documents[0].url, "https://www.the-first.com");+ let result = searcher.search(&query).expect("Search failed");+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(result.webpages[0].url, "https://www.the-first.com");}}diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 9c89757c..e0b05eae 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -287,10 +287,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 2);assert_eq!(res[0].url, "https://www.b.com");@@ -313,10 +310,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 1);assert_eq!(res[0].url, "https://www.a.com");@@ -338,10 +332,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 2);assert_eq!(res[0].url, "https://www.a.com");@@ -424,10 +415,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;let _ = searcher.search(&SearchQuery {@@ -438,10 +426,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;let _ = searcher.search(&SearchQuery {@@ -452,10 +437,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;}#[test]@@ -581,10 +563,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 2);assert_eq!(res[0].url, "https://www.a.com/this/is/a/pattern");@@ -741,10 +720,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 3);assert_eq!(res[0].url, "https://www.b.com/this/is/b/pattern");@@ -865,10 +841,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 1);assert_eq!(res[0].url, "https://www.b.com/");@@ -890,10 +863,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 1);assert_eq!(res[0].url, "https://www.b.com/");@@ -915,10 +885,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 1);assert_eq!(res[0].url, "https://www.a.com/");@@ -940,10 +907,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 1);assert_eq!(res[0].url, "https://www.b.com/");@@ -1018,10 +982,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap()- .webpages- .documents;+ .webpages;assert_eq!(res.len(), 0);}diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex a52b2aa1..a4ce8595 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -256,12 +256,10 @@ mod tests {query: "example".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();- assert_eq!(result.webpages.documents.len(), 2);- assert_eq!(result.webpages.documents[0].url, "https://www.b.com");- assert_eq!(result.webpages.documents[1].url, "https://www.a.com");+ .expect("Search failed");+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.b.com");+ assert_eq!(result.webpages[1].url, "https://www.a.com");}#[test]@@ -337,12 +335,10 @@ mod tests {query: "example".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();- assert_eq!(result.webpages.documents.len(), 2);- assert_eq!(result.webpages.documents[0].url, "https://www.b.com");- assert_eq!(result.webpages.documents[1].url, "https://www.a.com");+ .expect("Search failed");+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.b.com");+ assert_eq!(result.webpages[1].url, "https://www.a.com");}#[test]@@ -445,12 +441,10 @@ mod tests {query: "dr dk".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();+ .expect("Search failed");- assert_eq!(result.webpages.documents.len(), 3);- assert_eq!(result.webpages.documents[0].url, "https://www.dr.dk");+ assert_eq!(result.webpages.len(), 3);+ assert_eq!(result.webpages[0].url, "https://www.dr.dk");}#[test]@@ -525,11 +519,9 @@ mod tests {query: "title".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();+ .expect("Search failed");- assert_eq!(result.webpages.documents[0].url, "https://www.new.com");+ assert_eq!(result.webpages[0].url, "https://www.new.com");}#[test]@@ -610,12 +602,10 @@ mod tests {query: "test".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();+ .expect("Search failed");- assert_eq!(result.webpages.documents.len(), 2);- assert_eq!(result.webpages.documents[0].url, "https://www.first.com");+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.first.com");}#[test]@@ -688,12 +678,10 @@ mod tests {query: "test".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();+ .expect("Search failed");- assert_eq!(result.webpages.documents.len(), 2);- assert_eq!(result.webpages.documents[0].url, "https://www.first.com");+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.first.com");}#[test]@@ -805,12 +793,10 @@ mod tests {),..Default::default()})- .unwrap()- .into_websites().unwrap();- assert_eq!(res.webpages.num_docs, 3);- assert_eq!(&res.webpages.documents[0].url, "https://www.title.com");+ assert_eq!(res.num_hits, 3);+ assert_eq!(&res.webpages[0].url, "https://www.title.com");let res = searcher.search(&SearchQuery {@@ -827,12 +813,10 @@ mod tests {),..Default::default()})- .unwrap()- .into_websites().unwrap();- assert_eq!(res.webpages.num_docs, 3);- assert_eq!(&res.webpages.documents[0].url, "https://www.centrality.com");+ assert_eq!(res.num_hits, 3);+ assert_eq!(&res.webpages[0].url, "https://www.centrality.com");}#[test]@@ -936,16 +920,13 @@ mod tests {query: "termA termB".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;-- assert_eq!(result.num_docs, 3);- assert_eq!(result.documents.len(), 3);- assert_eq!(result.documents[0].url, "https://www.first.com");- assert_eq!(result.documents[1].url, "https://www.second.com");- assert_eq!(result.documents[2].url, "https://www.third.com");+ .expect("Search failed");++ assert_eq!(result.num_hits, 3);+ assert_eq!(result.webpages.len(), 3);+ assert_eq!(result.webpages[0].url, "https://www.first.com");+ assert_eq!(result.webpages[1].url, "https://www.second.com");+ assert_eq!(result.webpages[2].url, "https://www.third.com");}#[test]@@ -1020,15 +1001,12 @@ mod tests {query: "test".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;-- assert_eq!(result.num_docs, 2);- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.first.com");- assert_eq!(result.documents[1].url, "https://www.second.com");+ .expect("Search failed");++ assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.first.com");+ assert_eq!(result.webpages[1].url, "https://www.second.com");}#[test]@@ -1103,15 +1081,12 @@ mod tests {query: "test".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;-- assert_eq!(result.num_docs, 2);- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.first.com");- assert_eq!(result.documents[1].url, "https://www.second.com");+ .expect("Search failed");++ assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.first.com");+ assert_eq!(result.webpages[1].url, "https://www.second.com");}#[test]@@ -1271,29 +1246,23 @@ mod tests {query: "topic_a".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;+ .expect("Search failed");- assert_eq!(result.num_docs, 2);- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.a.com");- assert_eq!(result.documents[1].url, "https://www.wrong.com");+ assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.a.com");+ assert_eq!(result.webpages[1].url, "https://www.wrong.com");let result = searcher.search(&SearchQuery {query: "topic_b".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;-- assert_eq!(result.num_docs, 2);- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.b.com");- assert_eq!(result.documents[1].url, "https://www.wrong.com");+ .expect("Search failed");++ assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.b.com");+ assert_eq!(result.webpages[1].url, "https://www.wrong.com");}}diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 665c7230..8e65982b 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -201,16 +201,13 @@ mod tests {}),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;+ .expect("Search failed");- assert_eq!(result.num_docs, 3);- assert_eq!(result.documents.len(), 3);- assert_eq!(result.documents[0].url, "https://www.first.com");- assert_eq!(result.documents[1].url, "https://www.second.com");- assert_eq!(result.documents[2].url, "https://www.third.com");+ assert_eq!(result.num_hits, 3);+ assert_eq!(result.webpages.len(), 3);+ assert_eq!(result.webpages[0].url, "https://www.first.com");+ assert_eq!(result.webpages[1].url, "https://www.second.com");+ assert_eq!(result.webpages[2].url, "https://www.third.com");let result = searcher.search(&SearchQuery {@@ -222,16 +219,13 @@ mod tests {}),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;+ .expect("Search failed");- assert_eq!(result.num_docs, 3);- assert_eq!(result.documents.len(), 3);- assert_eq!(result.documents[0].url, "https://www.first.com");- assert_eq!(result.documents[1].url, "https://www.second.com");- assert_eq!(result.documents[2].url, "https://www.third.com");+ assert_eq!(result.num_hits, 3);+ assert_eq!(result.webpages.len(), 3);+ assert_eq!(result.webpages[0].url, "https://www.first.com");+ assert_eq!(result.webpages[1].url, "https://www.second.com");+ assert_eq!(result.webpages[2].url, "https://www.third.com");let result = searcher.search(&SearchQuery {@@ -243,14 +237,11 @@ mod tests {}),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap()- .webpages;+ .expect("Search failed");- assert_eq!(result.num_docs, 2);- assert_eq!(result.documents.len(), 2);- assert_eq!(result.documents[0].url, "https://www.third.com");- assert_eq!(result.documents[1].url, "https://www.second.com");+ assert_eq!(result.num_hits, 2);+ assert_eq!(result.webpages.len(), 2);+ assert_eq!(result.webpages[0].url, "https://www.third.com");+ assert_eq!(result.webpages[1].url, "https://www.second.com");}}diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rsindex d8735e6b..df1e1931 100644--- a/core/src/search_prettifier.rs+++ b/core/src/search_prettifier.rs@@ -23,9 +23,8 @@ use serde::{Deserialize, Serialize};use crate::{entity_index::{entity::Span, StoredEntity},inverted_index::RetrievedWebpage,- ranking::pipeline::RankingWebsite,- searcher::{self, LocalSearcher},- spell::CorrectionTerm,+ searcher::LocalSearcher,+ spell::{self, CorrectionTerm},webpage::{schema_org::{self, Item, OneOrMany, Property},Url,@@ -65,10 +64,14 @@ pub enum CodeOrText {Text(String),}-pub fn initial(result: searcher::local::InitialWebsiteResult) -> InitialWebsiteResult {- let sidebar = result.entity_sidebar;+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct HighlightedSpellCorrection {+ pub raw: String,+ pub highlighted: String,+}- let spell_corrected_query = result.spell_corrected_query.map(|correction| {+impl From<spell::Correction> for HighlightedSpellCorrection {+ fn from(correction: spell::Correction) -> Self {let mut highlighted = String::new();let mut raw = String::new();@@ -91,31 +94,10 @@ pub fn initial(result: searcher::local::InitialWebsiteResult) -> InitialWebsiteRraw = raw.trim_end().to_string();highlighted = highlighted.trim_end().to_string();- HighlightedSpellCorrection { raw, highlighted }- });-- InitialWebsiteResult {- spell_corrected_query,- websites: result.websites,- num_websites: result.num_websites,- entity_sidebar: sidebar,+ Self { raw, highlighted }}}-#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct HighlightedSpellCorrection {- pub raw: String,- pub highlighted: String,-}--#[derive(Debug, Serialize, Deserialize)]-pub struct InitialWebsiteResult {- pub spell_corrected_query: Option<HighlightedSpellCorrection>,- pub num_websites: usize,- pub websites: Vec<RankingWebsite>,- pub entity_sidebar: Option<DisplayedEntity>,-}-pub fn html_escape(s: &str) -> String {html_escape::decode_html_entities(s).chars()diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex eda6904f..6e61502a 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -15,14 +15,18 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::{+ bangs::{BangHit, Bangs},collector::{self, BucketCollector},exponential_backoff::ExponentialBackoff,inverted_index::{self, RetrievedWebpage},+ query,ranking::{models::cross_encoder::CrossEncoderModel,pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},},- search_prettifier::{create_stackoverflow_sidebar, DisplayedWebpage, Sidebar},+ search_prettifier::{+ create_stackoverflow_sidebar, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,+ },searcher::PrettifiedWebsitesResult,};@@ -40,7 +44,7 @@ use thiserror::Error;use crate::sonic;-use super::{InitialPrettifiedSearchResult, PrettifiedSearchResult, SearchQuery};+use super::{InitialWebsiteResult, SearchQuery, SearchResult};type Result<T> = std::result::Result<T, Error>;const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 100.0;@@ -62,16 +66,15 @@ pub enum Error {}impl RemoteSearcher {- async fn search(&self, query: &SearchQuery) -> Result<InitialPrettifiedSearchResult> {+ async fn search(&self, query: &SearchQuery) -> Result<InitialWebsiteResult> {for timeout in ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200)).take(5){if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await{- if let Ok(sonic::Response::Content(body)) = connection- .send(Request::SearchPrettified(query.clone()))- .await+ if let Ok(sonic::Response::Content(body)) =+ connection.send(Request::Search(query.clone())).await{return Ok(body);}@@ -109,7 +112,7 @@ impl RemoteSearcher {}#[derive(Clone, PartialEq, Eq)]-struct ShardId(u32);+struct ShardId(u64);pub struct Shard {id: ShardId,@@ -117,7 +120,7 @@ pub struct Shard {}impl Shard {- pub fn new(id: u32, replicas: Vec<String>) -> Self {+ pub fn new(id: u64, replicas: Vec<String>) -> Self {let mut parsed_replicas = Vec::new();for replica in replicas {@@ -169,14 +172,13 @@ impl Shard {}struct InitialSearchResultShard {- local_result: InitialPrettifiedSearchResult,+ local_result: InitialWebsiteResult,shard: ShardId,}#[derive(Serialize, Deserialize)]pub enum Request {Search(SearchQuery),- SearchPrettified(SearchQuery),RetrieveWebsites {websites: Vec<inverted_index::WebsitePointer>,query: String,@@ -186,6 +188,7 @@ pub enum Request {pub struct DistributedSearcher {shards: Vec<Shard>,cross_encoder: Arc<CrossEncoderModel>,+ bangs: Bangs,}#[derive(Clone)]@@ -219,10 +222,11 @@ impl collector::Doc for ScoredWebsitePointer {}impl DistributedSearcher {- pub fn new(shards: Vec<Shard>, model: CrossEncoderModel) -> Self {+ pub fn new(shards: Vec<Shard>, model: CrossEncoderModel, bangs: Bangs) -> Self {Self {shards,cross_encoder: Arc::new(model),+ bangs,}}@@ -238,15 +242,13 @@ impl DistributedSearcher {BucketCollector::new(pipeline.collector_top_n() + pipeline.collector_offset());for result in initial_results {- if let InitialPrettifiedSearchResult::Websites(local_result) = result.local_result {- for website in local_result.websites {- let pointer = ScoredWebsitePointer {- website,- shard: result.shard.clone(),- };-- collector.insert(pointer);- }+ for website in result.local_result.websites {+ let pointer = ScoredWebsitePointer {+ website,+ shard: result.shard.clone(),+ };++ collector.insert(pointer);}}@@ -317,15 +319,13 @@ impl DistributedSearcher {.search_initial(&query).await.into_iter()- .filter_map(|result| match result.local_result {- InitialPrettifiedSearchResult::Websites(websites) => {- if let Some(website) = websites.websites.first().cloned() {- Some((result.shard, website))- } else {- None- }- }- InitialPrettifiedSearchResult::Bang(_) => None,+ .filter_map(|result| {+ result+ .local_result+ .websites+ .first()+ .cloned()+ .map(|website| (result.shard, website))}).collect();@@ -350,13 +350,9 @@ impl DistributedSearcher {initial_results: &[InitialSearchResultShard],query: &SearchQuery,) -> Result<Option<Sidebar>> {- let entity = initial_results.first().and_then(|result| {- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.entity_sidebar.clone()- } else {- None- }- });+ let entity = initial_results+ .first()+ .and_then(|result| result.local_result.entity_sidebar.clone());match entity {Some(entity) => Ok(Some(Sidebar::Entity(entity))),@@ -364,7 +360,17 @@ impl DistributedSearcher {}}- pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {+ fn check_bangs(&self, query: &SearchQuery) -> Option<BangHit> {+ let parsed_terms = query::parser::parse(&query.query);++ self.bangs.get(&parsed_terms)+ }++ pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {+ if let Some(bang) = self.check_bangs(query) {+ return Ok(SearchResult::Bang(bang));+ }+let start = Instant::now();if query.is_empty() {@@ -375,35 +381,16 @@ impl DistributedSearcher {let initial_results = self.search_initial(query).await;- // check if any result has a bang hit- if let Some(result) = initial_results- .iter()- .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_)))- {- if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {- return Ok(PrettifiedSearchResult::Bang(bang.clone()));- }- }-let sidebar = self.sidebar(&initial_results, query).await?;- let spell_corrected_query = initial_results.first().and_then(|result| {- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.spell_corrected_query.clone()- } else {- None- }- });+ let spell_corrected_query = initial_results+ .first()+ .and_then(|result| result.local_result.spell_corrected_query.clone())+ .map(HighlightedSpellCorrection::from);let num_docs = initial_results.iter()- .map(|result| {- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.num_websites- } else {- 0- }- })+ .map(|result| result.local_result.num_websites).sum();let top_websites = self.combine_results(initial_results, &mut search_query)?;@@ -420,9 +407,9 @@ impl DistributedSearcher {return Err(Error::SearchFailed);}- Ok(PrettifiedSearchResult::Websites(PrettifiedWebsitesResult {+ Ok(SearchResult::Websites(PrettifiedWebsitesResult {spell_corrected_query,- num_docs,+ num_hits: num_docs,webpages: retrieved_webpages,sidebar,search_duration_ms: start.elapsed().as_millis(),diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex d16fda5f..92b5ba28 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -17,10 +17,8 @@use std::str::FromStr;use optics::Optic;-use serde::{Deserialize, Serialize};use uuid::Uuid;-use crate::bangs::{BangHit, Bangs};use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;@@ -29,20 +27,19 @@ use crate::ranking::centrality_store::CentralityStore;use crate::ranking::optics::CreateAggregator;use crate::ranking::pipeline::RankingWebsite;use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};-use crate::search_prettifier::{DisplayedEntity, Sidebar};+use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};use crate::spell::Correction;use crate::webgraph::centrality::topic::TopicCentrality;use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};-use super::WebsitesResult;-use super::{InitialSearchResult, SearchQuery, SearchResult};+use super::PrettifiedWebsitesResult;+use super::{InitialWebsiteResult, SearchQuery};pub struct LocalSearcher {index: Index,entity_index: Option<EntityIndex>,- bangs: Option<Bangs>,centrality_store: Option<CentralityStore>,topic_centrality: Option<TopicCentrality>,}@@ -58,7 +55,6 @@ impl LocalSearcher {LocalSearcher {index,entity_index: None,- bangs: None,centrality_store: None,topic_centrality: None,}@@ -68,10 +64,6 @@ impl LocalSearcher {self.entity_index = Some(entity_index);}- pub fn set_bangs(&mut self, bangs: Bangs) {- self.bangs = Some(bangs);- }-pub fn set_centrality_store(&mut self, centrality_store: CentralityStore) {self.centrality_store = Some(centrality_store);}@@ -215,16 +207,6 @@ impl LocalSearcher {Ok(self.index.spell_correction(&parsed_query.simple_terms()))}- fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {- let parsed_query = self.parse_query(query, None)?;-- if let Some(bangs) = self.bangs.as_ref() {- return Ok(bangs.get(&parsed_query));- }-- Ok(None)- }-fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {self.entity_index.as_ref()@@ -235,23 +217,19 @@ impl LocalSearcher {&self,query: &SearchQuery,de_rank_similar: bool,- ) -> Result<InitialSearchResult> {- if let Some(bang) = self.check_bangs(query)? {- return Ok(InitialSearchResult::Bang(bang));- }-+ ) -> Result<InitialWebsiteResult> {let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;let correction = self.spell_correction(query)?;let sidebar = self.entity_sidebar(query).map(|entity| DisplayedEntity::from(entity, self));- Ok(InitialSearchResult::Websites(InitialWebsiteResult {+ Ok(InitialWebsiteResult {spell_corrected_query: correction,websites,num_websites,entity_sidebar: sidebar,- }))+ })}pub fn retrieve_websites(@@ -273,12 +251,15 @@ impl LocalSearcher {self.index.retrieve_websites(websites, &query)}- pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {+ pub fn search(&self, query: &SearchQuery) -> Result<PrettifiedWebsitesResult> {use std::{sync::Arc, time::Instant};- use crate::ranking::{- models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},- pipeline::RankingPipeline,+ use crate::{+ ranking::{+ models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},+ pipeline::RankingPipeline,+ },+ search_prettifier::Sidebar,};let start = Instant::now();@@ -291,29 +272,28 @@ impl LocalSearcher {}};- let initial_result = self.search_initial(&search_query, true)?;-- match initial_result {- InitialSearchResult::Websites(search_result) => {- let top_websites = pipeline.apply(search_result.websites);- let pointers: Vec<_> = top_websites- .into_iter()- .map(|website| website.pointer)- .collect();-- let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;- Ok(SearchResult::Websites(WebsitesResult {- spell_corrected_query: search_result.spell_corrected_query,- webpages: inverted_index::SearchResult {- num_docs: search_result.num_websites,- documents: retrieved_sites,- },- sidebar: search_result.entity_sidebar.map(Sidebar::Entity),- search_duration_ms: start.elapsed().as_millis(),- }))- }- InitialSearchResult::Bang(bang) => Ok(SearchResult::Bang(bang)),- }+ let search_result = self.search_initial(&search_query, true)?;++ let top_websites = pipeline.apply(search_result.websites);+ let pointers: Vec<_> = top_websites+ .into_iter()+ .map(|website| website.pointer)+ .collect();++ let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;++ Ok(PrettifiedWebsitesResult {+ spell_corrected_query: search_result+ .spell_corrected_query+ .map(HighlightedSpellCorrection::from),+ num_hits: search_result.num_websites,+ webpages: retrieved_sites+ .into_iter()+ .map(DisplayedWebpage::from)+ .collect(),+ sidebar: search_result.entity_sidebar.map(Sidebar::Entity),+ search_duration_ms: start.elapsed().as_millis(),+ })}pub fn favicon(&self, site: &Url) -> Option<Image> {@@ -340,25 +320,6 @@ impl LocalSearcher {}}-#[derive(Debug, Serialize, Deserialize)]-pub struct InitialWebsiteResult {- pub spell_corrected_query: Option<Correction>,- pub num_websites: usize,- pub websites: Vec<RankingWebsite>,- pub entity_sidebar: Option<DisplayedEntity>,-}--impl SearchResult {- #[cfg(test)]- pub fn into_websites(self) -> Option<WebsitesResult> {- if let SearchResult::Websites(res) = self {- Some(res)- } else {- None- }- }-}-#[cfg(test)]mod tests {use crate::{@@ -417,10 +378,7 @@ mod tests {..Default::default()}).unwrap()- .into_websites()- .unwrap().webpages- .documents.into_iter().map(|page| page.url).collect();diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 384d5d58..4b3dfb2a 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -24,30 +24,16 @@ use serde::{Deserialize, Serialize};use crate::{bangs::BangHit,- inverted_index,- search_prettifier::{self, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},+ ranking::pipeline::RankingWebsite,+ search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},spell::Correction,webpage::region::Region,};pub const NUM_RESULTS_PER_PAGE: usize = 20;-#[derive(Debug, Serialize)]-pub struct WebsitesResult {- pub spell_corrected_query: Option<Correction>,- pub webpages: inverted_index::SearchResult,- pub sidebar: Option<Sidebar>,- pub search_duration_ms: u128,-}-#[derive(Debug, Serialize)]pub enum SearchResult {- Websites(WebsitesResult),- Bang(BangHit),-}--#[derive(Debug, Serialize)]-pub enum PrettifiedSearchResult {Websites(PrettifiedWebsitesResult),Bang(BangHit),}@@ -56,23 +42,11 @@ pub enum PrettifiedSearchResult {pub struct PrettifiedWebsitesResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,- pub num_docs: usize,+ pub num_hits: usize,pub sidebar: Option<Sidebar>,pub search_duration_ms: u128,}-#[derive(Debug, Serialize, Deserialize)]-pub enum InitialSearchResult {- Websites(local::InitialWebsiteResult),- Bang(BangHit),-}--#[derive(Debug, Serialize, Deserialize)]-pub enum InitialPrettifiedSearchResult {- Websites(search_prettifier::InitialWebsiteResult),- Bang(BangHit),-}-#[derive(Debug, Serialize, Deserialize, Clone)]pub struct SearchQuery {pub query: String,@@ -83,6 +57,14 @@ pub struct SearchQuery {pub site_rankings: Option<SiteRankings>,}+#[derive(Debug, Serialize, Deserialize)]+pub struct InitialWebsiteResult {+ pub spell_corrected_query: Option<Correction>,+ pub num_websites: usize,+ pub websites: Vec<RankingWebsite>,+ pub entity_sidebar: Option<DisplayedEntity>,+}+impl Default for SearchQuery {fn default() -> Self {Self {diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex 81863f34..2b2d2587 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -342,6 +342,7 @@ mod tests {use super::*;use crate::{index::Index,+ search_prettifier::Snippet,searcher::{LocalSearcher, SearchQuery},webpage::Webpage,};@@ -360,6 +361,13 @@ to the project are from community members.[15]Rust won first place for "most loved programming language" in the Stack Overflow DeveloperSurvey in 2016, 2017, and 2018."#;+ fn snippet_text(snippet: Snippet) -> String {+ match snippet {+ Snippet::Normal { date: _, text } => text,+ _ => panic!("The snippet was not text"),+ }+ }+#[test]fn snippet_during_search() {let mut index = Index::temporary().expect("Unable to open index");@@ -390,13 +398,11 @@ Survey in 2016, 2017, and 2018."#;query: "rust language".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();+ .expect("Search failed");- assert_eq!(result.webpages.num_docs, 1);- assert_eq!(result.webpages.documents.len(), 1);- assert_eq!(result.webpages.documents[0].snippet, format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its..."));+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(snippet_text(result.webpages[0].snippet.clone()), format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its..."));}#[test]@@ -429,13 +435,11 @@ Survey in 2016, 2017, and 2018."#;query: "describe".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();+ .expect("Search failed");- assert_eq!(result.webpages.num_docs, 1);- assert_eq!(result.webpages.documents.len(), 1);- assert_eq!(result.webpages.documents[0].snippet, format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its..."));+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);+ assert_eq!(snippet_text(result.webpages[0].snippet.clone()), format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its..."));}#[test]@@ -638,14 +642,12 @@ Survey in 2016, 2017, and 2018."#;query: "paradigms".to_string(),..Default::default()})- .expect("Search failed")- .into_websites()- .unwrap();+ .expect("Search failed");- assert_eq!(result.webpages.num_docs, 1);- assert_eq!(result.webpages.documents.len(), 1);+ assert_eq!(result.num_hits, 1);+ assert_eq!(result.webpages.len(), 1);assert_eq!(- result.webpages.documents[0].snippet,+ snippet_text(result.webpages[0].snippet.clone()),format!("Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural {HIGHLIGHTEN_PREFIX}paradigms{HIGHLIGHTEN_POSTFIX}. Rust is syntactically similar to C++[according to whom?], but its..."));}