Skip to content

web search done right

Cleanup search result types

Mikkel Denker9986d05629de
diff --git a/configs/frontend.toml b/configs/frontend.toml
index 3abc2979..e17cda94 100644
--- a/configs/frontend.toml
+++ b/configs/frontend.toml
@@ -1,4 +1,5 @@
queries_csv_path = "data/queries_us.csv"
host = "0.0.0.0:3000"
search_servers = [["0.0.0.0:3001"]]
-crossencoder_model_path = "data/cross_encoder"
\ No newline at end of file
+crossencoder_model_path = "data/cross_encoder"
+bangs_path = "data/bangs.json"
\ No newline at end of file
diff --git a/configs/search_server.toml b/configs/search_server.toml
index c35d3057..d2692b76 100644
--- a/configs/search_server.toml
+++ b/configs/search_server.toml
@@ -1,5 +1,4 @@
host = "0.0.0.0:3001"
index_path = "data/index"
entity_index_path = "data/entity"
-bangs_path = "data/bangs.json"
centrality_store_path = "data/centrality"
\ No newline at end of file
diff --git a/core/src/bangs.rs b/core/src/bangs.rs
index 18a41903..05175068 100644
--- a/core/src/bangs.rs
+++ b/core/src/bangs.rs
@@ -19,10 +19,7 @@ use std::{collections::HashMap, fs, path::Path};
use itertools::intersperse;
use serde::{Deserialize, Serialize};
-use crate::{
- query::{parser::Term, Query},
- webpage::Url,
-};
+use crate::{query::parser::Term, webpage::Url};
pub const BANG_PREFIX: char = '!';
@@ -78,8 +75,8 @@ impl Bangs {
}
}
- pub fn get(&self, query: &Query) -> Option<BangHit> {
- for possible_bang in query.terms().iter().filter_map(|term| {
+ pub fn get(&self, terms: &[Box<Term>]) -> Option<BangHit> {
+ for possible_bang in terms.iter().filter_map(|term| {
if let Term::PossibleBang(possible_bang) = term.as_ref() {
Some(possible_bang)
} else {
@@ -94,8 +91,7 @@ impl Bangs {
.replace(
"{{{s}}}",
intersperse(
- query
- .terms()
+ terms
.iter()
.filter(|term| {
if let Term::PossibleBang(bang) = term.as_ref() {
@@ -121,19 +117,13 @@ impl Bangs {
#[cfg(test)]
mod tests {
- use std::sync::Arc;
- use tantivy::tokenizer::TokenizerManager;
-
- use crate::{ranking::SignalAggregator, schema::create_schema};
+ use crate::query::parser::parse;
use super::*;
#[test]
fn simple_bang() {
- let schema = Arc::new(create_schema());
- let tokenizer_manager = TokenizerManager::new();
-
let bangs = Bangs::from_json(
r#"[{
"c": "Multimedia",
@@ -146,33 +136,11 @@ mod tests {
}]"#,
);
- let query = Query::parse(
- "no bangs",
- schema.clone(),
- &tokenizer_manager,
- &SignalAggregator::default(),
- )
- .unwrap();
- assert_eq!(bangs.get(&query), None);
-
- let query = Query::parse(
- "!no bangs",
- schema.clone(),
- &tokenizer_manager,
- &SignalAggregator::default(),
- )
- .unwrap();
- assert_eq!(bangs.get(&query), None);
-
- let query = Query::parse(
- "!ty bangs",
- schema,
- &tokenizer_manager,
- &SignalAggregator::default(),
- )
- .unwrap();
+ assert_eq!(bangs.get(&parse("no bangs")), None);
+ assert_eq!(bangs.get(&parse("!no bangs")), None);
+
assert_eq!(
- bangs.get(&query),
+ bangs.get(&parse("!ty bangs")),
Some(BangHit {
bang: Bang {
category: Some("Multimedia".to_string()),
diff --git a/core/src/entrypoint/frontend.rs b/core/src/entrypoint/frontend.rs
index 64f633dd..e3f0003f 100644
--- a/core/src/entrypoint/frontend.rs
+++ b/core/src/entrypoint/frontend.rs
@@ -16,16 +16,16 @@
use anyhow::Result;
-use crate::frontend::router;
+use crate::{frontend::router, FrontendConfig};
-pub async fn run(
- queries_csv_path: &str,
- crossencoder_model_path: &str,
- host: &str,
- shards: Vec<Vec<String>>,
-) -> Result<()> {
- let app = router(queries_csv_path, crossencoder_model_path, shards)?;
- let addr = host.parse()?;
+pub async fn run(config: FrontendConfig) -> Result<()> {
+ let app = router(
+ &config.queries_csv_path,
+ &config.crossencoder_model_path,
+ &config.bangs_path,
+ config.search_servers,
+ )?;
+ let addr = config.host.parse()?;
tracing::info!("listening on {}", addr);
axum::Server::bind(&addr)
.serve(app.into_make_service())
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 7bd9768b..4f0939ad 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -17,12 +17,10 @@
use std::net::SocketAddr;
use crate::{
- bangs::Bangs,
entity_index::EntityIndex,
index::Index,
inverted_index,
ranking::centrality_store::CentralityStore,
- search_prettifier::{self},
searcher::{self, LocalSearcher},
sonic, Result, SearchServerConfig,
};
@@ -35,7 +33,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
let entity_index = config
.entity_index_path
.map(|path| EntityIndex::open(path).unwrap());
- let bangs = config.bangs_path.map(Bangs::from_path);
let centrality_store = config.centrality_store_path.map(CentralityStore::open);
let search_index = Index::open(config.index_path)?;
@@ -45,10 +42,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
local_searcher.set_entity_index(entity_index);
}
- if let Some(bangs) = bangs {
- local_searcher.set_bangs(bangs);
- }
-
if let Some(centrality_store) = centrality_store {
local_searcher.set_centrality_store(centrality_store);
}
@@ -56,18 +49,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
loop {
if let Ok(req) = server.accept::<searcher::distributed::Request>().await {
match &req.body {
- searcher::Request::Search(query) => {
- match local_searcher.search_initial(query, false) {
- Ok(response) => {
- req.respond(sonic::Response::Content(response)).await.ok();
- }
- Err(_) => {
- req.respond::<searcher::InitialSearchResult>(sonic::Response::Empty)
- .await
- .ok();
- }
- }
- }
searcher::Request::RetrieveWebsites { websites, query } => {
match local_searcher.retrieve_websites(websites, query) {
Ok(response) => {
@@ -82,26 +63,11 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
}
}
}
- searcher::Request::SearchPrettified(query) => {
+ searcher::Request::Search(query) => {
match local_searcher.search_initial(query, false) {
- Ok(result) => match result {
- searcher::InitialSearchResult::Websites(result) => {
- let res = search_prettifier::initial(result);
-
- req.respond(sonic::Response::Content(
- searcher::InitialPrettifiedSearchResult::Websites(res),
- ))
- .await
- .ok();
- }
- searcher::InitialSearchResult::Bang(bang) => {
- req.respond(sonic::Response::Content(
- searcher::InitialSearchResult::Bang(bang),
- ))
- .await
- .ok();
- }
- },
+ Ok(result) => {
+ req.respond(sonic::Response::Content(result)).await.ok();
+ }
Err(_) => {
req.respond::<inverted_index::SearchResult>(sonic::Response::Empty)
.await
diff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rs
index 66f9891d..4cb5f44d 100644
--- a/core/src/frontend/mod.rs
+++ b/core/src/frontend/mod.rs
@@ -19,6 +19,7 @@ use tower_http::{compression::CompressionLayer, services::ServeDir};
use crate::{
autosuggest::Autosuggest,
+ bangs::Bangs,
ranking::models::cross_encoder::CrossEncoderModel,
searcher::{DistributedSearcher, Shard},
};
@@ -78,17 +79,19 @@ pub async fn favicon() -> impl IntoResponse {
pub fn router(
queries_csv_path: &str,
crossencoder_model_path: &str,
+ bangs_path: &str,
shards: Vec<Vec<String>>,
) -> Result<Router> {
let shards: Vec<_> = shards
.into_iter()
.enumerate()
- .map(|(id, replicas)| Shard::new(id as u32, replicas))
+ .map(|(id, replicas)| Shard::new(id as u64, replicas))
.collect();
let autosuggest = Autosuggest::load_csv(queries_csv_path)?;
let crossencoder = CrossEncoderModel::open(crossencoder_model_path)?;
- let searcher = DistributedSearcher::new(shards, crossencoder);
+ let bangs = Bangs::from_path(bangs_path);
+ let searcher = DistributedSearcher::new(shards, crossencoder, bangs);
let state = Arc::new(State {
searcher,
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rs
index 0adc6c2f..d8a11550 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/frontend/search.rs
@@ -25,7 +25,7 @@ use crate::{
thousand_sep_number, CodeOrText, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,
Snippet,
},
- searcher::{self, PrettifiedSearchResult, SearchQuery, NUM_RESULTS_PER_PAGE},
+ searcher::{self, SearchQuery, SearchResult, NUM_RESULTS_PER_PAGE},
webpage::region::{Region, ALL_REGIONS},
};
@@ -131,11 +131,11 @@ pub async fn route(
.await
{
Ok(result) => match result {
- PrettifiedSearchResult::Websites(result) => {
+ SearchResult::Websites(result) => {
let sidebar = result.sidebar;
let spell_correction = result.spell_corrected_query;
- let num_matches = thousand_sep_number(result.num_docs);
+ let num_matches = thousand_sep_number(result.num_hits);
let search_duration_sec =
format!("{:.2}", result.search_duration_ms as f64 / 1000.0);
@@ -196,9 +196,7 @@ pub async fn route(
HtmlTemplate(template).into_response()
}
- PrettifiedSearchResult::Bang(result) => {
- Redirect::to(&result.redirect_to.full()).into_response()
- }
+ SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),
},
Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),
Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
@@ -216,10 +214,8 @@ pub async fn api(
match state.searcher.search(&query).await {
Ok(result) => match result {
- PrettifiedSearchResult::Websites(result) => Json(result).into_response(),
- PrettifiedSearchResult::Bang(result) => {
- Redirect::to(&result.redirect_to.full()).into_response()
- }
+ SearchResult::Websites(result) => Json(result).into_response(),
+ SearchResult::Bang(result) => Redirect::to(&result.redirect_to.full()).into_response(),
},
Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),
Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
diff --git a/core/src/index.rs b/core/src/index.rs
index f0a09711..dc4e272e 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -402,14 +402,11 @@ mod tests {
query: "website".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
-
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
+ .expect("Search failed");
+
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.example.com");
}
#[test]
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index c58c4af1..fbda560b 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -206,7 +206,7 @@ impl InvertedIndex {
for pointer in pointers {
let doc = searcher.doc(pointer.address.into())?;
- top_websites.push(RankingWebsite::new(doc, pointer, aggregator))
+ top_websites.push(RankingWebsite::new(doc, pointer, aggregator));
}
Ok(top_websites)
@@ -237,7 +237,7 @@ impl InvertedIndex {
let mut webpages: Vec<RetrievedWebpage> = websites
.iter()
.map(|website| self.retrieve_doc(website.address, &searcher))
- .filter_map(|page| page.ok())
+ .filter_map(|res| res.ok())
.map(|mut doc| {
if let Some(image) = doc.primary_image.as_ref() {
if !query.simple_terms().into_iter().all(|term| {
@@ -438,14 +438,14 @@ impl From<Document> for RetrievedWebpage {
.value
.as_text()
.expect("Title field should be text")
- .to_string()
+ .to_string();
}
Field::Text(TextField::StemmedCleanBody) => {
webpage.body = value
.value
.as_text()
.expect("Body field should be text")
- .to_string()
+ .to_string();
}
Field::Text(TextField::Description) => {
let desc = value
@@ -461,7 +461,7 @@ impl From<Document> for RetrievedWebpage {
.value
.as_text()
.expect("Url field should be text")
- .to_string()
+ .to_string();
}
Field::Text(TextField::PrimaryImage) => {
webpage.primary_image = {
@@ -488,7 +488,7 @@ impl From<Document> for RetrievedWebpage {
.value
.as_text()
.expect("All body field should be text")
- .to_string()
+ .to_string();
}
Field::Fast(FastField::Region) => {
webpage.region = {
@@ -500,7 +500,7 @@ impl From<Document> for RetrievedWebpage {
let facet = value.value.as_facet().unwrap();
if !facet.is_root() {
- webpage.host_topic = Some(facet.clone().into())
+ webpage.host_topic = Some(facet.clone().into());
}
}
Field::Text(TextField::DmozDescription) => {
diff --git a/core/src/lib.rs b/core/src/lib.rs
index bf9f48a4..5349b34f 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -152,6 +152,7 @@ pub struct FrontendConfig {
pub queries_csv_path: String,
pub host: String,
pub crossencoder_model_path: String,
+ pub bangs_path: String,
pub search_servers: Vec<Vec<String>>,
}
@@ -159,7 +160,6 @@ pub struct FrontendConfig {
pub struct SearchServerConfig {
pub index_path: String,
pub entity_index_path: Option<String>,
- pub bangs_path: Option<String>,
pub centrality_store_path: Option<String>,
pub host: String,
}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 42daf788..ec661909 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -352,15 +352,10 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.second.com");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.second.com");
}
#[test]
@@ -404,43 +399,28 @@ mod tests {
query: "test site:first.com".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.first.com");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
let query = SearchQuery {
query: "test site:www.first.com".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.first.com");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
let query = SearchQuery {
query: "test -site:first.com".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.second.com");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.second.com");
}
#[test]
@@ -484,15 +464,10 @@ mod tests {
query: "intitle:website".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.first.com");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
}
#[test]
@@ -536,15 +511,10 @@ mod tests {
query: "test inurl:forum".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.first.com/forum");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.first.com/forum");
}
#[test]
@@ -618,41 +588,26 @@ mod tests {
query: "test site:first.com".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 0);
- assert_eq!(result.documents.len(), 0);
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 0);
+ assert_eq!(result.webpages.len(), 0);
let query = SearchQuery {
query: "test site:the-first.com".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.the-first.com");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.the-first.com");
let query = SearchQuery {
query: "test site:www.the-first.com".to_string(),
..Default::default()
};
- let result = searcher
- .search(&query)
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
- assert_eq!(result.num_docs, 1);
- assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.the-first.com");
+ let result = searcher.search(&query).expect("Search failed");
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(result.webpages[0].url, "https://www.the-first.com");
}
}
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 9c89757c..e0b05eae 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -287,10 +287,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 2);
assert_eq!(res[0].url, "https://www.b.com");
@@ -313,10 +310,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 1);
assert_eq!(res[0].url, "https://www.a.com");
@@ -338,10 +332,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 2);
assert_eq!(res[0].url, "https://www.a.com");
@@ -424,10 +415,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
let _ = searcher
.search(&SearchQuery {
@@ -438,10 +426,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
let _ = searcher
.search(&SearchQuery {
@@ -452,10 +437,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
}
#[test]
@@ -581,10 +563,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 2);
assert_eq!(res[0].url, "https://www.a.com/this/is/a/pattern");
@@ -741,10 +720,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 3);
assert_eq!(res[0].url, "https://www.b.com/this/is/b/pattern");
@@ -865,10 +841,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 1);
assert_eq!(res[0].url, "https://www.b.com/");
@@ -890,10 +863,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 1);
assert_eq!(res[0].url, "https://www.b.com/");
@@ -915,10 +885,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 1);
assert_eq!(res[0].url, "https://www.a.com/");
@@ -940,10 +907,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 1);
assert_eq!(res[0].url, "https://www.b.com/");
@@ -1018,10 +982,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
- .webpages
- .documents;
+ .webpages;
assert_eq!(res.len(), 0);
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index a52b2aa1..a4ce8595 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -256,12 +256,10 @@ mod tests {
query: "example".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
- assert_eq!(result.webpages.documents.len(), 2);
- assert_eq!(result.webpages.documents[0].url, "https://www.b.com");
- assert_eq!(result.webpages.documents[1].url, "https://www.a.com");
+ .expect("Search failed");
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.b.com");
+ assert_eq!(result.webpages[1].url, "https://www.a.com");
}
#[test]
@@ -337,12 +335,10 @@ mod tests {
query: "example".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
- assert_eq!(result.webpages.documents.len(), 2);
- assert_eq!(result.webpages.documents[0].url, "https://www.b.com");
- assert_eq!(result.webpages.documents[1].url, "https://www.a.com");
+ .expect("Search failed");
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.b.com");
+ assert_eq!(result.webpages[1].url, "https://www.a.com");
}
#[test]
@@ -445,12 +441,10 @@ mod tests {
query: "dr dk".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
+ .expect("Search failed");
- assert_eq!(result.webpages.documents.len(), 3);
- assert_eq!(result.webpages.documents[0].url, "https://www.dr.dk");
+ assert_eq!(result.webpages.len(), 3);
+ assert_eq!(result.webpages[0].url, "https://www.dr.dk");
}
#[test]
@@ -525,11 +519,9 @@ mod tests {
query: "title".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
+ .expect("Search failed");
- assert_eq!(result.webpages.documents[0].url, "https://www.new.com");
+ assert_eq!(result.webpages[0].url, "https://www.new.com");
}
#[test]
@@ -610,12 +602,10 @@ mod tests {
query: "test".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
+ .expect("Search failed");
- assert_eq!(result.webpages.documents.len(), 2);
- assert_eq!(result.webpages.documents[0].url, "https://www.first.com");
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
}
#[test]
@@ -688,12 +678,10 @@ mod tests {
query: "test".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
+ .expect("Search failed");
- assert_eq!(result.webpages.documents.len(), 2);
- assert_eq!(result.webpages.documents[0].url, "https://www.first.com");
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
}
#[test]
@@ -805,12 +793,10 @@ mod tests {
),
..Default::default()
})
- .unwrap()
- .into_websites()
.unwrap();
- assert_eq!(res.webpages.num_docs, 3);
- assert_eq!(&res.webpages.documents[0].url, "https://www.title.com");
+ assert_eq!(res.num_hits, 3);
+ assert_eq!(&res.webpages[0].url, "https://www.title.com");
let res = searcher
.search(&SearchQuery {
@@ -827,12 +813,10 @@ mod tests {
),
..Default::default()
})
- .unwrap()
- .into_websites()
.unwrap();
- assert_eq!(res.webpages.num_docs, 3);
- assert_eq!(&res.webpages.documents[0].url, "https://www.centrality.com");
+ assert_eq!(res.num_hits, 3);
+ assert_eq!(&res.webpages[0].url, "https://www.centrality.com");
}
#[test]
@@ -936,16 +920,13 @@ mod tests {
query: "termA termB".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
-
- assert_eq!(result.num_docs, 3);
- assert_eq!(result.documents.len(), 3);
- assert_eq!(result.documents[0].url, "https://www.first.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
- assert_eq!(result.documents[2].url, "https://www.third.com");
+ .expect("Search failed");
+
+ assert_eq!(result.num_hits, 3);
+ assert_eq!(result.webpages.len(), 3);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[1].url, "https://www.second.com");
+ assert_eq!(result.webpages[2].url, "https://www.third.com");
}
#[test]
@@ -1020,15 +1001,12 @@ mod tests {
query: "test".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
-
- assert_eq!(result.num_docs, 2);
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.first.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
+ .expect("Search failed");
+
+ assert_eq!(result.num_hits, 2);
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[1].url, "https://www.second.com");
}
#[test]
@@ -1103,15 +1081,12 @@ mod tests {
query: "test".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
-
- assert_eq!(result.num_docs, 2);
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.first.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
+ .expect("Search failed");
+
+ assert_eq!(result.num_hits, 2);
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[1].url, "https://www.second.com");
}
#[test]
@@ -1271,29 +1246,23 @@ mod tests {
query: "topic_a".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
+ .expect("Search failed");
- assert_eq!(result.num_docs, 2);
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.a.com");
- assert_eq!(result.documents[1].url, "https://www.wrong.com");
+ assert_eq!(result.num_hits, 2);
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.a.com");
+ assert_eq!(result.webpages[1].url, "https://www.wrong.com");
let result = searcher
.search(&SearchQuery {
query: "topic_b".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
-
- assert_eq!(result.num_docs, 2);
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.b.com");
- assert_eq!(result.documents[1].url, "https://www.wrong.com");
+ .expect("Search failed");
+
+ assert_eq!(result.num_hits, 2);
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.b.com");
+ assert_eq!(result.webpages[1].url, "https://www.wrong.com");
}
}
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 665c7230..8e65982b 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -201,16 +201,13 @@ mod tests {
}),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
+ .expect("Search failed");
- assert_eq!(result.num_docs, 3);
- assert_eq!(result.documents.len(), 3);
- assert_eq!(result.documents[0].url, "https://www.first.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
- assert_eq!(result.documents[2].url, "https://www.third.com");
+ assert_eq!(result.num_hits, 3);
+ assert_eq!(result.webpages.len(), 3);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[1].url, "https://www.second.com");
+ assert_eq!(result.webpages[2].url, "https://www.third.com");
let result = searcher
.search(&SearchQuery {
@@ -222,16 +219,13 @@ mod tests {
}),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
+ .expect("Search failed");
- assert_eq!(result.num_docs, 3);
- assert_eq!(result.documents.len(), 3);
- assert_eq!(result.documents[0].url, "https://www.first.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
- assert_eq!(result.documents[2].url, "https://www.third.com");
+ assert_eq!(result.num_hits, 3);
+ assert_eq!(result.webpages.len(), 3);
+ assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[1].url, "https://www.second.com");
+ assert_eq!(result.webpages[2].url, "https://www.third.com");
let result = searcher
.search(&SearchQuery {
@@ -243,14 +237,11 @@ mod tests {
}),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap()
- .webpages;
+ .expect("Search failed");
- assert_eq!(result.num_docs, 2);
- assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.third.com");
- assert_eq!(result.documents[1].url, "https://www.second.com");
+ assert_eq!(result.num_hits, 2);
+ assert_eq!(result.webpages.len(), 2);
+ assert_eq!(result.webpages[0].url, "https://www.third.com");
+ assert_eq!(result.webpages[1].url, "https://www.second.com");
}
}
diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rs
index d8735e6b..df1e1931 100644
--- a/core/src/search_prettifier.rs
+++ b/core/src/search_prettifier.rs
@@ -23,9 +23,8 @@ use serde::{Deserialize, Serialize};
use crate::{
entity_index::{entity::Span, StoredEntity},
inverted_index::RetrievedWebpage,
- ranking::pipeline::RankingWebsite,
- searcher::{self, LocalSearcher},
- spell::CorrectionTerm,
+ searcher::LocalSearcher,
+ spell::{self, CorrectionTerm},
webpage::{
schema_org::{self, Item, OneOrMany, Property},
Url,
@@ -65,10 +64,14 @@ pub enum CodeOrText {
Text(String),
}
-pub fn initial(result: searcher::local::InitialWebsiteResult) -> InitialWebsiteResult {
- let sidebar = result.entity_sidebar;
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct HighlightedSpellCorrection {
+ pub raw: String,
+ pub highlighted: String,
+}
- let spell_corrected_query = result.spell_corrected_query.map(|correction| {
+impl From<spell::Correction> for HighlightedSpellCorrection {
+ fn from(correction: spell::Correction) -> Self {
let mut highlighted = String::new();
let mut raw = String::new();
@@ -91,31 +94,10 @@ pub fn initial(result: searcher::local::InitialWebsiteResult) -> InitialWebsiteR
raw = raw.trim_end().to_string();
highlighted = highlighted.trim_end().to_string();
- HighlightedSpellCorrection { raw, highlighted }
- });
-
- InitialWebsiteResult {
- spell_corrected_query,
- websites: result.websites,
- num_websites: result.num_websites,
- entity_sidebar: sidebar,
+ Self { raw, highlighted }
}
}
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct HighlightedSpellCorrection {
- pub raw: String,
- pub highlighted: String,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct InitialWebsiteResult {
- pub spell_corrected_query: Option<HighlightedSpellCorrection>,
- pub num_websites: usize,
- pub websites: Vec<RankingWebsite>,
- pub entity_sidebar: Option<DisplayedEntity>,
-}
-
pub fn html_escape(s: &str) -> String {
html_escape::decode_html_entities(s)
.chars()
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index eda6904f..6e61502a 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -15,14 +15,18 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ bangs::{BangHit, Bangs},
collector::{self, BucketCollector},
exponential_backoff::ExponentialBackoff,
inverted_index::{self, RetrievedWebpage},
+ query,
ranking::{
models::cross_encoder::CrossEncoderModel,
pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},
},
- search_prettifier::{create_stackoverflow_sidebar, DisplayedWebpage, Sidebar},
+ search_prettifier::{
+ create_stackoverflow_sidebar, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,
+ },
searcher::PrettifiedWebsitesResult,
};
@@ -40,7 +44,7 @@ use thiserror::Error;
use crate::sonic;
-use super::{InitialPrettifiedSearchResult, PrettifiedSearchResult, SearchQuery};
+use super::{InitialWebsiteResult, SearchQuery, SearchResult};
type Result<T> = std::result::Result<T, Error>;
const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 100.0;
@@ -62,16 +66,15 @@ pub enum Error {
}
impl RemoteSearcher {
- async fn search(&self, query: &SearchQuery) -> Result<InitialPrettifiedSearchResult> {
+ async fn search(&self, query: &SearchQuery) -> Result<InitialWebsiteResult> {
for timeout in ExponentialBackoff::from_millis(30)
.with_limit(Duration::from_millis(200))
.take(5)
{
if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
{
- if let Ok(sonic::Response::Content(body)) = connection
- .send(Request::SearchPrettified(query.clone()))
- .await
+ if let Ok(sonic::Response::Content(body)) =
+ connection.send(Request::Search(query.clone())).await
{
return Ok(body);
}
@@ -109,7 +112,7 @@ impl RemoteSearcher {
}
#[derive(Clone, PartialEq, Eq)]
-struct ShardId(u32);
+struct ShardId(u64);
pub struct Shard {
id: ShardId,
@@ -117,7 +120,7 @@ pub struct Shard {
}
impl Shard {
- pub fn new(id: u32, replicas: Vec<String>) -> Self {
+ pub fn new(id: u64, replicas: Vec<String>) -> Self {
let mut parsed_replicas = Vec::new();
for replica in replicas {
@@ -169,14 +172,13 @@ impl Shard {
}
struct InitialSearchResultShard {
- local_result: InitialPrettifiedSearchResult,
+ local_result: InitialWebsiteResult,
shard: ShardId,
}
#[derive(Serialize, Deserialize)]
pub enum Request {
Search(SearchQuery),
- SearchPrettified(SearchQuery),
RetrieveWebsites {
websites: Vec<inverted_index::WebsitePointer>,
query: String,
@@ -186,6 +188,7 @@ pub enum Request {
pub struct DistributedSearcher {
shards: Vec<Shard>,
cross_encoder: Arc<CrossEncoderModel>,
+ bangs: Bangs,
}
#[derive(Clone)]
@@ -219,10 +222,11 @@ impl collector::Doc for ScoredWebsitePointer {
}
impl DistributedSearcher {
- pub fn new(shards: Vec<Shard>, model: CrossEncoderModel) -> Self {
+ pub fn new(shards: Vec<Shard>, model: CrossEncoderModel, bangs: Bangs) -> Self {
Self {
shards,
cross_encoder: Arc::new(model),
+ bangs,
}
}
@@ -238,15 +242,13 @@ impl DistributedSearcher {
BucketCollector::new(pipeline.collector_top_n() + pipeline.collector_offset());
for result in initial_results {
- if let InitialPrettifiedSearchResult::Websites(local_result) = result.local_result {
- for website in local_result.websites {
- let pointer = ScoredWebsitePointer {
- website,
- shard: result.shard.clone(),
- };
-
- collector.insert(pointer);
- }
+ for website in result.local_result.websites {
+ let pointer = ScoredWebsitePointer {
+ website,
+ shard: result.shard.clone(),
+ };
+
+ collector.insert(pointer);
}
}
@@ -317,15 +319,13 @@ impl DistributedSearcher {
.search_initial(&query)
.await
.into_iter()
- .filter_map(|result| match result.local_result {
- InitialPrettifiedSearchResult::Websites(websites) => {
- if let Some(website) = websites.websites.first().cloned() {
- Some((result.shard, website))
- } else {
- None
- }
- }
- InitialPrettifiedSearchResult::Bang(_) => None,
+ .filter_map(|result| {
+ result
+ .local_result
+ .websites
+ .first()
+ .cloned()
+ .map(|website| (result.shard, website))
})
.collect();
@@ -350,13 +350,9 @@ impl DistributedSearcher {
initial_results: &[InitialSearchResultShard],
query: &SearchQuery,
) -> Result<Option<Sidebar>> {
- let entity = initial_results.first().and_then(|result| {
- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.entity_sidebar.clone()
- } else {
- None
- }
- });
+ let entity = initial_results
+ .first()
+ .and_then(|result| result.local_result.entity_sidebar.clone());
match entity {
Some(entity) => Ok(Some(Sidebar::Entity(entity))),
@@ -364,7 +360,17 @@ impl DistributedSearcher {
}
}
- pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {
+ fn check_bangs(&self, query: &SearchQuery) -> Option<BangHit> {
+ let parsed_terms = query::parser::parse(&query.query);
+
+ self.bangs.get(&parsed_terms)
+ }
+
+ pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
+ if let Some(bang) = self.check_bangs(query) {
+ return Ok(SearchResult::Bang(bang));
+ }
+
let start = Instant::now();
if query.is_empty() {
@@ -375,35 +381,16 @@ impl DistributedSearcher {
let initial_results = self.search_initial(query).await;
- // check if any result has a bang hit
- if let Some(result) = initial_results
- .iter()
- .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_)))
- {
- if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {
- return Ok(PrettifiedSearchResult::Bang(bang.clone()));
- }
- }
-
let sidebar = self.sidebar(&initial_results, query).await?;
- let spell_corrected_query = initial_results.first().and_then(|result| {
- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.spell_corrected_query.clone()
- } else {
- None
- }
- });
+ let spell_corrected_query = initial_results
+ .first()
+ .and_then(|result| result.local_result.spell_corrected_query.clone())
+ .map(HighlightedSpellCorrection::from);
let num_docs = initial_results
.iter()
- .map(|result| {
- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.num_websites
- } else {
- 0
- }
- })
+ .map(|result| result.local_result.num_websites)
.sum();
let top_websites = self.combine_results(initial_results, &mut search_query)?;
@@ -420,9 +407,9 @@ impl DistributedSearcher {
return Err(Error::SearchFailed);
}
- Ok(PrettifiedSearchResult::Websites(PrettifiedWebsitesResult {
+ Ok(SearchResult::Websites(PrettifiedWebsitesResult {
spell_corrected_query,
- num_docs,
+ num_hits: num_docs,
webpages: retrieved_webpages,
sidebar,
search_duration_ms: start.elapsed().as_millis(),
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index d16fda5f..92b5ba28 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -17,10 +17,8 @@
use std::str::FromStr;
use optics::Optic;
-use serde::{Deserialize, Serialize};
use uuid::Uuid;
-use crate::bangs::{BangHit, Bangs};
use crate::entity_index::{EntityIndex, StoredEntity};
use crate::image_store::Image;
use crate::index::Index;
@@ -29,20 +27,19 @@ use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::optics::CreateAggregator;
use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
-use crate::search_prettifier::{DisplayedEntity, Sidebar};
+use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection};
use crate::spell::Correction;
use crate::webgraph::centrality::topic::TopicCentrality;
use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
-use super::WebsitesResult;
-use super::{InitialSearchResult, SearchQuery, SearchResult};
+use super::PrettifiedWebsitesResult;
+use super::{InitialWebsiteResult, SearchQuery};
pub struct LocalSearcher {
index: Index,
entity_index: Option<EntityIndex>,
- bangs: Option<Bangs>,
centrality_store: Option<CentralityStore>,
topic_centrality: Option<TopicCentrality>,
}
@@ -58,7 +55,6 @@ impl LocalSearcher {
LocalSearcher {
index,
entity_index: None,
- bangs: None,
centrality_store: None,
topic_centrality: None,
}
@@ -68,10 +64,6 @@ impl LocalSearcher {
self.entity_index = Some(entity_index);
}
- pub fn set_bangs(&mut self, bangs: Bangs) {
- self.bangs = Some(bangs);
- }
-
pub fn set_centrality_store(&mut self, centrality_store: CentralityStore) {
self.centrality_store = Some(centrality_store);
}
@@ -215,16 +207,6 @@ impl LocalSearcher {
Ok(self.index.spell_correction(&parsed_query.simple_terms()))
}
- fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {
- let parsed_query = self.parse_query(query, None)?;
-
- if let Some(bangs) = self.bangs.as_ref() {
- return Ok(bangs.get(&parsed_query));
- }
-
- Ok(None)
- }
-
fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {
self.entity_index
.as_ref()
@@ -235,23 +217,19 @@ impl LocalSearcher {
&self,
query: &SearchQuery,
de_rank_similar: bool,
- ) -> Result<InitialSearchResult> {
- if let Some(bang) = self.check_bangs(query)? {
- return Ok(InitialSearchResult::Bang(bang));
- }
-
+ ) -> Result<InitialWebsiteResult> {
let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;
let correction = self.spell_correction(query)?;
let sidebar = self
.entity_sidebar(query)
.map(|entity| DisplayedEntity::from(entity, self));
- Ok(InitialSearchResult::Websites(InitialWebsiteResult {
+ Ok(InitialWebsiteResult {
spell_corrected_query: correction,
websites,
num_websites,
entity_sidebar: sidebar,
- }))
+ })
}
pub fn retrieve_websites(
@@ -273,12 +251,15 @@ impl LocalSearcher {
self.index.retrieve_websites(websites, &query)
}
- pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
+ pub fn search(&self, query: &SearchQuery) -> Result<PrettifiedWebsitesResult> {
use std::{sync::Arc, time::Instant};
- use crate::ranking::{
- models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},
- pipeline::RankingPipeline,
+ use crate::{
+ ranking::{
+ models::cross_encoder::{CrossEncoderModel, DummyCrossEncoder},
+ pipeline::RankingPipeline,
+ },
+ search_prettifier::Sidebar,
};
let start = Instant::now();
@@ -291,29 +272,28 @@ impl LocalSearcher {
}
};
- let initial_result = self.search_initial(&search_query, true)?;
-
- match initial_result {
- InitialSearchResult::Websites(search_result) => {
- let top_websites = pipeline.apply(search_result.websites);
- let pointers: Vec<_> = top_websites
- .into_iter()
- .map(|website| website.pointer)
- .collect();
-
- let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;
- Ok(SearchResult::Websites(WebsitesResult {
- spell_corrected_query: search_result.spell_corrected_query,
- webpages: inverted_index::SearchResult {
- num_docs: search_result.num_websites,
- documents: retrieved_sites,
- },
- sidebar: search_result.entity_sidebar.map(Sidebar::Entity),
- search_duration_ms: start.elapsed().as_millis(),
- }))
- }
- InitialSearchResult::Bang(bang) => Ok(SearchResult::Bang(bang)),
- }
+ let search_result = self.search_initial(&search_query, true)?;
+
+ let top_websites = pipeline.apply(search_result.websites);
+ let pointers: Vec<_> = top_websites
+ .into_iter()
+ .map(|website| website.pointer)
+ .collect();
+
+ let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;
+
+ Ok(PrettifiedWebsitesResult {
+ spell_corrected_query: search_result
+ .spell_corrected_query
+ .map(HighlightedSpellCorrection::from),
+ num_hits: search_result.num_websites,
+ webpages: retrieved_sites
+ .into_iter()
+ .map(DisplayedWebpage::from)
+ .collect(),
+ sidebar: search_result.entity_sidebar.map(Sidebar::Entity),
+ search_duration_ms: start.elapsed().as_millis(),
+ })
}
pub fn favicon(&self, site: &Url) -> Option<Image> {
@@ -340,25 +320,6 @@ impl LocalSearcher {
}
}
-#[derive(Debug, Serialize, Deserialize)]
-pub struct InitialWebsiteResult {
- pub spell_corrected_query: Option<Correction>,
- pub num_websites: usize,
- pub websites: Vec<RankingWebsite>,
- pub entity_sidebar: Option<DisplayedEntity>,
-}
-
-impl SearchResult {
- #[cfg(test)]
- pub fn into_websites(self) -> Option<WebsitesResult> {
- if let SearchResult::Websites(res) = self {
- Some(res)
- } else {
- None
- }
- }
-}
-
#[cfg(test)]
mod tests {
use crate::{
@@ -417,10 +378,7 @@ mod tests {
..Default::default()
})
.unwrap()
- .into_websites()
- .unwrap()
.webpages
- .documents
.into_iter()
.map(|page| page.url)
.collect();
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 384d5d58..4b3dfb2a 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -24,30 +24,16 @@ use serde::{Deserialize, Serialize};
use crate::{
bangs::BangHit,
- inverted_index,
- search_prettifier::{self, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},
+ ranking::pipeline::RankingWebsite,
+ search_prettifier::{DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},
spell::Correction,
webpage::region::Region,
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
-#[derive(Debug, Serialize)]
-pub struct WebsitesResult {
- pub spell_corrected_query: Option<Correction>,
- pub webpages: inverted_index::SearchResult,
- pub sidebar: Option<Sidebar>,
- pub search_duration_ms: u128,
-}
-
#[derive(Debug, Serialize)]
pub enum SearchResult {
- Websites(WebsitesResult),
- Bang(BangHit),
-}
-
-#[derive(Debug, Serialize)]
-pub enum PrettifiedSearchResult {
Websites(PrettifiedWebsitesResult),
Bang(BangHit),
}
@@ -56,23 +42,11 @@ pub enum PrettifiedSearchResult {
pub struct PrettifiedWebsitesResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub webpages: Vec<DisplayedWebpage>,
- pub num_docs: usize,
+ pub num_hits: usize,
pub sidebar: Option<Sidebar>,
pub search_duration_ms: u128,
}
-#[derive(Debug, Serialize, Deserialize)]
-pub enum InitialSearchResult {
- Websites(local::InitialWebsiteResult),
- Bang(BangHit),
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub enum InitialPrettifiedSearchResult {
- Websites(search_prettifier::InitialWebsiteResult),
- Bang(BangHit),
-}
-
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SearchQuery {
pub query: String,
@@ -83,6 +57,14 @@ pub struct SearchQuery {
pub site_rankings: Option<SiteRankings>,
}
+#[derive(Debug, Serialize, Deserialize)]
+pub struct InitialWebsiteResult {
+ pub spell_corrected_query: Option<Correction>,
+ pub num_websites: usize,
+ pub websites: Vec<RankingWebsite>,
+ pub entity_sidebar: Option<DisplayedEntity>,
+}
+
impl Default for SearchQuery {
fn default() -> Self {
Self {
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 81863f34..2b2d2587 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -342,6 +342,7 @@ mod tests {
use super::*;
use crate::{
index::Index,
+ search_prettifier::Snippet,
searcher::{LocalSearcher, SearchQuery},
webpage::Webpage,
};
@@ -360,6 +361,13 @@ to the project are from community members.[15]
Rust won first place for "most loved programming language" in the Stack Overflow Developer
Survey in 2016, 2017, and 2018."#;
+ fn snippet_text(snippet: Snippet) -> String {
+ match snippet {
+ Snippet::Normal { date: _, text } => text,
+ _ => panic!("The snippet was not text"),
+ }
+ }
+
#[test]
fn snippet_during_search() {
let mut index = Index::temporary().expect("Unable to open index");
@@ -390,13 +398,11 @@ Survey in 2016, 2017, and 2018."#;
query: "rust language".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
+ .expect("Search failed");
- assert_eq!(result.webpages.num_docs, 1);
- assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its..."));
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(snippet_text(result.webpages[0].snippet.clone()), format!("{HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is a systems programming {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX} sponsored by Mozilla which describes it as a \"safe, concurrent, practical {HIGHLIGHTEN_PREFIX}language{HIGHLIGHTEN_POSTFIX}\", supporting functional and imperative-procedural paradigms. {HIGHLIGHTEN_PREFIX}Rust{HIGHLIGHTEN_POSTFIX} is syntactically similar to C++[according to whom?], but its..."));
}
#[test]
@@ -429,13 +435,11 @@ Survey in 2016, 2017, and 2018."#;
query: "describe".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
+ .expect("Search failed");
- assert_eq!(result.webpages.num_docs, 1);
- assert_eq!(result.webpages.documents.len(), 1);
- assert_eq!(result.webpages.documents[0].snippet, format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its..."));
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
+ assert_eq!(snippet_text(result.webpages[0].snippet.clone()), format!("Rust is a systems programming language sponsored by Mozilla which {HIGHLIGHTEN_PREFIX}describes{HIGHLIGHTEN_POSTFIX} it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural paradigms. Rust is syntactically similar to C++[according to whom?], but its..."));
}
#[test]
@@ -638,14 +642,12 @@ Survey in 2016, 2017, and 2018."#;
query: "paradigms".to_string(),
..Default::default()
})
- .expect("Search failed")
- .into_websites()
- .unwrap();
+ .expect("Search failed");
- assert_eq!(result.webpages.num_docs, 1);
- assert_eq!(result.webpages.documents.len(), 1);
+ assert_eq!(result.num_hits, 1);
+ assert_eq!(result.webpages.len(), 1);
assert_eq!(
- result.webpages.documents[0].snippet,
+ snippet_text(result.webpages[0].snippet.clone()),
format!("Rust is a systems programming language sponsored by Mozilla which describes it as a \"safe, concurrent, practical language\", supporting functional and imperative-procedural {HIGHLIGHTEN_PREFIX}paradigms{HIGHLIGHTEN_POSTFIX}. Rust is syntactically similar to C++[according to whom?], but its...")
);
}