Skip to content

web search done right

refactor searcher to have separate paths for api and html

Mikkel Denker2064cc4ad51a
diff --git a/benches/search-preindexed-goggle.rs b/benches/search-preindexed-goggle.rs
index 9258cdc5..54eba9f4 100644
--- a/benches/search-preindexed-goggle.rs
+++ b/benches/search-preindexed-goggle.rs
@@ -1,5 +1,8 @@
use criterion::{criterion_group, criterion_main, Criterion};
-use cuely::{index::Index, searcher::LocalSearcher};
+use cuely::{
+ index::Index,
+ searcher::{LocalSearcher, SearchQuery},
+};
const INDEX_PATH: &str = "data/index";
@@ -12,7 +15,12 @@ macro_rules! bench {
$c.bench_function(desc.as_str(), |b| {
b.iter(|| {
$searcher
- .search($query, None, Some($goggle.to_string()), None)
+ .search(&SearchQuery {
+ original: $query.to_string(),
+ selected_region: None,
+ goggle_program: Some($goggle.to_string()),
+ skip_pages: None,
+ })
.unwrap()
})
});
diff --git a/benches/search-preindexed.rs b/benches/search-preindexed.rs
index bbcdee67..87dfa85e 100644
--- a/benches/search-preindexed.rs
+++ b/benches/search-preindexed.rs
@@ -1,5 +1,8 @@
use criterion::{criterion_group, criterion_main, Criterion};
-use cuely::{index::Index, searcher::LocalSearcher};
+use cuely::{
+ index::Index,
+ searcher::{LocalSearcher, SearchQuery},
+};
const INDEX_PATH: &str = "data/index";
@@ -9,7 +12,16 @@ macro_rules! bench {
desc.push_str($query);
desc.push('\'');
$c.bench_function(desc.as_str(), |b| {
- b.iter(|| $searcher.search($query, None, None, None).unwrap())
+ b.iter(|| {
+ $searcher
+ .search(&SearchQuery {
+ original: $query.to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
+ .unwrap()
+ })
});
};
}
diff --git a/src/entrypoint/search_server.rs b/src/entrypoint/search_server.rs
index 63f90317..08c8209f 100644
--- a/src/entrypoint/search_server.rs
+++ b/src/entrypoint/search_server.rs
@@ -42,14 +42,8 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
loop {
if let Ok(req) = server.accept::<searcher::distributed::Request>().await {
match &req.body {
- searcher::Request::Search(search) => {
- match local_searcher.search_initial(
- &search.original,
- search.selected_region,
- search.goggle_program.clone(),
- search.skip_pages,
- false,
- ) {
+ searcher::Request::Search(query) => {
+ match local_searcher.search_initial(query, false) {
Ok(response) => {
req.respond(sonic::Response::Content(response)).await.ok();
}
@@ -74,30 +68,19 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
}
}
}
- searcher::Request::SearchPrettified(search) => {
- match local_searcher.search_initial(
- &search.original,
- search.selected_region,
- search.goggle_program.clone(),
- search.skip_pages,
- false,
- ) {
+ searcher::Request::SearchPrettified(query) => {
+ match local_searcher.search_initial(query, false) {
Ok(result) => match result {
- searcher::InitialSearchResult::Websites(
- searcher::InitialWebsitesFormatting::Raw(result),
- ) => {
+ searcher::InitialSearchResult::Websites(result) => {
let res = search_prettifier::initial(result, &local_searcher);
req.respond(sonic::Response::Content(
- searcher::InitialSearchResult::Websites(
- searcher::InitialWebsitesFormatting::Prettified(res),
- ),
+ searcher::InitialPrettifiedSearchResult::Websites(res),
))
.await
.ok();
}
searcher::InitialSearchResult::Bang(_) => todo!(),
- _ => unreachable!(),
},
Err(_) => {
req.respond::<inverted_index::SearchResult>(sonic::Response::Empty)
diff --git a/src/frontend/api.rs b/src/frontend/api.rs
index f935af6d..2fe812f5 100644
--- a/src/frontend/api.rs
+++ b/src/frontend/api.rs
@@ -16,7 +16,7 @@
use std::{collections::HashMap, sync::Arc};
-use crate::webpage::region::Region;
+use crate::{searcher::SearchQuery, webpage::region::Region};
use super::State;
use axum::{extract, response::IntoResponse, Extension, Json};
@@ -40,7 +40,12 @@ pub async fn search(
match state
.searcher
- .search_api(query.as_str(), selected_region, None, skip_pages)
+ .search_api(&SearchQuery {
+ original: query.to_string(),
+ selected_region,
+ goggle_program: None,
+ skip_pages,
+ })
.await
{
Ok(result) => Json(result),
diff --git a/src/frontend/search.rs b/src/frontend/search.rs
index d326921c..c0f52557 100644
--- a/src/frontend/search.rs
+++ b/src/frontend/search.rs
@@ -20,7 +20,7 @@ use axum::Extension;
use crate::{
search_prettifier::{thousand_sep_number, DisplayedEntity, DisplayedWebpage},
- searcher::{self, PrettifiedSearchResult},
+ searcher::{self, PrettifiedSearchResult, SearchQuery},
webpage::region::{Region, ALL_REGIONS},
};
@@ -91,7 +91,12 @@ pub async fn route(
match state
.searcher
- .search_prettified(query.as_str(), selected_region, goggle, skip_pages)
+ .search_prettified(&SearchQuery {
+ original: query.clone(),
+ selected_region,
+ goggle_program: goggle,
+ skip_pages,
+ })
.await
{
Ok(result) => match result {
diff --git a/src/ranking/goggles/mod.rs b/src/ranking/goggles/mod.rs
index 7fe3f21f..485267f7 100644
--- a/src/ranking/goggles/mod.rs
+++ b/src/ranking/goggles/mod.rs
@@ -294,7 +294,7 @@ mod tests {
use crate::{
index::Index,
schema::create_schema,
- searcher::LocalSearcher,
+ searcher::{LocalSearcher, SearchQuery},
webpage::{Html, Webpage},
};
@@ -361,7 +361,12 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search("website", None, None, None)
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap()
@@ -373,17 +378,17 @@ mod tests {
assert_eq!(res[1].url, "https://www.a.com");
let res = searcher
- .search(
- "website",
- None,
- Some(
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: Some(
r#"
- $discard,site=b.com
- "#
+ $discard,site=b.com
+ "#
.to_string(),
),
- None,
- )
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap()
@@ -394,17 +399,17 @@ mod tests {
assert_eq!(res[0].url, "https://www.a.com");
let res = searcher
- .search(
- "website",
- None,
- Some(
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: Some(
r#"
- $boost=10,site=a.com
- "#
+ $boost=10,site=a.com
+ "#
.to_string(),
),
- None,
- )
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap()
@@ -483,12 +488,14 @@ mod tests {
let searcher = LocalSearcher::from(index);
let _ = searcher
- .search(
- "website",
- None,
- Some(include_str!("../../../testcases/goggles/quickstart.goggle").to_string()),
- None,
- )
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: Some(
+ include_str!("../../../testcases/goggles/quickstart.goggle").to_string(),
+ ),
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap()
@@ -496,12 +503,14 @@ mod tests {
.documents;
let _ = searcher
- .search(
- "website",
- None,
- Some(include_str!("../../../testcases/goggles/hacker_news.goggle").to_string()),
- None,
- )
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: Some(
+ include_str!("../../../testcases/goggles/hacker_news.goggle").to_string(),
+ ),
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap()
@@ -509,14 +518,14 @@ mod tests {
.documents;
let _ = searcher
- .search(
- "website",
- None,
- Some(
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: Some(
include_str!("../../../testcases/goggles/copycats_removal.goggle").to_string(),
),
- None,
- )
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap()
@@ -609,10 +618,10 @@ mod tests {
let searcher = LocalSearcher::from(index);
let res = searcher
- .search(
- "website",
- None,
- Some(
+ .search(&SearchQuery {
+ original: "website".to_string(),
+ selected_region: None,
+ goggle_program: Some(
r#"
$discard
$site=a.com,boost=6
@@ -620,8 +629,8 @@ mod tests {
"#
.to_string(),
),
- None,
- )
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap()
diff --git a/src/ranking/mod.rs b/src/ranking/mod.rs
index 5151301b..8a5f5683 100644
--- a/src/ranking/mod.rs
+++ b/src/ranking/mod.rs
@@ -106,7 +106,7 @@ impl Ranker {
mod tests {
use crate::{
index::Index,
- searcher::LocalSearcher,
+ searcher::{LocalSearcher, SearchQuery},
webpage::{Html, Link, Webpage},
};
@@ -171,7 +171,12 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("example", None, None, None)
+ .search(&SearchQuery {
+ original: "example".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -239,7 +244,12 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("example", None, None, None)
+ .search(&SearchQuery {
+ original: "example".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -329,7 +339,12 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("dr dk", None, None, None)
+ .search(&SearchQuery {
+ original: "dr dk".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -398,7 +413,12 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("title", None, None, None)
+ .search(&SearchQuery {
+ original: "title".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -472,7 +492,12 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("test", None, None, None)
+ .search(&SearchQuery {
+ original: "test".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -539,7 +564,12 @@ mod tests {
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("test", None, None, None)
+ .search(&SearchQuery {
+ original: "test".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -630,18 +660,18 @@ mod tests {
let searcher = LocalSearcher::new(index, None, None);
let res = searcher
- .search(
- "example",
- None,
- Some(
+ .search(&SearchQuery {
+ original: "example".to_string(),
+ selected_region: None,
+ goggle_program: Some(
r#"
@field_title = 20000000
@host_centrality = 0
"#
.to_string(),
),
- None,
- )
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap();
@@ -650,18 +680,18 @@ mod tests {
assert_eq!(&res.webpages.documents[0].url, "https://www.title.com");
let res = searcher
- .search(
- "example",
- None,
- Some(
+ .search(&SearchQuery {
+ original: "example".to_string(),
+ selected_region: None,
+ goggle_program: Some(
r#"
- @field_all_body= 20000000
+ @field_all_body = 20000000
@host_centrality = 0
"#
.to_string(),
),
- None,
- )
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap();
@@ -670,12 +700,17 @@ mod tests {
assert_eq!(&res.webpages.documents[0].url, "https://www.body.com");
let res = searcher
- .search(
- "example",
- None,
- Some("@host_centrality= 2000000".to_string()),
- None,
- )
+ .search(&SearchQuery {
+ original: "example".to_string(),
+ selected_region: None,
+ goggle_program: Some(
+ r#"
+ @host_centrality = 2000000
+ "#
+ .to_string(),
+ ),
+ skip_pages: None,
+ })
.unwrap()
.into_websites()
.unwrap();
@@ -766,7 +801,12 @@ mod tests {
let searcher = LocalSearcher::new(index, None, None);
let result = searcher
- .search("termA termB", None, None, None)
+ .search(&SearchQuery {
+ original: "termA termB".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap()
@@ -837,7 +877,12 @@ mod tests {
let searcher = LocalSearcher::new(index, None, None);
let result = searcher
- .search("test", None, None, None)
+ .search(&SearchQuery {
+ original: "test".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap()
diff --git a/src/searcher/distributed.rs b/src/searcher/distributed.rs
index f9e17a72..745464eb 100644
--- a/src/searcher/distributed.rs
+++ b/src/searcher/distributed.rs
@@ -19,10 +19,7 @@ use crate::{
exponential_backoff::ExponentialBackoff,
inverted_index::{self, RetrievedWebpage},
search_prettifier::DisplayedWebpage,
- searcher::{
- InitialWebsitesFormatting, PrettifiedWebsitesResult, SearchResult, WebsitesResult,
- NUM_RESULTS_PER_PAGE,
- },
+ searcher::{PrettifiedWebsitesResult, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE},
};
use std::{cmp::Ordering, net::SocketAddr, time::Instant};
@@ -32,9 +29,11 @@ use futures::StreamExt;
use serde::{Deserialize, Serialize};
use thiserror::Error;
-use crate::{sonic, webpage::region::Region};
+use crate::sonic;
-use super::{InitialSearchResult, PrettifiedSearchResult};
+use super::{
+ InitialPrettifiedSearchResult, InitialSearchResult, PrettifiedSearchResult, SearchQuery,
+};
type Result<T> = std::result::Result<T, Error>;
@@ -67,7 +66,10 @@ impl RemoteSearcher {
Err(Error::SearchFailed)
}
- async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResult> {
+ async fn search_prettified(
+ &self,
+ query: &SearchQuery,
+ ) -> Result<InitialPrettifiedSearchResult> {
for timeout in ExponentialBackoff::from_millis(30).take(5) {
if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
{
@@ -171,7 +173,10 @@ impl Shard {
}
}
- async fn search_prettified(&self, query: &SearchQuery) -> Result<InitialSearchResultShard> {
+ async fn search_prettified(
+ &self,
+ query: &SearchQuery,
+ ) -> Result<InitialPrettifiedSearchResultShard> {
match self
.replicas
.iter()
@@ -180,7 +185,7 @@ impl Shard {
.next()
.await
{
- Some(result) => Ok(InitialSearchResultShard {
+ Some(result) => Ok(InitialPrettifiedSearchResultShard {
local_result: result?,
shard: self.id.clone(),
}),
@@ -230,12 +235,9 @@ struct InitialSearchResultShard {
shard: ShardId,
}
-#[derive(Serialize, Deserialize, Clone)]
-pub struct SearchQuery {
- pub original: String,
- pub selected_region: Option<Region>,
- pub goggle_program: Option<String>,
- pub skip_pages: Option<usize>,
+struct InitialPrettifiedSearchResultShard {
+ local_result: InitialPrettifiedSearchResult,
+ shard: ShardId,
}
#[derive(Serialize, Deserialize)]
@@ -323,31 +325,18 @@ impl DistributedSearcher {
Self { shards }
}
- pub async fn search_api(
- &self,
- query: &str,
- selected_region: Option<Region>,
- goggle_program: Option<String>,
- skip_pages: Option<usize>,
- ) -> Result<SearchResult> {
+ pub async fn search_api(&self, query: &SearchQuery) -> Result<SearchResult> {
let start = Instant::now();
if query.is_empty() {
return Err(Error::EmptyQuery);
}
- let query = SearchQuery {
- original: query.to_string(),
- selected_region,
- goggle_program,
- skip_pages,
- };
-
// search shards
let initial_results = self
.shards
.iter()
- .map(|shard| shard.search(&query))
+ .map(|shard| shard.search(query))
.collect::<FuturesUnordered<_>>()
.collect::<Vec<_>>()
.await
@@ -366,9 +355,7 @@ impl DistributedSearcher {
}
let spell_corrected_query = initial_results.first().and_then(|result| {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =
- &result.local_result
- {
+ if let InitialSearchResult::Websites(result) = &result.local_result {
result.spell_corrected_query.clone()
} else {
None
@@ -376,9 +363,7 @@ impl DistributedSearcher {
});
let entity = initial_results.first().and_then(|result| {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =
- &result.local_result
- {
+ if let InitialSearchResult::Websites(result) = &result.local_result {
result.entity.clone()
} else {
None
@@ -388,9 +373,7 @@ impl DistributedSearcher {
let num_docs = initial_results
.iter()
.map(|result| {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(result)) =
- &result.local_result
- {
+ if let InitialSearchResult::Websites(result) = &result.local_result {
result.websites.num_websites
} else {
0
@@ -399,13 +382,11 @@ impl DistributedSearcher {
.sum();
// combine results
- let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);
+ let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);
let mut collector = BucketCollector::new(top_n);
for result in initial_results {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(local_result)) =
- result.local_result
- {
+ if let InitialSearchResult::Websites(local_result) = result.local_result {
for website in local_result.websites.top_websites {
let pointer = ScoredWebsitePointer {
local_pointer: website,
@@ -420,7 +401,7 @@ impl DistributedSearcher {
let top_websites = collector
.into_sorted_vec(true)
.into_iter()
- .skip(skip_pages.unwrap_or(0))
+ .skip(query.skip_pages.unwrap_or(0))
.take(NUM_RESULTS_PER_PAGE)
.collect::<Vec<_>>();
@@ -465,31 +446,18 @@ impl DistributedSearcher {
}))
}
- pub async fn search_prettified(
- &self,
- query: &str,
- selected_region: Option<Region>,
- goggle_program: Option<String>,
- skip_pages: Option<usize>,
- ) -> Result<PrettifiedSearchResult> {
+ pub async fn search_prettified(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {
let start = Instant::now();
if query.is_empty() {
return Err(Error::EmptyQuery);
}
- let query = SearchQuery {
- original: query.to_string(),
- selected_region,
- goggle_program,
- skip_pages,
- };
-
// search shards
let initial_results = self
.shards
.iter()
- .map(|shard| shard.search_prettified(&query))
+ .map(|shard| shard.search_prettified(query))
.collect::<FuturesUnordered<_>>()
.collect::<Vec<_>>()
.await
@@ -500,17 +468,15 @@ impl DistributedSearcher {
// check if any result has a bang hit
if let Some(result) = initial_results
.iter()
- .find(|result| matches!(result.local_result, InitialSearchResult::Bang(_)))
+ .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_)))
{
- if let InitialSearchResult::Bang(bang) = &result.local_result {
+ if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {
return Ok(PrettifiedSearchResult::Bang(bang.clone()));
}
}
let spell_corrected_query = initial_results.first().and_then(|result| {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =
- &result.local_result
- {
+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
result.spell_corrected_query.clone()
} else {
None
@@ -518,9 +484,7 @@ impl DistributedSearcher {
});
let entity = initial_results.first().and_then(|result| {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(result)) =
- &result.local_result
- {
+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
result.entity.clone()
} else {
None
@@ -530,10 +494,7 @@ impl DistributedSearcher {
let num_docs = initial_results
.iter()
.map(|result| {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(
- result,
- )) = &result.local_result
- {
+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
result.websites.num_websites
} else {
0
@@ -542,14 +503,11 @@ impl DistributedSearcher {
.sum();
// combine results
- let top_n = NUM_RESULTS_PER_PAGE + skip_pages.unwrap_or(0);
+ let top_n = NUM_RESULTS_PER_PAGE + query.skip_pages.unwrap_or(0);
let mut collector = BucketCollector::new(top_n);
for result in initial_results {
- if let InitialSearchResult::Websites(InitialWebsitesFormatting::Prettified(
- local_result,
- )) = result.local_result
- {
+ if let InitialPrettifiedSearchResult::Websites(local_result) = result.local_result {
for website in local_result.websites.top_websites {
let pointer = ScoredWebsitePointer {
local_pointer: website,
@@ -564,7 +522,7 @@ impl DistributedSearcher {
let top_websites = collector
.into_sorted_vec(true)
.into_iter()
- .skip(skip_pages.unwrap_or(0))
+ .skip(query.skip_pages.unwrap_or(0))
.take(NUM_RESULTS_PER_PAGE)
.collect::<Vec<_>>();
diff --git a/src/searcher/local.rs b/src/searcher/local.rs
index f49ed41e..f352423c 100644
--- a/src/searcher/local.rs
+++ b/src/searcher/local.rs
@@ -31,10 +31,7 @@ use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
-use super::{
- InitialSearchResult, InitialWebsitesFormatting, SearchResult, WebsitesResult,
- NUM_RESULTS_PER_PAGE,
-};
+use super::{InitialSearchResult, SearchQuery, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE};
pub struct LocalSearcher {
index: Index,
@@ -66,17 +63,17 @@ impl LocalSearcher {
pub fn search_initial(
&self,
- query: &str,
- selected_region: Option<Region>,
- goggle_program: Option<String>,
- skip_pages: Option<usize>,
+ query: &SearchQuery,
de_rank_similar: bool,
) -> Result<InitialSearchResult> {
- let raw_query = query.to_string();
- let goggle = goggle_program.and_then(|program| goggles::parse(&program).ok());
+ let raw_query = query.original.clone();
+ let goggle = query
+ .goggle_program
+ .as_ref()
+ .and_then(|program| goggles::parse(program).ok());
- let mut query = Query::parse(
- query,
+ let mut parsed_query = Query::parse(
+ &query.original,
self.index.schema(),
self.index.tokenizers(),
goggle
@@ -85,18 +82,18 @@ impl LocalSearcher {
.unwrap_or(&SignalAggregator::default()),
)?;
- if query.is_empty() {
+ if parsed_query.is_empty() {
return Err(Error::EmptyQuery);
}
if let Some(bangs) = self.bangs.as_ref() {
- if let Some(bang) = bangs.get(&query) {
+ if let Some(bang) = bangs.get(&parsed_query) {
return Ok(InitialSearchResult::Bang(bang));
}
}
if let Some(goggle) = &goggle {
- query.set_goggle(goggle, &self.index.schema());
+ parsed_query.set_goggle(goggle, &self.index.schema());
}
let mut ranker = Ranker::new(
@@ -104,11 +101,11 @@ impl LocalSearcher {
goggle.map(|goggle| goggle.aggregator).unwrap_or_default(),
);
- if let Some(skip_pages) = skip_pages {
+ if let Some(skip_pages) = query.skip_pages {
ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);
}
- if let Some(region) = selected_region {
+ if let Some(region) = query.selected_region {
if region != Region::All {
ranker = ranker.with_region(region);
}
@@ -117,21 +114,21 @@ impl LocalSearcher {
ranker = ranker.with_max_docs(10_000_000, self.index.num_segments());
ranker.de_rank_similar(de_rank_similar);
- let webpages = self.index.search_initial(&query, ranker.collector())?;
- let correction = self.index.spell_correction(&query.simple_terms());
+ let webpages = self
+ .index
+ .search_initial(&parsed_query, ranker.collector())?;
+ let correction = self.index.spell_correction(&parsed_query.simple_terms());
let entity = self
.entity_index
.as_ref()
.and_then(|index| index.search(&raw_query));
- Ok(InitialSearchResult::Websites(
- InitialWebsitesFormatting::Raw(InitialWebsiteResult {
- spell_corrected_query: correction,
- websites: webpages,
- entity,
- }),
- ))
+ Ok(InitialSearchResult::Websites(InitialWebsiteResult {
+ spell_corrected_query: correction,
+ websites: webpages,
+ entity,
+ }))
}
pub fn retrieve_websites(
@@ -153,22 +150,16 @@ impl LocalSearcher {
self.index.retrieve_websites(websites, &query)
}
- pub fn search(
- &self,
- query: &str,
- selected_region: Option<Region>,
- goggle_program: Option<String>,
- skip_pages: Option<usize>,
- ) -> Result<SearchResult> {
+ pub fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
let start = Instant::now();
- let initial_result =
- self.search_initial(query, selected_region, goggle_program, skip_pages, true)?;
+ let query_text = query.original.clone();
+ let initial_result = self.search_initial(query, true)?;
match initial_result {
- InitialSearchResult::Websites(InitialWebsitesFormatting::Raw(search_result)) => {
+ InitialSearchResult::Websites(search_result) => {
let retrieved_sites =
- self.retrieve_websites(&search_result.websites.top_websites, query)?;
+ self.retrieve_websites(&search_result.websites.top_websites, &query_text)?;
Ok(SearchResult::Websites(WebsitesResult {
spell_corrected_query: search_result.spell_corrected_query,
@@ -181,7 +172,6 @@ impl LocalSearcher {
}))
}
InitialSearchResult::Bang(bang) => Ok(SearchResult::Bang(bang)),
- _ => unreachable!(),
}
}
@@ -264,7 +254,12 @@ mod tests {
for p in 0..NUM_PAGES {
let urls: Vec<_> = searcher
- .search("test", None, None, Some(p))
+ .search(&SearchQuery {
+ original: "test".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: Some(p),
+ })
.unwrap()
.into_websites()
.unwrap()
diff --git a/src/searcher/mod.rs b/src/searcher/mod.rs
index 6f9c3f04..61f0541a 100644
--- a/src/searcher/mod.rs
+++ b/src/searcher/mod.rs
@@ -26,6 +26,7 @@ use crate::{
entity_index::StoredEntity,
inverted_index,
search_prettifier::{self, DisplayedEntity, DisplayedWebpage},
+ webpage::region::Region,
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
@@ -61,12 +62,26 @@ pub struct PrettifiedWebsitesResult {
#[derive(Debug, Serialize, Deserialize)]
pub enum InitialSearchResult {
- Websites(InitialWebsitesFormatting),
+ Websites(local::InitialWebsiteResult),
Bang(BangHit),
}
#[derive(Debug, Serialize, Deserialize)]
-pub enum InitialWebsitesFormatting {
- Raw(local::InitialWebsiteResult),
- Prettified(search_prettifier::InitialWebsiteResult),
+pub enum InitialPrettifiedSearchResult {
+ Websites(search_prettifier::InitialWebsiteResult),
+ Bang(BangHit),
+}
+
+#[derive(Serialize, Deserialize, Clone)]
+pub struct SearchQuery {
+ pub original: String,
+ pub selected_region: Option<Region>,
+ pub goggle_program: Option<String>,
+ pub skip_pages: Option<usize>,
+}
+
+impl SearchQuery {
+ pub fn is_empty(&self) -> bool {
+ self.original.is_empty()
+ }
}
diff --git a/src/snippet.rs b/src/snippet.rs
index 264ca9c4..462befd4 100644
--- a/src/snippet.rs
+++ b/src/snippet.rs
@@ -320,7 +320,11 @@ pub fn generate(
#[cfg(test)]
mod tests {
use super::*;
- use crate::{index::Index, searcher::LocalSearcher, webpage::Webpage};
+ use crate::{
+ index::Index,
+ searcher::{LocalSearcher, SearchQuery},
+ webpage::Webpage,
+ };
use maplit::btreemap;
use tantivy::tokenizer::SimpleTokenizer;
@@ -363,7 +367,12 @@ Survey in 2016, 2017, and 2018."#;
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("rust language", None, None, None)
+ .search(&SearchQuery {
+ original: "rust language".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -400,7 +409,12 @@ Survey in 2016, 2017, and 2018."#;
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("describe", None, None, None)
+ .search(&SearchQuery {
+ original: "describe".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();
@@ -592,7 +606,12 @@ Survey in 2016, 2017, and 2018."#;
let searcher = LocalSearcher::from(index);
let result = searcher
- .search("paradigms", None, None, None)
+ .search(&SearchQuery {
+ original: "paradigms".to_string(),
+ selected_region: None,
+ goggle_program: None,
+ skip_pages: None,
+ })
.expect("Search failed")
.into_websites()
.unwrap();