web search done right
simple json api
Mikkel Denkereee628bea62d
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rsindex a2227c40..8dabe84c 100644--- a/core/benches/search-preindexed-optic.rs+++ b/core/benches/search-preindexed-optic.rs@@ -16,7 +16,7 @@ macro_rules! bench {b.iter(|| {$searcher.search(&SearchQuery {- original: $query.to_string(),+ query: $query.to_string(),optic_program: Some($optic.to_string()),..Default::default()})diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rsindex efe2ae82..af256ebc 100644--- a/core/benches/search-preindexed.rs+++ b/core/benches/search-preindexed.rs@@ -17,7 +17,7 @@ macro_rules! bench {b.iter(|| {$searcher.search(&SearchQuery {- original: $query.to_string(),+ query: $query.to_string(),..Default::default()}).unwrap()diff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rsindex 5275680a..bcb58874 100644--- a/core/src/frontend/mod.rs+++ b/core/src/frontend/mod.rs@@ -29,6 +29,7 @@ use axum::{http::StatusCode,response::{Html, IntoResponse, Response},routing::get,+ routing::post,};mod about;@@ -91,6 +92,7 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<RouterOk(Router::new().route("/", get(index::route)).route("/search", get(search::route))+ .route("/beta/api/search", post(search::api)).route("/autosuggest", get(autosuggest::route)).route("/autosuggest/browser", get(autosuggest::browser)).route("/favicon.ico", get(favicon))diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex eed51e39..a1a1d9ec 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -17,7 +17,7 @@use optics::SiteRankings;use std::{collections::HashMap, sync::Arc};-use axum::Extension;+use axum::{Extension, Json};use axum_macros::debug_handler;use crate::{@@ -121,7 +121,7 @@ pub async fn route(match state.searcher.search(&SearchQuery {- original: query.clone(),+ query: query.clone(),selected_region,optic_program: optic,offset,@@ -204,3 +204,24 @@ pub async fn route(Err(_) => panic!("Search failed"), // TODO: show 500 status to user here}}++#[allow(clippy::unused_async)]+#[allow(clippy::match_wild_err_arm)]+#[debug_handler]+pub async fn api(+ extract::Json(mut query): extract::Json<SearchQuery>,+ Extension(state): Extension<Arc<State>>,+) -> impl IntoResponse {+ query.num_results = query.num_results.min(50 * NUM_RESULTS_PER_PAGE);++ match state.searcher.search(&query).await {+ Ok(result) => match result {+ PrettifiedSearchResult::Websites(result) => Json(result).into_response(),+ PrettifiedSearchResult::Bang(result) => {+ Redirect::to(&result.redirect_to.full()).into_response()+ }+ },+ Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),+ Err(_) => panic!("Search failed"), // TODO: show 500 status to user here+ }+}diff --git a/core/src/index.rs b/core/src/index.rsindex 8784cc16..7273aad2 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -387,7 +387,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),..Default::default()}).expect("Search failed")diff --git a/core/src/kv/mod.rs b/core/src/kv/mod.rsindex cd7a32d4..9a5ca1c0 100644--- a/core/src/kv/mod.rs+++ b/core/src/kv/mod.rs@@ -83,7 +83,7 @@ whereunimplemented!()}- fn delete_raw(&mut self, key: &[u8]) {+ fn delete_raw(&mut self, _key: &[u8]) {unimplemented!()}diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex 389b8286..42daf788 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -315,7 +315,7 @@ mod tests {fn not_query() {let mut index = Index::temporary().expect("Unable to open index");let query = SearchQuery {- original: "test -website".to_string(),+ query: "test -website".to_string(),..Default::default()};@@ -401,7 +401,7 @@ mod tests {let searcher = LocalSearcher::from(index);let query = SearchQuery {- original: "test site:first.com".to_string(),+ query: "test site:first.com".to_string(),..Default::default()};let result = searcher@@ -415,7 +415,7 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.first.com");let query = SearchQuery {- original: "test site:www.first.com".to_string(),+ query: "test site:www.first.com".to_string(),..Default::default()};let result = searcher@@ -429,7 +429,7 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.first.com");let query = SearchQuery {- original: "test -site:first.com".to_string(),+ query: "test -site:first.com".to_string(),..Default::default()};let result = searcher@@ -481,7 +481,7 @@ mod tests {let searcher = LocalSearcher::from(index);let query = SearchQuery {- original: "intitle:website".to_string(),+ query: "intitle:website".to_string(),..Default::default()};let result = searcher@@ -533,7 +533,7 @@ mod tests {let searcher = LocalSearcher::from(index);let query = SearchQuery {- original: "test inurl:forum".to_string(),+ query: "test inurl:forum".to_string(),..Default::default()};let result = searcher@@ -615,7 +615,7 @@ mod tests {let searcher = LocalSearcher::from(index);let query = SearchQuery {- original: "test site:first.com".to_string(),+ query: "test site:first.com".to_string(),..Default::default()};let result = searcher@@ -628,7 +628,7 @@ mod tests {assert_eq!(result.documents.len(), 0);let query = SearchQuery {- original: "test site:the-first.com".to_string(),+ query: "test site:the-first.com".to_string(),..Default::default()};let result = searcher@@ -642,7 +642,7 @@ mod tests {assert_eq!(result.documents[0].url, "https://www.the-first.com");let query = SearchQuery {- original: "test site:www.the-first.com".to_string(),+ query: "test site:www.the-first.com".to_string(),..Default::default()};let result = searcherdiff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex eee53d67..9c89757c 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -283,7 +283,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),..Default::default()}).unwrap()@@ -298,7 +298,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"Rule {@@ -323,7 +323,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"Rule {@@ -417,7 +417,7 @@ mod tests {let _ = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(include_str!("../../../optics/testcases/quickstart.optic").to_string(),),@@ -431,7 +431,7 @@ mod tests {let _ = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(include_str!("../../../optics/testcases/hacker_news.optic").to_string(),),@@ -445,7 +445,7 @@ mod tests {let _ = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(include_str!("../../../optics/testcases/copycats_removal.optic").to_string(),),@@ -559,7 +559,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"DiscardNonMatching;@@ -729,7 +729,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"Like(Site("www.a.com"));@@ -850,7 +850,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"DiscardNonMatching;@@ -875,7 +875,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"DiscardNonMatching;@@ -900,7 +900,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"DiscardNonMatching;@@ -925,7 +925,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "website".to_string(),+ query: "website".to_string(),optic_program: Some(r#"DiscardNonMatching;@@ -990,7 +990,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "site:stackoverflow.com".to_string(),+ query: "site:stackoverflow.com".to_string(),optic_program: Some(r#"DiscardNonMatching;diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex 2ccea2c0..a6d3bea2 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -252,7 +252,7 @@ mod tests {let searcher = LocalSearcher::from(index);let result = searcher.search(&SearchQuery {- original: "example".to_string(),+ query: "example".to_string(),..Default::default()}).expect("Search failed")@@ -333,7 +333,7 @@ mod tests {let searcher = LocalSearcher::from(index);let result = searcher.search(&SearchQuery {- original: "example".to_string(),+ query: "example".to_string(),..Default::default()}).expect("Search failed")@@ -441,7 +441,7 @@ mod tests {let searcher = LocalSearcher::from(index);let result = searcher.search(&SearchQuery {- original: "dr dk".to_string(),+ query: "dr dk".to_string(),..Default::default()}).expect("Search failed")@@ -521,7 +521,7 @@ mod tests {let searcher = LocalSearcher::from(index);let result = searcher.search(&SearchQuery {- original: "title".to_string(),+ query: "title".to_string(),..Default::default()}).expect("Search failed")@@ -606,7 +606,7 @@ mod tests {let searcher = LocalSearcher::from(index);let result = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),..Default::default()}).expect("Search failed")@@ -684,7 +684,7 @@ mod tests {let searcher = LocalSearcher::from(index);let result = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),..Default::default()}).expect("Search failed")@@ -790,7 +790,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "example".to_string(),+ query: "example".to_string(),optic_program: Some(r#"RankingPipeline {@@ -813,7 +813,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "example".to_string(),+ query: "example".to_string(),optic_program: Some(r#"RankingPipeline {@@ -932,7 +932,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "termA termB".to_string(),+ query: "termA termB".to_string(),..Default::default()}).expect("Search failed")@@ -1016,7 +1016,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),..Default::default()}).expect("Search failed")@@ -1099,7 +1099,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),..Default::default()}).expect("Search failed")@@ -1267,7 +1267,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "topic_a".to_string(),+ query: "topic_a".to_string(),..Default::default()}).expect("Search failed")@@ -1282,7 +1282,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "topic_b".to_string(),+ query: "topic_b".to_string(),..Default::default()}).expect("Search failed")diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex 66cd855c..665c7230 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -193,7 +193,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),site_rankings: Some(SiteRankings {liked: vec!["www.first.com".to_string()],disliked: vec!["www.third.com".to_string()],@@ -214,7 +214,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),site_rankings: Some(SiteRankings {liked: vec!["www.first.com".to_string()],disliked: vec![],@@ -235,7 +235,7 @@ mod tests {let result = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),site_rankings: Some(SiteRankings {liked: vec![],disliked: vec!["www.second.com".to_string()],diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 70386ff3..3c80f5cf 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -309,7 +309,7 @@ impl DistributedSearcher {.map(|(idx, pointer)| (idx, pointer.website.pointer.clone())).unzip();- if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {+ if let Ok(websites) = shard.retrieve_websites(&pointers, &query.query).await {for (index, website) in indexes.into_iter().zip(websites.into_iter()) {retrieved_webpages[index] = Some(website);}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 2b842c14..deca8413 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -104,7 +104,7 @@ impl LocalSearcher {}let mut parsed_query = Query::parse(- &query.original,+ &query.query,self.index.schema(),self.index.tokenizers(),&query_aggregator,@@ -231,7 +231,7 @@ impl LocalSearcher {fn entity_sidebar(&self, query: &SearchQuery) -> Option<Sidebar> {self.entity_index.as_ref()- .and_then(|index| index.search(&query.original))+ .and_then(|index| index.search(&query.query)).map(Sidebar::Entity)}@@ -245,7 +245,7 @@ impl LocalSearcher {if num_websites > 0 && !top_websites.is_empty() {let top = top_websites.remove(0);if top.score > self.stackoverflow_sidebar_threshold {- let mut retrieved = self.retrieve_websites(&[top.pointer], &query.original)?;+ let mut retrieved = self.retrieve_websites(&[top.pointer], &query.query)?;let retrieved = retrieved.remove(0);return Ok(Some(Sidebar::StackOverflow {schema_org: retrieved.schema_org,@@ -326,7 +326,7 @@ impl LocalSearcher {.map(|website| website.pointer).collect();- let retrieved_sites = self.retrieve_websites(&pointers, &search_query.original)?;+ let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;Ok(SearchResult::Websites(WebsitesResult {spell_corrected_query: search_result.spell_corrected_query,webpages: inverted_index::SearchResult {@@ -437,7 +437,7 @@ mod tests {for p in 0..NUM_PAGES {let urls: Vec<_> = searcher.search(&SearchQuery {- original: "test".to_string(),+ query: "test".to_string(),offset: p * NUM_RESULTS_PER_PAGE,..Default::default()})@@ -479,7 +479,7 @@ mod tests {let res = searcher.search(&SearchQuery {- original: "regex parse html".to_string(),+ query: "regex parse html".to_string(),..Default::default()}).unwrap();diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex ce96059d..f9727f97 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -85,7 +85,7 @@ pub enum InitialPrettifiedSearchResult {#[derive(Debug, Serialize, Deserialize, Clone)]pub struct SearchQuery {- pub original: String,+ pub query: String,pub offset: usize,pub num_results: usize,pub selected_region: Option<Region>,@@ -96,7 +96,7 @@ pub struct SearchQuery {impl Default for SearchQuery {fn default() -> Self {Self {- original: Default::default(),+ query: Default::default(),offset: Default::default(),num_results: NUM_RESULTS_PER_PAGE,selected_region: Default::default(),@@ -108,6 +108,6 @@ impl Default for SearchQuery {impl SearchQuery {pub fn is_empty(&self) -> bool {- self.original.is_empty()+ self.query.is_empty()}}diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex 7d498e01..e292cc3c 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -388,7 +388,7 @@ Survey in 2016, 2017, and 2018."#;let result = searcher.search(&SearchQuery {- original: "rust language".to_string(),+ query: "rust language".to_string(),..Default::default()}).expect("Search failed")@@ -428,7 +428,7 @@ Survey in 2016, 2017, and 2018."#;let result = searcher.search(&SearchQuery {- original: "describe".to_string(),+ query: "describe".to_string(),..Default::default()}).expect("Search failed")@@ -638,7 +638,7 @@ Survey in 2016, 2017, and 2018."#;let result = searcher.search(&SearchQuery {- original: "paradigms".to_string(),+ query: "paradigms".to_string(),..Default::default()}).expect("Search failed")diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rsindex f8b7525b..64a0c143 100644--- a/core/src/webgraph/centrality/harmonic.rs+++ b/core/src/webgraph/centrality/harmonic.rs@@ -158,7 +158,7 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {}fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {- calculate_centrality(&graph)+ calculate_centrality(graph)}impl HarmonicCentrality {diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rsindex 173c7422..87acd6a2 100644--- a/core/src/webgraph/centrality/online_harmonic.rs+++ b/core/src/webgraph/centrality/online_harmonic.rs@@ -257,7 +257,7 @@ impl OnlineHarmonicCentrality {}let proxy_nodes: Vec<_> = nodes- .into_par_iter()+ .into_iter().take(num_proxy_nodes).filter(|node| node2id.contains_key(node)).map(|node| {