Skip to content

web search done right

simple json api

Mikkel Denkereee628bea62d
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rs
index a2227c40..8dabe84c 100644
--- a/core/benches/search-preindexed-optic.rs
+++ b/core/benches/search-preindexed-optic.rs
@@ -16,7 +16,7 @@ macro_rules! bench {
b.iter(|| {
$searcher
.search(&SearchQuery {
- original: $query.to_string(),
+ query: $query.to_string(),
optic_program: Some($optic.to_string()),
..Default::default()
})
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rs
index efe2ae82..af256ebc 100644
--- a/core/benches/search-preindexed.rs
+++ b/core/benches/search-preindexed.rs
@@ -17,7 +17,7 @@ macro_rules! bench {
b.iter(|| {
$searcher
.search(&SearchQuery {
- original: $query.to_string(),
+ query: $query.to_string(),
..Default::default()
})
.unwrap()
diff --git a/core/src/frontend/mod.rs b/core/src/frontend/mod.rs
index 5275680a..bcb58874 100644
--- a/core/src/frontend/mod.rs
+++ b/core/src/frontend/mod.rs
@@ -29,6 +29,7 @@ use axum::{
http::StatusCode,
response::{Html, IntoResponse, Response},
routing::get,
+ routing::post,
};
mod about;
@@ -91,6 +92,7 @@ pub fn router(queries_csv_path: &str, shards: Vec<Vec<String>>) -> Result<Router
Ok(Router::new()
.route("/", get(index::route))
.route("/search", get(search::route))
+ .route("/beta/api/search", post(search::api))
.route("/autosuggest", get(autosuggest::route))
.route("/autosuggest/browser", get(autosuggest::browser))
.route("/favicon.ico", get(favicon))
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rs
index eed51e39..a1a1d9ec 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/frontend/search.rs
@@ -17,7 +17,7 @@
use optics::SiteRankings;
use std::{collections::HashMap, sync::Arc};
-use axum::Extension;
+use axum::{Extension, Json};
use axum_macros::debug_handler;
use crate::{
@@ -121,7 +121,7 @@ pub async fn route(
match state
.searcher
.search(&SearchQuery {
- original: query.clone(),
+ query: query.clone(),
selected_region,
optic_program: optic,
offset,
@@ -204,3 +204,24 @@ pub async fn route(
Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
}
}
+
+#[allow(clippy::unused_async)]
+#[allow(clippy::match_wild_err_arm)]
+#[debug_handler]
+pub async fn api(
+ extract::Json(mut query): extract::Json<SearchQuery>,
+ Extension(state): Extension<Arc<State>>,
+) -> impl IntoResponse {
+ query.num_results = query.num_results.min(50 * NUM_RESULTS_PER_PAGE);
+
+ match state.searcher.search(&query).await {
+ Ok(result) => match result {
+ PrettifiedSearchResult::Websites(result) => Json(result).into_response(),
+ PrettifiedSearchResult::Bang(result) => {
+ Redirect::to(&result.redirect_to.full()).into_response()
+ }
+ },
+ Err(searcher::distributed::Error::EmptyQuery) => Redirect::to("/").into_response(),
+ Err(_) => panic!("Search failed"), // TODO: show 500 status to user here
+ }
+}
diff --git a/core/src/index.rs b/core/src/index.rs
index 8784cc16..7273aad2 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -387,7 +387,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
..Default::default()
})
.expect("Search failed")
diff --git a/core/src/kv/mod.rs b/core/src/kv/mod.rs
index cd7a32d4..9a5ca1c0 100644
--- a/core/src/kv/mod.rs
+++ b/core/src/kv/mod.rs
@@ -83,7 +83,7 @@ where
unimplemented!()
}
- fn delete_raw(&mut self, key: &[u8]) {
+ fn delete_raw(&mut self, _key: &[u8]) {
unimplemented!()
}
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 389b8286..42daf788 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -315,7 +315,7 @@ mod tests {
fn not_query() {
let mut index = Index::temporary().expect("Unable to open index");
let query = SearchQuery {
- original: "test -website".to_string(),
+ query: "test -website".to_string(),
..Default::default()
};
@@ -401,7 +401,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let query = SearchQuery {
- original: "test site:first.com".to_string(),
+ query: "test site:first.com".to_string(),
..Default::default()
};
let result = searcher
@@ -415,7 +415,7 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.first.com");
let query = SearchQuery {
- original: "test site:www.first.com".to_string(),
+ query: "test site:www.first.com".to_string(),
..Default::default()
};
let result = searcher
@@ -429,7 +429,7 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.first.com");
let query = SearchQuery {
- original: "test -site:first.com".to_string(),
+ query: "test -site:first.com".to_string(),
..Default::default()
};
let result = searcher
@@ -481,7 +481,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let query = SearchQuery {
- original: "intitle:website".to_string(),
+ query: "intitle:website".to_string(),
..Default::default()
};
let result = searcher
@@ -533,7 +533,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let query = SearchQuery {
- original: "test inurl:forum".to_string(),
+ query: "test inurl:forum".to_string(),
..Default::default()
};
let result = searcher
@@ -615,7 +615,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let query = SearchQuery {
- original: "test site:first.com".to_string(),
+ query: "test site:first.com".to_string(),
..Default::default()
};
let result = searcher
@@ -628,7 +628,7 @@ mod tests {
assert_eq!(result.documents.len(), 0);
let query = SearchQuery {
- original: "test site:the-first.com".to_string(),
+ query: "test site:the-first.com".to_string(),
..Default::default()
};
let result = searcher
@@ -642,7 +642,7 @@ mod tests {
assert_eq!(result.documents[0].url, "https://www.the-first.com");
let query = SearchQuery {
- original: "test site:www.the-first.com".to_string(),
+ query: "test site:www.the-first.com".to_string(),
..Default::default()
};
let result = searcher
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index eee53d67..9c89757c 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -283,7 +283,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
..Default::default()
})
.unwrap()
@@ -298,7 +298,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
Rule {
@@ -323,7 +323,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
Rule {
@@ -417,7 +417,7 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
include_str!("../../../optics/testcases/quickstart.optic").to_string(),
),
@@ -431,7 +431,7 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
include_str!("../../../optics/testcases/hacker_news.optic").to_string(),
),
@@ -445,7 +445,7 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
include_str!("../../../optics/testcases/copycats_removal.optic").to_string(),
),
@@ -559,7 +559,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
DiscardNonMatching;
@@ -729,7 +729,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
Like(Site("www.a.com"));
@@ -850,7 +850,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
DiscardNonMatching;
@@ -875,7 +875,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
DiscardNonMatching;
@@ -900,7 +900,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
DiscardNonMatching;
@@ -925,7 +925,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "website".to_string(),
+ query: "website".to_string(),
optic_program: Some(
r#"
DiscardNonMatching;
@@ -990,7 +990,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "site:stackoverflow.com".to_string(),
+ query: "site:stackoverflow.com".to_string(),
optic_program: Some(
r#"
DiscardNonMatching;
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index 2ccea2c0..a6d3bea2 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -252,7 +252,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let result = searcher
.search(&SearchQuery {
- original: "example".to_string(),
+ query: "example".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -333,7 +333,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let result = searcher
.search(&SearchQuery {
- original: "example".to_string(),
+ query: "example".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -441,7 +441,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let result = searcher
.search(&SearchQuery {
- original: "dr dk".to_string(),
+ query: "dr dk".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -521,7 +521,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let result = searcher
.search(&SearchQuery {
- original: "title".to_string(),
+ query: "title".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -606,7 +606,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let result = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -684,7 +684,7 @@ mod tests {
let searcher = LocalSearcher::from(index);
let result = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -790,7 +790,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "example".to_string(),
+ query: "example".to_string(),
optic_program: Some(
r#"
RankingPipeline {
@@ -813,7 +813,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "example".to_string(),
+ query: "example".to_string(),
optic_program: Some(
r#"
RankingPipeline {
@@ -932,7 +932,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "termA termB".to_string(),
+ query: "termA termB".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -1016,7 +1016,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -1099,7 +1099,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -1267,7 +1267,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "topic_a".to_string(),
+ query: "topic_a".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -1282,7 +1282,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "topic_b".to_string(),
+ query: "topic_b".to_string(),
..Default::default()
})
.expect("Search failed")
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index 66cd855c..665c7230 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -193,7 +193,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
site_rankings: Some(SiteRankings {
liked: vec!["www.first.com".to_string()],
disliked: vec!["www.third.com".to_string()],
@@ -214,7 +214,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
site_rankings: Some(SiteRankings {
liked: vec!["www.first.com".to_string()],
disliked: vec![],
@@ -235,7 +235,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
site_rankings: Some(SiteRankings {
liked: vec![],
disliked: vec!["www.second.com".to_string()],
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 70386ff3..3c80f5cf 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -309,7 +309,7 @@ impl DistributedSearcher {
.map(|(idx, pointer)| (idx, pointer.website.pointer.clone()))
.unzip();
- if let Ok(websites) = shard.retrieve_websites(&pointers, &query.original).await {
+ if let Ok(websites) = shard.retrieve_websites(&pointers, &query.query).await {
for (index, website) in indexes.into_iter().zip(websites.into_iter()) {
retrieved_webpages[index] = Some(website);
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 2b842c14..deca8413 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -104,7 +104,7 @@ impl LocalSearcher {
}
let mut parsed_query = Query::parse(
- &query.original,
+ &query.query,
self.index.schema(),
self.index.tokenizers(),
&query_aggregator,
@@ -231,7 +231,7 @@ impl LocalSearcher {
fn entity_sidebar(&self, query: &SearchQuery) -> Option<Sidebar> {
self.entity_index
.as_ref()
- .and_then(|index| index.search(&query.original))
+ .and_then(|index| index.search(&query.query))
.map(Sidebar::Entity)
}
@@ -245,7 +245,7 @@ impl LocalSearcher {
if num_websites > 0 && !top_websites.is_empty() {
let top = top_websites.remove(0);
if top.score > self.stackoverflow_sidebar_threshold {
- let mut retrieved = self.retrieve_websites(&[top.pointer], &query.original)?;
+ let mut retrieved = self.retrieve_websites(&[top.pointer], &query.query)?;
let retrieved = retrieved.remove(0);
return Ok(Some(Sidebar::StackOverflow {
schema_org: retrieved.schema_org,
@@ -326,7 +326,7 @@ impl LocalSearcher {
.map(|website| website.pointer)
.collect();
- let retrieved_sites = self.retrieve_websites(&pointers, &search_query.original)?;
+ let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;
Ok(SearchResult::Websites(WebsitesResult {
spell_corrected_query: search_result.spell_corrected_query,
webpages: inverted_index::SearchResult {
@@ -437,7 +437,7 @@ mod tests {
for p in 0..NUM_PAGES {
let urls: Vec<_> = searcher
.search(&SearchQuery {
- original: "test".to_string(),
+ query: "test".to_string(),
offset: p * NUM_RESULTS_PER_PAGE,
..Default::default()
})
@@ -479,7 +479,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
- original: "regex parse html".to_string(),
+ query: "regex parse html".to_string(),
..Default::default()
})
.unwrap();
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index ce96059d..f9727f97 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -85,7 +85,7 @@ pub enum InitialPrettifiedSearchResult {
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SearchQuery {
- pub original: String,
+ pub query: String,
pub offset: usize,
pub num_results: usize,
pub selected_region: Option<Region>,
@@ -96,7 +96,7 @@ pub struct SearchQuery {
impl Default for SearchQuery {
fn default() -> Self {
Self {
- original: Default::default(),
+ query: Default::default(),
offset: Default::default(),
num_results: NUM_RESULTS_PER_PAGE,
selected_region: Default::default(),
@@ -108,6 +108,6 @@ impl Default for SearchQuery {
impl SearchQuery {
pub fn is_empty(&self) -> bool {
- self.original.is_empty()
+ self.query.is_empty()
}
}
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 7d498e01..e292cc3c 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -388,7 +388,7 @@ Survey in 2016, 2017, and 2018."#;
let result = searcher
.search(&SearchQuery {
- original: "rust language".to_string(),
+ query: "rust language".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -428,7 +428,7 @@ Survey in 2016, 2017, and 2018."#;
let result = searcher
.search(&SearchQuery {
- original: "describe".to_string(),
+ query: "describe".to_string(),
..Default::default()
})
.expect("Search failed")
@@ -638,7 +638,7 @@ Survey in 2016, 2017, and 2018."#;
let result = searcher
.search(&SearchQuery {
- original: "paradigms".to_string(),
+ query: "paradigms".to_string(),
..Default::default()
})
.expect("Search failed")
diff --git a/core/src/webgraph/centrality/harmonic.rs b/core/src/webgraph/centrality/harmonic.rs
index f8b7525b..64a0c143 100644
--- a/core/src/webgraph/centrality/harmonic.rs
+++ b/core/src/webgraph/centrality/harmonic.rs
@@ -158,7 +158,7 @@ fn calculate_centrality(graph: &Webgraph) -> HashMap<Node, f64> {
}
fn calculate_host(graph: &Webgraph) -> HashMap<Node, f64> {
- calculate_centrality(&graph)
+ calculate_centrality(graph)
}
impl HarmonicCentrality {
diff --git a/core/src/webgraph/centrality/online_harmonic.rs b/core/src/webgraph/centrality/online_harmonic.rs
index 173c7422..87acd6a2 100644
--- a/core/src/webgraph/centrality/online_harmonic.rs
+++ b/core/src/webgraph/centrality/online_harmonic.rs
@@ -257,7 +257,7 @@ impl OnlineHarmonicCentrality {
}
let proxy_nodes: Vec<_> = nodes
- .into_par_iter()
+ .into_iter()
.take(num_proxy_nodes)
.filter(|node| node2id.contains_key(node))
.map(|node| {