web search done right
move spellcheck into separate api endpoint and only correct non-special terms. we don't ant to correct "site:...", "inurl:..." etc. parts of the query.
Mikkel Denkerc34849cae9df
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rsindex 18bc7f7c..f56352ed 100644--- a/crates/core/src/api/docs.rs+++ b/crates/core/src/api/docs.rs@@ -25,6 +25,7 @@ use utoipa_swagger_ui::SwaggerUi;search::search,search::widget,search::sidebar,+ search::spellcheck,webgraph::host::similar,webgraph::host::knows,webgraph::host::ingoing_hosts,@@ -44,6 +45,7 @@ use utoipa_swagger_ui::SwaggerUi;search::ApiSearchResult,search::WidgetQuery,search::SidebarQuery,+ search::SpellcheckQuery,crate::searcher::WebsitesResult,crate::search_prettifier::HighlightedSpellCorrection,crate::search_prettifier::DisplayedWebpage,diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rsindex e0a6d010..474eff5b 100644--- a/crates/core/src/api/mod.rs+++ b/crates/core/src/api/mod.rs@@ -210,6 +210,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {Router::new().route("/api/search/widget", post(search::widget)).route("/api/search/sidebar", post(search::sidebar))+ .route("/api/search/spellcheck", post(search::spellcheck)).route("/api/autosuggest", post(autosuggest::route)).route("/api/autosuggest/browser", get(autosuggest::browser)).route("/api/summarize", get(summarize::summarize_route))diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rsindex bf50db4d..3677fc04 100644--- a/crates/core/src/api/search.rs+++ b/crates/core/src/api/search.rs@@ -187,6 +187,27 @@ pub async fn sidebar(Json(state.searcher.sidebar(&req.query).await)}+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+pub struct SpellcheckQuery {+ pub query: String,+}++#[debug_handler]+#[utoipa::path(+ post,+ path = "/beta/api/search/spellcheck",+ request_body(content = SpellcheckQuery),+ responses(+ (status = 200, description = "The corrected string with the changes highlighted using <b>...<\\b> elements. Returns empty response if there is no correction to be made.", body = Option<HighlightedSpellCorrection>),+ )+)]+pub async fn spellcheck(+ extract::State(state): extract::State<Arc<State>>,+ extract::Json(req): extract::Json<SpellcheckQuery>,+) -> impl IntoResponse {+ Json(state.searcher.spell_check(&req.query))+}+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]#[serde(rename_all = "camelCase")]pub struct EntityImageParams {diff --git a/crates/core/src/query/parser.rs b/crates/core/src/query/parser.rsindex 1651261f..03cbe67c 100644--- a/crates/core/src/query/parser.rs+++ b/crates/core/src/query/parser.rs@@ -52,6 +52,11 @@ impl CompoundAwareTerm {#[derive(Debug, Clone, PartialEq, Eq, Hash)]pub struct SimpleTerm(String);+impl SimpleTerm {+ pub fn as_str(&self) -> &str {+ self.0.as_str()+ }+}impl From<String> for SimpleTerm {fn from(value: String) -> Self {diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rsindex 31dd68b9..70a26e88 100644--- a/crates/core/src/search_prettifier/mod.rs+++ b/crates/core/src/search_prettifier/mod.rs@@ -73,7 +73,10 @@ impl From<web_spell::Correction> for HighlightedSpellCorrection {for term in correction.terms {match term {- CorrectionTerm::Corrected(correction) => {+ CorrectionTerm::Corrected {+ orig: _,+ correction,+ } => {highlighted.push_str(&("<b><i>".to_string() + correction.as_str() + "</i></b>"));raw.push_str(&correction);diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rsindex 80d20b83..70333c03 100644--- a/crates/core/src/searcher/api/mod.rs+++ b/crates/core/src/searcher/api/mod.rs@@ -21,7 +21,7 @@ use std::collections::HashMap;use std::sync::Arc;use std::time::Instant;-use itertools::intersperse;+use itertools::{intersperse, Itertools};use url::Url;use crate::bangs::{Bang, BangHit};@@ -277,6 +277,63 @@ whereself.sidebar_manager.sidebar(query).await}+ pub fn spell_check(&self, query: &str) -> Option<HighlightedSpellCorrection> {+ let query = query.to_lowercase();++ let terms = query::parser::parse(&query);++ let simple_query = terms+ .clone()+ .into_iter()+ .filter_map(|term| match *term {+ query::parser::Term::Simple(t) => Some(String::from(t)),+ _ => None,+ })+ .join(" ");++ let corrections = self+ .spell_checker+ .as_ref()+ .and_then(|s| s.correct(&simple_query, &whatlang::Lang::Eng))?;++ let correction_map: HashMap<String, String> = corrections+ .terms+ .into_iter()+ .filter_map(|t| match t {+ crate::web_spell::CorrectionTerm::Corrected { orig, correction } => {+ Some((orig, correction))+ }+ crate::web_spell::CorrectionTerm::NotCorrected(_) => None,+ })+ .collect();++ let mut correction = crate::web_spell::Correction::empty(query);++ for term in terms {+ match *term {+ query::parser::Term::Simple(t) => {+ if let Some(term_correction) = correction_map.get(t.as_str()) {+ correction.push(crate::web_spell::CorrectionTerm::Corrected {+ orig: String::from(t),+ correction: term_correction.to_string(),+ });+ } else {+ correction.push(crate::web_spell::CorrectionTerm::NotCorrected(+ String::from(t),+ ));+ }+ }+ _ => {+ correction.push(crate::web_spell::CorrectionTerm::NotCorrected(+ term.to_string(),+ ));+ }+ }+ }++ Some(HighlightedSpellCorrection::from(correction))+ }+async fn retrieve_webpages(&self,query: &str,@@ -366,12 +423,6 @@ whereself.search_initial_from_live(&search_query),);- let spell_corrected_query = self- .spell_checker- .as_ref()- .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))- .map(HighlightedSpellCorrection::from);-let num_docs = initial_results.iter().map(|result| result.local_result.num_websites)@@ -432,7 +483,6 @@ wherelet search_duration_ms = start.elapsed().as_millis();Ok(WebsitesResult {- spell_corrected_query,num_hits: num_docs,webpages: retrieved_webpages,search_duration_ms,diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rsindex c25af51c..7786c41f 100644--- a/crates/core/src/searcher/local.rs+++ b/crates/core/src/searcher/local.rs@@ -425,7 +425,6 @@ whereOk(WebsitesResult {num_hits: search_result.num_websites,webpages,- spell_corrected_query: None,search_duration_ms: start.elapsed().as_millis(),has_more_results,})diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex a7a80d71..0a50dd58 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -26,11 +26,8 @@ use serde::{Deserialize, Serialize};use utoipa::ToSchema;use crate::{- bangs::BangHit,- config::defaults,- ranking::pipeline::RankingWebsite,- search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection},- webpage::region::Region,+ bangs::BangHit, config::defaults, ranking::pipeline::RankingWebsite,+ search_prettifier::DisplayedWebpage, webpage::region::Region,};pub const NUM_RESULTS_PER_PAGE: usize = 20;@@ -44,7 +41,6 @@ pub enum SearchResult {#[derive(Debug, Serialize, Deserialize, ToSchema)]#[serde(rename_all = "camelCase")]pub struct WebsitesResult {- pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,pub num_hits: Option<usize>,pub search_duration_ms: u128,diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rsindex c7e54acf..020b0435 100644--- a/crates/core/src/web_spell/mod.rs+++ b/crates/core/src/web_spell/mod.rs@@ -66,7 +66,7 @@ pub struct Correction {#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]pub enum CorrectionTerm {- Corrected(String),+ Corrected { orig: String, correction: String },NotCorrected(String),}@@ -74,7 +74,10 @@ impl From<Correction> for String {fn from(correction: Correction) -> Self {intersperse(correction.terms.into_iter().map(|term| match term {- CorrectionTerm::Corrected(correction) => correction,+ CorrectionTerm::Corrected {+ orig: _,+ correction,+ } => correction,CorrectionTerm::NotCorrected(orig) => orig,})," ".to_string(),diff --git a/crates/core/src/web_spell/spell_checker.rs b/crates/core/src/web_spell/spell_checker.rsindex 1ceb86a8..6a9a308e 100644--- a/crates/core/src/web_spell/spell_checker.rs+++ b/crates/core/src/web_spell/spell_checker.rs@@ -181,7 +181,10 @@ impl LangSpellChecker {if orig == possible_correction {res.push(CorrectionTerm::NotCorrected(orig));} else {- res.push(CorrectionTerm::Corrected(possible_correction));+ res.push(CorrectionTerm::Corrected {+ orig,+ correction: possible_correction,+ });}}@@ -263,7 +266,10 @@ mod tests {if orig == corrected {res.push(CorrectionTerm::NotCorrected(orig));} else {- res.push(CorrectionTerm::Corrected(corrected));+ res.push(CorrectionTerm::Corrected {+ orig,+ correction: corrected,+ });}}diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex aabd071d..9f578657 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -118,6 +118,8 @@ export const api = {requestJson<ApiSearchResult>('POST', `/beta/api/search`, body, options),searchSidebar: (body: SidebarQuery, options?: ApiOptions) =>requestJson<DisplayedSidebar>('POST', `/beta/api/search/sidebar`, body, options),+ searchSpellcheck: (body: SpellcheckQuery, options?: ApiOptions) =>+ requestJson<HighlightedSpellCorrection>('POST', `/beta/api/search/spellcheck`, body, options),searchWidget: (body: WidgetQuery, options?: ApiOptions) =>requestJson<Widget>('POST', `/beta/api/search/widget`, body, options),summarize: (@@ -359,6 +361,9 @@ export type Snippet =question: StackOverflowQuestion;type: 'stackOverflowQA';};+export type SpellcheckQuery = {+ query: string;+};export type StackOverflowAnswer = {accepted: boolean;body: CodeOrText[];@@ -394,7 +399,6 @@ export type WebsitesResult = {hasMoreResults: boolean;numHits?: number;searchDurationMs: number;- spellCorrectedQuery?: HighlightedSpellCorrection;webpages: DisplayedWebpage[];};export type Widget =diff --git a/frontend/src/lib/search.ts b/frontend/src/lib/search.tsindex 6324d48e..02841c37 100644--- a/frontend/src/lib/search.ts+++ b/frontend/src/lib/search.ts@@ -1,4 +1,12 @@-import type { Region, Widget, DisplayedSidebar, DisplayedWebpage, WebsitesResult, BangHit } from './api';+import type {+ Region,+ Widget,+ DisplayedSidebar,+ DisplayedWebpage,+ WebsitesResult,+ BangHit,+ HighlightedSpellCorrection,+} from './api';import { decompressRanked, type RankedSites } from './rankings';export type SearchParams = {@@ -11,16 +19,17 @@ export type SearchParams = {host_rankings: RankedSites | undefined;};-export type SearchResults =+export type SearchResults =| (WebsitesResult & {- type: 'websites';- widget?: Widget;- sidebar?: DisplayedSidebar;- discussions?: DisplayedWebpage[];- })+ type: 'websites';+ spellCorrection?: HighlightedSpellCorrection;+ widget?: Widget;+ sidebar?: DisplayedSidebar;+ discussions?: DisplayedWebpage[];+ })| (BangHit & {- type: 'bang';- });+ type: 'bang';+ });export const extractSearchParams = (searchParams: URLSearchParams | FormData): SearchParams => {const query = (searchParams.get('q') as string | undefined) ?? '';@@ -31,7 +40,9 @@ export const extractSearchParams = (searchParams: URLSearchParams | FormData): S| undefined;const safeSearch = (searchParams.get('ss') as string | undefined) == 'true';const compressedhost_rankings = (searchParams.get('sr') as string | undefined) || null;- const host_rankings = compressedhost_rankings ? decompressRanked(compressedhost_rankings) : void 0;+ const host_rankings = compressedhost_rankings+ ? decompressRanked(compressedhost_rankings)+ : void 0;return {query,@@ -81,4 +92,4 @@ Rule {Matches {Site("|sh.itjust.works|")}-};`;\ No newline at end of file+};`;diff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.tsindex f7f9c473..e8d7db2f 100644--- a/frontend/src/routes/search/+page.server.ts+++ b/frontend/src/routes/search/+page.server.ts@@ -80,11 +80,14 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {): { data: undefined };- const [websites, widget, sidebar, discussionsRes] = await Promise.all([+ const { data: spellcheckReq } = api.searchSpellcheck({ query: params.query });++ const [websites, widget, sidebar, discussionsRes, spellCorrection] = await Promise.all([websitesReq,widgetReq,sidebarReq,discussionsReq,+ spellcheckReq,]);const discussions = discussionsRes?.type == 'websites' ? discussionsRes.webpages : undefined;@@ -95,6 +98,7 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {widget,sidebar,discussions,+ spellCorrection,}: {...websites,diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelteindex 32c2d928..71be269d 100644--- a/frontend/src/routes/search/+page.svelte+++ b/frontend/src/routes/search/+page.svelte@@ -106,14 +106,14 @@</div></div><div class="col-start-1 flex min-w-0 max-w-2xl flex-col space-y-5">- {#if results.spellCorrectedQuery}+ {#if results.spellCorrection}<div><div>Did you mean:{' '}<aclass="font-medium"- href="/search?q={encodeURIComponent(results.spellCorrectedQuery.raw)}"- >{@html results.spellCorrectedQuery.highlighted}</a+ href="/search?q={encodeURIComponent(results.spellCorrection.raw)}"+ >{@html results.spellCorrection.highlighted}</a></div></div>