Skip to content

web search done right

move spellcheck into separate api endpoint and only correct non-special terms. we don't ant to correct "site:...", "inurl:..." etc. parts of the query.

Mikkel Denkerc34849cae9df
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index 18bc7f7c..f56352ed 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -25,6 +25,7 @@ use utoipa_swagger_ui::SwaggerUi;
search::search,
search::widget,
search::sidebar,
+ search::spellcheck,
webgraph::host::similar,
webgraph::host::knows,
webgraph::host::ingoing_hosts,
@@ -44,6 +45,7 @@ use utoipa_swagger_ui::SwaggerUi;
search::ApiSearchResult,
search::WidgetQuery,
search::SidebarQuery,
+ search::SpellcheckQuery,
crate::searcher::WebsitesResult,
crate::search_prettifier::HighlightedSpellCorrection,
crate::search_prettifier::DisplayedWebpage,
diff --git a/crates/core/src/api/mod.rs b/crates/core/src/api/mod.rs
index e0a6d010..474eff5b 100644
--- a/crates/core/src/api/mod.rs
+++ b/crates/core/src/api/mod.rs
@@ -210,6 +210,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
Router::new()
.route("/api/search/widget", post(search::widget))
.route("/api/search/sidebar", post(search::sidebar))
+ .route("/api/search/spellcheck", post(search::spellcheck))
.route("/api/autosuggest", post(autosuggest::route))
.route("/api/autosuggest/browser", get(autosuggest::browser))
.route("/api/summarize", get(summarize::summarize_route))
diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rs
index bf50db4d..3677fc04 100644
--- a/crates/core/src/api/search.rs
+++ b/crates/core/src/api/search.rs
@@ -187,6 +187,27 @@ pub async fn sidebar(
Json(state.searcher.sidebar(&req.query).await)
}
+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
+pub struct SpellcheckQuery {
+ pub query: String,
+}
+
+#[debug_handler]
+#[utoipa::path(
+ post,
+ path = "/beta/api/search/spellcheck",
+ request_body(content = SpellcheckQuery),
+ responses(
+ (status = 200, description = "The corrected string with the changes highlighted using <b>...<\\b> elements. Returns empty response if there is no correction to be made.", body = Option<HighlightedSpellCorrection>),
+ )
+)]
+pub async fn spellcheck(
+ extract::State(state): extract::State<Arc<State>>,
+ extract::Json(req): extract::Json<SpellcheckQuery>,
+) -> impl IntoResponse {
+ Json(state.searcher.spell_check(&req.query))
+}
+
#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct EntityImageParams {
diff --git a/crates/core/src/query/parser.rs b/crates/core/src/query/parser.rs
index 1651261f..03cbe67c 100644
--- a/crates/core/src/query/parser.rs
+++ b/crates/core/src/query/parser.rs
@@ -52,6 +52,11 @@ impl CompoundAwareTerm {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct SimpleTerm(String);
+impl SimpleTerm {
+ pub fn as_str(&self) -> &str {
+ self.0.as_str()
+ }
+}
impl From<String> for SimpleTerm {
fn from(value: String) -> Self {
diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rs
index 31dd68b9..70a26e88 100644
--- a/crates/core/src/search_prettifier/mod.rs
+++ b/crates/core/src/search_prettifier/mod.rs
@@ -73,7 +73,10 @@ impl From<web_spell::Correction> for HighlightedSpellCorrection {
for term in correction.terms {
match term {
- CorrectionTerm::Corrected(correction) => {
+ CorrectionTerm::Corrected {
+ orig: _,
+ correction,
+ } => {
highlighted
.push_str(&("<b><i>".to_string() + correction.as_str() + "</i></b>"));
raw.push_str(&correction);
diff --git a/crates/core/src/searcher/api/mod.rs b/crates/core/src/searcher/api/mod.rs
index 80d20b83..70333c03 100644
--- a/crates/core/src/searcher/api/mod.rs
+++ b/crates/core/src/searcher/api/mod.rs
@@ -21,7 +21,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::Instant;
-use itertools::intersperse;
+use itertools::{intersperse, Itertools};
use url::Url;
use crate::bangs::{Bang, BangHit};
@@ -277,6 +277,63 @@ where
self.sidebar_manager.sidebar(query).await
}
+ pub fn spell_check(&self, query: &str) -> Option<HighlightedSpellCorrection> {
+ let query = query.to_lowercase();
+
+ let terms = query::parser::parse(&query);
+
+ let simple_query = terms
+ .clone()
+ .into_iter()
+ .filter_map(|term| match *term {
+ query::parser::Term::Simple(t) => Some(String::from(t)),
+ _ => None,
+ })
+ .join(" ");
+
+ let corrections = self
+ .spell_checker
+ .as_ref()
+ .and_then(|s| s.correct(&simple_query, &whatlang::Lang::Eng))?;
+
+ let correction_map: HashMap<String, String> = corrections
+ .terms
+ .into_iter()
+ .filter_map(|t| match t {
+ crate::web_spell::CorrectionTerm::Corrected { orig, correction } => {
+ Some((orig, correction))
+ }
+ crate::web_spell::CorrectionTerm::NotCorrected(_) => None,
+ })
+ .collect();
+
+ let mut correction = crate::web_spell::Correction::empty(query);
+
+ for term in terms {
+ match *term {
+ query::parser::Term::Simple(t) => {
+ if let Some(term_correction) = correction_map.get(t.as_str()) {
+ correction.push(crate::web_spell::CorrectionTerm::Corrected {
+ orig: String::from(t),
+ correction: term_correction.to_string(),
+ });
+ } else {
+ correction.push(crate::web_spell::CorrectionTerm::NotCorrected(
+ String::from(t),
+ ));
+ }
+ }
+ _ => {
+ correction.push(crate::web_spell::CorrectionTerm::NotCorrected(
+ term.to_string(),
+ ));
+ }
+ }
+ }
+
+ Some(HighlightedSpellCorrection::from(correction))
+ }
+
async fn retrieve_webpages(
&self,
query: &str,
@@ -366,12 +423,6 @@ where
self.search_initial_from_live(&search_query),
);
- let spell_corrected_query = self
- .spell_checker
- .as_ref()
- .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))
- .map(HighlightedSpellCorrection::from);
-
let num_docs = initial_results
.iter()
.map(|result| result.local_result.num_websites)
@@ -432,7 +483,6 @@ where
let search_duration_ms = start.elapsed().as_millis();
Ok(WebsitesResult {
- spell_corrected_query,
num_hits: num_docs,
webpages: retrieved_webpages,
search_duration_ms,
diff --git a/crates/core/src/searcher/local.rs b/crates/core/src/searcher/local.rs
index c25af51c..7786c41f 100644
--- a/crates/core/src/searcher/local.rs
+++ b/crates/core/src/searcher/local.rs
@@ -425,7 +425,6 @@ where
Ok(WebsitesResult {
num_hits: search_result.num_websites,
webpages,
- spell_corrected_query: None,
search_duration_ms: start.elapsed().as_millis(),
has_more_results,
})
diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rs
index a7a80d71..0a50dd58 100644
--- a/crates/core/src/searcher/mod.rs
+++ b/crates/core/src/searcher/mod.rs
@@ -26,11 +26,8 @@ use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use crate::{
- bangs::BangHit,
- config::defaults,
- ranking::pipeline::RankingWebsite,
- search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection},
- webpage::region::Region,
+ bangs::BangHit, config::defaults, ranking::pipeline::RankingWebsite,
+ search_prettifier::DisplayedWebpage, webpage::region::Region,
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
@@ -44,7 +41,6 @@ pub enum SearchResult {
#[derive(Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct WebsitesResult {
- pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub webpages: Vec<DisplayedWebpage>,
pub num_hits: Option<usize>,
pub search_duration_ms: u128,
diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rs
index c7e54acf..020b0435 100644
--- a/crates/core/src/web_spell/mod.rs
+++ b/crates/core/src/web_spell/mod.rs
@@ -66,7 +66,7 @@ pub struct Correction {
#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]
pub enum CorrectionTerm {
- Corrected(String),
+ Corrected { orig: String, correction: String },
NotCorrected(String),
}
@@ -74,7 +74,10 @@ impl From<Correction> for String {
fn from(correction: Correction) -> Self {
intersperse(
correction.terms.into_iter().map(|term| match term {
- CorrectionTerm::Corrected(correction) => correction,
+ CorrectionTerm::Corrected {
+ orig: _,
+ correction,
+ } => correction,
CorrectionTerm::NotCorrected(orig) => orig,
}),
" ".to_string(),
diff --git a/crates/core/src/web_spell/spell_checker.rs b/crates/core/src/web_spell/spell_checker.rs
index 1ceb86a8..6a9a308e 100644
--- a/crates/core/src/web_spell/spell_checker.rs
+++ b/crates/core/src/web_spell/spell_checker.rs
@@ -181,7 +181,10 @@ impl LangSpellChecker {
if orig == possible_correction {
res.push(CorrectionTerm::NotCorrected(orig));
} else {
- res.push(CorrectionTerm::Corrected(possible_correction));
+ res.push(CorrectionTerm::Corrected {
+ orig,
+ correction: possible_correction,
+ });
}
}
@@ -263,7 +266,10 @@ mod tests {
if orig == corrected {
res.push(CorrectionTerm::NotCorrected(orig));
} else {
- res.push(CorrectionTerm::Corrected(corrected));
+ res.push(CorrectionTerm::Corrected {
+ orig,
+ correction: corrected,
+ });
}
}
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index aabd071d..9f578657 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -118,6 +118,8 @@ export const api = {
requestJson<ApiSearchResult>('POST', `/beta/api/search`, body, options),
searchSidebar: (body: SidebarQuery, options?: ApiOptions) =>
requestJson<DisplayedSidebar>('POST', `/beta/api/search/sidebar`, body, options),
+ searchSpellcheck: (body: SpellcheckQuery, options?: ApiOptions) =>
+ requestJson<HighlightedSpellCorrection>('POST', `/beta/api/search/spellcheck`, body, options),
searchWidget: (body: WidgetQuery, options?: ApiOptions) =>
requestJson<Widget>('POST', `/beta/api/search/widget`, body, options),
summarize: (
@@ -359,6 +361,9 @@ export type Snippet =
question: StackOverflowQuestion;
type: 'stackOverflowQA';
};
+export type SpellcheckQuery = {
+ query: string;
+};
export type StackOverflowAnswer = {
accepted: boolean;
body: CodeOrText[];
@@ -394,7 +399,6 @@ export type WebsitesResult = {
hasMoreResults: boolean;
numHits?: number;
searchDurationMs: number;
- spellCorrectedQuery?: HighlightedSpellCorrection;
webpages: DisplayedWebpage[];
};
export type Widget =
diff --git a/frontend/src/lib/search.ts b/frontend/src/lib/search.ts
index 6324d48e..02841c37 100644
--- a/frontend/src/lib/search.ts
+++ b/frontend/src/lib/search.ts
@@ -1,4 +1,12 @@
-import type { Region, Widget, DisplayedSidebar, DisplayedWebpage, WebsitesResult, BangHit } from './api';
+import type {
+ Region,
+ Widget,
+ DisplayedSidebar,
+ DisplayedWebpage,
+ WebsitesResult,
+ BangHit,
+ HighlightedSpellCorrection,
+} from './api';
import { decompressRanked, type RankedSites } from './rankings';
export type SearchParams = {
@@ -11,16 +19,17 @@ export type SearchParams = {
host_rankings: RankedSites | undefined;
};
-export type SearchResults =
+export type SearchResults =
| (WebsitesResult & {
- type: 'websites';
- widget?: Widget;
- sidebar?: DisplayedSidebar;
- discussions?: DisplayedWebpage[];
- })
+ type: 'websites';
+ spellCorrection?: HighlightedSpellCorrection;
+ widget?: Widget;
+ sidebar?: DisplayedSidebar;
+ discussions?: DisplayedWebpage[];
+ })
| (BangHit & {
- type: 'bang';
- });
+ type: 'bang';
+ });
export const extractSearchParams = (searchParams: URLSearchParams | FormData): SearchParams => {
const query = (searchParams.get('q') as string | undefined) ?? '';
@@ -31,7 +40,9 @@ export const extractSearchParams = (searchParams: URLSearchParams | FormData): S
| undefined;
const safeSearch = (searchParams.get('ss') as string | undefined) == 'true';
const compressedhost_rankings = (searchParams.get('sr') as string | undefined) || null;
- const host_rankings = compressedhost_rankings ? decompressRanked(compressedhost_rankings) : void 0;
+ const host_rankings = compressedhost_rankings
+ ? decompressRanked(compressedhost_rankings)
+ : void 0;
return {
query,
@@ -81,4 +92,4 @@ Rule {
Matches {
Site("|sh.itjust.works|")
}
-};`;
\ No newline at end of file
+};`;
diff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.ts
index f7f9c473..e8d7db2f 100644
--- a/frontend/src/routes/search/+page.server.ts
+++ b/frontend/src/routes/search/+page.server.ts
@@ -80,11 +80,14 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {
)
: { data: undefined };
- const [websites, widget, sidebar, discussionsRes] = await Promise.all([
+ const { data: spellcheckReq } = api.searchSpellcheck({ query: params.query });
+
+ const [websites, widget, sidebar, discussionsRes, spellCorrection] = await Promise.all([
websitesReq,
widgetReq,
sidebarReq,
discussionsReq,
+ spellcheckReq,
]);
const discussions = discussionsRes?.type == 'websites' ? discussionsRes.webpages : undefined;
@@ -95,6 +98,7 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {
widget,
sidebar,
discussions,
+ spellCorrection,
}
: {
...websites,
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte
index 32c2d928..71be269d 100644
--- a/frontend/src/routes/search/+page.svelte
+++ b/frontend/src/routes/search/+page.svelte
@@ -106,14 +106,14 @@
</div>
</div>
<div class="col-start-1 flex min-w-0 max-w-2xl flex-col space-y-5">
- {#if results.spellCorrectedQuery}
+ {#if results.spellCorrection}
<div>
<div>
Did you mean:{' '}
<a
class="font-medium"
- href="/search?q={encodeURIComponent(results.spellCorrectedQuery.raw)}"
- >{@html results.spellCorrectedQuery.highlighted}</a
+ href="/search?q={encodeURIComponent(results.spellCorrection.raw)}"
+ >{@html results.spellCorrection.highlighted}</a
>
</div>
</div>