web search done right
make discussions widget optional in api
Mikkel Denkerf7512a5595ea
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex 92a8ee15..936e2b33 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -177,6 +177,7 @@ impl Searcher {return_ranking_signals: false,flatten_response: false,safe_search: Some(false),+ fetch_discussions: false,};tracing::debug!("searching at {:?}: {:#?}", self.url, query);diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex e25e55df..d47f3738 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -65,6 +65,9 @@ pub struct ApiSearchQuery {#[serde(default = "defaults::SearchQuery::flatten_response")]pub flatten_response: bool,++ #[serde(default = "defaults::SearchQuery::fetch_discussions")]+ pub fetch_discussions: bool,}impl TryFrom<ApiSearchQuery> for SearchQuery {@@ -88,6 +91,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {site_rankings: api.site_rankings,return_ranking_signals: api.return_ranking_signals,safe_search: api.safe_search.unwrap_or(default.safe_search),+ fetch_discussions: api.fetch_discussions,})}}diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex 9ad0ae60..03aa17ac 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -128,4 +128,8 @@ impl SearchQuery {pub fn safe_search() -> bool {false}++ pub fn fetch_discussions() -> bool {+ false+ }}diff --git a/core/src/searcher/api.rs b/core/src/searcher/api.rsindex 4b231c58..c09501cd 100644--- a/core/src/searcher/api.rs+++ b/core/src/searcher/api.rs@@ -225,7 +225,7 @@ impl ApiSearcher {&self,query: &SearchQuery,) -> Result<Option<Vec<DisplayedWebpage>>> {- if query.optic.is_some() || query.page > 0 {+ if !query.fetch_discussions || query.optic.is_some() || query.page > 0 {return Ok(None);}@@ -333,6 +333,7 @@ impl ApiSearcher {.await;let sidebar = self.sidebar(&initial_results, query).await?;+let discussions = self.discussions_widget(query).await?;let widget = self.widget(query);diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex fcfab4a9..63a80860 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -69,6 +69,7 @@ pub struct SearchQuery {pub site_rankings: Option<SiteRankings>,pub return_ranking_signals: bool,pub safe_search: bool,+ pub fetch_discussions: bool,}#[derive(Debug, Clone, Serialize, Deserialize)]@@ -94,6 +95,7 @@ impl Default for SearchQuery {site_rankings: Default::default(),return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),safe_search: defaults::SearchQuery::safe_search(),+ fetch_discussions: defaults::SearchQuery::fetch_discussions(),}}}diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex aaac92ad..3439d834 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -145,7 +145,7 @@ export type SearchResults = components['schemas']['ApiSearchResult'];export type WebsitesResult = components['schemas']['WebsitesResult'];export type Webpage = components['schemas']['DisplayedWebpage'];export type TextSnippet = components['schemas']['TextSnippet'];-export type Sidebar = components['schemas']['Sidebar'];+export type Sidebar = components['schemas']['DisplayedSidebar'];export type Entity = components['schemas']['DisplayedEntity'];export type EntitySnippet = components['schemas']['EntitySnippet'];export type Widget = components['schemas']['Widget'];diff --git a/frontend/src/lib/api/schema.d.ts b/frontend/src/lib/api/schema.d.tsindex b14a0d66..96627d13 100644--- a/frontend/src/lib/api/schema.d.ts+++ b/frontend/src/lib/api/schema.d.ts@@ -3,42 +3,39 @@* Do not make direct changes to the file.*/+/** OneOf type helpers */type Without<T, U> = { [P in Exclude<keyof T, keyof U>]?: never };-type XOR<T, U> = T | U extends object ? (Without<T, U> & U) | (Without<U, T> & T) : T | U;-type OneOf<T extends any[]> = T extends [infer Only]- ? Only- : T extends [infer A, infer B, ...infer Rest]- ? OneOf<[XOR<A, B>, ...Rest]>- : never;+type XOR<T, U> = (T | U) extends object ? (Without<T, U> & U) | (Without<U, T> & T) : T | U;+type OneOf<T extends any[]> = T extends [infer Only] ? Only : T extends [infer A, infer B, ...infer Rest] ? OneOf<[XOR<A, B>, ...Rest]> : never;export interface paths {- '/beta/api/alice': {- get: operations['alice_route'];+ "/beta/api/alice": {+ get: operations["alice_route"];};- '/beta/api/autosuggest': {- post: operations['route'];+ "/beta/api/autosuggest": {+ post: operations["route"];};- '/beta/api/explore/export': {- post: operations['explore_export_optic'];+ "/beta/api/explore/export": {+ post: operations["explore_export_optic"];};- '/beta/api/fact_check': {- post: operations['fact_check_route'];+ "/beta/api/fact_check": {+ post: operations["fact_check_route"];};- '/beta/api/search': {- post: operations['api'];+ "/beta/api/search": {+ post: operations["api"];};- '/beta/api/sites/export': {- post: operations['sites_export_optic'];+ "/beta/api/sites/export": {+ post: operations["sites_export_optic"];};- '/beta/api/summarize': {- get: operations['summarize_route'];+ "/beta/api/summarize": {+ get: operations["summarize_route"];};- '/beta/api/webgraph/knows_site': {- post: operations['knows_site'];+ "/beta/api/webgraph/knows_site": {+ post: operations["knows_site"];};- '/beta/api/webgraph/similar_sites': {- post: operations['similar_sites'];+ "/beta/api/webgraph/similar_sites": {+ post: operations["similar_sites"];};}@@ -53,6 +50,7 @@ export interface components {* }*/ApiSearchQuery: {+ fetchDiscussions?: boolean;flattenResponse?: boolean;numResults?: number | null;optic?: string | null;@@ -60,18 +58,16 @@ export interface components {query: string;returnRankingSignals?: boolean;safeSearch?: boolean | null;- selectedRegion?: components['schemas']['Region'] | null;- siteRankings?: components['schemas']['SiteRankings'] | null;- };- ApiSearchResult:- | (components['schemas']['WebsitesResult'] & {- /** @enum {string} */- type: 'websites';- })- | (components['schemas']['BangHit'] & {- /** @enum {string} */- type: 'bang';- });+ selectedRegion?: components["schemas"]["Region"] | null;+ siteRankings?: components["schemas"]["SiteRankings"] | null;+ };+ ApiSearchResult: (components["schemas"]["WebsitesResult"] & {+ /** @enum {string} */+ type: "websites";+ }) | (components["schemas"]["BangHit"] & {+ /** @enum {string} */+ type: "bang";+ });Bang: {c?: string | null;d?: string | null;@@ -82,26 +78,24 @@ export interface components {u: string;};BangHit: {- bang: components['schemas']['Bang'];- redirectTo: components['schemas']['UrlWrapper'];+ bang: components["schemas"]["Bang"];+ redirectTo: components["schemas"]["UrlWrapper"];};Calculation: {- expr: components['schemas']['Expr'];+ expr: components["schemas"]["Expr"];input: string;/** Format: double */result: number;};- CodeOrText:- | {- /** @enum {string} */- type: 'code';- value: string;- }- | {- /** @enum {string} */- type: 'text';- value: string;- };+ CodeOrText: {+ /** @enum {string} */+ type: "code";+ value: string;+ } | {+ /** @enum {string} */+ type: "text";+ value: string;+ };DisplayedAnswer: {answer: string;prettyUrl: string;@@ -111,21 +105,33 @@ export interface components {};DisplayedEntity: {imageBase64?: string | null;- info: (string & components['schemas']['EntitySnippet'])[][];+ info: ((string & components["schemas"]["EntitySnippet"])[])[];/** Format: float */matchScore: number;- relatedEntities: components['schemas']['DisplayedEntity'][];- smallAbstract: components['schemas']['EntitySnippet'];+ relatedEntities: components["schemas"]["DisplayedEntity"][];+ smallAbstract: components["schemas"]["EntitySnippet"];title: string;};+ DisplayedSidebar: {+ /** @enum {string} */+ type: "entity";+ value: components["schemas"]["DisplayedEntity"];+ } | {+ /** @enum {string} */+ type: "stackOverflow";+ value: {+ answer: components["schemas"]["StackOverflowAnswer"];+ title: string;+ };+ };DisplayedWebpage: {domain: string;prettyUrl: string;rankingSignals?: {- [key: string]: components['schemas']['SignalScore'];+ [key: string]: components["schemas"]["SignalScore"];} | null;site: string;- snippet: components['schemas']['Snippet'];+ snippet: components["schemas"]["Snippet"];title: string;url: string;};@@ -133,57 +139,46 @@ export interface components {EncodedEncryptedState: string;EncodedSavedState: string;EntitySnippet: {- fragments: components['schemas']['EntitySnippetFragment'][];- };- EntitySnippetFragment:- | {- /** @enum {string} */- kind: 'normal';- text: string;- }- | {- href: string;- /** @enum {string} */- kind: 'link';- text: string;- };- ExecutionState:- | {- query: string;- /** @enum {string} */- type: 'beginSearch';- }- | {- query: string;- result: components['schemas']['SimplifiedWebsite'][];- /** @enum {string} */- type: 'searchResult';- }- | {- text: string;- /** @enum {string} */- type: 'speaking';- }- | {- state: components['schemas']['EncodedEncryptedState'];- /** @enum {string} */- type: 'done';- };+ fragments: components["schemas"]["EntitySnippetFragment"][];+ };+ EntitySnippetFragment: {+ /** @enum {string} */+ kind: "normal";+ text: string;+ } | {+ href: string;+ /** @enum {string} */+ kind: "link";+ text: string;+ };+ ExecutionState: {+ query: string;+ /** @enum {string} */+ type: "beginSearch";+ } | {+ query: string;+ result: components["schemas"]["SimplifiedWebsite"][];+ /** @enum {string} */+ type: "searchResult";+ } | {+ text: string;+ /** @enum {string} */+ type: "speaking";+ } | {+ state: components["schemas"]["EncodedEncryptedState"];+ /** @enum {string} */+ type: "done";+ };ExploreExportOpticParams: {chosenSites: string[];similarSites: string[];};- Expr: OneOf<- [- {- /** Format: double */- Number: number;- },- {- Op: Record<string, never>[];- },- ]- >;+ Expr: OneOf<[{+ /** Format: double */+ Number: number;+ }, {+ Op: Record<string, never>[];+ }]>;FactCheckParams: {claim: string;evidence: string;@@ -196,38 +191,22 @@ export interface components {highlighted: string;raw: string;};- KnowsSite:- | {- site: string;- /** @enum {string} */- type: 'known';- }- | {- /** @enum {string} */- type: 'unknown';- };+ KnowsSite: {+ site: string;+ /** @enum {string} */+ type: "known";+ } | {+ /** @enum {string} */+ type: "unknown";+ };/** @enum {string} */- Region: 'All' | 'Denmark' | 'France' | 'Germany' | 'Spain' | 'US';+ Region: "All" | "Denmark" | "France" | "Germany" | "Spain" | "US";ScoredSite: {description?: string | null;/** Format: double */score: number;site: string;};- Sidebar:- | {- /** @enum {string} */- type: 'entity';- value: components['schemas']['DisplayedEntity'];- }- | {- /** @enum {string} */- type: 'stackOverflow';- value: {- answer: components['schemas']['StackOverflowAnswer'];- title: string;- };- };SignalScore: {/** Format: double */coefficient: number;@@ -250,70 +229,61 @@ export interface components {liked: string[];};SitesExportOpticParams: {- siteRankings: components['schemas']['SiteRankings'];- };- Snippet:- | {- date?: string | null;- text: components['schemas']['TextSnippet'];- /** @enum {string} */- type: 'normal';- }- | {- answers: components['schemas']['StackOverflowAnswer'][];- question: components['schemas']['StackOverflowQuestion'];- /** @enum {string} */- type: 'stackOverflowQA';- };+ siteRankings: components["schemas"]["SiteRankings"];+ };+ Snippet: ({+ date?: string | null;+ text: components["schemas"]["TextSnippet"];+ /** @enum {string} */+ type: "normal";+ }) | {+ answers: components["schemas"]["StackOverflowAnswer"][];+ question: components["schemas"]["StackOverflowQuestion"];+ /** @enum {string} */+ type: "stackOverflowQA";+ };StackOverflowAnswer: {accepted: boolean;- body: components['schemas']['CodeOrText'][];+ body: components["schemas"]["CodeOrText"][];date: string;/** Format: int32 */upvotes: number;url: string;};StackOverflowQuestion: {- body: components['schemas']['CodeOrText'][];+ body: components["schemas"]["CodeOrText"][];};Suggestion: {highlighted: string;raw: string;};TextSnippet: {- fragments: components['schemas']['TextSnippetFragment'][];- };- TextSnippetFragment:- | {- /** @enum {string} */- kind: 'normal';- text: string;- }- | {- /** @enum {string} */- kind: 'highlighted';- text: string;- };+ fragments: components["schemas"]["TextSnippetFragment"][];+ };+ TextSnippetFragment: {+ kind: components["schemas"]["TextSnippetFragmentKind"];+ text: string;+ };/*** Url* @description Wrapper around `Url` that implements `ToSchema` for `Url`.*/UrlWrapper: string;WebsitesResult: {- directAnswer?: components['schemas']['DisplayedAnswer'] | null;- discussions?: components['schemas']['DisplayedWebpage'][] | null;+ directAnswer?: components["schemas"]["DisplayedAnswer"] | null;+ discussions?: components["schemas"]["DisplayedWebpage"][] | null;hasMoreResults: boolean;numHits: number;searchDurationMs: number;- sidebar?: components['schemas']['Sidebar'] | null;- spellCorrectedQuery?: components['schemas']['HighlightedSpellCorrection'] | null;- webpages: components['schemas']['DisplayedWebpage'][];- widget?: components['schemas']['Widget'] | null;+ sidebar?: components["schemas"]["DisplayedSidebar"] | null;+ spellCorrectedQuery?: components["schemas"]["HighlightedSpellCorrection"] | null;+ webpages: components["schemas"]["DisplayedWebpage"][];+ widget?: components["schemas"]["Widget"] | null;};Widget: {/** @enum {string} */- type: 'calculator';- value: components['schemas']['Calculation'];+ type: "calculator";+ value: components["schemas"]["Calculation"];};};responses: never;@@ -328,19 +298,20 @@ export type $defs = Record<string, never>;export type external = Record<string, never>;export interface operations {+alice_route: {parameters: {query: {message: string;optic?: string | null;- prevState?: components['schemas']['EncodedSavedState'] | null;+ prevState?: components["schemas"]["EncodedSavedState"] | null;};};responses: {/** @description Interact with Alice */200: {content: {- 'application/json': components['schemas']['ExecutionState'];+ "application/json": components["schemas"]["ExecutionState"];};};};@@ -355,7 +326,7 @@ export interface operations {/** @description Autosuggest */200: {content: {- 'application/json': components['schemas']['Suggestion'][];+ "application/json": components["schemas"]["Suggestion"][];};};};@@ -363,14 +334,14 @@ export interface operations {explore_export_optic: {requestBody: {content: {- 'application/json': components['schemas']['ExploreExportOpticParams'];+ "application/json": components["schemas"]["ExploreExportOpticParams"];};};responses: {/** @description Export explored sites as an optic */200: {content: {- 'text/plain': string;+ "text/plain": string;};};};@@ -378,14 +349,14 @@ export interface operations {fact_check_route: {requestBody: {content: {- 'application/json': components['schemas']['FactCheckParams'];+ "application/json": components["schemas"]["FactCheckParams"];};};responses: {/** @description Fact check the given claim against the given evidence */200: {content: {- 'application/json': components['schemas']['FactCheckResponse'];+ "application/json": components["schemas"]["FactCheckResponse"];};};};@@ -393,14 +364,14 @@ export interface operations {api: {requestBody: {content: {- 'application/json': components['schemas']['ApiSearchQuery'];+ "application/json": components["schemas"]["ApiSearchQuery"];};};responses: {/** @description Search results */200: {content: {- 'application/json': components['schemas']['ApiSearchResult'];+ "application/json": components["schemas"]["ApiSearchResult"];};};};@@ -408,14 +379,14 @@ export interface operations {sites_export_optic: {requestBody: {content: {- 'application/json': components['schemas']['SitesExportOpticParams'];+ "application/json": components["schemas"]["SitesExportOpticParams"];};};responses: {/** @description Export site rankings as an optic */200: {content: {- 'text/plain': string;+ "text/plain": string;};};};@@ -431,7 +402,7 @@ export interface operations {/** @description Summarize a website */200: {content: {- 'text/plain': string;+ "text/plain": string;};};};@@ -446,7 +417,7 @@ export interface operations {/** @description Whether the site is known */200: {content: {- 'application/json': components['schemas']['KnowsSite'];+ "application/json": components["schemas"]["KnowsSite"];};};};@@ -454,14 +425,14 @@ export interface operations {similar_sites: {requestBody: {content: {- 'application/json': components['schemas']['SimilarSitesParams'];+ "application/json": components["schemas"]["SimilarSitesParams"];};};responses: {/** @description List of similar sites */200: {content: {- 'application/json': components['schemas']['ScoredSite'][];+ "application/json": components["schemas"]["ScoredSite"][];};};};diff --git a/frontend/src/routes/search/+page.ts b/frontend/src/routes/search/+page.tsindex ea3cd4fc..b15ce62d 100644--- a/frontend/src/routes/search/+page.ts+++ b/frontend/src/routes/search/+page.ts@@ -21,6 +21,7 @@ export const load: PageLoad = async ({ fetch, url }) => {optic: params.optic && (await fetchRemoteOptic({ opticUrl: params.optic, fetch })),selectedRegion: params.selectedRegion,siteRankings: params.siteRankings,+ fetchDiscussions: true,},{ fetch },);