Skip to content

web search done right

refactor snippet into a normal snippet and a rich snippet. a normal snippet should always be returned from the api. an application can then choose to show the rich snippet instead if one is present. this gives more flexibility when building applications on top of stracts api

Mikkel Denker26eb16448271
diff --git a/crates/core/src/api/docs.rs b/crates/core/src/api/docs.rs
index f56352ed..89a08e31 100644
--- a/crates/core/src/api/docs.rs
+++ b/crates/core/src/api/docs.rs
@@ -53,6 +53,7 @@ use utoipa_swagger_ui::SwaggerUi;
crate::search_prettifier::DisplayedAnswer,
crate::search_prettifier::DisplayedSidebar,
crate::search_prettifier::Snippet,
+ crate::search_prettifier::RichSnippet,
crate::search_prettifier::StackOverflowAnswer,
crate::search_prettifier::StackOverflowQuestion,
crate::search_prettifier::CodeOrText,
diff --git a/crates/core/src/ranking/signal.rs b/crates/core/src/ranking/signal.rs
index 2904cb84..fb79110e 100644
--- a/crates/core/src/ranking/signal.rs
+++ b/crates/core/src/ranking/signal.rs
@@ -312,7 +312,7 @@ impl Signal {
Signal::IdfSumDomainIfHomepageNoTokenizer => 0.0036,
Signal::IdfSumTitleIfHomepage => 0.00022,
Signal::Bm25BacklinkText => 0.003,
- Signal::Bm25Keywords => 0.0005,
+ Signal::Bm25Keywords => 0.001,
Signal::CrossEncoderSnippet => 0.17,
Signal::CrossEncoderTitle => 0.17,
Signal::HostCentrality => 0.5,
diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rs
index 70a26e88..64fd9793 100644
--- a/crates/core/src/search_prettifier/mod.rs
+++ b/crates/core/src/search_prettifier/mod.rs
@@ -37,28 +37,22 @@ pub use entity::DisplayedEntity;
pub use self::stack_overflow::{stackoverflow_snippet, StackOverflowAnswer, StackOverflowQuestion};
+#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct Snippet {
+ pub date: Option<String>,
+ pub text: TextSnippet,
+}
+
#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]
#[serde(tag = "type", rename_all = "camelCase")]
-pub enum Snippet {
- Normal {
- date: Option<String>,
- text: TextSnippet,
- },
+pub enum RichSnippet {
StackOverflowQA {
question: StackOverflowQuestion,
answers: Vec<StackOverflowAnswer>,
},
}
-impl Snippet {
- pub fn text(&self) -> Option<&TextSnippet> {
- match self {
- Snippet::Normal { text, .. } => Some(text),
- _ => None,
- }
- }
-}
-
#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct HighlightedSpellCorrection {
@@ -146,6 +140,13 @@ fn prettify_date(date: NaiveDateTime) -> String {
fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {
let last_updated = webpage.updated_time.map(prettify_date);
+ Snippet {
+ date: last_updated,
+ text: webpage.snippet.clone(),
+ }
+}
+
+fn generate_rich_snippet(webpage: &RetrievedWebpage) -> Option<RichSnippet> {
let url = Url::parse(&webpage.url).unwrap();
if url.root_domain().unwrap_or_default() == "stackoverflow.com"
@@ -154,15 +155,12 @@ fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {
.iter()
.any(|item| item.types_contains("QAPage"))
{
- if let Ok(snippet) = stackoverflow_snippet(webpage) {
- return snippet;
+ if let Ok((question, answers)) = stackoverflow_snippet(webpage) {
+ return Some(RichSnippet::StackOverflowQA { question, answers });
}
}
- Snippet::Normal {
- date: last_updated,
- text: webpage.snippet.clone(),
- }
+ None
}
#[derive(Debug, Serialize, Deserialize, ToSchema)]
@@ -174,6 +172,7 @@ pub struct DisplayedWebpage {
pub domain: String,
pub pretty_url: String,
pub snippet: Snippet,
+ pub rich_snippet: Option<RichSnippet>,
pub ranking_signals: Option<HashMap<Signal, SignalScore>>,
pub score: Option<f64>,
pub likely_has_ads: bool,
@@ -193,6 +192,7 @@ pub struct DisplayedAnswer {
impl From<RetrievedWebpage> for DisplayedWebpage {
fn from(webpage: RetrievedWebpage) -> Self {
let snippet = generate_snippet(&webpage);
+ let rich_snippet = generate_rich_snippet(&webpage);
let url = Url::parse(&webpage.url).unwrap();
let domain = url.root_domain().unwrap_or_default().to_string();
@@ -209,6 +209,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
score: None,
likely_has_ads: webpage.likely_has_ads,
likely_has_paywall: webpage.likely_has_paywall,
+ rich_snippet,
}
}
}
diff --git a/crates/core/src/search_prettifier/stack_overflow.rs b/crates/core/src/search_prettifier/stack_overflow.rs
index 25dc8eab..c16c0b78 100644
--- a/crates/core/src/search_prettifier/stack_overflow.rs
+++ b/crates/core/src/search_prettifier/stack_overflow.rs
@@ -24,7 +24,7 @@ use crate::{
Error,
};
-use super::{DisplayedSidebar, Snippet};
+use super::DisplayedSidebar;
use crate::Result;
#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]
@@ -162,7 +162,9 @@ fn limit_chars(passages: &[CodeOrText], limit: usize) -> Vec<CodeOrText> {
res
}
-pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
+pub fn stackoverflow_snippet(
+ webpage: &RetrievedWebpage,
+) -> Result<(StackOverflowQuestion, Vec<StackOverflowAnswer>)> {
let schema_org = &webpage.schema_org;
match schema_org
@@ -225,7 +227,7 @@ pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
.take(3)
.collect();
- Ok(Snippet::StackOverflowQA { question, answers })
+ Ok((question, answers))
}
None => Err(Error::InvalidStackoverflowSchema.into()),
}
diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rs
index bd7065b2..1da0ef53 100644
--- a/crates/core/src/snippet.rs
+++ b/crates/core/src/snippet.rs
@@ -365,18 +365,11 @@ to the project are from community members.[15]
Rust won first place for "most loved programming language" in the Stack Overflow Developer
Survey in 2016, 2017, and 2018."#;
- fn text_snippet(snippet: Snippet) -> TextSnippet {
- match snippet {
- Snippet::Normal { date: _, text } => text,
- _ => panic!("The snippet was not text"),
- }
- }
-
const HIGHLIGHTEN_PREFIX: &str = "<b>";
const HIGHLIGHTEN_POSTFIX: &str = "</b>";
fn highlight(snippet: Snippet) -> String {
- let text = text_snippet(snippet);
+ let text = snippet.text;
text.fragments
.into_iter()
@@ -512,7 +505,7 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn empty_query() {
assert_eq!(
- highlight(Snippet::Normal {
+ highlight(Snippet {
date: None,
text: snippet_string(
"this is a test",
@@ -529,7 +522,7 @@ Survey in 2016, 2017, and 2018."#;
#[test]
fn empty_text() {
assert_eq!(
- highlight(Snippet::Normal {
+ highlight(Snippet {
date: None,
text: snippet_string(
"",
@@ -543,7 +536,7 @@ Survey in 2016, 2017, and 2018."#;
);
assert_eq!(
- highlight(Snippet::Normal {
+ highlight(Snippet {
date: None,
text: snippet_string("", &[], whatlang::Lang::Eng, SnippetConfig::default())
})
@@ -566,7 +559,7 @@ Survey in 2016, 2017, and 2018."#;
terms.insert("thisis".to_string());
assert_eq!(
- highlight(Snippet::Normal {
+ highlight(Snippet {
date: None,
text: snip.build()
})
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index 9f578657..81b7e300 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -267,6 +267,7 @@ export type DisplayedWebpage = {
likelyHasPaywall: boolean;
prettyUrl: string;
rankingSignals?: {};
+ richSnippet?: RichSnippet;
score?: number;
site: string;
snippet: Snippet;
@@ -334,6 +335,11 @@ export type PartOfSpeechMeaning = {
};
export type Region = 'All' | 'Denmark' | 'France' | 'Germany' | 'Spain' | 'US';
export const REGIONS = ['All', 'Denmark', 'France', 'Germany', 'Spain', 'US'] satisfies Region[];
+export type RichSnippet = {
+ answers: StackOverflowAnswer[];
+ question: StackOverflowQuestion;
+ type: 'stackOverflowQA';
+};
export type ScoredHost = {
description?: string;
host: string;
@@ -350,17 +356,10 @@ export type SimilarHostsParams = {
hosts: string[];
topN: number;
};
-export type Snippet =
- | {
- date?: string;
- text: TextSnippet;
- type: 'normal';
- }
- | {
- answers: StackOverflowAnswer[];
- question: StackOverflowQuestion;
- type: 'stackOverflowQA';
- };
+export type Snippet = {
+ date?: string;
+ text: TextSnippet;
+};
export type SpellcheckQuery = {
query: string;
};
diff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.ts
index 45c94a79..85f6e82a 100644
--- a/frontend/src/routes/search/+page.server.ts
+++ b/frontend/src/routes/search/+page.server.ts
@@ -5,11 +5,11 @@ import {
type SearchResults,
} from '$lib/search';
import { redirect } from '@sveltejs/kit';
-import type { Actions } from './$types';
+import type { Actions, PageServerLoadEvent } from './$types';
import { api } from '$lib/api';
import { fetchRemoteOptic } from '$lib/optics';
-export const load = async ({ locals, fetch, url, getClientAddress }) => {
+export const load = async ({ locals, fetch, url, getClientAddress }: PageServerLoadEvent) => {
const searchParams: SearchParams | undefined =
(locals['form'] && extractSearchParams(locals['form'])) || undefined;
diff --git a/frontend/src/routes/search/Discussions.svelte b/frontend/src/routes/search/Discussions.svelte
index 623dbd60..856e1d54 100644
--- a/frontend/src/routes/search/Discussions.svelte
+++ b/frontend/src/routes/search/Discussions.svelte
@@ -59,16 +59,14 @@
</span>
</summary>
- {#if discussion.snippet.type == 'normal'}
- <div class="mb-3 text-sm font-normal text-neutral-focus">
- {#if typeof discussion.snippet.date == 'string'}
- <span class="text-neutral">{discussion.snippet.date}</span>
- {/if}
- <span class="[&:nth-child(2)]:before:content-['—']">
- <TextSnippet snippet={discussion.snippet.text} />
- </span>
- </div>
- {/if}
+ <div class="mb-3 text-sm font-normal text-neutral-focus">
+ {#if typeof discussion.snippet.date == 'string'}
+ <span class="text-neutral">{discussion.snippet.date}</span>
+ {/if}
+ <span class="[&:nth-child(2)]:before:content-['—']">
+ <TextSnippet snippet={discussion.snippet.text} />
+ </span>
+ </div>
</details>
</div>
{/each}
diff --git a/frontend/src/routes/search/Result.svelte b/frontend/src/routes/search/Result.svelte
index 076b1767..c253ccc3 100644
--- a/frontend/src/routes/search/Result.svelte
+++ b/frontend/src/routes/search/Result.svelte
@@ -53,55 +53,51 @@
<AdjustVertical class="text-md" />
</button>
</div>
- <div class="text-sm font-normal text-neutral-focus [&>b]:font-bold">
+ <div class="snippet text-sm font-normal text-neutral-focus [&>b]:font-bold">
{#if $summary}
<Summary url={webpage.url} on:hide={() => clearSummary(webpage)} />
- {:else if webpage.snippet.type == 'normal'}
- <div class="snippet">
- <div class="line-clamp-3">
- <div class="inline">
- <span id="snippet-text" class="snippet-text">
- {#if webpage.likelyHasAds && $markPagesWithAdsStore && webpage.likelyHasPaywall && $markPagesWithPaywallStore}
- <span
- class="rounded border border-primary p-0.5 text-center text-xs text-neutral"
- title="page likely has ads and paywall"
- >
- has ads + paywall
- </span>
- {:else if webpage.likelyHasAds && $markPagesWithAdsStore}
- <span
- class="rounded border border-primary p-0.5 text-center text-xs text-neutral"
- title="page likely has ads"
- >
- has ads
- </span>
- {:else if webpage.likelyHasPaywall && $markPagesWithPaywallStore}
- <span
- class="rounded border border-primary p-0.5 text-center text-xs text-neutral"
- title="page likely has paywall"
- >
- paywall
- </span>
- {/if}
- {#if webpage.snippet.date}
- <span class="text-neutral">
- {webpage.snippet.date}
- </span> -
- {/if}
- <span>
- <TextSnippet snippet={webpage.snippet.text} />
+ {:else if webpage.richSnippet && webpage.richSnippet.type == 'stackOverflowQA'}
+ <StackOverflowSnippet
+ question={webpage.richSnippet.question}
+ answers={webpage.richSnippet.answers}
+ />
+ {:else}
+ <div class="line-clamp-3">
+ <div class="inline">
+ <span id="snippet-text" class="snippet-text">
+ {#if webpage.likelyHasAds && $markPagesWithAdsStore && webpage.likelyHasPaywall && $markPagesWithPaywallStore}
+ <span
+ class="rounded border border-primary p-0.5 text-center text-xs text-neutral"
+ title="page likely has ads and paywall"
+ >
+ has ads + paywall
</span>
+ {:else if webpage.likelyHasAds && $markPagesWithAdsStore}
+ <span
+ class="rounded border border-primary p-0.5 text-center text-xs text-neutral"
+ title="page likely has ads"
+ >
+ has ads
+ </span>
+ {:else if webpage.likelyHasPaywall && $markPagesWithPaywallStore}
+ <span
+ class="rounded border border-primary p-0.5 text-center text-xs text-neutral"
+ title="page likely has paywall"
+ >
+ paywall
+ </span>
+ {/if}
+ {#if webpage.snippet.date}
+ <span class="text-neutral">
+ {webpage.snippet.date}
+ </span> -
+ {/if}
+ <span>
+ <TextSnippet snippet={webpage.snippet.text} />
</span>
- </div>
+ </span>
</div>
</div>
- {:else if webpage.snippet.type == 'stackOverflowQA'}
- <div class="snippet">
- <StackOverflowSnippet
- question={webpage.snippet.question}
- answers={webpage.snippet.answers}
- />
- </div>
{/if}
</div>
</div>