web search done right
Discussions optic
Mikkel Denker5a0f2f60f6f7
diff --git a/core/src/frontend/optics.rs b/core/src/frontend/optics.rsindex aa77982d..0209a17f 100644--- a/core/src/frontend/optics.rs+++ b/core/src/frontend/optics.rs@@ -18,7 +18,7 @@ use super::HtmlTemplate;use askama::Template;use axum::response::IntoResponse;-pub const DEFAULT_OPTICS: [OpticLink; 2] = [+pub const DEFAULT_OPTICS: [OpticLink; 3] = [OpticLink {name: "Copycats removal",url: "https://raw.githubusercontent.com/Cuely/sample-optics/main/copycats_removal.optic",@@ -27,6 +27,10 @@ pub const DEFAULT_OPTICS: [OpticLink; 2] = [name: "Hacker News",url: "https://raw.githubusercontent.com/Cuely/sample-optics/main/hacker_news.optic",},+ OpticLink {+ name: "Discussions",+ url: "https://raw.githubusercontent.com/Cuely/sample-optics/main/discussions.optic",+ },];#[derive(Debug, Clone)]diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex bf156f6c..609877d5 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -44,6 +44,7 @@ use axum::{#[template(path = "search/index.html")]struct SearchTemplate {search_result: Vec<DisplayedWebpage>,+ discussions: Option<Vec<DisplayedWebpage>>,query: String,sidebar: Option<Sidebar>,spell_correction: Option<HighlightedSpellCorrection>,@@ -154,6 +155,7 @@ pub async fn route(let template = SearchTemplate {search_result: result.webpages,+ discussions: result.discussions,query,sidebar,spell_correction,diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex 2d081fdb..b35b2c42 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -121,7 +121,7 @@ struct ReRanker<M: CrossEncoder> {impl<M: CrossEncoder> ReRanker<M> {fn new(model: Arc<M>) -> Self {Self {- coefficient: 1.0,+ coefficient: 100.0,model,prev_score: 1.0,query: String::new(),@@ -133,8 +133,9 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {fn score(&self, websites: &mut [T]) {for website in websites {let mut website = website.as_mut_ranking();+ let text = website.title.clone() + ". " + &website.clean_body;website.score = self.prev_score * website.score- + self.coefficient * self.model.run(&self.query, &website.clean_body);+ + self.coefficient * self.model.run(&self.query, &text);}}diff --git a/core/src/schema.rs b/core/src/schema.rsindex 287689f4..3eda45ad 100644--- a/core/src/schema.rs+++ b/core/src/schema.rs@@ -386,7 +386,7 @@ impl Field {pub fn boost(&self) -> Option<f32> {match self {- Field::Text(TextField::Site) => Some(3.0),+ Field::Text(TextField::Site) => Some(2.0),Field::Text(TextField::TitleIfHomepage) => Some(3.0),Field::Text(TextField::DomainIfHomepage) => Some(6.0),Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => Some(30.0),diff --git a/core/src/searcher/discussions.optic b/core/src/searcher/discussions.opticnew file mode 100644index 00000000..f45065e6--- /dev/null+++ b/core/src/searcher/discussions.optic@@ -0,0 +1,20 @@+DiscardNonMatching;++Rule {+ Matches {+ Schema("QAPage"),+ }+};++Rule {+ Matches {+ Schema("DiscussionForumPosting"),+ }+};++Rule {+ Matches {+ Domain("reddit.com"),+ Url("comments"),+ }+}\ No newline at end of filediff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 11a2c57d..030b180a 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -47,7 +47,9 @@ use crate::sonic;use super::{InitialWebsiteResult, SearchQuery, SearchResult};type Result<T> = std::result::Result<T, Error>;+const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 100.0;+const DISCUSSIONS_WIDGET_THRESHOLD: f64 = 10.0;struct RemoteSearcher {addr: SocketAddr,@@ -366,6 +368,65 @@ impl DistributedSearcher {self.bangs.get(&parsed_terms)}+ async fn discussions_widget(+ &self,+ query: &SearchQuery,+ ) -> Result<Option<Vec<DisplayedWebpage>>> {+ if query.optic_program.is_some() || query.offset > 0 {+ return Ok(None);+ }++ const NUM_RESULTS: usize = 4;++ let mut query = SearchQuery {+ query: query.query.clone(),+ num_results: NUM_RESULTS,+ optic_program: Some(include_str!("discussions.optic").to_string()),+ ..Default::default()+ };++ let initial_results = self.search_initial(&query).await;++ if initial_results.is_empty() {+ return Ok(None);+ }++ let num_results: usize = initial_results+ .iter()+ .map(|res| res.local_result.num_websites)+ .sum();++ if num_results < NUM_RESULTS / 2 {+ return Ok(None);+ }++ let results = self.combine_results(initial_results, &mut query)?;++ let scores: Vec<_> = results+ .iter()+ .map(|pointer| pointer.website.score)+ .collect();++ let median = if scores.len() % 2 == 0 {+ (scores[(scores.len() / 2) - 1] + scores[(scores.len() / 2)]) / 2.0+ } else {+ scores[scores.len() / 2]+ };++ if median < DISCUSSIONS_WIDGET_THRESHOLD {+ return Ok(None);+ }++ let result = self+ .retrieve_webpages(&results, &query.query)+ .await+ .into_iter()+ .map(DisplayedWebpage::from)+ .collect();++ Ok(Some(result))+ }+pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {if let Some(bang) = self.check_bangs(query) {return Ok(SearchResult::Bang(bang));@@ -382,6 +443,7 @@ impl DistributedSearcher {let initial_results = self.search_initial(query).await;let sidebar = self.sidebar(&initial_results, query).await?;+ let discussions = self.discussions_widget(query).await?;let spell_corrected_query = initial_results.first()@@ -412,6 +474,7 @@ impl DistributedSearcher {num_hits: num_docs,webpages: retrieved_webpages,sidebar,+ discussions,search_duration_ms: start.elapsed().as_millis(),}))}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 2abf800f..46973b68 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -315,6 +315,7 @@ impl LocalSearcher {.into_iter().map(DisplayedWebpage::from).collect(),+ discussions: None,sidebar: search_result.entity_sidebar.map(Sidebar::Entity),search_duration_ms: start.elapsed().as_millis(),})diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 355bff52..8de162fb 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -44,6 +44,7 @@ pub struct WebsitesResult {pub webpages: Vec<DisplayedWebpage>,pub num_hits: usize,pub sidebar: Option<Sidebar>,+ pub discussions: Option<Vec<DisplayedWebpage>>,pub search_duration_ms: u128,}diff --git a/frontend/src/components/Discussions.astro b/frontend/src/components/Discussions.astronew file mode 100644index 00000000..5ea90662--- /dev/null+++ b/frontend/src/components/Discussions.astro@@ -0,0 +1,45 @@+---+import { askama } from "../askama";+import Hero from "./Hero.astro";++export interface Props {+ discussions: string;+}+---++{+askama.if_("let Some(discussions) = discussions", () => (+<div class="discussions relative overflow-hidden flex flex-col space-y-1.5" id="discussions">+ <div class="text-lg space-x-1 flex mb" id="discussions-header">+ <Hero class="w-4" icon="chat-alt-2" />+ <span>Discussions</span>+ </div>+ <div class="flex flex-col">+ {+ askama.for_("discussion in discussions", () => (+ <div class="discussion overflow-hidden">+ <div>+ <a class="text-sm hover:no-underline" href="{{ discussion.url }} ">+ {askama`discussion.domain`}+ </a>+ </div>+ <a+ class="discussion-title truncate text-md font-medium"+ title="{{ discussion.title }}"+ href="{{ discussion.url }}"+ >+ {askama`discussion.title $ {{lorem.sentence}}`}+ </a>+ </div>+ ))+ }+ </div>+</div>+))+}+<style>+ .discussion-title {+ display: inline-block;+ max-width: calc(100% - 10px);+ }+</style>\ No newline at end of filediff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex ee3cc7b2..3e7b4a55 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -7,6 +7,7 @@ import Hero from "../components/Hero.astro";import OpticSelector from "../components/OpticSelector.astro";import Header from "../components/Header.astro";import RankingModal from "../components/RankingModal.astro";+import Discussions from "../components/Discussions.astro";import { askama } from "../askama";---@@ -62,7 +63,7 @@ import { askama } from "../askama";</div><!-- Search results -->- <div class="col-start-1 flex min-w-0 max-w-4xl flex-col space-y-10">+ <div class="col-start-1 flex min-w-0 max-w-2xl flex-col space-y-10">{askama.if_("let Some(correction) = spell_correction", () => (<div>@@ -79,6 +80,12 @@ import { askama } from "../askama";{askama.for_("item in search_result", () => (+ <>+ {+ askama.if_("loop.index == 3", () => (+ <Discussions discussions="discussions" />+ ))+ }<div class="flex w-full"><div class="flex min-w-0 grow flex-col space-y-1"><div class="flex min-w-0">@@ -110,23 +117,8 @@ import { askama } from "../askama";<Snippet snippet={askama`item.snippet`} /></div></div>- <div class="flex h-full w-20 items-center">- <div class="h-20 w-20 py-1 pl-2">- {askama.if_(- "let Some(val) = item.primary_image_uuid",- <a href="{{ item.url|urlencode }}">- <img- class="h-full w-full rounded-full object-cover"- src={askama.fake(- "/image/{{ val|urlencode }}",- "{{image.avatar}}"- )}- />- </a>- )}- </div>- </div></div>+ </>))}@@ -180,7 +172,7 @@ import { askama } from "../askama";@apply text-blue-800 visited:text-purple-800;max-width: calc(100% - 30px);}-+.url {max-width: calc(100% - 100px);}