Skip to content

web search done right

Discussions optic

Mikkel Denker5a0f2f60f6f7
diff --git a/core/src/frontend/optics.rs b/core/src/frontend/optics.rs
index aa77982d..0209a17f 100644
--- a/core/src/frontend/optics.rs
+++ b/core/src/frontend/optics.rs
@@ -18,7 +18,7 @@ use super::HtmlTemplate;
use askama::Template;
use axum::response::IntoResponse;
-pub const DEFAULT_OPTICS: [OpticLink; 2] = [
+pub const DEFAULT_OPTICS: [OpticLink; 3] = [
OpticLink {
name: "Copycats removal",
url: "https://raw.githubusercontent.com/Cuely/sample-optics/main/copycats_removal.optic",
@@ -27,6 +27,10 @@ pub const DEFAULT_OPTICS: [OpticLink; 2] = [
name: "Hacker News",
url: "https://raw.githubusercontent.com/Cuely/sample-optics/main/hacker_news.optic",
},
+ OpticLink {
+ name: "Discussions",
+ url: "https://raw.githubusercontent.com/Cuely/sample-optics/main/discussions.optic",
+ },
];
#[derive(Debug, Clone)]
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rs
index bf156f6c..609877d5 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/frontend/search.rs
@@ -44,6 +44,7 @@ use axum::{
#[template(path = "search/index.html")]
struct SearchTemplate {
search_result: Vec<DisplayedWebpage>,
+ discussions: Option<Vec<DisplayedWebpage>>,
query: String,
sidebar: Option<Sidebar>,
spell_correction: Option<HighlightedSpellCorrection>,
@@ -154,6 +155,7 @@ pub async fn route(
let template = SearchTemplate {
search_result: result.webpages,
+ discussions: result.discussions,
query,
sidebar,
spell_correction,
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index 2d081fdb..b35b2c42 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -121,7 +121,7 @@ struct ReRanker<M: CrossEncoder> {
impl<M: CrossEncoder> ReRanker<M> {
fn new(model: Arc<M>) -> Self {
Self {
- coefficient: 1.0,
+ coefficient: 100.0,
model,
prev_score: 1.0,
query: String::new(),
@@ -133,8 +133,9 @@ impl<T: AsRankingWebsite, M: CrossEncoder> Scorer<T> for ReRanker<M> {
fn score(&self, websites: &mut [T]) {
for website in websites {
let mut website = website.as_mut_ranking();
+ let text = website.title.clone() + ". " + &website.clean_body;
website.score = self.prev_score * website.score
- + self.coefficient * self.model.run(&self.query, &website.clean_body);
+ + self.coefficient * self.model.run(&self.query, &text);
}
}
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 287689f4..3eda45ad 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -386,7 +386,7 @@ impl Field {
pub fn boost(&self) -> Option<f32> {
match self {
- Field::Text(TextField::Site) => Some(3.0),
+ Field::Text(TextField::Site) => Some(2.0),
Field::Text(TextField::TitleIfHomepage) => Some(3.0),
Field::Text(TextField::DomainIfHomepage) => Some(6.0),
Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => Some(30.0),
diff --git a/core/src/searcher/discussions.optic b/core/src/searcher/discussions.optic
new file mode 100644
index 00000000..f45065e6
--- /dev/null
+++ b/core/src/searcher/discussions.optic
@@ -0,0 +1,20 @@
+DiscardNonMatching;
+
+Rule {
+ Matches {
+ Schema("QAPage"),
+ }
+};
+
+Rule {
+ Matches {
+ Schema("DiscussionForumPosting"),
+ }
+};
+
+Rule {
+ Matches {
+ Domain("reddit.com"),
+ Url("comments"),
+ }
+}
\ No newline at end of file
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 11a2c57d..030b180a 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -47,7 +47,9 @@ use crate::sonic;
use super::{InitialWebsiteResult, SearchQuery, SearchResult};
type Result<T> = std::result::Result<T, Error>;
+
const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 100.0;
+const DISCUSSIONS_WIDGET_THRESHOLD: f64 = 10.0;
struct RemoteSearcher {
addr: SocketAddr,
@@ -366,6 +368,65 @@ impl DistributedSearcher {
self.bangs.get(&parsed_terms)
}
+ async fn discussions_widget(
+ &self,
+ query: &SearchQuery,
+ ) -> Result<Option<Vec<DisplayedWebpage>>> {
+ if query.optic_program.is_some() || query.offset > 0 {
+ return Ok(None);
+ }
+
+ const NUM_RESULTS: usize = 4;
+
+ let mut query = SearchQuery {
+ query: query.query.clone(),
+ num_results: NUM_RESULTS,
+ optic_program: Some(include_str!("discussions.optic").to_string()),
+ ..Default::default()
+ };
+
+ let initial_results = self.search_initial(&query).await;
+
+ if initial_results.is_empty() {
+ return Ok(None);
+ }
+
+ let num_results: usize = initial_results
+ .iter()
+ .map(|res| res.local_result.num_websites)
+ .sum();
+
+ if num_results < NUM_RESULTS / 2 {
+ return Ok(None);
+ }
+
+ let results = self.combine_results(initial_results, &mut query)?;
+
+ let scores: Vec<_> = results
+ .iter()
+ .map(|pointer| pointer.website.score)
+ .collect();
+
+ let median = if scores.len() % 2 == 0 {
+ (scores[(scores.len() / 2) - 1] + scores[(scores.len() / 2)]) / 2.0
+ } else {
+ scores[scores.len() / 2]
+ };
+
+ if median < DISCUSSIONS_WIDGET_THRESHOLD {
+ return Ok(None);
+ }
+
+ let result = self
+ .retrieve_webpages(&results, &query.query)
+ .await
+ .into_iter()
+ .map(DisplayedWebpage::from)
+ .collect();
+
+ Ok(Some(result))
+ }
+
pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
if let Some(bang) = self.check_bangs(query) {
return Ok(SearchResult::Bang(bang));
@@ -382,6 +443,7 @@ impl DistributedSearcher {
let initial_results = self.search_initial(query).await;
let sidebar = self.sidebar(&initial_results, query).await?;
+ let discussions = self.discussions_widget(query).await?;
let spell_corrected_query = initial_results
.first()
@@ -412,6 +474,7 @@ impl DistributedSearcher {
num_hits: num_docs,
webpages: retrieved_webpages,
sidebar,
+ discussions,
search_duration_ms: start.elapsed().as_millis(),
}))
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 2abf800f..46973b68 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -315,6 +315,7 @@ impl LocalSearcher {
.into_iter()
.map(DisplayedWebpage::from)
.collect(),
+ discussions: None,
sidebar: search_result.entity_sidebar.map(Sidebar::Entity),
search_duration_ms: start.elapsed().as_millis(),
})
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 355bff52..8de162fb 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -44,6 +44,7 @@ pub struct WebsitesResult {
pub webpages: Vec<DisplayedWebpage>,
pub num_hits: usize,
pub sidebar: Option<Sidebar>,
+ pub discussions: Option<Vec<DisplayedWebpage>>,
pub search_duration_ms: u128,
}
diff --git a/frontend/src/components/Discussions.astro b/frontend/src/components/Discussions.astro
new file mode 100644
index 00000000..5ea90662
--- /dev/null
+++ b/frontend/src/components/Discussions.astro
@@ -0,0 +1,45 @@
+---
+import { askama } from "../askama";
+import Hero from "./Hero.astro";
+
+export interface Props {
+ discussions: string;
+}
+---
+
+{
+askama.if_("let Some(discussions) = discussions", () => (
+<div class="discussions relative overflow-hidden flex flex-col space-y-1.5" id="discussions">
+ <div class="text-lg space-x-1 flex mb" id="discussions-header">
+ <Hero class="w-4" icon="chat-alt-2" />
+ <span>Discussions</span>
+ </div>
+ <div class="flex flex-col">
+ {
+ askama.for_("discussion in discussions", () => (
+ <div class="discussion overflow-hidden">
+ <div>
+ <a class="text-sm hover:no-underline" href="{{ discussion.url }} ">
+ {askama`discussion.domain`}
+ </a>
+ </div>
+ <a
+ class="discussion-title truncate text-md font-medium"
+ title="{{ discussion.title }}"
+ href="{{ discussion.url }}"
+ >
+ {askama`discussion.title $ {{lorem.sentence}}`}
+ </a>
+ </div>
+ ))
+ }
+ </div>
+</div>
+))
+}
+<style>
+ .discussion-title {
+ display: inline-block;
+ max-width: calc(100% - 10px);
+ }
+</style>
\ No newline at end of file
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index ee3cc7b2..3e7b4a55 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -7,6 +7,7 @@ import Hero from "../components/Hero.astro";
import OpticSelector from "../components/OpticSelector.astro";
import Header from "../components/Header.astro";
import RankingModal from "../components/RankingModal.astro";
+import Discussions from "../components/Discussions.astro";
import { askama } from "../askama";
---
@@ -62,7 +63,7 @@ import { askama } from "../askama";
</div>
<!-- Search results -->
- <div class="col-start-1 flex min-w-0 max-w-4xl flex-col space-y-10">
+ <div class="col-start-1 flex min-w-0 max-w-2xl flex-col space-y-10">
{
askama.if_("let Some(correction) = spell_correction", () => (
<div>
@@ -79,6 +80,12 @@ import { askama } from "../askama";
{
askama.for_("item in search_result", () => (
+ <>
+ {
+ askama.if_("loop.index == 3", () => (
+ <Discussions discussions="discussions" />
+ ))
+ }
<div class="flex w-full">
<div class="flex min-w-0 grow flex-col space-y-1">
<div class="flex min-w-0">
@@ -110,23 +117,8 @@ import { askama } from "../askama";
<Snippet snippet={askama`item.snippet`} />
</div>
</div>
- <div class="flex h-full w-20 items-center">
- <div class="h-20 w-20 py-1 pl-2">
- {askama.if_(
- "let Some(val) = item.primary_image_uuid",
- <a href="{{ item.url|urlencode }}">
- <img
- class="h-full w-full rounded-full object-cover"
- src={askama.fake(
- "/image/{{ val|urlencode }}",
- "{{image.avatar}}"
- )}
- />
- </a>
- )}
- </div>
- </div>
</div>
+ </>
))
}
@@ -180,7 +172,7 @@ import { askama } from "../askama";
@apply text-blue-800 visited:text-purple-800;
max-width: calc(100% - 30px);
}
-
+
.url {
max-width: calc(100% - 100px);
}