web search done right
Move widget, discussions etc out from api searcher and into separate api endpoints. This creates a better separation between the frontend and backend, and avoids e.g. 'fetch_discussions' abstraction leak. It also allows for these endpoints to be used in different contexts separately without having to perform a full search.
Mikkel Denker14e5466fb8f1
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rsindex 1a6608ba..aec936fb 100644--- a/core/src/alice/mod.rs+++ b/core/src/alice/mod.rs@@ -180,8 +180,6 @@ impl Searcher {return_ranking_signals: false,flatten_response: false,safe_search: Some(false),- fetch_discussions: false,- fetch_sidebar: false,count_results: false,};tracing::debug!("searching at {:?}: {:#?}", self.url, query);diff --git a/core/src/api/autosuggest.rs b/core/src/api/autosuggest.rsindex 9de14e4b..9a1d973e 100644--- a/core/src/api/autosuggest.rs+++ b/core/src/api/autosuggest.rs@@ -52,7 +52,6 @@ pub struct AutosuggestQuery {q: String,}-#[allow(clippy::unused_async)]#[utoipa::path(post,path = "/beta/api/autosuggest",diff --git a/core/src/api/docs.rs b/core/src/api/docs.rsindex 3785f3fa..18bc7f7c 100644--- a/core/src/api/docs.rs+++ b/core/src/api/docs.rs@@ -22,7 +22,9 @@ use utoipa_swagger_ui::SwaggerUi;#[derive(OpenApi)]#[openapi(paths(- search::api,+ search::search,+ search::widget,+ search::sidebar,webgraph::host::similar,webgraph::host::knows,webgraph::host::ingoing_hosts,@@ -40,6 +42,8 @@ use utoipa_swagger_ui::SwaggerUi;optics::HostRankings,search::ApiSearchQuery,search::ApiSearchResult,+ search::WidgetQuery,+ search::SidebarQuery,crate::searcher::WebsitesResult,crate::search_prettifier::HighlightedSpellCorrection,crate::search_prettifier::DisplayedWebpage,diff --git a/core/src/api/hosts.rs b/core/src/api/hosts.rsindex 9a7aee28..cf54d18c 100644--- a/core/src/api/hosts.rs+++ b/core/src/api/hosts.rs@@ -25,7 +25,6 @@ pub struct HostsExportOpticParams {host_rankings: HostRankings,}-#[allow(clippy::unused_async)]#[utoipa::path(post,path = "/beta/api/hosts/export",request_body(content = HostsExportOpticParams),diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rsindex 3a06187d..fb8cd112 100644--- a/core/src/api/improvement.rs+++ b/core/src/api/improvement.rs@@ -37,7 +37,6 @@ pub struct StoreParams {pub urls: Vec<String>,}-#[allow(clippy::unused_async)]pub async fn click(extract::Query(params): extract::Query<ClickParams>,extract::State(state): extract::State<Arc<State>>,@@ -62,7 +61,6 @@ impl TryFrom<StoreParams> for StoredQuery {}}-#[allow(clippy::unused_async)]pub async fn store(extract::State(state): extract::State<Arc<State>>,extract::Json(params): extract::Json<StoreParams>,diff --git a/core/src/api/metrics.rs b/core/src/api/metrics.rsindex 73fdc44d..c2425559 100644--- a/core/src/api/metrics.rs+++ b/core/src/api/metrics.rs@@ -19,7 +19,6 @@ use std::sync::Arc;use axum::{extract, response::IntoResponse};use axum_macros::debug_handler;-#[allow(clippy::unused_async)]#[allow(clippy::match_wild_err_arm)]#[debug_handler]pub async fn route(diff --git a/core/src/api/mod.rs b/core/src/api/mod.rsindex 8ba562fd..78ef9348 100644--- a/core/src/api/mod.rs+++ b/core/src/api/mod.rs@@ -96,7 +96,6 @@ pub struct State {pub cluster: Arc<Cluster>,}-#[allow(clippy::unused_async)]pub async fn favicon() -> impl IntoResponse {Response::builder().status(StatusCode::OK)@@ -201,7 +200,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {Ok(Router::new().merge(Router::new()- .route("/beta/api/search", post(search::api))+ .route("/beta/api/search", post(search::search)).route_layer(middleware::from_fn_with_state(state.clone(), search_metric)).layer(cors_layer()),)@@ -217,6 +216,8 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {.nest("/beta",Router::new()+ .route("/api/search/widget", post(search::widget))+ .route("/api/search/sidebar", post(search::sidebar)).route("/api/autosuggest", post(autosuggest::route)).route("/api/autosuggest/browser", get(autosuggest::browser)).route("/api/summarize", get(summarize::summarize_route))diff --git a/core/src/api/search.rs b/core/src/api/search.rsindex aa853e9b..bf50db4d 100644--- a/core/src/api/search.rs+++ b/core/src/api/search.rs@@ -51,12 +51,6 @@ pub struct ApiSearchQuery {#[serde(default = "defaults::SearchQuery::flatten_response")]pub flatten_response: bool,- #[serde(default = "defaults::SearchQuery::fetch_discussions")]- pub fetch_discussions: bool,-- #[serde(default = "defaults::SearchQuery::fetch_sidebar")]- pub fetch_sidebar: bool,-#[serde(default = "defaults::SearchQuery::count_results")]pub count_results: bool,}@@ -82,8 +76,6 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {host_rankings: api.host_rankings,return_ranking_signals: api.return_ranking_signals,safe_search: api.safe_search.unwrap_or(default.safe_search),- fetch_discussions: api.fetch_discussions,- fetch_sidebar: api.fetch_sidebar,count_results: api.count_results,})}@@ -105,8 +97,6 @@ impl From<SearchResult> for ApiSearchResult {}}-#[allow(clippy::unused_async)]-#[allow(clippy::match_wild_err_arm)]#[debug_handler]#[utoipa::path(post,@@ -116,7 +106,7 @@ impl From<SearchResult> for ApiSearchResult {(status = 200, description = "Search results", body = ApiSearchResult),))]-pub async fn api(+pub async fn search(extract::State(state): extract::State<Arc<State>>,extract::Json(query): extract::Json<ApiSearchQuery>,) -> Result<impl IntoResponse, StatusCode> {@@ -155,6 +145,48 @@ pub async fn api(}}+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+pub struct WidgetQuery {+ pub query: String,+}++#[debug_handler]+#[utoipa::path(+ post,+ path = "/beta/api/search/widget",+ request_body(content = WidgetQuery),+ responses(+ (status = 200, description = "The resulting widget if one matches the query", body = Option<Widget>),+ )+)]+pub async fn widget(+ extract::State(state): extract::State<Arc<State>>,+ extract::Json(req): extract::Json<WidgetQuery>,+) -> impl IntoResponse {+ Json(state.searcher.widget(&req.query).await)+}++#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+pub struct SidebarQuery {+ pub query: String,+}++#[debug_handler]+#[utoipa::path(+ post,+ path = "/beta/api/search/sidebar",+ request_body(content = SidebarQuery),+ responses(+ (status = 200, description = "The sidebar if one matches the query", body = Option<DisplayedSidebar>),+ )+)]+pub async fn sidebar(+ extract::State(state): extract::State<Arc<State>>,+ extract::Json(req): extract::Json<SidebarQuery>,+) -> impl IntoResponse {+ Json(state.searcher.sidebar(&req.query).await)+}+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]#[serde(rename_all = "camelCase")]pub struct EntityImageParams {diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rsindex 1c9faed8..015b8218 100644--- a/core/src/api/summarize.rs+++ b/core/src/api/summarize.rs@@ -67,7 +67,6 @@ async fn summarize(Err::<Sse<BoxStream<_>>, anyhow::Error>(anyhow::anyhow!("Summarization is not enabled"))}-#[allow(clippy::unused_async)]#[utoipa::path(get,path = "/beta/api/summarize",diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rsindex ba8ed83a..a4db60db 100644--- a/core/src/api/webgraph.rs+++ b/core/src/api/webgraph.rs@@ -73,7 +73,6 @@ pub mod host {pub host: String,}- #[allow(clippy::unused_async)]#[utoipa::path(post,path = "/beta/api/webgraph/host/similar",request_body(content = SimilarHostsParams),diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex a15ef45b..ee338dff 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -48,10 +48,6 @@ impl Api {pub fn entity_sidebar() -> f64 {10.0}-- pub fn discussions_widget() -> f64 {- 0.1- }}pub struct Snippet;@@ -129,14 +125,6 @@ impl SearchQuery {false}- pub fn fetch_discussions() -> bool {- false- }-- pub fn fetch_sidebar() -> bool {- false- }-pub fn count_results() -> bool {false}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex 005242cf..c1f7121b 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -175,9 +175,6 @@ pub struct ApiThresholds {#[serde(default = "defaults::Api::entity_sidebar")]pub entity_sidebar: f64,-- #[serde(default = "defaults::Api::discussions_widget")]- pub discussions_widget: f64,}impl Default for ApiThresholds {@@ -185,7 +182,6 @@ impl Default for ApiThresholds {Self {stackoverflow: defaults::Api::stackoverflow(),entity_sidebar: defaults::Api::entity_sidebar(),- discussions_widget: defaults::Api::discussions_widget(),}}}diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex 23d89ae1..a20995fa 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -27,7 +27,7 @@ use crate::{member::{Member, Service},sonic,},- entity_index::EntityIndex,+ entity_index::{EntityIndex, EntityMatch},image_store::Image,index::Index,inverted_index::{self, RetrievedWebpage},@@ -44,6 +44,7 @@ sonic_service!([RetrieveWebsites,Search,+ SearchEntity,GetWebpage,GetHomepageDescriptions,GetEntityImage,@@ -193,6 +194,18 @@ impl sonic::service::Message<SearchService> for GetEntityImage {}}+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct SearchEntity {+ pub query: String,+}++impl sonic::service::Message<SearchService> for SearchEntity {+ type Response = Option<EntityMatch>;+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {+ Ok(server.local_searcher.search_entity(&self.query))+ }+}+pub async fn run(config: config::SearchServerConfig) -> Result<()> {let addr = config.host;let server = SearchService::new(config).await?.bind(addr).await.unwrap();diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 32eeb0ae..ca6bff6f 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -1052,48 +1052,6 @@ mod tests {assert_eq!(res[0].url, "https://b.com/");}- #[test]- fn discussion_optic() {- let mut index = Index::temporary().expect("Unable to open index");-- index- .insert(Webpage {- html: Html::parse(- include_str!("../../testcases/schema_org/infinity_war.html"),- "https://a.com",- )- .unwrap(),- fetch_time_ms: 500,- ..Default::default()- })- .expect("failed to insert webpage");- index.commit().expect("failed to commit index");-- let searcher = LocalSearcher::from(index);- let res = searcher- .search(&SearchQuery {- query: "avengers endgame".to_string(),- ..Default::default()- })- .unwrap()- .webpages;-- assert!(!res.is_empty());- assert_eq!(&res[0].url, "https://a.com/");-- let res = searcher- .search(&SearchQuery {- query: "avengers endgame".to_string(),- optic: Some(- Optic::parse(include_str!("../searcher/api/discussions.optic")).unwrap(),- ),- ..Default::default()- })- .unwrap()- .webpages;- assert!(res.is_empty());- }-#[test]fn special_pattern_syntax() {let mut index = Index::temporary().expect("Unable to open index");diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rsindex fbd3830e..ee269bdc 100644--- a/core/src/ranking/pipeline.rs+++ b/core/src/ranking/pipeline.rs@@ -324,7 +324,11 @@ impl<T: AsRankingWebsite> RankingStage<T> {collector.insert(website);}- collector.into_sorted_vec(self.derank_similar)+ collector+ .into_sorted_vec(self.derank_similar)+ .into_iter()+ .take(top_n)+ .collect()}fn set_query_info(&mut self, query: &SearchQuery) {@@ -442,7 +446,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {}pub fn collector_top_n(&self) -> usize {- self.initial_top_n().max(self.top_n) + self.top_n * self.page+ (self.initial_top_n().max(self.top_n) + self.top_n * self.page) + 1}pub fn initial_top_n(&self) -> usize {@@ -510,7 +514,7 @@ mod tests {20,).unwrap();- assert_eq!(pipeline.collector_top_n(), 20);+ assert_eq!(pipeline.collector_top_n(), 20 + 1);let sample = sample_websites(pipeline.collector_top_n());let res: Vec<_> = pipelinediff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsindex b0468f22..31dd68b9 100644--- a/core/src/search_prettifier/mod.rs+++ b/core/src/search_prettifier/mod.rs@@ -172,6 +172,7 @@ pub struct DisplayedWebpage {pub pretty_url: String,pub snippet: Snippet,pub ranking_signals: Option<HashMap<Signal, SignalScore>>,+ pub score: Option<f64>,pub likely_has_ads: bool,pub likely_has_paywall: bool,}@@ -202,6 +203,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {domain,snippet,ranking_signals: None,+ score: None,likely_has_ads: webpage.likely_has_ads,likely_has_paywall: webpage.likely_has_paywall,}diff --git a/core/src/searcher/api/discussions.optic b/core/src/searcher/api/discussions.opticdeleted file mode 100644index 81ef9092..00000000--- a/core/src/searcher/api/discussions.optic+++ /dev/null@@ -1,39 +0,0 @@-DiscardNonMatching;--Rule {- Matches {- Schema("QAPage"),- }-};--Rule {- Matches {- Schema("DiscussionForumPosting"),- }-};--Rule {- Matches {- Domain("|reddit.com|"),- Url("comments"),- }-};--// source of instances: https://github.com/maltfield/awesome-lemmy-instances-Rule {- Matches {- Site("|lemmy.world|")- }-};--Rule {- Matches {- Site("|lemmy.ml|")- }-};--Rule {- Matches {- Site("|sh.itjust.works|")- }-};\ No newline at end of filediff --git a/core/src/searcher/api/mod.rs b/core/src/searcher/api/mod.rsindex 168aec9f..c12dc8fd 100644--- a/core/src/searcher/api/mod.rs+++ b/core/src/searcher/api/mod.rs@@ -32,9 +32,9 @@ use crate::inverted_index::RetrievedWebpage;use crate::ranking::models::cross_encoder::DummyCrossEncoder;use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite, RetrievedWebpageRanking};use crate::ranking::ALL_SIGNALS;-use crate::search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection};+use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};use crate::web_spell::SpellChecker;-use crate::widgets::Widgets;+use crate::widgets::{Widget, Widgets};use crate::{bangs::Bangs,collector::BucketCollector,@@ -79,10 +79,13 @@ pub fn combine_results() -> (Vec<ScoredWebsitePointer>, bool) {let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);- let mut num_sites: usize = 0;+ let mut has_more = false;for result in initial_results {+ if result.local_result.has_more {+ has_more = true;+ }+for website in result.local_result.websites {- num_sites += 1;let pointer = distributed::ScoredWebsitePointer {website,shard: result.shard,@@ -95,8 +98,11 @@ pub fn combine_results(}for result in live_results {+ if result.local_result.has_more {+ has_more = true;+ }+for website in result.local_result.websites {- num_sites += 1;let pointer = live::ScoredWebsitePointer {website,split_id: result.split_id.clone(),@@ -114,10 +120,7 @@ pub fn combine_results(.take(pipeline.collector_top_n()).collect::<Vec<_>>();- let offset = pipeline.offset();-let res = pipeline.apply(top_websites);- let has_more = num_sites.saturating_sub(offset) > res.len();(res, has_more)}@@ -131,6 +134,8 @@ pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[Scored}}+ website.score = Some(pointer.as_ranking().score);+website.ranking_signals = Some(signals);}}@@ -144,7 +149,7 @@ pub struct ApiSearcher<S, L> {lambda_model: Option<Arc<LambdaMART>>,bangs: Bangs,collector_config: CollectorConfig,- widget_manager: WidgetManager<S>,+ widget_manager: WidgetManager,spell_checker: Option<SpellChecker>,}@@ -156,7 +161,7 @@ pub struct ApiSearcher<S, L> {lambda_model: Option<Arc<LambdaMART>>,bangs: Bangs,collector_config: CollectorConfig,- widget_manager: WidgetManager<S>,+ widget_manager: WidgetManager,spell_checker: Option<SpellChecker>,}@@ -180,16 +185,8 @@ whereSidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());let lambda_model = lambda_model.map(Arc::new);- let qa_model = qa_model.map(Arc::new);-- let widget_manager = WidgetManager::new(- Widgets::new(config.widgets).unwrap(),- Arc::clone(&dist_searcher),- config.thresholds.clone(),- config.collector.clone(),- lambda_model.clone(),- qa_model,- );++ let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());Self {distributed_searcher: dist_searcher,@@ -219,13 +216,7 @@ whereSidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());let lambda_model = lambda_model.map(Arc::new);- let widget_manager = WidgetManager::new(- Widgets::new(config.widgets).unwrap(),- Arc::clone(&dist_searcher),- config.thresholds.clone(),- config.collector.clone(),- lambda_model.clone(),- );+ let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());Self {distributed_searcher: dist_searcher,@@ -279,6 +270,14 @@ whereOk(self.bangs.get(&parsed_terms))}+ pub async fn widget(&self, query: &str) -> Option<Widget> {+ self.widget_manager.widget(query).await+ }++ pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {+ self.sidebar_manager.sidebar(query).await+ }+async fn retrieve_webpages(&self,query: &str,@@ -352,6 +351,7 @@ where}let mut search_query = query.clone();+ let top_n = search_query.num_results;// This pipeline should be created before the first search is performed// so the query knows how many results to fetch from the indices@@ -359,33 +359,19 @@ where&mut search_query,self.lambda_model.clone(),self.collector_config.clone(),- 20,+ top_n,);- let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(++ let (initial_results, live_results) = tokio::join!(self.distributed_searcher.search_initial(&search_query),self.search_initial_from_live(&search_query),- self.widget_manager.widget(query),- self.widget_manager.discussions(query),- self.sidebar_manager.stackoverflow(query),);- let discussions = discussions?;- let stackoverflow = stackoverflow?;- let sidebar = if query.fetch_sidebar {- self.sidebar_manager- .sidebar(&initial_results, stackoverflow)- } else {- None- };-- let spell_corrected_query = if widget.is_none() {- self.spell_checker- .as_ref()- .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))- .map(HighlightedSpellCorrection::from)- } else {- None- };+ let spell_corrected_query = self+ .spell_checker+ .as_ref()+ .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))+ .map(HighlightedSpellCorrection::from);let num_docs = initial_results.iter()@@ -444,20 +430,12 @@ whereadd_ranking_signals(&mut retrieved_webpages, &top_websites);}- let direct_answer = self- .widget_manager- .answer(&query.query, &mut retrieved_webpages);-let search_duration_ms = start.elapsed().as_millis();Ok(WebsitesResult {spell_corrected_query,num_hits: num_docs,webpages: retrieved_webpages,- direct_answer,- sidebar,- widget,- discussions,search_duration_ms,has_more_results,})diff --git a/core/src/searcher/api/sidebar.rs b/core/src/searcher/api/sidebar.rsindex ebc86492..fbc9ad0f 100644--- a/core/src/searcher/api/sidebar.rs+++ b/core/src/searcher/api/sidebar.rs@@ -21,7 +21,7 @@ use optics::Optic;use url::Url;use crate::{- search_prettifier::{create_stackoverflow_sidebar, DisplayedEntity, DisplayedSidebar},+ search_prettifier::{create_stackoverflow_sidebar, DisplayedSidebar},searcher::{distributed, SearchQuery},};@@ -41,13 +41,9 @@ where}}- pub async fn stackoverflow(&self, query: &SearchQuery) -> Result<Option<DisplayedSidebar>> {- if !query.fetch_sidebar {- return Ok(None);- }-+ pub async fn stackoverflow(&self, query: &str) -> Result<Option<DisplayedSidebar>> {let query = SearchQuery {- query: query.query.clone(),+ query: query.to_string(),num_results: 1,optic: Some(Optic::parse(include_str!("stackoverflow.optic")).unwrap()),..Default::default()@@ -93,25 +89,18 @@ whereOk(None)}- pub fn sidebar(- &self,- initial_results: &[distributed::InitialSearchResultShard],- stackoverflow: Option<DisplayedSidebar>,- ) -> Option<DisplayedSidebar> {- let entity = initial_results- .iter()- .filter_map(|res| res.local_result.entity_sidebar.clone())- .map(DisplayedEntity::from)- .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)- .max_by(|a, b| {- a.match_score- .partial_cmp(&b.match_score)- .unwrap_or(Ordering::Equal)- });+ pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {+ let (entity, stackoverflow) = futures::join!(+ self.distributed_searcher.search_entity(query),+ self.stackoverflow(query)+ );- match entity {- Some(entity) => Some(DisplayedSidebar::Entity(entity)),- None => stackoverflow,+ if let Some(entity) = entity {+ if entity.score as f64 > self.thresholds.entity_sidebar {+ return Some(DisplayedSidebar::Entity(entity.into()));+ }}++ stackoverflow.ok().flatten()}}diff --git a/core/src/searcher/api/widget.rs b/core/src/searcher/api/widget.rsindex 53d312c5..4ee358a0 100644--- a/core/src/searcher/api/widget.rs+++ b/core/src/searcher/api/widget.rs@@ -14,184 +14,22 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use std::ops::Range;-use std::sync::Arc;-use itertools::Itertools;-use optics::Optic;--#[cfg(feature = "libtorch")]-use crate::qa_model::QaModel;-use crate::ranking::models::lambdamart::LambdaMART;-use crate::ranking::pipeline::{AsRankingWebsite, RankingPipeline};-use crate::search_prettifier::DisplayedAnswer;-use crate::search_prettifier::DisplayedWebpage;-use crate::searcher::api::{add_ranking_signals, combine_results, ScoredWebsitePointer};-use crate::searcher::{distributed, SearchQuery};-use crate::widgets::Widget;-use crate::{ceil_char_boundary, floor_char_boundary, query, Result};-use crate::{- config::{ApiThresholds, CollectorConfig},- widgets::Widgets,-};-#[cfg(feature = "libtorch")]-pub struct WidgetManager<S> {- widgets: Widgets,- distributed_searcher: Arc<S>,- thresholds: ApiThresholds,- collector_config: CollectorConfig,- lambda_model: Option<Arc<LambdaMART>>,- qa_model: Option<Arc<QaModel>>,-}+use crate::query;+use crate::widgets::{Widget, Widgets};-#[cfg(not(feature = "libtorch"))]-pub struct WidgetManager<S> {+pub struct WidgetManager {widgets: Widgets,- distributed_searcher: Arc<S>,- thresholds: ApiThresholds,- collector_config: CollectorConfig,- lambda_model: Option<Arc<LambdaMART>>,}-impl<S> WidgetManager<S>-where- S: distributed::SearchClient,-{- #[cfg(feature = "libtorch")]- pub fn new(- widgets: Widgets,- distributed_searcher: Arc<S>,- thresholds: ApiThresholds,- collector_config: CollectorConfig,- lambda_model: Option<Arc<LambdaMART>>,- qa_model: Option<Arc<QaModel>>,- ) -> WidgetManager<S> {- Self {- widgets,- distributed_searcher,- thresholds,- collector_config,- lambda_model,- qa_model,- }- }-- #[cfg(not(feature = "libtorch"))]- pub fn new(- widgets: Widgets,- distributed_searcher: Arc<S>,- thresholds: ApiThresholds,- collector_config: CollectorConfig,- lambda_model: Option<Arc<LambdaMART>>,- ) -> WidgetManager<S> {- Self {- widgets,- distributed_searcher,- thresholds,- collector_config,- lambda_model,- }- }-- pub async fn discussions(&self, query: &SearchQuery) -> Result<Option<Vec<DisplayedWebpage>>> {- if !query.fetch_discussions || query.optic.is_some() || query.page > 0 {- return Ok(None);- }-- const NUM_RESULTS: usize = 10;-- let mut query = SearchQuery {- query: query.query.clone(),- num_results: NUM_RESULTS,- optic: Some(Optic::parse(include_str!("discussions.optic")).unwrap()),- host_rankings: query.host_rankings.clone(),- return_ranking_signals: query.return_ranking_signals,- ..Default::default()- };-- // This pipeline should be created before the first search is performed- // so the query knows how many results to fetch from the indices- let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::recall_stage(- &mut query,- self.lambda_model.clone(),- self.collector_config.clone(),- NUM_RESULTS,- );-- let initial_results = self.distributed_searcher.search_initial(&query).await;-- if initial_results.is_empty() {- return Ok(None);- }-- let num_results: usize = initial_results- .iter()- .map(|res| res.local_result.websites.len())- .sum();-- if num_results < NUM_RESULTS / 2 {- return Ok(None);- }-- let (top_websites, _) = combine_results(- self.collector_config.clone(),- initial_results,- vec![],- pipeline,- );-- let scores: Vec<_> = top_websites- .iter()- .map(|pointer| pointer.as_ranking().score)- .collect();-- let median = if scores.len() % 2 == 0 {- (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0- } else {- scores[scores.len() / 2]- };-- if median < self.thresholds.discussions_widget {- return Ok(None);- }-- let mut result: Vec<DisplayedWebpage> = self- .distributed_searcher- .retrieve_webpages(- &top_websites- .clone()- .into_iter()- .filter_map(|pointer| {- if let ScoredWebsitePointer::Normal(p) = pointer {- Some(p)- } else {- None- }- })- .enumerate()- .collect_vec(),- &query.query,- )- .await- .into_iter()- .map(|(_, webpage)| webpage.into_retrieved_webpage())- .map(DisplayedWebpage::from)- .collect();-- if query.return_ranking_signals {- add_ranking_signals(&mut result, &top_websites);- }-- Ok(Some(result))+impl WidgetManager {+ pub fn new(widgets: Widgets) -> WidgetManager {+ Self { widgets }}- pub async fn widget(&self, query: &SearchQuery) -> Option<Widget> {- if query.page > 0 {- return None;- }-- let parsed_terms = query::parser::parse(&query.query);+ pub async fn widget(&self, query: &str) -> Option<Widget> {+ let parsed_terms = query::parser::parse(query);self.widgets.widget(parsed_terms@@ -207,164 +45,4 @@ where.as_str(),)}-- #[cfg(feature = "libtorch")]- pub fn answer(- &self,- query: &str,- webpages: &mut Vec<DisplayedWebpage>,- ) -> Option<DisplayedAnswer> {- self.qa_model.as_ref().and_then(|qa_model| {- let contexts: Vec<_> = webpages- .iter()- .take(1)- .filter_map(|webpage| webpage.snippet.text())- .map(|t| t.fragments.iter().map(|f| f.text()).join(""))- .collect();-- match qa_model.run(query, &contexts) {- Some(answer) => {- let answer_webpage = webpages.remove(answer.context_idx);- let snip = answer_webpage- .snippet- .text()- .unwrap()- .fragments- .iter()- .map(|f| f.text())- .join("");- Some(DisplayedAnswer {- title: answer_webpage.title,- url: answer_webpage.url,- pretty_url: answer_webpage.pretty_url,- snippet: generate_answer_snippet(&snip, answer.offset.clone()),- answer: snip[answer.offset].to_string(),- })- }- None => None,- }- })- }-- #[cfg(not(feature = "libtorch"))]- pub fn answer(- &self,- query: &str,- webpages: &mut Vec<DisplayedWebpage>,- ) -> Option<DisplayedAnswer> {- None- }-}--fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {- let mut best_start = 0;- let mut best_end = 0;- const SNIPPET_LENGTH: usize = 200;-- if body.is_empty() || answer_offset.start > body.len() - 1 {- return body.to_string();- }-- for (idx, _) in body.char_indices().filter(|(_, c)| *c == '.') {- if idx > answer_offset.end + SNIPPET_LENGTH {- break;- }-- if idx < answer_offset.start {- best_start = idx;- }-- if idx > answer_offset.end {- best_end = idx;- }- }-- if (answer_offset.end - best_start > SNIPPET_LENGTH) || (best_start >= best_end) {- if answer_offset.end - answer_offset.start >= SNIPPET_LENGTH {- let end = floor_char_boundary(body, answer_offset.start + SNIPPET_LENGTH);-- return "<b>".to_string() + &body[answer_offset.start..end] + "</b>";- }-- let chars_either_side = (SNIPPET_LENGTH - (answer_offset.end - answer_offset.start)) / 2;-- let start = ceil_char_boundary(- body,- answer_offset- .start- .checked_sub(chars_either_side)- .unwrap_or_default(),- );- let mut end = ceil_char_boundary(body, answer_offset.end + chars_either_side);-- if end >= body.len() {- end = floor_char_boundary(body, body.len());- }-- body[start..answer_offset.start].to_string()- + "<b>"- + &body[answer_offset.clone()]- + "</b>"- + &body[answer_offset.end..end]- } else {- let mut res = body[best_start..answer_offset.start].to_string()- + "<b>"- + &body[answer_offset.clone()]- + "</b>";-- let remaining_chars = SNIPPET_LENGTH - (res.len() - 7);- let end = ceil_char_boundary(body, (remaining_chars + answer_offset.end).min(best_end));-- res += &body[answer_offset.end..end];-- res- }-}--#[cfg(test)]-mod tests {- use super::*;- #[test]- fn test_generate_answer_snippet() {- assert_eq!(- generate_answer_snippet("this is a test", 0..4),- "<b>this</b> is a test".to_string()- );-- assert_eq!(- generate_answer_snippet("this is a test", 0..1000),- "<b>this is a test</b>".to_string()- );- assert_eq!(- generate_answer_snippet("this is a test", 1000..2000),- "this is a test".to_string()- );- let input = r#"-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test- "#;-- let res = generate_answer_snippet(input, 0..500);- assert!(!res.is_empty());- assert!(res.len() > 100);- assert!(res.len() < input.len());- assert!(res.starts_with("<b>"));- assert!(res.ends_with("</b>"));-- assert_eq!(generate_answer_snippet("", 0..2000), "".to_string());- }}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 205bab7a..da2f0403 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -23,6 +23,7 @@ use crate::{ReplicatedClient, Shard, ShardIdentifier, ShardedClient, SpecificShardSelector,},},+ entity_index::EntityMatch,entrypoint::search_server::{self, SearchService},image_store::Image,inverted_index::{RetrievedWebpage, WebsitePointer},@@ -270,6 +271,25 @@ impl SearchClient for DistributedSearcher {Ok(res.pop().flatten())}++ async fn search_entity(&self, query: &str) -> Option<EntityMatch> {+ let client = self.client().await;++ let (_, mut res) = client+ .send(+ &search_server::SearchEntity {+ query: query.to_string(),+ },+ &RandomShardSelector,+ &RandomReplicaSelector,+ )+ .await+ .ok()?+ .pop()+ .unwrap();++ res.pop().flatten()+ }}pub trait SearchClient {@@ -277,19 +297,25 @@ pub trait SearchClient {&self,query: &SearchQuery,) -> impl Future<Output = Vec<InitialSearchResultShard>> + Send;+fn retrieve_webpages(&self,top_websites: &[(usize, ScoredWebsitePointer)],query: &str,) -> impl Future<Output = Vec<(usize, RetrievedWebpageRanking)>> + Send;++ fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;+fn get_webpage(&self,url: &str,) -> impl Future<Output = Result<Option<RetrievedWebpage>>> + Send;+fn get_homepage_descriptions(&self,urls: &[Url],) -> impl Future<Output = HashMap<Url, String>> + Send;+fn get_entity_image(&self,image_id: &str,diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex b04534ce..5d32c8e0 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -33,7 +33,7 @@ use crate::ranking::models::linear::LinearRegression;use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS};use crate::search_ctx::Ctx;-use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage};+use crate::search_prettifier::DisplayedWebpage;use crate::webgraph::Node;use crate::{inverted_index, live_index, Error, Result};@@ -311,9 +311,9 @@ where)?;let pipe_top_n = pipeline.top_n;- let ranking_websites = pipeline.apply(ranking_websites);+ let has_more = ranking_websites.len() > pipe_top_n;- let has_more = pipe_top_n == ranking_websites.len();+ let ranking_websites = pipeline.apply(ranking_websites);Ok(InvertedIndexResult {webpages: ranking_websites,@@ -322,10 +322,10 @@ where})}- fn entity_sidebar(&self, query: &SearchQuery) -> Option<EntityMatch> {+ pub fn search_entity(&self, query: &str) -> Option<EntityMatch> {self.entity_index.as_ref()- .and_then(|index| index.search(&query.query))+ .and_then(|index| index.search(query))}pub fn search_initial(@@ -337,13 +337,11 @@ wherelet ctx = guard.inverted_index().local_search_ctx();let inverted_index_result =self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;- let sidebar = self.entity_sidebar(query);Ok(InitialWebsiteResult {websites: inverted_index_result.webpages,num_websites: inverted_index_result.num_hits,has_more: inverted_index_result.has_more,- entity_sidebar: sidebar,})}@@ -371,8 +369,6 @@ wherepub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {use std::time::Instant;- use crate::search_prettifier::DisplayedSidebar;-let start = Instant::now();let mut search_query = query.clone();@@ -444,14 +440,7 @@ whereOk(WebsitesResult {num_hits: search_result.num_websites,webpages,- discussions: None,- widget: None,spell_corrected_query: None,- direct_answer: None,- sidebar: search_result- .entity_sidebar- .map(DisplayedEntity::from)- .map(DisplayedSidebar::Entity),search_duration_ms: start.elapsed().as_millis(),has_more_results,})diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex b9897445..a7a80d71 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -28,13 +28,9 @@ use utoipa::ToSchema;use crate::{bangs::BangHit,config::defaults,- entity_index::EntityMatch,ranking::pipeline::RankingWebsite,- search_prettifier::{- DisplayedAnswer, DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection,- },+ search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection},webpage::region::Region,- widgets::Widget,};pub const NUM_RESULTS_PER_PAGE: usize = 20;@@ -51,10 +47,6 @@ pub struct WebsitesResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,pub num_hits: Option<usize>,- pub sidebar: Option<DisplayedSidebar>,- pub widget: Option<Widget>,- pub direct_answer: Option<DisplayedAnswer>,- pub discussions: Option<Vec<DisplayedWebpage>>,pub search_duration_ms: u128,pub has_more_results: bool,}@@ -69,8 +61,6 @@ pub struct SearchQuery {pub host_rankings: Option<HostRankings>,pub return_ranking_signals: bool,pub safe_search: bool,- pub fetch_discussions: bool,- pub fetch_sidebar: bool,pub count_results: bool,}@@ -79,7 +69,6 @@ pub struct InitialWebsiteResult {pub num_websites: Option<usize>,pub websites: Vec<RankingWebsite>,pub has_more: bool,- pub entity_sidebar: Option<EntityMatch>,}impl Default for SearchQuery {@@ -96,8 +85,6 @@ impl Default for SearchQuery {host_rankings: Default::default(),return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),safe_search: defaults::SearchQuery::safe_search(),- fetch_discussions: defaults::SearchQuery::fetch_discussions(),- fetch_sidebar: defaults::SearchQuery::fetch_sidebar(),count_results: defaults::SearchQuery::count_results(),}}diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.tsindex f576974a..aabd071d 100644--- a/frontend/src/lib/api/index.ts+++ b/frontend/src/lib/api/index.ts@@ -116,6 +116,10 @@ export const api = {requestPlain('POST', `/beta/api/hosts/export`, body, options),search: (body: ApiSearchQuery, options?: ApiOptions) =>requestJson<ApiSearchResult>('POST', `/beta/api/search`, body, options),+ searchSidebar: (body: SidebarQuery, options?: ApiOptions) =>+ requestJson<DisplayedSidebar>('POST', `/beta/api/search/sidebar`, body, options),+ searchWidget: (body: WidgetQuery, options?: ApiOptions) =>+ requestJson<Widget>('POST', `/beta/api/search/widget`, body, options),summarize: (query: {query: string;@@ -184,8 +188,6 @@ export const api = {export type ApiSearchQuery = {countResults?: boolean;- fetchDiscussions?: boolean;- fetchSidebar?: boolean;flattenResponse?: boolean;hostRankings?: HostRankings;numResults?: number;@@ -263,6 +265,7 @@ export type DisplayedWebpage = {likelyHasPaywall: boolean;prettyUrl: string;rankingSignals?: {};+ score?: number;site: string;snippet: Snippet;title: string;@@ -334,6 +337,9 @@ export type ScoredHost = {host: string;score: number;};+export type SidebarQuery = {+ query: string;+};export type SignalScore = {coefficient: number;value: number;@@ -385,15 +391,11 @@ export type ThesaurusWidget = {};export type UrlWrapper = string;export type WebsitesResult = {- directAnswer?: DisplayedAnswer;- discussions?: DisplayedWebpage[];hasMoreResults: boolean;numHits?: number;searchDurationMs: number;- sidebar?: DisplayedSidebar;spellCorrectedQuery?: HighlightedSpellCorrection;webpages: DisplayedWebpage[];- widget?: Widget;};export type Widget =| {@@ -404,6 +406,9 @@ export type Widget =type: 'thesaurus';value: ThesaurusWidget;};+export type WidgetQuery = {+ query: string;+};export type WordMeaning = {definition: Definition;examples: Example[];diff --git a/frontend/src/lib/search.ts b/frontend/src/lib/search.tsindex af2ff414..6324d48e 100644--- a/frontend/src/lib/search.ts+++ b/frontend/src/lib/search.ts@@ -1,4 +1,4 @@-import type { Region } from './api';+import type { Region, Widget, DisplayedSidebar, DisplayedWebpage, WebsitesResult, BangHit } from './api';import { decompressRanked, type RankedSites } from './rankings';export type SearchParams = {@@ -11,6 +11,17 @@ export type SearchParams = {host_rankings: RankedSites | undefined;};+export type SearchResults =+ | (WebsitesResult & {+ type: 'websites';+ widget?: Widget;+ sidebar?: DisplayedSidebar;+ discussions?: DisplayedWebpage[];+ })+ | (BangHit & {+ type: 'bang';+ });+export const extractSearchParams = (searchParams: URLSearchParams | FormData): SearchParams => {const query = (searchParams.get('q') as string | undefined) ?? '';const currentPage = parseInt((searchParams.get('p') as string | undefined) ?? '1') || 1;@@ -32,3 +43,42 @@ export const extractSearchParams = (searchParams: URLSearchParams | FormData): Shost_rankings,};};++export const discussionsOptic: string = `DiscardNonMatching;++Rule {+ Matches {+ Schema("QAPage"),+ }+};++Rule {+ Matches {+ Schema("DiscussionForumPosting"),+ }+};++Rule {+ Matches {+ Domain("|reddit.com|"),+ Url("comments"),+ }+};++Rule {+ Matches {+ Site("|lemmy.world|")+ }+};++Rule {+ Matches {+ Site("|lemmy.ml|")+ }+};++Rule {+ Matches {+ Site("|sh.itjust.works|")+ }+};`;\ No newline at end of filediff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.tsindex a2b6426c..67356118 100644--- a/frontend/src/routes/search/+page.server.ts+++ b/frontend/src/routes/search/+page.server.ts@@ -1,4 +1,4 @@-import { extractSearchParams, type SearchParams } from '$lib/search';+import { discussionsOptic, extractSearchParams, type SearchParams, type SearchResults } from '$lib/search';import { redirect } from '@sveltejs/kit';import type { Actions } from './$types';import { api } from '$lib/api';@@ -23,7 +23,9 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {redirect(301, '/');}- const { data } = api.search(+ let start = Date.now();++ const { data: websitesReq } = api.search({query: params.query,page: params.currentPage - 1,@@ -31,18 +33,65 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {optic: params.optic && (await fetchRemoteOptic({ opticUrl: params.optic, fetch })),selectedRegion: params.selectedRegion,hostRankings: params.host_rankings,- fetchDiscussions: true,- fetchSidebar: true,countResults: true,},{ fetch, headers: { 'X-Forwarded-For': getClientAddress() } },);- const results = await data;+ const { data: widgetReq } = params.currentPage == 1 ?+ api.searchWidget(+ {+ query: params.query,+ },+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },+ )+ : { data: undefined };++ const { data: sidebarReq } = params.currentPage == 1 ?+ api.searchSidebar(+ {+ query: params.query,+ },+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },+ )+ : { data: undefined };++ const { data: discussionsReq } = params.currentPage == 1 ?+ api.search(+ {+ query: params.query,+ optic: discussionsOptic,+ numResults: 10,+ safeSearch: params.safeSearch,+ selectedRegion: params.selectedRegion,+ hostRankings: params.host_rankings,+ returnRankingSignals: true,+ countResults: false,+ },+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },+ )+ : { data: undefined };++ const [websites, widget, sidebar, discussionsRes] = await Promise.all([websitesReq, widgetReq, sidebarReq, discussionsReq]);+ const discussions = discussionsRes?.type == "websites" ? discussionsRes.webpages : undefined;++ let results: SearchResults = websites.type == "websites" ? {+ ...websites,+ widget,+ sidebar,+ discussions,+ } : {+ ...websites,+ };++ if (results.type == "websites") {+ results.searchDurationMs = Date.now() - start;+ }return { form: searchParams, results };};+export const actions: Actions = {default: async (event) => {const { request } = event;diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelteindex 23b88c0b..867a90f1 100644--- a/frontend/src/routes/search/+page.svelte+++ b/frontend/src/routes/search/+page.svelte@@ -63,10 +63,6 @@modal = void 0;}- // $: {- // results.discussions = results.webpages;- // }-$: {if (browser && results.type == 'websites') updateQueryId({ query, webpages: results.webpages });}@@ -126,10 +122,6 @@<Widget widget={results.widget} />{/if}- {#if results.directAnswer}- <DirectAnswer directAnswer={results.directAnswer} />- {/if}-{#if results.webpages}<div class="grid grid-cols-1 space-y-10 place-self-start">{#each results.webpages as webpage, resultIndex (`${query}-${webpage.url}`)}@@ -138,9 +130,7 @@</div>{/each}{#if results.discussions}- <div class="row-start-5">- <Discussions discussions={results.discussions} />- </div>+ <Discussions discussions={results.discussions} />{/if}</div>{/if}diff --git a/frontend/src/routes/search/+page.ts b/frontend/src/routes/search/+page.tsindex 4469dd15..cc9d85f4 100644--- a/frontend/src/routes/search/+page.ts+++ b/frontend/src/routes/search/+page.ts@@ -34,6 +34,7 @@ export const load: PageLoad = async (req) => {return newParams;}).otherwise(() => {});+const nextPageSearchParams = match(results.type == 'websites' && results.hasMoreResults).with(true, () => {const newParams = new URLSearchParams(url.searchParams);diff --git a/frontend/src/routes/search/Discussions.svelte b/frontend/src/routes/search/Discussions.svelteindex 286ab087..a85e641a 100644--- a/frontend/src/routes/search/Discussions.svelte+++ b/frontend/src/routes/search/Discussions.svelte@@ -5,14 +5,34 @@import TextSnippet from '$lib/components/TextSnippet.svelte';import Button from '$lib/components/Button.svelte';- export let discussions: DisplayedWebpage[];+ export let discussions: DisplayedWebpage[] | undefined;let showMore = false;- $: shownDiscussions = showMore ? discussions : discussions.slice(0, 4);+ $: {+ let medianScore = 0;+ if (discussions) {+ const scores = discussions.map((d) => d.score || 0);+ scores.sort((a, b) => a - b);+ const mid = Math.floor(scores.length / 2);+ medianScore = scores.length % 2 !== 0 ? scores[mid] : (scores[mid - 1] + scores[mid]) / 2;+ }++ if (medianScore < 0.1) {+ discussions = undefined;+ }++ let numDiscussions = discussions?.length || 0;+ if (numDiscussions < 5) {+ discussions = undefined;+ }+ }++ $: shownDiscussions = showMore ? discussions : discussions?.slice(0, 4);</script>-<div class="flex flex-col space-y-1.5 overflow-hidden">+{#if shownDiscussions && shownDiscussions.length > 0}+<div class="flex flex-col space-y-1.5 overflow-hidden row-start-5"><div class="flex items-center space-x-1 text-lg"><ChatBubbleLeftRight class="text-sm text-neutral" /><span>Discussions</span>@@ -63,4 +83,5 @@summary::-webkit-details-marker {display: none;}-</style>\ No newline at end of file+</style>+{/if}\ No newline at end of filediff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelteindex 62ac4672..f7ca1d71 100644--- a/frontend/src/routes/search/Modal.svelte+++ b/frontend/src/routes/search/Modal.svelte@@ -19,7 +19,7 @@let innerWidth = 0;let left = modal.left;- let top = modal.top + 32;+ $: top = modal.top + 32;$: {if (innerWidth < mediumWidthCutoff) {