Skip to content

web search done right

Move widget, discussions etc out from api searcher and into separate api endpoints. This creates a better separation between the frontend and backend, and avoids e.g. 'fetch_discussions' abstraction leak. It also allows for these endpoints to be used in different contexts separately without having to perform a full search.

Mikkel Denker14e5466fb8f1
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index 1a6608ba..aec936fb 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -180,8 +180,6 @@ impl Searcher {
return_ranking_signals: false,
flatten_response: false,
safe_search: Some(false),
- fetch_discussions: false,
- fetch_sidebar: false,
count_results: false,
};
tracing::debug!("searching at {:?}: {:#?}", self.url, query);
diff --git a/core/src/api/autosuggest.rs b/core/src/api/autosuggest.rs
index 9de14e4b..9a1d973e 100644
--- a/core/src/api/autosuggest.rs
+++ b/core/src/api/autosuggest.rs
@@ -52,7 +52,6 @@ pub struct AutosuggestQuery {
q: String,
}
-#[allow(clippy::unused_async)]
#[utoipa::path(
post,
path = "/beta/api/autosuggest",
diff --git a/core/src/api/docs.rs b/core/src/api/docs.rs
index 3785f3fa..18bc7f7c 100644
--- a/core/src/api/docs.rs
+++ b/core/src/api/docs.rs
@@ -22,7 +22,9 @@ use utoipa_swagger_ui::SwaggerUi;
#[derive(OpenApi)]
#[openapi(
paths(
- search::api,
+ search::search,
+ search::widget,
+ search::sidebar,
webgraph::host::similar,
webgraph::host::knows,
webgraph::host::ingoing_hosts,
@@ -40,6 +42,8 @@ use utoipa_swagger_ui::SwaggerUi;
optics::HostRankings,
search::ApiSearchQuery,
search::ApiSearchResult,
+ search::WidgetQuery,
+ search::SidebarQuery,
crate::searcher::WebsitesResult,
crate::search_prettifier::HighlightedSpellCorrection,
crate::search_prettifier::DisplayedWebpage,
diff --git a/core/src/api/hosts.rs b/core/src/api/hosts.rs
index 9a7aee28..cf54d18c 100644
--- a/core/src/api/hosts.rs
+++ b/core/src/api/hosts.rs
@@ -25,7 +25,6 @@ pub struct HostsExportOpticParams {
host_rankings: HostRankings,
}
-#[allow(clippy::unused_async)]
#[utoipa::path(post,
path = "/beta/api/hosts/export",
request_body(content = HostsExportOpticParams),
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rs
index 3a06187d..fb8cd112 100644
--- a/core/src/api/improvement.rs
+++ b/core/src/api/improvement.rs
@@ -37,7 +37,6 @@ pub struct StoreParams {
pub urls: Vec<String>,
}
-#[allow(clippy::unused_async)]
pub async fn click(
extract::Query(params): extract::Query<ClickParams>,
extract::State(state): extract::State<Arc<State>>,
@@ -62,7 +61,6 @@ impl TryFrom<StoreParams> for StoredQuery {
}
}
-#[allow(clippy::unused_async)]
pub async fn store(
extract::State(state): extract::State<Arc<State>>,
extract::Json(params): extract::Json<StoreParams>,
diff --git a/core/src/api/metrics.rs b/core/src/api/metrics.rs
index 73fdc44d..c2425559 100644
--- a/core/src/api/metrics.rs
+++ b/core/src/api/metrics.rs
@@ -19,7 +19,6 @@ use std::sync::Arc;
use axum::{extract, response::IntoResponse};
use axum_macros::debug_handler;
-#[allow(clippy::unused_async)]
#[allow(clippy::match_wild_err_arm)]
#[debug_handler]
pub async fn route(
diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs
index 8ba562fd..78ef9348 100644
--- a/core/src/api/mod.rs
+++ b/core/src/api/mod.rs
@@ -96,7 +96,6 @@ pub struct State {
pub cluster: Arc<Cluster>,
}
-#[allow(clippy::unused_async)]
pub async fn favicon() -> impl IntoResponse {
Response::builder()
.status(StatusCode::OK)
@@ -201,7 +200,7 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
Ok(Router::new()
.merge(
Router::new()
- .route("/beta/api/search", post(search::api))
+ .route("/beta/api/search", post(search::search))
.route_layer(middleware::from_fn_with_state(state.clone(), search_metric))
.layer(cors_layer()),
)
@@ -217,6 +216,8 @@ pub async fn router(config: &ApiConfig, counters: Counters) -> Result<Router> {
.nest(
"/beta",
Router::new()
+ .route("/api/search/widget", post(search::widget))
+ .route("/api/search/sidebar", post(search::sidebar))
.route("/api/autosuggest", post(autosuggest::route))
.route("/api/autosuggest/browser", get(autosuggest::browser))
.route("/api/summarize", get(summarize::summarize_route))
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index aa853e9b..bf50db4d 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -51,12 +51,6 @@ pub struct ApiSearchQuery {
#[serde(default = "defaults::SearchQuery::flatten_response")]
pub flatten_response: bool,
- #[serde(default = "defaults::SearchQuery::fetch_discussions")]
- pub fetch_discussions: bool,
-
- #[serde(default = "defaults::SearchQuery::fetch_sidebar")]
- pub fetch_sidebar: bool,
-
#[serde(default = "defaults::SearchQuery::count_results")]
pub count_results: bool,
}
@@ -82,8 +76,6 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {
host_rankings: api.host_rankings,
return_ranking_signals: api.return_ranking_signals,
safe_search: api.safe_search.unwrap_or(default.safe_search),
- fetch_discussions: api.fetch_discussions,
- fetch_sidebar: api.fetch_sidebar,
count_results: api.count_results,
})
}
@@ -105,8 +97,6 @@ impl From<SearchResult> for ApiSearchResult {
}
}
-#[allow(clippy::unused_async)]
-#[allow(clippy::match_wild_err_arm)]
#[debug_handler]
#[utoipa::path(
post,
@@ -116,7 +106,7 @@ impl From<SearchResult> for ApiSearchResult {
(status = 200, description = "Search results", body = ApiSearchResult),
)
)]
-pub async fn api(
+pub async fn search(
extract::State(state): extract::State<Arc<State>>,
extract::Json(query): extract::Json<ApiSearchQuery>,
) -> Result<impl IntoResponse, StatusCode> {
@@ -155,6 +145,48 @@ pub async fn api(
}
}
+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
+pub struct WidgetQuery {
+ pub query: String,
+}
+
+#[debug_handler]
+#[utoipa::path(
+ post,
+ path = "/beta/api/search/widget",
+ request_body(content = WidgetQuery),
+ responses(
+ (status = 200, description = "The resulting widget if one matches the query", body = Option<Widget>),
+ )
+)]
+pub async fn widget(
+ extract::State(state): extract::State<Arc<State>>,
+ extract::Json(req): extract::Json<WidgetQuery>,
+) -> impl IntoResponse {
+ Json(state.searcher.widget(&req.query).await)
+}
+
+#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
+pub struct SidebarQuery {
+ pub query: String,
+}
+
+#[debug_handler]
+#[utoipa::path(
+ post,
+ path = "/beta/api/search/sidebar",
+ request_body(content = SidebarQuery),
+ responses(
+ (status = 200, description = "The sidebar if one matches the query", body = Option<DisplayedSidebar>),
+ )
+)]
+pub async fn sidebar(
+ extract::State(state): extract::State<Arc<State>>,
+ extract::Json(req): extract::Json<SidebarQuery>,
+) -> impl IntoResponse {
+ Json(state.searcher.sidebar(&req.query).await)
+}
+
#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]
#[serde(rename_all = "camelCase")]
pub struct EntityImageParams {
diff --git a/core/src/api/summarize.rs b/core/src/api/summarize.rs
index 1c9faed8..015b8218 100644
--- a/core/src/api/summarize.rs
+++ b/core/src/api/summarize.rs
@@ -67,7 +67,6 @@ async fn summarize(
Err::<Sse<BoxStream<_>>, anyhow::Error>(anyhow::anyhow!("Summarization is not enabled"))
}
-#[allow(clippy::unused_async)]
#[utoipa::path(
get,
path = "/beta/api/summarize",
diff --git a/core/src/api/webgraph.rs b/core/src/api/webgraph.rs
index ba8ed83a..a4db60db 100644
--- a/core/src/api/webgraph.rs
+++ b/core/src/api/webgraph.rs
@@ -73,7 +73,6 @@ pub mod host {
pub host: String,
}
- #[allow(clippy::unused_async)]
#[utoipa::path(post,
path = "/beta/api/webgraph/host/similar",
request_body(content = SimilarHostsParams),
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index a15ef45b..ee338dff 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -48,10 +48,6 @@ impl Api {
pub fn entity_sidebar() -> f64 {
10.0
}
-
- pub fn discussions_widget() -> f64 {
- 0.1
- }
}
pub struct Snippet;
@@ -129,14 +125,6 @@ impl SearchQuery {
false
}
- pub fn fetch_discussions() -> bool {
- false
- }
-
- pub fn fetch_sidebar() -> bool {
- false
- }
-
pub fn count_results() -> bool {
false
}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index 005242cf..c1f7121b 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -175,9 +175,6 @@ pub struct ApiThresholds {
#[serde(default = "defaults::Api::entity_sidebar")]
pub entity_sidebar: f64,
-
- #[serde(default = "defaults::Api::discussions_widget")]
- pub discussions_widget: f64,
}
impl Default for ApiThresholds {
@@ -185,7 +182,6 @@ impl Default for ApiThresholds {
Self {
stackoverflow: defaults::Api::stackoverflow(),
entity_sidebar: defaults::Api::entity_sidebar(),
- discussions_widget: defaults::Api::discussions_widget(),
}
}
}
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index 23d89ae1..a20995fa 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -27,7 +27,7 @@ use crate::{
member::{Member, Service},
sonic,
},
- entity_index::EntityIndex,
+ entity_index::{EntityIndex, EntityMatch},
image_store::Image,
index::Index,
inverted_index::{self, RetrievedWebpage},
@@ -44,6 +44,7 @@ sonic_service!(
[
RetrieveWebsites,
Search,
+ SearchEntity,
GetWebpage,
GetHomepageDescriptions,
GetEntityImage,
@@ -193,6 +194,18 @@ impl sonic::service::Message<SearchService> for GetEntityImage {
}
}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SearchEntity {
+ pub query: String,
+}
+
+impl sonic::service::Message<SearchService> for SearchEntity {
+ type Response = Option<EntityMatch>;
+ async fn handle(self, server: &SearchService) -> sonic::Result<Self::Response> {
+ Ok(server.local_searcher.search_entity(&self.query))
+ }
+}
+
pub async fn run(config: config::SearchServerConfig) -> Result<()> {
let addr = config.host;
let server = SearchService::new(config).await?.bind(addr).await.unwrap();
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 32eeb0ae..ca6bff6f 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -1052,48 +1052,6 @@ mod tests {
assert_eq!(res[0].url, "https://b.com/");
}
- #[test]
- fn discussion_optic() {
- let mut index = Index::temporary().expect("Unable to open index");
-
- index
- .insert(Webpage {
- html: Html::parse(
- include_str!("../../testcases/schema_org/infinity_war.html"),
- "https://a.com",
- )
- .unwrap(),
- fetch_time_ms: 500,
- ..Default::default()
- })
- .expect("failed to insert webpage");
- index.commit().expect("failed to commit index");
-
- let searcher = LocalSearcher::from(index);
- let res = searcher
- .search(&SearchQuery {
- query: "avengers endgame".to_string(),
- ..Default::default()
- })
- .unwrap()
- .webpages;
-
- assert!(!res.is_empty());
- assert_eq!(&res[0].url, "https://a.com/");
-
- let res = searcher
- .search(&SearchQuery {
- query: "avengers endgame".to_string(),
- optic: Some(
- Optic::parse(include_str!("../searcher/api/discussions.optic")).unwrap(),
- ),
- ..Default::default()
- })
- .unwrap()
- .webpages;
- assert!(res.is_empty());
- }
-
#[test]
fn special_pattern_syntax() {
let mut index = Index::temporary().expect("Unable to open index");
diff --git a/core/src/ranking/pipeline.rs b/core/src/ranking/pipeline.rs
index fbd3830e..ee269bdc 100644
--- a/core/src/ranking/pipeline.rs
+++ b/core/src/ranking/pipeline.rs
@@ -324,7 +324,11 @@ impl<T: AsRankingWebsite> RankingStage<T> {
collector.insert(website);
}
- collector.into_sorted_vec(self.derank_similar)
+ collector
+ .into_sorted_vec(self.derank_similar)
+ .into_iter()
+ .take(top_n)
+ .collect()
}
fn set_query_info(&mut self, query: &SearchQuery) {
@@ -442,7 +446,7 @@ impl<T: AsRankingWebsite> RankingPipeline<T> {
}
pub fn collector_top_n(&self) -> usize {
- self.initial_top_n().max(self.top_n) + self.top_n * self.page
+ (self.initial_top_n().max(self.top_n) + self.top_n * self.page) + 1
}
pub fn initial_top_n(&self) -> usize {
@@ -510,7 +514,7 @@ mod tests {
20,
)
.unwrap();
- assert_eq!(pipeline.collector_top_n(), 20);
+ assert_eq!(pipeline.collector_top_n(), 20 + 1);
let sample = sample_websites(pipeline.collector_top_n());
let res: Vec<_> = pipeline
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index b0468f22..31dd68b9 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -172,6 +172,7 @@ pub struct DisplayedWebpage {
pub pretty_url: String,
pub snippet: Snippet,
pub ranking_signals: Option<HashMap<Signal, SignalScore>>,
+ pub score: Option<f64>,
pub likely_has_ads: bool,
pub likely_has_paywall: bool,
}
@@ -202,6 +203,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
domain,
snippet,
ranking_signals: None,
+ score: None,
likely_has_ads: webpage.likely_has_ads,
likely_has_paywall: webpage.likely_has_paywall,
}
diff --git a/core/src/searcher/api/discussions.optic b/core/src/searcher/api/discussions.optic
deleted file mode 100644
index 81ef9092..00000000
--- a/core/src/searcher/api/discussions.optic
+++ /dev/null
@@ -1,39 +0,0 @@
-DiscardNonMatching;
-
-Rule {
- Matches {
- Schema("QAPage"),
- }
-};
-
-Rule {
- Matches {
- Schema("DiscussionForumPosting"),
- }
-};
-
-Rule {
- Matches {
- Domain("|reddit.com|"),
- Url("comments"),
- }
-};
-
-// source of instances: https://github.com/maltfield/awesome-lemmy-instances
-Rule {
- Matches {
- Site("|lemmy.world|")
- }
-};
-
-Rule {
- Matches {
- Site("|lemmy.ml|")
- }
-};
-
-Rule {
- Matches {
- Site("|sh.itjust.works|")
- }
-};
\ No newline at end of file
diff --git a/core/src/searcher/api/mod.rs b/core/src/searcher/api/mod.rs
index 168aec9f..c12dc8fd 100644
--- a/core/src/searcher/api/mod.rs
+++ b/core/src/searcher/api/mod.rs
@@ -32,9 +32,9 @@ use crate::inverted_index::RetrievedWebpage;
use crate::ranking::models::cross_encoder::DummyCrossEncoder;
use crate::ranking::pipeline::{AsRankingWebsite, RankingWebsite, RetrievedWebpageRanking};
use crate::ranking::ALL_SIGNALS;
-use crate::search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection};
+use crate::search_prettifier::{DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection};
use crate::web_spell::SpellChecker;
-use crate::widgets::Widgets;
+use crate::widgets::{Widget, Widgets};
use crate::{
bangs::Bangs,
collector::BucketCollector,
@@ -79,10 +79,13 @@ pub fn combine_results(
) -> (Vec<ScoredWebsitePointer>, bool) {
let mut collector = BucketCollector::new(pipeline.collector_top_n(), collector_config);
- let mut num_sites: usize = 0;
+ let mut has_more = false;
for result in initial_results {
+ if result.local_result.has_more {
+ has_more = true;
+ }
+
for website in result.local_result.websites {
- num_sites += 1;
let pointer = distributed::ScoredWebsitePointer {
website,
shard: result.shard,
@@ -95,8 +98,11 @@ pub fn combine_results(
}
for result in live_results {
+ if result.local_result.has_more {
+ has_more = true;
+ }
+
for website in result.local_result.websites {
- num_sites += 1;
let pointer = live::ScoredWebsitePointer {
website,
split_id: result.split_id.clone(),
@@ -114,10 +120,7 @@ pub fn combine_results(
.take(pipeline.collector_top_n())
.collect::<Vec<_>>();
- let offset = pipeline.offset();
-
let res = pipeline.apply(top_websites);
- let has_more = num_sites.saturating_sub(offset) > res.len();
(res, has_more)
}
@@ -131,6 +134,8 @@ pub fn add_ranking_signals(websites: &mut [DisplayedWebpage], pointers: &[Scored
}
}
+ website.score = Some(pointer.as_ranking().score);
+
website.ranking_signals = Some(signals);
}
}
@@ -144,7 +149,7 @@ pub struct ApiSearcher<S, L> {
lambda_model: Option<Arc<LambdaMART>>,
bangs: Bangs,
collector_config: CollectorConfig,
- widget_manager: WidgetManager<S>,
+ widget_manager: WidgetManager,
spell_checker: Option<SpellChecker>,
}
@@ -156,7 +161,7 @@ pub struct ApiSearcher<S, L> {
lambda_model: Option<Arc<LambdaMART>>,
bangs: Bangs,
collector_config: CollectorConfig,
- widget_manager: WidgetManager<S>,
+ widget_manager: WidgetManager,
spell_checker: Option<SpellChecker>,
}
@@ -180,16 +185,8 @@ where
SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());
let lambda_model = lambda_model.map(Arc::new);
- let qa_model = qa_model.map(Arc::new);
-
- let widget_manager = WidgetManager::new(
- Widgets::new(config.widgets).unwrap(),
- Arc::clone(&dist_searcher),
- config.thresholds.clone(),
- config.collector.clone(),
- lambda_model.clone(),
- qa_model,
- );
+
+ let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());
Self {
distributed_searcher: dist_searcher,
@@ -219,13 +216,7 @@ where
SidebarManager::new(Arc::clone(&dist_searcher), config.thresholds.clone());
let lambda_model = lambda_model.map(Arc::new);
- let widget_manager = WidgetManager::new(
- Widgets::new(config.widgets).unwrap(),
- Arc::clone(&dist_searcher),
- config.thresholds.clone(),
- config.collector.clone(),
- lambda_model.clone(),
- );
+ let widget_manager = WidgetManager::new(Widgets::new(config.widgets).unwrap());
Self {
distributed_searcher: dist_searcher,
@@ -279,6 +270,14 @@ where
Ok(self.bangs.get(&parsed_terms))
}
+ pub async fn widget(&self, query: &str) -> Option<Widget> {
+ self.widget_manager.widget(query).await
+ }
+
+ pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {
+ self.sidebar_manager.sidebar(query).await
+ }
+
async fn retrieve_webpages(
&self,
query: &str,
@@ -352,6 +351,7 @@ where
}
let mut search_query = query.clone();
+ let top_n = search_query.num_results;
// This pipeline should be created before the first search is performed
// so the query knows how many results to fetch from the indices
@@ -359,33 +359,19 @@ where
&mut search_query,
self.lambda_model.clone(),
self.collector_config.clone(),
- 20,
+ top_n,
);
- let (initial_results, live_results, widget, discussions, stackoverflow) = tokio::join!(
+
+ let (initial_results, live_results) = tokio::join!(
self.distributed_searcher.search_initial(&search_query),
self.search_initial_from_live(&search_query),
- self.widget_manager.widget(query),
- self.widget_manager.discussions(query),
- self.sidebar_manager.stackoverflow(query),
);
- let discussions = discussions?;
- let stackoverflow = stackoverflow?;
- let sidebar = if query.fetch_sidebar {
- self.sidebar_manager
- .sidebar(&initial_results, stackoverflow)
- } else {
- None
- };
-
- let spell_corrected_query = if widget.is_none() {
- self.spell_checker
- .as_ref()
- .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))
- .map(HighlightedSpellCorrection::from)
- } else {
- None
- };
+ let spell_corrected_query = self
+ .spell_checker
+ .as_ref()
+ .and_then(|s| s.correct(&query.query, &whatlang::Lang::Eng))
+ .map(HighlightedSpellCorrection::from);
let num_docs = initial_results
.iter()
@@ -444,20 +430,12 @@ where
add_ranking_signals(&mut retrieved_webpages, &top_websites);
}
- let direct_answer = self
- .widget_manager
- .answer(&query.query, &mut retrieved_webpages);
-
let search_duration_ms = start.elapsed().as_millis();
Ok(WebsitesResult {
spell_corrected_query,
num_hits: num_docs,
webpages: retrieved_webpages,
- direct_answer,
- sidebar,
- widget,
- discussions,
search_duration_ms,
has_more_results,
})
diff --git a/core/src/searcher/api/sidebar.rs b/core/src/searcher/api/sidebar.rs
index ebc86492..fbc9ad0f 100644
--- a/core/src/searcher/api/sidebar.rs
+++ b/core/src/searcher/api/sidebar.rs
@@ -21,7 +21,7 @@ use optics::Optic;
use url::Url;
use crate::{
- search_prettifier::{create_stackoverflow_sidebar, DisplayedEntity, DisplayedSidebar},
+ search_prettifier::{create_stackoverflow_sidebar, DisplayedSidebar},
searcher::{distributed, SearchQuery},
};
@@ -41,13 +41,9 @@ where
}
}
- pub async fn stackoverflow(&self, query: &SearchQuery) -> Result<Option<DisplayedSidebar>> {
- if !query.fetch_sidebar {
- return Ok(None);
- }
-
+ pub async fn stackoverflow(&self, query: &str) -> Result<Option<DisplayedSidebar>> {
let query = SearchQuery {
- query: query.query.clone(),
+ query: query.to_string(),
num_results: 1,
optic: Some(Optic::parse(include_str!("stackoverflow.optic")).unwrap()),
..Default::default()
@@ -93,25 +89,18 @@ where
Ok(None)
}
- pub fn sidebar(
- &self,
- initial_results: &[distributed::InitialSearchResultShard],
- stackoverflow: Option<DisplayedSidebar>,
- ) -> Option<DisplayedSidebar> {
- let entity = initial_results
- .iter()
- .filter_map(|res| res.local_result.entity_sidebar.clone())
- .map(DisplayedEntity::from)
- .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)
- .max_by(|a, b| {
- a.match_score
- .partial_cmp(&b.match_score)
- .unwrap_or(Ordering::Equal)
- });
+ pub async fn sidebar(&self, query: &str) -> Option<DisplayedSidebar> {
+ let (entity, stackoverflow) = futures::join!(
+ self.distributed_searcher.search_entity(query),
+ self.stackoverflow(query)
+ );
- match entity {
- Some(entity) => Some(DisplayedSidebar::Entity(entity)),
- None => stackoverflow,
+ if let Some(entity) = entity {
+ if entity.score as f64 > self.thresholds.entity_sidebar {
+ return Some(DisplayedSidebar::Entity(entity.into()));
+ }
}
+
+ stackoverflow.ok().flatten()
}
}
diff --git a/core/src/searcher/api/widget.rs b/core/src/searcher/api/widget.rs
index 53d312c5..4ee358a0 100644
--- a/core/src/searcher/api/widget.rs
+++ b/core/src/searcher/api/widget.rs
@@ -14,184 +14,22 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use std::ops::Range;
-use std::sync::Arc;
-
use itertools::Itertools;
-use optics::Optic;
-
-#[cfg(feature = "libtorch")]
-use crate::qa_model::QaModel;
-use crate::ranking::models::lambdamart::LambdaMART;
-use crate::ranking::pipeline::{AsRankingWebsite, RankingPipeline};
-use crate::search_prettifier::DisplayedAnswer;
-use crate::search_prettifier::DisplayedWebpage;
-use crate::searcher::api::{add_ranking_signals, combine_results, ScoredWebsitePointer};
-use crate::searcher::{distributed, SearchQuery};
-use crate::widgets::Widget;
-use crate::{ceil_char_boundary, floor_char_boundary, query, Result};
-use crate::{
- config::{ApiThresholds, CollectorConfig},
- widgets::Widgets,
-};
-#[cfg(feature = "libtorch")]
-pub struct WidgetManager<S> {
- widgets: Widgets,
- distributed_searcher: Arc<S>,
- thresholds: ApiThresholds,
- collector_config: CollectorConfig,
- lambda_model: Option<Arc<LambdaMART>>,
- qa_model: Option<Arc<QaModel>>,
-}
+use crate::query;
+use crate::widgets::{Widget, Widgets};
-#[cfg(not(feature = "libtorch"))]
-pub struct WidgetManager<S> {
+pub struct WidgetManager {
widgets: Widgets,
- distributed_searcher: Arc<S>,
- thresholds: ApiThresholds,
- collector_config: CollectorConfig,
- lambda_model: Option<Arc<LambdaMART>>,
}
-impl<S> WidgetManager<S>
-where
- S: distributed::SearchClient,
-{
- #[cfg(feature = "libtorch")]
- pub fn new(
- widgets: Widgets,
- distributed_searcher: Arc<S>,
- thresholds: ApiThresholds,
- collector_config: CollectorConfig,
- lambda_model: Option<Arc<LambdaMART>>,
- qa_model: Option<Arc<QaModel>>,
- ) -> WidgetManager<S> {
- Self {
- widgets,
- distributed_searcher,
- thresholds,
- collector_config,
- lambda_model,
- qa_model,
- }
- }
-
- #[cfg(not(feature = "libtorch"))]
- pub fn new(
- widgets: Widgets,
- distributed_searcher: Arc<S>,
- thresholds: ApiThresholds,
- collector_config: CollectorConfig,
- lambda_model: Option<Arc<LambdaMART>>,
- ) -> WidgetManager<S> {
- Self {
- widgets,
- distributed_searcher,
- thresholds,
- collector_config,
- lambda_model,
- }
- }
-
- pub async fn discussions(&self, query: &SearchQuery) -> Result<Option<Vec<DisplayedWebpage>>> {
- if !query.fetch_discussions || query.optic.is_some() || query.page > 0 {
- return Ok(None);
- }
-
- const NUM_RESULTS: usize = 10;
-
- let mut query = SearchQuery {
- query: query.query.clone(),
- num_results: NUM_RESULTS,
- optic: Some(Optic::parse(include_str!("discussions.optic")).unwrap()),
- host_rankings: query.host_rankings.clone(),
- return_ranking_signals: query.return_ranking_signals,
- ..Default::default()
- };
-
- // This pipeline should be created before the first search is performed
- // so the query knows how many results to fetch from the indices
- let pipeline: RankingPipeline<ScoredWebsitePointer> = RankingPipeline::recall_stage(
- &mut query,
- self.lambda_model.clone(),
- self.collector_config.clone(),
- NUM_RESULTS,
- );
-
- let initial_results = self.distributed_searcher.search_initial(&query).await;
-
- if initial_results.is_empty() {
- return Ok(None);
- }
-
- let num_results: usize = initial_results
- .iter()
- .map(|res| res.local_result.websites.len())
- .sum();
-
- if num_results < NUM_RESULTS / 2 {
- return Ok(None);
- }
-
- let (top_websites, _) = combine_results(
- self.collector_config.clone(),
- initial_results,
- vec![],
- pipeline,
- );
-
- let scores: Vec<_> = top_websites
- .iter()
- .map(|pointer| pointer.as_ranking().score)
- .collect();
-
- let median = if scores.len() % 2 == 0 {
- (scores[(scores.len() / 2) - 1] + scores[scores.len() / 2]) / 2.0
- } else {
- scores[scores.len() / 2]
- };
-
- if median < self.thresholds.discussions_widget {
- return Ok(None);
- }
-
- let mut result: Vec<DisplayedWebpage> = self
- .distributed_searcher
- .retrieve_webpages(
- &top_websites
- .clone()
- .into_iter()
- .filter_map(|pointer| {
- if let ScoredWebsitePointer::Normal(p) = pointer {
- Some(p)
- } else {
- None
- }
- })
- .enumerate()
- .collect_vec(),
- &query.query,
- )
- .await
- .into_iter()
- .map(|(_, webpage)| webpage.into_retrieved_webpage())
- .map(DisplayedWebpage::from)
- .collect();
-
- if query.return_ranking_signals {
- add_ranking_signals(&mut result, &top_websites);
- }
-
- Ok(Some(result))
+impl WidgetManager {
+ pub fn new(widgets: Widgets) -> WidgetManager {
+ Self { widgets }
}
- pub async fn widget(&self, query: &SearchQuery) -> Option<Widget> {
- if query.page > 0 {
- return None;
- }
-
- let parsed_terms = query::parser::parse(&query.query);
+ pub async fn widget(&self, query: &str) -> Option<Widget> {
+ let parsed_terms = query::parser::parse(query);
self.widgets.widget(
parsed_terms
@@ -207,164 +45,4 @@ where
.as_str(),
)
}
-
- #[cfg(feature = "libtorch")]
- pub fn answer(
- &self,
- query: &str,
- webpages: &mut Vec<DisplayedWebpage>,
- ) -> Option<DisplayedAnswer> {
- self.qa_model.as_ref().and_then(|qa_model| {
- let contexts: Vec<_> = webpages
- .iter()
- .take(1)
- .filter_map(|webpage| webpage.snippet.text())
- .map(|t| t.fragments.iter().map(|f| f.text()).join(""))
- .collect();
-
- match qa_model.run(query, &contexts) {
- Some(answer) => {
- let answer_webpage = webpages.remove(answer.context_idx);
- let snip = answer_webpage
- .snippet
- .text()
- .unwrap()
- .fragments
- .iter()
- .map(|f| f.text())
- .join("");
- Some(DisplayedAnswer {
- title: answer_webpage.title,
- url: answer_webpage.url,
- pretty_url: answer_webpage.pretty_url,
- snippet: generate_answer_snippet(&snip, answer.offset.clone()),
- answer: snip[answer.offset].to_string(),
- })
- }
- None => None,
- }
- })
- }
-
- #[cfg(not(feature = "libtorch"))]
- pub fn answer(
- &self,
- query: &str,
- webpages: &mut Vec<DisplayedWebpage>,
- ) -> Option<DisplayedAnswer> {
- None
- }
-}
-
-fn generate_answer_snippet(body: &str, answer_offset: Range<usize>) -> String {
- let mut best_start = 0;
- let mut best_end = 0;
- const SNIPPET_LENGTH: usize = 200;
-
- if body.is_empty() || answer_offset.start > body.len() - 1 {
- return body.to_string();
- }
-
- for (idx, _) in body.char_indices().filter(|(_, c)| *c == '.') {
- if idx > answer_offset.end + SNIPPET_LENGTH {
- break;
- }
-
- if idx < answer_offset.start {
- best_start = idx;
- }
-
- if idx > answer_offset.end {
- best_end = idx;
- }
- }
-
- if (answer_offset.end - best_start > SNIPPET_LENGTH) || (best_start >= best_end) {
- if answer_offset.end - answer_offset.start >= SNIPPET_LENGTH {
- let end = floor_char_boundary(body, answer_offset.start + SNIPPET_LENGTH);
-
- return "<b>".to_string() + &body[answer_offset.start..end] + "</b>";
- }
-
- let chars_either_side = (SNIPPET_LENGTH - (answer_offset.end - answer_offset.start)) / 2;
-
- let start = ceil_char_boundary(
- body,
- answer_offset
- .start
- .checked_sub(chars_either_side)
- .unwrap_or_default(),
- );
- let mut end = ceil_char_boundary(body, answer_offset.end + chars_either_side);
-
- if end >= body.len() {
- end = floor_char_boundary(body, body.len());
- }
-
- body[start..answer_offset.start].to_string()
- + "<b>"
- + &body[answer_offset.clone()]
- + "</b>"
- + &body[answer_offset.end..end]
- } else {
- let mut res = body[best_start..answer_offset.start].to_string()
- + "<b>"
- + &body[answer_offset.clone()]
- + "</b>";
-
- let remaining_chars = SNIPPET_LENGTH - (res.len() - 7);
- let end = ceil_char_boundary(body, (remaining_chars + answer_offset.end).min(best_end));
-
- res += &body[answer_offset.end..end];
-
- res
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- #[test]
- fn test_generate_answer_snippet() {
- assert_eq!(
- generate_answer_snippet("this is a test", 0..4),
- "<b>this</b> is a test".to_string()
- );
-
- assert_eq!(
- generate_answer_snippet("this is a test", 0..1000),
- "<b>this is a test</b>".to_string()
- );
- assert_eq!(
- generate_answer_snippet("this is a test", 1000..2000),
- "this is a test".to_string()
- );
- let input = r#"
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
-this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test this is a long test
- "#;
-
- let res = generate_answer_snippet(input, 0..500);
- assert!(!res.is_empty());
- assert!(res.len() > 100);
- assert!(res.len() < input.len());
- assert!(res.starts_with("<b>"));
- assert!(res.ends_with("</b>"));
-
- assert_eq!(generate_answer_snippet("", 0..2000), "".to_string());
- }
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 205bab7a..da2f0403 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -23,6 +23,7 @@ use crate::{
ReplicatedClient, Shard, ShardIdentifier, ShardedClient, SpecificShardSelector,
},
},
+ entity_index::EntityMatch,
entrypoint::search_server::{self, SearchService},
image_store::Image,
inverted_index::{RetrievedWebpage, WebsitePointer},
@@ -270,6 +271,25 @@ impl SearchClient for DistributedSearcher {
Ok(res.pop().flatten())
}
+
+ async fn search_entity(&self, query: &str) -> Option<EntityMatch> {
+ let client = self.client().await;
+
+ let (_, mut res) = client
+ .send(
+ &search_server::SearchEntity {
+ query: query.to_string(),
+ },
+ &RandomShardSelector,
+ &RandomReplicaSelector,
+ )
+ .await
+ .ok()?
+ .pop()
+ .unwrap();
+
+ res.pop().flatten()
+ }
}
pub trait SearchClient {
@@ -277,19 +297,25 @@ pub trait SearchClient {
&self,
query: &SearchQuery,
) -> impl Future<Output = Vec<InitialSearchResultShard>> + Send;
+
fn retrieve_webpages(
&self,
top_websites: &[(usize, ScoredWebsitePointer)],
query: &str,
) -> impl Future<Output = Vec<(usize, RetrievedWebpageRanking)>> + Send;
+
+ fn search_entity(&self, query: &str) -> impl Future<Output = Option<EntityMatch>> + Send;
+
fn get_webpage(
&self,
url: &str,
) -> impl Future<Output = Result<Option<RetrievedWebpage>>> + Send;
+
fn get_homepage_descriptions(
&self,
urls: &[Url],
) -> impl Future<Output = HashMap<Url, String>> + Send;
+
fn get_entity_image(
&self,
image_id: &str,
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index b04534ce..5d32c8e0 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -33,7 +33,7 @@ use crate::ranking::models::linear::LinearRegression;
use crate::ranking::pipeline::{RankingPipeline, RankingWebsite};
use crate::ranking::{query_centrality, Ranker, Signal, SignalAggregator, ALL_SIGNALS};
use crate::search_ctx::Ctx;
-use crate::search_prettifier::{DisplayedEntity, DisplayedWebpage};
+use crate::search_prettifier::DisplayedWebpage;
use crate::webgraph::Node;
use crate::{inverted_index, live_index, Error, Result};
@@ -311,9 +311,9 @@ where
)?;
let pipe_top_n = pipeline.top_n;
- let ranking_websites = pipeline.apply(ranking_websites);
+ let has_more = ranking_websites.len() > pipe_top_n;
- let has_more = pipe_top_n == ranking_websites.len();
+ let ranking_websites = pipeline.apply(ranking_websites);
Ok(InvertedIndexResult {
webpages: ranking_websites,
@@ -322,10 +322,10 @@ where
})
}
- fn entity_sidebar(&self, query: &SearchQuery) -> Option<EntityMatch> {
+ pub fn search_entity(&self, query: &str) -> Option<EntityMatch> {
self.entity_index
.as_ref()
- .and_then(|index| index.search(&query.query))
+ .and_then(|index| index.search(query))
}
pub fn search_initial(
@@ -337,13 +337,11 @@ where
let ctx = guard.inverted_index().local_search_ctx();
let inverted_index_result =
self.search_inverted_index(&ctx, &guard, query, de_rank_similar)?;
- let sidebar = self.entity_sidebar(query);
Ok(InitialWebsiteResult {
websites: inverted_index_result.webpages,
num_websites: inverted_index_result.num_hits,
has_more: inverted_index_result.has_more,
- entity_sidebar: sidebar,
})
}
@@ -371,8 +369,6 @@ where
pub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {
use std::time::Instant;
- use crate::search_prettifier::DisplayedSidebar;
-
let start = Instant::now();
let mut search_query = query.clone();
@@ -444,14 +440,7 @@ where
Ok(WebsitesResult {
num_hits: search_result.num_websites,
webpages,
- discussions: None,
- widget: None,
spell_corrected_query: None,
- direct_answer: None,
- sidebar: search_result
- .entity_sidebar
- .map(DisplayedEntity::from)
- .map(DisplayedSidebar::Entity),
search_duration_ms: start.elapsed().as_millis(),
has_more_results,
})
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index b9897445..a7a80d71 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -28,13 +28,9 @@ use utoipa::ToSchema;
use crate::{
bangs::BangHit,
config::defaults,
- entity_index::EntityMatch,
ranking::pipeline::RankingWebsite,
- search_prettifier::{
- DisplayedAnswer, DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection,
- },
+ search_prettifier::{DisplayedWebpage, HighlightedSpellCorrection},
webpage::region::Region,
- widgets::Widget,
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
@@ -51,10 +47,6 @@ pub struct WebsitesResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub webpages: Vec<DisplayedWebpage>,
pub num_hits: Option<usize>,
- pub sidebar: Option<DisplayedSidebar>,
- pub widget: Option<Widget>,
- pub direct_answer: Option<DisplayedAnswer>,
- pub discussions: Option<Vec<DisplayedWebpage>>,
pub search_duration_ms: u128,
pub has_more_results: bool,
}
@@ -69,8 +61,6 @@ pub struct SearchQuery {
pub host_rankings: Option<HostRankings>,
pub return_ranking_signals: bool,
pub safe_search: bool,
- pub fetch_discussions: bool,
- pub fetch_sidebar: bool,
pub count_results: bool,
}
@@ -79,7 +69,6 @@ pub struct InitialWebsiteResult {
pub num_websites: Option<usize>,
pub websites: Vec<RankingWebsite>,
pub has_more: bool,
- pub entity_sidebar: Option<EntityMatch>,
}
impl Default for SearchQuery {
@@ -96,8 +85,6 @@ impl Default for SearchQuery {
host_rankings: Default::default(),
return_ranking_signals: defaults::SearchQuery::return_ranking_signals(),
safe_search: defaults::SearchQuery::safe_search(),
- fetch_discussions: defaults::SearchQuery::fetch_discussions(),
- fetch_sidebar: defaults::SearchQuery::fetch_sidebar(),
count_results: defaults::SearchQuery::count_results(),
}
}
diff --git a/frontend/src/lib/api/index.ts b/frontend/src/lib/api/index.ts
index f576974a..aabd071d 100644
--- a/frontend/src/lib/api/index.ts
+++ b/frontend/src/lib/api/index.ts
@@ -116,6 +116,10 @@ export const api = {
requestPlain('POST', `/beta/api/hosts/export`, body, options),
search: (body: ApiSearchQuery, options?: ApiOptions) =>
requestJson<ApiSearchResult>('POST', `/beta/api/search`, body, options),
+ searchSidebar: (body: SidebarQuery, options?: ApiOptions) =>
+ requestJson<DisplayedSidebar>('POST', `/beta/api/search/sidebar`, body, options),
+ searchWidget: (body: WidgetQuery, options?: ApiOptions) =>
+ requestJson<Widget>('POST', `/beta/api/search/widget`, body, options),
summarize: (
query: {
query: string;
@@ -184,8 +188,6 @@ export const api = {
export type ApiSearchQuery = {
countResults?: boolean;
- fetchDiscussions?: boolean;
- fetchSidebar?: boolean;
flattenResponse?: boolean;
hostRankings?: HostRankings;
numResults?: number;
@@ -263,6 +265,7 @@ export type DisplayedWebpage = {
likelyHasPaywall: boolean;
prettyUrl: string;
rankingSignals?: {};
+ score?: number;
site: string;
snippet: Snippet;
title: string;
@@ -334,6 +337,9 @@ export type ScoredHost = {
host: string;
score: number;
};
+export type SidebarQuery = {
+ query: string;
+};
export type SignalScore = {
coefficient: number;
value: number;
@@ -385,15 +391,11 @@ export type ThesaurusWidget = {
};
export type UrlWrapper = string;
export type WebsitesResult = {
- directAnswer?: DisplayedAnswer;
- discussions?: DisplayedWebpage[];
hasMoreResults: boolean;
numHits?: number;
searchDurationMs: number;
- sidebar?: DisplayedSidebar;
spellCorrectedQuery?: HighlightedSpellCorrection;
webpages: DisplayedWebpage[];
- widget?: Widget;
};
export type Widget =
| {
@@ -404,6 +406,9 @@ export type Widget =
type: 'thesaurus';
value: ThesaurusWidget;
};
+export type WidgetQuery = {
+ query: string;
+};
export type WordMeaning = {
definition: Definition;
examples: Example[];
diff --git a/frontend/src/lib/search.ts b/frontend/src/lib/search.ts
index af2ff414..6324d48e 100644
--- a/frontend/src/lib/search.ts
+++ b/frontend/src/lib/search.ts
@@ -1,4 +1,4 @@
-import type { Region } from './api';
+import type { Region, Widget, DisplayedSidebar, DisplayedWebpage, WebsitesResult, BangHit } from './api';
import { decompressRanked, type RankedSites } from './rankings';
export type SearchParams = {
@@ -11,6 +11,17 @@ export type SearchParams = {
host_rankings: RankedSites | undefined;
};
+export type SearchResults =
+ | (WebsitesResult & {
+ type: 'websites';
+ widget?: Widget;
+ sidebar?: DisplayedSidebar;
+ discussions?: DisplayedWebpage[];
+ })
+ | (BangHit & {
+ type: 'bang';
+ });
+
export const extractSearchParams = (searchParams: URLSearchParams | FormData): SearchParams => {
const query = (searchParams.get('q') as string | undefined) ?? '';
const currentPage = parseInt((searchParams.get('p') as string | undefined) ?? '1') || 1;
@@ -32,3 +43,42 @@ export const extractSearchParams = (searchParams: URLSearchParams | FormData): S
host_rankings,
};
};
+
+export const discussionsOptic: string = `DiscardNonMatching;
+
+Rule {
+ Matches {
+ Schema("QAPage"),
+ }
+};
+
+Rule {
+ Matches {
+ Schema("DiscussionForumPosting"),
+ }
+};
+
+Rule {
+ Matches {
+ Domain("|reddit.com|"),
+ Url("comments"),
+ }
+};
+
+Rule {
+ Matches {
+ Site("|lemmy.world|")
+ }
+};
+
+Rule {
+ Matches {
+ Site("|lemmy.ml|")
+ }
+};
+
+Rule {
+ Matches {
+ Site("|sh.itjust.works|")
+ }
+};`;
\ No newline at end of file
diff --git a/frontend/src/routes/search/+page.server.ts b/frontend/src/routes/search/+page.server.ts
index a2b6426c..67356118 100644
--- a/frontend/src/routes/search/+page.server.ts
+++ b/frontend/src/routes/search/+page.server.ts
@@ -1,4 +1,4 @@
-import { extractSearchParams, type SearchParams } from '$lib/search';
+import { discussionsOptic, extractSearchParams, type SearchParams, type SearchResults } from '$lib/search';
import { redirect } from '@sveltejs/kit';
import type { Actions } from './$types';
import { api } from '$lib/api';
@@ -23,7 +23,9 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {
redirect(301, '/');
}
- const { data } = api.search(
+ let start = Date.now();
+
+ const { data: websitesReq } = api.search(
{
query: params.query,
page: params.currentPage - 1,
@@ -31,18 +33,65 @@ export const load = async ({ locals, fetch, url, getClientAddress }) => {
optic: params.optic && (await fetchRemoteOptic({ opticUrl: params.optic, fetch })),
selectedRegion: params.selectedRegion,
hostRankings: params.host_rankings,
- fetchDiscussions: true,
- fetchSidebar: true,
countResults: true,
},
{ fetch, headers: { 'X-Forwarded-For': getClientAddress() } },
);
- const results = await data;
+ const { data: widgetReq } = params.currentPage == 1 ?
+ api.searchWidget(
+ {
+ query: params.query,
+ },
+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },
+ )
+ : { data: undefined };
+
+ const { data: sidebarReq } = params.currentPage == 1 ?
+ api.searchSidebar(
+ {
+ query: params.query,
+ },
+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },
+ )
+ : { data: undefined };
+
+ const { data: discussionsReq } = params.currentPage == 1 ?
+ api.search(
+ {
+ query: params.query,
+ optic: discussionsOptic,
+ numResults: 10,
+ safeSearch: params.safeSearch,
+ selectedRegion: params.selectedRegion,
+ hostRankings: params.host_rankings,
+ returnRankingSignals: true,
+ countResults: false,
+ },
+ { fetch, headers: { 'X-Forwarded-For': getClientAddress() } },
+ )
+ : { data: undefined };
+
+ const [websites, widget, sidebar, discussionsRes] = await Promise.all([websitesReq, widgetReq, sidebarReq, discussionsReq]);
+ const discussions = discussionsRes?.type == "websites" ? discussionsRes.webpages : undefined;
+
+ let results: SearchResults = websites.type == "websites" ? {
+ ...websites,
+ widget,
+ sidebar,
+ discussions,
+ } : {
+ ...websites,
+ };
+
+ if (results.type == "websites") {
+ results.searchDurationMs = Date.now() - start;
+ }
return { form: searchParams, results };
};
+
export const actions: Actions = {
default: async (event) => {
const { request } = event;
diff --git a/frontend/src/routes/search/+page.svelte b/frontend/src/routes/search/+page.svelte
index 23b88c0b..867a90f1 100644
--- a/frontend/src/routes/search/+page.svelte
+++ b/frontend/src/routes/search/+page.svelte
@@ -63,10 +63,6 @@
modal = void 0;
}
- // $: {
- // results.discussions = results.webpages;
- // }
-
$: {
if (browser && results.type == 'websites') updateQueryId({ query, webpages: results.webpages });
}
@@ -126,10 +122,6 @@
<Widget widget={results.widget} />
{/if}
- {#if results.directAnswer}
- <DirectAnswer directAnswer={results.directAnswer} />
- {/if}
-
{#if results.webpages}
<div class="grid grid-cols-1 space-y-10 place-self-start">
{#each results.webpages as webpage, resultIndex (`${query}-${webpage.url}`)}
@@ -138,9 +130,7 @@
</div>
{/each}
{#if results.discussions}
- <div class="row-start-5">
- <Discussions discussions={results.discussions} />
- </div>
+ <Discussions discussions={results.discussions} />
{/if}
</div>
{/if}
diff --git a/frontend/src/routes/search/+page.ts b/frontend/src/routes/search/+page.ts
index 4469dd15..cc9d85f4 100644
--- a/frontend/src/routes/search/+page.ts
+++ b/frontend/src/routes/search/+page.ts
@@ -34,6 +34,7 @@ export const load: PageLoad = async (req) => {
return newParams;
})
.otherwise(() => {});
+
const nextPageSearchParams = match(results.type == 'websites' && results.hasMoreResults)
.with(true, () => {
const newParams = new URLSearchParams(url.searchParams);
diff --git a/frontend/src/routes/search/Discussions.svelte b/frontend/src/routes/search/Discussions.svelte
index 286ab087..a85e641a 100644
--- a/frontend/src/routes/search/Discussions.svelte
+++ b/frontend/src/routes/search/Discussions.svelte
@@ -5,14 +5,34 @@
import TextSnippet from '$lib/components/TextSnippet.svelte';
import Button from '$lib/components/Button.svelte';
- export let discussions: DisplayedWebpage[];
+ export let discussions: DisplayedWebpage[] | undefined;
let showMore = false;
- $: shownDiscussions = showMore ? discussions : discussions.slice(0, 4);
+ $: {
+ let medianScore = 0;
+ if (discussions) {
+ const scores = discussions.map((d) => d.score || 0);
+ scores.sort((a, b) => a - b);
+ const mid = Math.floor(scores.length / 2);
+ medianScore = scores.length % 2 !== 0 ? scores[mid] : (scores[mid - 1] + scores[mid]) / 2;
+ }
+
+ if (medianScore < 0.1) {
+ discussions = undefined;
+ }
+
+ let numDiscussions = discussions?.length || 0;
+ if (numDiscussions < 5) {
+ discussions = undefined;
+ }
+ }
+
+ $: shownDiscussions = showMore ? discussions : discussions?.slice(0, 4);
</script>
-<div class="flex flex-col space-y-1.5 overflow-hidden">
+{#if shownDiscussions && shownDiscussions.length > 0}
+<div class="flex flex-col space-y-1.5 overflow-hidden row-start-5">
<div class="flex items-center space-x-1 text-lg">
<ChatBubbleLeftRight class="text-sm text-neutral" />
<span>Discussions</span>
@@ -63,4 +83,5 @@
summary::-webkit-details-marker {
display: none;
}
-</style>
\ No newline at end of file
+</style>
+{/if}
\ No newline at end of file
diff --git a/frontend/src/routes/search/Modal.svelte b/frontend/src/routes/search/Modal.svelte
index 62ac4672..f7ca1d71 100644
--- a/frontend/src/routes/search/Modal.svelte
+++ b/frontend/src/routes/search/Modal.svelte
@@ -19,7 +19,7 @@
let innerWidth = 0;
let left = modal.left;
- let top = modal.top + 32;
+ $: top = modal.top + 32;
$: {
if (innerWidth < mediumWidthCutoff) {