web search done right
stackoverflow sidebar
Mikkel Denkerb518cf20b0c5
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rsindex fe7ad414..a2227c40 100644--- a/core/benches/search-preindexed-optic.rs+++ b/core/benches/search-preindexed-optic.rs@@ -17,10 +17,8 @@ macro_rules! bench {$searcher.search(&SearchQuery {original: $query.to_string(),- selected_region: None,optic_program: Some($optic.to_string()),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap()})diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rsindex 81643947..1a6996a7 100644--- a/core/benches/search-preindexed.rs+++ b/core/benches/search-preindexed.rs@@ -20,10 +20,7 @@ macro_rules! bench {$searcher.search(&SearchQuery {original: $query.to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap()})diff --git a/core/src/frontend/api.rs b/core/src/frontend/api.rsindex 6c1fff04..be360bce 100644--- a/core/src/frontend/api.rs+++ b/core/src/frontend/api.rs@@ -43,9 +43,8 @@ pub async fn search(.search_api(&SearchQuery {original: query.to_string(),selected_region,- optic_program: None,- site_rankings: None,skip_pages,+ ..Default::default()}).await{diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex 8ec4f719..8a97a0d3 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -22,7 +22,7 @@ use axum_macros::debug_handler;use crate::{search_prettifier::{- thousand_sep_number, CodeOrText, DisplayedEntity, DisplayedWebpage,+ thousand_sep_number, CodeOrText, DisplayedSidebar, DisplayedWebpage,HighlightedSpellCorrection, Snippet,},searcher::{self, PrettifiedSearchResult, SearchQuery},@@ -44,7 +44,7 @@ use axum::{struct SearchTemplate {search_result: Vec<DisplayedWebpage>,query: String,- entity: Option<DisplayedEntity>,+ sidebar: Option<DisplayedSidebar>,spell_correction: Option<HighlightedSpellCorrection>,num_matches: String,search_duration_sec: String,@@ -122,12 +122,13 @@ pub async fn route(optic_program: optic,skip_pages,site_rankings,+ custom_num_results: None,}).await{Ok(result) => match result {PrettifiedSearchResult::Websites(result) => {- let entity = result.entity;+ let sidebar = result.sidebar;let spell_correction = result.spell_corrected_query;let num_matches = thousand_sep_number(result.num_docs);@@ -178,7 +179,7 @@ pub async fn route(let template = SearchTemplate {search_result: result.webpages,query,- entity,+ sidebar,spell_correction,num_matches,search_duration_sec,diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex ff046874..ff86e8e9 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -17,6 +17,7 @@use crate::{ranking::SignalAggregator,schema::{Field, TextField},+ searcher::NUM_RESULTS_PER_PAGE,Result,};use optics::Optic;@@ -46,6 +47,7 @@ pub struct Query {terms: Vec<Box<Term>>,simple_terms_text: Vec<String>,tantivy_query: Box<BooleanQuery>,+ num_results: usize,}fn proximity_queries(@@ -153,6 +155,7 @@ impl Query {terms,simple_terms_text,tantivy_query,+ num_results: NUM_RESULTS_PER_PAGE,})}@@ -168,6 +171,10 @@ impl Query {self.simple_terms_text.clone()}+ pub fn set_num_results(&mut self, num_results: usize) {+ self.num_results = num_results;+ }+pub fn terms(&self) -> &[Box<Term>] {&self.terms}@@ -175,6 +182,10 @@ impl Query {pub fn is_empty(&self) -> bool {self.terms.is_empty()}++ pub fn num_results(&self) -> usize {+ self.num_results+ }}impl tantivy::query::Query for Query {diff --git a/core/src/query/optic.rs b/core/src/query/optic.rsindex 0656f6d5..68d02efe 100644--- a/core/src/query/optic.rs+++ b/core/src/query/optic.rs@@ -227,10 +227,7 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -245,7 +242,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"Rule {@@ -257,8 +253,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -272,7 +267,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"Rule {@@ -284,8 +278,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -375,12 +368,10 @@ mod tests {let _ = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(include_str!("../../../optics/testcases/quickstart.optic").to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -391,12 +382,10 @@ mod tests {let _ = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(include_str!("../../../optics/testcases/hacker_news.optic").to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -407,12 +396,10 @@ mod tests {let _ = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(include_str!("../../../optics/testcases/copycats_removal.optic").to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -523,7 +510,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"DiscardNonMatching;@@ -542,8 +528,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -698,7 +683,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"Like(Site("www.a.com"));@@ -707,8 +691,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -821,7 +804,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"DiscardNonMatching;@@ -833,8 +815,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -848,7 +829,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"DiscardNonMatching;@@ -860,8 +840,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -875,7 +854,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"DiscardNonMatching;@@ -887,8 +865,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -902,7 +879,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "website".to_string(),- selected_region: None,optic_program: Some(r#"DiscardNonMatching;@@ -914,8 +890,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -969,7 +944,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "site:stackoverflow.com".to_string(),- selected_region: None,optic_program: Some(r#"DiscardNonMatching;@@ -994,8 +968,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex a9abbc3a..ca96c14e 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -228,10 +228,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "example".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -312,10 +309,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "example".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -423,10 +417,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "dr dk".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -506,10 +497,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "title".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -594,10 +582,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -675,10 +660,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -784,7 +766,6 @@ mod tests {let res = searcher.search(&SearchQuery {original: "example".to_string(),- selected_region: None,optic_program: Some(r#"Ranking{Field("title"), 20000000};@@ -792,8 +773,7 @@ mod tests {"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -805,15 +785,13 @@ mod tests {let res = searcher.search(&SearchQuery {original: "example".to_string(),- selected_region: None,optic_program: Some(r#"Ranking{Signal("host_centrality"), 2000000};"#.to_string(),),- skip_pages: None,- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()@@ -922,10 +900,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "termA termB".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -1009,10 +984,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -1095,10 +1067,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -1268,10 +1237,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "topic_a".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -1286,10 +1252,7 @@ mod tests {let result = searcher.search(&SearchQuery {original: "topic_b".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rsindex a62b4280..7f2124a7 100644--- a/core/src/ranking/optics.rs+++ b/core/src/ranking/optics.rs@@ -27,7 +27,7 @@ use crate::{use super::{Signal, SignalAggregator};-pub const SCALE: f32 = 500.0;+pub const SCALE: f32 = 200.0;pub trait CreateAggregator {fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator;@@ -231,14 +231,12 @@ mod tests {let result = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,site_rankings: Some(SiteRankings {liked: vec!["www.first.com".to_string()],disliked: vec!["www.third.com".to_string()],blocked: vec![],}),+ ..Default::default()}).expect("Search failed").into_websites()@@ -254,14 +252,12 @@ mod tests {let result = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,site_rankings: Some(SiteRankings {liked: vec!["www.first.com".to_string()],disliked: vec![],blocked: vec![],}),+ ..Default::default()}).expect("Search failed").into_websites()@@ -277,14 +273,12 @@ mod tests {let result = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,site_rankings: Some(SiteRankings {liked: vec![],disliked: vec!["www.second.com".to_string()],blocked: vec!["www.first.com".to_string()],}),+ ..Default::default()}).expect("Search failed").into_websites()diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rsindex 2b3f350d..2d2f8802 100644--- a/core/src/search_prettifier.rs+++ b/core/src/search_prettifier.rs@@ -19,14 +19,15 @@ use std::collections::BTreeMap;use chrono::{NaiveDate, NaiveDateTime, Utc};use itertools::{intersperse, Itertools};use serde::{Deserialize, Serialize};+use tracing::log::debug;use crate::{entity_index::{entity::Span, StoredEntity},inverted_index::{self, RetrievedWebpage},- searcher::{self, LocalSearcher},+ searcher::{self, LocalSearcher, Sidebar},spell::CorrectionTerm,webpage::{- schema_org::{OneOrMany, Property},+ schema_org::{self, OneOrMany, Property},Url,},Error, Result,@@ -36,9 +37,15 @@ pub fn initial(result: searcher::local::InitialWebsiteResult,local_searcher: &LocalSearcher,) -> InitialWebsiteResult {- let entity = result- .entity- .map(|entity| DisplayedEntity::from(entity, local_searcher));+ let sidebar = result.sidebar.and_then(|sidebar| {+ let res = DisplayedSidebar::from(sidebar, local_searcher);+ if let Ok(sidebar) = res {+ Some(sidebar)+ } else {+ debug!("Failed to parse sidebar information: {:?}", res);+ None+ }+ });let spell_corrected_query = result.spell_corrected_query.map(|correction| {let mut highlighted = String::new();@@ -69,7 +76,7 @@ pub fn initial(InitialWebsiteResult {spell_corrected_query,websites: result.websites,- entity,+ sidebar,}}@@ -106,7 +113,7 @@ pub struct HighlightedSpellCorrection {pub struct InitialWebsiteResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub websites: inverted_index::InitialSearchResult,- pub entity: Option<DisplayedEntity>,+ pub sidebar: Option<DisplayedSidebar>,}pub fn html_escape(s: &str) -> String {@@ -244,6 +251,29 @@ fn parse_so_answer(})}+fn schema_item_to_stackoverflow_answer(+ item: schema_org::Item,+ url: Url,+ accepted: bool,+) -> Option<StackOverflowAnswer> {+ match (+ item.properties.get("text"),+ item.properties.get("dateCreated"),+ item.properties.get("upvoteCount"),+ item.properties.get("url"),+ ) {+ (Some(text), Some(date), Some(upvotes), Some(answer_url)) => parse_so_answer(+ text.clone(),+ date.clone(),+ upvotes.clone(),+ answer_url.clone(),+ url,+ accepted,+ ),+ _ => None,+ }+}+fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {match webpage.schema_org@@ -281,22 +311,7 @@ fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {.and_then(|ans| ans.one()).and_then(|prop| prop.try_into_item()).and_then(|item| {- match (- item.properties.get("text"),- item.properties.get("dateCreated"),- item.properties.get("upvoteCount"),- item.properties.get("url"),- ) {- (Some(text), Some(date), Some(upvotes), Some(url)) => parse_so_answer(- text.clone(),- date.clone(),- upvotes.clone(),- url.clone(),- Url::from(webpage.url.clone()),- true,- ),- _ => None,- }+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), true)}){answers.push(ans);@@ -311,22 +326,7 @@ fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {.into_iter().filter_map(|prop| prop.try_into_item()).filter_map(|item| {- match (- item.properties.get("text"),- item.properties.get("dateCreated"),- item.properties.get("upvoteCount"),- item.properties.get("url"),- ) {- (Some(text), Some(date), Some(upvotes), Some(url)) => parse_so_answer(- text.clone(),- date.clone(),- upvotes.clone(),- url.clone(),- Url::from(webpage.url.clone()),- false,- ),- _ => None,- }+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), false)}){answers.push(answer);@@ -405,6 +405,55 @@ impl From<RetrievedWebpage> for DisplayedWebpage {}}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum DisplayedSidebar {+ Entity(DisplayedEntity),+ StackOverflow {+ title: String,+ answer: StackOverflowAnswer,+ },+}++impl DisplayedSidebar {+ fn from(sidebar: Sidebar, searcher: &LocalSearcher) -> Result<Self> {+ match sidebar {+ Sidebar::Entity(entity) => Ok(DisplayedSidebar::Entity(DisplayedEntity::from(+ entity, searcher,+ ))),+ Sidebar::StackOverflow { schema_org, url } => {+ if let Some(item) = schema_org+ .into_iter()+ .find(|item| item.types_contains("QAPage"))+ .and_then(|item| item.properties.get("mainEntity").cloned())+ .and_then(|properties| properties.one())+ .and_then(|property| property.try_into_item())+ {+ let title = item+ .properties+ .get("name")+ .cloned()+ .and_then(|prop| prop.one())+ .and_then(|prop| prop.try_into_string())+ .ok_or(Error::InvalidStackoverflowSchema)?;++ item.properties+ .get("acceptedAnswer")+ .cloned()+ .and_then(|ans| ans.one())+ .and_then(|prop| prop.try_into_item())+ .and_then(|item| {+ schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)+ })+ .map(|answer| DisplayedSidebar::StackOverflow { title, answer })+ .ok_or(Error::InvalidStackoverflowSchema)+ } else {+ Err(Error::InvalidStackoverflowSchema)+ }+ }+ }+ }+}+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct DisplayedEntity {pub title: String,@@ -450,6 +499,7 @@ fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(| "website"| "logo"| "image caption"+ | "alt")}).take(5)diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 7408c27a..609a9377 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -320,9 +320,9 @@ impl DistributedSearcher {}});- let entity = initial_results.first().and_then(|result| {+ let sidebar = initial_results.first().and_then(|result| {if let InitialSearchResult::Websites(result) = &result.local_result {- result.entity.clone()+ result.sidebar.clone()} else {None}@@ -399,7 +399,7 @@ impl DistributedSearcher {num_docs,documents: retrieved_webpages,},- entity,+ sidebar,search_duration_ms: start.elapsed().as_millis(),}))}@@ -441,9 +441,9 @@ impl DistributedSearcher {}});- let entity = initial_results.first().and_then(|result| {+ let sidebar = initial_results.first().and_then(|result| {if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.entity.clone()+ result.sidebar.clone()} else {None}@@ -521,7 +521,7 @@ impl DistributedSearcher {spell_corrected_query,num_docs,webpages: retrieved_webpages,- entity,+ sidebar,search_duration_ms: start.elapsed().as_millis(),}))}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 5009d9a8..08dc2872 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -21,8 +21,8 @@ use optics::Optic;use serde::{Deserialize, Serialize};use uuid::Uuid;-use crate::bangs::Bangs;-use crate::entity_index::{EntityIndex, StoredEntity};+use crate::bangs::{BangHit, Bangs};+use crate::entity_index::EntityIndex;use crate::image_store::Image;use crate::index::Index;use crate::query::Query;@@ -35,7 +35,11 @@ use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};-use super::{InitialSearchResult, SearchQuery, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE};+use super::{+ InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult, NUM_RESULTS_PER_PAGE,+};++const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;pub struct LocalSearcher {index: Index,@@ -78,17 +82,7 @@ impl LocalSearcher {self.topic_centrality = Some(topic_centrality);}- pub fn search_initial(- &self,- query: &SearchQuery,- de_rank_similar: bool,- ) -> Result<InitialSearchResult> {- let raw_query = query.original.clone();- let optic = match query.optic_program.as_ref() {- Some(program) => Some(optics::parse(program)?),- None => None,- };-+ fn parse_query(&self, query: &SearchQuery, optic: Option<&Optic>) -> Result<Query> {let query_aggregator = optic.as_ref().map(|optic| {@@ -107,15 +101,28 @@ impl LocalSearcher {&query_aggregator,)?;- if parsed_query.is_empty() {- return Err(Error::EmptyQuery);+ if let Some(num_results) = query.custom_num_results {+ parsed_query.set_num_results(num_results)}- if let Some(bangs) = self.bangs.as_ref() {- if let Some(bang) = bangs.get(&parsed_query) {- return Ok(InitialSearchResult::Bang(bang));- }+ if parsed_query.is_empty() {+ Err(Error::EmptyQuery)+ } else {+ Ok(parsed_query)}+ }++ fn search_inverted_index(+ &self,+ query: &SearchQuery,+ de_rank_similar: bool,+ ) -> Result<inverted_index::InitialSearchResult> {+ let optic = match query.optic_program.as_ref() {+ Some(program) => Some(optics::parse(program)?),+ None => None,+ };++ let mut parsed_query = self.parse_query(query, optic.as_ref())?;let mut optics = Vec::new();@@ -172,26 +179,88 @@ impl LocalSearcher {ranker = ranker.with_max_docs(10_000_000, self.index.num_segments())- .with_num_results(NUM_RESULTS_PER_PAGE);+ .with_num_results(parsed_query.num_results());if let Some(skip_pages) = query.skip_pages {ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);}- let webpages = self- .index- .search_initial(&parsed_query, ranker.collector())?;- let correction = self.index.spell_correction(&parsed_query.simple_terms());+ self.index.search_initial(&parsed_query, ranker.collector())+ }- let entity = self- .entity_index+ fn spell_correction(&self, query: &SearchQuery) -> Result<Option<Correction>> {+ let parsed_query = self.parse_query(query, None)?;+ Ok(self.index.spell_correction(&parsed_query.simple_terms()))+ }++ fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {+ let parsed_query = self.parse_query(query, None)?;++ if let Some(bangs) = self.bangs.as_ref() {+ return Ok(bangs.get(&parsed_query));+ }++ Ok(None)+ }++ fn entity_sidebar(&self, query: &SearchQuery) -> Option<Sidebar> {+ self.entity_index.as_ref()- .and_then(|index| index.search(&raw_query));+ .and_then(|index| index.search(&query.original))+ .map(Sidebar::Entity)+ }++ fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {+ let mut query = query.clone();+ query.optic_program = Some(include_str!("stackoverflow.optic").to_string());+ query.custom_num_results = Some(1);++ let mut res = self.search_inverted_index(&query, false)?;++ if res.num_websites > 0 && !res.top_websites.is_empty() {+ let top = res.top_websites.remove(0);+ if top.score > STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD {+ let mut retrieved = self.retrieve_websites(&[top], &query.original)?;+ let retrieved = retrieved.remove(0);+ return Ok(Some(Sidebar::StackOverflow {+ schema_org: retrieved.schema_org,+ url: retrieved.url,+ }));+ }+ }++ Ok(None)+ }++ fn sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {+ if let Some(entity) = self.entity_sidebar(query) {+ return Ok(Some(entity));+ }++ if let Some(stackoverflow) = self.stackoverflow_sidebar(query)? {+ return Ok(Some(stackoverflow));+ }++ Ok(None)+ }++ pub fn search_initial(+ &self,+ query: &SearchQuery,+ de_rank_similar: bool,+ ) -> Result<InitialSearchResult> {+ if let Some(bang) = self.check_bangs(query)? {+ return Ok(InitialSearchResult::Bang(bang));+ }++ let webpages = self.search_inverted_index(query, de_rank_similar)?;+ let correction = self.spell_correction(query)?;+ let sidebar = self.sidebar(query)?;Ok(InitialSearchResult::Websites(InitialWebsiteResult {spell_corrected_query: correction,websites: webpages,- entity,+ sidebar,}))}@@ -231,7 +300,7 @@ impl LocalSearcher {num_docs: search_result.websites.num_websites,documents: retrieved_sites,},- entity: search_result.entity,+ sidebar: search_result.sidebar,search_duration_ms: start.elapsed().as_millis(),}))}@@ -267,7 +336,7 @@ impl LocalSearcher {pub struct InitialWebsiteResult {pub spell_corrected_query: Option<Correction>,pub websites: inverted_index::InitialSearchResult,- pub entity: Option<StoredEntity>,+ pub sidebar: Option<Sidebar>,}impl SearchResult {@@ -332,10 +401,8 @@ mod tests {let urls: Vec<_> = searcher.search(&SearchQuery {original: "test".to_string(),- selected_region: None,- optic_program: None,skip_pages: Some(p),- site_rankings: None,+ ..Default::default()}).unwrap().into_websites()diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex b73f4b2c..5f4ab6fd 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -26,9 +26,9 @@ use crate::{bangs::BangHit,entity_index::StoredEntity,inverted_index,- search_prettifier::{self, DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection},+ search_prettifier::{self, DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection},spell::Correction,- webpage::region::Region,+ webpage::{region::Region, schema_org},};pub const NUM_RESULTS_PER_PAGE: usize = 20;@@ -37,10 +37,19 @@ pub const NUM_RESULTS_PER_PAGE: usize = 20;pub struct WebsitesResult {pub spell_corrected_query: Option<Correction>,pub webpages: inverted_index::SearchResult,- pub entity: Option<StoredEntity>,+ pub sidebar: Option<Sidebar>,pub search_duration_ms: u128,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum Sidebar {+ Entity(StoredEntity),+ StackOverflow {+ schema_org: Vec<schema_org::Item>,+ url: String,+ },+}+#[derive(Debug, Serialize)]pub enum SearchResult {Websites(WebsitesResult),@@ -58,7 +67,7 @@ pub struct PrettifiedWebsitesResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,pub num_docs: usize,- pub entity: Option<DisplayedEntity>,+ pub sidebar: Option<DisplayedSidebar>,pub search_duration_ms: u128,}@@ -74,13 +83,14 @@ pub enum InitialPrettifiedSearchResult {Bang(BangHit),}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, Serialize, Deserialize, Clone, Default)]pub struct SearchQuery {pub original: String,pub selected_region: Option<Region>,pub optic_program: Option<String>,pub skip_pages: Option<usize>,pub site_rankings: Option<SiteRankings>,+ pub custom_num_results: Option<usize>,}impl SearchQuery {diff --git a/core/src/searcher/stackoverflow.optic b/core/src/searcher/stackoverflow.opticnew file mode 100644index 00000000..03bfbab9--- /dev/null+++ b/core/src/searcher/stackoverflow.optic@@ -0,0 +1,9 @@+DiscardNonMatching;++Rule {+ Matches {+ Domain("stackoverflow.com"),+ Schema("QAPage"),+ Schema("acceptedAnswer"),+ }+}\ No newline at end of filediff --git a/core/src/snippet.rs b/core/src/snippet.rsindex feee7ff3..7d498e01 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -389,10 +389,7 @@ Survey in 2016, 2017, and 2018."#;let result = searcher.search(&SearchQuery {original: "rust language".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -432,10 +429,7 @@ Survey in 2016, 2017, and 2018."#;let result = searcher.search(&SearchQuery {original: "describe".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()@@ -645,10 +639,7 @@ Survey in 2016, 2017, and 2018."#;let result = searcher.search(&SearchQuery {original: "paradigms".to_string(),- selected_region: None,- optic_program: None,- skip_pages: None,- site_rankings: None,+ ..Default::default()}).expect("Search failed").into_websites()diff --git a/frontend/src/components/Sidebar.astro b/frontend/src/components/Sidebar.astronew file mode 100644index 00000000..f316141d--- /dev/null+++ b/frontend/src/components/Sidebar.astro@@ -0,0 +1,21 @@+---+import { askama } from "../askama";+import Entity from "../components/Entity.astro";+import StackOverflow from "../components/StackOverflow.astro";++export interface Props {+ sidebar: string;+}+---+<div>+{+ askama.match("sidebar", {+ "DisplayedSidebar::Entity(entity)": (+ <Entity entity = "entity" />+ ),+ "DisplayedSidebar::StackOverflow{title, answer}": (+ <StackOverflow title = "title" answer = "answer" />+ )+ })+}+</div>\ No newline at end of filediff --git a/frontend/src/components/Snippet.astro b/frontend/src/components/Snippet.astroindex b12db1d2..d692b853 100644--- a/frontend/src/components/Snippet.astro+++ b/frontend/src/components/Snippet.astro@@ -6,8 +6,6 @@ export interface Props {snippet: string;}----<link rel="stylesheet" href="/css/highlightjs-default.min.css"/>-<script is:inline src="/js/highlight.min.js" /><div>{askama.match("item.snippet", {@@ -53,11 +51,13 @@ export interface Props {<div class="flex justify-between grow mb-2"><div>{askama`answer.date`}</div><div class="flex space-x-2">- <div class="flex space-x-1">- <div>+ <div class="flex items-center space-x-1">+ <span class="h-fit">{askama`answer.upvotes`}+ </span>+ <div class="h-fit">+ <Hero class="w-3" icon="thumb-up" /></div>- <Hero class="w-3" icon="thumb-up" /></div>{askama.if_("answer.accepted", () => (@@ -95,5 +95,4 @@ export interface Props {)})}-</div>-<script is:inline>hljs.highlightAll();</script>\ No newline at end of file+</div>\ No newline at end of filediff --git a/frontend/src/components/StackOverflow.astro b/frontend/src/components/StackOverflow.astronew file mode 100644index 00000000..4936c9f6--- /dev/null+++ b/frontend/src/components/StackOverflow.astro@@ -0,0 +1,49 @@+---+import { askama } from "../askama";+import Hero from "./Hero.astro";++export interface Props {+ title: string;+ answer: string;+}+---+<div class="flex flex-col space-y-5 border p-5 rounded-lg max-w-lg">+ <div class="flex flex-col space-y-1">+ <div class="flex justify-between space-x-2 grow">+ <div>+ <a class="text-lg font-medium" href={askama`answer.url`}>{askama`title`}</a>+ </div>+ <div class="flex space-x-1 items-center">+ <span class="h-fit">+ {askama`answer.upvotes`}+ </span>+ <div class="h-fit">+ <Hero class="w-4" icon="thumb-up" />+ </div>+ </div>+ </div>+ <div class="flex justify-between space-x-2 grow">+ <div><a href={askama`answer.url`}>{askama`answer.url`}</a></div>+ <div>{askama`answer.date`}</div>+ </div>+ </div>+ <hr />+ <div class="flex flex-col space-y-3">+ {+ askama.for_("part in answer.body", () => (+ <div>+ {+ askama.match("part", {+ "CodeOrText::Text(text)": (+ <span>{askama`text`}</span>+ ),+ "CodeOrText::Code(code)": (+ <pre class="bg-slate-50 rounded-lg"><code style="background: none">{askama `code`}</code></pre>+ )+ })+ }+ </div>+ ))+ }+ </div>+</div>\ No newline at end of filediff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astroindex 3f68fb18..d02d5293 100644--- a/frontend/src/pages/search.astro+++ b/frontend/src/pages/search.astro@@ -1,7 +1,7 @@---import Layout from "../layouts/Layout.astro";import Footer from "../components/Footer.astro";-import Entity from "../components/Entity.astro";+import Sidebar from "../components/Sidebar.astro";import Snippet from "../components/Snippet.astro";import Hero from "../components/Hero.astro";import OpticSelector from "../components/OpticSelector.astro";@@ -142,12 +142,12 @@ import { askama } from "../askama";</div></div>- <!-- Entity -->+ <!-- Sidebar -->{askama.if_(- "let Some(entity) = entity",+ "let Some(sidebar) = sidebar",<div class="row-start-2 flex justify-center pt-10 md:col-start-2 md:row-span-2 md:row-start-1">- <Entity entity="entity" />+ <Sidebar sidebar="sidebar" /></div>)}@@ -205,6 +205,8 @@ import { askama } from "../askama";</style></noscript>+<link rel="stylesheet" href="/css/highlightjs-default.min.css"/>+<script is:inline src="/js/highlight.min.js" /><script is:inline>var rankingAdjustButtons = document.querySelectorAll(".adjust-btn");var rankingModal = document.getElementById("ranking-modal");@@ -234,4 +236,5 @@ import { askama } from "../askama";}, 0);})});-</script>\ No newline at end of file+</script>+<script is:inline>hljs.highlightAll();</script>\ No newline at end of file