Skip to content

web search done right

stackoverflow sidebar

Mikkel Denkerb518cf20b0c5
diff --git a/core/benches/search-preindexed-optic.rs b/core/benches/search-preindexed-optic.rs
index fe7ad414..a2227c40 100644
--- a/core/benches/search-preindexed-optic.rs
+++ b/core/benches/search-preindexed-optic.rs
@@ -17,10 +17,8 @@ macro_rules! bench {
$searcher
.search(&SearchQuery {
original: $query.to_string(),
- selected_region: None,
optic_program: Some($optic.to_string()),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
})
diff --git a/core/benches/search-preindexed.rs b/core/benches/search-preindexed.rs
index 81643947..1a6996a7 100644
--- a/core/benches/search-preindexed.rs
+++ b/core/benches/search-preindexed.rs
@@ -20,10 +20,7 @@ macro_rules! bench {
$searcher
.search(&SearchQuery {
original: $query.to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
})
diff --git a/core/src/frontend/api.rs b/core/src/frontend/api.rs
index 6c1fff04..be360bce 100644
--- a/core/src/frontend/api.rs
+++ b/core/src/frontend/api.rs
@@ -43,9 +43,8 @@ pub async fn search(
.search_api(&SearchQuery {
original: query.to_string(),
selected_region,
- optic_program: None,
- site_rankings: None,
skip_pages,
+ ..Default::default()
})
.await
{
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rs
index 8ec4f719..8a97a0d3 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/frontend/search.rs
@@ -22,7 +22,7 @@ use axum_macros::debug_handler;
use crate::{
search_prettifier::{
- thousand_sep_number, CodeOrText, DisplayedEntity, DisplayedWebpage,
+ thousand_sep_number, CodeOrText, DisplayedSidebar, DisplayedWebpage,
HighlightedSpellCorrection, Snippet,
},
searcher::{self, PrettifiedSearchResult, SearchQuery},
@@ -44,7 +44,7 @@ use axum::{
struct SearchTemplate {
search_result: Vec<DisplayedWebpage>,
query: String,
- entity: Option<DisplayedEntity>,
+ sidebar: Option<DisplayedSidebar>,
spell_correction: Option<HighlightedSpellCorrection>,
num_matches: String,
search_duration_sec: String,
@@ -122,12 +122,13 @@ pub async fn route(
optic_program: optic,
skip_pages,
site_rankings,
+ custom_num_results: None,
})
.await
{
Ok(result) => match result {
PrettifiedSearchResult::Websites(result) => {
- let entity = result.entity;
+ let sidebar = result.sidebar;
let spell_correction = result.spell_corrected_query;
let num_matches = thousand_sep_number(result.num_docs);
@@ -178,7 +179,7 @@ pub async fn route(
let template = SearchTemplate {
search_result: result.webpages,
query,
- entity,
+ sidebar,
spell_correction,
num_matches,
search_duration_sec,
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index ff046874..ff86e8e9 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -17,6 +17,7 @@
use crate::{
ranking::SignalAggregator,
schema::{Field, TextField},
+ searcher::NUM_RESULTS_PER_PAGE,
Result,
};
use optics::Optic;
@@ -46,6 +47,7 @@ pub struct Query {
terms: Vec<Box<Term>>,
simple_terms_text: Vec<String>,
tantivy_query: Box<BooleanQuery>,
+ num_results: usize,
}
fn proximity_queries(
@@ -153,6 +155,7 @@ impl Query {
terms,
simple_terms_text,
tantivy_query,
+ num_results: NUM_RESULTS_PER_PAGE,
})
}
@@ -168,6 +171,10 @@ impl Query {
self.simple_terms_text.clone()
}
+ pub fn set_num_results(&mut self, num_results: usize) {
+ self.num_results = num_results;
+ }
+
pub fn terms(&self) -> &[Box<Term>] {
&self.terms
}
@@ -175,6 +182,10 @@ impl Query {
pub fn is_empty(&self) -> bool {
self.terms.is_empty()
}
+
+ pub fn num_results(&self) -> usize {
+ self.num_results
+ }
}
impl tantivy::query::Query for Query {
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 0656f6d5..68d02efe 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -227,10 +227,7 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -245,7 +242,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
Rule {
@@ -257,8 +253,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -272,7 +267,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
Rule {
@@ -284,8 +278,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -375,12 +368,10 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
include_str!("../../../optics/testcases/quickstart.optic").to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -391,12 +382,10 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
include_str!("../../../optics/testcases/hacker_news.optic").to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -407,12 +396,10 @@ mod tests {
let _ = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
include_str!("../../../optics/testcases/copycats_removal.optic").to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -523,7 +510,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
DiscardNonMatching;
@@ -542,8 +528,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -698,7 +683,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
Like(Site("www.a.com"));
@@ -707,8 +691,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -821,7 +804,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
DiscardNonMatching;
@@ -833,8 +815,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -848,7 +829,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
DiscardNonMatching;
@@ -860,8 +840,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -875,7 +854,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
DiscardNonMatching;
@@ -887,8 +865,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -902,7 +879,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "website".to_string(),
- selected_region: None,
optic_program: Some(
r#"
DiscardNonMatching;
@@ -914,8 +890,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -969,7 +944,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "site:stackoverflow.com".to_string(),
- selected_region: None,
optic_program: Some(
r#"
DiscardNonMatching;
@@ -994,8 +968,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index a9abbc3a..ca96c14e 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -228,10 +228,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "example".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -312,10 +309,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "example".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -423,10 +417,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "dr dk".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -506,10 +497,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "title".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -594,10 +582,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -675,10 +660,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -784,7 +766,6 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "example".to_string(),
- selected_region: None,
optic_program: Some(
r#"
Ranking{Field("title"), 20000000};
@@ -792,8 +773,7 @@ mod tests {
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -805,15 +785,13 @@ mod tests {
let res = searcher
.search(&SearchQuery {
original: "example".to_string(),
- selected_region: None,
optic_program: Some(
r#"
Ranking{Signal("host_centrality"), 2000000};
"#
.to_string(),
),
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
@@ -922,10 +900,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "termA termB".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -1009,10 +984,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -1095,10 +1067,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -1268,10 +1237,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "topic_a".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -1286,10 +1252,7 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "topic_b".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index a62b4280..7f2124a7 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -27,7 +27,7 @@ use crate::{
use super::{Signal, SignalAggregator};
-pub const SCALE: f32 = 500.0;
+pub const SCALE: f32 = 200.0;
pub trait CreateAggregator {
fn aggregator(&self, approx: Option<&ApproximatedHarmonicCentrality>) -> SignalAggregator;
@@ -231,14 +231,12 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
site_rankings: Some(SiteRankings {
liked: vec!["www.first.com".to_string()],
disliked: vec!["www.third.com".to_string()],
blocked: vec![],
}),
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -254,14 +252,12 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
site_rankings: Some(SiteRankings {
liked: vec!["www.first.com".to_string()],
disliked: vec![],
blocked: vec![],
}),
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -277,14 +273,12 @@ mod tests {
let result = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
site_rankings: Some(SiteRankings {
liked: vec![],
disliked: vec!["www.second.com".to_string()],
blocked: vec!["www.first.com".to_string()],
}),
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rs
index 2b3f350d..2d2f8802 100644
--- a/core/src/search_prettifier.rs
+++ b/core/src/search_prettifier.rs
@@ -19,14 +19,15 @@ use std::collections::BTreeMap;
use chrono::{NaiveDate, NaiveDateTime, Utc};
use itertools::{intersperse, Itertools};
use serde::{Deserialize, Serialize};
+use tracing::log::debug;
use crate::{
entity_index::{entity::Span, StoredEntity},
inverted_index::{self, RetrievedWebpage},
- searcher::{self, LocalSearcher},
+ searcher::{self, LocalSearcher, Sidebar},
spell::CorrectionTerm,
webpage::{
- schema_org::{OneOrMany, Property},
+ schema_org::{self, OneOrMany, Property},
Url,
},
Error, Result,
@@ -36,9 +37,15 @@ pub fn initial(
result: searcher::local::InitialWebsiteResult,
local_searcher: &LocalSearcher,
) -> InitialWebsiteResult {
- let entity = result
- .entity
- .map(|entity| DisplayedEntity::from(entity, local_searcher));
+ let sidebar = result.sidebar.and_then(|sidebar| {
+ let res = DisplayedSidebar::from(sidebar, local_searcher);
+ if let Ok(sidebar) = res {
+ Some(sidebar)
+ } else {
+ debug!("Failed to parse sidebar information: {:?}", res);
+ None
+ }
+ });
let spell_corrected_query = result.spell_corrected_query.map(|correction| {
let mut highlighted = String::new();
@@ -69,7 +76,7 @@ pub fn initial(
InitialWebsiteResult {
spell_corrected_query,
websites: result.websites,
- entity,
+ sidebar,
}
}
@@ -106,7 +113,7 @@ pub struct HighlightedSpellCorrection {
pub struct InitialWebsiteResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub websites: inverted_index::InitialSearchResult,
- pub entity: Option<DisplayedEntity>,
+ pub sidebar: Option<DisplayedSidebar>,
}
pub fn html_escape(s: &str) -> String {
@@ -244,6 +251,29 @@ fn parse_so_answer(
})
}
+fn schema_item_to_stackoverflow_answer(
+ item: schema_org::Item,
+ url: Url,
+ accepted: bool,
+) -> Option<StackOverflowAnswer> {
+ match (
+ item.properties.get("text"),
+ item.properties.get("dateCreated"),
+ item.properties.get("upvoteCount"),
+ item.properties.get("url"),
+ ) {
+ (Some(text), Some(date), Some(upvotes), Some(answer_url)) => parse_so_answer(
+ text.clone(),
+ date.clone(),
+ upvotes.clone(),
+ answer_url.clone(),
+ url,
+ accepted,
+ ),
+ _ => None,
+ }
+}
+
fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
match webpage
.schema_org
@@ -281,22 +311,7 @@ fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
.and_then(|ans| ans.one())
.and_then(|prop| prop.try_into_item())
.and_then(|item| {
- match (
- item.properties.get("text"),
- item.properties.get("dateCreated"),
- item.properties.get("upvoteCount"),
- item.properties.get("url"),
- ) {
- (Some(text), Some(date), Some(upvotes), Some(url)) => parse_so_answer(
- text.clone(),
- date.clone(),
- upvotes.clone(),
- url.clone(),
- Url::from(webpage.url.clone()),
- true,
- ),
- _ => None,
- }
+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), true)
})
{
answers.push(ans);
@@ -311,22 +326,7 @@ fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
.into_iter()
.filter_map(|prop| prop.try_into_item())
.filter_map(|item| {
- match (
- item.properties.get("text"),
- item.properties.get("dateCreated"),
- item.properties.get("upvoteCount"),
- item.properties.get("url"),
- ) {
- (Some(text), Some(date), Some(upvotes), Some(url)) => parse_so_answer(
- text.clone(),
- date.clone(),
- upvotes.clone(),
- url.clone(),
- Url::from(webpage.url.clone()),
- false,
- ),
- _ => None,
- }
+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), false)
})
{
answers.push(answer);
@@ -405,6 +405,55 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
}
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum DisplayedSidebar {
+ Entity(DisplayedEntity),
+ StackOverflow {
+ title: String,
+ answer: StackOverflowAnswer,
+ },
+}
+
+impl DisplayedSidebar {
+ fn from(sidebar: Sidebar, searcher: &LocalSearcher) -> Result<Self> {
+ match sidebar {
+ Sidebar::Entity(entity) => Ok(DisplayedSidebar::Entity(DisplayedEntity::from(
+ entity, searcher,
+ ))),
+ Sidebar::StackOverflow { schema_org, url } => {
+ if let Some(item) = schema_org
+ .into_iter()
+ .find(|item| item.types_contains("QAPage"))
+ .and_then(|item| item.properties.get("mainEntity").cloned())
+ .and_then(|properties| properties.one())
+ .and_then(|property| property.try_into_item())
+ {
+ let title = item
+ .properties
+ .get("name")
+ .cloned()
+ .and_then(|prop| prop.one())
+ .and_then(|prop| prop.try_into_string())
+ .ok_or(Error::InvalidStackoverflowSchema)?;
+
+ item.properties
+ .get("acceptedAnswer")
+ .cloned()
+ .and_then(|ans| ans.one())
+ .and_then(|prop| prop.try_into_item())
+ .and_then(|item| {
+ schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)
+ })
+ .map(|answer| DisplayedSidebar::StackOverflow { title, answer })
+ .ok_or(Error::InvalidStackoverflowSchema)
+ } else {
+ Err(Error::InvalidStackoverflowSchema)
+ }
+ }
+ }
+ }
+}
+
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct DisplayedEntity {
pub title: String,
@@ -450,6 +499,7 @@ fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(
| "website"
| "logo"
| "image caption"
+ | "alt"
)
})
.take(5)
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 7408c27a..609a9377 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -320,9 +320,9 @@ impl DistributedSearcher {
}
});
- let entity = initial_results.first().and_then(|result| {
+ let sidebar = initial_results.first().and_then(|result| {
if let InitialSearchResult::Websites(result) = &result.local_result {
- result.entity.clone()
+ result.sidebar.clone()
} else {
None
}
@@ -399,7 +399,7 @@ impl DistributedSearcher {
num_docs,
documents: retrieved_webpages,
},
- entity,
+ sidebar,
search_duration_ms: start.elapsed().as_millis(),
}))
}
@@ -441,9 +441,9 @@ impl DistributedSearcher {
}
});
- let entity = initial_results.first().and_then(|result| {
+ let sidebar = initial_results.first().and_then(|result| {
if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.entity.clone()
+ result.sidebar.clone()
} else {
None
}
@@ -521,7 +521,7 @@ impl DistributedSearcher {
spell_corrected_query,
num_docs,
webpages: retrieved_webpages,
- entity,
+ sidebar,
search_duration_ms: start.elapsed().as_millis(),
}))
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 5009d9a8..08dc2872 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -21,8 +21,8 @@ use optics::Optic;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
-use crate::bangs::Bangs;
-use crate::entity_index::{EntityIndex, StoredEntity};
+use crate::bangs::{BangHit, Bangs};
+use crate::entity_index::EntityIndex;
use crate::image_store::Image;
use crate::index::Index;
use crate::query::Query;
@@ -35,7 +35,11 @@ use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
-use super::{InitialSearchResult, SearchQuery, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE};
+use super::{
+ InitialSearchResult, SearchQuery, SearchResult, Sidebar, WebsitesResult, NUM_RESULTS_PER_PAGE,
+};
+
+const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;
pub struct LocalSearcher {
index: Index,
@@ -78,17 +82,7 @@ impl LocalSearcher {
self.topic_centrality = Some(topic_centrality);
}
- pub fn search_initial(
- &self,
- query: &SearchQuery,
- de_rank_similar: bool,
- ) -> Result<InitialSearchResult> {
- let raw_query = query.original.clone();
- let optic = match query.optic_program.as_ref() {
- Some(program) => Some(optics::parse(program)?),
- None => None,
- };
-
+ fn parse_query(&self, query: &SearchQuery, optic: Option<&Optic>) -> Result<Query> {
let query_aggregator = optic
.as_ref()
.map(|optic| {
@@ -107,15 +101,28 @@ impl LocalSearcher {
&query_aggregator,
)?;
- if parsed_query.is_empty() {
- return Err(Error::EmptyQuery);
+ if let Some(num_results) = query.custom_num_results {
+ parsed_query.set_num_results(num_results)
}
- if let Some(bangs) = self.bangs.as_ref() {
- if let Some(bang) = bangs.get(&parsed_query) {
- return Ok(InitialSearchResult::Bang(bang));
- }
+ if parsed_query.is_empty() {
+ Err(Error::EmptyQuery)
+ } else {
+ Ok(parsed_query)
}
+ }
+
+ fn search_inverted_index(
+ &self,
+ query: &SearchQuery,
+ de_rank_similar: bool,
+ ) -> Result<inverted_index::InitialSearchResult> {
+ let optic = match query.optic_program.as_ref() {
+ Some(program) => Some(optics::parse(program)?),
+ None => None,
+ };
+
+ let mut parsed_query = self.parse_query(query, optic.as_ref())?;
let mut optics = Vec::new();
@@ -172,26 +179,88 @@ impl LocalSearcher {
ranker = ranker
.with_max_docs(10_000_000, self.index.num_segments())
- .with_num_results(NUM_RESULTS_PER_PAGE);
+ .with_num_results(parsed_query.num_results());
if let Some(skip_pages) = query.skip_pages {
ranker = ranker.with_offset(NUM_RESULTS_PER_PAGE * skip_pages);
}
- let webpages = self
- .index
- .search_initial(&parsed_query, ranker.collector())?;
- let correction = self.index.spell_correction(&parsed_query.simple_terms());
+ self.index.search_initial(&parsed_query, ranker.collector())
+ }
- let entity = self
- .entity_index
+ fn spell_correction(&self, query: &SearchQuery) -> Result<Option<Correction>> {
+ let parsed_query = self.parse_query(query, None)?;
+ Ok(self.index.spell_correction(&parsed_query.simple_terms()))
+ }
+
+ fn check_bangs(&self, query: &SearchQuery) -> Result<Option<BangHit>> {
+ let parsed_query = self.parse_query(query, None)?;
+
+ if let Some(bangs) = self.bangs.as_ref() {
+ return Ok(bangs.get(&parsed_query));
+ }
+
+ Ok(None)
+ }
+
+ fn entity_sidebar(&self, query: &SearchQuery) -> Option<Sidebar> {
+ self.entity_index
.as_ref()
- .and_then(|index| index.search(&raw_query));
+ .and_then(|index| index.search(&query.original))
+ .map(Sidebar::Entity)
+ }
+
+ fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {
+ let mut query = query.clone();
+ query.optic_program = Some(include_str!("stackoverflow.optic").to_string());
+ query.custom_num_results = Some(1);
+
+ let mut res = self.search_inverted_index(&query, false)?;
+
+ if res.num_websites > 0 && !res.top_websites.is_empty() {
+ let top = res.top_websites.remove(0);
+ if top.score > STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD {
+ let mut retrieved = self.retrieve_websites(&[top], &query.original)?;
+ let retrieved = retrieved.remove(0);
+ return Ok(Some(Sidebar::StackOverflow {
+ schema_org: retrieved.schema_org,
+ url: retrieved.url,
+ }));
+ }
+ }
+
+ Ok(None)
+ }
+
+ fn sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {
+ if let Some(entity) = self.entity_sidebar(query) {
+ return Ok(Some(entity));
+ }
+
+ if let Some(stackoverflow) = self.stackoverflow_sidebar(query)? {
+ return Ok(Some(stackoverflow));
+ }
+
+ Ok(None)
+ }
+
+ pub fn search_initial(
+ &self,
+ query: &SearchQuery,
+ de_rank_similar: bool,
+ ) -> Result<InitialSearchResult> {
+ if let Some(bang) = self.check_bangs(query)? {
+ return Ok(InitialSearchResult::Bang(bang));
+ }
+
+ let webpages = self.search_inverted_index(query, de_rank_similar)?;
+ let correction = self.spell_correction(query)?;
+ let sidebar = self.sidebar(query)?;
Ok(InitialSearchResult::Websites(InitialWebsiteResult {
spell_corrected_query: correction,
websites: webpages,
- entity,
+ sidebar,
}))
}
@@ -231,7 +300,7 @@ impl LocalSearcher {
num_docs: search_result.websites.num_websites,
documents: retrieved_sites,
},
- entity: search_result.entity,
+ sidebar: search_result.sidebar,
search_duration_ms: start.elapsed().as_millis(),
}))
}
@@ -267,7 +336,7 @@ impl LocalSearcher {
pub struct InitialWebsiteResult {
pub spell_corrected_query: Option<Correction>,
pub websites: inverted_index::InitialSearchResult,
- pub entity: Option<StoredEntity>,
+ pub sidebar: Option<Sidebar>,
}
impl SearchResult {
@@ -332,10 +401,8 @@ mod tests {
let urls: Vec<_> = searcher
.search(&SearchQuery {
original: "test".to_string(),
- selected_region: None,
- optic_program: None,
skip_pages: Some(p),
- site_rankings: None,
+ ..Default::default()
})
.unwrap()
.into_websites()
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index b73f4b2c..5f4ab6fd 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -26,9 +26,9 @@ use crate::{
bangs::BangHit,
entity_index::StoredEntity,
inverted_index,
- search_prettifier::{self, DisplayedEntity, DisplayedWebpage, HighlightedSpellCorrection},
+ search_prettifier::{self, DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection},
spell::Correction,
- webpage::region::Region,
+ webpage::{region::Region, schema_org},
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
@@ -37,10 +37,19 @@ pub const NUM_RESULTS_PER_PAGE: usize = 20;
pub struct WebsitesResult {
pub spell_corrected_query: Option<Correction>,
pub webpages: inverted_index::SearchResult,
- pub entity: Option<StoredEntity>,
+ pub sidebar: Option<Sidebar>,
pub search_duration_ms: u128,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Sidebar {
+ Entity(StoredEntity),
+ StackOverflow {
+ schema_org: Vec<schema_org::Item>,
+ url: String,
+ },
+}
+
#[derive(Debug, Serialize)]
pub enum SearchResult {
Websites(WebsitesResult),
@@ -58,7 +67,7 @@ pub struct PrettifiedWebsitesResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub webpages: Vec<DisplayedWebpage>,
pub num_docs: usize,
- pub entity: Option<DisplayedEntity>,
+ pub sidebar: Option<DisplayedSidebar>,
pub search_duration_ms: u128,
}
@@ -74,13 +83,14 @@ pub enum InitialPrettifiedSearchResult {
Bang(BangHit),
}
-#[derive(Debug, Serialize, Deserialize, Clone)]
+#[derive(Debug, Serialize, Deserialize, Clone, Default)]
pub struct SearchQuery {
pub original: String,
pub selected_region: Option<Region>,
pub optic_program: Option<String>,
pub skip_pages: Option<usize>,
pub site_rankings: Option<SiteRankings>,
+ pub custom_num_results: Option<usize>,
}
impl SearchQuery {
diff --git a/core/src/searcher/stackoverflow.optic b/core/src/searcher/stackoverflow.optic
new file mode 100644
index 00000000..03bfbab9
--- /dev/null
+++ b/core/src/searcher/stackoverflow.optic
@@ -0,0 +1,9 @@
+DiscardNonMatching;
+
+Rule {
+ Matches {
+ Domain("stackoverflow.com"),
+ Schema("QAPage"),
+ Schema("acceptedAnswer"),
+ }
+}
\ No newline at end of file
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index feee7ff3..7d498e01 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -389,10 +389,7 @@ Survey in 2016, 2017, and 2018."#;
let result = searcher
.search(&SearchQuery {
original: "rust language".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -432,10 +429,7 @@ Survey in 2016, 2017, and 2018."#;
let result = searcher
.search(&SearchQuery {
original: "describe".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
@@ -645,10 +639,7 @@ Survey in 2016, 2017, and 2018."#;
let result = searcher
.search(&SearchQuery {
original: "paradigms".to_string(),
- selected_region: None,
- optic_program: None,
- skip_pages: None,
- site_rankings: None,
+ ..Default::default()
})
.expect("Search failed")
.into_websites()
diff --git a/frontend/src/components/Sidebar.astro b/frontend/src/components/Sidebar.astro
new file mode 100644
index 00000000..f316141d
--- /dev/null
+++ b/frontend/src/components/Sidebar.astro
@@ -0,0 +1,21 @@
+---
+import { askama } from "../askama";
+import Entity from "../components/Entity.astro";
+import StackOverflow from "../components/StackOverflow.astro";
+
+export interface Props {
+ sidebar: string;
+}
+---
+<div>
+{
+ askama.match("sidebar", {
+ "DisplayedSidebar::Entity(entity)": (
+ <Entity entity = "entity" />
+ ),
+ "DisplayedSidebar::StackOverflow{title, answer}": (
+ <StackOverflow title = "title" answer = "answer" />
+ )
+ })
+}
+</div>
\ No newline at end of file
diff --git a/frontend/src/components/Snippet.astro b/frontend/src/components/Snippet.astro
index b12db1d2..d692b853 100644
--- a/frontend/src/components/Snippet.astro
+++ b/frontend/src/components/Snippet.astro
@@ -6,8 +6,6 @@ export interface Props {
snippet: string;
}
---
-<link rel="stylesheet" href="/css/highlightjs-default.min.css"/>
-<script is:inline src="/js/highlight.min.js" />
<div>
{
askama.match("item.snippet", {
@@ -53,11 +51,13 @@ export interface Props {
<div class="flex justify-between grow mb-2">
<div>{askama`answer.date`}</div>
<div class="flex space-x-2">
- <div class="flex space-x-1">
- <div>
+ <div class="flex items-center space-x-1">
+ <span class="h-fit">
{askama`answer.upvotes`}
+ </span>
+ <div class="h-fit">
+ <Hero class="w-3" icon="thumb-up" />
</div>
- <Hero class="w-3" icon="thumb-up" />
</div>
{
askama.if_("answer.accepted", () => (
@@ -95,5 +95,4 @@ export interface Props {
)
})
}
-</div>
-<script is:inline>hljs.highlightAll();</script>
\ No newline at end of file
+</div>
\ No newline at end of file
diff --git a/frontend/src/components/StackOverflow.astro b/frontend/src/components/StackOverflow.astro
new file mode 100644
index 00000000..4936c9f6
--- /dev/null
+++ b/frontend/src/components/StackOverflow.astro
@@ -0,0 +1,49 @@
+---
+import { askama } from "../askama";
+import Hero from "./Hero.astro";
+
+export interface Props {
+ title: string;
+ answer: string;
+}
+---
+<div class="flex flex-col space-y-5 border p-5 rounded-lg max-w-lg">
+ <div class="flex flex-col space-y-1">
+ <div class="flex justify-between space-x-2 grow">
+ <div>
+ <a class="text-lg font-medium" href={askama`answer.url`}>{askama`title`}</a>
+ </div>
+ <div class="flex space-x-1 items-center">
+ <span class="h-fit">
+ {askama`answer.upvotes`}
+ </span>
+ <div class="h-fit">
+ <Hero class="w-4" icon="thumb-up" />
+ </div>
+ </div>
+ </div>
+ <div class="flex justify-between space-x-2 grow">
+ <div><a href={askama`answer.url`}>{askama`answer.url`}</a></div>
+ <div>{askama`answer.date`}</div>
+ </div>
+ </div>
+ <hr />
+ <div class="flex flex-col space-y-3">
+ {
+ askama.for_("part in answer.body", () => (
+ <div>
+ {
+ askama.match("part", {
+ "CodeOrText::Text(text)": (
+ <span>{askama`text`}</span>
+ ),
+ "CodeOrText::Code(code)": (
+ <pre class="bg-slate-50 rounded-lg"><code style="background: none">{askama `code`}</code></pre>
+ )
+ })
+ }
+ </div>
+ ))
+ }
+ </div>
+</div>
\ No newline at end of file
diff --git a/frontend/src/pages/search.astro b/frontend/src/pages/search.astro
index 3f68fb18..d02d5293 100644
--- a/frontend/src/pages/search.astro
+++ b/frontend/src/pages/search.astro
@@ -1,7 +1,7 @@
---
import Layout from "../layouts/Layout.astro";
import Footer from "../components/Footer.astro";
-import Entity from "../components/Entity.astro";
+import Sidebar from "../components/Sidebar.astro";
import Snippet from "../components/Snippet.astro";
import Hero from "../components/Hero.astro";
import OpticSelector from "../components/OpticSelector.astro";
@@ -142,12 +142,12 @@ import { askama } from "../askama";
</div>
</div>
- <!-- Entity -->
+ <!-- Sidebar -->
{
askama.if_(
- "let Some(entity) = entity",
+ "let Some(sidebar) = sidebar",
<div class="row-start-2 flex justify-center pt-10 md:col-start-2 md:row-span-2 md:row-start-1">
- <Entity entity="entity" />
+ <Sidebar sidebar="sidebar" />
</div>
)
}
@@ -205,6 +205,8 @@ import { askama } from "../askama";
</style>
</noscript>
+<link rel="stylesheet" href="/css/highlightjs-default.min.css"/>
+<script is:inline src="/js/highlight.min.js" />
<script is:inline>
var rankingAdjustButtons = document.querySelectorAll(".adjust-btn");
var rankingModal = document.getElementById("ranking-modal");
@@ -234,4 +236,5 @@ import { askama } from "../askama";
}, 0);
})
});
-</script>
\ No newline at end of file
+</script>
+<script is:inline>hljs.highlightAll();</script>
\ No newline at end of file