web search done right
Major refactor. Generate stackoverflow sidebar in distributed searcher as this allows us to re-use the existing search infrastructure
Mikkel Denker6cffad6e2870
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rsindex ce665b87..7bd9768b 100644--- a/core/src/entrypoint/search_server.rs+++ b/core/src/entrypoint/search_server.rs@@ -68,7 +68,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {}}}- searcher::Request::RetrieveWebites { websites, query } => {+ searcher::Request::RetrieveWebsites { websites, query } => {match local_searcher.retrieve_websites(websites, query) {Ok(response) => {req.respond(sonic::Response::Content(response)).await.ok();@@ -86,7 +86,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {match local_searcher.search_initial(query, false) {Ok(result) => match result {searcher::InitialSearchResult::Websites(result) => {- let res = search_prettifier::initial(result, &local_searcher);+ let res = search_prettifier::initial(result);req.respond(sonic::Response::Content(searcher::InitialPrettifiedSearchResult::Websites(res),@@ -109,25 +109,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {}}}- searcher::Request::RetrievePrettifiedWebites { websites, query } => {- match local_searcher.retrieve_websites(websites, query) {- Ok(result) => {- req.respond(sonic::Response::Content(search_prettifier::retrieve(- result,- &local_searcher,- )))- .await- .ok();- }- Err(_) => {- req.respond::<Vec<search_prettifier::DisplayedWebpage>>(- sonic::Response::Empty,- )- .await- .ok();- }- }- }}}}diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex a1a1d9ec..0adc6c2f 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -22,8 +22,8 @@ use axum_macros::debug_handler;use crate::{search_prettifier::{- thousand_sep_number, CodeOrText, DisplayedSidebar, DisplayedWebpage,- HighlightedSpellCorrection, Snippet,+ thousand_sep_number, CodeOrText, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,+ Snippet,},searcher::{self, PrettifiedSearchResult, SearchQuery, NUM_RESULTS_PER_PAGE},webpage::region::{Region, ALL_REGIONS},@@ -44,7 +44,7 @@ use axum::{struct SearchTemplate {search_result: Vec<DisplayedWebpage>,query: String,- sidebar: Option<DisplayedSidebar>,+ sidebar: Option<Sidebar>,spell_correction: Option<HighlightedSpellCorrection>,num_matches: String,search_duration_sec: String,diff --git a/core/src/index.rs b/core/src/index.rsindex 7273aad2..f0a09711 100644--- a/core/src/index.rs+++ b/core/src/index.rs@@ -152,7 +152,19 @@ impl Index {websites: &[inverted_index::WebsitePointer],query: &Query,) -> Result<Vec<inverted_index::RetrievedWebpage>> {- self.inverted_index.retrieve_websites(websites, query)+ let mut websites = self.inverted_index.retrieve_websites(websites, query)?;++ for website in &mut websites {+ let url = Url::from(website.url.clone());+ website.favicon = self.retrieve_favicon(&url);+ if let Some(uuid) = website.primary_image.as_ref().map(|image| &image.uuid) {+ if self.retrieve_primary_image(uuid).is_none() {+ website.primary_image = None;+ }+ }+ }++ Ok(websites)}pub fn merge(mut self, other: Self) -> Self {diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rsindex 9016d33a..c58c4af1 100644--- a/core/src/inverted_index.rs+++ b/core/src/inverted_index.rs@@ -414,8 +414,8 @@ pub struct SearchResult {pub struct RetrievedWebpage {pub title: String,pub url: String,- pub snippet: String,pub body: String,+ pub snippet: String,pub dirty_body: String,pub description: Option<String>,pub dmoz_description: Option<String>,diff --git a/core/src/main.rs b/core/src/main.rsindex 519d5fe3..51d6d485 100644--- a/core/src/main.rs+++ b/core/src/main.rs@@ -96,8 +96,8 @@ enum WebgraphOptions {config_path: String,},Merge {- path: String,- other_path: Option<String>,+ #[clap(required = true)]+ paths: Vec<String>,#[clap(long)]num_segments: Option<usize>,},@@ -195,13 +195,12 @@ fn main() -> Result<()> {entrypoint::Webgraph::run_locally(&config)?;}WebgraphOptions::Merge {- path,- other_path,+ mut paths,num_segments,} => {- let mut webgraph = WebgraphBuilder::new(path).open();+ let mut webgraph = WebgraphBuilder::new(paths.remove(0)).open();- if let Some(other_path) = other_path {+ for other_path in paths {let other = WebgraphBuilder::new(other_path).open();webgraph.merge(other);}diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rsindex 9ea8f7a2..3a498bd0 100644--- a/core/src/ranking/models/cross_encoder.rs+++ b/core/src/ranking/models/cross_encoder.rs@@ -114,8 +114,8 @@ mod tests {));}- Err(err) => {- dbg!(err);+ Err(_err) => {+ // dbg!(err);// panic!();}}diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rsindex ec6d3a21..d8735e6b 100644--- a/core/src/search_prettifier.rs+++ b/core/src/search_prettifier.rs@@ -19,34 +19,54 @@ use std::collections::BTreeMap;use chrono::{NaiveDate, NaiveDateTime, Utc};use itertools::{intersperse, Itertools};use serde::{Deserialize, Serialize};-use tracing::log::debug;use crate::{entity_index::{entity::Span, StoredEntity},inverted_index::RetrievedWebpage,ranking::pipeline::RankingWebsite,- searcher::{self, LocalSearcher, Sidebar},+ searcher::{self, LocalSearcher},spell::CorrectionTerm,webpage::{- schema_org::{self, OneOrMany, Property},+ schema_org::{self, Item, OneOrMany, Property},Url,},Error, Result,};-pub fn initial(- result: searcher::local::InitialWebsiteResult,- local_searcher: &LocalSearcher,-) -> InitialWebsiteResult {- let sidebar = result.sidebar.and_then(|sidebar| {- let res = DisplayedSidebar::from(sidebar, local_searcher);- if let Ok(sidebar) = res {- Some(sidebar)- } else {- debug!("Failed to parse sidebar information: {:?}", res);- None- }- });+#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum Snippet {+ Normal {+ date: Option<String>,+ text: String,+ },+ StackOverflowQA {+ question: StackOverflowQuestion,+ answers: Vec<StackOverflowAnswer>,+ },+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct StackOverflowAnswer {+ pub body: Vec<CodeOrText>,+ pub date: String,+ pub url: String,+ pub upvotes: u32,+ pub accepted: bool,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct StackOverflowQuestion {+ pub body: Vec<CodeOrText>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum CodeOrText {+ Code(String),+ Text(String),+}++pub fn initial(result: searcher::local::InitialWebsiteResult) -> InitialWebsiteResult {+ let sidebar = result.entity_sidebar;let spell_corrected_query = result.spell_corrected_query.map(|correction| {let mut highlighted = String::new();@@ -78,33 +98,10 @@ pub fn initial(spell_corrected_query,websites: result.websites,num_websites: result.num_websites,- sidebar,+ entity_sidebar: sidebar,}}-pub fn retrieve(result: Vec<RetrievedWebpage>, searcher: &LocalSearcher) -> Vec<DisplayedWebpage> {- result- .into_iter()- .map(|mut webpage| {- webpage.primary_image = webpage.primary_image.and_then(|image| {- if searcher- .primary_image(image.uuid.clone().to_string())- .is_some()- {- Some(image)- } else {- None- }- });-- let url: Url = webpage.url.clone().into();- webpage.favicon = searcher.favicon(&url.site().to_string().into());- webpage- })- .map(DisplayedWebpage::from)- .collect()-}-#[derive(Debug, Serialize, Deserialize, Clone)]pub struct HighlightedSpellCorrection {pub raw: String,@@ -116,7 +113,7 @@ pub struct InitialWebsiteResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub num_websites: usize,pub websites: Vec<RankingWebsite>,- pub sidebar: Option<DisplayedSidebar>,+ pub entity_sidebar: Option<DisplayedEntity>,}pub fn html_escape(s: &str) -> String {@@ -167,38 +164,6 @@ fn prettify_date(date: NaiveDateTime) -> String {format!("{}", date.format("%d. %b. %Y"))}-#[derive(Debug, Serialize, Deserialize, Clone)]-pub enum CodeOrText {- Code(String),- Text(String),-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct StackOverflowAnswer {- pub body: Vec<CodeOrText>,- pub date: String,- pub url: String,- pub upvotes: u32,- pub accepted: bool,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct StackOverflowQuestion {- pub body: Vec<CodeOrText>,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub enum Snippet {- Normal {- date: Option<String>,- text: String,- },- StackOverflowQA {- question: StackOverflowQuestion,- answers: Vec<StackOverflowAnswer>,- },-}-fn parse_so_answer(text: OneOrMany<Property>,date: OneOrMany<Property>,@@ -409,7 +374,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {}#[derive(Debug, Serialize, Deserialize, Clone)]-pub enum DisplayedSidebar {+pub enum Sidebar {Entity(DisplayedEntity),StackOverflow {title: String,@@ -417,43 +382,34 @@ pub enum DisplayedSidebar {},}-impl DisplayedSidebar {- fn from(sidebar: Sidebar, searcher: &LocalSearcher) -> Result<Self> {- match sidebar {- Sidebar::Entity(entity) => Ok(DisplayedSidebar::Entity(DisplayedEntity::from(- entity, searcher,- ))),- Sidebar::StackOverflow { schema_org, url } => {- if let Some(item) = schema_org- .into_iter()- .find(|item| item.types_contains("QAPage"))- .and_then(|item| item.properties.get("mainEntity").cloned())- .and_then(|properties| properties.one())- .and_then(|property| property.try_into_item())- {- let title = item- .properties- .get("name")- .cloned()- .and_then(|prop| prop.one())- .and_then(|prop| prop.try_into_string())- .ok_or(Error::InvalidStackoverflowSchema)?;-- item.properties- .get("acceptedAnswer")- .cloned()- .and_then(|ans| ans.one())- .and_then(|prop| prop.try_into_item())- .and_then(|item| {- schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)- })- .map(|answer| DisplayedSidebar::StackOverflow { title, answer })- .ok_or(Error::InvalidStackoverflowSchema)- } else {- Err(Error::InvalidStackoverflowSchema)- }- }- }+pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Result<Sidebar> {+ if let Some(item) = schema_org+ .into_iter()+ .find(|item| item.types_contains("QAPage"))+ .and_then(|item| item.properties.get("mainEntity").cloned())+ .and_then(|properties| properties.one())+ .and_then(|property| property.try_into_item())+ {+ let title = item+ .properties+ .get("name")+ .cloned()+ .and_then(|prop| prop.one())+ .and_then(|prop| prop.try_into_string())+ .ok_or(Error::InvalidStackoverflowSchema)?;++ item.properties+ .get("acceptedAnswer")+ .cloned()+ .and_then(|ans| ans.one())+ .and_then(|prop| prop.try_into_item())+ .and_then(|item| {+ schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)+ })+ .map(|answer| Sidebar::StackOverflow { title, answer })+ .ok_or(Error::InvalidStackoverflowSchema)+ } else {+ Err(Error::InvalidStackoverflowSchema)}}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 01ae6857..eda6904f 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -17,16 +17,17 @@use crate::{collector::{self, BucketCollector},exponential_backoff::ExponentialBackoff,- inverted_index::{self},+ inverted_index::{self, RetrievedWebpage},ranking::{models::cross_encoder::CrossEncoderModel,pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},},- search_prettifier::DisplayedWebpage,+ search_prettifier::{create_stackoverflow_sidebar, DisplayedWebpage, Sidebar},searcher::PrettifiedWebsitesResult,};use std::{+ cmp::Ordering,net::SocketAddr,sync::Arc,time::{Duration, Instant},@@ -42,6 +43,7 @@ use crate::sonic;use super::{InitialPrettifiedSearchResult, PrettifiedSearchResult, SearchQuery};type Result<T> = std::result::Result<T, Error>;+const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 100.0;struct RemoteSearcher {addr: SocketAddr,@@ -83,7 +85,7 @@ impl RemoteSearcher {&self,pointers: &[inverted_index::WebsitePointer],original_query: &str,- ) -> Result<Vec<DisplayedWebpage>> {+ ) -> Result<Vec<RetrievedWebpage>> {for timeout in ExponentialBackoff::from_millis(30).with_limit(Duration::from_millis(200)).take(5)@@ -91,7 +93,7 @@ impl RemoteSearcher {if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await{if let Ok(sonic::Response::Content(body)) = connection- .send(Request::RetrievePrettifiedWebites {+ .send(Request::RetrieveWebsites {websites: pointers.to_vec(),query: original_query.to_string(),})@@ -151,7 +153,7 @@ impl Shard {&self,pointers: &[inverted_index::WebsitePointer],original_query: &str,- ) -> Result<Vec<DisplayedWebpage>> {+ ) -> Result<Vec<RetrievedWebpage>> {match self.replicas.iter()@@ -175,11 +177,7 @@ struct InitialSearchResultShard {pub enum Request {Search(SearchQuery),SearchPrettified(SearchQuery),- RetrieveWebites {- websites: Vec<inverted_index::WebsitePointer>,- query: String,- },- RetrievePrettifiedWebites {+ RetrieveWebsites {websites: Vec<inverted_index::WebsitePointer>,query: String,},@@ -228,67 +226,14 @@ impl DistributedSearcher {}}- pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {- let start = Instant::now();-- if query.is_empty() {- return Err(Error::EmptyQuery);- }-- let mut search_query = query.clone();+ fn combine_results(+ &self,+ initial_results: Vec<InitialSearchResultShard>,+ search_query: &mut SearchQuery,+ ) -> Result<Vec<ScoredWebsitePointer>> {let pipeline: RankingPipeline<ScoredWebsitePointer> =- RankingPipeline::for_query(&mut search_query, self.cross_encoder.clone())?;+ RankingPipeline::for_query(search_query, self.cross_encoder.clone())?;- // search shards- let initial_results = self- .shards- .iter()- .map(|shard| shard.search(&search_query))- .collect::<FuturesUnordered<_>>()- .collect::<Vec<_>>()- .await- .into_iter()- .filter_map(|result| result.ok())- .collect::<Vec<_>>();-- // check if any result has a bang hit- if let Some(result) = initial_results- .iter()- .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_)))- {- if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {- return Ok(PrettifiedSearchResult::Bang(bang.clone()));- }- }-- let spell_corrected_query = initial_results.first().and_then(|result| {- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.spell_corrected_query.clone()- } else {- None- }- });-- let sidebar = initial_results.first().and_then(|result| {- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.sidebar.clone()- } else {- None- }- });-- let num_docs = initial_results- .iter()- .map(|result| {- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {- result.num_websites- } else {- 0- }- })- .sum();-- // combine resultslet mut collector =BucketCollector::new(pipeline.collector_top_n() + pipeline.collector_offset());@@ -312,9 +257,14 @@ impl DistributedSearcher {.take(pipeline.collector_top_n()).collect::<Vec<_>>();- let top_websites = pipeline.apply(top_websites);+ Ok(pipeline.apply(top_websites))+ }- // retrieve webpages+ async fn retrieve_webpages(+ &self,+ top_websites: &[ScoredWebsitePointer],+ query: &str,+ ) -> Vec<RetrievedWebpage> {let mut retrieved_webpages = Vec::new();for _ in 0..top_websites.len() {@@ -329,7 +279,7 @@ impl DistributedSearcher {.map(|(idx, pointer)| (idx, pointer.website.pointer.clone())).unzip();- if let Ok(websites) = shard.retrieve_websites(&pointers, &query.query).await {+ if let Ok(websites) = shard.retrieve_websites(&pointers, query).await {for (index, website) in indexes.into_iter().zip(websites.into_iter()) {retrieved_webpages[index] = Some(website);}@@ -340,6 +290,132 @@ impl DistributedSearcher {debug_assert_eq!(retrieved_webpages.len(), top_websites.len());+ retrieved_webpages+ }++ async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {+ self.shards+ .iter()+ .map(|shard| shard.search(query))+ .collect::<FuturesUnordered<_>>()+ .collect::<Vec<_>>()+ .await+ .into_iter()+ .filter_map(|result| result.ok())+ .collect::<Vec<_>>()+ }++ async fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {+ let query = SearchQuery {+ query: query.query.clone(),+ num_results: 1,+ optic_program: Some(include_str!("stackoverflow.optic").to_string()),+ ..Default::default()+ };++ let mut results: Vec<_> = self+ .search_initial(&query)+ .await+ .into_iter()+ .filter_map(|result| match result.local_result {+ InitialPrettifiedSearchResult::Websites(websites) => {+ if let Some(website) = websites.websites.first().cloned() {+ Some((result.shard, website))+ } else {+ None+ }+ }+ InitialPrettifiedSearchResult::Bang(_) => None,+ })+ .collect();++ results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));++ if let Some((shard, website)) = results.pop() {+ if website.score > STACKOVERFLOW_SIDEBAR_THRESHOLD {+ let scored_websites = vec![ScoredWebsitePointer { website, shard }];+ let mut retrieved = self.retrieve_webpages(&scored_websites, &query.query).await;++ if let Some(res) = retrieved.pop() {+ return Ok(Some(create_stackoverflow_sidebar(res.schema_org, res.url)?));+ }+ }+ }++ Ok(None)+ }++ async fn sidebar(+ &self,+ initial_results: &[InitialSearchResultShard],+ query: &SearchQuery,+ ) -> Result<Option<Sidebar>> {+ let entity = initial_results.first().and_then(|result| {+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {+ result.entity_sidebar.clone()+ } else {+ None+ }+ });++ match entity {+ Some(entity) => Ok(Some(Sidebar::Entity(entity))),+ None => Ok(self.stackoverflow_sidebar(query).await?),+ }+ }++ pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {+ let start = Instant::now();++ if query.is_empty() {+ return Err(Error::EmptyQuery);+ }++ let mut search_query = query.clone();++ let initial_results = self.search_initial(query).await;++ // check if any result has a bang hit+ if let Some(result) = initial_results+ .iter()+ .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_)))+ {+ if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {+ return Ok(PrettifiedSearchResult::Bang(bang.clone()));+ }+ }++ let sidebar = self.sidebar(&initial_results, query).await?;++ let spell_corrected_query = initial_results.first().and_then(|result| {+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {+ result.spell_corrected_query.clone()+ } else {+ None+ }+ });++ let num_docs = initial_results+ .iter()+ .map(|result| {+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {+ result.num_websites+ } else {+ 0+ }+ })+ .sum();++ let top_websites = self.combine_results(initial_results, &mut search_query)?;++ // retrieve webpages+ let retrieved_webpages: Vec<_> = self+ .retrieve_webpages(&top_websites, &query.query)+ .await+ .into_iter()+ .map(DisplayedWebpage::from)+ .collect();+if retrieved_webpages.is_empty() && !top_websites.is_empty() {return Err(Error::SearchFailed);}diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 3bd3088a..d16fda5f 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize};use uuid::Uuid;use crate::bangs::{BangHit, Bangs};-use crate::entity_index::EntityIndex;+use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;use crate::query::Query;@@ -29,6 +29,7 @@ use crate::ranking::centrality_store::CentralityStore;use crate::ranking::optics::CreateAggregator;use crate::ranking::pipeline::RankingWebsite;use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};+use crate::search_prettifier::{DisplayedEntity, Sidebar};use crate::spell::Correction;use crate::webgraph::centrality::topic::TopicCentrality;use crate::webpage::region::Region;@@ -36,9 +37,7 @@ use crate::webpage::Url;use crate::{inverted_index, Error, Result};use super::WebsitesResult;-use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};--const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;+use super::{InitialSearchResult, SearchQuery, SearchResult};pub struct LocalSearcher {index: Index,@@ -46,7 +45,6 @@ pub struct LocalSearcher {bangs: Option<Bangs>,centrality_store: Option<CentralityStore>,topic_centrality: Option<TopicCentrality>,- stackoverflow_sidebar_threshold: f64,}impl From<Index> for LocalSearcher {@@ -63,7 +61,6 @@ impl LocalSearcher {bangs: None,centrality_store: None,topic_centrality: None,- stackoverflow_sidebar_threshold: STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD,}}@@ -228,45 +225,10 @@ impl LocalSearcher {Ok(None)}- fn entity_sidebar(&self, query: &SearchQuery) -> Option<Sidebar> {+ fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {self.entity_index.as_ref().and_then(|index| index.search(&query.query))- .map(Sidebar::Entity)- }-- fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {- let mut query = query.clone();- query.optic_program = Some(include_str!("stackoverflow.optic").to_string());- query.num_results = 1;-- let (mut top_websites, num_websites) = self.search_inverted_index(&query, false)?;-- if num_websites > 0 && !top_websites.is_empty() {- let top = top_websites.remove(0);- if top.score > self.stackoverflow_sidebar_threshold {- let mut retrieved = self.retrieve_websites(&[top.pointer], &query.query)?;- let retrieved = retrieved.remove(0);- return Ok(Some(Sidebar::StackOverflow {- schema_org: retrieved.schema_org,- url: retrieved.url,- }));- }- }-- Ok(None)- }-- fn sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {- if let Some(entity) = self.entity_sidebar(query) {- return Ok(Some(entity));- }-- if let Some(stackoverflow) = self.stackoverflow_sidebar(query)? {- return Ok(Some(stackoverflow));- }-- Ok(None)}pub fn search_initial(@@ -280,13 +242,15 @@ impl LocalSearcher {let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;let correction = self.spell_correction(query)?;- let sidebar = self.sidebar(query)?;+ let sidebar = self+ .entity_sidebar(query)+ .map(|entity| DisplayedEntity::from(entity, self));Ok(InitialSearchResult::Websites(InitialWebsiteResult {spell_corrected_query: correction,websites,num_websites,- sidebar,+ entity_sidebar: sidebar,}))}@@ -344,7 +308,7 @@ impl LocalSearcher {num_docs: search_result.num_websites,documents: retrieved_sites,},- sidebar: search_result.sidebar,+ sidebar: search_result.entity_sidebar.map(Sidebar::Entity),search_duration_ms: start.elapsed().as_millis(),}))}@@ -381,7 +345,7 @@ pub struct InitialWebsiteResult {pub spell_corrected_query: Option<Correction>,pub num_websites: usize,pub websites: Vec<RankingWebsite>,- pub sidebar: Option<Sidebar>,+ pub entity_sidebar: Option<DisplayedEntity>,}impl SearchResult {@@ -471,36 +435,4 @@ mod tests {}}}-- #[test]- fn stackoverflow_sidebar() {- let mut index = Index::temporary().expect("Unable to open index");-- index- .insert(Webpage::new(- include_str!("../../testcases/schema_org/stackoverflow.html"),- "https://www.stackoverflow.com",- ))- .expect("failed to insert webpage");-- index.commit().unwrap();-- let mut searcher = LocalSearcher::new(index);- searcher.stackoverflow_sidebar_threshold = 0.0;-- let res = searcher- .search(&SearchQuery {- query: "regex parse html".to_string(),- ..Default::default()- })- .unwrap();-- assert!(matches!(- res.into_websites().unwrap().sidebar.unwrap(),- Sidebar::StackOverflow {- schema_org: _,- url: _- }- ));- }}diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex f9727f97..384d5d58 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -24,11 +24,10 @@ use serde::{Deserialize, Serialize};use crate::{bangs::BangHit,- entity_index::StoredEntity,inverted_index,- search_prettifier::{self, DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection},+ search_prettifier::{self, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},spell::Correction,- webpage::{region::Region, schema_org},+ webpage::region::Region,};pub const NUM_RESULTS_PER_PAGE: usize = 20;@@ -41,15 +40,6 @@ pub struct WebsitesResult {pub search_duration_ms: u128,}-#[derive(Debug, Serialize, Deserialize, Clone)]-pub enum Sidebar {- Entity(StoredEntity),- StackOverflow {- schema_org: Vec<schema_org::Item>,- url: String,- },-}-#[derive(Debug, Serialize)]pub enum SearchResult {Websites(WebsitesResult),@@ -67,7 +57,7 @@ pub struct PrettifiedWebsitesResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,pub num_docs: usize,- pub sidebar: Option<DisplayedSidebar>,+ pub sidebar: Option<Sidebar>,pub search_duration_ms: u128,}diff --git a/core/src/snippet.rs b/core/src/snippet.rsindex e4c6bf6c..81863f34 100644--- a/core/src/snippet.rs+++ b/core/src/snippet.rs@@ -26,7 +26,7 @@ use tantivy::tokenizer::{TextAnalyzer, Token};use tantivy::{Score, Searcher};use whatlang::Lang;-/// For now we use the snippet generator from tantivy with a minor modifications.+/// For now we use the snippet generator from tantivy with modifications./// In the future we want to implement something closer to the method described in <https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf>./// This will require us to store each paragraph of the webpage separately to get adequate performance./// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.@@ -74,7 +74,7 @@ impl FragmentCandidate {/// `Snippet`/// Contains a fragment of a document, and some highlighed parts inside it.#[derive(Debug)]-struct Snippet {+struct SnippetString {fragment: String,highlighted: Vec<Range<usize>>,}@@ -82,7 +82,7 @@ struct Snippet {const HIGHLIGHTEN_PREFIX: &str = "<b style=\"font-weight: 700;\">";const HIGHLIGHTEN_POSTFIX: &str = "</b>";-impl Snippet {+impl SnippetString {/// Returns a hignlightned html from the `Snippet`.fn to_html(&self) -> String {let mut html = String::new();@@ -156,7 +156,7 @@ fn search_fragments(////// Takes a vector of `FragmentCandidate`s and the text./// Figures out the best fragment from it and creates a snippet.-fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str) -> Snippet {+fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str) -> SnippetString {let best_fragment_opt = fragments.iter().max_by(|left, right| {let cmp_score = left.score@@ -175,14 +175,14 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str).iter().map(|item| item.start - fragment.start_offset..item.end - fragment.start_offset).collect();- Snippet {+ SnippetString {fragment: fragment_text.to_string(),highlighted,}} else {// when there no fragments to chose from,// for now create a empty snippet- Snippet {+ SnippetString {fragment: String::new(),highlighted: vec![],}@@ -246,7 +246,7 @@ impl SnippetGenerator {}/// Generates a snippet for the given text.- fn snippet(&self, text: &str) -> Snippet {+ fn snippet(&self, text: &str) -> SnippetString {let fragment_candidates =search_fragments(&self.tokenizer, text, &self.terms_text, self.max_num_chars);select_best_fragment_combination(&fragment_candidates[..], text)@@ -324,7 +324,7 @@ fn generate_snippet_without_stem(searcher: &Searcher,query: &Query,field: Field,-) -> Result<Snippet> {+) -> Result<SnippetString> {let tokenizer = Tokenizer::default().into();let generator = SnippetGenerator::create(searcher, query, tokenizer, field)?;diff --git a/core/src/sonic.rs b/core/src/sonic.rsindex 3f87e570..2a3a4800 100644--- a/core/src/sonic.rs+++ b/core/src/sonic.rs@@ -59,7 +59,7 @@ unsafe fn any_as_u8_slice<T: Sized>(p: &T) -> &[u8] {impl<T> Request<T> {pub async fn respond<R: Serialize>(mut self, response: Response<R>) -> Result<()> {- let bytes = bincode::serialize(&response)?;+ let bytes = bincode::serialize(&response).unwrap();let header = Header {body_size: bytes.len(),};@@ -102,7 +102,7 @@ impl Server {stream.read_exact(&mut buf).await?;tracing::debug!("received bytes: {:?}", &buf);- let body = bincode::deserialize(&buf)?;+ let body = bincode::deserialize(&buf).unwrap();Ok(Request { stream, body })}@@ -134,7 +134,7 @@ impl Connection {mut self,request: T,) -> Result<Response<R>> {- let bytes = bincode::serialize(&request)?;+ let bytes = bincode::serialize(&request).unwrap();let header = Header {body_size: bytes.len(),@@ -155,7 +155,7 @@ impl Connection {self.stream.shutdown().await?;- Ok(bincode::deserialize(&buf)?)+ Ok(bincode::deserialize(&buf).unwrap())}pub async fn send<T: Serialize, R: DeserializeOwned + Serialize>(diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 7d37975a..b0a648d6 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -36,6 +36,8 @@ mod executor;use self::executor::Executor;use self::segment::{CachedStore, SegmentNodeID};+const MAX_NUM_SEGMENTS: usize = 200;+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]pub struct NodeID(pub u64);@@ -458,6 +460,8 @@ impl<S: Store> Webgraph<S> {self.save_metadata();self.node2id.flush();self.id2node.flush();++ self.merge_segments(MAX_NUM_SEGMENTS);}pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rsindex 032768c2..b9866086 100644--- a/core/src/webpage/mod.rs+++ b/core/src/webpage/mod.rs@@ -17,8 +17,7 @@ use crate::{human_website_annotations::Topic,prehashed::{hash, split_u128},schema::{FastField, TextField},- simhash,- tokenizer::{self, FlattenedJson},+ simhash, tokenizer,webgraph::NodeID,Error, Result,};@@ -709,17 +708,11 @@ impl Html {let site = self.pretokenize_site();let description = self.pretokenize_description();- let schemas: Vec<_> = self- .schema_org()- .into_iter()- .map(|schema| schema.into_single_map())- .collect();+ let schemas: Vec<_> = self.schema_org();- let schema_json = serde_json::to_string(&self.schema_org())- .ok()- .unwrap_or_default();+ let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();- let pretokenized_schema_json = match FlattenedJson::new(&schemas) {+ let pretokenized_schema_json = match schema_org::flattened_json(schemas) {Ok(f) => {let mut tokens = Vec::new();diff --git a/core/src/webpage/schema_org/json_ld.rs b/core/src/webpage/schema_org/json_ld.rsindex c3d8a02d..55c8bd0b 100644--- a/core/src/webpage/schema_org/json_ld.rs+++ b/core/src/webpage/schema_org/json_ld.rs@@ -16,7 +16,7 @@use kuchiki::NodeRef;-use super::Item;+use super::RawItem;pub fn convert_all_ints_to_strings(json: &str) -> Result<String, serde_json::Error> {use serde_json::Value;@@ -38,7 +38,7 @@ pub fn convert_all_ints_to_strings(json: &str) -> Result<String, serde_json::Err})}-pub fn parse(root: NodeRef) -> Vec<Item> {+pub(crate) fn parse(root: NodeRef) -> Vec<RawItem> {let mut res = Vec::new();for node in root.select("script").unwrap().filter(|node| {@@ -65,7 +65,7 @@ mod tests {use kuchiki::traits::TendrilSink;use maplit::hashmap;- use crate::webpage::schema_org::{OneOrMany, Property};+ use crate::webpage::schema_org::{RawOneOrMany, RawProperty};use super::*;@@ -100,16 +100,16 @@ mod tests {assert_eq!(res,- vec![Item {- itemtype: Some(OneOrMany::One("ImageObject".to_string())),+ vec![RawItem {+ itemtype: Some(RawOneOrMany::One("ImageObject".to_string())),properties: hashmap! {- "@context".to_string() => OneOrMany::One(Property::String("https://schema.org".to_string())),- "author".to_string() => OneOrMany::One(Property::String("Jane Doe".to_string())),- "contentLocation".to_string() => OneOrMany::One(Property::String("Puerto Vallarta, Mexico".to_string())),- "contentUrl".to_string() => OneOrMany::One(Property::String("mexico-beach.jpg".to_string())),- "datePublished".to_string() => OneOrMany::One(Property::String("2008-01-25".to_string())),- "description".to_string() => OneOrMany::One(Property::String("I took this picture while on vacation last year.".to_string())),- "name".to_string() => OneOrMany::One(Property::String("Beach in Mexico".to_string())),+ "@context".to_string() => RawOneOrMany::One(RawProperty::String("https://schema.org".to_string())),+ "author".to_string() => RawOneOrMany::One(RawProperty::String("Jane Doe".to_string())),+ "contentLocation".to_string() => RawOneOrMany::One(RawProperty::String("Puerto Vallarta, Mexico".to_string())),+ "contentUrl".to_string() => RawOneOrMany::One(RawProperty::String("mexico-beach.jpg".to_string())),+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2008-01-25".to_string())),+ "description".to_string() => RawOneOrMany::One(RawProperty::String("I took this picture while on vacation last year.".to_string())),+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Beach in Mexico".to_string())),}}]);@@ -162,11 +162,11 @@ mod tests {assert_eq!(res,- vec![Item {- itemtype: Some(OneOrMany::One("test".to_string())),+ vec![RawItem {+ itemtype: Some(RawOneOrMany::One("test".to_string())),properties: hashmap! {- "@context".to_string() => OneOrMany::One(Property::String("https://schema.org".to_string())),- "cost".to_string() => OneOrMany::One(Property::String("123".to_string())),+ "@context".to_string() => RawOneOrMany::One(RawProperty::String("https://schema.org".to_string())),+ "cost".to_string() => RawOneOrMany::One(RawProperty::String("123".to_string())),}}]);diff --git a/core/src/webpage/schema_org/microdata.rs b/core/src/webpage/schema_org/microdata.rsindex c77bf063..228e4b0a 100644--- a/core/src/webpage/schema_org/microdata.rs+++ b/core/src/webpage/schema_org/microdata.rs@@ -20,7 +20,7 @@ use kuchiki::NodeRef;use std::collections::HashMap;use thiserror::Error;-use super::{Item, OneOrMany, Property};+use super::{RawItem, RawOneOrMany, RawProperty};pub type Result<T> = std::result::Result<T, Error>;@@ -36,7 +36,7 @@ pub enum Error {Json(#[from] serde_json::Error),}-fn rec_text_contents(node: NodeRef) -> Vec<Property> {+fn rec_text_contents(node: NodeRef) -> Vec<RawProperty> {let mut res = Vec::new();for child in node.children() {@@ -46,11 +46,11 @@ fn rec_text_contents(node: NodeRef) -> Vec<Property> {let mut properties = HashMap::new();properties.insert("text".to_string(),- OneOrMany::One(Property::String(child.text_contents())),+ RawOneOrMany::One(RawProperty::String(child.text_contents())),);- res.push(Property::Item(Item {- itemtype: Some(OneOrMany::One("SourceCode".to_string())),+ res.push(RawProperty::Item(RawItem {+ itemtype: Some(RawOneOrMany::One("SourceCode".to_string())),properties,}))} else if !elem.attributes.borrow().contains("itemprop")@@ -67,7 +67,7 @@ fn rec_text_contents(node: NodeRef) -> Vec<Property> {if had_line_ending {s.push('\n');}- res.push(Property::String(s));+ res.push(RawProperty::String(s));}_ => {}}@@ -76,7 +76,7 @@ fn rec_text_contents(node: NodeRef) -> Vec<Property> {res}-fn text_contents(node: NodeRef) -> Vec<Property> {+fn text_contents(node: NodeRef) -> Vec<RawProperty> {let properties = rec_text_contents(node);// merge consecutive strings@@ -85,15 +85,15 @@ fn text_contents(node: NodeRef) -> Vec<Property> {for prop in properties {match &prop {- Property::String(s) => {+ RawProperty::String(s) => {current = match current {Some(current) => match current {- Property::String(mut current_str) => {+ RawProperty::String(mut current_str) => {current_str.push(' ');current_str.push_str(s);- Some(Property::String(current_str))+ Some(RawProperty::String(current_str))}- Property::Item(_) => {+ RawProperty::Item(_) => {res.push(current);Some(prop)}@@ -101,7 +101,7 @@ fn text_contents(node: NodeRef) -> Vec<Property> {None => Some(prop),}}- Property::Item(_) => {+ RawProperty::Item(_) => {if let Some(current) = current {res.push(current);}@@ -117,7 +117,7 @@ fn text_contents(node: NodeRef) -> Vec<Property> {// trim stringsfor prop in &mut res {- if let Property::String(s) = prop {+ if let RawProperty::String(s) = prop {let had_line_ending = s.ends_with('\n');*s = itertools::intersperse(@@ -142,7 +142,7 @@ fn text_contents(node: NodeRef) -> Vec<Property> {/// implementation of https://html.spec.whatwg.org/multipage/microdata.html#associating-names-with-items/// TODO: handle itemrefs-fn parse_item(root: NodeRef) -> Result<Item> {+fn parse_item(root: NodeRef) -> Result<RawItem> {if !root.as_element().unwrap()@@ -163,13 +163,13 @@ fn parse_item(root: NodeRef) -> Result<Item> {let itemtype: Vec<_> = s.split_ascii_whitespace().map(String::from).collect();if itemtype.len() == 1 {- OneOrMany::One(itemtype.into_iter().next().unwrap())+ RawOneOrMany::One(itemtype.into_iter().next().unwrap())} else {- OneOrMany::Many(itemtype)+ RawOneOrMany::Many(itemtype)}});- let mut properties: HashMap<String, Vec<Property>> = HashMap::new();+ let mut properties: HashMap<String, Vec<RawProperty>> = HashMap::new();let mut pending: Vec<_> = root.children().collect();while let Some(current) = pending.pop() {@@ -180,10 +180,10 @@ fn parse_item(root: NodeRef) -> Result<Item> {if let Some(itemprop) = elem.attributes.borrow().get("itemprop") {let properties_for_prop = if elem.attributes.borrow().contains("itemscope") {- vec![Property::Item(parse_item(current)?)]+ vec![RawProperty::Item(parse_item(current)?)]} else {match elem.name.local.to_string().as_str() {- "meta" => vec![Property::String(+ "meta" => vec![RawProperty::String(elem.attributes.borrow().get("content")@@ -192,26 +192,26 @@ fn parse_item(root: NodeRef) -> Result<Item> {)],"audio" | "embed" | "iframe" | "img" | "source" | "track" | "video" => {if let Some(url) = elem.attributes.borrow().get("src") {- vec![Property::String(url.to_string())]+ vec![RawProperty::String(url.to_string())]} else {- vec![Property::String(String::new())]+ vec![RawProperty::String(String::new())]}}"a" | "area" | "link" => {if let Some(url) = elem.attributes.borrow().get("href") {- vec![Property::String(url.to_string())]+ vec![RawProperty::String(url.to_string())]} else {- vec![Property::String(String::new())]+ vec![RawProperty::String(String::new())]}}- "object" => vec![Property::String(+ "object" => vec![RawProperty::String(elem.attributes.borrow().get("data").map(String::from).unwrap_or_default(),)],- "data" | "meter" => vec![Property::String(+ "data" | "meter" => vec![RawProperty::String(elem.attributes.borrow().get("value")@@ -226,7 +226,7 @@ fn parse_item(root: NodeRef) -> Result<Item> {.map(String::from).unwrap_or_else(|| current.text_contents());- vec![Property::String(time)]+ vec![RawProperty::String(time)]}_ => text_contents(current.clone()),}@@ -242,7 +242,7 @@ fn parse_item(root: NodeRef) -> Result<Item> {}}- Ok(Item {+ Ok(RawItem {itemtype,properties: properties.into_iter()@@ -252,16 +252,19 @@ fn parse_item(root: NodeRef) -> Result<Item> {if properties.is_empty() {None} else if properties.len() == 1 {- Some((name, OneOrMany::One(properties.into_iter().next().unwrap())))+ Some((+ name,+ RawOneOrMany::One(properties.into_iter().next().unwrap()),+ ))} else {- Some((name, OneOrMany::Many(properties)))+ Some((name, RawOneOrMany::Many(properties)))}}).collect(),})}-fn parse(root: NodeRef) -> Vec<Item> {+fn parse(root: NodeRef) -> Vec<RawItem> {let mut res = Vec::new();let mut pending: Vec<_> = root.inclusive_descendants().collect();@@ -280,10 +283,10 @@ fn parse(root: NodeRef) -> Vec<Item> {res}-fn fix_type_for_schema(mut item: Item) -> Item {- if let Some(OneOrMany::One(itemtype)) = &item.itemtype {+fn fix_type_for_schema(mut item: RawItem) -> RawItem {+ if let Some(RawOneOrMany::One(itemtype)) = &item.itemtype {if let Some(last) = itemtype.split('/').last() {- item.itemtype = Some(OneOrMany::One(last.to_string()));+ item.itemtype = Some(RawOneOrMany::One(last.to_string()));}}@@ -291,24 +294,24 @@ fn fix_type_for_schema(mut item: Item) -> Item {.properties.into_iter().map(|(key, properties)| match properties {- OneOrMany::One(property) => {- if let Property::Item(subitem) = property {+ RawOneOrMany::One(property) => {+ if let RawProperty::Item(subitem) = property {(key,- OneOrMany::One(Property::Item(fix_type_for_schema(subitem))),+ RawOneOrMany::One(RawProperty::Item(fix_type_for_schema(subitem))),)} else {- (key, OneOrMany::One(property))+ (key, RawOneOrMany::One(property))}}- OneOrMany::Many(properties) => (+ RawOneOrMany::Many(properties) => (key,- OneOrMany::Many(+ RawOneOrMany::Many(properties.into_iter().map(|property| {- if let Property::Item(subitem) = property {- Property::Item(fix_type_for_schema(subitem))+ if let RawProperty::Item(subitem) = property {+ RawProperty::Item(fix_type_for_schema(subitem))} else {property}@@ -322,7 +325,7 @@ fn fix_type_for_schema(mut item: Item) -> Item {item}-pub fn parse_schema(root: NodeRef) -> Vec<Item> {+pub fn parse_schema(root: NodeRef) -> Vec<RawItem> {parse(root).into_iter().map(fix_type_for_schema).collect()}@@ -331,7 +334,7 @@ mod tests {use kuchiki::traits::TendrilSink;use maplit::hashmap;- use crate::webpage::schema_org::OneOrMany;+ use crate::webpage::schema_org::RawOneOrMany;use super::*;@@ -353,11 +356,11 @@ mod tests {assert_eq!(parse_item(root).unwrap(),- Item {- itemtype: Some(OneOrMany::One(String::from("http://n.whatwg.org/work"))),+ RawItem {+ itemtype: Some(RawOneOrMany::One(String::from("http://n.whatwg.org/work"))),properties: hashmap! {- "work".to_string() => OneOrMany::One(Property::String("images/house.jpeg".to_string())),- "title".to_string() => OneOrMany::One(Property::String("The house I found.".to_string())),+ "work".to_string() => RawOneOrMany::One(RawProperty::String("images/house.jpeg".to_string())),+ "title".to_string() => RawOneOrMany::One(RawProperty::String("The house I found.".to_string())),}});@@ -390,25 +393,25 @@ mod tests {.as_node().clone();- let expected = Item {- itemtype: Some(OneOrMany::One(String::from(+ let expected = RawItem {+ itemtype: Some(RawOneOrMany::One(String::from("http://schema.org/BlogPosting",))),properties: hashmap! {- "comment".to_string() => OneOrMany::One(- Property::Item(- Item {- itemtype: Some(OneOrMany::One("http://schema.org/UserComments".to_string())),+ "comment".to_string() => RawOneOrMany::One(+ RawProperty::Item(+ RawItem {+ itemtype: Some(RawOneOrMany::One("http://schema.org/UserComments".to_string())),properties: hashmap! {- "url".to_string() => OneOrMany::One(Property::String("#c1".to_string())),- "creator".to_string() => OneOrMany::One(- Property::Item(Item {- itemtype: Some(OneOrMany::One("http://schema.org/Person".to_string())),+ "url".to_string() => RawOneOrMany::One(RawProperty::String("#c1".to_string())),+ "creator".to_string() => RawOneOrMany::One(+ RawProperty::Item(RawItem {+ itemtype: Some(RawOneOrMany::One("http://schema.org/Person".to_string())),properties: hashmap! {- "name".to_string() => OneOrMany::One(Property::String("Greg".to_string()))+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Greg".to_string()))}})),- "commentTime".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string()))+ "commentTime".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string()))}})),},@@ -497,43 +500,43 @@ mod tests {let res = parse(root);assert_eq!(res.len(), 2);- let expected_article = Item {- itemtype: Some(OneOrMany::One(String::from(+ let expected_article = RawItem {+ itemtype: Some(RawOneOrMany::One(String::from("http://schema.org/BlogPosting",))),properties: hashmap! {- "headline".to_string() => OneOrMany::One(Property::String(String::from("Progress report"))),- "datePublished".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string())),- "url".to_string() => OneOrMany::One(Property::String("?comments=0".to_string())),- "comment".to_string() => OneOrMany::Many(vec![- Property::Item(- Item {- itemtype: Some(OneOrMany::One("http://schema.org/UserComments".to_string())),+ "headline".to_string() => RawOneOrMany::One(RawProperty::String(String::from("Progress report"))),+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string())),+ "url".to_string() => RawOneOrMany::One(RawProperty::String("?comments=0".to_string())),+ "comment".to_string() => RawOneOrMany::Many(vec![+ RawProperty::Item(+ RawItem {+ itemtype: Some(RawOneOrMany::One("http://schema.org/UserComments".to_string())),properties: hashmap! {- "url".to_string() => OneOrMany::One(Property::String("#c1".to_string())),- "creator".to_string() => OneOrMany::One(- Property::Item(Item {- itemtype: Some(OneOrMany::One("http://schema.org/Person".to_string())),+ "url".to_string() => RawOneOrMany::One(RawProperty::String("#c1".to_string())),+ "creator".to_string() => RawOneOrMany::One(+ RawProperty::Item(RawItem {+ itemtype: Some(RawOneOrMany::One("http://schema.org/Person".to_string())),properties: hashmap! {- "name".to_string() => OneOrMany::One(Property::String("Greg".to_string()))+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Greg".to_string()))}})),- "commentTime".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string()))+ "commentTime".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string()))}}),- Property::Item(- Item {- itemtype: Some(OneOrMany::One("http://schema.org/UserComments".to_string())),+ RawProperty::Item(+ RawItem {+ itemtype: Some(RawOneOrMany::One("http://schema.org/UserComments".to_string())),properties: hashmap! {- "url".to_string() => OneOrMany::One(Property::String("#c2".to_string())),- "creator".to_string() => OneOrMany::One(- Property::Item(Item {- itemtype: Some(OneOrMany::One("http://schema.org/Person".to_string())),+ "url".to_string() => RawOneOrMany::One(RawProperty::String("#c2".to_string())),+ "creator".to_string() => RawOneOrMany::One(+ RawProperty::Item(RawItem {+ itemtype: Some(RawOneOrMany::One("http://schema.org/Person".to_string())),properties: hashmap! {- "name".to_string() => OneOrMany::One(Property::String("Charlotte".to_string()))+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Charlotte".to_string()))}})),- "commentTime".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string()))+ "commentTime".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string()))}})]),@@ -616,15 +619,15 @@ mod tests {assert_eq!(res,- vec![Item {- itemtype: Some(OneOrMany::One("ImageObject".to_string())),+ vec![RawItem {+ itemtype: Some(RawOneOrMany::One("ImageObject".to_string())),properties: hashmap! {- "author".to_string() => OneOrMany::One(Property::String("Jane Doe".to_string())),- "contentLocation".to_string() => OneOrMany::One(Property::String("Puerto Vallarta, Mexico".to_string())),- "contentUrl".to_string() => OneOrMany::One(Property::String("mexico-beach.jpg".to_string())),- "datePublished".to_string() => OneOrMany::One(Property::String("2008-01-25".to_string())),- "description".to_string() => OneOrMany::One(Property::String("I took this picture while on vacation last year.".to_string())),- "name".to_string() => OneOrMany::One(Property::String("Beach in Mexico".to_string())),+ "author".to_string() => RawOneOrMany::One(RawProperty::String("Jane Doe".to_string())),+ "contentLocation".to_string() => RawOneOrMany::One(RawProperty::String("Puerto Vallarta, Mexico".to_string())),+ "contentUrl".to_string() => RawOneOrMany::One(RawProperty::String("mexico-beach.jpg".to_string())),+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2008-01-25".to_string())),+ "description".to_string() => RawOneOrMany::One(RawProperty::String("I took this picture while on vacation last year.".to_string())),+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Beach in Mexico".to_string())),}}]);diff --git a/core/src/webpage/schema_org/mod.rs b/core/src/webpage/schema_org/mod.rsindex 936e8c8d..37def264 100644--- a/core/src/webpage/schema_org/mod.rs+++ b/core/src/webpage/schema_org/mod.rs@@ -19,39 +19,38 @@ use std::collections::HashMap;use kuchiki::NodeRef;use serde::{Deserialize, Serialize};+use crate::tokenizer::FlattenedJson;+use crate::Result;+mod json_ld;mod microdata;#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]-#[serde(untagged)]pub enum Property {String(String),Item(Item),}-impl Property {- pub fn try_into_string(self) -> Option<String> {- if let Property::String(s) = self {- Some(s)- } else {- None+ pub(crate) fn try_into_string(&self) -> Option<String> {+ match self {+ Property::String(s) => Some(s.clone()),+ Property::Item(_) => None,}}- pub fn try_into_item(self) -> Option<Item> {- if let Property::Item(item) = self {- Some(item)- } else {- None+ pub(crate) fn try_into_item(&self) -> Option<Item> {+ match self {+ Property::String(_) => None,+ Property::Item(it) => Some(it.clone()),}}}#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]#[serde(untagged)]-pub enum SingleMap {+enum SingleMap {Leaf(String),- Node(HashMap<OneOrMany<String>, HashMap<String, OneOrMany<SingleMap>>>),+ Node(HashMap<RawOneOrMany<String>, HashMap<String, RawOneOrMany<SingleMap>>>),}impl From<Property> for SingleMap {@@ -66,15 +65,18 @@ impl From<Property> for SingleMap {.properties.into_iter().map(|(key, val)| match val {- OneOrMany::One(one) => (key, OneOrMany::One(SingleMap::from(one))),+ OneOrMany::One(one) => (key, RawOneOrMany::One(SingleMap::from(one))),OneOrMany::Many(many) => (key,- OneOrMany::Many(many.into_iter().map(SingleMap::from).collect()),+ RawOneOrMany::Many(many.into_iter().map(SingleMap::from).collect()),),}).collect();- res.insert(tt, recursive);+ match tt {+ OneOrMany::One(one) => res.insert(RawOneOrMany::One(one), recursive),+ OneOrMany::Many(many) => res.insert(RawOneOrMany::Many(many), recursive),+ };}SingleMap::Node(res)@@ -83,11 +85,50 @@ impl From<Property> for SingleMap {}}+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]+pub struct RawItem {+ #[serde(rename = "@type")]+ itemtype: Option<RawOneOrMany<String>>,+ #[serde(flatten)]+ properties: HashMap<String, RawOneOrMany<RawProperty>>,+}++#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]+#[serde(untagged)]+enum RawProperty {+ String(String),+ Item(RawItem),+}+impl RawProperty {+ #[cfg(test)]+ fn try_into_item(&self) -> Option<RawItem> {+ match self {+ RawProperty::String(_) => None,+ RawProperty::Item(it) => Some(it.clone()),+ }+ }++ #[cfg(test)]+ fn try_into_string(&self) -> Option<String> {+ match self {+ RawProperty::String(s) => Some(s.clone()),+ RawProperty::Item(_) => None,+ }+ }+}++impl From<RawProperty> for Property {+ fn from(value: RawProperty) -> Self {+ match value {+ RawProperty::String(s) => Self::String(s),+ RawProperty::Item(it) => Self::Item(Item::from(it)),+ }+ }+}+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct Item {- #[serde(rename = "@type")]pub itemtype: Option<OneOrMany<String>>,- #[serde(flatten)]pub properties: HashMap<String, OneOrMany<Property>>,}@@ -102,13 +143,34 @@ impl Item {}}- pub fn into_single_map(self) -> SingleMap {+ fn into_single_map(self) -> SingleMap {SingleMap::from(Property::Item(self))}}+impl From<RawItem> for Item {+ fn from(value: RawItem) -> Self {+ Self {+ itemtype: value.itemtype.map(|tt| match tt {+ RawOneOrMany::One(one) => OneOrMany::One(one),+ RawOneOrMany::Many(many) => OneOrMany::Many(many),+ }),+ properties: value+ .properties+ .into_iter()+ .map(|(key, val)| match val {+ RawOneOrMany::One(prop) => (key, OneOrMany::One(Property::from(prop))),+ RawOneOrMany::Many(props) => (+ key,+ OneOrMany::Many(props.into_iter().map(Property::from).collect()),+ ),+ })+ .collect(),+ }+ }+}+#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Hash)]-#[serde(untagged)]pub enum OneOrMany<T> {One(T),Many(Vec<T>),@@ -130,11 +192,36 @@ impl<T> OneOrMany<T> {}}+#[derive(Deserialize, Serialize, PartialEq, Eq, Debug, Clone, Hash)]+#[serde(untagged)]+enum RawOneOrMany<T> {+ One(T),+ Many(Vec<T>),+}++impl<T> RawOneOrMany<T> {+ #[cfg(test)]+ pub fn one(self) -> Option<T> {+ match self {+ RawOneOrMany::One(one) => Some(one),+ RawOneOrMany::Many(many) => many.into_iter().next(),+ }+ }++ #[cfg(test)]+ pub fn many(self) -> Vec<T> {+ match self {+ RawOneOrMany::One(one) => vec![one],+ RawOneOrMany::Many(many) => many,+ }+ }+}+pub fn parse(root: NodeRef) -> Vec<Item> {let mut res = self::json_ld::parse(root.clone());res.append(&mut self::microdata::parse_schema(root));- res+ res.into_iter().map(Item::from).collect()}#[cfg(test)]@@ -160,19 +247,19 @@ mod tests {}"#;- let item: Item = serde_json::from_str(json).unwrap();+ let item: RawItem = serde_json::from_str(json).unwrap();assert_eq!(item,- Item {- itemtype: Some(OneOrMany::One("ImageObject".to_string())),+ RawItem {+ itemtype: Some(RawOneOrMany::One("ImageObject".to_string())),properties: hashmap! {- "@context".to_string() => OneOrMany::One(Property::String("https://schema.org".to_string())),- "author".to_string() => OneOrMany::One(Property::String("Jane Doe".to_string())),- "contentLocation".to_string() => OneOrMany::One(Property::String("Puerto Vallarta, Mexico".to_string())),- "contentUrl".to_string() => OneOrMany::One(Property::String("mexico-beach.jpg".to_string())),- "datePublished".to_string() => OneOrMany::One(Property::String("2008-01-25".to_string())),- "description".to_string() => OneOrMany::One(Property::String("I took this picture while on vacation last year.".to_string())),- "name".to_string() => OneOrMany::One(Property::String("Beach in Mexico".to_string())),+ "@context".to_string() => RawOneOrMany::One(RawProperty::String("https://schema.org".to_string())),+ "author".to_string() => RawOneOrMany::One(RawProperty::String("Jane Doe".to_string())),+ "contentLocation".to_string() => RawOneOrMany::One(RawProperty::String("Puerto Vallarta, Mexico".to_string())),+ "contentUrl".to_string() => RawOneOrMany::One(RawProperty::String("mexico-beach.jpg".to_string())),+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2008-01-25".to_string())),+ "description".to_string() => RawOneOrMany::One(RawProperty::String("I took this picture while on vacation last year.".to_string())),+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Beach in Mexico".to_string())),}});@@ -190,7 +277,7 @@ mod tests {assert!(res[0].properties.contains_key("primaryImageOfPage"));assert_eq!(res[0].properties.get("name"),- Some(&OneOrMany::One(Property::String(+ Some(&RawOneOrMany::One(RawProperty::String("RegEx match open tags except XHTML self-contained tags".to_string()))));@@ -205,16 +292,19 @@ mod tests {.try_into_item().unwrap();- assert_eq!(main.itemtype, Some(OneOrMany::One("Question".to_string())));+ assert_eq!(+ main.itemtype,+ Some(RawOneOrMany::One("Question".to_string()))+ );assert_eq!(main.properties.get("name"),- Some(&OneOrMany::One(Property::String(+ Some(&RawOneOrMany::One(RawProperty::String("RegEx match open tags except XHTML self-contained tags".to_string()))));assert_eq!(main.properties.get("dateCreated"),- Some(&OneOrMany::One(Property::String(+ Some(&RawOneOrMany::One(RawProperty::String("2009-11-13T22:38:26".to_string()))));@@ -233,14 +323,14 @@ mod tests {let text = main.properties.get("text").unwrap().clone().many();- assert_eq!(text[0], Property::String("Locked . Comments on this question have been disabled, but it is still accepting new answers and other interactions. Learn more .\nI need to match all of these opening tags:\n".to_string()));+ assert_eq!(text[0], RawProperty::String("Locked . Comments on this question have been disabled, but it is still accepting new answers and other interactions. Learn more .\nI need to match all of these opening tags:\n".to_string()));assert_eq!(text[1],- Property::Item(Item {- itemtype: Some(OneOrMany::One("SourceCode".to_string())),+ RawProperty::Item(RawItem {+ itemtype: Some(RawOneOrMany::One("SourceCode".to_string())),properties: hashmap! {- "text".to_string() => OneOrMany::One(Property::String("<p>\n<a href=\"foo\">\n".to_string()))+ "text".to_string() => RawOneOrMany::One(RawProperty::String("<p>\n<a href=\"foo\">\n".to_string()))}}));@@ -267,45 +357,47 @@ mod tests {assert_eq!(recipe.properties.get("recipeCategory"),- Some(&OneOrMany::One(Property::String("Aftensmad".to_string())))+ Some(&RawOneOrMany::One(RawProperty::String(+ "Aftensmad".to_string()+ ))));assert_eq!(recipe.properties.get("name"),- Some(&OneOrMany::One(Property::String(+ Some(&RawOneOrMany::One(RawProperty::String("One Pot Pasta med chorizo".to_string()))));assert_eq!(recipe.properties.get("recipeYield"),- Some(&OneOrMany::One(Property::String("4".to_string())))+ Some(&RawOneOrMany::One(RawProperty::String("4".to_string()))));assert_eq!(recipe.properties.get("cookTime"),- Some(&OneOrMany::One(Property::String("PT25M".to_string())))+ Some(&RawOneOrMany::One(RawProperty::String("PT25M".to_string()))));assert_eq!(recipe.properties.get("recipeIngredient"),- Some(&OneOrMany::Many(vec![- Property::String("400 g spaghetti".to_string()),- Property::String("1 dåse hakkede tomater".to_string()),- Property::String("1 håndfuld frisk basilikum, bladene herfra".to_string()),- Property::String("1 løg, finthakket".to_string()),- Property::String("2 fed hvidløg, finthakket".to_string()),- Property::String("20 cherrytomater, skåret i både".to_string()),- Property::String("0,50 squash, groftrevet".to_string()),- Property::String("1 tsk oregano, tørret".to_string()),- Property::String("50 g chorizo, finthakket".to_string()),- Property::String("5 dl grøntsagsbouillon".to_string()),- Property::String("2 spsk olivenolie".to_string()),- Property::String("1 tsk chiliflager, kan undlades".to_string()),- Property::String("1 tsk salt".to_string()),- Property::String("sort peber, friskkværnet".to_string()),- Property::String("50 g parmesan, friskrevet til servering".to_string()),- Property::String("1 håndfuld frisk basilikum, blade".to_string()),+ Some(&RawOneOrMany::Many(vec![+ RawProperty::String("400 g spaghetti".to_string()),+ RawProperty::String("1 dåse hakkede tomater".to_string()),+ RawProperty::String("1 håndfuld frisk basilikum, bladene herfra".to_string()),+ RawProperty::String("1 løg, finthakket".to_string()),+ RawProperty::String("2 fed hvidløg, finthakket".to_string()),+ RawProperty::String("20 cherrytomater, skåret i både".to_string()),+ RawProperty::String("0,50 squash, groftrevet".to_string()),+ RawProperty::String("1 tsk oregano, tørret".to_string()),+ RawProperty::String("50 g chorizo, finthakket".to_string()),+ RawProperty::String("5 dl grøntsagsbouillon".to_string()),+ RawProperty::String("2 spsk olivenolie".to_string()),+ RawProperty::String("1 tsk chiliflager, kan undlades".to_string()),+ RawProperty::String("1 tsk salt".to_string()),+ RawProperty::String("sort peber, friskkværnet".to_string()),+ RawProperty::String("50 g parmesan, friskrevet til servering".to_string()),+ RawProperty::String("1 håndfuld frisk basilikum, blade".to_string()),])));@@ -334,3 +426,11 @@ mod tests {assert_eq!(&instructions, "Helt enkelt som navnet antyder, så kom alle ingredienserne i en stor gryde på én gang. Kog retten op, rør godt rundt i gryden og skru ned for varmen. Lad det simrekoge under låg i 10-12 minutter, til spaghettien er perfekt kogt – al dente med lidt bid i. Server med revet parmesan og basilikum.");}}++pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {+ let single_maps: Vec<_> = schemas+ .into_iter()+ .map(|item| item.into_single_map())+ .collect();+ FlattenedJson::new(&single_maps)+}diff --git a/frontend/src/components/Sidebar.astro b/frontend/src/components/Sidebar.astroindex ea397600..3da2cae1 100644--- a/frontend/src/components/Sidebar.astro+++ b/frontend/src/components/Sidebar.astro@@ -11,8 +11,8 @@ export interface Props {<div>{askama.match("sidebar", {- "DisplayedSidebar::Entity(entity)": <Entity entity="entity" />,- "DisplayedSidebar::StackOverflow{title, answer}": (+ "Sidebar::Entity(entity)": <Entity entity="entity" />,+ "Sidebar::StackOverflow{title, answer}": (<StackOverflow title="title" answer="answer" />),})