Skip to content

web search done right

Major refactor. Generate stackoverflow sidebar in distributed searcher as this allows us to re-use the existing search infrastructure

Mikkel Denker6cffad6e2870
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index ce665b87..7bd9768b 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -68,7 +68,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
}
}
}
- searcher::Request::RetrieveWebites { websites, query } => {
+ searcher::Request::RetrieveWebsites { websites, query } => {
match local_searcher.retrieve_websites(websites, query) {
Ok(response) => {
req.respond(sonic::Response::Content(response)).await.ok();
@@ -86,7 +86,7 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
match local_searcher.search_initial(query, false) {
Ok(result) => match result {
searcher::InitialSearchResult::Websites(result) => {
- let res = search_prettifier::initial(result, &local_searcher);
+ let res = search_prettifier::initial(result);
req.respond(sonic::Response::Content(
searcher::InitialPrettifiedSearchResult::Websites(res),
@@ -109,25 +109,6 @@ pub async fn run(config: SearchServerConfig) -> Result<()> {
}
}
}
- searcher::Request::RetrievePrettifiedWebites { websites, query } => {
- match local_searcher.retrieve_websites(websites, query) {
- Ok(result) => {
- req.respond(sonic::Response::Content(search_prettifier::retrieve(
- result,
- &local_searcher,
- )))
- .await
- .ok();
- }
- Err(_) => {
- req.respond::<Vec<search_prettifier::DisplayedWebpage>>(
- sonic::Response::Empty,
- )
- .await
- .ok();
- }
- }
- }
}
}
}
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rs
index a1a1d9ec..0adc6c2f 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/frontend/search.rs
@@ -22,8 +22,8 @@ use axum_macros::debug_handler;
use crate::{
search_prettifier::{
- thousand_sep_number, CodeOrText, DisplayedSidebar, DisplayedWebpage,
- HighlightedSpellCorrection, Snippet,
+ thousand_sep_number, CodeOrText, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,
+ Snippet,
},
searcher::{self, PrettifiedSearchResult, SearchQuery, NUM_RESULTS_PER_PAGE},
webpage::region::{Region, ALL_REGIONS},
@@ -44,7 +44,7 @@ use axum::{
struct SearchTemplate {
search_result: Vec<DisplayedWebpage>,
query: String,
- sidebar: Option<DisplayedSidebar>,
+ sidebar: Option<Sidebar>,
spell_correction: Option<HighlightedSpellCorrection>,
num_matches: String,
search_duration_sec: String,
diff --git a/core/src/index.rs b/core/src/index.rs
index 7273aad2..f0a09711 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -152,7 +152,19 @@ impl Index {
websites: &[inverted_index::WebsitePointer],
query: &Query,
) -> Result<Vec<inverted_index::RetrievedWebpage>> {
- self.inverted_index.retrieve_websites(websites, query)
+ let mut websites = self.inverted_index.retrieve_websites(websites, query)?;
+
+ for website in &mut websites {
+ let url = Url::from(website.url.clone());
+ website.favicon = self.retrieve_favicon(&url);
+ if let Some(uuid) = website.primary_image.as_ref().map(|image| &image.uuid) {
+ if self.retrieve_primary_image(uuid).is_none() {
+ website.primary_image = None;
+ }
+ }
+ }
+
+ Ok(websites)
}
pub fn merge(mut self, other: Self) -> Self {
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 9016d33a..c58c4af1 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -414,8 +414,8 @@ pub struct SearchResult {
pub struct RetrievedWebpage {
pub title: String,
pub url: String,
- pub snippet: String,
pub body: String,
+ pub snippet: String,
pub dirty_body: String,
pub description: Option<String>,
pub dmoz_description: Option<String>,
diff --git a/core/src/main.rs b/core/src/main.rs
index 519d5fe3..51d6d485 100644
--- a/core/src/main.rs
+++ b/core/src/main.rs
@@ -96,8 +96,8 @@ enum WebgraphOptions {
config_path: String,
},
Merge {
- path: String,
- other_path: Option<String>,
+ #[clap(required = true)]
+ paths: Vec<String>,
#[clap(long)]
num_segments: Option<usize>,
},
@@ -195,13 +195,12 @@ fn main() -> Result<()> {
entrypoint::Webgraph::run_locally(&config)?;
}
WebgraphOptions::Merge {
- path,
- other_path,
+ mut paths,
num_segments,
} => {
- let mut webgraph = WebgraphBuilder::new(path).open();
+ let mut webgraph = WebgraphBuilder::new(paths.remove(0)).open();
- if let Some(other_path) = other_path {
+ for other_path in paths {
let other = WebgraphBuilder::new(other_path).open();
webgraph.merge(other);
}
diff --git a/core/src/ranking/models/cross_encoder.rs b/core/src/ranking/models/cross_encoder.rs
index 9ea8f7a2..3a498bd0 100644
--- a/core/src/ranking/models/cross_encoder.rs
+++ b/core/src/ranking/models/cross_encoder.rs
@@ -114,8 +114,8 @@ mod tests {
)
);
}
- Err(err) => {
- dbg!(err);
+ Err(_err) => {
+ // dbg!(err);
// panic!();
}
}
diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rs
index ec6d3a21..d8735e6b 100644
--- a/core/src/search_prettifier.rs
+++ b/core/src/search_prettifier.rs
@@ -19,34 +19,54 @@ use std::collections::BTreeMap;
use chrono::{NaiveDate, NaiveDateTime, Utc};
use itertools::{intersperse, Itertools};
use serde::{Deserialize, Serialize};
-use tracing::log::debug;
use crate::{
entity_index::{entity::Span, StoredEntity},
inverted_index::RetrievedWebpage,
ranking::pipeline::RankingWebsite,
- searcher::{self, LocalSearcher, Sidebar},
+ searcher::{self, LocalSearcher},
spell::CorrectionTerm,
webpage::{
- schema_org::{self, OneOrMany, Property},
+ schema_org::{self, Item, OneOrMany, Property},
Url,
},
Error, Result,
};
-pub fn initial(
- result: searcher::local::InitialWebsiteResult,
- local_searcher: &LocalSearcher,
-) -> InitialWebsiteResult {
- let sidebar = result.sidebar.and_then(|sidebar| {
- let res = DisplayedSidebar::from(sidebar, local_searcher);
- if let Ok(sidebar) = res {
- Some(sidebar)
- } else {
- debug!("Failed to parse sidebar information: {:?}", res);
- None
- }
- });
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Snippet {
+ Normal {
+ date: Option<String>,
+ text: String,
+ },
+ StackOverflowQA {
+ question: StackOverflowQuestion,
+ answers: Vec<StackOverflowAnswer>,
+ },
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct StackOverflowAnswer {
+ pub body: Vec<CodeOrText>,
+ pub date: String,
+ pub url: String,
+ pub upvotes: u32,
+ pub accepted: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct StackOverflowQuestion {
+ pub body: Vec<CodeOrText>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum CodeOrText {
+ Code(String),
+ Text(String),
+}
+
+pub fn initial(result: searcher::local::InitialWebsiteResult) -> InitialWebsiteResult {
+ let sidebar = result.entity_sidebar;
let spell_corrected_query = result.spell_corrected_query.map(|correction| {
let mut highlighted = String::new();
@@ -78,33 +98,10 @@ pub fn initial(
spell_corrected_query,
websites: result.websites,
num_websites: result.num_websites,
- sidebar,
+ entity_sidebar: sidebar,
}
}
-pub fn retrieve(result: Vec<RetrievedWebpage>, searcher: &LocalSearcher) -> Vec<DisplayedWebpage> {
- result
- .into_iter()
- .map(|mut webpage| {
- webpage.primary_image = webpage.primary_image.and_then(|image| {
- if searcher
- .primary_image(image.uuid.clone().to_string())
- .is_some()
- {
- Some(image)
- } else {
- None
- }
- });
-
- let url: Url = webpage.url.clone().into();
- webpage.favicon = searcher.favicon(&url.site().to_string().into());
- webpage
- })
- .map(DisplayedWebpage::from)
- .collect()
-}
-
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct HighlightedSpellCorrection {
pub raw: String,
@@ -116,7 +113,7 @@ pub struct InitialWebsiteResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub num_websites: usize,
pub websites: Vec<RankingWebsite>,
- pub sidebar: Option<DisplayedSidebar>,
+ pub entity_sidebar: Option<DisplayedEntity>,
}
pub fn html_escape(s: &str) -> String {
@@ -167,38 +164,6 @@ fn prettify_date(date: NaiveDateTime) -> String {
format!("{}", date.format("%d. %b. %Y"))
}
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum CodeOrText {
- Code(String),
- Text(String),
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct StackOverflowAnswer {
- pub body: Vec<CodeOrText>,
- pub date: String,
- pub url: String,
- pub upvotes: u32,
- pub accepted: bool,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct StackOverflowQuestion {
- pub body: Vec<CodeOrText>,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum Snippet {
- Normal {
- date: Option<String>,
- text: String,
- },
- StackOverflowQA {
- question: StackOverflowQuestion,
- answers: Vec<StackOverflowAnswer>,
- },
-}
-
fn parse_so_answer(
text: OneOrMany<Property>,
date: OneOrMany<Property>,
@@ -409,7 +374,7 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
}
#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum DisplayedSidebar {
+pub enum Sidebar {
Entity(DisplayedEntity),
StackOverflow {
title: String,
@@ -417,43 +382,34 @@ pub enum DisplayedSidebar {
},
}
-impl DisplayedSidebar {
- fn from(sidebar: Sidebar, searcher: &LocalSearcher) -> Result<Self> {
- match sidebar {
- Sidebar::Entity(entity) => Ok(DisplayedSidebar::Entity(DisplayedEntity::from(
- entity, searcher,
- ))),
- Sidebar::StackOverflow { schema_org, url } => {
- if let Some(item) = schema_org
- .into_iter()
- .find(|item| item.types_contains("QAPage"))
- .and_then(|item| item.properties.get("mainEntity").cloned())
- .and_then(|properties| properties.one())
- .and_then(|property| property.try_into_item())
- {
- let title = item
- .properties
- .get("name")
- .cloned()
- .and_then(|prop| prop.one())
- .and_then(|prop| prop.try_into_string())
- .ok_or(Error::InvalidStackoverflowSchema)?;
-
- item.properties
- .get("acceptedAnswer")
- .cloned()
- .and_then(|ans| ans.one())
- .and_then(|prop| prop.try_into_item())
- .and_then(|item| {
- schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)
- })
- .map(|answer| DisplayedSidebar::StackOverflow { title, answer })
- .ok_or(Error::InvalidStackoverflowSchema)
- } else {
- Err(Error::InvalidStackoverflowSchema)
- }
- }
- }
+pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Result<Sidebar> {
+ if let Some(item) = schema_org
+ .into_iter()
+ .find(|item| item.types_contains("QAPage"))
+ .and_then(|item| item.properties.get("mainEntity").cloned())
+ .and_then(|properties| properties.one())
+ .and_then(|property| property.try_into_item())
+ {
+ let title = item
+ .properties
+ .get("name")
+ .cloned()
+ .and_then(|prop| prop.one())
+ .and_then(|prop| prop.try_into_string())
+ .ok_or(Error::InvalidStackoverflowSchema)?;
+
+ item.properties
+ .get("acceptedAnswer")
+ .cloned()
+ .and_then(|ans| ans.one())
+ .and_then(|prop| prop.try_into_item())
+ .and_then(|item| {
+ schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)
+ })
+ .map(|answer| Sidebar::StackOverflow { title, answer })
+ .ok_or(Error::InvalidStackoverflowSchema)
+ } else {
+ Err(Error::InvalidStackoverflowSchema)
}
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 01ae6857..eda6904f 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -17,16 +17,17 @@
use crate::{
collector::{self, BucketCollector},
exponential_backoff::ExponentialBackoff,
- inverted_index::{self},
+ inverted_index::{self, RetrievedWebpage},
ranking::{
models::cross_encoder::CrossEncoderModel,
pipeline::{AsRankingWebsite, RankingPipeline, RankingWebsite},
},
- search_prettifier::DisplayedWebpage,
+ search_prettifier::{create_stackoverflow_sidebar, DisplayedWebpage, Sidebar},
searcher::PrettifiedWebsitesResult,
};
use std::{
+ cmp::Ordering,
net::SocketAddr,
sync::Arc,
time::{Duration, Instant},
@@ -42,6 +43,7 @@ use crate::sonic;
use super::{InitialPrettifiedSearchResult, PrettifiedSearchResult, SearchQuery};
type Result<T> = std::result::Result<T, Error>;
+const STACKOVERFLOW_SIDEBAR_THRESHOLD: f64 = 100.0;
struct RemoteSearcher {
addr: SocketAddr,
@@ -83,7 +85,7 @@ impl RemoteSearcher {
&self,
pointers: &[inverted_index::WebsitePointer],
original_query: &str,
- ) -> Result<Vec<DisplayedWebpage>> {
+ ) -> Result<Vec<RetrievedWebpage>> {
for timeout in ExponentialBackoff::from_millis(30)
.with_limit(Duration::from_millis(200))
.take(5)
@@ -91,7 +93,7 @@ impl RemoteSearcher {
if let Ok(connection) = sonic::Connection::create_with_timeout(self.addr, timeout).await
{
if let Ok(sonic::Response::Content(body)) = connection
- .send(Request::RetrievePrettifiedWebites {
+ .send(Request::RetrieveWebsites {
websites: pointers.to_vec(),
query: original_query.to_string(),
})
@@ -151,7 +153,7 @@ impl Shard {
&self,
pointers: &[inverted_index::WebsitePointer],
original_query: &str,
- ) -> Result<Vec<DisplayedWebpage>> {
+ ) -> Result<Vec<RetrievedWebpage>> {
match self
.replicas
.iter()
@@ -175,11 +177,7 @@ struct InitialSearchResultShard {
pub enum Request {
Search(SearchQuery),
SearchPrettified(SearchQuery),
- RetrieveWebites {
- websites: Vec<inverted_index::WebsitePointer>,
- query: String,
- },
- RetrievePrettifiedWebites {
+ RetrieveWebsites {
websites: Vec<inverted_index::WebsitePointer>,
query: String,
},
@@ -228,67 +226,14 @@ impl DistributedSearcher {
}
}
- pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {
- let start = Instant::now();
-
- if query.is_empty() {
- return Err(Error::EmptyQuery);
- }
-
- let mut search_query = query.clone();
+ fn combine_results(
+ &self,
+ initial_results: Vec<InitialSearchResultShard>,
+ search_query: &mut SearchQuery,
+ ) -> Result<Vec<ScoredWebsitePointer>> {
let pipeline: RankingPipeline<ScoredWebsitePointer> =
- RankingPipeline::for_query(&mut search_query, self.cross_encoder.clone())?;
+ RankingPipeline::for_query(search_query, self.cross_encoder.clone())?;
- // search shards
- let initial_results = self
- .shards
- .iter()
- .map(|shard| shard.search(&search_query))
- .collect::<FuturesUnordered<_>>()
- .collect::<Vec<_>>()
- .await
- .into_iter()
- .filter_map(|result| result.ok())
- .collect::<Vec<_>>();
-
- // check if any result has a bang hit
- if let Some(result) = initial_results
- .iter()
- .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_)))
- {
- if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {
- return Ok(PrettifiedSearchResult::Bang(bang.clone()));
- }
- }
-
- let spell_corrected_query = initial_results.first().and_then(|result| {
- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.spell_corrected_query.clone()
- } else {
- None
- }
- });
-
- let sidebar = initial_results.first().and_then(|result| {
- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.sidebar.clone()
- } else {
- None
- }
- });
-
- let num_docs = initial_results
- .iter()
- .map(|result| {
- if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
- result.num_websites
- } else {
- 0
- }
- })
- .sum();
-
- // combine results
let mut collector =
BucketCollector::new(pipeline.collector_top_n() + pipeline.collector_offset());
@@ -312,9 +257,14 @@ impl DistributedSearcher {
.take(pipeline.collector_top_n())
.collect::<Vec<_>>();
- let top_websites = pipeline.apply(top_websites);
+ Ok(pipeline.apply(top_websites))
+ }
- // retrieve webpages
+ async fn retrieve_webpages(
+ &self,
+ top_websites: &[ScoredWebsitePointer],
+ query: &str,
+ ) -> Vec<RetrievedWebpage> {
let mut retrieved_webpages = Vec::new();
for _ in 0..top_websites.len() {
@@ -329,7 +279,7 @@ impl DistributedSearcher {
.map(|(idx, pointer)| (idx, pointer.website.pointer.clone()))
.unzip();
- if let Ok(websites) = shard.retrieve_websites(&pointers, &query.query).await {
+ if let Ok(websites) = shard.retrieve_websites(&pointers, query).await {
for (index, website) in indexes.into_iter().zip(websites.into_iter()) {
retrieved_webpages[index] = Some(website);
}
@@ -340,6 +290,132 @@ impl DistributedSearcher {
debug_assert_eq!(retrieved_webpages.len(), top_websites.len());
+ retrieved_webpages
+ }
+
+ async fn search_initial(&self, query: &SearchQuery) -> Vec<InitialSearchResultShard> {
+ self.shards
+ .iter()
+ .map(|shard| shard.search(query))
+ .collect::<FuturesUnordered<_>>()
+ .collect::<Vec<_>>()
+ .await
+ .into_iter()
+ .filter_map(|result| result.ok())
+ .collect::<Vec<_>>()
+ }
+
+ async fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {
+ let query = SearchQuery {
+ query: query.query.clone(),
+ num_results: 1,
+ optic_program: Some(include_str!("stackoverflow.optic").to_string()),
+ ..Default::default()
+ };
+
+ let mut results: Vec<_> = self
+ .search_initial(&query)
+ .await
+ .into_iter()
+ .filter_map(|result| match result.local_result {
+ InitialPrettifiedSearchResult::Websites(websites) => {
+ if let Some(website) = websites.websites.first().cloned() {
+ Some((result.shard, website))
+ } else {
+ None
+ }
+ }
+ InitialPrettifiedSearchResult::Bang(_) => None,
+ })
+ .collect();
+
+ results.sort_by(|(_, a), (_, b)| a.score.partial_cmp(&b.score).unwrap_or(Ordering::Equal));
+
+ if let Some((shard, website)) = results.pop() {
+ if website.score > STACKOVERFLOW_SIDEBAR_THRESHOLD {
+ let scored_websites = vec![ScoredWebsitePointer { website, shard }];
+ let mut retrieved = self.retrieve_webpages(&scored_websites, &query.query).await;
+
+ if let Some(res) = retrieved.pop() {
+ return Ok(Some(create_stackoverflow_sidebar(res.schema_org, res.url)?));
+ }
+ }
+ }
+
+ Ok(None)
+ }
+
+ async fn sidebar(
+ &self,
+ initial_results: &[InitialSearchResultShard],
+ query: &SearchQuery,
+ ) -> Result<Option<Sidebar>> {
+ let entity = initial_results.first().and_then(|result| {
+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
+ result.entity_sidebar.clone()
+ } else {
+ None
+ }
+ });
+
+ match entity {
+ Some(entity) => Ok(Some(Sidebar::Entity(entity))),
+ None => Ok(self.stackoverflow_sidebar(query).await?),
+ }
+ }
+
+ pub async fn search(&self, query: &SearchQuery) -> Result<PrettifiedSearchResult> {
+ let start = Instant::now();
+
+ if query.is_empty() {
+ return Err(Error::EmptyQuery);
+ }
+
+ let mut search_query = query.clone();
+
+ let initial_results = self.search_initial(query).await;
+
+ // check if any result has a bang hit
+ if let Some(result) = initial_results
+ .iter()
+ .find(|result| matches!(result.local_result, InitialPrettifiedSearchResult::Bang(_)))
+ {
+ if let InitialPrettifiedSearchResult::Bang(bang) = &result.local_result {
+ return Ok(PrettifiedSearchResult::Bang(bang.clone()));
+ }
+ }
+
+ let sidebar = self.sidebar(&initial_results, query).await?;
+
+ let spell_corrected_query = initial_results.first().and_then(|result| {
+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
+ result.spell_corrected_query.clone()
+ } else {
+ None
+ }
+ });
+
+ let num_docs = initial_results
+ .iter()
+ .map(|result| {
+ if let InitialPrettifiedSearchResult::Websites(result) = &result.local_result {
+ result.num_websites
+ } else {
+ 0
+ }
+ })
+ .sum();
+
+ let top_websites = self.combine_results(initial_results, &mut search_query)?;
+
+ // retrieve webpages
+ let retrieved_webpages: Vec<_> = self
+ .retrieve_webpages(&top_websites, &query.query)
+ .await
+ .into_iter()
+ .map(DisplayedWebpage::from)
+ .collect();
+
if retrieved_webpages.is_empty() && !top_websites.is_empty() {
return Err(Error::SearchFailed);
}
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 3bd3088a..d16fda5f 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::bangs::{BangHit, Bangs};
-use crate::entity_index::EntityIndex;
+use crate::entity_index::{EntityIndex, StoredEntity};
use crate::image_store::Image;
use crate::index::Index;
use crate::query::Query;
@@ -29,6 +29,7 @@ use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::optics::CreateAggregator;
use crate::ranking::pipeline::RankingWebsite;
use crate::ranking::{online_centrality_scorer, Ranker, SignalAggregator};
+use crate::search_prettifier::{DisplayedEntity, Sidebar};
use crate::spell::Correction;
use crate::webgraph::centrality::topic::TopicCentrality;
use crate::webpage::region::Region;
@@ -36,9 +37,7 @@ use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
use super::WebsitesResult;
-use super::{InitialSearchResult, SearchQuery, SearchResult, Sidebar};
-
-const STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD: f64 = 250.0;
+use super::{InitialSearchResult, SearchQuery, SearchResult};
pub struct LocalSearcher {
index: Index,
@@ -46,7 +45,6 @@ pub struct LocalSearcher {
bangs: Option<Bangs>,
centrality_store: Option<CentralityStore>,
topic_centrality: Option<TopicCentrality>,
- stackoverflow_sidebar_threshold: f64,
}
impl From<Index> for LocalSearcher {
@@ -63,7 +61,6 @@ impl LocalSearcher {
bangs: None,
centrality_store: None,
topic_centrality: None,
- stackoverflow_sidebar_threshold: STACKOVERFLOW_SIDEBAR_SCORE_THRESHOLD,
}
}
@@ -228,45 +225,10 @@ impl LocalSearcher {
Ok(None)
}
- fn entity_sidebar(&self, query: &SearchQuery) -> Option<Sidebar> {
+ fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {
self.entity_index
.as_ref()
.and_then(|index| index.search(&query.query))
- .map(Sidebar::Entity)
- }
-
- fn stackoverflow_sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {
- let mut query = query.clone();
- query.optic_program = Some(include_str!("stackoverflow.optic").to_string());
- query.num_results = 1;
-
- let (mut top_websites, num_websites) = self.search_inverted_index(&query, false)?;
-
- if num_websites > 0 && !top_websites.is_empty() {
- let top = top_websites.remove(0);
- if top.score > self.stackoverflow_sidebar_threshold {
- let mut retrieved = self.retrieve_websites(&[top.pointer], &query.query)?;
- let retrieved = retrieved.remove(0);
- return Ok(Some(Sidebar::StackOverflow {
- schema_org: retrieved.schema_org,
- url: retrieved.url,
- }));
- }
- }
-
- Ok(None)
- }
-
- fn sidebar(&self, query: &SearchQuery) -> Result<Option<Sidebar>> {
- if let Some(entity) = self.entity_sidebar(query) {
- return Ok(Some(entity));
- }
-
- if let Some(stackoverflow) = self.stackoverflow_sidebar(query)? {
- return Ok(Some(stackoverflow));
- }
-
- Ok(None)
}
pub fn search_initial(
@@ -280,13 +242,15 @@ impl LocalSearcher {
let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;
let correction = self.spell_correction(query)?;
- let sidebar = self.sidebar(query)?;
+ let sidebar = self
+ .entity_sidebar(query)
+ .map(|entity| DisplayedEntity::from(entity, self));
Ok(InitialSearchResult::Websites(InitialWebsiteResult {
spell_corrected_query: correction,
websites,
num_websites,
- sidebar,
+ entity_sidebar: sidebar,
}))
}
@@ -344,7 +308,7 @@ impl LocalSearcher {
num_docs: search_result.num_websites,
documents: retrieved_sites,
},
- sidebar: search_result.sidebar,
+ sidebar: search_result.entity_sidebar.map(Sidebar::Entity),
search_duration_ms: start.elapsed().as_millis(),
}))
}
@@ -381,7 +345,7 @@ pub struct InitialWebsiteResult {
pub spell_corrected_query: Option<Correction>,
pub num_websites: usize,
pub websites: Vec<RankingWebsite>,
- pub sidebar: Option<Sidebar>,
+ pub entity_sidebar: Option<DisplayedEntity>,
}
impl SearchResult {
@@ -471,36 +435,4 @@ mod tests {
}
}
}
-
- #[test]
- fn stackoverflow_sidebar() {
- let mut index = Index::temporary().expect("Unable to open index");
-
- index
- .insert(Webpage::new(
- include_str!("../../testcases/schema_org/stackoverflow.html"),
- "https://www.stackoverflow.com",
- ))
- .expect("failed to insert webpage");
-
- index.commit().unwrap();
-
- let mut searcher = LocalSearcher::new(index);
- searcher.stackoverflow_sidebar_threshold = 0.0;
-
- let res = searcher
- .search(&SearchQuery {
- query: "regex parse html".to_string(),
- ..Default::default()
- })
- .unwrap();
-
- assert!(matches!(
- res.into_websites().unwrap().sidebar.unwrap(),
- Sidebar::StackOverflow {
- schema_org: _,
- url: _
- }
- ));
- }
}
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index f9727f97..384d5d58 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -24,11 +24,10 @@ use serde::{Deserialize, Serialize};
use crate::{
bangs::BangHit,
- entity_index::StoredEntity,
inverted_index,
- search_prettifier::{self, DisplayedSidebar, DisplayedWebpage, HighlightedSpellCorrection},
+ search_prettifier::{self, DisplayedWebpage, HighlightedSpellCorrection, Sidebar},
spell::Correction,
- webpage::{region::Region, schema_org},
+ webpage::region::Region,
};
pub const NUM_RESULTS_PER_PAGE: usize = 20;
@@ -41,15 +40,6 @@ pub struct WebsitesResult {
pub search_duration_ms: u128,
}
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum Sidebar {
- Entity(StoredEntity),
- StackOverflow {
- schema_org: Vec<schema_org::Item>,
- url: String,
- },
-}
-
#[derive(Debug, Serialize)]
pub enum SearchResult {
Websites(WebsitesResult),
@@ -67,7 +57,7 @@ pub struct PrettifiedWebsitesResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub webpages: Vec<DisplayedWebpage>,
pub num_docs: usize,
- pub sidebar: Option<DisplayedSidebar>,
+ pub sidebar: Option<Sidebar>,
pub search_duration_ms: u128,
}
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index e4c6bf6c..81863f34 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -26,7 +26,7 @@ use tantivy::tokenizer::{TextAnalyzer, Token};
use tantivy::{Score, Searcher};
use whatlang::Lang;
-/// For now we use the snippet generator from tantivy with a minor modifications.
+/// For now we use the snippet generator from tantivy with modifications.
/// In the future we want to implement something closer to the method described in <https://cs.pomona.edu/~dkauchak/ir_project/whitepapers/Snippet-IL.pdf>.
/// This will require us to store each paragraph of the webpage separately to get adequate performance.
/// Implementing SnippetIL will also allow us to correctly add "..." to the snippet.
@@ -74,7 +74,7 @@ impl FragmentCandidate {
/// `Snippet`
/// Contains a fragment of a document, and some highlighed parts inside it.
#[derive(Debug)]
-struct Snippet {
+struct SnippetString {
fragment: String,
highlighted: Vec<Range<usize>>,
}
@@ -82,7 +82,7 @@ struct Snippet {
const HIGHLIGHTEN_PREFIX: &str = "<b style=\"font-weight: 700;\">";
const HIGHLIGHTEN_POSTFIX: &str = "</b>";
-impl Snippet {
+impl SnippetString {
/// Returns a hignlightned html from the `Snippet`.
fn to_html(&self) -> String {
let mut html = String::new();
@@ -156,7 +156,7 @@ fn search_fragments(
///
/// Takes a vector of `FragmentCandidate`s and the text.
/// Figures out the best fragment from it and creates a snippet.
-fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str) -> Snippet {
+fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str) -> SnippetString {
let best_fragment_opt = fragments.iter().max_by(|left, right| {
let cmp_score = left
.score
@@ -175,14 +175,14 @@ fn select_best_fragment_combination(fragments: &[FragmentCandidate], text: &str)
.iter()
.map(|item| item.start - fragment.start_offset..item.end - fragment.start_offset)
.collect();
- Snippet {
+ SnippetString {
fragment: fragment_text.to_string(),
highlighted,
}
} else {
// when there no fragments to chose from,
// for now create a empty snippet
- Snippet {
+ SnippetString {
fragment: String::new(),
highlighted: vec![],
}
@@ -246,7 +246,7 @@ impl SnippetGenerator {
}
/// Generates a snippet for the given text.
- fn snippet(&self, text: &str) -> Snippet {
+ fn snippet(&self, text: &str) -> SnippetString {
let fragment_candidates =
search_fragments(&self.tokenizer, text, &self.terms_text, self.max_num_chars);
select_best_fragment_combination(&fragment_candidates[..], text)
@@ -324,7 +324,7 @@ fn generate_snippet_without_stem(
searcher: &Searcher,
query: &Query,
field: Field,
-) -> Result<Snippet> {
+) -> Result<SnippetString> {
let tokenizer = Tokenizer::default().into();
let generator = SnippetGenerator::create(searcher, query, tokenizer, field)?;
diff --git a/core/src/sonic.rs b/core/src/sonic.rs
index 3f87e570..2a3a4800 100644
--- a/core/src/sonic.rs
+++ b/core/src/sonic.rs
@@ -59,7 +59,7 @@ unsafe fn any_as_u8_slice<T: Sized>(p: &T) -> &[u8] {
impl<T> Request<T> {
pub async fn respond<R: Serialize>(mut self, response: Response<R>) -> Result<()> {
- let bytes = bincode::serialize(&response)?;
+ let bytes = bincode::serialize(&response).unwrap();
let header = Header {
body_size: bytes.len(),
};
@@ -102,7 +102,7 @@ impl Server {
stream.read_exact(&mut buf).await?;
tracing::debug!("received bytes: {:?}", &buf);
- let body = bincode::deserialize(&buf)?;
+ let body = bincode::deserialize(&buf).unwrap();
Ok(Request { stream, body })
}
@@ -134,7 +134,7 @@ impl Connection {
mut self,
request: T,
) -> Result<Response<R>> {
- let bytes = bincode::serialize(&request)?;
+ let bytes = bincode::serialize(&request).unwrap();
let header = Header {
body_size: bytes.len(),
@@ -155,7 +155,7 @@ impl Connection {
self.stream.shutdown().await?;
- Ok(bincode::deserialize(&buf)?)
+ Ok(bincode::deserialize(&buf).unwrap())
}
pub async fn send<T: Serialize, R: DeserializeOwned + Serialize>(
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 7d37975a..b0a648d6 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -36,6 +36,8 @@ mod executor;
use self::executor::Executor;
use self::segment::{CachedStore, SegmentNodeID};
+const MAX_NUM_SEGMENTS: usize = 200;
+
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct NodeID(pub u64);
@@ -458,6 +460,8 @@ impl<S: Store> Webgraph<S> {
self.save_metadata();
self.node2id.flush();
self.id2node.flush();
+
+ self.merge_segments(MAX_NUM_SEGMENTS);
}
pub fn ingoing_edges(&self, node: Node) -> Vec<FullEdge> {
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index 032768c2..b9866086 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -17,8 +17,7 @@ use crate::{
human_website_annotations::Topic,
prehashed::{hash, split_u128},
schema::{FastField, TextField},
- simhash,
- tokenizer::{self, FlattenedJson},
+ simhash, tokenizer,
webgraph::NodeID,
Error, Result,
};
@@ -709,17 +708,11 @@ impl Html {
let site = self.pretokenize_site();
let description = self.pretokenize_description();
- let schemas: Vec<_> = self
- .schema_org()
- .into_iter()
- .map(|schema| schema.into_single_map())
- .collect();
+ let schemas: Vec<_> = self.schema_org();
- let schema_json = serde_json::to_string(&self.schema_org())
- .ok()
- .unwrap_or_default();
+ let schema_json = serde_json::to_string(&schemas).ok().unwrap_or_default();
- let pretokenized_schema_json = match FlattenedJson::new(&schemas) {
+ let pretokenized_schema_json = match schema_org::flattened_json(schemas) {
Ok(f) => {
let mut tokens = Vec::new();
diff --git a/core/src/webpage/schema_org/json_ld.rs b/core/src/webpage/schema_org/json_ld.rs
index c3d8a02d..55c8bd0b 100644
--- a/core/src/webpage/schema_org/json_ld.rs
+++ b/core/src/webpage/schema_org/json_ld.rs
@@ -16,7 +16,7 @@
use kuchiki::NodeRef;
-use super::Item;
+use super::RawItem;
pub fn convert_all_ints_to_strings(json: &str) -> Result<String, serde_json::Error> {
use serde_json::Value;
@@ -38,7 +38,7 @@ pub fn convert_all_ints_to_strings(json: &str) -> Result<String, serde_json::Err
})
}
-pub fn parse(root: NodeRef) -> Vec<Item> {
+pub(crate) fn parse(root: NodeRef) -> Vec<RawItem> {
let mut res = Vec::new();
for node in root.select("script").unwrap().filter(|node| {
@@ -65,7 +65,7 @@ mod tests {
use kuchiki::traits::TendrilSink;
use maplit::hashmap;
- use crate::webpage::schema_org::{OneOrMany, Property};
+ use crate::webpage::schema_org::{RawOneOrMany, RawProperty};
use super::*;
@@ -100,16 +100,16 @@ mod tests {
assert_eq!(
res,
- vec![Item {
- itemtype: Some(OneOrMany::One("ImageObject".to_string())),
+ vec![RawItem {
+ itemtype: Some(RawOneOrMany::One("ImageObject".to_string())),
properties: hashmap! {
- "@context".to_string() => OneOrMany::One(Property::String("https://schema.org".to_string())),
- "author".to_string() => OneOrMany::One(Property::String("Jane Doe".to_string())),
- "contentLocation".to_string() => OneOrMany::One(Property::String("Puerto Vallarta, Mexico".to_string())),
- "contentUrl".to_string() => OneOrMany::One(Property::String("mexico-beach.jpg".to_string())),
- "datePublished".to_string() => OneOrMany::One(Property::String("2008-01-25".to_string())),
- "description".to_string() => OneOrMany::One(Property::String("I took this picture while on vacation last year.".to_string())),
- "name".to_string() => OneOrMany::One(Property::String("Beach in Mexico".to_string())),
+ "@context".to_string() => RawOneOrMany::One(RawProperty::String("https://schema.org".to_string())),
+ "author".to_string() => RawOneOrMany::One(RawProperty::String("Jane Doe".to_string())),
+ "contentLocation".to_string() => RawOneOrMany::One(RawProperty::String("Puerto Vallarta, Mexico".to_string())),
+ "contentUrl".to_string() => RawOneOrMany::One(RawProperty::String("mexico-beach.jpg".to_string())),
+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2008-01-25".to_string())),
+ "description".to_string() => RawOneOrMany::One(RawProperty::String("I took this picture while on vacation last year.".to_string())),
+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Beach in Mexico".to_string())),
}
}]
);
@@ -162,11 +162,11 @@ mod tests {
assert_eq!(
res,
- vec![Item {
- itemtype: Some(OneOrMany::One("test".to_string())),
+ vec![RawItem {
+ itemtype: Some(RawOneOrMany::One("test".to_string())),
properties: hashmap! {
- "@context".to_string() => OneOrMany::One(Property::String("https://schema.org".to_string())),
- "cost".to_string() => OneOrMany::One(Property::String("123".to_string())),
+ "@context".to_string() => RawOneOrMany::One(RawProperty::String("https://schema.org".to_string())),
+ "cost".to_string() => RawOneOrMany::One(RawProperty::String("123".to_string())),
}
}]
);
diff --git a/core/src/webpage/schema_org/microdata.rs b/core/src/webpage/schema_org/microdata.rs
index c77bf063..228e4b0a 100644
--- a/core/src/webpage/schema_org/microdata.rs
+++ b/core/src/webpage/schema_org/microdata.rs
@@ -20,7 +20,7 @@ use kuchiki::NodeRef;
use std::collections::HashMap;
use thiserror::Error;
-use super::{Item, OneOrMany, Property};
+use super::{RawItem, RawOneOrMany, RawProperty};
pub type Result<T> = std::result::Result<T, Error>;
@@ -36,7 +36,7 @@ pub enum Error {
Json(#[from] serde_json::Error),
}
-fn rec_text_contents(node: NodeRef) -> Vec<Property> {
+fn rec_text_contents(node: NodeRef) -> Vec<RawProperty> {
let mut res = Vec::new();
for child in node.children() {
@@ -46,11 +46,11 @@ fn rec_text_contents(node: NodeRef) -> Vec<Property> {
let mut properties = HashMap::new();
properties.insert(
"text".to_string(),
- OneOrMany::One(Property::String(child.text_contents())),
+ RawOneOrMany::One(RawProperty::String(child.text_contents())),
);
- res.push(Property::Item(Item {
- itemtype: Some(OneOrMany::One("SourceCode".to_string())),
+ res.push(RawProperty::Item(RawItem {
+ itemtype: Some(RawOneOrMany::One("SourceCode".to_string())),
properties,
}))
} else if !elem.attributes.borrow().contains("itemprop")
@@ -67,7 +67,7 @@ fn rec_text_contents(node: NodeRef) -> Vec<Property> {
if had_line_ending {
s.push('\n');
}
- res.push(Property::String(s));
+ res.push(RawProperty::String(s));
}
_ => {}
}
@@ -76,7 +76,7 @@ fn rec_text_contents(node: NodeRef) -> Vec<Property> {
res
}
-fn text_contents(node: NodeRef) -> Vec<Property> {
+fn text_contents(node: NodeRef) -> Vec<RawProperty> {
let properties = rec_text_contents(node);
// merge consecutive strings
@@ -85,15 +85,15 @@ fn text_contents(node: NodeRef) -> Vec<Property> {
for prop in properties {
match &prop {
- Property::String(s) => {
+ RawProperty::String(s) => {
current = match current {
Some(current) => match current {
- Property::String(mut current_str) => {
+ RawProperty::String(mut current_str) => {
current_str.push(' ');
current_str.push_str(s);
- Some(Property::String(current_str))
+ Some(RawProperty::String(current_str))
}
- Property::Item(_) => {
+ RawProperty::Item(_) => {
res.push(current);
Some(prop)
}
@@ -101,7 +101,7 @@ fn text_contents(node: NodeRef) -> Vec<Property> {
None => Some(prop),
}
}
- Property::Item(_) => {
+ RawProperty::Item(_) => {
if let Some(current) = current {
res.push(current);
}
@@ -117,7 +117,7 @@ fn text_contents(node: NodeRef) -> Vec<Property> {
// trim strings
for prop in &mut res {
- if let Property::String(s) = prop {
+ if let RawProperty::String(s) = prop {
let had_line_ending = s.ends_with('\n');
*s = itertools::intersperse(
@@ -142,7 +142,7 @@ fn text_contents(node: NodeRef) -> Vec<Property> {
/// implementation of https://html.spec.whatwg.org/multipage/microdata.html#associating-names-with-items
/// TODO: handle itemrefs
-fn parse_item(root: NodeRef) -> Result<Item> {
+fn parse_item(root: NodeRef) -> Result<RawItem> {
if !root
.as_element()
.unwrap()
@@ -163,13 +163,13 @@ fn parse_item(root: NodeRef) -> Result<Item> {
let itemtype: Vec<_> = s.split_ascii_whitespace().map(String::from).collect();
if itemtype.len() == 1 {
- OneOrMany::One(itemtype.into_iter().next().unwrap())
+ RawOneOrMany::One(itemtype.into_iter().next().unwrap())
} else {
- OneOrMany::Many(itemtype)
+ RawOneOrMany::Many(itemtype)
}
});
- let mut properties: HashMap<String, Vec<Property>> = HashMap::new();
+ let mut properties: HashMap<String, Vec<RawProperty>> = HashMap::new();
let mut pending: Vec<_> = root.children().collect();
while let Some(current) = pending.pop() {
@@ -180,10 +180,10 @@ fn parse_item(root: NodeRef) -> Result<Item> {
if let Some(itemprop) = elem.attributes.borrow().get("itemprop") {
let properties_for_prop = if elem.attributes.borrow().contains("itemscope") {
- vec![Property::Item(parse_item(current)?)]
+ vec![RawProperty::Item(parse_item(current)?)]
} else {
match elem.name.local.to_string().as_str() {
- "meta" => vec![Property::String(
+ "meta" => vec![RawProperty::String(
elem.attributes
.borrow()
.get("content")
@@ -192,26 +192,26 @@ fn parse_item(root: NodeRef) -> Result<Item> {
)],
"audio" | "embed" | "iframe" | "img" | "source" | "track" | "video" => {
if let Some(url) = elem.attributes.borrow().get("src") {
- vec![Property::String(url.to_string())]
+ vec![RawProperty::String(url.to_string())]
} else {
- vec![Property::String(String::new())]
+ vec![RawProperty::String(String::new())]
}
}
"a" | "area" | "link" => {
if let Some(url) = elem.attributes.borrow().get("href") {
- vec![Property::String(url.to_string())]
+ vec![RawProperty::String(url.to_string())]
} else {
- vec![Property::String(String::new())]
+ vec![RawProperty::String(String::new())]
}
}
- "object" => vec![Property::String(
+ "object" => vec![RawProperty::String(
elem.attributes
.borrow()
.get("data")
.map(String::from)
.unwrap_or_default(),
)],
- "data" | "meter" => vec![Property::String(
+ "data" | "meter" => vec![RawProperty::String(
elem.attributes
.borrow()
.get("value")
@@ -226,7 +226,7 @@ fn parse_item(root: NodeRef) -> Result<Item> {
.map(String::from)
.unwrap_or_else(|| current.text_contents());
- vec![Property::String(time)]
+ vec![RawProperty::String(time)]
}
_ => text_contents(current.clone()),
}
@@ -242,7 +242,7 @@ fn parse_item(root: NodeRef) -> Result<Item> {
}
}
- Ok(Item {
+ Ok(RawItem {
itemtype,
properties: properties
.into_iter()
@@ -252,16 +252,19 @@ fn parse_item(root: NodeRef) -> Result<Item> {
if properties.is_empty() {
None
} else if properties.len() == 1 {
- Some((name, OneOrMany::One(properties.into_iter().next().unwrap())))
+ Some((
+ name,
+ RawOneOrMany::One(properties.into_iter().next().unwrap()),
+ ))
} else {
- Some((name, OneOrMany::Many(properties)))
+ Some((name, RawOneOrMany::Many(properties)))
}
})
.collect(),
})
}
-fn parse(root: NodeRef) -> Vec<Item> {
+fn parse(root: NodeRef) -> Vec<RawItem> {
let mut res = Vec::new();
let mut pending: Vec<_> = root.inclusive_descendants().collect();
@@ -280,10 +283,10 @@ fn parse(root: NodeRef) -> Vec<Item> {
res
}
-fn fix_type_for_schema(mut item: Item) -> Item {
- if let Some(OneOrMany::One(itemtype)) = &item.itemtype {
+fn fix_type_for_schema(mut item: RawItem) -> RawItem {
+ if let Some(RawOneOrMany::One(itemtype)) = &item.itemtype {
if let Some(last) = itemtype.split('/').last() {
- item.itemtype = Some(OneOrMany::One(last.to_string()));
+ item.itemtype = Some(RawOneOrMany::One(last.to_string()));
}
}
@@ -291,24 +294,24 @@ fn fix_type_for_schema(mut item: Item) -> Item {
.properties
.into_iter()
.map(|(key, properties)| match properties {
- OneOrMany::One(property) => {
- if let Property::Item(subitem) = property {
+ RawOneOrMany::One(property) => {
+ if let RawProperty::Item(subitem) = property {
(
key,
- OneOrMany::One(Property::Item(fix_type_for_schema(subitem))),
+ RawOneOrMany::One(RawProperty::Item(fix_type_for_schema(subitem))),
)
} else {
- (key, OneOrMany::One(property))
+ (key, RawOneOrMany::One(property))
}
}
- OneOrMany::Many(properties) => (
+ RawOneOrMany::Many(properties) => (
key,
- OneOrMany::Many(
+ RawOneOrMany::Many(
properties
.into_iter()
.map(|property| {
- if let Property::Item(subitem) = property {
- Property::Item(fix_type_for_schema(subitem))
+ if let RawProperty::Item(subitem) = property {
+ RawProperty::Item(fix_type_for_schema(subitem))
} else {
property
}
@@ -322,7 +325,7 @@ fn fix_type_for_schema(mut item: Item) -> Item {
item
}
-pub fn parse_schema(root: NodeRef) -> Vec<Item> {
+pub fn parse_schema(root: NodeRef) -> Vec<RawItem> {
parse(root).into_iter().map(fix_type_for_schema).collect()
}
@@ -331,7 +334,7 @@ mod tests {
use kuchiki::traits::TendrilSink;
use maplit::hashmap;
- use crate::webpage::schema_org::OneOrMany;
+ use crate::webpage::schema_org::RawOneOrMany;
use super::*;
@@ -353,11 +356,11 @@ mod tests {
assert_eq!(
parse_item(root).unwrap(),
- Item {
- itemtype: Some(OneOrMany::One(String::from("http://n.whatwg.org/work"))),
+ RawItem {
+ itemtype: Some(RawOneOrMany::One(String::from("http://n.whatwg.org/work"))),
properties: hashmap! {
- "work".to_string() => OneOrMany::One(Property::String("images/house.jpeg".to_string())),
- "title".to_string() => OneOrMany::One(Property::String("The house I found.".to_string())),
+ "work".to_string() => RawOneOrMany::One(RawProperty::String("images/house.jpeg".to_string())),
+ "title".to_string() => RawOneOrMany::One(RawProperty::String("The house I found.".to_string())),
}
}
);
@@ -390,25 +393,25 @@ mod tests {
.as_node()
.clone();
- let expected = Item {
- itemtype: Some(OneOrMany::One(String::from(
+ let expected = RawItem {
+ itemtype: Some(RawOneOrMany::One(String::from(
"http://schema.org/BlogPosting",
))),
properties: hashmap! {
- "comment".to_string() => OneOrMany::One(
- Property::Item(
- Item {
- itemtype: Some(OneOrMany::One("http://schema.org/UserComments".to_string())),
+ "comment".to_string() => RawOneOrMany::One(
+ RawProperty::Item(
+ RawItem {
+ itemtype: Some(RawOneOrMany::One("http://schema.org/UserComments".to_string())),
properties: hashmap! {
- "url".to_string() => OneOrMany::One(Property::String("#c1".to_string())),
- "creator".to_string() => OneOrMany::One(
- Property::Item(Item {
- itemtype: Some(OneOrMany::One("http://schema.org/Person".to_string())),
+ "url".to_string() => RawOneOrMany::One(RawProperty::String("#c1".to_string())),
+ "creator".to_string() => RawOneOrMany::One(
+ RawProperty::Item(RawItem {
+ itemtype: Some(RawOneOrMany::One("http://schema.org/Person".to_string())),
properties: hashmap! {
- "name".to_string() => OneOrMany::One(Property::String("Greg".to_string()))
+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Greg".to_string()))
}
})),
- "commentTime".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string()))
+ "commentTime".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string()))
}
})),
},
@@ -497,43 +500,43 @@ mod tests {
let res = parse(root);
assert_eq!(res.len(), 2);
- let expected_article = Item {
- itemtype: Some(OneOrMany::One(String::from(
+ let expected_article = RawItem {
+ itemtype: Some(RawOneOrMany::One(String::from(
"http://schema.org/BlogPosting",
))),
properties: hashmap! {
- "headline".to_string() => OneOrMany::One(Property::String(String::from("Progress report"))),
- "datePublished".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string())),
- "url".to_string() => OneOrMany::One(Property::String("?comments=0".to_string())),
- "comment".to_string() => OneOrMany::Many(vec![
- Property::Item(
- Item {
- itemtype: Some(OneOrMany::One("http://schema.org/UserComments".to_string())),
+ "headline".to_string() => RawOneOrMany::One(RawProperty::String(String::from("Progress report"))),
+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string())),
+ "url".to_string() => RawOneOrMany::One(RawProperty::String("?comments=0".to_string())),
+ "comment".to_string() => RawOneOrMany::Many(vec![
+ RawProperty::Item(
+ RawItem {
+ itemtype: Some(RawOneOrMany::One("http://schema.org/UserComments".to_string())),
properties: hashmap! {
- "url".to_string() => OneOrMany::One(Property::String("#c1".to_string())),
- "creator".to_string() => OneOrMany::One(
- Property::Item(Item {
- itemtype: Some(OneOrMany::One("http://schema.org/Person".to_string())),
+ "url".to_string() => RawOneOrMany::One(RawProperty::String("#c1".to_string())),
+ "creator".to_string() => RawOneOrMany::One(
+ RawProperty::Item(RawItem {
+ itemtype: Some(RawOneOrMany::One("http://schema.org/Person".to_string())),
properties: hashmap! {
- "name".to_string() => OneOrMany::One(Property::String("Greg".to_string()))
+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Greg".to_string()))
}
})),
- "commentTime".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string()))
+ "commentTime".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string()))
}
}),
- Property::Item(
- Item {
- itemtype: Some(OneOrMany::One("http://schema.org/UserComments".to_string())),
+ RawProperty::Item(
+ RawItem {
+ itemtype: Some(RawOneOrMany::One("http://schema.org/UserComments".to_string())),
properties: hashmap! {
- "url".to_string() => OneOrMany::One(Property::String("#c2".to_string())),
- "creator".to_string() => OneOrMany::One(
- Property::Item(Item {
- itemtype: Some(OneOrMany::One("http://schema.org/Person".to_string())),
+ "url".to_string() => RawOneOrMany::One(RawProperty::String("#c2".to_string())),
+ "creator".to_string() => RawOneOrMany::One(
+ RawProperty::Item(RawItem {
+ itemtype: Some(RawOneOrMany::One("http://schema.org/Person".to_string())),
properties: hashmap! {
- "name".to_string() => OneOrMany::One(Property::String("Charlotte".to_string()))
+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Charlotte".to_string()))
}
})),
- "commentTime".to_string() => OneOrMany::One(Property::String("2013-08-29".to_string()))
+ "commentTime".to_string() => RawOneOrMany::One(RawProperty::String("2013-08-29".to_string()))
}
})
]),
@@ -616,15 +619,15 @@ mod tests {
assert_eq!(
res,
- vec![Item {
- itemtype: Some(OneOrMany::One("ImageObject".to_string())),
+ vec![RawItem {
+ itemtype: Some(RawOneOrMany::One("ImageObject".to_string())),
properties: hashmap! {
- "author".to_string() => OneOrMany::One(Property::String("Jane Doe".to_string())),
- "contentLocation".to_string() => OneOrMany::One(Property::String("Puerto Vallarta, Mexico".to_string())),
- "contentUrl".to_string() => OneOrMany::One(Property::String("mexico-beach.jpg".to_string())),
- "datePublished".to_string() => OneOrMany::One(Property::String("2008-01-25".to_string())),
- "description".to_string() => OneOrMany::One(Property::String("I took this picture while on vacation last year.".to_string())),
- "name".to_string() => OneOrMany::One(Property::String("Beach in Mexico".to_string())),
+ "author".to_string() => RawOneOrMany::One(RawProperty::String("Jane Doe".to_string())),
+ "contentLocation".to_string() => RawOneOrMany::One(RawProperty::String("Puerto Vallarta, Mexico".to_string())),
+ "contentUrl".to_string() => RawOneOrMany::One(RawProperty::String("mexico-beach.jpg".to_string())),
+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2008-01-25".to_string())),
+ "description".to_string() => RawOneOrMany::One(RawProperty::String("I took this picture while on vacation last year.".to_string())),
+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Beach in Mexico".to_string())),
}
}]
);
diff --git a/core/src/webpage/schema_org/mod.rs b/core/src/webpage/schema_org/mod.rs
index 936e8c8d..37def264 100644
--- a/core/src/webpage/schema_org/mod.rs
+++ b/core/src/webpage/schema_org/mod.rs
@@ -19,39 +19,38 @@ use std::collections::HashMap;
use kuchiki::NodeRef;
use serde::{Deserialize, Serialize};
+use crate::tokenizer::FlattenedJson;
+use crate::Result;
+
mod json_ld;
mod microdata;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(untagged)]
pub enum Property {
String(String),
Item(Item),
}
-
impl Property {
- pub fn try_into_string(self) -> Option<String> {
- if let Property::String(s) = self {
- Some(s)
- } else {
- None
+ pub(crate) fn try_into_string(&self) -> Option<String> {
+ match self {
+ Property::String(s) => Some(s.clone()),
+ Property::Item(_) => None,
}
}
- pub fn try_into_item(self) -> Option<Item> {
- if let Property::Item(item) = self {
- Some(item)
- } else {
- None
+ pub(crate) fn try_into_item(&self) -> Option<Item> {
+ match self {
+ Property::String(_) => None,
+ Property::Item(it) => Some(it.clone()),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(untagged)]
-pub enum SingleMap {
+enum SingleMap {
Leaf(String),
- Node(HashMap<OneOrMany<String>, HashMap<String, OneOrMany<SingleMap>>>),
+ Node(HashMap<RawOneOrMany<String>, HashMap<String, RawOneOrMany<SingleMap>>>),
}
impl From<Property> for SingleMap {
@@ -66,15 +65,18 @@ impl From<Property> for SingleMap {
.properties
.into_iter()
.map(|(key, val)| match val {
- OneOrMany::One(one) => (key, OneOrMany::One(SingleMap::from(one))),
+ OneOrMany::One(one) => (key, RawOneOrMany::One(SingleMap::from(one))),
OneOrMany::Many(many) => (
key,
- OneOrMany::Many(many.into_iter().map(SingleMap::from).collect()),
+ RawOneOrMany::Many(many.into_iter().map(SingleMap::from).collect()),
),
})
.collect();
- res.insert(tt, recursive);
+ match tt {
+ OneOrMany::One(one) => res.insert(RawOneOrMany::One(one), recursive),
+ OneOrMany::Many(many) => res.insert(RawOneOrMany::Many(many), recursive),
+ };
}
SingleMap::Node(res)
@@ -83,11 +85,50 @@ impl From<Property> for SingleMap {
}
}
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
+pub struct RawItem {
+ #[serde(rename = "@type")]
+ itemtype: Option<RawOneOrMany<String>>,
+ #[serde(flatten)]
+ properties: HashMap<String, RawOneOrMany<RawProperty>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
+#[serde(untagged)]
+enum RawProperty {
+ String(String),
+ Item(RawItem),
+}
+impl RawProperty {
+ #[cfg(test)]
+ fn try_into_item(&self) -> Option<RawItem> {
+ match self {
+ RawProperty::String(_) => None,
+ RawProperty::Item(it) => Some(it.clone()),
+ }
+ }
+
+ #[cfg(test)]
+ fn try_into_string(&self) -> Option<String> {
+ match self {
+ RawProperty::String(s) => Some(s.clone()),
+ RawProperty::Item(_) => None,
+ }
+ }
+}
+
+impl From<RawProperty> for Property {
+ fn from(value: RawProperty) -> Self {
+ match value {
+ RawProperty::String(s) => Self::String(s),
+ RawProperty::Item(it) => Self::Item(Item::from(it)),
+ }
+ }
+}
+
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Item {
- #[serde(rename = "@type")]
pub itemtype: Option<OneOrMany<String>>,
- #[serde(flatten)]
pub properties: HashMap<String, OneOrMany<Property>>,
}
@@ -102,13 +143,34 @@ impl Item {
}
}
- pub fn into_single_map(self) -> SingleMap {
+ fn into_single_map(self) -> SingleMap {
SingleMap::from(Property::Item(self))
}
}
+impl From<RawItem> for Item {
+ fn from(value: RawItem) -> Self {
+ Self {
+ itemtype: value.itemtype.map(|tt| match tt {
+ RawOneOrMany::One(one) => OneOrMany::One(one),
+ RawOneOrMany::Many(many) => OneOrMany::Many(many),
+ }),
+ properties: value
+ .properties
+ .into_iter()
+ .map(|(key, val)| match val {
+ RawOneOrMany::One(prop) => (key, OneOrMany::One(Property::from(prop))),
+ RawOneOrMany::Many(props) => (
+ key,
+ OneOrMany::Many(props.into_iter().map(Property::from).collect()),
+ ),
+ })
+ .collect(),
+ }
+ }
+}
+
#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Hash)]
-#[serde(untagged)]
pub enum OneOrMany<T> {
One(T),
Many(Vec<T>),
@@ -130,11 +192,36 @@ impl<T> OneOrMany<T> {
}
}
+#[derive(Deserialize, Serialize, PartialEq, Eq, Debug, Clone, Hash)]
+#[serde(untagged)]
+enum RawOneOrMany<T> {
+ One(T),
+ Many(Vec<T>),
+}
+
+impl<T> RawOneOrMany<T> {
+ #[cfg(test)]
+ pub fn one(self) -> Option<T> {
+ match self {
+ RawOneOrMany::One(one) => Some(one),
+ RawOneOrMany::Many(many) => many.into_iter().next(),
+ }
+ }
+
+ #[cfg(test)]
+ pub fn many(self) -> Vec<T> {
+ match self {
+ RawOneOrMany::One(one) => vec![one],
+ RawOneOrMany::Many(many) => many,
+ }
+ }
+}
+
pub fn parse(root: NodeRef) -> Vec<Item> {
let mut res = self::json_ld::parse(root.clone());
res.append(&mut self::microdata::parse_schema(root));
- res
+ res.into_iter().map(Item::from).collect()
}
#[cfg(test)]
@@ -160,19 +247,19 @@ mod tests {
}
"#;
- let item: Item = serde_json::from_str(json).unwrap();
+ let item: RawItem = serde_json::from_str(json).unwrap();
assert_eq!(
item,
- Item {
- itemtype: Some(OneOrMany::One("ImageObject".to_string())),
+ RawItem {
+ itemtype: Some(RawOneOrMany::One("ImageObject".to_string())),
properties: hashmap! {
- "@context".to_string() => OneOrMany::One(Property::String("https://schema.org".to_string())),
- "author".to_string() => OneOrMany::One(Property::String("Jane Doe".to_string())),
- "contentLocation".to_string() => OneOrMany::One(Property::String("Puerto Vallarta, Mexico".to_string())),
- "contentUrl".to_string() => OneOrMany::One(Property::String("mexico-beach.jpg".to_string())),
- "datePublished".to_string() => OneOrMany::One(Property::String("2008-01-25".to_string())),
- "description".to_string() => OneOrMany::One(Property::String("I took this picture while on vacation last year.".to_string())),
- "name".to_string() => OneOrMany::One(Property::String("Beach in Mexico".to_string())),
+ "@context".to_string() => RawOneOrMany::One(RawProperty::String("https://schema.org".to_string())),
+ "author".to_string() => RawOneOrMany::One(RawProperty::String("Jane Doe".to_string())),
+ "contentLocation".to_string() => RawOneOrMany::One(RawProperty::String("Puerto Vallarta, Mexico".to_string())),
+ "contentUrl".to_string() => RawOneOrMany::One(RawProperty::String("mexico-beach.jpg".to_string())),
+ "datePublished".to_string() => RawOneOrMany::One(RawProperty::String("2008-01-25".to_string())),
+ "description".to_string() => RawOneOrMany::One(RawProperty::String("I took this picture while on vacation last year.".to_string())),
+ "name".to_string() => RawOneOrMany::One(RawProperty::String("Beach in Mexico".to_string())),
}
}
);
@@ -190,7 +277,7 @@ mod tests {
assert!(res[0].properties.contains_key("primaryImageOfPage"));
assert_eq!(
res[0].properties.get("name"),
- Some(&OneOrMany::One(Property::String(
+ Some(&RawOneOrMany::One(RawProperty::String(
"RegEx match open tags except XHTML self-contained tags".to_string()
)))
);
@@ -205,16 +292,19 @@ mod tests {
.try_into_item()
.unwrap();
- assert_eq!(main.itemtype, Some(OneOrMany::One("Question".to_string())));
+ assert_eq!(
+ main.itemtype,
+ Some(RawOneOrMany::One("Question".to_string()))
+ );
assert_eq!(
main.properties.get("name"),
- Some(&OneOrMany::One(Property::String(
+ Some(&RawOneOrMany::One(RawProperty::String(
"RegEx match open tags except XHTML self-contained tags".to_string()
)))
);
assert_eq!(
main.properties.get("dateCreated"),
- Some(&OneOrMany::One(Property::String(
+ Some(&RawOneOrMany::One(RawProperty::String(
"2009-11-13T22:38:26".to_string()
)))
);
@@ -233,14 +323,14 @@ mod tests {
let text = main.properties.get("text").unwrap().clone().many();
- assert_eq!(text[0], Property::String("Locked . Comments on this question have been disabled, but it is still accepting new answers and other interactions. Learn more .\nI need to match all of these opening tags:\n".to_string()));
+ assert_eq!(text[0], RawProperty::String("Locked . Comments on this question have been disabled, but it is still accepting new answers and other interactions. Learn more .\nI need to match all of these opening tags:\n".to_string()));
assert_eq!(
text[1],
- Property::Item(Item {
- itemtype: Some(OneOrMany::One("SourceCode".to_string())),
+ RawProperty::Item(RawItem {
+ itemtype: Some(RawOneOrMany::One("SourceCode".to_string())),
properties: hashmap! {
- "text".to_string() => OneOrMany::One(Property::String("<p>\n<a href=\"foo\">\n".to_string()))
+ "text".to_string() => RawOneOrMany::One(RawProperty::String("<p>\n<a href=\"foo\">\n".to_string()))
}
})
);
@@ -267,45 +357,47 @@ mod tests {
assert_eq!(
recipe.properties.get("recipeCategory"),
- Some(&OneOrMany::One(Property::String("Aftensmad".to_string())))
+ Some(&RawOneOrMany::One(RawProperty::String(
+ "Aftensmad".to_string()
+ )))
);
assert_eq!(
recipe.properties.get("name"),
- Some(&OneOrMany::One(Property::String(
+ Some(&RawOneOrMany::One(RawProperty::String(
"One Pot Pasta med chorizo".to_string()
)))
);
assert_eq!(
recipe.properties.get("recipeYield"),
- Some(&OneOrMany::One(Property::String("4".to_string())))
+ Some(&RawOneOrMany::One(RawProperty::String("4".to_string())))
);
assert_eq!(
recipe.properties.get("cookTime"),
- Some(&OneOrMany::One(Property::String("PT25M".to_string())))
+ Some(&RawOneOrMany::One(RawProperty::String("PT25M".to_string())))
);
assert_eq!(
recipe.properties.get("recipeIngredient"),
- Some(&OneOrMany::Many(vec![
- Property::String("400 g spaghetti".to_string()),
- Property::String("1 dåse hakkede tomater".to_string()),
- Property::String("1 håndfuld frisk basilikum, bladene herfra".to_string()),
- Property::String("1 løg, finthakket".to_string()),
- Property::String("2 fed hvidløg, finthakket".to_string()),
- Property::String("20 cherrytomater, skåret i både".to_string()),
- Property::String("0,50 squash, groftrevet".to_string()),
- Property::String("1 tsk oregano, tørret".to_string()),
- Property::String("50 g chorizo, finthakket".to_string()),
- Property::String("5 dl grøntsagsbouillon".to_string()),
- Property::String("2 spsk olivenolie".to_string()),
- Property::String("1 tsk chiliflager, kan undlades".to_string()),
- Property::String("1 tsk salt".to_string()),
- Property::String("sort peber, friskkværnet".to_string()),
- Property::String("50 g parmesan, friskrevet til servering".to_string()),
- Property::String("1 håndfuld frisk basilikum, blade".to_string()),
+ Some(&RawOneOrMany::Many(vec![
+ RawProperty::String("400 g spaghetti".to_string()),
+ RawProperty::String("1 dåse hakkede tomater".to_string()),
+ RawProperty::String("1 håndfuld frisk basilikum, bladene herfra".to_string()),
+ RawProperty::String("1 løg, finthakket".to_string()),
+ RawProperty::String("2 fed hvidløg, finthakket".to_string()),
+ RawProperty::String("20 cherrytomater, skåret i både".to_string()),
+ RawProperty::String("0,50 squash, groftrevet".to_string()),
+ RawProperty::String("1 tsk oregano, tørret".to_string()),
+ RawProperty::String("50 g chorizo, finthakket".to_string()),
+ RawProperty::String("5 dl grøntsagsbouillon".to_string()),
+ RawProperty::String("2 spsk olivenolie".to_string()),
+ RawProperty::String("1 tsk chiliflager, kan undlades".to_string()),
+ RawProperty::String("1 tsk salt".to_string()),
+ RawProperty::String("sort peber, friskkværnet".to_string()),
+ RawProperty::String("50 g parmesan, friskrevet til servering".to_string()),
+ RawProperty::String("1 håndfuld frisk basilikum, blade".to_string()),
]))
);
@@ -334,3 +426,11 @@ mod tests {
assert_eq!(&instructions, "Helt enkelt som navnet antyder, så kom alle ingredienserne i en stor gryde på én gang. Kog retten op, rør godt rundt i gryden og skru ned for varmen. Lad det simrekoge under låg i 10-12 minutter, til spaghettien er perfekt kogt – al dente med lidt bid i. Server med revet parmesan og basilikum.");
}
}
+
+pub(crate) fn flattened_json(schemas: Vec<Item>) -> Result<FlattenedJson> {
+ let single_maps: Vec<_> = schemas
+ .into_iter()
+ .map(|item| item.into_single_map())
+ .collect();
+ FlattenedJson::new(&single_maps)
+}
diff --git a/frontend/src/components/Sidebar.astro b/frontend/src/components/Sidebar.astro
index ea397600..3da2cae1 100644
--- a/frontend/src/components/Sidebar.astro
+++ b/frontend/src/components/Sidebar.astro
@@ -11,8 +11,8 @@ export interface Props {
<div>
{
askama.match("sidebar", {
- "DisplayedSidebar::Entity(entity)": <Entity entity="entity" />,
- "DisplayedSidebar::StackOverflow{title, answer}": (
+ "Sidebar::Entity(entity)": <Entity entity="entity" />,
+ "Sidebar::StackOverflow{title, answer}": (
<StackOverflow title="title" answer="answer" />
),
})