Skip to content

web search done right

refactor

Mikkel Denker5d20184b88bd
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rs
index d8a11550..bf156f6c 100644
--- a/core/src/frontend/search.rs
+++ b/core/src/frontend/search.rs
@@ -14,6 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
+use http::Uri;
use optics::SiteRankings;
use std::{collections::HashMap, sync::Arc};
@@ -61,6 +62,27 @@ enum RegionSelection {
Unselected(Region),
}
+fn extract_site_rankings(params: &HashMap<String, String>) -> Option<SiteRankings> {
+ match params.get("sr") {
+ Some(sr) => {
+ if !sr.is_empty() {
+ if let Ok(site_rankings) = base64::decode(sr) {
+ if let Ok(site_rankings) = std::str::from_utf8(&site_rankings) {
+ serde_json::from_str(site_rankings).ok()
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ } else {
+ None
+ }
+ }
+ None => None,
+ }
+}
+
#[allow(clippy::unused_async)]
#[allow(clippy::match_wild_err_arm)]
#[debug_handler]
@@ -78,6 +100,7 @@ pub async fn route(
let offset = skip_pages * NUM_RESULTS_PER_PAGE;
let mut optic = None;
+ // should only be set if optic was successfully downloaded
let mut current_optic_url = None;
if let Some(url) = params.get("optic") {
@@ -99,24 +122,7 @@ pub async fn route(
}
});
- let site_rankings: Option<SiteRankings> = match params.get("sr") {
- Some(sr) => {
- if !sr.is_empty() {
- if let Ok(site_rankings) = base64::decode(sr) {
- if let Ok(site_rankings) = std::str::from_utf8(&site_rankings) {
- serde_json::from_str(site_rankings).ok()
- } else {
- None
- }
- } else {
- None
- }
- } else {
- None
- }
- }
- None => None,
- };
+ let site_rankings = extract_site_rankings(&params);
match state
.searcher
@@ -140,45 +146,12 @@ pub async fn route(
let search_duration_sec =
format!("{:.2}", result.search_duration_ms as f64 / 1000.0);
- let all_regions = ALL_REGIONS
- .into_iter()
- .map(|region| {
- if let Some(selected_region) = selected_region {
- if region == selected_region {
- RegionSelection::Selected(region)
- } else {
- RegionSelection::Unselected(region)
- }
- } else {
- RegionSelection::Unselected(region)
- }
- })
- .collect();
+ let all_regions = generate_regions(selected_region);
+ let next_page_url = next_page_url(&uri, params.clone(), skip_pages);
+ let prev_page_url = prev_page_url(&uri, params, skip_pages);
let current_page = skip_pages + 1;
- let mut next_page_params = params.clone();
- next_page_params.insert("p".to_string(), (skip_pages + 1).to_string());
- let next_page_url = uri.path().to_string()
- + "?"
- + serde_urlencoded::to_string(&next_page_params)
- .unwrap()
- .as_str();
-
- let prev_page_url = if current_page > 1 {
- let mut prev_page_params = params;
- prev_page_params.insert("p".to_string(), (skip_pages - 1).to_string());
- Some(
- uri.path().to_string()
- + "?"
- + serde_urlencoded::to_string(&prev_page_params)
- .unwrap()
- .as_str(),
- )
- } else {
- None
- };
-
let template = SearchTemplate {
search_result: result.webpages,
query,
@@ -203,6 +176,51 @@ pub async fn route(
}
}
+fn generate_regions(selected_region: Option<Region>) -> Vec<RegionSelection> {
+ ALL_REGIONS
+ .into_iter()
+ .map(|region| {
+ if let Some(selected_region) = selected_region {
+ if region == selected_region {
+ RegionSelection::Selected(region)
+ } else {
+ RegionSelection::Unselected(region)
+ }
+ } else {
+ RegionSelection::Unselected(region)
+ }
+ })
+ .collect()
+}
+
+fn prev_page_url(uri: &Uri, params: HashMap<String, String>, skip_pages: usize) -> Option<String> {
+ if skip_pages > 0 {
+ let mut prev_page_params = params;
+ prev_page_params.insert("p".to_string(), (skip_pages - 1).to_string());
+ Some(
+ uri.path().to_string()
+ + "?"
+ + serde_urlencoded::to_string(&prev_page_params)
+ .unwrap()
+ .as_str(),
+ )
+ } else {
+ None
+ }
+}
+
+fn next_page_url(uri: &Uri, params: HashMap<String, String>, skip_pages: usize) -> String {
+ let mut next_page_params = params;
+ next_page_params.insert("p".to_string(), (skip_pages + 1).to_string());
+ let next_page_url = uri.path().to_string()
+ + "?"
+ + serde_urlencoded::to_string(&next_page_params)
+ .unwrap()
+ .as_str();
+
+ next_page_url
+}
+
#[allow(clippy::unused_async)]
#[allow(clippy::match_wild_err_arm)]
#[debug_handler]
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index ec661909..5f678ef7 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -19,6 +19,7 @@ use crate::{
ranking::SignalAggregator,
schema::{Field, TextField},
searcher::NUM_RESULTS_PER_PAGE,
+ webpage::region::Region,
Result,
};
use optics::Optic;
@@ -48,6 +49,8 @@ pub struct Query {
terms: Vec<Box<Term>>,
simple_terms_text: Vec<String>,
tantivy_query: Box<BooleanQuery>,
+ offset: usize,
+ region: Option<Region>,
num_results: usize,
}
@@ -156,6 +159,8 @@ impl Query {
terms,
simple_terms_text,
tantivy_query,
+ offset: 0,
+ region: None,
num_results: NUM_RESULTS_PER_PAGE,
})
}
@@ -167,7 +172,7 @@ impl Query {
&mut optic.as_multiple_tantivy(&index.schema(), index.inverted_index.fastfield_cache()),
);
- self.tantivy_query = Box::new(BooleanQuery::new(subqueries))
+ self.tantivy_query = Box::new(BooleanQuery::new(subqueries));
}
pub fn simple_terms(&self) -> Vec<String> {
@@ -178,6 +183,14 @@ impl Query {
self.num_results = num_results;
}
+ pub fn set_offset(&mut self, offset: usize) {
+ self.offset = offset;
+ }
+
+ pub fn set_region(&mut self, region: Region) {
+ self.region = Some(region);
+ }
+
pub fn terms(&self) -> &[Box<Term>] {
&self.terms
}
@@ -189,6 +202,14 @@ impl Query {
pub fn num_results(&self) -> usize {
self.num_results
}
+
+ pub fn offset(&self) -> usize {
+ self.offset
+ }
+
+ pub fn region(&self) -> Option<&Region> {
+ self.region.as_ref()
+ }
}
impl tantivy::query::Query for Query {
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index a4ce8595..1ae5f243 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -48,6 +48,7 @@ use crate::{
pub use self::signal::*;
+#[derive(Clone)]
pub struct Ranker {
region_count: Arc<RegionCount>,
selected_region: Option<Region>,
diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rs
deleted file mode 100644
index df1e1931..00000000
--- a/core/src/search_prettifier.rs
+++ /dev/null
@@ -1,683 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::collections::BTreeMap;
-
-use chrono::{NaiveDate, NaiveDateTime, Utc};
-use itertools::{intersperse, Itertools};
-use serde::{Deserialize, Serialize};
-
-use crate::{
- entity_index::{entity::Span, StoredEntity},
- inverted_index::RetrievedWebpage,
- searcher::LocalSearcher,
- spell::{self, CorrectionTerm},
- webpage::{
- schema_org::{self, Item, OneOrMany, Property},
- Url,
- },
- Error, Result,
-};
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum Snippet {
- Normal {
- date: Option<String>,
- text: String,
- },
- StackOverflowQA {
- question: StackOverflowQuestion,
- answers: Vec<StackOverflowAnswer>,
- },
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct StackOverflowAnswer {
- pub body: Vec<CodeOrText>,
- pub date: String,
- pub url: String,
- pub upvotes: u32,
- pub accepted: bool,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct StackOverflowQuestion {
- pub body: Vec<CodeOrText>,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum CodeOrText {
- Code(String),
- Text(String),
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct HighlightedSpellCorrection {
- pub raw: String,
- pub highlighted: String,
-}
-
-impl From<spell::Correction> for HighlightedSpellCorrection {
- fn from(correction: spell::Correction) -> Self {
- let mut highlighted = String::new();
- let mut raw = String::new();
-
- for term in correction.terms {
- match term {
- CorrectionTerm::Corrected(correction) => {
- highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));
- raw.push_str(&correction);
- }
- CorrectionTerm::NotCorrected(orig) => {
- highlighted.push_str(&orig);
- raw.push_str(&orig);
- }
- }
-
- raw.push(' ');
- highlighted.push(' ');
- }
-
- raw = raw.trim_end().to_string();
- highlighted = highlighted.trim_end().to_string();
-
- Self { raw, highlighted }
- }
-}
-
-pub fn html_escape(s: &str) -> String {
- html_escape::decode_html_entities(s)
- .chars()
- .filter(|c| !matches!(c, '<' | '>' | '&'))
- .collect()
-}
-
-fn prettify_url(url: &Url) -> String {
- let mut pretty_url = url.strip_query().to_string();
-
- if pretty_url.ends_with('/') {
- pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
- }
-
- let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
- pretty_url = Url::from(pretty_url.clone())
- .strip_protocol()
- .replace('/', " › ");
- pretty_url = protocol + &pretty_url;
-
- pretty_url
-}
-
-fn prettify_date(date: NaiveDateTime) -> String {
- let current_time = Utc::now().naive_utc();
- let diff = current_time.signed_duration_since(date);
-
- let num_hours = diff.num_hours() + 1;
- if num_hours < 24 {
- if num_hours <= 1 {
- return "1 hour ago".to_string();
- }
-
- return format!("{num_hours} hours ago");
- }
-
- let num_days = diff.num_days();
- if num_days < 30 {
- if num_days <= 1 {
- return "1 day ago".to_string();
- }
-
- return format!("{num_days} days ago");
- }
-
- format!("{}", date.format("%d. %b. %Y"))
-}
-
-fn parse_so_answer(
- text: OneOrMany<Property>,
- date: OneOrMany<Property>,
- upvotes: OneOrMany<Property>,
- url: OneOrMany<Property>,
- webpage_url: Url,
- accepted: bool,
-) -> Option<StackOverflowAnswer> {
- let text: Vec<_> = text
- .many()
- .into_iter()
- .map(|prop| match prop {
- Property::String(s) => CodeOrText::Text(s),
- Property::Item(item) => CodeOrText::Code(
- item.properties
- .get("text")
- .and_then(|p| p.clone().one())
- .and_then(|prop| prop.try_into_string())
- .unwrap_or_default(),
- ),
- })
- .collect();
-
- let date = chrono::NaiveDateTime::parse_from_str(
- date.one()
- .and_then(|prop| prop.try_into_string())
- .unwrap_or_default()
- .as_str(),
- "%Y-%m-%dT%H:%M:%S",
- )
- .ok()?;
-
- let upvotes = upvotes
- .one()
- .and_then(|prop| prop.try_into_string())
- .and_then(|s| s.parse().ok())?;
-
- let url = url
- .one()
- .and_then(|prop| prop.try_into_string())
- .map(Url::from)
- .map(|mut url| {
- url.prefix_with(&Url::from(webpage_url.site()));
- url
- })?;
-
- Some(StackOverflowAnswer {
- body: text,
- date: format!("{}", date.date().format("%b %d, %Y")),
- upvotes,
- url: url.full(),
- accepted,
- })
-}
-
-fn schema_item_to_stackoverflow_answer(
- item: schema_org::Item,
- url: Url,
- accepted: bool,
-) -> Option<StackOverflowAnswer> {
- match (
- item.properties.get("text"),
- item.properties.get("dateCreated"),
- item.properties.get("upvoteCount"),
- item.properties.get("url"),
- ) {
- (Some(text), Some(date), Some(upvotes), Some(answer_url)) => parse_so_answer(
- text.clone(),
- date.clone(),
- upvotes.clone(),
- answer_url.clone(),
- url,
- accepted,
- ),
- _ => None,
- }
-}
-
-fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
- match webpage
- .schema_org
- .iter()
- .find(|item| item.types_contains("QAPage"))
- .and_then(|item| item.properties.get("mainEntity"))
- .and_then(|properties| properties.clone().one())
- .and_then(|property| property.try_into_item())
- {
- Some(item) => {
- let question: Vec<CodeOrText> = item
- .properties
- .get("text")
- .map(|item| item.clone().many())
- .unwrap_or_default()
- .into_iter()
- .map(|prop| match prop {
- Property::String(s) => CodeOrText::Text(s),
- Property::Item(item) => CodeOrText::Code(
- item.properties
- .get("text")
- .and_then(|p| p.clone().one())
- .and_then(|prop| prop.try_into_string())
- .unwrap_or_default(),
- ),
- })
- .collect();
-
- let mut answers = Vec::new();
-
- if let Some(ans) = item
- .properties
- .get("acceptedAnswer")
- .cloned()
- .and_then(|ans| ans.one())
- .and_then(|prop| prop.try_into_item())
- .and_then(|item| {
- schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), true)
- })
- {
- answers.push(ans);
- }
-
- for answer in item
- .properties
- .get("suggestedAnswer")
- .cloned()
- .map(|answers| answers.many())
- .unwrap_or_default()
- .into_iter()
- .filter_map(|prop| prop.try_into_item())
- .filter_map(|item| {
- schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), false)
- })
- {
- answers.push(answer);
- }
-
- Ok(Snippet::StackOverflowQA {
- question: StackOverflowQuestion { body: question },
- answers: answers.into_iter().take(3).collect(),
- })
- }
- None => Err(Error::InvalidStackoverflowSchema),
- }
-}
-
-fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {
- let last_updated = webpage.updated_time.map(prettify_date);
-
- let url = Url::from(webpage.url.clone());
-
- if url.domain() == "stackoverflow.com"
- && webpage
- .schema_org
- .iter()
- .any(|item| item.types_contains("QAPage"))
- {
- if let Ok(snippet) = stackoverflow_snippet(webpage) {
- return snippet;
- }
- }
-
- Snippet::Normal {
- date: last_updated,
- text: webpage.snippet.clone(),
- }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct DisplayedWebpage {
- pub title: String,
- pub url: String,
- pub site: String,
- pub favicon_base64: String,
- pub domain: String,
- pub pretty_url: String,
- pub snippet: Snippet,
- pub body: String,
- pub primary_image_uuid: Option<String>,
-}
-
-impl From<RetrievedWebpage> for DisplayedWebpage {
- fn from(webpage: RetrievedWebpage) -> Self {
- let snippet = generate_snippet(&webpage);
-
- let url: Url = webpage.url.clone().into();
- let domain = url.domain().to_string();
- let pretty_url = prettify_url(&url);
-
- let title = html_escape(&webpage.title);
-
- let favicon_bytes = webpage
- .favicon
- .map(|favicon| favicon.as_raw_bytes())
- .unwrap_or_else(|| include_bytes!("../../frontend/dist/images/globe.png").to_vec());
-
- Self {
- title,
- site: url.site().to_string(),
- url: webpage.url,
- pretty_url,
- domain,
- favicon_base64: base64::encode(favicon_bytes),
- snippet,
- body: webpage.body,
- primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),
- }
- }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub enum Sidebar {
- Entity(DisplayedEntity),
- StackOverflow {
- title: String,
- answer: StackOverflowAnswer,
- },
-}
-
-pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Result<Sidebar> {
- if let Some(item) = schema_org
- .into_iter()
- .find(|item| item.types_contains("QAPage"))
- .and_then(|item| item.properties.get("mainEntity").cloned())
- .and_then(|properties| properties.one())
- .and_then(|property| property.try_into_item())
- {
- let title = item
- .properties
- .get("name")
- .cloned()
- .and_then(|prop| prop.one())
- .and_then(|prop| prop.try_into_string())
- .ok_or(Error::InvalidStackoverflowSchema)?;
-
- item.properties
- .get("acceptedAnswer")
- .cloned()
- .and_then(|ans| ans.one())
- .and_then(|prop| prop.try_into_item())
- .and_then(|item| {
- schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)
- })
- .map(|answer| Sidebar::StackOverflow { title, answer })
- .ok_or(Error::InvalidStackoverflowSchema)
- } else {
- Err(Error::InvalidStackoverflowSchema)
- }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct DisplayedEntity {
- pub title: String,
- pub small_abstract: String,
- pub image_base64: Option<String>,
- pub related_entities: Vec<DisplayedEntity>,
- pub info: Vec<(String, String)>,
-}
-
-fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {
- let mut info: Vec<_> = info.into_iter().collect();
-
- info.sort_by(|(a, _), (b, _)| {
- searcher
- .attribute_occurrence(b)
- .unwrap_or(0)
- .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))
- });
-
- info.into_iter()
- .map(|(key, value)| {
- let mut value = entity_link_to_html(value, 150).replace('*', "•");
-
- if value.starts_with('•') || value.starts_with("\n•") {
- if let Some(first_bullet) = value.find('•') {
- value = value.chars().skip(first_bullet + 1).collect();
- }
- }
-
- let value = maybe_prettify_entity_date(value);
-
- (key.replace('_', " "), value)
- })
- .filter(|(key, _)| {
- !matches!(
- key.as_str(),
- "caption"
- | "image size"
- | "label"
- | "landscape"
- | "signature"
- | "name"
- | "website"
- | "logo"
- | "image caption"
- | "alt"
- )
- })
- .take(5)
- .collect()
-}
-
-impl DisplayedEntity {
- pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {
- let entity_abstract = Span {
- text: entity.entity_abstract,
- links: entity.links,
- };
-
- let small_abstract = entity_link_to_html(entity_abstract, 300);
-
- let image_base64 = entity
- .image
- .and_then(|image| searcher.entity_image(image))
- .map(|image| base64::encode(image.as_raw_bytes()));
-
- Self {
- title: entity.title,
- small_abstract,
- image_base64,
- related_entities: entity
- .related_entities
- .into_iter()
- .map(|entity| DisplayedEntity::from(entity, searcher))
- .collect(),
- info: prepare_info(entity.info, searcher),
- }
- }
-}
-
-fn maybe_prettify_entity_date(value: String) -> String {
- if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {
- return format!("{}", date.format("%d/%m/%Y"));
- }
-
- if value.split_whitespace().count() == 6 {
- let first_date = NaiveDate::parse_from_str(
- itertools::intersperse(value.split_whitespace().take(3), " ")
- .collect::<String>()
- .as_str(),
- "%Y %-m %-d",
- );
- let second_date = NaiveDate::parse_from_str(
- itertools::intersperse(value.split_whitespace().skip(3), " ")
- .collect::<String>()
- .as_str(),
- "%Y %-m %-d",
- );
-
- if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {
- // the dates are reversed from the parser, so we need to return second_date before first_date
- return format!("{}", second_date.format("%d/%m/%Y"))
- + " - "
- + format!("{}", first_date.format("%d/%m/%Y")).as_str();
- }
- }
-
- value
-}
-
-pub fn thousand_sep_number(num: usize) -> String {
- let s = num.to_string();
- let c = s.chars().rev().chunks(3);
- let chunks = c.into_iter().map(|chunk| {
- chunk
- .into_iter()
- .collect::<Vec<char>>()
- .into_iter()
- .rev()
- .collect::<String>()
- });
-
- intersperse(
- chunks.collect::<Vec<_>>().into_iter().rev(),
- ".".to_string(),
- )
- .collect()
-}
-
-fn entity_link_to_html(span: Span, trunace_to: usize) -> String {
- let mut s = span.text;
-
- let truncated = s.len() > trunace_to;
- if truncated {
- s = s.chars().take(trunace_to).collect();
- }
-
- let chars = s.chars();
- let num_chars = chars.clone().count();
-
- let mut res = String::new();
-
- let mut last_link_end = 0;
- for link in span.links {
- if link.start > num_chars {
- break;
- }
-
- res += chars
- .clone()
- .skip(last_link_end)
- .take(link.start - last_link_end)
- .collect::<String>()
- .as_str();
-
- let link_text: String = chars
- .clone()
- .skip(link.start)
- .take(link.end - link.start)
- .collect();
-
- res += format!(
- "<a href=\"https://en.wikipedia.org/wiki/{}\">",
- link.target.replace(' ', "_")
- )
- .as_str();
-
- res += link_text.as_str();
-
- res += "</a>";
-
- last_link_end = link.end;
- }
-
- res += chars
- .clone()
- .skip(last_link_end)
- .collect::<String>()
- .as_str();
-
- if truncated {
- res += "...";
- }
-
- res
-}
-
-#[cfg(test)]
-mod tests {
- use chrono::{DateTime, Utc};
-
- use crate::entity_index::entity::Link;
-
- use super::*;
-
- #[test]
- fn prettify_date_in_hours() {
- let date = Utc::now().naive_utc();
- assert_eq!(prettify_date(date), "1 hour ago".to_string());
-
- let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();
- assert_eq!(prettify_date(date), "2 hours ago".to_string());
- }
-
- #[test]
- fn prettify_date_days() {
- let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();
- assert_eq!(prettify_date(date), "1 day ago".to_string());
-
- let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();
- assert_eq!(prettify_date(date), "2 days ago".to_string());
- }
-
- #[test]
- fn prettify_date_rest() {
- let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")
- .unwrap()
- .naive_local();
- assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());
- }
-
- #[test]
- fn simple_link_to_html() {
- assert_eq!(
- entity_link_to_html(
- Span {
- text: "some text with a link".to_string(),
- links: vec![Link {
- start: 5,
- end: 9,
- target: "text article".to_string()
- }]
- },
- 10000
- ),
- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"
- .to_string()
- );
- }
-
- #[test]
- fn truncated_link_to_html() {
- assert_eq!(
- entity_link_to_html(
- Span {
- text: "some text".to_string(),
- links: vec![Link {
- start: 5,
- end: 9,
- target: "text article".to_string()
- }]
- },
- 7
- ),
- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()
- );
- }
-
- #[test]
- fn einstein_date() {
- assert_eq!(
- maybe_prettify_entity_date("1879 3 14 ".to_string()),
- "14/03/1879".to_string()
- );
- }
-
- #[test]
- fn entity_date_span_prettify() {
- assert_eq!(
- maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),
- "14/03/1879 - 27/05/1999".to_string()
- );
- }
-
- #[test]
- fn sep_number() {
- assert_eq!(thousand_sep_number(0), "0".to_string());
- assert_eq!(thousand_sep_number(10), "10".to_string());
- assert_eq!(thousand_sep_number(100), "100".to_string());
- assert_eq!(thousand_sep_number(1000), "1.000".to_string());
- assert_eq!(thousand_sep_number(10_000), "10.000".to_string());
- assert_eq!(thousand_sep_number(100_000), "100.000".to_string());
- assert_eq!(thousand_sep_number(512_854), "512.854".to_string());
- assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());
- }
-}
diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rs
new file mode 100644
index 00000000..d0889d56
--- /dev/null
+++ b/core/src/search_prettifier/entity.rs
@@ -0,0 +1,253 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use std::collections::BTreeMap;
+
+use chrono::NaiveDate;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ entity_index::{entity::Span, StoredEntity},
+ searcher::LocalSearcher,
+};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct DisplayedEntity {
+ pub title: String,
+ pub small_abstract: String,
+ pub image_base64: Option<String>,
+ pub related_entities: Vec<DisplayedEntity>,
+ pub info: Vec<(String, String)>,
+}
+
+fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {
+ let mut info: Vec<_> = info.into_iter().collect();
+
+ info.sort_by(|(a, _), (b, _)| {
+ searcher
+ .attribute_occurrence(b)
+ .unwrap_or(0)
+ .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))
+ });
+
+ info.into_iter()
+ .map(|(key, value)| {
+ let mut value = entity_link_to_html(value, 150).replace('*', "•");
+
+ if value.starts_with('•') || value.starts_with("\n•") {
+ if let Some(first_bullet) = value.find('•') {
+ value = value.chars().skip(first_bullet + 1).collect();
+ }
+ }
+
+ let value = maybe_prettify_entity_date(value);
+
+ (key.replace('_', " "), value)
+ })
+ .filter(|(key, _)| {
+ !matches!(
+ key.as_str(),
+ "caption"
+ | "image size"
+ | "label"
+ | "landscape"
+ | "signature"
+ | "name"
+ | "website"
+ | "logo"
+ | "image caption"
+ | "alt"
+ )
+ })
+ .take(5)
+ .collect()
+}
+
+impl DisplayedEntity {
+ pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {
+ let entity_abstract = Span {
+ text: entity.entity_abstract,
+ links: entity.links,
+ };
+
+ let small_abstract = entity_link_to_html(entity_abstract, 300);
+
+ let image_base64 = entity
+ .image
+ .and_then(|image| searcher.entity_image(image))
+ .map(|image| base64::encode(image.as_raw_bytes()));
+
+ Self {
+ title: entity.title,
+ small_abstract,
+ image_base64,
+ related_entities: entity
+ .related_entities
+ .into_iter()
+ .map(|entity| DisplayedEntity::from(entity, searcher))
+ .collect(),
+ info: prepare_info(entity.info, searcher),
+ }
+ }
+}
+
+fn maybe_prettify_entity_date(value: String) -> String {
+ if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {
+ return format!("{}", date.format("%d/%m/%Y"));
+ }
+
+ if value.split_whitespace().count() == 6 {
+ let first_date = NaiveDate::parse_from_str(
+ itertools::intersperse(value.split_whitespace().take(3), " ")
+ .collect::<String>()
+ .as_str(),
+ "%Y %-m %-d",
+ );
+ let second_date = NaiveDate::parse_from_str(
+ itertools::intersperse(value.split_whitespace().skip(3), " ")
+ .collect::<String>()
+ .as_str(),
+ "%Y %-m %-d",
+ );
+
+ if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {
+ // the dates are reversed from the parser, so we need to return second_date before first_date
+ return format!("{}", second_date.format("%d/%m/%Y"))
+ + " - "
+ + format!("{}", first_date.format("%d/%m/%Y")).as_str();
+ }
+ }
+
+ value
+}
+
+fn entity_link_to_html(span: Span, trunace_to: usize) -> String {
+ let mut s = span.text;
+
+ let truncated = s.len() > trunace_to;
+ if truncated {
+ s = s.chars().take(trunace_to).collect();
+ }
+
+ let chars = s.chars();
+ let num_chars = chars.clone().count();
+
+ let mut res = String::new();
+
+ let mut last_link_end = 0;
+ for link in span.links {
+ if link.start > num_chars {
+ break;
+ }
+
+ res += chars
+ .clone()
+ .skip(last_link_end)
+ .take(link.start - last_link_end)
+ .collect::<String>()
+ .as_str();
+
+ let link_text: String = chars
+ .clone()
+ .skip(link.start)
+ .take(link.end - link.start)
+ .collect();
+
+ res += format!(
+ "<a href=\"https://en.wikipedia.org/wiki/{}\">",
+ link.target.replace(' ', "_")
+ )
+ .as_str();
+
+ res += link_text.as_str();
+
+ res += "</a>";
+
+ last_link_end = link.end;
+ }
+
+ res += chars
+ .clone()
+ .skip(last_link_end)
+ .collect::<String>()
+ .as_str();
+
+ if truncated {
+ res += "...";
+ }
+
+ res
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::entity_index::entity::Link;
+
+ use super::*;
+
+ #[test]
+ fn simple_link_to_html() {
+ assert_eq!(
+ entity_link_to_html(
+ Span {
+ text: "some text with a link".to_string(),
+ links: vec![Link {
+ start: 5,
+ end: 9,
+ target: "text article".to_string()
+ }]
+ },
+ 10000
+ ),
+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"
+ .to_string()
+ );
+ }
+
+ #[test]
+ fn truncated_link_to_html() {
+ assert_eq!(
+ entity_link_to_html(
+ Span {
+ text: "some text".to_string(),
+ links: vec![Link {
+ start: 5,
+ end: 9,
+ target: "text article".to_string()
+ }]
+ },
+ 7
+ ),
+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()
+ );
+ }
+
+ #[test]
+ fn einstein_date() {
+ assert_eq!(
+ maybe_prettify_entity_date("1879 3 14 ".to_string()),
+ "14/03/1879".to_string()
+ );
+ }
+
+ #[test]
+ fn entity_date_span_prettify() {
+ assert_eq!(
+ maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),
+ "14/03/1879 - 27/05/1999".to_string()
+ );
+ }
+}
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
new file mode 100644
index 00000000..2c4de6c7
--- /dev/null
+++ b/core/src/search_prettifier/mod.rs
@@ -0,0 +1,263 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+mod entity;
+mod stack_overflow;
+
+use chrono::{NaiveDateTime, Utc};
+use itertools::{intersperse, Itertools};
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ inverted_index::RetrievedWebpage,
+ spell::{self, CorrectionTerm},
+ webpage::Url,
+};
+
+pub use self::stack_overflow::{create_stackoverflow_sidebar, CodeOrText};
+pub use entity::DisplayedEntity;
+
+use self::stack_overflow::{stackoverflow_snippet, StackOverflowAnswer, StackOverflowQuestion};
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Snippet {
+ Normal {
+ date: Option<String>,
+ text: String,
+ },
+ StackOverflowQA {
+ question: StackOverflowQuestion,
+ answers: Vec<StackOverflowAnswer>,
+ },
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct HighlightedSpellCorrection {
+ pub raw: String,
+ pub highlighted: String,
+}
+
+impl From<spell::Correction> for HighlightedSpellCorrection {
+ fn from(correction: spell::Correction) -> Self {
+ let mut highlighted = String::new();
+ let mut raw = String::new();
+
+ for term in correction.terms {
+ match term {
+ CorrectionTerm::Corrected(correction) => {
+ highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));
+ raw.push_str(&correction);
+ }
+ CorrectionTerm::NotCorrected(orig) => {
+ highlighted.push_str(&orig);
+ raw.push_str(&orig);
+ }
+ }
+
+ raw.push(' ');
+ highlighted.push(' ');
+ }
+
+ raw = raw.trim_end().to_string();
+ highlighted = highlighted.trim_end().to_string();
+
+ Self { raw, highlighted }
+ }
+}
+
+pub fn html_escape(s: &str) -> String {
+ html_escape::decode_html_entities(s)
+ .chars()
+ .filter(|c| !matches!(c, '<' | '>' | '&'))
+ .collect()
+}
+
+fn prettify_url(url: &Url) -> String {
+ let mut pretty_url = url.strip_query().to_string();
+
+ if pretty_url.ends_with('/') {
+ pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
+ }
+
+ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
+ pretty_url = Url::from(pretty_url.clone())
+ .strip_protocol()
+ .replace('/', " › ");
+ pretty_url = protocol + &pretty_url;
+
+ pretty_url
+}
+
+fn prettify_date(date: NaiveDateTime) -> String {
+ let current_time = Utc::now().naive_utc();
+ let diff = current_time.signed_duration_since(date);
+
+ let num_hours = diff.num_hours() + 1;
+ if num_hours < 24 {
+ if num_hours <= 1 {
+ return "1 hour ago".to_string();
+ }
+
+ return format!("{num_hours} hours ago");
+ }
+
+ let num_days = diff.num_days();
+ if num_days < 30 {
+ if num_days <= 1 {
+ return "1 day ago".to_string();
+ }
+
+ return format!("{num_days} days ago");
+ }
+
+ format!("{}", date.format("%d. %b. %Y"))
+}
+
+fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {
+ let last_updated = webpage.updated_time.map(prettify_date);
+
+ let url = Url::from(webpage.url.clone());
+
+ if url.domain() == "stackoverflow.com"
+ && webpage
+ .schema_org
+ .iter()
+ .any(|item| item.types_contains("QAPage"))
+ {
+ if let Ok(snippet) = stackoverflow_snippet(webpage) {
+ return snippet;
+ }
+ }
+
+ Snippet::Normal {
+ date: last_updated,
+ text: webpage.snippet.clone(),
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct DisplayedWebpage {
+ pub title: String,
+ pub url: String,
+ pub site: String,
+ pub favicon_base64: String,
+ pub domain: String,
+ pub pretty_url: String,
+ pub snippet: Snippet,
+ pub body: String,
+ pub primary_image_uuid: Option<String>,
+}
+
+impl From<RetrievedWebpage> for DisplayedWebpage {
+ fn from(webpage: RetrievedWebpage) -> Self {
+ let snippet = generate_snippet(&webpage);
+
+ let url: Url = webpage.url.clone().into();
+ let domain = url.domain().to_string();
+ let pretty_url = prettify_url(&url);
+
+ let title = html_escape(&webpage.title);
+
+ let favicon_bytes = webpage
+ .favicon
+ .map(|favicon| favicon.as_raw_bytes())
+ .unwrap_or_else(|| include_bytes!("../../../frontend/dist/images/globe.png").to_vec());
+
+ Self {
+ title,
+ site: url.site().to_string(),
+ url: webpage.url,
+ pretty_url,
+ domain,
+ favicon_base64: base64::encode(favicon_bytes),
+ snippet,
+ body: webpage.body,
+ primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),
+ }
+ }
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum Sidebar {
+ Entity(DisplayedEntity),
+ StackOverflow {
+ title: String,
+ answer: StackOverflowAnswer,
+ },
+}
+
+pub fn thousand_sep_number(num: usize) -> String {
+ let s = num.to_string();
+ let c = s.chars().rev().chunks(3);
+ let chunks = c.into_iter().map(|chunk| {
+ chunk
+ .into_iter()
+ .collect::<Vec<char>>()
+ .into_iter()
+ .rev()
+ .collect::<String>()
+ });
+
+ intersperse(
+ chunks.collect::<Vec<_>>().into_iter().rev(),
+ ".".to_string(),
+ )
+ .collect()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use chrono::{DateTime, Utc};
+
+ #[test]
+ fn prettify_date_in_hours() {
+ let date = Utc::now().naive_utc();
+ assert_eq!(prettify_date(date), "1 hour ago".to_string());
+
+ let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();
+ assert_eq!(prettify_date(date), "2 hours ago".to_string());
+ }
+
+ #[test]
+ fn prettify_date_days() {
+ let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();
+ assert_eq!(prettify_date(date), "1 day ago".to_string());
+
+ let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();
+ assert_eq!(prettify_date(date), "2 days ago".to_string());
+ }
+
+ #[test]
+ fn prettify_date_rest() {
+ let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")
+ .unwrap()
+ .naive_local();
+ assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());
+ }
+
+ #[test]
+ fn sep_number() {
+ assert_eq!(thousand_sep_number(0), "0".to_string());
+ assert_eq!(thousand_sep_number(10), "10".to_string());
+ assert_eq!(thousand_sep_number(100), "100".to_string());
+ assert_eq!(thousand_sep_number(1000), "1.000".to_string());
+ assert_eq!(thousand_sep_number(10_000), "10.000".to_string());
+ assert_eq!(thousand_sep_number(100_000), "100.000".to_string());
+ assert_eq!(thousand_sep_number(512_854), "512.854".to_string());
+ assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());
+ }
+}
diff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rs
new file mode 100644
index 00000000..ec291f21
--- /dev/null
+++ b/core/src/search_prettifier/stack_overflow.rs
@@ -0,0 +1,225 @@
+// Cuely is an open source web search engine.
+// Copyright (C) 2022 Cuely ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+use serde::{Deserialize, Serialize};
+
+use crate::{
+ inverted_index::RetrievedWebpage,
+ webpage::{
+ schema_org::{self, Item, OneOrMany, Property},
+ Url,
+ },
+ Error,
+};
+
+use super::{Sidebar, Snippet};
+use crate::Result;
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct StackOverflowAnswer {
+ pub body: Vec<CodeOrText>,
+ pub date: String,
+ pub url: String,
+ pub upvotes: u32,
+ pub accepted: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct StackOverflowQuestion {
+ pub body: Vec<CodeOrText>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub enum CodeOrText {
+ Code(String),
+ Text(String),
+}
+
+fn parse_so_answer(
+ text: OneOrMany<Property>,
+ date: OneOrMany<Property>,
+ upvotes: OneOrMany<Property>,
+ url: OneOrMany<Property>,
+ webpage_url: Url,
+ accepted: bool,
+) -> Option<StackOverflowAnswer> {
+ let text: Vec<_> = text
+ .many()
+ .into_iter()
+ .map(|prop| match prop {
+ Property::String(s) => CodeOrText::Text(s),
+ Property::Item(item) => CodeOrText::Code(
+ item.properties
+ .get("text")
+ .and_then(|p| p.clone().one())
+ .and_then(|prop| prop.try_into_string())
+ .unwrap_or_default(),
+ ),
+ })
+ .collect();
+
+ let date = chrono::NaiveDateTime::parse_from_str(
+ date.one()
+ .and_then(|prop| prop.try_into_string())
+ .unwrap_or_default()
+ .as_str(),
+ "%Y-%m-%dT%H:%M:%S",
+ )
+ .ok()?;
+
+ let upvotes = upvotes
+ .one()
+ .and_then(|prop| prop.try_into_string())
+ .and_then(|s| s.parse().ok())?;
+
+ let url = url
+ .one()
+ .and_then(|prop| prop.try_into_string())
+ .map(Url::from)
+ .map(|mut url| {
+ url.prefix_with(&Url::from(webpage_url.site()));
+ url
+ })?;
+
+ Some(StackOverflowAnswer {
+ body: text,
+ date: format!("{}", date.date().format("%b %d, %Y")),
+ upvotes,
+ url: url.full(),
+ accepted,
+ })
+}
+
+fn schema_item_to_stackoverflow_answer(
+ item: schema_org::Item,
+ url: Url,
+ accepted: bool,
+) -> Option<StackOverflowAnswer> {
+ match (
+ item.properties.get("text"),
+ item.properties.get("dateCreated"),
+ item.properties.get("upvoteCount"),
+ item.properties.get("url"),
+ ) {
+ (Some(text), Some(date), Some(upvotes), Some(answer_url)) => parse_so_answer(
+ text.clone(),
+ date.clone(),
+ upvotes.clone(),
+ answer_url.clone(),
+ url,
+ accepted,
+ ),
+ _ => None,
+ }
+}
+
+pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
+ match webpage
+ .schema_org
+ .iter()
+ .find(|item| item.types_contains("QAPage"))
+ .and_then(|item| item.properties.get("mainEntity"))
+ .and_then(|properties| properties.clone().one())
+ .and_then(|property| property.try_into_item())
+ {
+ Some(item) => {
+ let question: Vec<CodeOrText> = item
+ .properties
+ .get("text")
+ .map(|item| item.clone().many())
+ .unwrap_or_default()
+ .into_iter()
+ .map(|prop| match prop {
+ Property::String(s) => CodeOrText::Text(s),
+ Property::Item(item) => CodeOrText::Code(
+ item.properties
+ .get("text")
+ .and_then(|p| p.clone().one())
+ .and_then(|prop| prop.try_into_string())
+ .unwrap_or_default(),
+ ),
+ })
+ .collect();
+
+ let mut answers = Vec::new();
+
+ if let Some(ans) = item
+ .properties
+ .get("acceptedAnswer")
+ .cloned()
+ .and_then(|ans| ans.one())
+ .and_then(|prop| prop.try_into_item())
+ .and_then(|item| {
+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), true)
+ })
+ {
+ answers.push(ans);
+ }
+
+ for answer in item
+ .properties
+ .get("suggestedAnswer")
+ .cloned()
+ .map(|answers| answers.many())
+ .unwrap_or_default()
+ .into_iter()
+ .filter_map(|prop| prop.try_into_item())
+ .filter_map(|item| {
+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), false)
+ })
+ {
+ answers.push(answer);
+ }
+
+ Ok(Snippet::StackOverflowQA {
+ question: StackOverflowQuestion { body: question },
+ answers: answers.into_iter().take(3).collect(),
+ })
+ }
+ None => Err(Error::InvalidStackoverflowSchema),
+ }
+}
+
+pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Result<Sidebar> {
+ if let Some(item) = schema_org
+ .into_iter()
+ .find(|item| item.types_contains("QAPage"))
+ .and_then(|item| item.properties.get("mainEntity").cloned())
+ .and_then(|properties| properties.one())
+ .and_then(|property| property.try_into_item())
+ {
+ let title = item
+ .properties
+ .get("name")
+ .cloned()
+ .and_then(|prop| prop.one())
+ .and_then(|prop| prop.try_into_string())
+ .ok_or(Error::InvalidStackoverflowSchema)?;
+
+ item.properties
+ .get("acceptedAnswer")
+ .cloned()
+ .and_then(|ans| ans.one())
+ .and_then(|prop| prop.try_into_item())
+ .and_then(|item| {
+ schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)
+ })
+ .map(|answer| Sidebar::StackOverflow { title, answer })
+ .ok_or(Error::InvalidStackoverflowSchema)
+ } else {
+ Err(Error::InvalidStackoverflowSchema)
+ }
+}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 6e61502a..11a2c57d 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -27,7 +27,7 @@ use crate::{
search_prettifier::{
create_stackoverflow_sidebar, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,
},
- searcher::PrettifiedWebsitesResult,
+ searcher::WebsitesResult,
};
use std::{
@@ -407,7 +407,7 @@ impl DistributedSearcher {
return Err(Error::SearchFailed);
}
- Ok(SearchResult::Websites(PrettifiedWebsitesResult {
+ Ok(SearchResult::Websites(WebsitesResult {
spell_corrected_query,
num_hits: num_docs,
webpages: retrieved_webpages,
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 92b5ba28..2abf800f 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -22,7 +22,8 @@ use uuid::Uuid;
use crate::entity_index::{EntityIndex, StoredEntity};
use crate::image_store::Image;
use crate::index::Index;
-use crate::query::Query;
+use crate::query::parser::Term;
+use crate::query::{self, Query};
use crate::ranking::centrality_store::CentralityStore;
use crate::ranking::optics::CreateAggregator;
use crate::ranking::pipeline::RankingWebsite;
@@ -34,7 +35,7 @@ use crate::webpage::region::Region;
use crate::webpage::Url;
use crate::{inverted_index, Error, Result};
-use super::PrettifiedWebsitesResult;
+use super::WebsitesResult;
use super::{InitialWebsiteResult, SearchQuery};
pub struct LocalSearcher {
@@ -72,8 +73,14 @@ impl LocalSearcher {
self.topic_centrality = Some(topic_centrality);
}
- fn parse_query(&self, query: &SearchQuery, optic: Option<&Optic>) -> Result<Query> {
+ fn parse_query(&self, query: &SearchQuery) -> Result<(Query, Optic)> {
+ let optic = match query.optic_program.as_ref() {
+ Some(program) => Some(optics::parse(program)?),
+ None => None,
+ };
+
let mut query_aggregator = optic
+ .as_ref()
.and_then(|optic| {
optic
.pipeline
@@ -83,7 +90,7 @@ impl LocalSearcher {
.unwrap_or_default();
if let (Some(optic), Some(harmonic)) = (
- optic,
+ optic.as_ref(),
self.centrality_store
.as_ref()
.map(|store| &store.online_harmonic),
@@ -100,30 +107,16 @@ impl LocalSearcher {
)?;
parsed_query.set_num_results(query.num_results);
+ parsed_query.set_offset(query.offset);
- if parsed_query.is_empty() {
- Err(Error::EmptyQuery)
- } else {
- Ok(parsed_query)
+ if let Some(region) = query.selected_region {
+ parsed_query.set_region(region);
}
- }
-
- fn search_inverted_index(
- &self,
- query: &SearchQuery,
- de_rank_similar: bool,
- ) -> Result<(Vec<RankingWebsite>, usize)> {
- let optic = match query.optic_program.as_ref() {
- Some(program) => Some(optics::parse(program)?),
- None => None,
- };
-
- let mut parsed_query = self.parse_query(query, optic.as_ref())?;
let mut optics = Vec::new();
- if let Some(optic) = &optic {
- optics.push(optic.clone());
+ if let Some(optic) = optic {
+ optics.push(optic);
}
if let Some(site_rankings) = &query.site_rankings {
@@ -138,34 +131,33 @@ impl LocalSearcher {
parsed_query.set_optic(&optic, &self.index);
- let mut aggregator = optic
- .pipeline
- .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
- .unwrap_or_default();
-
- if let Some(harmonic) = self
- .centrality_store
- .as_ref()
- .map(|store| &store.online_harmonic)
- {
- aggregator
- .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));
+ if parsed_query.is_empty() {
+ Err(Error::EmptyQuery)
+ } else {
+ Ok((parsed_query, optic))
}
+ }
+ fn ranker(
+ &self,
+ query: &Query,
+ de_rank_similar: bool,
+ aggregator: SignalAggregator,
+ ) -> Result<Ranker> {
let mut ranker = Ranker::new(
self.index.region_count.clone(),
aggregator,
self.index.inverted_index.fastfield_cache(),
);
- if let Some(region) = query.selected_region {
- if region != Region::All {
- ranker = ranker.with_region(region);
+ if let Some(region) = query.region() {
+ if *region != Region::All {
+ ranker = ranker.with_region(*region);
}
}
if let Some(topic_centrality) = self.topic_centrality.as_ref() {
- let topic_scorer = topic_centrality.scorer(&parsed_query);
+ let topic_scorer = topic_centrality.scorer(query);
ranker.set_topic_scorer(topic_scorer);
}
@@ -176,7 +168,7 @@ impl LocalSearcher {
.with_max_docs(10_000, self.index.num_segments())
.with_num_results(100);
- let top_host_nodes = self.index.top_nodes(&parsed_query, ranker.collector())?;
+ let top_host_nodes = self.index.top_nodes(query, ranker.collector())?;
if !top_host_nodes.is_empty() {
let harmonic = centrality_store
.online_harmonic
@@ -185,10 +177,34 @@ impl LocalSearcher {
}
}
- ranker = ranker
+ Ok(ranker
.with_max_docs(10_000_000, self.index.num_segments())
- .with_num_results(parsed_query.num_results())
- .with_offset(query.offset);
+ .with_num_results(query.num_results())
+ .with_offset(query.offset()))
+ }
+
+ fn search_inverted_index(
+ &self,
+ query: &SearchQuery,
+ de_rank_similar: bool,
+ ) -> Result<(Vec<RankingWebsite>, usize)> {
+ let (parsed_query, optic) = self.parse_query(query)?;
+
+ let mut aggregator = optic
+ .pipeline
+ .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))
+ .unwrap_or_default();
+
+ if let Some(harmonic) = self
+ .centrality_store
+ .as_ref()
+ .map(|store| &store.online_harmonic)
+ {
+ aggregator
+ .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));
+ }
+
+ let ranker = self.ranker(&parsed_query, de_rank_similar, aggregator)?;
let res = self
.index
@@ -202,9 +218,16 @@ impl LocalSearcher {
Ok((ranking_websites, res.num_websites))
}
- fn spell_correction(&self, query: &SearchQuery) -> Result<Option<Correction>> {
- let parsed_query = self.parse_query(query, None)?;
- Ok(self.index.spell_correction(&parsed_query.simple_terms()))
+ fn spell_correction(&self, query: &SearchQuery) -> Option<Correction> {
+ let terms: Vec<_> = query::parser::parse(&query.query)
+ .into_iter()
+ .filter_map(|term| match *term {
+ Term::Simple(s) => Some(s),
+ _ => None,
+ })
+ .collect();
+
+ self.index.spell_correction(&terms)
}
fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {
@@ -219,7 +242,7 @@ impl LocalSearcher {
de_rank_similar: bool,
) -> Result<InitialWebsiteResult> {
let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;
- let correction = self.spell_correction(query)?;
+ let correction = self.spell_correction(query);
let sidebar = self
.entity_sidebar(query)
.map(|entity| DisplayedEntity::from(entity, self));
@@ -251,7 +274,8 @@ impl LocalSearcher {
self.index.retrieve_websites(websites, &query)
}
- pub fn search(&self, query: &SearchQuery) -> Result<PrettifiedWebsitesResult> {
+ /// This function is mainly used for tests and benchmarks
+ pub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {
use std::{sync::Arc, time::Instant};
use crate::{
@@ -282,7 +306,7 @@ impl LocalSearcher {
let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;
- Ok(PrettifiedWebsitesResult {
+ Ok(WebsitesResult {
spell_corrected_query: search_result
.spell_corrected_query
.map(HighlightedSpellCorrection::from),
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index 4b3dfb2a..355bff52 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -34,12 +34,12 @@ pub const NUM_RESULTS_PER_PAGE: usize = 20;
#[derive(Debug, Serialize)]
pub enum SearchResult {
- Websites(PrettifiedWebsitesResult),
+ Websites(WebsitesResult),
Bang(BangHit),
}
#[derive(Debug, Serialize)]
-pub struct PrettifiedWebsitesResult {
+pub struct WebsitesResult {
pub spell_corrected_query: Option<HighlightedSpellCorrection>,
pub webpages: Vec<DisplayedWebpage>,
pub num_hits: usize,
@@ -67,6 +67,9 @@ pub struct InitialWebsiteResult {
impl Default for SearchQuery {
fn default() -> Self {
+ // This does not use `..Default::default()` as there should be
+ // an explicit compile error when new fields are added to the `SearchQuery` struct
+ // to ensure the developer considers what the default should be.
Self {
query: Default::default(),
offset: Default::default(),
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 1eab9a9f..3aab829f 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -540,7 +540,7 @@ impl<S: Store> Webgraph<S> {
}
self.segments
- .sort_by_key(|b| std::cmp::Reverse(b.num_nodes()));
+ .sort_by_key(|segment| std::cmp::Reverse(segment.num_nodes()));
let mut candidates = Vec::with_capacity(num_segments);