web search done right
refactor
Mikkel Denker5d20184b88bd
diff --git a/core/src/frontend/search.rs b/core/src/frontend/search.rsindex d8a11550..bf156f6c 100644--- a/core/src/frontend/search.rs+++ b/core/src/frontend/search.rs@@ -14,6 +14,7 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.+use http::Uri;use optics::SiteRankings;use std::{collections::HashMap, sync::Arc};@@ -61,6 +62,27 @@ enum RegionSelection {Unselected(Region),}+fn extract_site_rankings(params: &HashMap<String, String>) -> Option<SiteRankings> {+ match params.get("sr") {+ Some(sr) => {+ if !sr.is_empty() {+ if let Ok(site_rankings) = base64::decode(sr) {+ if let Ok(site_rankings) = std::str::from_utf8(&site_rankings) {+ serde_json::from_str(site_rankings).ok()+ } else {+ None+ }+ } else {+ None+ }+ } else {+ None+ }+ }+ None => None,+ }+}+#[allow(clippy::unused_async)]#[allow(clippy::match_wild_err_arm)]#[debug_handler]@@ -78,6 +100,7 @@ pub async fn route(let offset = skip_pages * NUM_RESULTS_PER_PAGE;let mut optic = None;+ // should only be set if optic was successfully downloadedlet mut current_optic_url = None;if let Some(url) = params.get("optic") {@@ -99,24 +122,7 @@ pub async fn route(}});- let site_rankings: Option<SiteRankings> = match params.get("sr") {- Some(sr) => {- if !sr.is_empty() {- if let Ok(site_rankings) = base64::decode(sr) {- if let Ok(site_rankings) = std::str::from_utf8(&site_rankings) {- serde_json::from_str(site_rankings).ok()- } else {- None- }- } else {- None- }- } else {- None- }- }- None => None,- };+ let site_rankings = extract_site_rankings(¶ms);match state.searcher@@ -140,45 +146,12 @@ pub async fn route(let search_duration_sec =format!("{:.2}", result.search_duration_ms as f64 / 1000.0);- let all_regions = ALL_REGIONS- .into_iter()- .map(|region| {- if let Some(selected_region) = selected_region {- if region == selected_region {- RegionSelection::Selected(region)- } else {- RegionSelection::Unselected(region)- }- } else {- RegionSelection::Unselected(region)- }- })- .collect();+ let all_regions = generate_regions(selected_region);+ let next_page_url = next_page_url(&uri, params.clone(), skip_pages);+ let prev_page_url = prev_page_url(&uri, params, skip_pages);let current_page = skip_pages + 1;- let mut next_page_params = params.clone();- next_page_params.insert("p".to_string(), (skip_pages + 1).to_string());- let next_page_url = uri.path().to_string()- + "?"- + serde_urlencoded::to_string(&next_page_params)- .unwrap()- .as_str();-- let prev_page_url = if current_page > 1 {- let mut prev_page_params = params;- prev_page_params.insert("p".to_string(), (skip_pages - 1).to_string());- Some(- uri.path().to_string()- + "?"- + serde_urlencoded::to_string(&prev_page_params)- .unwrap()- .as_str(),- )- } else {- None- };-let template = SearchTemplate {search_result: result.webpages,query,@@ -203,6 +176,51 @@ pub async fn route(}}+fn generate_regions(selected_region: Option<Region>) -> Vec<RegionSelection> {+ ALL_REGIONS+ .into_iter()+ .map(|region| {+ if let Some(selected_region) = selected_region {+ if region == selected_region {+ RegionSelection::Selected(region)+ } else {+ RegionSelection::Unselected(region)+ }+ } else {+ RegionSelection::Unselected(region)+ }+ })+ .collect()+}++fn prev_page_url(uri: &Uri, params: HashMap<String, String>, skip_pages: usize) -> Option<String> {+ if skip_pages > 0 {+ let mut prev_page_params = params;+ prev_page_params.insert("p".to_string(), (skip_pages - 1).to_string());+ Some(+ uri.path().to_string()+ + "?"+ + serde_urlencoded::to_string(&prev_page_params)+ .unwrap()+ .as_str(),+ )+ } else {+ None+ }+}++fn next_page_url(uri: &Uri, params: HashMap<String, String>, skip_pages: usize) -> String {+ let mut next_page_params = params;+ next_page_params.insert("p".to_string(), (skip_pages + 1).to_string());+ let next_page_url = uri.path().to_string()+ + "?"+ + serde_urlencoded::to_string(&next_page_params)+ .unwrap()+ .as_str();++ next_page_url+}+#[allow(clippy::unused_async)]#[allow(clippy::match_wild_err_arm)]#[debug_handler]diff --git a/core/src/query/mod.rs b/core/src/query/mod.rsindex ec661909..5f678ef7 100644--- a/core/src/query/mod.rs+++ b/core/src/query/mod.rs@@ -19,6 +19,7 @@ use crate::{ranking::SignalAggregator,schema::{Field, TextField},searcher::NUM_RESULTS_PER_PAGE,+ webpage::region::Region,Result,};use optics::Optic;@@ -48,6 +49,8 @@ pub struct Query {terms: Vec<Box<Term>>,simple_terms_text: Vec<String>,tantivy_query: Box<BooleanQuery>,+ offset: usize,+ region: Option<Region>,num_results: usize,}@@ -156,6 +159,8 @@ impl Query {terms,simple_terms_text,tantivy_query,+ offset: 0,+ region: None,num_results: NUM_RESULTS_PER_PAGE,})}@@ -167,7 +172,7 @@ impl Query {&mut optic.as_multiple_tantivy(&index.schema(), index.inverted_index.fastfield_cache()),);- self.tantivy_query = Box::new(BooleanQuery::new(subqueries))+ self.tantivy_query = Box::new(BooleanQuery::new(subqueries));}pub fn simple_terms(&self) -> Vec<String> {@@ -178,6 +183,14 @@ impl Query {self.num_results = num_results;}+ pub fn set_offset(&mut self, offset: usize) {+ self.offset = offset;+ }++ pub fn set_region(&mut self, region: Region) {+ self.region = Some(region);+ }+pub fn terms(&self) -> &[Box<Term>] {&self.terms}@@ -189,6 +202,14 @@ impl Query {pub fn num_results(&self) -> usize {self.num_results}++ pub fn offset(&self) -> usize {+ self.offset+ }++ pub fn region(&self) -> Option<&Region> {+ self.region.as_ref()+ }}impl tantivy::query::Query for Query {diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rsindex a4ce8595..1ae5f243 100644--- a/core/src/ranking/mod.rs+++ b/core/src/ranking/mod.rs@@ -48,6 +48,7 @@ use crate::{pub use self::signal::*;+#[derive(Clone)]pub struct Ranker {region_count: Arc<RegionCount>,selected_region: Option<Region>,diff --git a/core/src/search_prettifier.rs b/core/src/search_prettifier.rsdeleted file mode 100644index df1e1931..00000000--- a/core/src/search_prettifier.rs+++ /dev/null@@ -1,683 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::collections::BTreeMap;--use chrono::{NaiveDate, NaiveDateTime, Utc};-use itertools::{intersperse, Itertools};-use serde::{Deserialize, Serialize};--use crate::{- entity_index::{entity::Span, StoredEntity},- inverted_index::RetrievedWebpage,- searcher::LocalSearcher,- spell::{self, CorrectionTerm},- webpage::{- schema_org::{self, Item, OneOrMany, Property},- Url,- },- Error, Result,-};--#[derive(Debug, Serialize, Deserialize, Clone)]-pub enum Snippet {- Normal {- date: Option<String>,- text: String,- },- StackOverflowQA {- question: StackOverflowQuestion,- answers: Vec<StackOverflowAnswer>,- },-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct StackOverflowAnswer {- pub body: Vec<CodeOrText>,- pub date: String,- pub url: String,- pub upvotes: u32,- pub accepted: bool,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct StackOverflowQuestion {- pub body: Vec<CodeOrText>,-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub enum CodeOrText {- Code(String),- Text(String),-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct HighlightedSpellCorrection {- pub raw: String,- pub highlighted: String,-}--impl From<spell::Correction> for HighlightedSpellCorrection {- fn from(correction: spell::Correction) -> Self {- let mut highlighted = String::new();- let mut raw = String::new();-- for term in correction.terms {- match term {- CorrectionTerm::Corrected(correction) => {- highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));- raw.push_str(&correction);- }- CorrectionTerm::NotCorrected(orig) => {- highlighted.push_str(&orig);- raw.push_str(&orig);- }- }-- raw.push(' ');- highlighted.push(' ');- }-- raw = raw.trim_end().to_string();- highlighted = highlighted.trim_end().to_string();-- Self { raw, highlighted }- }-}--pub fn html_escape(s: &str) -> String {- html_escape::decode_html_entities(s)- .chars()- .filter(|c| !matches!(c, '<' | '>' | '&'))- .collect()-}--fn prettify_url(url: &Url) -> String {- let mut pretty_url = url.strip_query().to_string();-- if pretty_url.ends_with('/') {- pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();- }-- let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";- pretty_url = Url::from(pretty_url.clone())- .strip_protocol()- .replace('/', " › ");- pretty_url = protocol + &pretty_url;-- pretty_url-}--fn prettify_date(date: NaiveDateTime) -> String {- let current_time = Utc::now().naive_utc();- let diff = current_time.signed_duration_since(date);-- let num_hours = diff.num_hours() + 1;- if num_hours < 24 {- if num_hours <= 1 {- return "1 hour ago".to_string();- }-- return format!("{num_hours} hours ago");- }-- let num_days = diff.num_days();- if num_days < 30 {- if num_days <= 1 {- return "1 day ago".to_string();- }-- return format!("{num_days} days ago");- }-- format!("{}", date.format("%d. %b. %Y"))-}--fn parse_so_answer(- text: OneOrMany<Property>,- date: OneOrMany<Property>,- upvotes: OneOrMany<Property>,- url: OneOrMany<Property>,- webpage_url: Url,- accepted: bool,-) -> Option<StackOverflowAnswer> {- let text: Vec<_> = text- .many()- .into_iter()- .map(|prop| match prop {- Property::String(s) => CodeOrText::Text(s),- Property::Item(item) => CodeOrText::Code(- item.properties- .get("text")- .and_then(|p| p.clone().one())- .and_then(|prop| prop.try_into_string())- .unwrap_or_default(),- ),- })- .collect();-- let date = chrono::NaiveDateTime::parse_from_str(- date.one()- .and_then(|prop| prop.try_into_string())- .unwrap_or_default()- .as_str(),- "%Y-%m-%dT%H:%M:%S",- )- .ok()?;-- let upvotes = upvotes- .one()- .and_then(|prop| prop.try_into_string())- .and_then(|s| s.parse().ok())?;-- let url = url- .one()- .and_then(|prop| prop.try_into_string())- .map(Url::from)- .map(|mut url| {- url.prefix_with(&Url::from(webpage_url.site()));- url- })?;-- Some(StackOverflowAnswer {- body: text,- date: format!("{}", date.date().format("%b %d, %Y")),- upvotes,- url: url.full(),- accepted,- })-}--fn schema_item_to_stackoverflow_answer(- item: schema_org::Item,- url: Url,- accepted: bool,-) -> Option<StackOverflowAnswer> {- match (- item.properties.get("text"),- item.properties.get("dateCreated"),- item.properties.get("upvoteCount"),- item.properties.get("url"),- ) {- (Some(text), Some(date), Some(upvotes), Some(answer_url)) => parse_so_answer(- text.clone(),- date.clone(),- upvotes.clone(),- answer_url.clone(),- url,- accepted,- ),- _ => None,- }-}--fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {- match webpage- .schema_org- .iter()- .find(|item| item.types_contains("QAPage"))- .and_then(|item| item.properties.get("mainEntity"))- .and_then(|properties| properties.clone().one())- .and_then(|property| property.try_into_item())- {- Some(item) => {- let question: Vec<CodeOrText> = item- .properties- .get("text")- .map(|item| item.clone().many())- .unwrap_or_default()- .into_iter()- .map(|prop| match prop {- Property::String(s) => CodeOrText::Text(s),- Property::Item(item) => CodeOrText::Code(- item.properties- .get("text")- .and_then(|p| p.clone().one())- .and_then(|prop| prop.try_into_string())- .unwrap_or_default(),- ),- })- .collect();-- let mut answers = Vec::new();-- if let Some(ans) = item- .properties- .get("acceptedAnswer")- .cloned()- .and_then(|ans| ans.one())- .and_then(|prop| prop.try_into_item())- .and_then(|item| {- schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), true)- })- {- answers.push(ans);- }-- for answer in item- .properties- .get("suggestedAnswer")- .cloned()- .map(|answers| answers.many())- .unwrap_or_default()- .into_iter()- .filter_map(|prop| prop.try_into_item())- .filter_map(|item| {- schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), false)- })- {- answers.push(answer);- }-- Ok(Snippet::StackOverflowQA {- question: StackOverflowQuestion { body: question },- answers: answers.into_iter().take(3).collect(),- })- }- None => Err(Error::InvalidStackoverflowSchema),- }-}--fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {- let last_updated = webpage.updated_time.map(prettify_date);-- let url = Url::from(webpage.url.clone());-- if url.domain() == "stackoverflow.com"- && webpage- .schema_org- .iter()- .any(|item| item.types_contains("QAPage"))- {- if let Ok(snippet) = stackoverflow_snippet(webpage) {- return snippet;- }- }-- Snippet::Normal {- date: last_updated,- text: webpage.snippet.clone(),- }-}--#[derive(Debug, Serialize, Deserialize)]-pub struct DisplayedWebpage {- pub title: String,- pub url: String,- pub site: String,- pub favicon_base64: String,- pub domain: String,- pub pretty_url: String,- pub snippet: Snippet,- pub body: String,- pub primary_image_uuid: Option<String>,-}--impl From<RetrievedWebpage> for DisplayedWebpage {- fn from(webpage: RetrievedWebpage) -> Self {- let snippet = generate_snippet(&webpage);-- let url: Url = webpage.url.clone().into();- let domain = url.domain().to_string();- let pretty_url = prettify_url(&url);-- let title = html_escape(&webpage.title);-- let favicon_bytes = webpage- .favicon- .map(|favicon| favicon.as_raw_bytes())- .unwrap_or_else(|| include_bytes!("../../frontend/dist/images/globe.png").to_vec());-- Self {- title,- site: url.site().to_string(),- url: webpage.url,- pretty_url,- domain,- favicon_base64: base64::encode(favicon_bytes),- snippet,- body: webpage.body,- primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),- }- }-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub enum Sidebar {- Entity(DisplayedEntity),- StackOverflow {- title: String,- answer: StackOverflowAnswer,- },-}--pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Result<Sidebar> {- if let Some(item) = schema_org- .into_iter()- .find(|item| item.types_contains("QAPage"))- .and_then(|item| item.properties.get("mainEntity").cloned())- .and_then(|properties| properties.one())- .and_then(|property| property.try_into_item())- {- let title = item- .properties- .get("name")- .cloned()- .and_then(|prop| prop.one())- .and_then(|prop| prop.try_into_string())- .ok_or(Error::InvalidStackoverflowSchema)?;-- item.properties- .get("acceptedAnswer")- .cloned()- .and_then(|ans| ans.one())- .and_then(|prop| prop.try_into_item())- .and_then(|item| {- schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)- })- .map(|answer| Sidebar::StackOverflow { title, answer })- .ok_or(Error::InvalidStackoverflowSchema)- } else {- Err(Error::InvalidStackoverflowSchema)- }-}--#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct DisplayedEntity {- pub title: String,- pub small_abstract: String,- pub image_base64: Option<String>,- pub related_entities: Vec<DisplayedEntity>,- pub info: Vec<(String, String)>,-}--fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {- let mut info: Vec<_> = info.into_iter().collect();-- info.sort_by(|(a, _), (b, _)| {- searcher- .attribute_occurrence(b)- .unwrap_or(0)- .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))- });-- info.into_iter()- .map(|(key, value)| {- let mut value = entity_link_to_html(value, 150).replace('*', "•");-- if value.starts_with('•') || value.starts_with("\n•") {- if let Some(first_bullet) = value.find('•') {- value = value.chars().skip(first_bullet + 1).collect();- }- }-- let value = maybe_prettify_entity_date(value);-- (key.replace('_', " "), value)- })- .filter(|(key, _)| {- !matches!(- key.as_str(),- "caption"- | "image size"- | "label"- | "landscape"- | "signature"- | "name"- | "website"- | "logo"- | "image caption"- | "alt"- )- })- .take(5)- .collect()-}--impl DisplayedEntity {- pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {- let entity_abstract = Span {- text: entity.entity_abstract,- links: entity.links,- };-- let small_abstract = entity_link_to_html(entity_abstract, 300);-- let image_base64 = entity- .image- .and_then(|image| searcher.entity_image(image))- .map(|image| base64::encode(image.as_raw_bytes()));-- Self {- title: entity.title,- small_abstract,- image_base64,- related_entities: entity- .related_entities- .into_iter()- .map(|entity| DisplayedEntity::from(entity, searcher))- .collect(),- info: prepare_info(entity.info, searcher),- }- }-}--fn maybe_prettify_entity_date(value: String) -> String {- if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {- return format!("{}", date.format("%d/%m/%Y"));- }-- if value.split_whitespace().count() == 6 {- let first_date = NaiveDate::parse_from_str(- itertools::intersperse(value.split_whitespace().take(3), " ")- .collect::<String>()- .as_str(),- "%Y %-m %-d",- );- let second_date = NaiveDate::parse_from_str(- itertools::intersperse(value.split_whitespace().skip(3), " ")- .collect::<String>()- .as_str(),- "%Y %-m %-d",- );-- if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {- // the dates are reversed from the parser, so we need to return second_date before first_date- return format!("{}", second_date.format("%d/%m/%Y"))- + " - "- + format!("{}", first_date.format("%d/%m/%Y")).as_str();- }- }-- value-}--pub fn thousand_sep_number(num: usize) -> String {- let s = num.to_string();- let c = s.chars().rev().chunks(3);- let chunks = c.into_iter().map(|chunk| {- chunk- .into_iter()- .collect::<Vec<char>>()- .into_iter()- .rev()- .collect::<String>()- });-- intersperse(- chunks.collect::<Vec<_>>().into_iter().rev(),- ".".to_string(),- )- .collect()-}--fn entity_link_to_html(span: Span, trunace_to: usize) -> String {- let mut s = span.text;-- let truncated = s.len() > trunace_to;- if truncated {- s = s.chars().take(trunace_to).collect();- }-- let chars = s.chars();- let num_chars = chars.clone().count();-- let mut res = String::new();-- let mut last_link_end = 0;- for link in span.links {- if link.start > num_chars {- break;- }-- res += chars- .clone()- .skip(last_link_end)- .take(link.start - last_link_end)- .collect::<String>()- .as_str();-- let link_text: String = chars- .clone()- .skip(link.start)- .take(link.end - link.start)- .collect();-- res += format!(- "<a href=\"https://en.wikipedia.org/wiki/{}\">",- link.target.replace(' ', "_")- )- .as_str();-- res += link_text.as_str();-- res += "</a>";-- last_link_end = link.end;- }-- res += chars- .clone()- .skip(last_link_end)- .collect::<String>()- .as_str();-- if truncated {- res += "...";- }-- res-}--#[cfg(test)]-mod tests {- use chrono::{DateTime, Utc};-- use crate::entity_index::entity::Link;-- use super::*;-- #[test]- fn prettify_date_in_hours() {- let date = Utc::now().naive_utc();- assert_eq!(prettify_date(date), "1 hour ago".to_string());-- let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();- assert_eq!(prettify_date(date), "2 hours ago".to_string());- }-- #[test]- fn prettify_date_days() {- let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();- assert_eq!(prettify_date(date), "1 day ago".to_string());-- let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();- assert_eq!(prettify_date(date), "2 days ago".to_string());- }-- #[test]- fn prettify_date_rest() {- let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")- .unwrap()- .naive_local();- assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());- }-- #[test]- fn simple_link_to_html() {- assert_eq!(- entity_link_to_html(- Span {- text: "some text with a link".to_string(),- links: vec![Link {- start: 5,- end: 9,- target: "text article".to_string()- }]- },- 10000- ),- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"- .to_string()- );- }-- #[test]- fn truncated_link_to_html() {- assert_eq!(- entity_link_to_html(- Span {- text: "some text".to_string(),- links: vec![Link {- start: 5,- end: 9,- target: "text article".to_string()- }]- },- 7- ),- "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()- );- }-- #[test]- fn einstein_date() {- assert_eq!(- maybe_prettify_entity_date("1879 3 14 ".to_string()),- "14/03/1879".to_string()- );- }-- #[test]- fn entity_date_span_prettify() {- assert_eq!(- maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),- "14/03/1879 - 27/05/1999".to_string()- );- }-- #[test]- fn sep_number() {- assert_eq!(thousand_sep_number(0), "0".to_string());- assert_eq!(thousand_sep_number(10), "10".to_string());- assert_eq!(thousand_sep_number(100), "100".to_string());- assert_eq!(thousand_sep_number(1000), "1.000".to_string());- assert_eq!(thousand_sep_number(10_000), "10.000".to_string());- assert_eq!(thousand_sep_number(100_000), "100.000".to_string());- assert_eq!(thousand_sep_number(512_854), "512.854".to_string());- assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());- }-}diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rsnew file mode 100644index 00000000..d0889d56--- /dev/null+++ b/core/src/search_prettifier/entity.rs@@ -0,0 +1,253 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::collections::BTreeMap;++use chrono::NaiveDate;+use serde::{Deserialize, Serialize};++use crate::{+ entity_index::{entity::Span, StoredEntity},+ searcher::LocalSearcher,+};++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct DisplayedEntity {+ pub title: String,+ pub small_abstract: String,+ pub image_base64: Option<String>,+ pub related_entities: Vec<DisplayedEntity>,+ pub info: Vec<(String, String)>,+}++fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {+ let mut info: Vec<_> = info.into_iter().collect();++ info.sort_by(|(a, _), (b, _)| {+ searcher+ .attribute_occurrence(b)+ .unwrap_or(0)+ .cmp(&searcher.attribute_occurrence(a).unwrap_or(0))+ });++ info.into_iter()+ .map(|(key, value)| {+ let mut value = entity_link_to_html(value, 150).replace('*', "•");++ if value.starts_with('•') || value.starts_with("\n•") {+ if let Some(first_bullet) = value.find('•') {+ value = value.chars().skip(first_bullet + 1).collect();+ }+ }++ let value = maybe_prettify_entity_date(value);++ (key.replace('_', " "), value)+ })+ .filter(|(key, _)| {+ !matches!(+ key.as_str(),+ "caption"+ | "image size"+ | "label"+ | "landscape"+ | "signature"+ | "name"+ | "website"+ | "logo"+ | "image caption"+ | "alt"+ )+ })+ .take(5)+ .collect()+}++impl DisplayedEntity {+ pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {+ let entity_abstract = Span {+ text: entity.entity_abstract,+ links: entity.links,+ };++ let small_abstract = entity_link_to_html(entity_abstract, 300);++ let image_base64 = entity+ .image+ .and_then(|image| searcher.entity_image(image))+ .map(|image| base64::encode(image.as_raw_bytes()));++ Self {+ title: entity.title,+ small_abstract,+ image_base64,+ related_entities: entity+ .related_entities+ .into_iter()+ .map(|entity| DisplayedEntity::from(entity, searcher))+ .collect(),+ info: prepare_info(entity.info, searcher),+ }+ }+}++fn maybe_prettify_entity_date(value: String) -> String {+ if let Ok(date) = NaiveDate::parse_from_str(value.trim(), "%Y %-m %-d") {+ return format!("{}", date.format("%d/%m/%Y"));+ }++ if value.split_whitespace().count() == 6 {+ let first_date = NaiveDate::parse_from_str(+ itertools::intersperse(value.split_whitespace().take(3), " ")+ .collect::<String>()+ .as_str(),+ "%Y %-m %-d",+ );+ let second_date = NaiveDate::parse_from_str(+ itertools::intersperse(value.split_whitespace().skip(3), " ")+ .collect::<String>()+ .as_str(),+ "%Y %-m %-d",+ );++ if let (Ok(first_date), Ok(second_date)) = (first_date, second_date) {+ // the dates are reversed from the parser, so we need to return second_date before first_date+ return format!("{}", second_date.format("%d/%m/%Y"))+ + " - "+ + format!("{}", first_date.format("%d/%m/%Y")).as_str();+ }+ }++ value+}++fn entity_link_to_html(span: Span, trunace_to: usize) -> String {+ let mut s = span.text;++ let truncated = s.len() > trunace_to;+ if truncated {+ s = s.chars().take(trunace_to).collect();+ }++ let chars = s.chars();+ let num_chars = chars.clone().count();++ let mut res = String::new();++ let mut last_link_end = 0;+ for link in span.links {+ if link.start > num_chars {+ break;+ }++ res += chars+ .clone()+ .skip(last_link_end)+ .take(link.start - last_link_end)+ .collect::<String>()+ .as_str();++ let link_text: String = chars+ .clone()+ .skip(link.start)+ .take(link.end - link.start)+ .collect();++ res += format!(+ "<a href=\"https://en.wikipedia.org/wiki/{}\">",+ link.target.replace(' ', "_")+ )+ .as_str();++ res += link_text.as_str();++ res += "</a>";++ last_link_end = link.end;+ }++ res += chars+ .clone()+ .skip(last_link_end)+ .collect::<String>()+ .as_str();++ if truncated {+ res += "...";+ }++ res+}++#[cfg(test)]+mod tests {+ use crate::entity_index::entity::Link;++ use super::*;++ #[test]+ fn simple_link_to_html() {+ assert_eq!(+ entity_link_to_html(+ Span {+ text: "some text with a link".to_string(),+ links: vec![Link {+ start: 5,+ end: 9,+ target: "text article".to_string()+ }]+ },+ 10000+ ),+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">text</a> with a link"+ .to_string()+ );+ }++ #[test]+ fn truncated_link_to_html() {+ assert_eq!(+ entity_link_to_html(+ Span {+ text: "some text".to_string(),+ links: vec![Link {+ start: 5,+ end: 9,+ target: "text article".to_string()+ }]+ },+ 7+ ),+ "some <a href=\"https://en.wikipedia.org/wiki/text_article\">te</a>...".to_string()+ );+ }++ #[test]+ fn einstein_date() {+ assert_eq!(+ maybe_prettify_entity_date("1879 3 14 ".to_string()),+ "14/03/1879".to_string()+ );+ }++ #[test]+ fn entity_date_span_prettify() {+ assert_eq!(+ maybe_prettify_entity_date(" 1999 5 27 1879 3 14 ".to_string()),+ "14/03/1879 - 27/05/1999".to_string()+ );+ }+}diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rsnew file mode 100644index 00000000..2c4de6c7--- /dev/null+++ b/core/src/search_prettifier/mod.rs@@ -0,0 +1,263 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++mod entity;+mod stack_overflow;++use chrono::{NaiveDateTime, Utc};+use itertools::{intersperse, Itertools};+use serde::{Deserialize, Serialize};++use crate::{+ inverted_index::RetrievedWebpage,+ spell::{self, CorrectionTerm},+ webpage::Url,+};++pub use self::stack_overflow::{create_stackoverflow_sidebar, CodeOrText};+pub use entity::DisplayedEntity;++use self::stack_overflow::{stackoverflow_snippet, StackOverflowAnswer, StackOverflowQuestion};++#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum Snippet {+ Normal {+ date: Option<String>,+ text: String,+ },+ StackOverflowQA {+ question: StackOverflowQuestion,+ answers: Vec<StackOverflowAnswer>,+ },+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct HighlightedSpellCorrection {+ pub raw: String,+ pub highlighted: String,+}++impl From<spell::Correction> for HighlightedSpellCorrection {+ fn from(correction: spell::Correction) -> Self {+ let mut highlighted = String::new();+ let mut raw = String::new();++ for term in correction.terms {+ match term {+ CorrectionTerm::Corrected(correction) => {+ highlighted.push_str(&("<b><i>".to_string() + &correction + "</i></b>"));+ raw.push_str(&correction);+ }+ CorrectionTerm::NotCorrected(orig) => {+ highlighted.push_str(&orig);+ raw.push_str(&orig);+ }+ }++ raw.push(' ');+ highlighted.push(' ');+ }++ raw = raw.trim_end().to_string();+ highlighted = highlighted.trim_end().to_string();++ Self { raw, highlighted }+ }+}++pub fn html_escape(s: &str) -> String {+ html_escape::decode_html_entities(s)+ .chars()+ .filter(|c| !matches!(c, '<' | '>' | '&'))+ .collect()+}++fn prettify_url(url: &Url) -> String {+ let mut pretty_url = url.strip_query().to_string();++ if pretty_url.ends_with('/') {+ pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();+ }++ let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";+ pretty_url = Url::from(pretty_url.clone())+ .strip_protocol()+ .replace('/', " › ");+ pretty_url = protocol + &pretty_url;++ pretty_url+}++fn prettify_date(date: NaiveDateTime) -> String {+ let current_time = Utc::now().naive_utc();+ let diff = current_time.signed_duration_since(date);++ let num_hours = diff.num_hours() + 1;+ if num_hours < 24 {+ if num_hours <= 1 {+ return "1 hour ago".to_string();+ }++ return format!("{num_hours} hours ago");+ }++ let num_days = diff.num_days();+ if num_days < 30 {+ if num_days <= 1 {+ return "1 day ago".to_string();+ }++ return format!("{num_days} days ago");+ }++ format!("{}", date.format("%d. %b. %Y"))+}++fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {+ let last_updated = webpage.updated_time.map(prettify_date);++ let url = Url::from(webpage.url.clone());++ if url.domain() == "stackoverflow.com"+ && webpage+ .schema_org+ .iter()+ .any(|item| item.types_contains("QAPage"))+ {+ if let Ok(snippet) = stackoverflow_snippet(webpage) {+ return snippet;+ }+ }++ Snippet::Normal {+ date: last_updated,+ text: webpage.snippet.clone(),+ }+}++#[derive(Debug, Serialize, Deserialize)]+pub struct DisplayedWebpage {+ pub title: String,+ pub url: String,+ pub site: String,+ pub favicon_base64: String,+ pub domain: String,+ pub pretty_url: String,+ pub snippet: Snippet,+ pub body: String,+ pub primary_image_uuid: Option<String>,+}++impl From<RetrievedWebpage> for DisplayedWebpage {+ fn from(webpage: RetrievedWebpage) -> Self {+ let snippet = generate_snippet(&webpage);++ let url: Url = webpage.url.clone().into();+ let domain = url.domain().to_string();+ let pretty_url = prettify_url(&url);++ let title = html_escape(&webpage.title);++ let favicon_bytes = webpage+ .favicon+ .map(|favicon| favicon.as_raw_bytes())+ .unwrap_or_else(|| include_bytes!("../../../frontend/dist/images/globe.png").to_vec());++ Self {+ title,+ site: url.site().to_string(),+ url: webpage.url,+ pretty_url,+ domain,+ favicon_base64: base64::encode(favicon_bytes),+ snippet,+ body: webpage.body,+ primary_image_uuid: webpage.primary_image.map(|image| image.uuid.to_string()),+ }+ }+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum Sidebar {+ Entity(DisplayedEntity),+ StackOverflow {+ title: String,+ answer: StackOverflowAnswer,+ },+}++pub fn thousand_sep_number(num: usize) -> String {+ let s = num.to_string();+ let c = s.chars().rev().chunks(3);+ let chunks = c.into_iter().map(|chunk| {+ chunk+ .into_iter()+ .collect::<Vec<char>>()+ .into_iter()+ .rev()+ .collect::<String>()+ });++ intersperse(+ chunks.collect::<Vec<_>>().into_iter().rev(),+ ".".to_string(),+ )+ .collect()+}++#[cfg(test)]+mod tests {+ use super::*;+ use chrono::{DateTime, Utc};++ #[test]+ fn prettify_date_in_hours() {+ let date = Utc::now().naive_utc();+ assert_eq!(prettify_date(date), "1 hour ago".to_string());++ let date = (Utc::now() - chrono::Duration::seconds(4000)).naive_utc();+ assert_eq!(prettify_date(date), "2 hours ago".to_string());+ }++ #[test]+ fn prettify_date_days() {+ let date = (Utc::now() - chrono::Duration::days(1)).naive_utc();+ assert_eq!(prettify_date(date), "1 day ago".to_string());++ let date = (Utc::now() - chrono::Duration::days(2)).naive_utc();+ assert_eq!(prettify_date(date), "2 days ago".to_string());+ }++ #[test]+ fn prettify_date_rest() {+ let date = DateTime::parse_from_rfc3339("1996-12-19T16:39:57-08:00")+ .unwrap()+ .naive_local();+ assert_eq!(prettify_date(date), "19. Dec. 1996".to_string());+ }++ #[test]+ fn sep_number() {+ assert_eq!(thousand_sep_number(0), "0".to_string());+ assert_eq!(thousand_sep_number(10), "10".to_string());+ assert_eq!(thousand_sep_number(100), "100".to_string());+ assert_eq!(thousand_sep_number(1000), "1.000".to_string());+ assert_eq!(thousand_sep_number(10_000), "10.000".to_string());+ assert_eq!(thousand_sep_number(100_000), "100.000".to_string());+ assert_eq!(thousand_sep_number(512_854), "512.854".to_string());+ assert_eq!(thousand_sep_number(9_512_854), "9.512.854".to_string());+ }+}diff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rsnew file mode 100644index 00000000..ec291f21--- /dev/null+++ b/core/src/search_prettifier/stack_overflow.rs@@ -0,0 +1,225 @@+// Cuely is an open source web search engine.+// Copyright (C) 2022 Cuely ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use serde::{Deserialize, Serialize};++use crate::{+ inverted_index::RetrievedWebpage,+ webpage::{+ schema_org::{self, Item, OneOrMany, Property},+ Url,+ },+ Error,+};++use super::{Sidebar, Snippet};+use crate::Result;++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct StackOverflowAnswer {+ pub body: Vec<CodeOrText>,+ pub date: String,+ pub url: String,+ pub upvotes: u32,+ pub accepted: bool,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct StackOverflowQuestion {+ pub body: Vec<CodeOrText>,+}++#[derive(Debug, Serialize, Deserialize, Clone)]+pub enum CodeOrText {+ Code(String),+ Text(String),+}++fn parse_so_answer(+ text: OneOrMany<Property>,+ date: OneOrMany<Property>,+ upvotes: OneOrMany<Property>,+ url: OneOrMany<Property>,+ webpage_url: Url,+ accepted: bool,+) -> Option<StackOverflowAnswer> {+ let text: Vec<_> = text+ .many()+ .into_iter()+ .map(|prop| match prop {+ Property::String(s) => CodeOrText::Text(s),+ Property::Item(item) => CodeOrText::Code(+ item.properties+ .get("text")+ .and_then(|p| p.clone().one())+ .and_then(|prop| prop.try_into_string())+ .unwrap_or_default(),+ ),+ })+ .collect();++ let date = chrono::NaiveDateTime::parse_from_str(+ date.one()+ .and_then(|prop| prop.try_into_string())+ .unwrap_or_default()+ .as_str(),+ "%Y-%m-%dT%H:%M:%S",+ )+ .ok()?;++ let upvotes = upvotes+ .one()+ .and_then(|prop| prop.try_into_string())+ .and_then(|s| s.parse().ok())?;++ let url = url+ .one()+ .and_then(|prop| prop.try_into_string())+ .map(Url::from)+ .map(|mut url| {+ url.prefix_with(&Url::from(webpage_url.site()));+ url+ })?;++ Some(StackOverflowAnswer {+ body: text,+ date: format!("{}", date.date().format("%b %d, %Y")),+ upvotes,+ url: url.full(),+ accepted,+ })+}++fn schema_item_to_stackoverflow_answer(+ item: schema_org::Item,+ url: Url,+ accepted: bool,+) -> Option<StackOverflowAnswer> {+ match (+ item.properties.get("text"),+ item.properties.get("dateCreated"),+ item.properties.get("upvoteCount"),+ item.properties.get("url"),+ ) {+ (Some(text), Some(date), Some(upvotes), Some(answer_url)) => parse_so_answer(+ text.clone(),+ date.clone(),+ upvotes.clone(),+ answer_url.clone(),+ url,+ accepted,+ ),+ _ => None,+ }+}++pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {+ match webpage+ .schema_org+ .iter()+ .find(|item| item.types_contains("QAPage"))+ .and_then(|item| item.properties.get("mainEntity"))+ .and_then(|properties| properties.clone().one())+ .and_then(|property| property.try_into_item())+ {+ Some(item) => {+ let question: Vec<CodeOrText> = item+ .properties+ .get("text")+ .map(|item| item.clone().many())+ .unwrap_or_default()+ .into_iter()+ .map(|prop| match prop {+ Property::String(s) => CodeOrText::Text(s),+ Property::Item(item) => CodeOrText::Code(+ item.properties+ .get("text")+ .and_then(|p| p.clone().one())+ .and_then(|prop| prop.try_into_string())+ .unwrap_or_default(),+ ),+ })+ .collect();++ let mut answers = Vec::new();++ if let Some(ans) = item+ .properties+ .get("acceptedAnswer")+ .cloned()+ .and_then(|ans| ans.one())+ .and_then(|prop| prop.try_into_item())+ .and_then(|item| {+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), true)+ })+ {+ answers.push(ans);+ }++ for answer in item+ .properties+ .get("suggestedAnswer")+ .cloned()+ .map(|answers| answers.many())+ .unwrap_or_default()+ .into_iter()+ .filter_map(|prop| prop.try_into_item())+ .filter_map(|item| {+ schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), false)+ })+ {+ answers.push(answer);+ }++ Ok(Snippet::StackOverflowQA {+ question: StackOverflowQuestion { body: question },+ answers: answers.into_iter().take(3).collect(),+ })+ }+ None => Err(Error::InvalidStackoverflowSchema),+ }+}++pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Result<Sidebar> {+ if let Some(item) = schema_org+ .into_iter()+ .find(|item| item.types_contains("QAPage"))+ .and_then(|item| item.properties.get("mainEntity").cloned())+ .and_then(|properties| properties.one())+ .and_then(|property| property.try_into_item())+ {+ let title = item+ .properties+ .get("name")+ .cloned()+ .and_then(|prop| prop.one())+ .and_then(|prop| prop.try_into_string())+ .ok_or(Error::InvalidStackoverflowSchema)?;++ item.properties+ .get("acceptedAnswer")+ .cloned()+ .and_then(|ans| ans.one())+ .and_then(|prop| prop.try_into_item())+ .and_then(|item| {+ schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)+ })+ .map(|answer| Sidebar::StackOverflow { title, answer })+ .ok_or(Error::InvalidStackoverflowSchema)+ } else {+ Err(Error::InvalidStackoverflowSchema)+ }+}diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rsindex 6e61502a..11a2c57d 100644--- a/core/src/searcher/distributed.rs+++ b/core/src/searcher/distributed.rs@@ -27,7 +27,7 @@ use crate::{search_prettifier::{create_stackoverflow_sidebar, DisplayedWebpage, HighlightedSpellCorrection, Sidebar,},- searcher::PrettifiedWebsitesResult,+ searcher::WebsitesResult,};use std::{@@ -407,7 +407,7 @@ impl DistributedSearcher {return Err(Error::SearchFailed);}- Ok(SearchResult::Websites(PrettifiedWebsitesResult {+ Ok(SearchResult::Websites(WebsitesResult {spell_corrected_query,num_hits: num_docs,webpages: retrieved_webpages,diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex 92b5ba28..2abf800f 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -22,7 +22,8 @@ use uuid::Uuid;use crate::entity_index::{EntityIndex, StoredEntity};use crate::image_store::Image;use crate::index::Index;-use crate::query::Query;+use crate::query::parser::Term;+use crate::query::{self, Query};use crate::ranking::centrality_store::CentralityStore;use crate::ranking::optics::CreateAggregator;use crate::ranking::pipeline::RankingWebsite;@@ -34,7 +35,7 @@ use crate::webpage::region::Region;use crate::webpage::Url;use crate::{inverted_index, Error, Result};-use super::PrettifiedWebsitesResult;+use super::WebsitesResult;use super::{InitialWebsiteResult, SearchQuery};pub struct LocalSearcher {@@ -72,8 +73,14 @@ impl LocalSearcher {self.topic_centrality = Some(topic_centrality);}- fn parse_query(&self, query: &SearchQuery, optic: Option<&Optic>) -> Result<Query> {+ fn parse_query(&self, query: &SearchQuery) -> Result<(Query, Optic)> {+ let optic = match query.optic_program.as_ref() {+ Some(program) => Some(optics::parse(program)?),+ None => None,+ };+let mut query_aggregator = optic+ .as_ref().and_then(|optic| {optic.pipeline@@ -83,7 +90,7 @@ impl LocalSearcher {.unwrap_or_default();if let (Some(optic), Some(harmonic)) = (- optic,+ optic.as_ref(),self.centrality_store.as_ref().map(|store| &store.online_harmonic),@@ -100,30 +107,16 @@ impl LocalSearcher {)?;parsed_query.set_num_results(query.num_results);+ parsed_query.set_offset(query.offset);- if parsed_query.is_empty() {- Err(Error::EmptyQuery)- } else {- Ok(parsed_query)+ if let Some(region) = query.selected_region {+ parsed_query.set_region(region);}- }-- fn search_inverted_index(- &self,- query: &SearchQuery,- de_rank_similar: bool,- ) -> Result<(Vec<RankingWebsite>, usize)> {- let optic = match query.optic_program.as_ref() {- Some(program) => Some(optics::parse(program)?),- None => None,- };-- let mut parsed_query = self.parse_query(query, optic.as_ref())?;let mut optics = Vec::new();- if let Some(optic) = &optic {- optics.push(optic.clone());+ if let Some(optic) = optic {+ optics.push(optic);}if let Some(site_rankings) = &query.site_rankings {@@ -138,34 +131,33 @@ impl LocalSearcher {parsed_query.set_optic(&optic, &self.index);- let mut aggregator = optic- .pipeline- .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))- .unwrap_or_default();-- if let Some(harmonic) = self- .centrality_store- .as_ref()- .map(|store| &store.online_harmonic)- {- aggregator- .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));+ if parsed_query.is_empty() {+ Err(Error::EmptyQuery)+ } else {+ Ok((parsed_query, optic))}+ }+ fn ranker(+ &self,+ query: &Query,+ de_rank_similar: bool,+ aggregator: SignalAggregator,+ ) -> Result<Ranker> {let mut ranker = Ranker::new(self.index.region_count.clone(),aggregator,self.index.inverted_index.fastfield_cache(),);- if let Some(region) = query.selected_region {- if region != Region::All {- ranker = ranker.with_region(region);+ if let Some(region) = query.region() {+ if *region != Region::All {+ ranker = ranker.with_region(*region);}}if let Some(topic_centrality) = self.topic_centrality.as_ref() {- let topic_scorer = topic_centrality.scorer(&parsed_query);+ let topic_scorer = topic_centrality.scorer(query);ranker.set_topic_scorer(topic_scorer);}@@ -176,7 +168,7 @@ impl LocalSearcher {.with_max_docs(10_000, self.index.num_segments()).with_num_results(100);- let top_host_nodes = self.index.top_nodes(&parsed_query, ranker.collector())?;+ let top_host_nodes = self.index.top_nodes(query, ranker.collector())?;if !top_host_nodes.is_empty() {let harmonic = centrality_store.online_harmonic@@ -185,10 +177,34 @@ impl LocalSearcher {}}- ranker = ranker+ Ok(ranker.with_max_docs(10_000_000, self.index.num_segments())- .with_num_results(parsed_query.num_results())- .with_offset(query.offset);+ .with_num_results(query.num_results())+ .with_offset(query.offset()))+ }++ fn search_inverted_index(+ &self,+ query: &SearchQuery,+ de_rank_similar: bool,+ ) -> Result<(Vec<RankingWebsite>, usize)> {+ let (parsed_query, optic) = self.parse_query(query)?;++ let mut aggregator = optic+ .pipeline+ .and_then(|pipeline| pipeline.stages.first().map(|stage| stage.aggregator()))+ .unwrap_or_default();++ if let Some(harmonic) = self+ .centrality_store+ .as_ref()+ .map(|store| &store.online_harmonic)+ {+ aggregator+ .add_personal_harmonic(online_centrality_scorer(&optic.site_rankings, harmonic));+ }++ let ranker = self.ranker(&parsed_query, de_rank_similar, aggregator)?;let res = self.index@@ -202,9 +218,16 @@ impl LocalSearcher {Ok((ranking_websites, res.num_websites))}- fn spell_correction(&self, query: &SearchQuery) -> Result<Option<Correction>> {- let parsed_query = self.parse_query(query, None)?;- Ok(self.index.spell_correction(&parsed_query.simple_terms()))+ fn spell_correction(&self, query: &SearchQuery) -> Option<Correction> {+ let terms: Vec<_> = query::parser::parse(&query.query)+ .into_iter()+ .filter_map(|term| match *term {+ Term::Simple(s) => Some(s),+ _ => None,+ })+ .collect();++ self.index.spell_correction(&terms)}fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {@@ -219,7 +242,7 @@ impl LocalSearcher {de_rank_similar: bool,) -> Result<InitialWebsiteResult> {let (websites, num_websites) = self.search_inverted_index(query, de_rank_similar)?;- let correction = self.spell_correction(query)?;+ let correction = self.spell_correction(query);let sidebar = self.entity_sidebar(query).map(|entity| DisplayedEntity::from(entity, self));@@ -251,7 +274,8 @@ impl LocalSearcher {self.index.retrieve_websites(websites, &query)}- pub fn search(&self, query: &SearchQuery) -> Result<PrettifiedWebsitesResult> {+ /// This function is mainly used for tests and benchmarks+ pub fn search(&self, query: &SearchQuery) -> Result<WebsitesResult> {use std::{sync::Arc, time::Instant};use crate::{@@ -282,7 +306,7 @@ impl LocalSearcher {let retrieved_sites = self.retrieve_websites(&pointers, &search_query.query)?;- Ok(PrettifiedWebsitesResult {+ Ok(WebsitesResult {spell_corrected_query: search_result.spell_corrected_query.map(HighlightedSpellCorrection::from),diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rsindex 4b3dfb2a..355bff52 100644--- a/core/src/searcher/mod.rs+++ b/core/src/searcher/mod.rs@@ -34,12 +34,12 @@ pub const NUM_RESULTS_PER_PAGE: usize = 20;#[derive(Debug, Serialize)]pub enum SearchResult {- Websites(PrettifiedWebsitesResult),+ Websites(WebsitesResult),Bang(BangHit),}#[derive(Debug, Serialize)]-pub struct PrettifiedWebsitesResult {+pub struct WebsitesResult {pub spell_corrected_query: Option<HighlightedSpellCorrection>,pub webpages: Vec<DisplayedWebpage>,pub num_hits: usize,@@ -67,6 +67,9 @@ pub struct InitialWebsiteResult {impl Default for SearchQuery {fn default() -> Self {+ // This does not use `..Default::default()` as there should be+ // an explicit compile error when new fields are added to the `SearchQuery` struct+ // to ensure the developer considers what the default should be.Self {query: Default::default(),offset: Default::default(),diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex 1eab9a9f..3aab829f 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -540,7 +540,7 @@ impl<S: Store> Webgraph<S> {}self.segments- .sort_by_key(|b| std::cmp::Reverse(b.num_nodes()));+ .sort_by_key(|segment| std::cmp::Reverse(segment.num_nodes()));let mut candidates = Vec::with_capacity(num_segments);