web search done right
entity sidebar threshold
Mikkel Denker8ee456c272da
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rsindex 429c1da7..fce55059 100644--- a/core/src/config/defaults.rs+++ b/core/src/config/defaults.rs@@ -37,6 +37,10 @@ impl Frontend {0.5}+ pub fn entity_sidebar() -> f64 {+ 10.0+ }+pub fn discussions_widget() -> f64 {0.1}diff --git a/core/src/config/mod.rs b/core/src/config/mod.rsindex bc204257..65281ffd 100644--- a/core/src/config/mod.rs+++ b/core/src/config/mod.rs@@ -170,6 +170,9 @@ pub struct FrontendThresholds {#[serde(default = "defaults::Frontend::stackoverflow")]pub stackoverflow: f64,+ #[serde(default = "defaults::Frontend::entity_sidebar")]+ pub entity_sidebar: f64,+#[serde(default = "defaults::Frontend::discussions_widget")]pub discussions_widget: f64,}@@ -178,6 +181,7 @@ impl Default for FrontendThresholds {fn default() -> Self {Self {stackoverflow: defaults::Frontend::stackoverflow(),+ entity_sidebar: defaults::Frontend::entity_sidebar(),discussions_widget: defaults::Frontend::discussions_widget(),}}diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rsindex 34379369..779f3ed8 100644--- a/core/src/crawler/coordinator.rs+++ b/core/src/crawler/coordinator.rs@@ -82,7 +82,7 @@ impl CrawlCoordinator {.unwrap();}- tracing::debug!("inserted responses in {:?}", start.elapsed());+ tracing::info!("inserted responses in {:?}", start.elapsed());Ok(())}diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rsindex d370bf9e..9cc547e6 100644--- a/core/src/entity_index/mod.rs+++ b/core/src/entity_index/mod.rs@@ -145,16 +145,22 @@ fn wikipedify_url(url: &Url) -> Vec<Url> {]}-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]+#[derive(Debug, Serialize, Deserialize, Clone)]pub struct StoredEntity {pub title: String,pub entity_abstract: String,pub image: Option<String>,- pub related_entities: Vec<StoredEntity>,+ pub related_entities: Vec<EntityMatch>,pub info: BTreeMap<String, Span>,pub links: Vec<Link>,}+#[derive(Debug, Serialize, Deserialize, Clone)]+pub struct EntityMatch {+ pub entity: StoredEntity,+ pub score: f32,+}+impl EntityIndex {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {if !path.as_ref().exists() {@@ -230,7 +236,7 @@ impl EntityIndex {self.image_downloader.download(&mut self.image_store);}- fn related_entities(&self, doc: DocAddress) -> Vec<StoredEntity> {+ fn related_entities(&self, doc: DocAddress) -> Vec<EntityMatch> {let searcher = self.reader.searcher();let more_like_this_query = MoreLikeThisQuery::builder().with_min_doc_frequency(1)@@ -253,17 +259,20 @@ impl EntityIndex {Ok(result) => result.into_iter().filter(|(_, related_doc)| doc != *related_doc)- .map(|(_, doc_address)| {- self.retrieve_stored_entity(&searcher, doc_address, false, false, false)+ .map(|(score, doc_address)| {+ let entity =+ self.retrieve_stored_entity(&searcher, doc_address, false, false, false);++ EntityMatch { entity, score }})- .filter(|entity| entity.image.is_some())+ .filter(|m| m.entity.image.is_some()).take(4).collect(),Err(_) => Vec::new(),}}- pub fn search(&self, query: &str) -> Option<StoredEntity> {+ pub fn search(&self, query: &str) -> Option<EntityMatch> {let searcher = self.reader.searcher();let title = self.schema.get_field("title").unwrap();@@ -301,8 +310,13 @@ impl EntityIndex {.search(&query, &TopDocs::with_limit(1)).unwrap().first()- .map(|(_score, doc_address)| {- self.retrieve_stored_entity(&searcher, *doc_address, true, true, true)+ .map(|(score, doc_address)| {+ let entity = self.retrieve_stored_entity(&searcher, *doc_address, true, true, true);++ EntityMatch {+ entity,+ score: *score,+ }})}@@ -439,10 +453,13 @@ mod tests {index.commit();- assert_eq!(index.search("the"), None);- assert_eq!(index.search("ashes").unwrap().title.as_str(), "the ashes");+ assert!(index.search("the").is_none());+ assert_eq!(+ index.search("ashes").unwrap().entity.title.as_str(),+ "the ashes"+ );assert_eq!(- index.search("the ashes").unwrap().title.as_str(),+ index.search("the ashes").unwrap().entity.title.as_str(),"the ashes");}diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rsindex 2430f247..14424341 100644--- a/core/src/search_prettifier/entity.rs+++ b/core/src/search_prettifier/entity.rs@@ -20,7 +20,7 @@ use chrono::NaiveDate;use serde::{Deserialize, Serialize};use crate::{- entity_index::{entity::Span, StoredEntity},+ entity_index::{entity::Span, EntityMatch},searcher::LocalSearcher,};@@ -31,6 +31,7 @@ pub struct DisplayedEntity {pub image_base64: Option<String>,pub related_entities: Vec<DisplayedEntity>,pub info: Vec<(String, String)>,+ pub match_score: f32,}fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {@@ -77,29 +78,32 @@ fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(}impl DisplayedEntity {- pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {+ pub fn from(m: EntityMatch, searcher: &LocalSearcher) -> Self {let entity_abstract = Span {- text: entity.entity_abstract,- links: entity.links,+ text: m.entity.entity_abstract,+ links: m.entity.links,};let small_abstract = entity_link_to_html(entity_abstract, 300);- let image_base64 = entity+ let image_base64 = m+ .entity.image.and_then(|image| searcher.entity_image(image)).map(|image| base64::encode(image.as_raw_bytes()));Self {- title: entity.title,+ title: m.entity.title,small_abstract,image_base64,- related_entities: entity+ related_entities: m+ .entity.related_entities.into_iter()- .map(|entity| DisplayedEntity::from(entity, searcher))+ .map(|m| DisplayedEntity::from(m, searcher)).collect(),- info: prepare_info(entity.info, searcher),+ info: prepare_info(m.entity.info, searcher),+ match_score: m.score,}}}diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rsindex 77be5be9..53c80460 100644--- a/core/src/searcher/frontend.rs+++ b/core/src/searcher/frontend.rs@@ -163,8 +163,14 @@ impl FrontendSearcher {query: &SearchQuery,) -> Result<Option<Sidebar>> {let entity = initial_results- .first()- .and_then(|result| result.local_result.entity_sidebar.clone());+ .iter()+ .filter_map(|res| res.local_result.entity_sidebar.clone())+ .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)+ .max_by(|a, b| {+ a.match_score+ .partial_cmp(&b.match_score)+ .unwrap_or(Ordering::Equal)+ });match entity {Some(entity) => Ok(Some(Sidebar::Entity(entity))),diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rsindex ea7f8eb4..8f66b799 100644--- a/core/src/searcher/local.rs+++ b/core/src/searcher/local.rs@@ -18,7 +18,7 @@ use std::collections::HashMap;use std::sync::Arc;use crate::config::{CollectorConfig, SnippetConfig};-use crate::entity_index::{EntityIndex, StoredEntity};+use crate::entity_index::{EntityIndex, EntityMatch};use crate::image_store::Image;use crate::index::Index;use crate::inverted_index::RetrievedWebpage;@@ -254,7 +254,7 @@ impl LocalSearcher {})}- fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {+ fn entity_sidebar(&self, query: &SearchQuery) -> Option<EntityMatch> {self.entity_index.as_ref().and_then(|index| index.search(&query.query))