Skip to content

web search done right

entity sidebar threshold

Mikkel Denker8ee456c272da
diff --git a/core/src/config/defaults.rs b/core/src/config/defaults.rs
index 429c1da7..fce55059 100644
--- a/core/src/config/defaults.rs
+++ b/core/src/config/defaults.rs
@@ -37,6 +37,10 @@ impl Frontend {
0.5
}
+ pub fn entity_sidebar() -> f64 {
+ 10.0
+ }
+
pub fn discussions_widget() -> f64 {
0.1
}
diff --git a/core/src/config/mod.rs b/core/src/config/mod.rs
index bc204257..65281ffd 100644
--- a/core/src/config/mod.rs
+++ b/core/src/config/mod.rs
@@ -170,6 +170,9 @@ pub struct FrontendThresholds {
#[serde(default = "defaults::Frontend::stackoverflow")]
pub stackoverflow: f64,
+ #[serde(default = "defaults::Frontend::entity_sidebar")]
+ pub entity_sidebar: f64,
+
#[serde(default = "defaults::Frontend::discussions_widget")]
pub discussions_widget: f64,
}
@@ -178,6 +181,7 @@ impl Default for FrontendThresholds {
fn default() -> Self {
Self {
stackoverflow: defaults::Frontend::stackoverflow(),
+ entity_sidebar: defaults::Frontend::entity_sidebar(),
discussions_widget: defaults::Frontend::discussions_widget(),
}
}
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 34379369..779f3ed8 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -82,7 +82,7 @@ impl CrawlCoordinator {
.unwrap();
}
- tracing::debug!("inserted responses in {:?}", start.elapsed());
+ tracing::info!("inserted responses in {:?}", start.elapsed());
Ok(())
}
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index d370bf9e..9cc547e6 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -145,16 +145,22 @@ fn wikipedify_url(url: &Url) -> Vec<Url> {
]
}
-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct StoredEntity {
pub title: String,
pub entity_abstract: String,
pub image: Option<String>,
- pub related_entities: Vec<StoredEntity>,
+ pub related_entities: Vec<EntityMatch>,
pub info: BTreeMap<String, Span>,
pub links: Vec<Link>,
}
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct EntityMatch {
+ pub entity: StoredEntity,
+ pub score: f32,
+}
+
impl EntityIndex {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
if !path.as_ref().exists() {
@@ -230,7 +236,7 @@ impl EntityIndex {
self.image_downloader.download(&mut self.image_store);
}
- fn related_entities(&self, doc: DocAddress) -> Vec<StoredEntity> {
+ fn related_entities(&self, doc: DocAddress) -> Vec<EntityMatch> {
let searcher = self.reader.searcher();
let more_like_this_query = MoreLikeThisQuery::builder()
.with_min_doc_frequency(1)
@@ -253,17 +259,20 @@ impl EntityIndex {
Ok(result) => result
.into_iter()
.filter(|(_, related_doc)| doc != *related_doc)
- .map(|(_, doc_address)| {
- self.retrieve_stored_entity(&searcher, doc_address, false, false, false)
+ .map(|(score, doc_address)| {
+ let entity =
+ self.retrieve_stored_entity(&searcher, doc_address, false, false, false);
+
+ EntityMatch { entity, score }
})
- .filter(|entity| entity.image.is_some())
+ .filter(|m| m.entity.image.is_some())
.take(4)
.collect(),
Err(_) => Vec::new(),
}
}
- pub fn search(&self, query: &str) -> Option<StoredEntity> {
+ pub fn search(&self, query: &str) -> Option<EntityMatch> {
let searcher = self.reader.searcher();
let title = self.schema.get_field("title").unwrap();
@@ -301,8 +310,13 @@ impl EntityIndex {
.search(&query, &TopDocs::with_limit(1))
.unwrap()
.first()
- .map(|(_score, doc_address)| {
- self.retrieve_stored_entity(&searcher, *doc_address, true, true, true)
+ .map(|(score, doc_address)| {
+ let entity = self.retrieve_stored_entity(&searcher, *doc_address, true, true, true);
+
+ EntityMatch {
+ entity,
+ score: *score,
+ }
})
}
@@ -439,10 +453,13 @@ mod tests {
index.commit();
- assert_eq!(index.search("the"), None);
- assert_eq!(index.search("ashes").unwrap().title.as_str(), "the ashes");
+ assert!(index.search("the").is_none());
+ assert_eq!(
+ index.search("ashes").unwrap().entity.title.as_str(),
+ "the ashes"
+ );
assert_eq!(
- index.search("the ashes").unwrap().title.as_str(),
+ index.search("the ashes").unwrap().entity.title.as_str(),
"the ashes"
);
}
diff --git a/core/src/search_prettifier/entity.rs b/core/src/search_prettifier/entity.rs
index 2430f247..14424341 100644
--- a/core/src/search_prettifier/entity.rs
+++ b/core/src/search_prettifier/entity.rs
@@ -20,7 +20,7 @@ use chrono::NaiveDate;
use serde::{Deserialize, Serialize};
use crate::{
- entity_index::{entity::Span, StoredEntity},
+ entity_index::{entity::Span, EntityMatch},
searcher::LocalSearcher,
};
@@ -31,6 +31,7 @@ pub struct DisplayedEntity {
pub image_base64: Option<String>,
pub related_entities: Vec<DisplayedEntity>,
pub info: Vec<(String, String)>,
+ pub match_score: f32,
}
fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(String, String)> {
@@ -77,29 +78,32 @@ fn prepare_info(info: BTreeMap<String, Span>, searcher: &LocalSearcher) -> Vec<(
}
impl DisplayedEntity {
- pub fn from(entity: StoredEntity, searcher: &LocalSearcher) -> Self {
+ pub fn from(m: EntityMatch, searcher: &LocalSearcher) -> Self {
let entity_abstract = Span {
- text: entity.entity_abstract,
- links: entity.links,
+ text: m.entity.entity_abstract,
+ links: m.entity.links,
};
let small_abstract = entity_link_to_html(entity_abstract, 300);
- let image_base64 = entity
+ let image_base64 = m
+ .entity
.image
.and_then(|image| searcher.entity_image(image))
.map(|image| base64::encode(image.as_raw_bytes()));
Self {
- title: entity.title,
+ title: m.entity.title,
small_abstract,
image_base64,
- related_entities: entity
+ related_entities: m
+ .entity
.related_entities
.into_iter()
- .map(|entity| DisplayedEntity::from(entity, searcher))
+ .map(|m| DisplayedEntity::from(m, searcher))
.collect(),
- info: prepare_info(entity.info, searcher),
+ info: prepare_info(m.entity.info, searcher),
+ match_score: m.score,
}
}
}
diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rs
index 77be5be9..53c80460 100644
--- a/core/src/searcher/frontend.rs
+++ b/core/src/searcher/frontend.rs
@@ -163,8 +163,14 @@ impl FrontendSearcher {
query: &SearchQuery,
) -> Result<Option<Sidebar>> {
let entity = initial_results
- .first()
- .and_then(|result| result.local_result.entity_sidebar.clone());
+ .iter()
+ .filter_map(|res| res.local_result.entity_sidebar.clone())
+ .filter(|entity| entity.match_score as f64 > self.thresholds.entity_sidebar)
+ .max_by(|a, b| {
+ a.match_score
+ .partial_cmp(&b.match_score)
+ .unwrap_or(Ordering::Equal)
+ });
match entity {
Some(entity) => Ok(Some(Sidebar::Entity(entity))),
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index ea7f8eb4..8f66b799 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -18,7 +18,7 @@ use std::collections::HashMap;
use std::sync::Arc;
use crate::config::{CollectorConfig, SnippetConfig};
-use crate::entity_index::{EntityIndex, StoredEntity};
+use crate::entity_index::{EntityIndex, EntityMatch};
use crate::image_store::Image;
use crate::index::Index;
use crate::inverted_index::RetrievedWebpage;
@@ -254,7 +254,7 @@ impl LocalSearcher {
})
}
- fn entity_sidebar(&self, query: &SearchQuery) -> Option<StoredEntity> {
+ fn entity_sidebar(&self, query: &SearchQuery) -> Option<EntityMatch> {
self.entity_index
.as_ref()
.and_then(|index| index.search(&query.query))