Skip to content

web search done right

skip disambiguation pages when constructing entity index

Mikkel Denker6d3af18bac4c
diff --git a/crates/core/src/entity_index/entity.rs b/crates/core/src/entity_index/entity.rs
index 9e6a8eb2..9359de33 100644
--- a/crates/core/src/entity_index/entity.rs
+++ b/crates/core/src/entity_index/entity.rs
@@ -20,6 +20,7 @@ use utoipa::ToSchema;
#[derive(Debug)]
pub struct Entity {
+ pub is_disambiguation: bool,
pub title: String,
pub page_abstract: Span,
pub info: Vec<(String, Span)>,
diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rs
index b2dfcd6e..49a27be6 100644
--- a/crates/core/src/entity_index/mod.rs
+++ b/crates/core/src/entity_index/mod.rs
@@ -411,6 +411,7 @@ mod tests {
index.prepare_writer();
index.insert(Entity {
+ is_disambiguation: false,
title: "the ashes".to_string(),
page_abstract: Span {
text: String::new(),
@@ -439,6 +440,7 @@ mod tests {
index.prepare_writer();
index.insert(Entity {
+ is_disambiguation: false,
title: "the ashes".to_string(),
page_abstract: Span {
text: String::new(),
diff --git a/crates/core/src/entrypoint/entity.rs b/crates/core/src/entrypoint/entity.rs
index df818b71..41c795c3 100644
--- a/crates/core/src/entrypoint/entity.rs
+++ b/crates/core/src/entrypoint/entity.rs
@@ -99,7 +99,10 @@ fn article_to_entity(article: Article) -> Entity {
.map(|n| node_into_span(n.as_node()))
.unwrap_or_default();
+ let is_disambiguation = root.select_first("#disambigbox").is_some();
+
Entity {
+ is_disambiguation,
title,
page_abstract,
image,
@@ -177,7 +180,7 @@ impl EntityIndexer {
let mut inserts = 0;
- for entity in EntityIterator::new(&zim)? {
+ for entity in EntityIterator::new(&zim)?.filter(|e| !e.is_disambiguation) {
index.insert(entity);
inserts += 1;